adrianbonpin/deckyvault
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

perf: parallel bulk sync with 5 concurrent requests, remove 100-game limit

Browse Source
This commit is contained in:
adrianbonpin
2026-04-30 22:59:28 +08:00
parent 0461beca4e
commit 5f037481ec
3 changed files with 110 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+3
View File
@@ -70,6 +70,9 @@ All notable changes to DeckyVault will be documented in this file.
- Added custom system indicator
### Changed
- Sync All now processes games in parallel (5 concurrent) instead of one-by-one
- Removed 100-game limit for Sync All (now syncs all Steam games)
- Sync Selected now uses bulk endpoint for faster processing
- Manage page now shows dashboard by default instead of redirecting to users
- Games list now supports 12 filter dimensions and 7 sort options
- Game details page now shows anti-cheat, playability, and Steam reviews prominently
app/(manage)/manage/games/games-client.tsx
+49 -147
View File
@@ -107,179 +107,81 @@ export function GamesClient() {
isRunning: true,
current: 0,
total: gamesToSync.length,
currentGame: null,
currentGame: "Preparing sync...",
synced: 0,
failed: 0,
results: new Map(),
})
let synced = 0
let failed = 0
const results = new Map<string, { success: boolean; error?: string }>()
try {
// Use the bulk sync endpoint with selected game IDs
const res = await fetch("/api/games/sync/bulk", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
mode: "selected",
gameIds: gamesToSync.map((g) => g.id),
}),
})
for (let i = 0; i < gamesToSync.length; i++) {
const game = gamesToSync[i]
setSyncProgress((prev) => ({
...prev,
current: i + 1,
currentGame: game.title,
}))
try {
const res = await fetch(`/api/games/${game.id}/sync`, { method: "POST" })
if (res.ok) {
const data = await res.json()
if (data.status === "synced") {
synced++
results.set(game.id, { success: true })
} else {
failed++
results.set(game.id, { success: false, error: data.error })
}
} else {
failed++
results.set(game.id, { success: false, error: `HTTP ${res.status}` })
}
} catch (error) {
failed++
results.set(game.id, { success: false, error: String(error) })
if (!res.ok) {
throw new Error(`Sync failed: HTTP ${res.status}`)
}
const data = await res.json()
setSyncProgress((prev) => ({
...prev,
synced,
failed,
results: new Map(results),
isRunning: false,
synced: data.synced,
failed: data.failed,
currentGame: null,
}))
// Update the game in the list immediately
const result = results.get(game.id)
if (result?.success) {
setGames((prev) =>
prev.map((g) =>
g.id === game.id
? { ...g, syncStatus: "synced", lastSync: new Date().toISOString(), syncError: null }
: g
)
)
} else {
setGames((prev) =>
prev.map((g) =>
g.id === game.id
? { ...g, syncStatus: "failed", syncError: result?.error || "Sync failed" }
: g
)
)
}
// Rate limit: 500ms between syncs (skip on last)
if (i < gamesToSync.length - 1) {
await new Promise((resolve) => setTimeout(resolve, 500))
}
setSyncCompleted(true)
setSelectedIds(new Set())
} catch (error) {
console.error("Sync selected failed:", error)
alert("Sync failed. Check console for details.")
} finally {
setSyncing(false)
}
setSyncProgress((prev) => ({
...prev,
isRunning: false,
currentGame: null,
}))
setSyncing(false)
setSelectedIds(new Set())
setSyncCompleted(true)
}
const handleSyncAll = async () => {
if (!confirm("This will sync all Steam games. Continue?")) return
// Fetch all Steam games
setSyncing(true)
setSyncCompleted(false)
setSyncProgress({
isRunning: true,
current: 0,
total: 0,
currentGame: "Preparing sync...",
synced: 0,
failed: 0,
results: new Map(),
})
try {
const res = await fetch("/api/games?limit=1000&filter_source=steam")
if (!res.ok) {
alert("Failed to fetch games list")
setSyncing(false)
return
}
const data = await res.json()
const allSteamGames = data.data.filter((g: Game) => g.steamAppId)
if (allSteamGames.length === 0) {
alert("No Steam games to sync")
setSyncing(false)
return
}
setSyncCompleted(false)
setSyncProgress({
isRunning: true,
current: 0,
total: allSteamGames.length,
currentGame: null,
synced: 0,
failed: 0,
results: new Map(),
// Use the bulk sync endpoint (processes in parallel)
const res = await fetch("/api/games/sync/bulk", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ mode: "all" }),
})
let synced = 0
let failed = 0
const results = new Map<string, { success: boolean; error?: string }>()
for (let i = 0; i < allSteamGames.length; i++) {
const game = allSteamGames[i]
setSyncProgress((prev) => ({
...prev,
current: i + 1,
currentGame: game.title,
}))
try {
const syncRes = await fetch(`/api/games/${game.id}/sync`, { method: "POST" })
if (syncRes.ok) {
const syncData = await syncRes.json()
if (syncData.status === "synced") {
synced++
results.set(game.id, { success: true })
} else {
failed++
results.set(game.id, { success: false, error: syncData.error })
}
} else {
failed++
results.set(game.id, { success: false, error: `HTTP ${syncRes.status}` })
}
} catch (error) {
failed++
results.set(game.id, { success: false, error: String(error) })
}
setSyncProgress((prev) => ({
...prev,
synced,
failed,
results: new Map(results),
}))
// Update the game in the list if it's currently visible
const result = results.get(game.id)
setGames((prev) =>
prev.map((g) =>
g.id === game.id
? result?.success
? { ...g, syncStatus: "synced", lastSync: new Date().toISOString(), syncError: null }
: { ...g, syncStatus: "failed", syncError: result?.error || "Sync failed" }
: g
)
)
// Rate limit: 500ms between syncs (skip on last)
if (i < allSteamGames.length - 1) {
await new Promise((resolve) => setTimeout(resolve, 500))
}
if (!res.ok) {
throw new Error(`Sync failed: HTTP ${res.status}`)
}
const data = await res.json()
setSyncProgress((prev) => ({
...prev,
isRunning: false,
total: data.total,
synced: data.synced,
failed: data.failed,
currentGame: null,
}))
setSyncCompleted(true)
lib/api/games.ts
+58 -20
View File
@@ -189,7 +189,60 @@ export const gameVersionsRoutes = new Elysia({ prefix: "/games/:gameId/versions"
)
// ── Game Sync Routes ────────────────────────────────────────────────
const MAX_BULK_SYNC = 100
const MAX_BULK_SYNC = 1000
const SYNC_CONCURRENCY = 5 // Number of parallel syncs
const SYNC_BATCH_DELAY_MS = 100 // Delay between batches to respect rate limits
/**
* Process syncs in parallel with controlled concurrency.
* Processes items in batches of `concurrency` size.
*/
async function syncInParallel(
gamesToSync: { id: string; steamAppId: number | null }[],
concurrency: number = SYNC_CONCURRENCY
): Promise<{ synced: number; failed: number; results: Map<string, { success: boolean; error?: string }> }> {
let synced = 0
let failed = 0
const results = new Map<string, { success: boolean; error?: string }>()
// Process in batches
for (let i = 0; i < gamesToSync.length; i += concurrency) {
const batch = gamesToSync.slice(i, i + concurrency)
// Process batch in parallel
const batchResults = await Promise.allSettled(
batch
.filter((g) => g.steamAppId)
.map(async (game) => {
const result = await syncSteamGame(game.steamAppId!, { forceRetry: true })
return { gameId: game.id, ...result }
})
)
// Collect results
for (const result of batchResults) {
if (result.status === "fulfilled") {
const { gameId, success, error } = result.value
if (success) {
synced++
results.set(gameId, { success: true })
} else {
failed++
results.set(gameId, { success: false, error })
}
} else {
failed++
}
}
// Small delay between batches to avoid hammering Steam API
if (i + concurrency < gamesToSync.length) {
await new Promise((resolve) => setTimeout(resolve, SYNC_BATCH_DELAY_MS))
}
}
return { synced, failed, results }
}
export const gameSyncRoutes = new Elysia({ prefix: "/games" })
// Bulk sync (defined before /:gameId/sync to avoid route conflict)
@@ -208,8 +261,7 @@ export const gameSyncRoutes = new Elysia({ prefix: "/games" })
gamesToSync = await db
.select({ id: games.id, steamAppId: games.steamAppId })
.from(games)
.where(sql`${games.steamAppId} IS NOT NULL`)
.limit(MAX_BULK_SYNC);
.where(sql`${games.steamAppId} IS NOT NULL`);
} else if (body.mode === "stale") {
gamesToSync = await db
.select({ id: games.id, steamAppId: games.steamAppId })
@@ -222,8 +274,7 @@ export const gameSyncRoutes = new Elysia({ prefix: "/games" })
sql`${games.lastSync} < NOW() - INTERVAL '7 days'`
)
)
)
.limit(MAX_BULK_SYNC);
);
} else {
const gameIds = (body.gameIds || []).slice(0, MAX_BULK_SYNC);
if (gameIds.length > 0) {
@@ -243,21 +294,8 @@ export const gameSyncRoutes = new Elysia({ prefix: "/games" })
return { total: 0, synced: 0, failed: 0, message: "No games to sync" };
}
// Process syncs sequentially with delay
let synced = 0;
let failed = 0;
for (const game of gamesToSync) {
if (!game.steamAppId) continue;
const result = await syncSteamGame(game.steamAppId, { forceRetry: true });
if (result.success) synced++;
else failed++;
// Rate limit protection (skip delay on last item)
if (game !== gamesToSync[gamesToSync.length - 1]) {
await new Promise((resolve) => setTimeout(resolve, 500));
}
}
// Process syncs in parallel with controlled concurrency
const { synced, failed } = await syncInParallel(gamesToSync, SYNC_CONCURRENCY);
return {
total: gamesToSync.length,