From 5f037481ec105ed045a15170c8c6815dacf69a3b Mon Sep 17 00:00:00 2001 From: Adrian Bonpin Date: Thu, 30 Apr 2026 22:59:28 +0800 Subject: [PATCH] perf: parallel bulk sync with 5 concurrent requests, remove 100-game limit --- CHANGELOG.md | 3 + app/(manage)/manage/games/games-client.tsx | 196 ++++++--------------- lib/api/games.ts | 78 +++++--- 3 files changed, 110 insertions(+), 167 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa4a69f..44f5e57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,9 @@ All notable changes to DeckyVault will be documented in this file. - Added custom system indicator ### Changed +- Sync All now processes games in parallel (5 concurrent) instead of one-by-one +- Removed 100-game limit for Sync All (now syncs all Steam games) +- Sync Selected now uses bulk endpoint for faster processing - Manage page now shows dashboard by default instead of redirecting to users - Games list now supports 12 filter dimensions and 7 sort options - Game details page now shows anti-cheat, playability, and Steam reviews prominently diff --git a/app/(manage)/manage/games/games-client.tsx b/app/(manage)/manage/games/games-client.tsx index 5532d9d..b39534f 100644 --- a/app/(manage)/manage/games/games-client.tsx +++ b/app/(manage)/manage/games/games-client.tsx @@ -107,179 +107,81 @@ export function GamesClient() { isRunning: true, current: 0, total: gamesToSync.length, - currentGame: null, + currentGame: "Preparing sync...", synced: 0, failed: 0, results: new Map(), }) - let synced = 0 - let failed = 0 - const results = new Map() + try { + // Use the bulk sync endpoint with selected game IDs + const res = await fetch("/api/games/sync/bulk", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + mode: "selected", + gameIds: gamesToSync.map((g) => g.id), + }), + }) - for (let i = 0; i < gamesToSync.length; i++) { - const game = gamesToSync[i] - setSyncProgress((prev) => ({ - ...prev, - current: i + 1, - currentGame: game.title, - })) - - try { - const res = await fetch(`/api/games/${game.id}/sync`, { method: "POST" }) - if (res.ok) { - const data = await res.json() - if (data.status === "synced") { - synced++ - results.set(game.id, { success: true }) - } else { - failed++ - results.set(game.id, { success: false, error: data.error }) - } - } else { - failed++ - results.set(game.id, { success: false, error: `HTTP ${res.status}` }) - } - } catch (error) { - failed++ - results.set(game.id, { success: false, error: String(error) }) + if (!res.ok) { + throw new Error(`Sync failed: HTTP ${res.status}`) } + const data = await res.json() + setSyncProgress((prev) => ({ ...prev, - synced, - failed, - results: new Map(results), + isRunning: false, + synced: data.synced, + failed: data.failed, + currentGame: null, })) - - // Update the game in the list immediately - const result = results.get(game.id) - if (result?.success) { - setGames((prev) => - prev.map((g) => - g.id === game.id - ? { ...g, syncStatus: "synced", lastSync: new Date().toISOString(), syncError: null } - : g - ) - ) - } else { - setGames((prev) => - prev.map((g) => - g.id === game.id - ? { ...g, syncStatus: "failed", syncError: result?.error || "Sync failed" } - : g - ) - ) - } - - // Rate limit: 500ms between syncs (skip on last) - if (i < gamesToSync.length - 1) { - await new Promise((resolve) => setTimeout(resolve, 500)) - } + setSyncCompleted(true) + setSelectedIds(new Set()) + } catch (error) { + console.error("Sync selected failed:", error) + alert("Sync failed. Check console for details.") + } finally { + setSyncing(false) } - - setSyncProgress((prev) => ({ - ...prev, - isRunning: false, - currentGame: null, - })) - setSyncing(false) - setSelectedIds(new Set()) - setSyncCompleted(true) } const handleSyncAll = async () => { if (!confirm("This will sync all Steam games. Continue?")) return - // Fetch all Steam games setSyncing(true) + setSyncCompleted(false) + setSyncProgress({ + isRunning: true, + current: 0, + total: 0, + currentGame: "Preparing sync...", + synced: 0, + failed: 0, + results: new Map(), + }) + try { - const res = await fetch("/api/games?limit=1000&filter_source=steam") - if (!res.ok) { - alert("Failed to fetch games list") - setSyncing(false) - return - } - const data = await res.json() - const allSteamGames = data.data.filter((g: Game) => g.steamAppId) - - if (allSteamGames.length === 0) { - alert("No Steam games to sync") - setSyncing(false) - return - } - - setSyncCompleted(false) - setSyncProgress({ - isRunning: true, - current: 0, - total: allSteamGames.length, - currentGame: null, - synced: 0, - failed: 0, - results: new Map(), + // Use the bulk sync endpoint (processes in parallel) + const res = await fetch("/api/games/sync/bulk", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ mode: "all" }), }) - let synced = 0 - let failed = 0 - const results = new Map() - - for (let i = 0; i < allSteamGames.length; i++) { - const game = allSteamGames[i] - setSyncProgress((prev) => ({ - ...prev, - current: i + 1, - currentGame: game.title, - })) - - try { - const syncRes = await fetch(`/api/games/${game.id}/sync`, { method: "POST" }) - if (syncRes.ok) { - const syncData = await syncRes.json() - if (syncData.status === "synced") { - synced++ - results.set(game.id, { success: true }) - } else { - failed++ - results.set(game.id, { success: false, error: syncData.error }) - } - } else { - failed++ - results.set(game.id, { success: false, error: `HTTP ${syncRes.status}` }) - } - } catch (error) { - failed++ - results.set(game.id, { success: false, error: String(error) }) - } - - setSyncProgress((prev) => ({ - ...prev, - synced, - failed, - results: new Map(results), - })) - - // Update the game in the list if it's currently visible - const result = results.get(game.id) - setGames((prev) => - prev.map((g) => - g.id === game.id - ? result?.success - ? { ...g, syncStatus: "synced", lastSync: new Date().toISOString(), syncError: null } - : { ...g, syncStatus: "failed", syncError: result?.error || "Sync failed" } - : g - ) - ) - - // Rate limit: 500ms between syncs (skip on last) - if (i < allSteamGames.length - 1) { - await new Promise((resolve) => setTimeout(resolve, 500)) - } + if (!res.ok) { + throw new Error(`Sync failed: HTTP ${res.status}`) } + const data = await res.json() + setSyncProgress((prev) => ({ ...prev, isRunning: false, + total: data.total, + synced: data.synced, + failed: data.failed, currentGame: null, })) setSyncCompleted(true) diff --git a/lib/api/games.ts b/lib/api/games.ts index 26bd61d..53d4e2c 100644 --- a/lib/api/games.ts +++ b/lib/api/games.ts @@ -189,7 +189,60 @@ export const gameVersionsRoutes = new Elysia({ prefix: "/games/:gameId/versions" ) // ── Game Sync Routes ──────────────────────────────────────────────── -const MAX_BULK_SYNC = 100 +const MAX_BULK_SYNC = 1000 +const SYNC_CONCURRENCY = 5 // Number of parallel syncs +const SYNC_BATCH_DELAY_MS = 100 // Delay between batches to respect rate limits + +/** + * Process syncs in parallel with controlled concurrency. + * Processes items in batches of `concurrency` size. + */ +async function syncInParallel( + gamesToSync: { id: string; steamAppId: number | null }[], + concurrency: number = SYNC_CONCURRENCY +): Promise<{ synced: number; failed: number; results: Map }> { + let synced = 0 + let failed = 0 + const results = new Map() + + // Process in batches + for (let i = 0; i < gamesToSync.length; i += concurrency) { + const batch = gamesToSync.slice(i, i + concurrency) + + // Process batch in parallel + const batchResults = await Promise.allSettled( + batch + .filter((g) => g.steamAppId) + .map(async (game) => { + const result = await syncSteamGame(game.steamAppId!, { forceRetry: true }) + return { gameId: game.id, ...result } + }) + ) + + // Collect results + for (const result of batchResults) { + if (result.status === "fulfilled") { + const { gameId, success, error } = result.value + if (success) { + synced++ + results.set(gameId, { success: true }) + } else { + failed++ + results.set(gameId, { success: false, error }) + } + } else { + failed++ + } + } + + // Small delay between batches to avoid hammering Steam API + if (i + concurrency < gamesToSync.length) { + await new Promise((resolve) => setTimeout(resolve, SYNC_BATCH_DELAY_MS)) + } + } + + return { synced, failed, results } +} export const gameSyncRoutes = new Elysia({ prefix: "/games" }) // Bulk sync (defined before /:gameId/sync to avoid route conflict) @@ -208,8 +261,7 @@ export const gameSyncRoutes = new Elysia({ prefix: "/games" }) gamesToSync = await db .select({ id: games.id, steamAppId: games.steamAppId }) .from(games) - .where(sql`${games.steamAppId} IS NOT NULL`) - .limit(MAX_BULK_SYNC); + .where(sql`${games.steamAppId} IS NOT NULL`); } else if (body.mode === "stale") { gamesToSync = await db .select({ id: games.id, steamAppId: games.steamAppId }) @@ -222,8 +274,7 @@ export const gameSyncRoutes = new Elysia({ prefix: "/games" }) sql`${games.lastSync} < NOW() - INTERVAL '7 days'` ) ) - ) - .limit(MAX_BULK_SYNC); + ); } else { const gameIds = (body.gameIds || []).slice(0, MAX_BULK_SYNC); if (gameIds.length > 0) { @@ -243,21 +294,8 @@ export const gameSyncRoutes = new Elysia({ prefix: "/games" }) return { total: 0, synced: 0, failed: 0, message: "No games to sync" }; } - // Process syncs sequentially with delay - let synced = 0; - let failed = 0; - - for (const game of gamesToSync) { - if (!game.steamAppId) continue; - const result = await syncSteamGame(game.steamAppId, { forceRetry: true }); - if (result.success) synced++; - else failed++; - - // Rate limit protection (skip delay on last item) - if (game !== gamesToSync[gamesToSync.length - 1]) { - await new Promise((resolve) => setTimeout(resolve, 500)); - } - } + // Process syncs in parallel with controlled concurrency + const { synced, failed } = await syncInParallel(gamesToSync, SYNC_CONCURRENCY); return { total: gamesToSync.length,