perf: parallel bulk sync with 5 concurrent requests, remove 100-game limit

CHANGELOG.md

@@ -70,6 +70,9 @@ All notable changes to DeckyVault will be documented in this file.
 - Added custom system indicator
 
 ### Changed
+- Sync All now processes games in parallel (5 concurrent) instead of one-by-one
+- Removed 100-game limit for Sync All (now syncs all Steam games)
+- Sync Selected now uses bulk endpoint for faster processing
 - Manage page now shows dashboard by default instead of redirecting to users
 - Games list now supports 12 filter dimensions and 7 sort options
 - Game details page now shows anti-cheat, playability, and Steam reviews prominently

app/(manage)/manage/games/games-client.tsx

@@ -107,179 +107,81 @@ export function GamesClient() {
       isRunning: true,
       current: 0,
       total: gamesToSync.length,
-      currentGame: null,
+      currentGame: "Preparing sync...",
       synced: 0,
       failed: 0,
       results: new Map(),
     })
 
-    let synced = 0
-    let failed = 0
-    const results = new Map<string, { success: boolean; error?: string }>()
-
-    for (let i = 0; i < gamesToSync.length; i++) {
-      const game = gamesToSync[i]
-      setSyncProgress((prev) => ({
-        ...prev,
-        current: i + 1,
-        currentGame: game.title,
-      }))
-
     try {
-        const res = await fetch(`/api/games/${game.id}/sync`, { method: "POST" })
+      // Use the bulk sync endpoint with selected game IDs
-        if (res.ok) {
+      const res = await fetch("/api/games/sync/bulk", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+          mode: "selected",
+          gameIds: gamesToSync.map((g) => g.id),
+        }),
+      })
+
+      if (!res.ok) {
+        throw new Error(`Sync failed: HTTP ${res.status}`)
+      }
+
       const data = await res.json()
-          if (data.status === "synced") {
-            synced++
-            results.set(game.id, { success: true })
-          } else {
-            failed++
-            results.set(game.id, { success: false, error: data.error })
-          }
-        } else {
-          failed++
-          results.set(game.id, { success: false, error: `HTTP ${res.status}` })
-        }
-      } catch (error) {
-        failed++
-        results.set(game.id, { success: false, error: String(error) })
-      }
-
-      setSyncProgress((prev) => ({
-        ...prev,
-        synced,
-        failed,
-        results: new Map(results),
-      }))
-
-      // Update the game in the list immediately
-      const result = results.get(game.id)
-      if (result?.success) {
-        setGames((prev) =>
-          prev.map((g) =>
-            g.id === game.id
-              ? { ...g, syncStatus: "synced", lastSync: new Date().toISOString(), syncError: null }
-              : g
-          )
-        )
-      } else {
-        setGames((prev) =>
-          prev.map((g) =>
-            g.id === game.id
-              ? { ...g, syncStatus: "failed", syncError: result?.error || "Sync failed" }
-              : g
-          )
-        )
-      }
-
-      // Rate limit: 500ms between syncs (skip on last)
-      if (i < gamesToSync.length - 1) {
-        await new Promise((resolve) => setTimeout(resolve, 500))
-      }
-    }
 
       setSyncProgress((prev) => ({
         ...prev,
         isRunning: false,
+        synced: data.synced,
+        failed: data.failed,
         currentGame: null,
       }))
-    setSyncing(false)
-    setSelectedIds(new Set())
       setSyncCompleted(true)
+      setSelectedIds(new Set())
+    } catch (error) {
+      console.error("Sync selected failed:", error)
+      alert("Sync failed. Check console for details.")
+    } finally {
+      setSyncing(false)
+    }
   }
 
   const handleSyncAll = async () => {
     if (!confirm("This will sync all Steam games. Continue?")) return
 
-    // Fetch all Steam games
     setSyncing(true)
-    try {
-      const res = await fetch("/api/games?limit=1000&filter_source=steam")
-      if (!res.ok) {
-        alert("Failed to fetch games list")
-        setSyncing(false)
-        return
-      }
-      const data = await res.json()
-      const allSteamGames = data.data.filter((g: Game) => g.steamAppId)
-
-      if (allSteamGames.length === 0) {
-        alert("No Steam games to sync")
-        setSyncing(false)
-        return
-      }
-
     setSyncCompleted(false)
     setSyncProgress({
       isRunning: true,
       current: 0,
-        total: allSteamGames.length,
+      total: 0,
-        currentGame: null,
+      currentGame: "Preparing sync...",
       synced: 0,
       failed: 0,
       results: new Map(),
     })
 
-      let synced = 0
-      let failed = 0
-      const results = new Map<string, { success: boolean; error?: string }>()
-
-      for (let i = 0; i < allSteamGames.length; i++) {
-        const game = allSteamGames[i]
-        setSyncProgress((prev) => ({
-          ...prev,
-          current: i + 1,
-          currentGame: game.title,
-        }))
-
     try {
-          const syncRes = await fetch(`/api/games/${game.id}/sync`, { method: "POST" })
+      // Use the bulk sync endpoint (processes in parallel)
-          if (syncRes.ok) {
+      const res = await fetch("/api/games/sync/bulk", {
-            const syncData = await syncRes.json()
+        method: "POST",
-            if (syncData.status === "synced") {
+        headers: { "Content-Type": "application/json" },
-              synced++
+        body: JSON.stringify({ mode: "all" }),
-              results.set(game.id, { success: true })
+      })
-            } else {
+
-              failed++
+      if (!res.ok) {
-              results.set(game.id, { success: false, error: syncData.error })
+        throw new Error(`Sync failed: HTTP ${res.status}`)
-            }
-          } else {
-            failed++
-            results.set(game.id, { success: false, error: `HTTP ${syncRes.status}` })
-          }
-        } catch (error) {
-          failed++
-          results.set(game.id, { success: false, error: String(error) })
       }
 
-        setSyncProgress((prev) => ({
+      const data = await res.json()
-          ...prev,
-          synced,
-          failed,
-          results: new Map(results),
-        }))
-
-        // Update the game in the list if it's currently visible
-        const result = results.get(game.id)
-        setGames((prev) =>
-          prev.map((g) =>
-            g.id === game.id
-              ? result?.success
-                ? { ...g, syncStatus: "synced", lastSync: new Date().toISOString(), syncError: null }
-                : { ...g, syncStatus: "failed", syncError: result?.error || "Sync failed" }
-              : g
-          )
-        )
-
-        // Rate limit: 500ms between syncs (skip on last)
-        if (i < allSteamGames.length - 1) {
-          await new Promise((resolve) => setTimeout(resolve, 500))
-        }
-      }
 
       setSyncProgress((prev) => ({
         ...prev,
         isRunning: false,
+        total: data.total,
+        synced: data.synced,
+        failed: data.failed,
         currentGame: null,
       }))
       setSyncCompleted(true)

lib/api/games.ts

@@ -189,7 +189,60 @@ export const gameVersionsRoutes = new Elysia({ prefix: "/games/:gameId/versions"
   )
 
 // ── Game Sync Routes ────────────────────────────────────────────────
-const MAX_BULK_SYNC = 100
+const MAX_BULK_SYNC = 1000
+const SYNC_CONCURRENCY = 5 // Number of parallel syncs
+const SYNC_BATCH_DELAY_MS = 100 // Delay between batches to respect rate limits
+
+/**
+ * Process syncs in parallel with controlled concurrency.
+ * Processes items in batches of `concurrency` size.
+ */
+async function syncInParallel(
+  gamesToSync: { id: string; steamAppId: number | null }[],
+  concurrency: number = SYNC_CONCURRENCY
+): Promise<{ synced: number; failed: number; results: Map<string, { success: boolean; error?: string }> }> {
+  let synced = 0
+  let failed = 0
+  const results = new Map<string, { success: boolean; error?: string }>()
+
+  // Process in batches
+  for (let i = 0; i < gamesToSync.length; i += concurrency) {
+    const batch = gamesToSync.slice(i, i + concurrency)
+
+    // Process batch in parallel
+    const batchResults = await Promise.allSettled(
+      batch
+        .filter((g) => g.steamAppId)
+        .map(async (game) => {
+          const result = await syncSteamGame(game.steamAppId!, { forceRetry: true })
+          return { gameId: game.id, ...result }
+        })
+    )
+
+    // Collect results
+    for (const result of batchResults) {
+      if (result.status === "fulfilled") {
+        const { gameId, success, error } = result.value
+        if (success) {
+          synced++
+          results.set(gameId, { success: true })
+        } else {
+          failed++
+          results.set(gameId, { success: false, error })
+        }
+      } else {
+        failed++
+      }
+    }
+
+    // Small delay between batches to avoid hammering Steam API
+    if (i + concurrency < gamesToSync.length) {
+      await new Promise((resolve) => setTimeout(resolve, SYNC_BATCH_DELAY_MS))
+    }
+  }
+
+  return { synced, failed, results }
+}
 
 export const gameSyncRoutes = new Elysia({ prefix: "/games" })
   // Bulk sync (defined before /:gameId/sync to avoid route conflict)
@@ -208,8 +261,7 @@ export const gameSyncRoutes = new Elysia({ prefix: "/games" })
         gamesToSync = await db
           .select({ id: games.id, steamAppId: games.steamAppId })
           .from(games)
-          .where(sql`${games.steamAppId} IS NOT NULL`)
+          .where(sql`${games.steamAppId} IS NOT NULL`);
-          .limit(MAX_BULK_SYNC);
       } else if (body.mode === "stale") {
         gamesToSync = await db
           .select({ id: games.id, steamAppId: games.steamAppId })
@@ -222,8 +274,7 @@ export const gameSyncRoutes = new Elysia({ prefix: "/games" })
                 sql`${games.lastSync} < NOW() - INTERVAL '7 days'`
               )
             )
-          )
+          );
-          .limit(MAX_BULK_SYNC);
       } else {
         const gameIds = (body.gameIds || []).slice(0, MAX_BULK_SYNC);
         if (gameIds.length > 0) {
@@ -243,21 +294,8 @@ export const gameSyncRoutes = new Elysia({ prefix: "/games" })
         return { total: 0, synced: 0, failed: 0, message: "No games to sync" };
       }
 
-      // Process syncs sequentially with delay
+      // Process syncs in parallel with controlled concurrency
-      let synced = 0;
+      const { synced, failed } = await syncInParallel(gamesToSync, SYNC_CONCURRENCY);
-      let failed = 0;
-
-      for (const game of gamesToSync) {
-        if (!game.steamAppId) continue;
-        const result = await syncSteamGame(game.steamAppId, { forceRetry: true });
-        if (result.success) synced++;
-        else failed++;
-        
-        // Rate limit protection (skip delay on last item)
-        if (game !== gamesToSync[gamesToSync.length - 1]) {
-          await new Promise((resolve) => setTimeout(resolve, 500));
-        }
-      }
 
       return {
         total: gamesToSync.length,