fix: sequential sync with 1.5s delay to avoid Steam rate limiting

This commit is contained in:
2026-04-30 23:13:28 +08:00
parent 583486de1a
commit 293aadbff0
2 changed files with 40 additions and 59 deletions
+30 -53
View File
@@ -191,52 +191,39 @@ export const gameVersionsRoutes = new Elysia({ prefix: "/games/:gameId/versions"
// ── Game Sync Routes ──────────────────────────────────────────────── // ── Game Sync Routes ────────────────────────────────────────────────
const MAX_BULK_SYNC = 1000 const MAX_BULK_SYNC = 1000
const SYNC_CONCURRENCY = 5 // Number of parallel syncs const SYNC_CONCURRENCY = 5 // Number of parallel syncs
const SYNC_BATCH_DELAY_MS = 100 // Delay between batches to respect rate limits const SYNC_BATCH_DELAY_MS = 2000 // Delay between batches to respect rate limits (2 seconds)
/** /**
* Process syncs in parallel with controlled concurrency. * Process syncs sequentially with delay to avoid Steam rate limiting.
* Processes items in batches of `concurrency` size. * Steam API has aggressive rate limiting - parallel requests get blocked quickly.
*/ */
async function syncInParallel( async function syncSequentially(
gamesToSync: { id: string; steamAppId: number | null }[], gamesToSync: { id: string; steamAppId: number | null }[]
concurrency: number = SYNC_CONCURRENCY
): Promise<{ synced: number; failed: number; results: Map<string, { success: boolean; error?: string }> }> { ): Promise<{ synced: number; failed: number; results: Map<string, { success: boolean; error?: string }> }> {
let synced = 0 let synced = 0
let failed = 0 let failed = 0
const results = new Map<string, { success: boolean; error?: string }>() const results = new Map<string, { success: boolean; error?: string }>()
// Process in batches for (let i = 0; i < gamesToSync.length; i++) {
for (let i = 0; i < gamesToSync.length; i += concurrency) { const game = gamesToSync[i]
const batch = gamesToSync.slice(i, i + concurrency) if (!game.steamAppId) continue
// Process batch in parallel try {
const batchResults = await Promise.allSettled( const result = await syncSteamGame(game.steamAppId, { forceRetry: true })
batch if (result.success) {
.filter((g) => g.steamAppId)
.map(async (game) => {
const result = await syncSteamGame(game.steamAppId!, { forceRetry: true })
return { gameId: game.id, ...result }
})
)
// Collect results
for (const result of batchResults) {
if (result.status === "fulfilled") {
const { gameId, success, error } = result.value
if (success) {
synced++ synced++
results.set(gameId, { success: true }) results.set(game.id, { success: true })
} else { } else {
failed++ failed++
results.set(gameId, { success: false, error }) results.set(game.id, { success: false, error: result.error })
} }
} else { } catch (e) {
failed++ failed++
} results.set(game.id, { success: false, error: String(e) })
} }
// Small delay between batches to avoid hammering Steam API // Delay between syncs to avoid rate limiting
if (i + concurrency < gamesToSync.length) { if (i < gamesToSync.length - 1) {
await new Promise((resolve) => setTimeout(resolve, SYNC_BATCH_DELAY_MS)) await new Promise((resolve) => setTimeout(resolve, SYNC_BATCH_DELAY_MS))
} }
} }
@@ -309,42 +296,32 @@ export const gameSyncRoutes = new Elysia({ prefix: "/games" })
let failed = 0 let failed = 0
const batchSize = SYNC_CONCURRENCY const batchSize = SYNC_CONCURRENCY
for (let i = 0; i < gamesToSync.length; i += batchSize) { // Process syncs sequentially with delay to avoid rate limiting
const batch = gamesToSync.slice(i, i + batchSize) for (let i = 0; i < gamesToSync.length; i++) {
const game = gamesToSync[i]
if (!game.steamAppId) continue
// Process batch in parallel try {
const batchResults = await Promise.allSettled( const result = await syncSteamGame(game.steamAppId, { forceRetry: true })
batch if (result.success) synced++
.filter((g) => g.steamAppId)
.map(async (game) => {
const result = await syncSteamGame(game.steamAppId!, { forceRetry: true })
return { gameId: game.id, gameTitle: game.id, ...result }
})
)
// Collect results and send progress
for (const result of batchResults) {
if (result.status === "fulfilled") {
if (result.value.success) synced++
else failed++ else failed++
} else { } catch (e) {
failed++ failed++
} }
}
// Send progress update after each batch // Send progress update
send({ send({
type: "progress", type: "progress",
current: Math.min(i + batchSize, gamesToSync.length), current: i + 1,
total: gamesToSync.length, total: gamesToSync.length,
synced, synced,
failed, failed,
currentGame: null, currentGame: null,
}) })
// Small delay between batches // Delay between syncs to avoid rate limiting (1.5 seconds)
if (i + batchSize < gamesToSync.length) { if (i < gamesToSync.length - 1) {
await new Promise((resolve) => setTimeout(resolve, SYNC_BATCH_DELAY_MS)) await new Promise((resolve) => setTimeout(resolve, 1500))
} }
} }
+6 -2
View File
@@ -171,8 +171,12 @@ export async function syncSteamGame(
url.searchParams.set("l", "en") url.searchParams.set("l", "en")
const res = await fetch(url.toString(), { const res = await fetch(url.toString(), {
headers: { Accept: "application/json" }, headers: {
signal: AbortSignal.timeout(10000), Accept: "application/json",
"User-Agent": "DeckyVault/1.0",
"Accept-Language": "en-US,en;q=0.9",
},
signal: AbortSignal.timeout(15000),
}) })
if (res.status === 429) { if (res.status === 429) {