fix: sequential sync with 1.5s delay to avoid Steam rate limiting
This commit is contained in:
+30
-53
@@ -191,52 +191,39 @@ export const gameVersionsRoutes = new Elysia({ prefix: "/games/:gameId/versions"
|
|||||||
// ── Game Sync Routes ────────────────────────────────────────────────
|
// ── Game Sync Routes ────────────────────────────────────────────────
|
||||||
const MAX_BULK_SYNC = 1000
|
const MAX_BULK_SYNC = 1000
|
||||||
const SYNC_CONCURRENCY = 5 // Number of parallel syncs
|
const SYNC_CONCURRENCY = 5 // Number of parallel syncs
|
||||||
const SYNC_BATCH_DELAY_MS = 100 // Delay between batches to respect rate limits
|
const SYNC_BATCH_DELAY_MS = 2000 // Delay between batches to respect rate limits (2 seconds)
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Process syncs in parallel with controlled concurrency.
|
* Process syncs sequentially with delay to avoid Steam rate limiting.
|
||||||
* Processes items in batches of `concurrency` size.
|
* Steam API has aggressive rate limiting - parallel requests get blocked quickly.
|
||||||
*/
|
*/
|
||||||
async function syncInParallel(
|
async function syncSequentially(
|
||||||
gamesToSync: { id: string; steamAppId: number | null }[],
|
gamesToSync: { id: string; steamAppId: number | null }[]
|
||||||
concurrency: number = SYNC_CONCURRENCY
|
|
||||||
): Promise<{ synced: number; failed: number; results: Map<string, { success: boolean; error?: string }> }> {
|
): Promise<{ synced: number; failed: number; results: Map<string, { success: boolean; error?: string }> }> {
|
||||||
let synced = 0
|
let synced = 0
|
||||||
let failed = 0
|
let failed = 0
|
||||||
const results = new Map<string, { success: boolean; error?: string }>()
|
const results = new Map<string, { success: boolean; error?: string }>()
|
||||||
|
|
||||||
// Process in batches
|
for (let i = 0; i < gamesToSync.length; i++) {
|
||||||
for (let i = 0; i < gamesToSync.length; i += concurrency) {
|
const game = gamesToSync[i]
|
||||||
const batch = gamesToSync.slice(i, i + concurrency)
|
if (!game.steamAppId) continue
|
||||||
|
|
||||||
// Process batch in parallel
|
try {
|
||||||
const batchResults = await Promise.allSettled(
|
const result = await syncSteamGame(game.steamAppId, { forceRetry: true })
|
||||||
batch
|
if (result.success) {
|
||||||
.filter((g) => g.steamAppId)
|
|
||||||
.map(async (game) => {
|
|
||||||
const result = await syncSteamGame(game.steamAppId!, { forceRetry: true })
|
|
||||||
return { gameId: game.id, ...result }
|
|
||||||
})
|
|
||||||
)
|
|
||||||
|
|
||||||
// Collect results
|
|
||||||
for (const result of batchResults) {
|
|
||||||
if (result.status === "fulfilled") {
|
|
||||||
const { gameId, success, error } = result.value
|
|
||||||
if (success) {
|
|
||||||
synced++
|
synced++
|
||||||
results.set(gameId, { success: true })
|
results.set(game.id, { success: true })
|
||||||
} else {
|
} else {
|
||||||
failed++
|
failed++
|
||||||
results.set(gameId, { success: false, error })
|
results.set(game.id, { success: false, error: result.error })
|
||||||
}
|
}
|
||||||
} else {
|
} catch (e) {
|
||||||
failed++
|
failed++
|
||||||
}
|
results.set(game.id, { success: false, error: String(e) })
|
||||||
}
|
}
|
||||||
|
|
||||||
// Small delay between batches to avoid hammering Steam API
|
// Delay between syncs to avoid rate limiting
|
||||||
if (i + concurrency < gamesToSync.length) {
|
if (i < gamesToSync.length - 1) {
|
||||||
await new Promise((resolve) => setTimeout(resolve, SYNC_BATCH_DELAY_MS))
|
await new Promise((resolve) => setTimeout(resolve, SYNC_BATCH_DELAY_MS))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -309,42 +296,32 @@ export const gameSyncRoutes = new Elysia({ prefix: "/games" })
|
|||||||
let failed = 0
|
let failed = 0
|
||||||
const batchSize = SYNC_CONCURRENCY
|
const batchSize = SYNC_CONCURRENCY
|
||||||
|
|
||||||
for (let i = 0; i < gamesToSync.length; i += batchSize) {
|
// Process syncs sequentially with delay to avoid rate limiting
|
||||||
const batch = gamesToSync.slice(i, i + batchSize)
|
for (let i = 0; i < gamesToSync.length; i++) {
|
||||||
|
const game = gamesToSync[i]
|
||||||
|
if (!game.steamAppId) continue
|
||||||
|
|
||||||
// Process batch in parallel
|
try {
|
||||||
const batchResults = await Promise.allSettled(
|
const result = await syncSteamGame(game.steamAppId, { forceRetry: true })
|
||||||
batch
|
if (result.success) synced++
|
||||||
.filter((g) => g.steamAppId)
|
|
||||||
.map(async (game) => {
|
|
||||||
const result = await syncSteamGame(game.steamAppId!, { forceRetry: true })
|
|
||||||
return { gameId: game.id, gameTitle: game.id, ...result }
|
|
||||||
})
|
|
||||||
)
|
|
||||||
|
|
||||||
// Collect results and send progress
|
|
||||||
for (const result of batchResults) {
|
|
||||||
if (result.status === "fulfilled") {
|
|
||||||
if (result.value.success) synced++
|
|
||||||
else failed++
|
else failed++
|
||||||
} else {
|
} catch (e) {
|
||||||
failed++
|
failed++
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Send progress update after each batch
|
// Send progress update
|
||||||
send({
|
send({
|
||||||
type: "progress",
|
type: "progress",
|
||||||
current: Math.min(i + batchSize, gamesToSync.length),
|
current: i + 1,
|
||||||
total: gamesToSync.length,
|
total: gamesToSync.length,
|
||||||
synced,
|
synced,
|
||||||
failed,
|
failed,
|
||||||
currentGame: null,
|
currentGame: null,
|
||||||
})
|
})
|
||||||
|
|
||||||
// Small delay between batches
|
// Delay between syncs to avoid rate limiting (1.5 seconds)
|
||||||
if (i + batchSize < gamesToSync.length) {
|
if (i < gamesToSync.length - 1) {
|
||||||
await new Promise((resolve) => setTimeout(resolve, SYNC_BATCH_DELAY_MS))
|
await new Promise((resolve) => setTimeout(resolve, 1500))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+6
-2
@@ -171,8 +171,12 @@ export async function syncSteamGame(
|
|||||||
url.searchParams.set("l", "en")
|
url.searchParams.set("l", "en")
|
||||||
|
|
||||||
const res = await fetch(url.toString(), {
|
const res = await fetch(url.toString(), {
|
||||||
headers: { Accept: "application/json" },
|
headers: {
|
||||||
signal: AbortSignal.timeout(10000),
|
Accept: "application/json",
|
||||||
|
"User-Agent": "DeckyVault/1.0",
|
||||||
|
"Accept-Language": "en-US,en;q=0.9",
|
||||||
|
},
|
||||||
|
signal: AbortSignal.timeout(15000),
|
||||||
})
|
})
|
||||||
|
|
||||||
if (res.status === 429) {
|
if (res.status === 429) {
|
||||||
|
|||||||
Reference in New Issue
Block a user