feat: streaming progress for bulk sync - real-time updates
This commit is contained in:
@@ -114,7 +114,7 @@ export function GamesClient() {
|
||||
})
|
||||
|
||||
try {
|
||||
// Use the bulk sync endpoint with selected game IDs
|
||||
// Use the bulk sync endpoint with streaming progress
|
||||
const res = await fetch("/api/games/sync/bulk", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
@@ -129,17 +129,52 @@ export function GamesClient() {
|
||||
throw new Error(errorData?.error || `Sync failed: HTTP ${res.status}`)
|
||||
}
|
||||
|
||||
const data = await res.json()
|
||||
// Read streaming response
|
||||
const reader = res.body?.getReader()
|
||||
const decoder = new TextDecoder()
|
||||
let buffer = ""
|
||||
|
||||
setSyncProgress((prev) => ({
|
||||
...prev,
|
||||
isRunning: false,
|
||||
synced: data.synced || 0,
|
||||
failed: data.failed || 0,
|
||||
currentGame: null,
|
||||
}))
|
||||
setSyncCompleted(true)
|
||||
setSelectedIds(new Set())
|
||||
if (reader) {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
|
||||
buffer += decoder.decode(value, { stream: true })
|
||||
const lines = buffer.split("\n")
|
||||
buffer = lines.pop() || ""
|
||||
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue
|
||||
try {
|
||||
const data = JSON.parse(line)
|
||||
|
||||
if (data.type === "progress") {
|
||||
setSyncProgress((prev) => ({
|
||||
...prev,
|
||||
current: data.current,
|
||||
total: data.total,
|
||||
synced: data.synced,
|
||||
failed: data.failed,
|
||||
currentGame: data.currentGame,
|
||||
}))
|
||||
} else if (data.type === "complete") {
|
||||
setSyncProgress((prev) => ({
|
||||
...prev,
|
||||
isRunning: false,
|
||||
total: data.total,
|
||||
synced: data.synced,
|
||||
failed: data.failed,
|
||||
currentGame: null,
|
||||
}))
|
||||
setSyncCompleted(true)
|
||||
setSelectedIds(new Set())
|
||||
}
|
||||
} catch (e) {
|
||||
// Ignore parse errors
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Sync selected failed:", error)
|
||||
alert(`Sync failed: ${error instanceof Error ? error.message : String(error)}`)
|
||||
@@ -169,7 +204,7 @@ export function GamesClient() {
|
||||
})
|
||||
|
||||
try {
|
||||
// Use the bulk sync endpoint (processes in parallel)
|
||||
// Use the bulk sync endpoint with streaming progress
|
||||
const res = await fetch("/api/games/sync/bulk", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
@@ -181,17 +216,51 @@ export function GamesClient() {
|
||||
throw new Error(errorData?.error || `Sync failed: HTTP ${res.status}`)
|
||||
}
|
||||
|
||||
const data = await res.json()
|
||||
// Read streaming response
|
||||
const reader = res.body?.getReader()
|
||||
const decoder = new TextDecoder()
|
||||
let buffer = ""
|
||||
|
||||
setSyncProgress((prev) => ({
|
||||
...prev,
|
||||
isRunning: false,
|
||||
total: data.total || 0,
|
||||
synced: data.synced || 0,
|
||||
failed: data.failed || 0,
|
||||
currentGame: null,
|
||||
}))
|
||||
setSyncCompleted(true)
|
||||
if (reader) {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
|
||||
buffer += decoder.decode(value, { stream: true })
|
||||
const lines = buffer.split("\n")
|
||||
buffer = lines.pop() || ""
|
||||
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue
|
||||
try {
|
||||
const data = JSON.parse(line)
|
||||
|
||||
if (data.type === "progress") {
|
||||
setSyncProgress((prev) => ({
|
||||
...prev,
|
||||
current: data.current,
|
||||
total: data.total,
|
||||
synced: data.synced,
|
||||
failed: data.failed,
|
||||
currentGame: data.currentGame,
|
||||
}))
|
||||
} else if (data.type === "complete") {
|
||||
setSyncProgress((prev) => ({
|
||||
...prev,
|
||||
isRunning: false,
|
||||
total: data.total,
|
||||
synced: data.synced,
|
||||
failed: data.failed,
|
||||
currentGame: null,
|
||||
}))
|
||||
setSyncCompleted(true)
|
||||
}
|
||||
} catch (e) {
|
||||
// Ignore parse errors
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Sync all failed:", error)
|
||||
alert(`Sync failed: ${error instanceof Error ? error.message : String(error)}`)
|
||||
|
||||
@@ -3,90 +3,109 @@
|
||||
import Link from "next/link"
|
||||
import { usePathname } from "next/navigation"
|
||||
import {
|
||||
UsersIcon,
|
||||
CpuIcon,
|
||||
Gamepad2Icon,
|
||||
MessageSquareIcon,
|
||||
FlagIcon,
|
||||
BarChart3Icon,
|
||||
LayoutDashboardIcon,
|
||||
UsersIcon,
|
||||
CpuIcon,
|
||||
Gamepad2Icon,
|
||||
MessageSquareIcon,
|
||||
FlagIcon,
|
||||
BarChart3Icon,
|
||||
LayoutDashboardIcon,
|
||||
} from "lucide-react"
|
||||
|
||||
type NavItem =
|
||||
| { type: "section"; label: string }
|
||||
| { type: "divider" }
|
||||
| { type: "link"; href: string; label: string; icon: React.ElementType }
|
||||
| { type: "section"; label: string }
|
||||
| { type: "divider" }
|
||||
| { type: "link"; href: string; label: string; icon: React.ElementType }
|
||||
|
||||
const navItems: NavItem[] = [
|
||||
{ type: "section", label: "Overview" },
|
||||
{ type: "link", href: "/manage", label: "Dashboard", icon: LayoutDashboardIcon },
|
||||
{ type: "divider" },
|
||||
{ type: "section", label: "Management" },
|
||||
{ type: "link", href: "/manage/users", label: "Users", icon: UsersIcon },
|
||||
{ type: "link", href: "/manage/hardware", label: "Hardware", icon: CpuIcon },
|
||||
{ type: "link", href: "/manage/games", label: "Games", icon: Gamepad2Icon },
|
||||
{ type: "divider" },
|
||||
{ type: "section", label: "Moderation" },
|
||||
{ type: "link", href: "/manage/reports", label: "Reports", icon: FlagIcon },
|
||||
{ type: "link", href: "/manage/benchmarks", label: "Benchmarks", icon: BarChart3Icon },
|
||||
{ type: "link", href: "/manage/comments", label: "Comments", icon: MessageSquareIcon },
|
||||
{ type: "section", label: "Overview" },
|
||||
{
|
||||
type: "link",
|
||||
href: "/manage",
|
||||
label: "Dashboard",
|
||||
icon: LayoutDashboardIcon,
|
||||
},
|
||||
{ type: "divider" },
|
||||
{ type: "section", label: "Management" },
|
||||
{ type: "link", href: "/manage/users", label: "Users", icon: UsersIcon },
|
||||
{
|
||||
type: "link",
|
||||
href: "/manage/hardware",
|
||||
label: "Hardware",
|
||||
icon: CpuIcon,
|
||||
},
|
||||
{ type: "link", href: "/manage/games", label: "Games", icon: Gamepad2Icon },
|
||||
{ type: "divider" },
|
||||
{ type: "section", label: "Moderation" },
|
||||
{ type: "link", href: "/manage/reports", label: "Reports", icon: FlagIcon },
|
||||
{
|
||||
type: "link",
|
||||
href: "/manage/benchmarks",
|
||||
label: "Benchmarks",
|
||||
icon: BarChart3Icon,
|
||||
},
|
||||
{
|
||||
type: "link",
|
||||
href: "/manage/comments",
|
||||
label: "Comments",
|
||||
icon: MessageSquareIcon,
|
||||
},
|
||||
]
|
||||
|
||||
export function ManageSidebar() {
|
||||
const pathname = usePathname()
|
||||
const pathname = usePathname()
|
||||
|
||||
return (
|
||||
<nav className="md:w-56 shrink-0">
|
||||
<div className="hidden md:flex items-center gap-2 px-3 py-2 mb-2 text-sm font-semibold text-text/70">
|
||||
<LayoutDashboardIcon className="h-4 w-4" />
|
||||
Manage Panel
|
||||
</div>
|
||||
return (
|
||||
<nav className='md:w-56 shrink-0'>
|
||||
<div className='hidden md:flex items-center gap-2 px-3 py-2 mb-2 text-sm font-semibold text-text/70'>
|
||||
<LayoutDashboardIcon className='h-4 w-4' />
|
||||
Manage Panel
|
||||
</div>
|
||||
|
||||
<div className="flex md:flex-col gap-1 overflow-x-auto md:overflow-visible pb-2 md:pb-0 md:border-r md:border-border md:pr-3">
|
||||
{navItems.map((item, index) => {
|
||||
if (item.type === "section") {
|
||||
return (
|
||||
<div
|
||||
key={`section-${item.label}`}
|
||||
className="hidden md:block px-3 py-1 text-[10px] font-semibold uppercase tracking-widest text-text/30"
|
||||
>
|
||||
{item.label}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
<div className='flex md:flex-col gap-1 overflow-x-auto md:overflow-visible pb-2 md:pb-0 md:border-r md:border-border md:pr-3'>
|
||||
{navItems.map((item, index) => {
|
||||
if (item.type === "section") {
|
||||
return (
|
||||
<div
|
||||
key={`section-${item.label}`}
|
||||
className='hidden md:block px-3 py-1 text-[10px] font-semibold uppercase tracking-widest text-text/30'
|
||||
>
|
||||
{item.label}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (item.type === "divider") {
|
||||
return [
|
||||
<div
|
||||
key={`div-m-${index}`}
|
||||
className="w-px bg-border shrink-0 self-stretch md:hidden"
|
||||
/>,
|
||||
<div
|
||||
key={`div-d-${index}`}
|
||||
className="hidden md:block h-px bg-border"
|
||||
/>,
|
||||
]
|
||||
}
|
||||
if (item.type === "divider") {
|
||||
return [
|
||||
<div
|
||||
key={`div-m-${index}`}
|
||||
className='w-px bg-border shrink-0 self-stretch md:hidden'
|
||||
/>,
|
||||
<div
|
||||
key={`div-d-${index}`}
|
||||
className='hidden md:block h-px bg-border'
|
||||
/>,
|
||||
]
|
||||
}
|
||||
|
||||
const isActive =
|
||||
pathname === item.href || pathname.startsWith(item.href + "/")
|
||||
const isActive = pathname === item.href
|
||||
|
||||
return (
|
||||
<Link
|
||||
key={item.href}
|
||||
href={item.href}
|
||||
className={`flex items-center gap-2 px-4 md:px-3 py-2.5 text-sm font-medium transition-colors whitespace-nowrap rounded-lg ${
|
||||
isActive
|
||||
? "bg-primary/10 text-primary"
|
||||
: "text-text/50 hover:text-text/70 hover:bg-text/5"
|
||||
}`}
|
||||
>
|
||||
<item.icon className="h-4 w-4 shrink-0" />
|
||||
{item.label}
|
||||
</Link>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</nav>
|
||||
)
|
||||
return (
|
||||
<Link
|
||||
key={item.href}
|
||||
href={item.href}
|
||||
className={`flex items-center gap-2 px-4 md:px-3 py-2.5 text-sm font-medium transition-colors whitespace-nowrap rounded-lg ${
|
||||
isActive
|
||||
? "bg-primary/10 text-primary"
|
||||
: "text-text/50 hover:text-text/70 hover:bg-text/5"
|
||||
}`}
|
||||
>
|
||||
<item.icon className='h-4 w-4 shrink-0' />
|
||||
{item.label}
|
||||
</Link>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</nav>
|
||||
)
|
||||
}
|
||||
|
||||
+65
-11
@@ -245,7 +245,7 @@ async function syncInParallel(
|
||||
}
|
||||
|
||||
export const gameSyncRoutes = new Elysia({ prefix: "/games" })
|
||||
// Bulk sync (defined before /:gameId/sync to avoid route conflict)
|
||||
// Bulk sync with streaming progress (defined before /:gameId/sync to avoid route conflict)
|
||||
.post(
|
||||
"/sync/bulk",
|
||||
async ({ body, request, set }) => {
|
||||
@@ -294,19 +294,73 @@ export const gameSyncRoutes = new Elysia({ prefix: "/games" })
|
||||
return { total: 0, synced: 0, failed: 0, message: "No games to sync" };
|
||||
}
|
||||
|
||||
console.log(`[Bulk Sync] Starting sync of ${gamesToSync.length} games with concurrency ${SYNC_CONCURRENCY}`)
|
||||
// Return streaming response for real-time progress
|
||||
const encoder = new TextEncoder()
|
||||
const stream = new ReadableStream({
|
||||
async start(controller) {
|
||||
const send = (data: any) => {
|
||||
controller.enqueue(encoder.encode(JSON.stringify(data) + "\n"))
|
||||
}
|
||||
|
||||
// Process syncs in parallel with controlled concurrency
|
||||
const { synced, failed } = await syncInParallel(gamesToSync, SYNC_CONCURRENCY);
|
||||
// Send initial progress
|
||||
send({ type: "progress", current: 0, total: gamesToSync.length, synced: 0, failed: 0, currentGame: null })
|
||||
|
||||
console.log(`[Bulk Sync] Complete: ${synced} synced, ${failed} failed`)
|
||||
let synced = 0
|
||||
let failed = 0
|
||||
const batchSize = SYNC_CONCURRENCY
|
||||
|
||||
return {
|
||||
total: gamesToSync.length,
|
||||
synced,
|
||||
failed,
|
||||
message: `Synced ${synced} games, ${failed} failed`,
|
||||
};
|
||||
for (let i = 0; i < gamesToSync.length; i += batchSize) {
|
||||
const batch = gamesToSync.slice(i, i + batchSize)
|
||||
|
||||
// Process batch in parallel
|
||||
const batchResults = await Promise.allSettled(
|
||||
batch
|
||||
.filter((g) => g.steamAppId)
|
||||
.map(async (game) => {
|
||||
const result = await syncSteamGame(game.steamAppId!, { forceRetry: true })
|
||||
return { gameId: game.id, gameTitle: game.id, ...result }
|
||||
})
|
||||
)
|
||||
|
||||
// Collect results and send progress
|
||||
for (const result of batchResults) {
|
||||
if (result.status === "fulfilled") {
|
||||
if (result.value.success) synced++
|
||||
else failed++
|
||||
} else {
|
||||
failed++
|
||||
}
|
||||
}
|
||||
|
||||
// Send progress update after each batch
|
||||
send({
|
||||
type: "progress",
|
||||
current: Math.min(i + batchSize, gamesToSync.length),
|
||||
total: gamesToSync.length,
|
||||
synced,
|
||||
failed,
|
||||
currentGame: null,
|
||||
})
|
||||
|
||||
// Small delay between batches
|
||||
if (i + batchSize < gamesToSync.length) {
|
||||
await new Promise((resolve) => setTimeout(resolve, SYNC_BATCH_DELAY_MS))
|
||||
}
|
||||
}
|
||||
|
||||
// Send completion
|
||||
send({ type: "complete", total: gamesToSync.length, synced, failed })
|
||||
controller.close()
|
||||
}
|
||||
})
|
||||
|
||||
return new Response(stream, {
|
||||
headers: {
|
||||
"Content-Type": "text/event-stream",
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
},
|
||||
})
|
||||
},
|
||||
{
|
||||
body: t.Object({
|
||||
|
||||
Reference in New Issue
Block a user