From 6009366fb4ac2e710ccc2aa0e2f7985c69836a05 Mon Sep 17 00:00:00 2001 From: Adrian Bonpin Date: Sat, 9 May 2026 12:50:16 +0800 Subject: [PATCH] feat: add modular daily cron system with storage cleanup and orphan detection --- .env.example | 28 ++++---- lib/api/app.ts | 3 + lib/api/cron.ts | 162 +++++++++++++++++++++++++++++++++++++++++++++++ lib/api/index.ts | 1 + 4 files changed, 179 insertions(+), 15 deletions(-) create mode 100644 lib/api/cron.ts diff --git a/.env.example b/.env.example index 4489d97..dd76ff6 100644 --- a/.env.example +++ b/.env.example @@ -50,13 +50,20 @@ DISCORD_CLIENT_ID="" DISCORD_CLIENT_SECRET="" # ----------------------------------------------------------------------------- -# STORAGE (R2) +# STORAGE (CLOUDFLARE R2) # ----------------------------------------------------------------------------- -R2_ACCOUNT_ID="account_id_here" -R2_ACCESS_KEY_ID="access_key_id_here" -R2_SECRET_ACCESS_KEY="secret_access_key_here" -R2_BUCKET_NAME="deckyvault" -R2_PUBLIC_URL="https://cdn.deckyvault.xyz" +R2_ACCOUNT_ID= +R2_ACCESS_KEY_ID= +R2_SECRET_ACCESS_KEY= +R2_BUCKET_NAME=deckyvault +R2_PUBLIC_URL= + +# ----------------------------------------------------------------------------- +# CRON +# ----------------------------------------------------------------------------- +# Secret token for daily cron endpoint (generate with: openssl rand -base64 32) +# Leave empty to disable cron endpoints +CRON_SECRET= # ----------------------------------------------------------------------------- # PASSKEY (WebAuthn) @@ -88,12 +95,3 @@ STEAMGRIDDB_API_KEY="your_steamgriddb_api_key_here" # Discord webhook URL for contact/report submissions DISCORD_WEBHOOK_URL="" -# ----------------------------------------------------------------------------- -# SITEMAP -# ----------------------------------------------------------------------------- -# Secret token for on-demand sitemap revalidation (generate with: openssl rand -base64 32) -# Leave empty to disable the revalidation endpoint -REVALIDATE_SECRET="" - -# ISR revalidation interval in seconds for sitemap.xml (default: 3600 = 1 hour) -SITEMAP_REVALIDATE_SECONDS="" \ No newline at end of file diff --git a/lib/api/app.ts b/lib/api/app.ts index 356b1fb..3a4e126 100644 --- a/lib/api/app.ts +++ b/lib/api/app.ts @@ -33,6 +33,7 @@ import { gamesListingRoutes } from "@/lib/api/games-listing" import { steamgridProxyRoutes } from "@/lib/api/steamgrid-proxy" import { dashboardRoutes } from "@/lib/api/dashboard" import { savedFilterRoutes } from "@/lib/api/saved-filters" +import { cronRoutes } from "@/lib/api/cron" const betterAuth = new Elysia({ name: "better-auth" }) .mount(auth.handler) @@ -114,6 +115,8 @@ export const app = new Elysia({ prefix: "/api" }) .use(communitySuggestionRoutes) // Saved filters .use(savedFilterRoutes) + // Cron + .use(cronRoutes) // Dashboard .use(dashboardRoutes) // Root diff --git a/lib/api/cron.ts b/lib/api/cron.ts new file mode 100644 index 0000000..1d73e63 --- /dev/null +++ b/lib/api/cron.ts @@ -0,0 +1,162 @@ +import { Elysia, t } from "elysia" +import { db } from "@/lib/db/index" +import { storageObjects } from "@/lib/db/schema" +import { eq, and, isNull, sql } from "drizzle-orm" +import { deleteObject, isR2Configured } from "@/lib/storage" + +// ── Task Result Type ──────────────────────────────────────────────── +interface CronTaskResult { + name: string + status: "success" | "skipped" | "error" + durationMs: number + details: Record +} + +// ── Task Registry ─────────────────────────────────────────────────── +type CronTask = () => Promise + +const taskRegistry = new Map() + +export function registerCronTask(name: string, task: CronTask): void { + taskRegistry.set(name, task) +} + +// ── Storage Cleanup Task ──────────────────────────────────────────── +registerCronTask("storage_cleanup", async () => { + const start = Date.now() + const details: Record = {} + + if (!isR2Configured()) { + return { name: "storage_cleanup", status: "skipped" as const, durationMs: Date.now() - start, details: { reason: "R2 not configured" } } + } + + try { + // Find orphaned storage objects + const orphaned = await db + .select() + .from(storageObjects) + .where(eq(storageObjects.isOrphaned, true)) + + let deletedCount = 0 + let errorCount = 0 + + // Process in batches of 100 + for (let i = 0; i < orphaned.length; i += 100) { + const batch = orphaned.slice(i, i + 100) + await Promise.allSettled( + batch.map(async (obj) => { + try { + await deleteObject(obj.key) + deletedCount++ + } catch { + errorCount++ + } + }), + ) + } + + // Remove deleted objects from tracking table + if (orphaned.length > 0) { + await db + .delete(storageObjects) + .where(eq(storageObjects.isOrphaned, true)) + } + + details.deletedCount = deletedCount + details.errorCount = errorCount + details.totalOrphans = orphaned.length + + return { name: "storage_cleanup", status: "success" as const, durationMs: Date.now() - start, details } + } catch (err) { + details.error = err instanceof Error ? err.message : String(err) + return { name: "storage_cleanup", status: "error" as const, durationMs: Date.now() - start, details } + } +}) + +// ── Orphan Detection (separate task for future extensibility) ────── +// This marks objects as orphaned based on their entityType/entityId references +registerCronTask("orphan_detection", async () => { + const start = Date.now() + const details: Record = {} + + try { + // Avatar orphans: storage_objects with entityType='avatar' where user doesn't exist + // or user.image doesn't contain this object's key + const avatarOrphans = await db.execute(sql` + UPDATE storage_objects + SET is_orphaned = true + WHERE entity_type = 'avatar' + AND is_orphaned = false + AND entity_id NOT IN (SELECT id FROM "user") + `) + + details.avatarOrphansMarked = avatarOrphans.rowCount ?? 0 + + // Game cover orphans: storage_objects with entityType='game_cover' where game doesn't exist + const gameOrphans = await db.execute(sql` + UPDATE storage_objects + SET is_orphaned = true + WHERE entity_type = 'game_cover' + AND is_orphaned = false + AND entity_id NOT IN (SELECT id FROM games) + `) + + details.gameCoverOrphansMarked = gameOrphans.rowCount ?? 0 + + return { name: "orphan_detection", status: "success" as const, durationMs: Date.now() - start, details } + } catch (err) { + details.error = err instanceof Error ? err.message : String(err) + return { name: "orphan_detection", status: "error" as const, durationMs: Date.now() - start, details } + } +}) + +// ── Cron Route ────────────────────────────────────────────────────── +export const cronRoutes = new Elysia({ prefix: "/cron" }).post( + "/daily", + async ({ query, set }) => { + const cronSecret = process.env.CRON_SECRET + + // If CRON_SECRET is not configured, disable the endpoint + if (!cronSecret) { + set.status = 404 + return { error: "Cron endpoint not configured" } + } + + // Validate cron secret from query parameter + if (query.secret !== cronSecret) { + set.status = 401 + return { error: "Unauthorized" } + } + + const overallStart = Date.now() + + // Determine which tasks to run + const taskNames = query.tasks + ? query.tasks.split(",").filter((t) => taskRegistry.has(t)) + : Array.from(taskRegistry.keys()) + + const results: CronTaskResult[] = [] + + for (const taskName of taskNames) { + const task = taskRegistry.get(taskName) + if (task) { + const result = await task() + results.push(result) + } + } + + const overallDuration = Date.now() - overallStart + + return { + success: true, + duration: overallDuration, + tasks: results, + } + }, + { + query: t.Object({ + secret: t.String(), + tasks: t.Optional(t.String()), + }), + }, +) \ No newline at end of file diff --git a/lib/api/index.ts b/lib/api/index.ts index 516d557..82f8ff4 100644 --- a/lib/api/index.ts +++ b/lib/api/index.ts @@ -22,3 +22,4 @@ export { steamReviewRoutes } from "./steam-reviews" export { communitySuggestionRoutes } from "./community-suggestions" export { savedFilterRoutes } from "./saved-filters" export { dashboardRoutes } from "./dashboard" +export { cronRoutes } from "./cron"