feat: add multi-strategy version fetchers, disable auto-fetch by default

- Add 4 version-fetch strategies (UpToDateCheck API, Store Page Scrape,
  Community Hub Scrape, Store API Heuristic) — no Steam API key needed
- Create orchestrator that tries all strategies, prefers named version
  over build ID
- Add test page at /test-version-fetchers for comparing strategies
- Add standalone /api/version-test endpoint (direct Steam App ID, no DB)
- Update /api/games/:id/steamdb-version to use new strategies
- Disable auto-fetch on submit page by default (NEXT_PUBLIC_VERSION_AUTO_FETCH=true to enable)
- SteamDB scraping blocked by Cloudflare; strategies preserved for later enablement
This commit is contained in:
2026-05-16 20:13:23 +08:00
parent 88a098f387
commit e5f671a992
13 changed files with 1356 additions and 37 deletions
+140
View File
@@ -0,0 +1,140 @@
/**
* Strategy 3: Steam Community Hub Scraping
*
* Fetches steamcommunity.com/app/{appid} and extracts version/build info
* from update announcements, embedded data, and the app sidebar.
*
* Community hub is less aggressively rate-limited than the store.
*/
import type { VersionFetchResult } from "./types"
const COMMUNITY_URL = "https://steamcommunity.com/app"
export async function fetchCommunityHub(
steamAppId: number,
): Promise<VersionFetchResult> {
const source = "Community Hub Scrape"
try {
const res = await fetch(`${COMMUNITY_URL}/${steamAppId}`, {
headers: {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
Accept: "text/html",
},
signal: AbortSignal.timeout(10000),
})
if (!res.ok) {
return {
versionString: null,
buildId: null,
source,
success: false,
error: `HTTP ${res.status}`,
}
}
const html = await res.text()
let versionString: string | null = null
let buildId: string | null = null
// ── Extract from embedded community data ──────────────────
// The community hub embeds JSON in data attributes
const communityDataMatch = html.match(
/data-community="([^"]+)"/i,
)
if (communityDataMatch) {
try {
const decoded = communityDataMatch[1]
.replace(/&quot;/g, '"')
.replace(/&amp;/g, "&")
const data = JSON.parse(decoded)
// Community data usually has CLANSTEAMID, APPID, IS_OGG, etc.
// Not version info, but checked for completeness
if (data.APPID && !buildId) {
// APPID is just the same app ID — not useful
}
} catch {
// Non-fatal
}
}
// ── Extract from "View Stats" link ────────────────────────
// Sometimes game stats pages have more info
// ── Parse update announcements for version numbers ────────
// Look for patterns like "Counter-Strike 2 Update" or "Game Update 1.4.2"
const updateTitlePatterns = [
// "Game Name Update" — generic, skip
// "Update 1.4.2" pattern
/(?:Update|Patch)\s+(\d+\.\d+(?:\.\d+)?(?:[a-z]\d*)?)/gi,
// "Version 1.4.2" pattern
/Version\s+(\d+\.\d+(?:\.\d+)?(?:[a-z]\d*)?)/gi,
// "Build 12345678" pattern
/Build\s+(\d{5,10})/gi,
]
for (const pattern of updateTitlePatterns) {
const match = pattern.exec(html)
if (match) {
const val = match[1]
if (/^\d{5,10}$/.test(val)) {
if (!buildId) buildId = val
} else if (/^\d+\.\d+/.test(val)) {
if (!versionString) versionString = val
}
}
}
// ── Extract build ID from JS globals ──────────────────────
const buildTimestampMatch = html.match(
/"BUILD_TIMESTAMP"[:\s]+(\d{9,10})/,
)
if (buildTimestampMatch && !buildId) {
// BUILD_TIMESTAMP is the page build time, not game build
// But if we have nothing else, it's a signal
// Skipping — too noisy
}
// ── Try apphub sidebar for build/version info ──────────────
const apphubMatch = html.match(
/apphub_AppInfo[^>]*>([\s\S]*?)<\/div>/i,
)
if (apphubMatch) {
const apphubHtml = apphubMatch[1]
// Look for "Build ID" or "Current version" labels
const buildLabelMatch = apphubHtml.match(
/Build\s*ID[:\s]*<\/span>\s*([\d,]+)/i,
)
if (buildLabelMatch && !buildId) {
buildId = buildLabelMatch[1].replace(/,/g, "")
}
const versionLabelMatch = apphubHtml.match(
/(?:Current|Latest)\s*Version[:\s]*<\/span>\s*([\d.]+)/i,
)
if (versionLabelMatch && !versionString) {
versionString = versionLabelMatch[1]
}
}
const success = !!(versionString || buildId)
return {
versionString,
buildId,
source,
success,
error: success ? undefined : "No version data found on community hub",
}
} catch (err) {
return {
versionString: null,
buildId: null,
source,
success: false,
error: err instanceof Error ? err.message : "Unknown error",
}
}
}
+109
View File
@@ -0,0 +1,109 @@
/**
* Version Fetcher Orchestrator
*
* Tries multiple strategies to fetch game version/build info without a Steam API key.
*
* Strategies (tried in order):
* 1. UpToDateCheck API — Fastest, works for Valve dedicated-server games
* 2. Store Page Scrape — Extracts from embedded JS/JSON-LD on store page
* 3. Community Hub Scrape — Extracts from community hub update announcements
* 4. Store API Heuristic — Uses store API + header image timestamps
*
* The first strategy that returns a versionString wins.
* If no strategy returns a versionString, we fall back to the first strategy
* that returns a buildId.
*
* Client-side preferred strategies (2, 3, 4) should be called from the browser
* to avoid server-wide rate limiting. Strategy 1 (UpToDateCheck) is safe
* for server-side use.
*/
import type { VersionFetchResult, VersionFetchStrategy } from "./types"
import { fetchUpToDateCheck } from "./uptodate-api"
import { fetchStorePage } from "./store-page"
import { fetchCommunityHub } from "./community-hub"
import { fetchStoreApi } from "./store-api"
export type { VersionFetchResult, VersionFetchStrategy }
/** All available strategies */
export const ALL_STRATEGIES: Array<{
name: string
fn: VersionFetchStrategy
/** If true, this strategy should be called from the client (browser), not server */
preferClient: boolean
}> = [
{ name: "UpToDateCheck API", fn: fetchUpToDateCheck, preferClient: false },
{ name: "Store Page Scrape", fn: fetchStorePage, preferClient: true },
{ name: "Community Hub Scrape", fn: fetchCommunityHub, preferClient: true },
{ name: "Store API Heuristic", fn: fetchStoreApi, preferClient: true },
]
/** Server-safe strategies (won't trigger rate limits on server IP) */
export const SERVER_STRATEGIES = ALL_STRATEGIES.filter((s) => !s.preferClient)
/** Client-side strategies (use browser IP to avoid server-wide rate limits) */
export const CLIENT_STRATEGIES = ALL_STRATEGIES.filter((s) => s.preferClient)
/**
* Run all strategies and return the best result.
*
* Priority:
* 1. Result with versionString
* 2. Result with buildId
* 3. First failure
*/
export async function fetchAllVersions(
steamAppId: number,
strategies = ALL_STRATEGIES,
): Promise<{
/** Best combined result */
best: VersionFetchResult
/** Individual results from each strategy */
all: VersionFetchResult[]
/** Which strategy produced the best result */
bestSource: string
}> {
const results = await Promise.all(
strategies.map((s) => s.fn(steamAppId)),
)
// Find best: prefer versionString > buildId > nothing
let best: VersionFetchResult = results[0]
for (const result of results) {
if (result.versionString && !best.versionString) {
best = result
}
if (!best.versionString && !best.buildId && result.buildId) {
best = result
}
}
return {
best,
all: results,
bestSource: best.source,
}
}
/**
* Run a single strategy by name.
*/
export async function fetchVersionByStrategy(
steamAppId: number,
strategyName: string,
): Promise<VersionFetchResult> {
const strategy = ALL_STRATEGIES.find(
(s) => s.name === strategyName,
)
if (!strategy) {
return {
versionString: null,
buildId: null,
source: strategyName,
success: false,
error: `Unknown strategy: ${strategyName}`,
}
}
return strategy.fn(steamAppId)
}
+133
View File
@@ -0,0 +1,133 @@
/**
* Strategy 4: Steam Store API (appdetails) + Heuristics
*
* Uses the public store.steampowered.com/api/appdetails endpoint (no API key).
* While it doesn't directly contain version/build IDs, we can extract:
* - The ?t= cache-busting timestamp from image URLs (changes with updates)
* - Release date information
* - Any version hints in the game description
*/
import type { VersionFetchResult } from "./types"
const APPDETAILS_URL = "https://store.steampowered.com/api/appdetails"
interface AppDetailsResponse {
[appId: string]: {
success: boolean
data?: {
name: string
header_image?: string
release_date?: {
coming_soon: boolean
date: string
}
detailed_description?: string
about_the_game?: string
// Steam may include additional fields
[key: string]: unknown
}
}
}
export async function fetchStoreApi(
steamAppId: number,
): Promise<VersionFetchResult> {
const source = "Store API Heuristic"
try {
const res = await fetch(
`${APPDETAILS_URL}?appids=${steamAppId}`,
{
headers: {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
Accept: "application/json",
},
signal: AbortSignal.timeout(10000),
},
)
if (!res.ok) {
return {
versionString: null,
buildId: null,
source,
success: false,
error: `HTTP ${res.status}`,
}
}
const data: AppDetailsResponse = await res.json()
const appData = data[String(steamAppId)]
if (!appData?.success || !appData.data) {
return {
versionString: null,
buildId: null,
source,
success: false,
error: "App not found or not available",
}
}
const { data: details } = appData
let versionString: string | null = null
let buildId: string | null = null
// ── Extract timestamp from header_image URL ────────────────
// e.g. header.jpg?t=1749053861 — changes with every game update
if (details.header_image) {
const tsMatch = details.header_image.match(/\?t=(\d{9,10})/)
if (tsMatch) {
buildId = tsMatch[1]
}
}
// ── Search description for version mentions ────────────────
const textToSearch = [
details.about_the_game,
details.detailed_description,
]
.filter(Boolean)
.join(" ")
if (textToSearch) {
// Strip HTML tags
const cleanText = textToSearch.replace(/<[^>]+>/g, " ")
// Look for version patterns in description
const versionPatterns = [
/(?:version|patch|update)\s*[:#]?\s*(\d+\.\d+(?:\.\d+)?(?:[a-z]\d*)?)/i,
/v?(\d+\.\d+\.\d+(?:[a-z]\d*)?)/i,
]
for (const pattern of versionPatterns) {
const match = cleanText.match(pattern)
if (match) {
versionString = match[1]
break
}
}
}
const success = !!(versionString || buildId)
return {
versionString,
buildId,
source,
success,
error: success
? undefined
: "No version hints found in store API data",
}
} catch (err) {
return {
versionString: null,
buildId: null,
source,
success: false,
error: err instanceof Error ? err.message : "Unknown error",
}
}
}
+201
View File
@@ -0,0 +1,201 @@
/**
* Strategy 2: Steam Store Page Scraping
*
* Fetches the store.steampowered.com/app/{appid} HTML page and extracts
* version/build info from embedded JavaScript data, JSON-LD structured data,
* and meta tags.
*
* Designed for client-side use (uses the browser's IP to avoid server-wide rate limits).
* Server-side calls may be rate-limited by Steam.
*/
import type { VersionFetchResult } from "./types"
const STORE_URL = "https://store.steampowered.com/app"
export async function fetchStorePage(
steamAppId: number,
): Promise<VersionFetchResult> {
const source = "Store Page Scrape"
try {
const res = await fetch(`${STORE_URL}/${steamAppId}`, {
headers: {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
Accept: "text/html",
},
signal: AbortSignal.timeout(10000),
})
if (!res.ok) {
return {
versionString: null,
buildId: null,
source,
success: false,
error: `HTTP ${res.status}`,
}
}
const html = await res.text()
let versionString: string | null = null
let buildId: string | null = null
// ── Extract from JSON-LD structured data ──────────────────
const ldJsonMatch = html.match(
/<script[^>]*type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/i,
)
if (ldJsonMatch) {
try {
const ld = JSON.parse(ldJsonMatch[1])
// Look for version in SoftwareApplication schema
if (ld.version && typeof ld.version === "string") {
versionString = ld.version
}
// Sometimes build info is in description or other fields
if (ld.description && !versionString) {
const verMatch = ld.description.match(
/(?:version|build)\s*[:#]?\s*([\d.]+[\w.]*)/i,
)
if (verMatch) versionString = verMatch[1]
}
} catch {
// JSON-LD parse failure — non-fatal
}
}
// ── Extract from embedded JS data objects ─────────────────
// Pattern: g_rgAssetData, g_rgAppData, etc.
const jsDataPatterns = [
/(?:g_rgAppData|g_rgAssetData|g_rgDepotData)\s*=\s*(\{[\s\S]*?\});/gi,
/data-ds-appdata\s*=\s*['"]([^'"]+)['"]/gi,
]
for (const pattern of jsDataPatterns) {
const match = pattern.exec(html)
if (match) {
try {
const data = JSON.parse(match[1])
// Search for version/build in nested objects
const found = findVersionInObject(data)
if (found.versionString && !versionString)
versionString = found.versionString
if (found.buildId && !buildId) buildId = found.buildId
} catch {
// Non-fatal
}
}
}
// ── Extract from meta tags ────────────────────────────────
if (!versionString) {
const metaVersion = html.match(
/<meta[^>]+name="[^"]*version[^"]*"[^>]+content="([^"]+)"/i,
)
if (metaVersion) versionString = metaVersion[1]
}
// ── Extract build ID from image URLs ──────────────────────
// The ?t= timestamp in header image URLs changes with each update
if (!buildId) {
const imgMatch = html.match(
/header\.jpg\?t=(\d{9,10})/i,
)
if (imgMatch) {
// This is a timestamp, not a build ID per se, but it changes with updates
// Use it as a build-ish identifier
buildId = imgMatch[1]
}
}
// ── Extract version from changelog/update sections ────────
if (!versionString) {
// Look for "Update X.Y" or "Patch X.Y" in the page text
const updateMatch = html.match(
/(?:Update|Patch|Version)\s+(\d+\.\d+(?:\.\d+)?(?:[a-z]\d*)?)/i,
)
if (updateMatch) versionString = updateMatch[1]
}
const success = !!(versionString || buildId)
return {
versionString,
buildId,
source,
success,
error: success ? undefined : "No version data found on store page",
}
} catch (err) {
return {
versionString: null,
buildId: null,
source,
success: false,
error: err instanceof Error ? err.message : "Unknown error",
}
}
}
/** Recursively search an object for version/build strings */
function findVersionInObject(
obj: unknown,
depth = 0,
): { versionString: string | null; buildId: string | null } {
if (depth > 4 || !obj || typeof obj !== "object") {
return { versionString: null, buildId: null }
}
const record = obj as Record<string, unknown>
let versionString: string | null = null
let buildId: string | null = null
const versionKeys = [
"version",
"clientversion",
"gameversion",
"app_version",
"displayVersion",
]
const buildKeys = [
"buildid",
"build_id",
"build",
"app_build",
"publicbuild",
]
for (const key of Object.keys(record)) {
const val = record[key]
if (typeof val === "string") {
const lowerKey = key.toLowerCase()
if (
!versionString &&
versionKeys.some((k) => lowerKey.includes(k)) &&
/^\d+\.\d+/.test(val)
) {
versionString = val
}
if (
!buildId &&
buildKeys.some((k) => lowerKey.includes(k)) &&
/^\d{3,10}$/.test(val)
) {
buildId = val
}
}
if (typeof val === "number" && val > 1000 && val < 99999999) {
const lowerKey = key.toLowerCase()
if (!buildId && buildKeys.some((k) => lowerKey.includes(k))) {
buildId = String(val)
}
}
if (val && typeof val === "object" && !Array.isArray(val)) {
const nested = findVersionInObject(val, depth + 1)
if (!versionString) versionString = nested.versionString
if (!buildId) buildId = nested.buildId
}
}
return { versionString, buildId }
}
+18
View File
@@ -0,0 +1,18 @@
/** Result from a single version-fetch strategy */
export interface VersionFetchResult {
/** Human-readable version string like "1.2.3" or "Patch 4.0" */
versionString: string | null
/** Numeric build ID from Steam */
buildId: string | null
/** Which strategy produced this result */
source: string
/** Whether the strategy succeeded (even if partial — e.g. buildId only) */
success: boolean
/** Error message if strategy failed completely */
error?: string
}
/** A version-fetch strategy function */
export type VersionFetchStrategy = (
steamAppId: number,
) => Promise<VersionFetchResult>
+103
View File
@@ -0,0 +1,103 @@
/**
* Strategy 1: Steam UpToDateCheck API
*
* Uses the public ISteamApps/UpToDateCheck endpoint (no API key required).
* Primarily works for Valve games with dedicated servers (CS2, TF2, Dota 2, etc.).
*
* Returns:
* - required_version: numeric version (used as buildId fallback)
* - message: sometimes contains a named version like "1.41.6.1"
*
* Non-Valve games typically return { success: false }.
*/
import type { VersionFetchResult } from "./types"
const UPTODATE_URL =
"https://api.steampowered.com/ISteamApps/UpToDateCheck/v1"
interface UpToDateResponse {
response: {
success: boolean
up_to_date?: boolean
version_is_listable?: boolean
required_version?: number
message?: string
error?: string
}
}
export async function fetchUpToDateCheck(
steamAppId: number,
): Promise<VersionFetchResult> {
const source = "UpToDateCheck API"
try {
const res = await fetch(
`${UPTODATE_URL}?appid=${steamAppId}&version=0`,
{
headers: {
"User-Agent": "DeckyVault/1.0 (deckyvault.xyz; game version lookup)",
},
signal: AbortSignal.timeout(8000),
},
)
if (!res.ok) {
return {
versionString: null,
buildId: null,
source,
success: false,
error: `HTTP ${res.status}`,
}
}
const data: UpToDateResponse = await res.json()
if (!data.response.success) {
return {
versionString: null,
buildId: null,
source,
success: false,
error: data.response.error ?? "App not supported",
}
}
let versionString: string | null = null
let buildId: string | null = null
// Extract named version from message
// e.g. "Server version required: 1.41.6.1" → "1.41.6.1"
if (data.response.message) {
const namedMatch = data.response.message.match(
/(?:version|required)[:\s]+([\d.]+)/i,
)
if (namedMatch) {
versionString = namedMatch[1]
}
}
// Use required_version as buildId
if (data.response.required_version) {
buildId = String(data.response.required_version)
}
const success = !!(versionString || buildId)
return {
versionString,
buildId,
source,
success,
error: success ? undefined : "No version data in response",
}
} catch (err) {
return {
versionString: null,
buildId: null,
source,
success: false,
error: err instanceof Error ? err.message : "Unknown error",
}
}
}