From 0f583c3f73d5d84949af99703c6484d1f651bfaf Mon Sep 17 00:00:00 2001 From: FalconChipp <112320693+FalconChipp@users.noreply.github.com> Date: Mon, 11 May 2026 15:42:09 +0100 Subject: [PATCH 1/2] Add TCGTracking pricing integration and scripts Introduce TCGTracking-based pricing pipeline and server support. - Add new scripts: scripts/preloadTCGTracking.ts (caches TCGTracking set pricing), scripts/updatePricingFromTCGTracking.ts (resolves cached pricing to a resolved-pricing artifact), and scripts/utils-data/tcgtracking.ts (helpers, fetcher, cache path helpers, category mapping, import URL fix). - Extend scripts/preloadTCGPlayer.ts to also download TCGCSV prices into var/models/tcgplayer/prices as a fallback. - Add set ID override support, daily cache freshness checks, polite fetch delays, failure reporting (var/reports/tcgtracking-preload-failures.json), and --strict/--apply flags for control. - New pricing resolver supports three modes via PRICING_SOURCE: tcgtracking (default), tcgcsv (old fallback), tcgtracking-only; updatePricingFromTCGTracking supports dry-run and apply and writes var/models/tcgtracking/resolved-pricing.json when applied. - Wire the server to use the new provider: add server/src/libs/providers/tcgplayer/tcgtracking.ts and load it by default in server/src/libs/providers/tcgplayer/index.ts unless official credentials or PRICING_SOURCE=tcgcsv are present. Notes: The scripts do not mutate card source files or create/remap variants; caching filenames use existing TCGPlayer set IDs to remain compatible with existing data. --- package.json | 16 +- scripts/preloadTCGPlayer.ts | 14 + scripts/preloadTCGTracking.ts | 384 +++++++ scripts/updatePricingFromTCGTracking.ts | 948 ++++++++++++++++++ scripts/utils-data/tcgtracking.ts | 327 ++++++ server/src/libs/providers/tcgplayer/index.ts | 14 +- .../libs/providers/tcgplayer/tcgtracking.ts | 126 +++ 7 files changed, 1825 insertions(+), 4 deletions(-) create mode 100644 scripts/preloadTCGTracking.ts create mode 100644 scripts/updatePricingFromTCGTracking.ts create mode 100644 scripts/utils-data/tcgtracking.ts create mode 100644 server/src/libs/providers/tcgplayer/tcgtracking.ts diff --git a/package.json b/package.json index b9ec2112dd..c44b29e9b6 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,19 @@ "dex:audit:report": "bun scripts/pokedexIdFixer/audit-dex-ids.ts --report", "dex:fix:dry-run": "bun scripts/pokedexIdFixer/fix-dex-ids.ts --dry-run", "dex:fix:apply": "bun scripts/pokedexIdFixer/fix-dex-ids.ts --apply", - "dex:lint": "bun scripts/pokedexIdFixer/lint-dex-ids.ts" + "dex:lint": "bun scripts/pokedexIdFixer/lint-dex-ids.ts", + + "tcgtracking:preload": "bun scripts/preloadTCGTracking.ts", + "tcgtracking:update-pricing": "bun scripts/updatePricingFromTCGTracking.ts", + "tcgtracking:update-pricing:dry-run": "bun scripts/updatePricingFromTCGTracking.ts --dry-run", + "tcgtracking:update-pricing:apply": "bun scripts/updatePricingFromTCGTracking.ts --apply", + + "pricing:preload": "bun scripts/preloadTCGTracking.ts", + "pricing:preload:backup": "bun scripts/preloadTCGPlayer.ts", + "pricing:update:dry-run": "bun scripts/updatePricingFromTCGTracking.ts --dry-run", + "pricing:update:apply": "bun scripts/updatePricingFromTCGTracking.ts --apply", + "pricing:sync": "bun scripts/preloadTCGTracking.ts && bun scripts/updatePricingFromTCGTracking.ts --apply", + "pricing:sync:with-backup": "bun scripts/preloadTCGTracking.ts && bun scripts/preloadTCGPlayer.ts && bun scripts/updatePricingFromTCGTracking.ts --apply", }, "devDependencies": { "@dzeio/object-util": "^1.9.2", @@ -22,4 +34,4 @@ "ts-node": "^10.0.0", "typescript": "^5.0.0" } -} +} \ No newline at end of file diff --git a/scripts/preloadTCGPlayer.ts b/scripts/preloadTCGPlayer.ts index affb658216..159b7c25e5 100644 --- a/scripts/preloadTCGPlayer.ts +++ b/scripts/preloadTCGPlayer.ts @@ -54,6 +54,20 @@ try { await fs.writeFile(`${folder}/${id}.json`, JSON.stringify(products)) } + // Load prices (used as TCGCSV fallback by updatePricingFromTCGTracking) + console.log('Loading prices...') + const pricesFolder = baseFolder + '/prices' + await fs.mkdir(pricesFolder, { recursive: true }) + + for (const id of ids) { + console.log('Loading prices', id) + const prices = await fetch(`https://tcgcsv.com/tcgplayer/3/${id}/prices`, { + headers: { 'User-Agent': userAgent } + }) + .then((it) => it.json()) + await fs.writeFile(`${pricesFolder}/${id}.json`, JSON.stringify(prices)) + } + console.log('done') } catch (error) { diff --git a/scripts/preloadTCGTracking.ts b/scripts/preloadTCGTracking.ts new file mode 100644 index 0000000000..ea1685ba73 --- /dev/null +++ b/scripts/preloadTCGTracking.ts @@ -0,0 +1,384 @@ +import fs from 'node:fs/promises' +import path from 'node:path' + +import { + fetchTCGTrackingSetPricing, + getCategoryIdForSetFile, + getTCGTrackingPricingCacheFile, + sleep, + toImportUrl, + writeJsonFile, + type TCGTrackingSetCacheTarget, +} from './utils-data/tcgtracking' + +interface TCGDexSetModule { + default?: { + id?: string + name?: string + thirdParty?: { + tcgplayer?: number + } + } +} + +interface TCGTrackingPreloadFailure { + categoryId: number + tcgplayerSetId: number + tcgTrackingSetId?: number + sourceFile: string + status: 'missing-endpoint' | 'fetch-failed' + error: string +} + +/** + * TCGTracking sometimes exposes a set under a different ID than the + * existing TCGDex set.thirdParty.tcgplayer ID. + * + * Key: existing TCGDex / TCGPlayer set ID + * Value: TCGTracking set ID used for API fetching + * + * IMPORTANT: + * The cache file is still written using the original TCGDex / TCGPlayer set ID. + * That keeps updatePricingFromTCGTracking.ts compatible with existing card data. + */ +const TCGTRACKING_SET_ID_OVERRIDES: Record = { + // Black Bolt + 22325: 24325, +} + +/** + * CHANGE: + * Added a TCGTracking pricing preloader. + * + * This script does not edit card files. + * This script does not create or map variants. + * This script does not remove TCGCSV. + * + * It only fetches pricing from TCGTracking and caches it locally. + * + * CHANGE: + * Failed fetches now write a report instead of causing the script to fail by default. + * + * Use --strict if you want failures to exit with code 1. + * + * CHANGE: + * Added TCGTracking set ID overrides. + * + * This allows sets like Black Bolt to be fetched from TCGTracking using their + * current TCGTracking set ID while still caching under the existing TCGDex + * thirdParty.tcgplayer set ID. + */ +async function main(): Promise { + const setFiles = await findSetFiles([ + path.join(process.cwd(), 'data'), + path.join(process.cwd(), 'data-asia'), + ]) + + const failures: TCGTrackingPreloadFailure[] = [] + const targets = await getTCGTrackingCacheTargets(setFiles) + + console.log(`Found ${targets.length} TCGDex sets with thirdParty.tcgplayer IDs.`) + + let fetched = 0 + let skipped = 0 + let failed = 0 + + for (const target of targets) { + const cacheFile = getTCGTrackingPricingCacheFile( + target.categoryId, + target.tcgplayerSetId, + ) + + const fetchSetId = target.tcgTrackingSetId ?? target.tcgplayerSetId + const hasOverride = fetchSetId !== target.tcgplayerSetId + + try { + const shouldSkip = await isFreshCache(cacheFile) + + if (shouldSkip) { + skipped += 1 + console.log( + `Skipping category ${target.categoryId}, set ${target.tcgplayerSetId}; cache is fresh.`, + ) + continue + } + + console.log( + `Fetching TCGTracking pricing: category ${target.categoryId}, set ${fetchSetId}` + + (hasOverride + ? ` (override for TCGDex set ${target.tcgplayerSetId})` + : ''), + ) + + const pricing = await fetchTCGTrackingSetPricing( + target.categoryId, + fetchSetId, + ) + + await writeJsonFile(cacheFile, { + source: 'tcgtracking', + categoryId: target.categoryId, + + // Existing TCGDex / TCGPlayer set ID. + // This is used for the cache filename and updater lookup. + tcgplayerSetId: target.tcgplayerSetId, + + // Actual TCGTracking set ID used when fetching. + tcgTrackingSetId: fetchSetId, + + sourceFile: normalizePathForReport(target.sourceFile), + fetchedAt: new Date().toISOString(), + data: pricing, + }) + + fetched += 1 + + // Polite delay between requests. + await sleep(100) + } catch (error) { + failed += 1 + + const errorMessage = getErrorMessage(error) + + failures.push({ + categoryId: target.categoryId, + tcgplayerSetId: target.tcgplayerSetId, + tcgTrackingSetId: hasOverride ? fetchSetId : undefined, + sourceFile: normalizePathForReport(target.sourceFile), + status: errorMessage.includes('404 Not Found') + ? 'missing-endpoint' + : 'fetch-failed', + error: errorMessage, + }) + + console.error( + `Failed to fetch category ${target.categoryId}, set ${fetchSetId}` + + (hasOverride + ? ` (override for TCGDex set ${target.tcgplayerSetId})` + : ''), + ) + console.error(errorMessage) + } + } + + if (failures.length > 0) { + const reportFile = path.join( + process.cwd(), + 'var', + 'reports', + 'tcgtracking-preload-failures.json', + ) + + await writeFailureReport(reportFile, failures) + + console.log('') + console.log(`Failure report written to ${normalizePathForReport(reportFile)}`) + } + + console.log('') + console.log('TCGTracking preload complete.') + console.log(`Fetched: ${fetched}`) + console.log(`Skipped: ${skipped}`) + console.log(`Failed: ${failed}`) + + const isStrict = process.argv.includes('--strict') + + if (failed > 0 && isStrict) { + process.exitCode = 1 + } +} + +/** + * Finds set files. + * + * The existing repo structure has scripts inside: + * + * H:\cards-database\scripts + * + * and data folders at repo root: + * + * data + * data-asia + * + * This scans both. + */ +async function findSetFiles(rootFolders: string[]): Promise { + const files: string[] = [] + + for (const rootFolder of rootFolders) { + const exists = await pathExists(rootFolder) + + if (!exists) { + continue + } + + await walk(rootFolder, files) + } + + return files.filter((file) => { + if (!file.endsWith('.ts')) { + return false + } + + // CHANGE: + // Avoid card files where possible by only allowing shallow set files. + // + // Existing scripts often scan data/*/*.ts for set files. + // This keeps the same intent but is slightly safer. + const relative = path.relative(process.cwd(), file).replace(/\\/g, '/') + const parts = relative.split('/') + + // data/Serie/Set.ts + // data-asia/Serie/Set.ts + return ( + (parts[0] === 'data' || parts[0] === 'data-asia') && + parts.length === 3 + ) + }) +} + +async function walk(folder: string, files: string[]): Promise { + const entries = await fs.readdir(folder, { + withFileTypes: true, + }) + + for (const entry of entries) { + const fullPath = path.join(folder, entry.name) + + if (entry.isDirectory()) { + await walk(fullPath, files) + continue + } + + files.push(fullPath) + } +} + +/** + * Reads TCGDex set files and extracts thirdParty.tcgplayer set IDs. + * + * IMPORTANT: + * We are using the existing TCGDex thirdParty.tcgplayer set IDs. + * We are not doing new set matching. + */ +async function getTCGTrackingCacheTargets( + setFiles: string[], +): Promise { + const targets: TCGTrackingSetCacheTarget[] = [] + + for (const setFile of setFiles) { + const setModule = (await import(toImportUrl(setFile))) as TCGDexSetModule + const setData = setModule.default + + const tcgplayerSetId = setData?.thirdParty?.tcgplayer + + if (typeof tcgplayerSetId !== 'number') { + continue + } + + targets.push({ + categoryId: getCategoryIdForSetFile(setFile), + tcgplayerSetId, + tcgTrackingSetId: TCGTRACKING_SET_ID_OVERRIDES[tcgplayerSetId], + sourceFile: setFile, + }) + } + + return dedupeTargets(targets) +} + +function dedupeTargets( + targets: TCGTrackingSetCacheTarget[], +): TCGTrackingSetCacheTarget[] { + const seen = new Set() + const deduped: TCGTrackingSetCacheTarget[] = [] + + for (const target of targets) { + const fetchSetId = target.tcgTrackingSetId ?? target.tcgplayerSetId + const key = `${target.categoryId}:${target.tcgplayerSetId}:${fetchSetId}` + + if (seen.has(key)) { + continue + } + + seen.add(key) + deduped.push(target) + } + + return deduped +} + +/** + * Keeps pricing cache fresh for one day. + * + * CHANGE: + * This mirrors the idea that pricing should update daily, + * while avoiding unnecessary full refetches. + */ +async function isFreshCache(filePath: string): Promise { + try { + const stat = await fs.stat(filePath) + const ageMs = Date.now() - stat.mtimeMs + const oneDayMs = 24 * 60 * 60 * 1000 + + return ageMs < oneDayMs + } catch (error) { + if (isNodeError(error) && error.code === 'ENOENT') { + return false + } + + throw error + } +} + +async function writeFailureReport( + reportFile: string, + failures: TCGTrackingPreloadFailure[], +): Promise { + await fs.mkdir(path.dirname(reportFile), { + recursive: true, + }) + + await fs.writeFile( + reportFile, + `${JSON.stringify( + { + createdAt: new Date().toISOString(), + failures, + }, + null, + '\t', + )}\n`, + 'utf8', + ) +} + +async function pathExists(filePath: string): Promise { + try { + await fs.access(filePath) + return true + } catch { + return false + } +} + +function normalizePathForReport(filePath: string): string { + return path.relative(process.cwd(), filePath).replace(/\\/g, '/') +} + +function getErrorMessage(error: unknown): string { + if (error instanceof Error) { + return error.message + } + + return String(error) +} + +function isNodeError(error: unknown): error is NodeJS.ErrnoException { + return error instanceof Error && 'code' in error +} + +main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) \ No newline at end of file diff --git a/scripts/updatePricingFromTCGTracking.ts b/scripts/updatePricingFromTCGTracking.ts new file mode 100644 index 0000000000..df95a5e047 --- /dev/null +++ b/scripts/updatePricingFromTCGTracking.ts @@ -0,0 +1,948 @@ +import fs from 'node:fs/promises' +import path from 'node:path' + +import { + getCategoryIdForSetFile, + getTCGTrackingPricingCacheFile, + toImportUrl, + type TCGTrackingCategoryId, +} from './utils-data/tcgtracking' + +interface TCGDexSet { + id?: string + name?: Record + serie?: { + id?: string + name?: Record + } + thirdParty?: { + tcgplayer?: number + cardmarket?: number + } +} + +interface TCGDexVariant { + type?: string + subtype?: string + size?: string + stamp?: string[] + foil?: string + thirdParty?: { + tcgplayer?: number + cardmarket?: number + } +} + +interface TCGDexCard { + name?: Record + set?: TCGDexSet + thirdParty?: { + tcgplayer?: number + cardmarket?: number + } + variants?: Record | TCGDexVariant[] +} + +interface TCGDexCardModule { + default?: TCGDexCard +} + +interface CachedTCGTrackingPricingFile { + source: 'tcgtracking' + categoryId: TCGTrackingCategoryId + tcgplayerSetId: number + sourceFile: string + fetchedAt: string + data: { + set_id: number + updated?: string + prices?: Record + } +} + +interface TCGTrackingProductPricing { + tcg?: Record +} + +interface TCGTrackingPriceBucket { + low?: number + market?: number +} + +interface TCGCSVPricesResult { + productId: number + lowPrice: number | null + midPrice: number | null + highPrice: number | null + marketPrice: number | null + directLowPrice: number | null + subTypeName: string +} + +interface TCGCSVPricesFile { + totalItems: number + success: boolean + errors: unknown[] + results: TCGCSVPricesResult[] +} + +interface ResolvedPrice { + productId: number + priceType: string + low?: number + market?: number +} + +interface ResolvedCardPrice { + cardId: string + cardFile: string + cardName?: string + setId?: string + setName?: string + categoryId: number + tcgplayerSetId: number + sourceUpdated?: string + sourceFetchedAt: string + prices: ResolvedCardPriceEntry[] +} + +interface ResolvedCardPriceEntry { + target: 'card' | 'variant' + variantIndex?: number + variant?: { + type?: string + subtype?: string + size?: string + stamp?: string[] + foil?: string + } + tcgplayerProductId: number + priceType: string + low?: number + market?: number +} + +interface MissingPrice { + cardId: string + cardFile: string + cardName?: string + setId?: string + setName?: string + categoryId?: number + tcgplayerSetId?: number + tcgplayerProductId?: number + target: 'card' | 'variant' | 'set-cache' + variantIndex?: number + reason: + | 'missing-card-tcgplayer-id' + | 'missing-variant-tcgplayer-id' + | 'missing-set-tcgplayer-id' + | 'missing-cache' + | 'missing-product-price' + | 'missing-tcg-prices' +} + +interface CountByReason { + reason: MissingPrice['reason'] + count: number +} + +interface MissingCacheSummary { + categoryId?: number + tcgplayerSetId?: number + setId?: string + setName?: string + count: number +} + +interface CountByTarget { + target: ResolvedCardPriceEntry['target'] + count: number +} + +interface CountByPriceType { + priceType: string + count: number +} + +interface PricingUpdateReport { + createdAt: string + mode: 'dry-run' | 'apply' + summary: { + cardFilesScanned: number + cardFilesWithPrices: number + priceEntriesResolved: number + missing: number + + /** + * CHANGE: + * This is the number of card records that could not find a cache file. + * + * It is not the number of unique missing cache files. + */ + missingCacheRecords: number + + /** + * CHANGE: + * This is the actual number of unique set cache files missing. + */ + uniqueMissingCaches: number + + outputWritten: boolean + } + resolvedByTarget: CountByTarget[] + priceTypes: CountByPriceType[] + missingByReason: CountByReason[] + missingCaches: MissingCacheSummary[] + resolved: ResolvedCardPrice[] + missing: MissingPrice[] +} + +/** + * Controls which pricing source is primary. + * + * tcgtracking (default): TCGTracking is primary; TCGCSV cache is used as fallback + * when TCGTracking has no data for a set. + * tcgcsv: TCGCSV-only; skips TCGTracking cache entirely. + * tcgtracking-only: TCGTracking only; no TCGCSV fallback. + * + * Set via environment variable: + * PRICING_SOURCE=tcgtracking bun scripts/updatePricingFromTCGTracking.ts --apply + */ +const PRICING_SOURCE = (process.env.PRICING_SOURCE ?? 'tcgtracking') as + | 'tcgtracking' + | 'tcgcsv' + | 'tcgtracking-only' + +/** + * CHANGE: + * Added TCGTracking pricing resolver. + * + * This script: + * - reads cached TCGTracking pricing files created by preloadTCGTracking.ts + * - reads existing TCGDex card files + * - uses existing thirdParty.tcgplayer IDs + * - resolves prices from TCGTracking + * - writes a resolved pricing artifact when --apply is used + * + * This script does NOT: + * - create variants + * - remap variants + * - edit card source files + * - remove TCGCSV fallback + * + * Why: + * TCGTracking pricing cache is keyed by existing TCGPlayer product IDs: + * + * data.prices[tcgplayerProductId].tcg[priceType].market + * data.prices[tcgplayerProductId].tcg[priceType].low + */ +async function main(): Promise { + const mode = getMode() + + const cardFiles = await findCardFiles([ + path.join(process.cwd(), 'data'), + path.join(process.cwd(), 'data-asia'), + ]) + + const resolved: ResolvedCardPrice[] = [] + const missing: MissingPrice[] = [] + + let missingCacheRecords = 0 + let priceEntriesResolved = 0 + + const cacheByKey = new Map() + + for (const cardFile of cardFiles) { + const cardId = getCardIdFromFile(cardFile) + const cardModule = (await import(toImportUrl(cardFile))) as TCGDexCardModule + const card = cardModule.default + + if (!card) { + continue + } + + const categoryId = getCategoryIdForSetFile(cardFile) + const tcgplayerSetId = card.set?.thirdParty?.tcgplayer + const cardName = getEnglishName(card.name) + const setId = card.set?.id + const setName = getEnglishName(card.set?.name) + + if (typeof tcgplayerSetId !== 'number') { + missing.push({ + cardId, + cardFile: normalizePathForReport(cardFile), + cardName, + setId, + setName, + categoryId, + target: 'set-cache', + reason: 'missing-set-tcgplayer-id', + }) + continue + } + + const cache = await getCachedPricingFile({ + categoryId, + tcgplayerSetId, + cacheByKey, + }) + + if (!cache) { + missingCacheRecords += 1 + + missing.push({ + cardId, + cardFile: normalizePathForReport(cardFile), + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + target: 'set-cache', + reason: 'missing-cache', + }) + continue + } + + const cardPriceEntries = resolvePricesForCard({ + card, + cardId, + cardFile, + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + cache, + missing, + }) + + if (cardPriceEntries.length === 0) { + continue + } + + priceEntriesResolved += cardPriceEntries.length + + resolved.push({ + cardId, + cardFile: normalizePathForReport(cardFile), + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + sourceUpdated: cache.data.updated, + sourceFetchedAt: cache.fetchedAt, + prices: cardPriceEntries, + }) + } + + const missingByReason = groupMissingByReason(missing) + const missingCaches = groupMissingCaches(missing) + const resolvedByTarget = groupResolvedByTarget(resolved) + const priceTypes = groupPriceTypes(resolved) + + const report: PricingUpdateReport = { + createdAt: new Date().toISOString(), + mode, + summary: { + cardFilesScanned: cardFiles.length, + cardFilesWithPrices: resolved.length, + priceEntriesResolved, + missing: missing.length, + missingCacheRecords, + uniqueMissingCaches: missingCaches.length, + outputWritten: mode === 'apply', + }, + resolvedByTarget, + priceTypes, + missingByReason, + missingCaches, + resolved, + missing, + } + + await writeReport(report) + + if (mode === 'apply') { + await writeResolvedPricingArtifact(resolved) + } + + printSummary(report) +} + +/** + * Finds card files. + * + * Set files are shallow: + * + * data/Serie/Set.ts + * + * Card files are deeper: + * + * data/Serie/Set/Card.ts + */ +async function findCardFiles(rootFolders: string[]): Promise { + const files: string[] = [] + + for (const rootFolder of rootFolders) { + const exists = await pathExists(rootFolder) + + if (!exists) { + continue + } + + await walk(rootFolder, files) + } + + return files.filter((file) => { + if (!file.endsWith('.ts')) { + return false + } + + const relative = path.relative(process.cwd(), file).replace(/\\/g, '/') + const parts = relative.split('/') + + // data/Serie/Set/Card.ts + // data-asia/Serie/Set/Card.ts + return ( + (parts[0] === 'data' || parts[0] === 'data-asia') && + parts.length === 4 + ) + }) +} + +async function walk(folder: string, files: string[]): Promise { + const entries = await fs.readdir(folder, { + withFileTypes: true, + }) + + for (const entry of entries) { + const fullPath = path.join(folder, entry.name) + + if (entry.isDirectory()) { + await walk(fullPath, files) + continue + } + + files.push(fullPath) + } +} + +async function getCachedPricingFile({ + categoryId, + tcgplayerSetId, + cacheByKey, +}: { + categoryId: TCGTrackingCategoryId + tcgplayerSetId: number + cacheByKey: Map +}): Promise { + const key = `${categoryId}:${tcgplayerSetId}` + + if (cacheByKey.has(key)) { + return cacheByKey.get(key) ?? null + } + + if (PRICING_SOURCE !== 'tcgcsv') { + const cacheFile = getTCGTrackingPricingCacheFile(categoryId, tcgplayerSetId) + + try { + const raw = await fs.readFile(cacheFile, 'utf8') + const parsed = JSON.parse(raw) as CachedTCGTrackingPricingFile + + cacheByKey.set(key, parsed) + + return parsed + } catch (error) { + if (!isNodeError(error) || error.code !== 'ENOENT') { + throw error + } + } + } + + // Fall back to TCGCSV cache unless explicitly using tcgtracking-only. + if (PRICING_SOURCE !== 'tcgtracking-only') { + const fallback = await readTCGCSVPricingCache(categoryId, tcgplayerSetId) + + cacheByKey.set(key, fallback) + + return fallback + } + + cacheByKey.set(key, null) + + return null +} + +/** + * Reads a TCGCSV price cache file (var/models/tcgplayer/prices/{setId}.json) + * and converts it to the CachedTCGTrackingPricingFile shape so the same + * resolver can consume it without changes. + * + * Populated by: bun scripts/preloadTCGPlayer.ts (pricing:preload:backup) + */ +async function readTCGCSVPricingCache( + categoryId: TCGTrackingCategoryId, + tcgplayerSetId: number, +): Promise { + const cacheFile = path.join( + process.cwd(), + 'var', + 'models', + 'tcgplayer', + 'prices', + `${tcgplayerSetId}.json`, + ) + + try { + const raw = await fs.readFile(cacheFile, 'utf8') + const tcgcsv = JSON.parse(raw) as TCGCSVPricesFile + + if (!tcgcsv.success || !tcgcsv.results?.length) { + return null + } + + const prices: Record }> = {} + + for (const result of tcgcsv.results) { + const id = String(result.productId) + + if (!prices[id]) { + prices[id] = { tcg: {} } + } + + const subType = result.subTypeName || 'Normal' + + prices[id].tcg[subType] = { + low: result.lowPrice ?? undefined, + market: result.marketPrice ?? undefined, + } + } + + return { + source: 'tcgtracking', + categoryId, + tcgplayerSetId, + sourceFile: `tcgcsv-fallback:${tcgplayerSetId}`, + fetchedAt: new Date().toISOString(), + data: { + set_id: tcgplayerSetId, + prices, + }, + } + } catch (error) { + if (isNodeError(error) && error.code === 'ENOENT') { + return null + } + + throw error + } +} + +function resolvePricesForCard({ + card, + cardId, + cardFile, + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + cache, + missing, +}: { + card: TCGDexCard + cardId: string + cardFile: string + cardName?: string + setId?: string + setName?: string + categoryId: number + tcgplayerSetId: number + cache: CachedTCGTrackingPricingFile + missing: MissingPrice[] +}): ResolvedCardPriceEntry[] { + const entries: ResolvedCardPriceEntry[] = [] + + /** + * If card.variants is an array, the repo is using detailed variants. + * In that case, prefer variant-level thirdParty.tcgplayer IDs. + */ + if (Array.isArray(card.variants)) { + card.variants.forEach((variant, variantIndex) => { + const tcgplayerProductId = variant.thirdParty?.tcgplayer + + if (typeof tcgplayerProductId !== 'number') { + missing.push({ + cardId, + cardFile: normalizePathForReport(cardFile), + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + target: 'variant', + variantIndex, + reason: 'missing-variant-tcgplayer-id', + }) + + return + } + + const prices = getPricesForProduct(cache, tcgplayerProductId) + + if (prices.length === 0) { + missing.push({ + cardId, + cardFile: normalizePathForReport(cardFile), + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + tcgplayerProductId, + target: 'variant', + variantIndex, + reason: 'missing-product-price', + }) + + return + } + + for (const price of prices) { + entries.push({ + target: 'variant', + variantIndex, + variant: { + type: variant.type, + subtype: variant.subtype, + size: variant.size, + stamp: variant.stamp, + foil: variant.foil, + }, + tcgplayerProductId, + priceType: price.priceType, + low: price.low, + market: price.market, + }) + } + }) + + return entries + } + + /** + * If card.variants is not an array, use the card-level product ID. + * This keeps compatibility with older card files. + */ + const tcgplayerProductId = card.thirdParty?.tcgplayer + + if (typeof tcgplayerProductId !== 'number') { + missing.push({ + cardId, + cardFile: normalizePathForReport(cardFile), + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + target: 'card', + reason: 'missing-card-tcgplayer-id', + }) + + return entries + } + + const prices = getPricesForProduct(cache, tcgplayerProductId) + + if (prices.length === 0) { + missing.push({ + cardId, + cardFile: normalizePathForReport(cardFile), + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + tcgplayerProductId, + target: 'card', + reason: 'missing-product-price', + }) + + return entries + } + + for (const price of prices) { + entries.push({ + target: 'card', + tcgplayerProductId, + priceType: price.priceType, + low: price.low, + market: price.market, + }) + } + + return entries +} + +function getPricesForProduct( + cache: CachedTCGTrackingPricingFile, + tcgplayerProductId: number, +): ResolvedPrice[] { + const product = cache.data.prices?.[String(tcgplayerProductId)] + + if (!product) { + return [] + } + + if (!product.tcg) { + return [] + } + + return Object.entries(product.tcg) + .map(([priceType, price]) => ({ + productId: tcgplayerProductId, + priceType, + low: price.low, + market: price.market, + })) + .filter((price) => { + return typeof price.low === 'number' || typeof price.market === 'number' + }) +} + +function groupMissingByReason(missing: MissingPrice[]): CountByReason[] { + const counts = new Map() + + for (const item of missing) { + counts.set(item.reason, (counts.get(item.reason) ?? 0) + 1) + } + + return Array.from(counts.entries()) + .map(([reason, count]) => ({ + reason, + count, + })) + .sort((a, b) => b.count - a.count) +} + +function groupMissingCaches(missing: MissingPrice[]): MissingCacheSummary[] { + const groups = new Map() + + for (const item of missing) { + if (item.reason !== 'missing-cache') { + continue + } + + const key = `${item.categoryId ?? 'unknown'}:${item.tcgplayerSetId ?? 'unknown'}` + const existing = groups.get(key) + + if (existing) { + existing.count += 1 + continue + } + + groups.set(key, { + categoryId: item.categoryId, + tcgplayerSetId: item.tcgplayerSetId, + setId: item.setId, + setName: item.setName, + count: 1, + }) + } + + return Array.from(groups.values()).sort((a, b) => b.count - a.count) +} + +function groupResolvedByTarget(resolved: ResolvedCardPrice[]): CountByTarget[] { + const counts = new Map() + + for (const card of resolved) { + for (const price of card.prices) { + counts.set(price.target, (counts.get(price.target) ?? 0) + 1) + } + } + + return Array.from(counts.entries()) + .map(([target, count]) => ({ + target, + count, + })) + .sort((a, b) => b.count - a.count) +} + +function groupPriceTypes(resolved: ResolvedCardPrice[]): CountByPriceType[] { + const counts = new Map() + + for (const card of resolved) { + for (const price of card.prices) { + counts.set(price.priceType, (counts.get(price.priceType) ?? 0) + 1) + } + } + + return Array.from(counts.entries()) + .map(([priceType, count]) => ({ + priceType, + count, + })) + .sort((a, b) => b.count - a.count) +} + +async function writeReport(report: PricingUpdateReport): Promise { + const reportFile = path.join( + process.cwd(), + 'var', + 'reports', + 'tcgtracking-pricing-update.json', + ) + + await fs.mkdir(path.dirname(reportFile), { + recursive: true, + }) + + await fs.writeFile( + reportFile, + `${JSON.stringify(report, null, '\t')}\n`, + 'utf8', + ) +} + +/** + * CHANGE: + * This writes the resolved pricing artifact consumed by the next integration step. + * + * It intentionally does not mutate card source files because the current source files + * store marketplace IDs, while pricing should be generated from the cached provider data. + */ +async function writeResolvedPricingArtifact( + resolved: ResolvedCardPrice[], +): Promise { + const outputFile = path.join( + process.cwd(), + 'var', + 'models', + 'tcgtracking', + 'resolved-pricing.json', + ) + + await fs.mkdir(path.dirname(outputFile), { + recursive: true, + }) + + await fs.writeFile( + outputFile, + `${JSON.stringify( + { + source: 'tcgtracking', + createdAt: new Date().toISOString(), + cards: resolved, + }, + null, + '\t', + )}\n`, + 'utf8', + ) +} + +function printSummary(report: PricingUpdateReport): void { + console.log('') + console.log('TCGTracking pricing update complete.') + console.log(`Mode: ${report.mode}`) + console.log(`Pricing source: ${PRICING_SOURCE}`) + console.log(`Card files scanned: ${report.summary.cardFilesScanned}`) + console.log(`Cards with prices: ${report.summary.cardFilesWithPrices}`) + console.log(`Price entries resolved: ${report.summary.priceEntriesResolved}`) + console.log(`Missing records: ${report.summary.missing}`) + console.log(`Missing cache records: ${report.summary.missingCacheRecords}`) + console.log(`Unique missing caches: ${report.summary.uniqueMissingCaches}`) + console.log(`Output written: ${report.summary.outputWritten ? 'yes' : 'no'}`) + + if (report.resolvedByTarget.length > 0) { + console.log('') + console.log('Resolved by target:') + + for (const item of report.resolvedByTarget) { + console.log(`- ${item.target}: ${item.count}`) + } + } + + if (report.priceTypes.length > 0) { + console.log('') + console.log('Resolved price types:') + + for (const item of report.priceTypes.slice(0, 20)) { + console.log(`- ${item.priceType}: ${item.count}`) + } + } + + if (report.missingByReason.length > 0) { + console.log('') + console.log('Missing by reason:') + + for (const item of report.missingByReason) { + console.log(`- ${item.reason}: ${item.count}`) + } + } + + if (report.missingCaches.length > 0) { + console.log('') + console.log('Missing cache sets:') + + for (const item of report.missingCaches.slice(0, 20)) { + console.log( + `- ${item.setName ?? item.setId ?? 'Unknown set'} (${item.tcgplayerSetId ?? 'unknown'}): ${item.count}`, + ) + } + } + + console.log('') + console.log('Report:') + console.log('var/reports/tcgtracking-pricing-update.json') + + if (report.mode === 'apply') { + console.log('') + console.log('Resolved pricing artifact:') + console.log('var/models/tcgtracking/resolved-pricing.json') + } +} + +function getMode(): 'dry-run' | 'apply' { + if (process.argv.includes('--apply')) { + return 'apply' + } + + return 'dry-run' +} + +async function pathExists(filePath: string): Promise { + try { + await fs.access(filePath) + return true + } catch { + return false + } +} + +function getCardIdFromFile(filePath: string): string { + const filename = path.basename(filePath) + return filename.slice(0, filename.lastIndexOf('.')) +} + +function getEnglishName(value?: Record): string | undefined { + return value?.en ?? Object.values(value ?? {})[0] +} + +function normalizePathForReport(filePath: string): string { + return path.relative(process.cwd(), filePath).replace(/\\/g, '/') +} + +function isNodeError(error: unknown): error is NodeJS.ErrnoException { + return error instanceof Error && 'code' in error +} + +main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) \ No newline at end of file diff --git a/scripts/utils-data/tcgtracking.ts b/scripts/utils-data/tcgtracking.ts new file mode 100644 index 0000000000..07ba814ea0 --- /dev/null +++ b/scripts/utils-data/tcgtracking.ts @@ -0,0 +1,327 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import { pathToFileURL } from 'node:url' + +/** + * TCGTracking Open TCG API base URL. + * + * This replaces the old TCGCSV fetch source for pricing data, + * but does not remove the old TCGCSV scripts yet. + */ +export const TCGTRACKING_API_BASE_URL = 'https://tcgtracking.com/tcgapi/v1' + +/** + * TCGTracking category IDs. + * + * 3 = Pokémon + * 85 = Pokémon Japan + * + * These are used only to fetch pricing data. + * They do not decide card variants, finishes, or canonical card data. + */ +export const TCGTRACKING_CATEGORIES = { + pokemon: 3, + pokemonJapan: 85, +} as const + +export type TCGTrackingCategoryId = + (typeof TCGTRACKING_CATEGORIES)[keyof typeof TCGTRACKING_CATEGORIES] + +/** + * The cached pricing record is intentionally loose because TCGTracking may expose + * price records with short keys, long keys, or nested response wrappers. + * + * The preload script stores the original raw JSON as-is. + * Normalisation happens when reading the cache later. + */ +export type TCGTrackingRawPricingResponse = unknown + +export interface TCGTrackingSetCacheTarget { + categoryId: TCGTrackingCategoryId + + /** + * Existing TCGDex / TCGPlayer set ID. + * + * This is used for cache file naming so the updater can still look up pricing + * by card.set.thirdParty.tcgplayer. + */ + tcgplayerSetId: number + + /** + * Optional TCGTracking set ID override. + * + * This is used only when fetching from TCGTracking if their set ID differs + * from the existing TCGDex / TCGPlayer set ID. + * + * Example: + * - TCGDex Black Bolt set ID: 22325 + * - TCGTracking Black Bolt set ID: 24325 + */ + tcgTrackingSetId?: number + + sourceFile: string +} + +/** + * Cached TCGTracking pricing file written by scripts/preloadTCGTracking.ts. + * + * Real cache shape: + * + * data.prices[tcgplayerProductId].tcg[priceType].low + * data.prices[tcgplayerProductId].tcg[priceType].market + * + * Example priceType values from the uploaded cache include: + * + * Normal + * Holofoil + * Reverse Holofoil + */ +export interface CachedTCGTrackingPricingFile { + source: 'tcgtracking' + categoryId: TCGTrackingCategoryId + + /** + * Existing TCGDex / TCGPlayer set ID. + * + * This should match card.set.thirdParty.tcgplayer. + */ + tcgplayerSetId: number + + /** + * Actual TCGTracking set ID used when fetching. + * + * Usually the same as tcgplayerSetId, but can differ for sets such as + * Black Bolt. + * + * Known missing TCGTracking pricing endpoints: + * + * - DP trainer Kit (Manaphy), TCGPlayer set ID 609 + * - DP trainer Kit (Lucario), TCGPlayer set ID 610 + * + * TCGTracking search currently returns no matching set for: + * "DP Trainer Kit Manaphy/Lucario" + * + * These should fall back to existing/TCGCSV pricing. + */ + tcgTrackingSetId?: number + + sourceFile: string + fetchedAt: string + data: { + set_id: number + updated?: string + prices?: Record + } +} + +export interface TCGTrackingProductPricing { + tcg?: Record +} + +export interface TCGTrackingVariantPrice { + low?: number + market?: number +} + +export interface ResolvedTCGTrackingPrice { + productId: number + priceType: string + low?: number + market?: number +} + +/** + * CHANGE: + * Added a dedicated TCGTracking cache folder. + * + * This avoids changing the old TCGCSV cache: + * + * var/models/tcgplayer/... + * + * New cache: + * + * var/models/tcgtracking/category-3/pricing/{setId}.json + * var/models/tcgtracking/category-85/pricing/{setId}.json + * + * IMPORTANT: + * The {setId}.json filename should use the existing TCGDex / TCGPlayer set ID, + * not the override ID. This keeps the updater compatible with existing card data. + */ +export function getTCGTrackingPricingCacheFile( + categoryId: TCGTrackingCategoryId, + tcgplayerSetId: number, +): string { + return path.join( + process.cwd(), + 'var', + 'models', + 'tcgtracking', + `category-${categoryId}`, + 'pricing', + `${tcgplayerSetId}.json`, + ) +} + +export async function ensureParentFolder(filePath: string): Promise { + await fs.mkdir(path.dirname(filePath), { recursive: true }) +} + +export async function writeJsonFile(filePath: string, data: unknown): Promise { + await ensureParentFolder(filePath) + + await fs.writeFile( + filePath, + `${JSON.stringify(data, null, '\t')}\n`, + 'utf8', + ) +} + +export async function readJsonFile(filePath: string): Promise { + try { + const raw = await fs.readFile(filePath, 'utf8') + return JSON.parse(raw) as T + } catch (error) { + if (isNodeError(error) && error.code === 'ENOENT') { + return null + } + + throw error + } +} + +export function isNodeError(error: unknown): error is NodeJS.ErrnoException { + return error instanceof Error && 'code' in error +} + +export async function fetchTCGTrackingSetPricing( + categoryId: TCGTrackingCategoryId, + tcgTrackingSetId: number, +): Promise { + const url = `${TCGTRACKING_API_BASE_URL}/${categoryId}/sets/${tcgTrackingSetId}/pricing` + + const response = await fetch(url, { + headers: { + Accept: 'application/json', + }, + }) + + if (!response.ok) { + throw new Error( + `TCGTracking pricing fetch failed for category ${categoryId}, set ${tcgTrackingSetId}: ${response.status} ${response.statusText}`, + ) + } + + return response.json() +} + +/** + * Small delay to avoid hammering the API when fetching many sets. + * + * TCGTracking is CDN-backed, but keeping this polite makes the script safer + * for repeated maintainer use and CI. + */ +export async function sleep(ms: number): Promise { + await new Promise((resolve) => { + setTimeout(resolve, ms) + }) +} + +/** + * CHANGE: + * Converts a filesystem path into a valid file URL for dynamic import. + * + * The previous manual file:/// conversion broke on paths with special characters. + * + * Example broken path: + * + * H:\cards-database\data\EX\Unseen Forces Unown Collection\?.ts + * + * A literal ? inside an import URL is treated as the start of a query string unless + * it is encoded. pathToFileURL handles that correctly. + */ +export function toImportUrl(filePath: string): string { + return pathToFileURL(path.resolve(filePath)).href +} + +/** + * Decide which TCGTracking category to use based on the file location. + * + * CHANGE: + * This is only for selecting the pricing source category. + * It does not change card data or map variants. + */ +export function getCategoryIdForSetFile(filePath: string): TCGTrackingCategoryId { + const normalized = filePath.replace(/\\/g, '/') + + if (normalized.includes('/data-asia/')) { + return TCGTRACKING_CATEGORIES.pokemonJapan + } + + return TCGTRACKING_CATEGORIES.pokemon +} + +/** + * Reads a cached TCGTracking pricing file. + */ +export async function readTCGTrackingPricingCache( + categoryId: TCGTrackingCategoryId, + tcgplayerSetId: number, +): Promise { + const cacheFile = getTCGTrackingPricingCacheFile(categoryId, tcgplayerSetId) + + return readJsonFile(cacheFile) +} + +/** + * Finds prices for an existing TCGPlayer product ID. + * + * This does not decide what variants exist. + * It only returns the price buckets TCGTracking has for that product. + */ +export function getPricesForTCGPlayerProduct( + cache: CachedTCGTrackingPricingFile, + tcgplayerProductId: number, +): ResolvedTCGTrackingPrice[] { + const productPricing = cache.data.prices?.[String(tcgplayerProductId)] + + if (!productPricing?.tcg) { + return [] + } + + return Object.entries(productPricing.tcg) + .map(([priceType, price]) => ({ + productId: tcgplayerProductId, + priceType, + low: price.low, + market: price.market, + })) + .filter((price) => { + return typeof price.low === 'number' || typeof price.market === 'number' + }) +} + +/** + * Gets the best single price for a product. + * + * Preference: + * 1. market + * 2. low + * + * This is useful if the current TCGDex pricing shape only stores one price value. + */ +export function getBestPriceForTCGPlayerProduct( + cache: CachedTCGTrackingPricingFile, + tcgplayerProductId: number, +): ResolvedTCGTrackingPrice | null { + const prices = getPricesForTCGPlayerProduct(cache, tcgplayerProductId) + + const withMarket = prices.find((price) => typeof price.market === 'number') + + if (withMarket) { + return withMarket + } + + const withLow = prices.find((price) => typeof price.low === 'number') + + return withLow ?? null +} \ No newline at end of file diff --git a/server/src/libs/providers/tcgplayer/index.ts b/server/src/libs/providers/tcgplayer/index.ts index 0ba969bc97..504ce35d31 100644 --- a/server/src/libs/providers/tcgplayer/index.ts +++ b/server/src/libs/providers/tcgplayer/index.ts @@ -1,8 +1,12 @@ import * as OfficialTCGPlayer from './official' import * as Fallback from './fallback' +import * as TCGTracking from './tcgtracking' import type RFC7807 from '../../RFCs/RFC7807' -let source: (typeof OfficialTCGPlayer) | (typeof Fallback) = Fallback +type Source = typeof OfficialTCGPlayer | typeof Fallback | typeof TCGTracking + +let source: Source + if ( process.env.TCGPLAYER_CLIENT_ID && process.env.TCGPLAYER_CLIENT_SECRET @@ -10,8 +14,14 @@ if ( ) { console.log('loading official TCGPlayer backend') source = OfficialTCGPlayer -} else { +} else if (process.env.PRICING_SOURCE === 'tcgcsv') { console.log('loading fallback TCGPlayer backend') + source = Fallback +} else { + // TCGTracking is the default — reads from var/models/tcgtracking/resolved-pricing.json + // Set PRICING_SOURCE=tcgcsv to revert to the old live TCGCSV fetch. + console.log('loading TCGTracking backend') + source = TCGTracking } diff --git a/server/src/libs/providers/tcgplayer/tcgtracking.ts b/server/src/libs/providers/tcgplayer/tcgtracking.ts new file mode 100644 index 0000000000..418bf42fb5 --- /dev/null +++ b/server/src/libs/providers/tcgplayer/tcgtracking.ts @@ -0,0 +1,126 @@ +import fs from 'node:fs/promises' +import path from 'node:path' + +interface ResolvedPricingEntry { + target: 'card' | 'variant' + tcgplayerProductId: number + priceType: string + low?: number + market?: number +} + +interface ResolvedPricingCard { + cardId: string + sourceFetchedAt: string + sourceUpdated?: string + prices: ResolvedPricingEntry[] +} + +interface ResolvedPricingFile { + source: 'tcgtracking' + createdAt: string + cards: ResolvedPricingCard[] +} + +export interface PriceResult { + productId: number + lowPrice: number + midPrice: number + highPrice: number + marketPrice?: number + directLowPrice?: number +} + +// Index: productId → normalised-price-type → PriceResult +let cache: Record> = {} +let lastLoaded: Date | undefined = undefined +let lastUpdated: string | undefined = undefined + +// Can be overridden via TCGTRACKING_PRICING_PATH env var for Docker / custom setups. +// Default: one level above the server directory (standard repo layout). +const RESOLVED_PRICING_PATH = process.env.TCGTRACKING_PRICING_PATH ?? path.join( + process.cwd(), + '..', + 'var', + 'models', + 'tcgtracking', + 'resolved-pricing.json', +) + +export async function updateTCGPlayerDatas(): Promise { + try { + const stat = await fs.stat(RESOLVED_PRICING_PATH) + + // Skip reload when file hasn't changed since last load + if (lastLoaded && stat.mtimeMs <= lastLoaded.getTime()) { + return false + } + + const raw = await fs.readFile(RESOLVED_PRICING_PATH, 'utf8') + const data = JSON.parse(raw) as ResolvedPricingFile + + if (data.source !== 'tcgtracking') { + console.warn('TCGTracking: unexpected source in resolved-pricing.json:', data.source) + return false + } + + const newCache: Record> = {} + + for (const card of data.cards) { + for (const price of card.prices) { + const productId = price.tcgplayerProductId + const key = price.priceType.toLowerCase().replaceAll(' ', '-') + + if (!newCache[productId]) { + newCache[productId] = {} + } + + newCache[productId][key] = { + productId, + lowPrice: price.low ?? 0, + midPrice: 0, + highPrice: 0, + marketPrice: price.market, + } + } + } + + cache = newCache + lastLoaded = new Date() + lastUpdated = data.createdAt + + return true + } catch (error) { + if (isNodeError(error) && error.code === 'ENOENT') { + console.warn('TCGTracking: resolved-pricing.json not found:', RESOLVED_PRICING_PATH) + return false + } + throw error + } +} + +export async function getTCGPlayerPrice(card: { thirdParty: { tcgplayer?: number } }): Promise<{ + unit: 'USD' + updated: string + [key: string]: unknown +} | null> { + if (!lastLoaded || typeof card.thirdParty?.tcgplayer !== 'number') { + return null + } + + const variants = cache[card.thirdParty.tcgplayer] + + if (!variants) { + return null + } + + return { + updated: lastUpdated ?? lastLoaded.toISOString(), + unit: 'USD', + ...variants, + } +} + +function isNodeError(error: unknown): error is NodeJS.ErrnoException { + return error instanceof Error && 'code' in error +} From 81aed4b39ac04601134b675d9e4e74b0567488b7 Mon Sep 17 00:00:00 2001 From: FalconChipp <112320693+FalconChipp@users.noreply.github.com> Date: Mon, 11 May 2026 19:35:54 +0100 Subject: [PATCH 2/2] Introduce pricing providers and updater script Add a unified pricing subsystem and updater: new files under scripts/pricing (cache, sources, tcgplayer, tcgtracking, types) and a top-level scripts/updatePricing.ts that scans card files, resolves prices from configured providers, and writes reports/debug artifacts. Implements a TTLCache, provider factory/ordering (TCGDEX_PRICING_SOURCES), TCGPlayer and TCGTracking providers with caching and inflight deduplication, and set ID overrides for TCGTracking. Remove legacy TCGTracking-specific preload/update scripts (scripts/preloadTCGTracking.ts and scripts/updatePricingFromTCGTracking.ts) and update package.json scripts to use pricing:update / pricing:update:debug. --- package.json | 13 +- scripts/preloadTCGTracking.ts | 384 ------- scripts/pricing/cache.ts | 35 + scripts/pricing/sources.ts | 26 + scripts/pricing/tcgplayer.ts | 124 +++ scripts/pricing/tcgtracking.ts | 126 +++ scripts/pricing/types.ts | 18 + scripts/updatePricing.ts | 592 +++++++++++ scripts/updatePricingFromTCGTracking.ts | 948 ------------------ scripts/utils-data/tcgtracking.ts | 255 +---- .../libs/providers/tcgplayer/tcgtracking.ts | 157 +-- 11 files changed, 1009 insertions(+), 1669 deletions(-) delete mode 100644 scripts/preloadTCGTracking.ts create mode 100644 scripts/pricing/cache.ts create mode 100644 scripts/pricing/sources.ts create mode 100644 scripts/pricing/tcgplayer.ts create mode 100644 scripts/pricing/tcgtracking.ts create mode 100644 scripts/pricing/types.ts create mode 100644 scripts/updatePricing.ts delete mode 100644 scripts/updatePricingFromTCGTracking.ts diff --git a/package.json b/package.json index c44b29e9b6..15dc02db02 100644 --- a/package.json +++ b/package.json @@ -11,17 +11,8 @@ "dex:fix:apply": "bun scripts/pokedexIdFixer/fix-dex-ids.ts --apply", "dex:lint": "bun scripts/pokedexIdFixer/lint-dex-ids.ts", - "tcgtracking:preload": "bun scripts/preloadTCGTracking.ts", - "tcgtracking:update-pricing": "bun scripts/updatePricingFromTCGTracking.ts", - "tcgtracking:update-pricing:dry-run": "bun scripts/updatePricingFromTCGTracking.ts --dry-run", - "tcgtracking:update-pricing:apply": "bun scripts/updatePricingFromTCGTracking.ts --apply", - - "pricing:preload": "bun scripts/preloadTCGTracking.ts", - "pricing:preload:backup": "bun scripts/preloadTCGPlayer.ts", - "pricing:update:dry-run": "bun scripts/updatePricingFromTCGTracking.ts --dry-run", - "pricing:update:apply": "bun scripts/updatePricingFromTCGTracking.ts --apply", - "pricing:sync": "bun scripts/preloadTCGTracking.ts && bun scripts/updatePricingFromTCGTracking.ts --apply", - "pricing:sync:with-backup": "bun scripts/preloadTCGTracking.ts && bun scripts/preloadTCGPlayer.ts && bun scripts/updatePricingFromTCGTracking.ts --apply", + "pricing:update": "bun scripts/updatePricing.ts", + "pricing:update:debug": "bun scripts/updatePricing.ts --write-debug-artifact", }, "devDependencies": { "@dzeio/object-util": "^1.9.2", diff --git a/scripts/preloadTCGTracking.ts b/scripts/preloadTCGTracking.ts deleted file mode 100644 index ea1685ba73..0000000000 --- a/scripts/preloadTCGTracking.ts +++ /dev/null @@ -1,384 +0,0 @@ -import fs from 'node:fs/promises' -import path from 'node:path' - -import { - fetchTCGTrackingSetPricing, - getCategoryIdForSetFile, - getTCGTrackingPricingCacheFile, - sleep, - toImportUrl, - writeJsonFile, - type TCGTrackingSetCacheTarget, -} from './utils-data/tcgtracking' - -interface TCGDexSetModule { - default?: { - id?: string - name?: string - thirdParty?: { - tcgplayer?: number - } - } -} - -interface TCGTrackingPreloadFailure { - categoryId: number - tcgplayerSetId: number - tcgTrackingSetId?: number - sourceFile: string - status: 'missing-endpoint' | 'fetch-failed' - error: string -} - -/** - * TCGTracking sometimes exposes a set under a different ID than the - * existing TCGDex set.thirdParty.tcgplayer ID. - * - * Key: existing TCGDex / TCGPlayer set ID - * Value: TCGTracking set ID used for API fetching - * - * IMPORTANT: - * The cache file is still written using the original TCGDex / TCGPlayer set ID. - * That keeps updatePricingFromTCGTracking.ts compatible with existing card data. - */ -const TCGTRACKING_SET_ID_OVERRIDES: Record = { - // Black Bolt - 22325: 24325, -} - -/** - * CHANGE: - * Added a TCGTracking pricing preloader. - * - * This script does not edit card files. - * This script does not create or map variants. - * This script does not remove TCGCSV. - * - * It only fetches pricing from TCGTracking and caches it locally. - * - * CHANGE: - * Failed fetches now write a report instead of causing the script to fail by default. - * - * Use --strict if you want failures to exit with code 1. - * - * CHANGE: - * Added TCGTracking set ID overrides. - * - * This allows sets like Black Bolt to be fetched from TCGTracking using their - * current TCGTracking set ID while still caching under the existing TCGDex - * thirdParty.tcgplayer set ID. - */ -async function main(): Promise { - const setFiles = await findSetFiles([ - path.join(process.cwd(), 'data'), - path.join(process.cwd(), 'data-asia'), - ]) - - const failures: TCGTrackingPreloadFailure[] = [] - const targets = await getTCGTrackingCacheTargets(setFiles) - - console.log(`Found ${targets.length} TCGDex sets with thirdParty.tcgplayer IDs.`) - - let fetched = 0 - let skipped = 0 - let failed = 0 - - for (const target of targets) { - const cacheFile = getTCGTrackingPricingCacheFile( - target.categoryId, - target.tcgplayerSetId, - ) - - const fetchSetId = target.tcgTrackingSetId ?? target.tcgplayerSetId - const hasOverride = fetchSetId !== target.tcgplayerSetId - - try { - const shouldSkip = await isFreshCache(cacheFile) - - if (shouldSkip) { - skipped += 1 - console.log( - `Skipping category ${target.categoryId}, set ${target.tcgplayerSetId}; cache is fresh.`, - ) - continue - } - - console.log( - `Fetching TCGTracking pricing: category ${target.categoryId}, set ${fetchSetId}` + - (hasOverride - ? ` (override for TCGDex set ${target.tcgplayerSetId})` - : ''), - ) - - const pricing = await fetchTCGTrackingSetPricing( - target.categoryId, - fetchSetId, - ) - - await writeJsonFile(cacheFile, { - source: 'tcgtracking', - categoryId: target.categoryId, - - // Existing TCGDex / TCGPlayer set ID. - // This is used for the cache filename and updater lookup. - tcgplayerSetId: target.tcgplayerSetId, - - // Actual TCGTracking set ID used when fetching. - tcgTrackingSetId: fetchSetId, - - sourceFile: normalizePathForReport(target.sourceFile), - fetchedAt: new Date().toISOString(), - data: pricing, - }) - - fetched += 1 - - // Polite delay between requests. - await sleep(100) - } catch (error) { - failed += 1 - - const errorMessage = getErrorMessage(error) - - failures.push({ - categoryId: target.categoryId, - tcgplayerSetId: target.tcgplayerSetId, - tcgTrackingSetId: hasOverride ? fetchSetId : undefined, - sourceFile: normalizePathForReport(target.sourceFile), - status: errorMessage.includes('404 Not Found') - ? 'missing-endpoint' - : 'fetch-failed', - error: errorMessage, - }) - - console.error( - `Failed to fetch category ${target.categoryId}, set ${fetchSetId}` + - (hasOverride - ? ` (override for TCGDex set ${target.tcgplayerSetId})` - : ''), - ) - console.error(errorMessage) - } - } - - if (failures.length > 0) { - const reportFile = path.join( - process.cwd(), - 'var', - 'reports', - 'tcgtracking-preload-failures.json', - ) - - await writeFailureReport(reportFile, failures) - - console.log('') - console.log(`Failure report written to ${normalizePathForReport(reportFile)}`) - } - - console.log('') - console.log('TCGTracking preload complete.') - console.log(`Fetched: ${fetched}`) - console.log(`Skipped: ${skipped}`) - console.log(`Failed: ${failed}`) - - const isStrict = process.argv.includes('--strict') - - if (failed > 0 && isStrict) { - process.exitCode = 1 - } -} - -/** - * Finds set files. - * - * The existing repo structure has scripts inside: - * - * H:\cards-database\scripts - * - * and data folders at repo root: - * - * data - * data-asia - * - * This scans both. - */ -async function findSetFiles(rootFolders: string[]): Promise { - const files: string[] = [] - - for (const rootFolder of rootFolders) { - const exists = await pathExists(rootFolder) - - if (!exists) { - continue - } - - await walk(rootFolder, files) - } - - return files.filter((file) => { - if (!file.endsWith('.ts')) { - return false - } - - // CHANGE: - // Avoid card files where possible by only allowing shallow set files. - // - // Existing scripts often scan data/*/*.ts for set files. - // This keeps the same intent but is slightly safer. - const relative = path.relative(process.cwd(), file).replace(/\\/g, '/') - const parts = relative.split('/') - - // data/Serie/Set.ts - // data-asia/Serie/Set.ts - return ( - (parts[0] === 'data' || parts[0] === 'data-asia') && - parts.length === 3 - ) - }) -} - -async function walk(folder: string, files: string[]): Promise { - const entries = await fs.readdir(folder, { - withFileTypes: true, - }) - - for (const entry of entries) { - const fullPath = path.join(folder, entry.name) - - if (entry.isDirectory()) { - await walk(fullPath, files) - continue - } - - files.push(fullPath) - } -} - -/** - * Reads TCGDex set files and extracts thirdParty.tcgplayer set IDs. - * - * IMPORTANT: - * We are using the existing TCGDex thirdParty.tcgplayer set IDs. - * We are not doing new set matching. - */ -async function getTCGTrackingCacheTargets( - setFiles: string[], -): Promise { - const targets: TCGTrackingSetCacheTarget[] = [] - - for (const setFile of setFiles) { - const setModule = (await import(toImportUrl(setFile))) as TCGDexSetModule - const setData = setModule.default - - const tcgplayerSetId = setData?.thirdParty?.tcgplayer - - if (typeof tcgplayerSetId !== 'number') { - continue - } - - targets.push({ - categoryId: getCategoryIdForSetFile(setFile), - tcgplayerSetId, - tcgTrackingSetId: TCGTRACKING_SET_ID_OVERRIDES[tcgplayerSetId], - sourceFile: setFile, - }) - } - - return dedupeTargets(targets) -} - -function dedupeTargets( - targets: TCGTrackingSetCacheTarget[], -): TCGTrackingSetCacheTarget[] { - const seen = new Set() - const deduped: TCGTrackingSetCacheTarget[] = [] - - for (const target of targets) { - const fetchSetId = target.tcgTrackingSetId ?? target.tcgplayerSetId - const key = `${target.categoryId}:${target.tcgplayerSetId}:${fetchSetId}` - - if (seen.has(key)) { - continue - } - - seen.add(key) - deduped.push(target) - } - - return deduped -} - -/** - * Keeps pricing cache fresh for one day. - * - * CHANGE: - * This mirrors the idea that pricing should update daily, - * while avoiding unnecessary full refetches. - */ -async function isFreshCache(filePath: string): Promise { - try { - const stat = await fs.stat(filePath) - const ageMs = Date.now() - stat.mtimeMs - const oneDayMs = 24 * 60 * 60 * 1000 - - return ageMs < oneDayMs - } catch (error) { - if (isNodeError(error) && error.code === 'ENOENT') { - return false - } - - throw error - } -} - -async function writeFailureReport( - reportFile: string, - failures: TCGTrackingPreloadFailure[], -): Promise { - await fs.mkdir(path.dirname(reportFile), { - recursive: true, - }) - - await fs.writeFile( - reportFile, - `${JSON.stringify( - { - createdAt: new Date().toISOString(), - failures, - }, - null, - '\t', - )}\n`, - 'utf8', - ) -} - -async function pathExists(filePath: string): Promise { - try { - await fs.access(filePath) - return true - } catch { - return false - } -} - -function normalizePathForReport(filePath: string): string { - return path.relative(process.cwd(), filePath).replace(/\\/g, '/') -} - -function getErrorMessage(error: unknown): string { - if (error instanceof Error) { - return error.message - } - - return String(error) -} - -function isNodeError(error: unknown): error is NodeJS.ErrnoException { - return error instanceof Error && 'code' in error -} - -main().catch((error) => { - console.error(error) - process.exitCode = 1 -}) \ No newline at end of file diff --git a/scripts/pricing/cache.ts b/scripts/pricing/cache.ts new file mode 100644 index 0000000000..e76a722838 --- /dev/null +++ b/scripts/pricing/cache.ts @@ -0,0 +1,35 @@ +interface CacheEntry { + value: V + expiresAt: number +} + +export class TTLCache { + private map = new Map>() + + set(key: K, value: V, ttlMs: number): void { + this.map.set(key, { value, expiresAt: Date.now() + ttlMs }) + } + + get(key: K): V | undefined { + const entry = this.map.get(key) + + if (!entry) { + return undefined + } + + if (Date.now() > entry.expiresAt) { + this.map.delete(key) + return undefined + } + + return entry.value + } + + has(key: K): boolean { + return this.get(key) !== undefined + } + + delete(key: K): void { + this.map.delete(key) + } +} diff --git a/scripts/pricing/sources.ts b/scripts/pricing/sources.ts new file mode 100644 index 0000000000..b926bee4d6 --- /dev/null +++ b/scripts/pricing/sources.ts @@ -0,0 +1,26 @@ +import { TCGTrackingProvider } from './tcgtracking' +import { TCGPlayerProvider } from './tcgplayer' +import type { PricingProvider, PricingSourceName } from './types' + +const PROVIDER_FACTORIES: Record PricingProvider> = { + tcgtracking: () => new TCGTrackingProvider(), + tcgplayer: () => new TCGPlayerProvider(), +} + +/** + * Returns pricing providers in the order defined by TCGDEX_PRICING_SOURCES. + * + * Default: tcgtracking,tcgplayer + * + * The first provider to return prices for a product wins. Providers that + * fail to load or have no data for a product are silently skipped. + */ +export function getPricingProvidersFromEnv(): PricingProvider[] { + const raw = process.env.TCGDEX_PRICING_SOURCES ?? 'tcgtracking,tcgplayer' + + return raw + .split(',') + .map((s) => s.trim() as PricingSourceName) + .filter((name) => name in PROVIDER_FACTORIES) + .map((name) => PROVIDER_FACTORIES[name]()) +} diff --git a/scripts/pricing/tcgplayer.ts b/scripts/pricing/tcgplayer.ts new file mode 100644 index 0000000000..51afb8f71f --- /dev/null +++ b/scripts/pricing/tcgplayer.ts @@ -0,0 +1,124 @@ +import { TTLCache } from './cache' +import type { PricingProvider, ResolvedPrice } from './types' + +// Same TTL as TCGTracking — prices update daily +const PRICING_TTL_MS = 12 * 60 * 60 * 1000 + +interface TCGCSVPricesResult { + productId: number + lowPrice: number | null + marketPrice: number | null + subTypeName: string +} + +interface TCGCSVPricesFile { + success: boolean + results: TCGCSVPricesResult[] +} + +type ProductPriceMap = Map + +export class TCGPlayerProvider implements PricingProvider { + readonly name = 'tcgplayer' as const + + // productId → normalised-price-type → prices + private cache = new TTLCache() + // Deduplicate concurrent fetches for the same set + private inflight = new Map>() + + async getPrices(input: { + categoryId: number + tcgplayerSetId: number + tcgplayerProductId: number + }): Promise { + const productPricing = await this.loadProduct( + input.tcgplayerSetId, + input.tcgplayerProductId, + ) + + if (!productPricing) { + return [] + } + + const result: ResolvedPrice[] = [] + + for (const [priceType, price] of productPricing) { + if (typeof price.low === 'number' || typeof price.market === 'number') { + result.push({ + source: 'tcgplayer', + productId: input.tcgplayerProductId, + priceType, + low: price.low, + market: price.market, + }) + } + } + + return result + } + + private async loadProduct( + tcgplayerSetId: number, + tcgplayerProductId: number, + ): Promise { + if (this.cache.has(tcgplayerProductId)) { + return this.cache.get(tcgplayerProductId) ?? null + } + + // Fetch the whole set so all products in it are cached together + let pending = this.inflight.get(tcgplayerSetId) + + if (!pending) { + pending = this.fetchSet(tcgplayerSetId) + this.inflight.set(tcgplayerSetId, pending) + } + + try { + await pending + } finally { + this.inflight.delete(tcgplayerSetId) + } + + return this.cache.get(tcgplayerProductId) ?? null + } + + private async fetchSet(tcgplayerSetId: number): Promise { + const userAgent = process.env.TCGCSV_USER_AGENT + + if (!userAgent) { + return + } + + try { + const res = await fetch( + `https://tcgcsv.com/tcgplayer/3/${tcgplayerSetId}/prices`, + { headers: { 'User-Agent': userAgent } }, + ) + + if (!res.ok) { + console.warn( + `TCGPlayer: set ${tcgplayerSetId} returned ${res.status} ${res.statusText}`, + ) + return + } + + const data = (await res.json()) as TCGCSVPricesFile + + for (const result of data.results ?? []) { + const priceType = result.subTypeName.toLowerCase().replaceAll(' ', '-') + const map: ProductPriceMap = this.cache.get(result.productId) ?? new Map() + + map.set(priceType, { + low: result.lowPrice ?? undefined, + market: result.marketPrice ?? undefined, + }) + + this.cache.set(result.productId, map, PRICING_TTL_MS) + } + } catch (error) { + console.warn( + `TCGPlayer: failed to load set ${tcgplayerSetId}: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } +} diff --git a/scripts/pricing/tcgtracking.ts b/scripts/pricing/tcgtracking.ts new file mode 100644 index 0000000000..33b6250820 --- /dev/null +++ b/scripts/pricing/tcgtracking.ts @@ -0,0 +1,126 @@ +import { + fetchTCGTrackingSetPricing, + type TCGTrackingCategoryId, + type TCGTrackingProductPricing, +} from '../utils-data/tcgtracking' +import { TTLCache } from './cache' +import type { PricingProvider, ResolvedPrice } from './types' + +// Prices update daily ~8 AM EST; 12-hour TTL means at most one stale window per day +const PRICING_TTL_MS = 12 * 60 * 60 * 1000 + +/** + * TCGTracking sometimes exposes a set under a different ID than the + * existing TCGDex / TCGPlayer set ID. + * + * Key: existing TCGDex / TCGPlayer set ID + * Value: TCGTracking set ID used for API fetching + * + * Lookups are still keyed by the original TCGDex ID so getPrices() + * stays compatible with existing card data. + */ +const TCGTRACKING_SET_ID_OVERRIDES: Record = { + // Black Bolt: TCGDex/TCGPlayer ID 22325, TCGTracking ID 24325 + 22325: 24325, +} + +interface SetPricingResponse { + set_id: number + updated: string + prices: Record +} + +export class TCGTrackingProvider implements PricingProvider { + readonly name = 'tcgtracking' as const + + private pricingCache = new TTLCache() + private inflight = new Map>() + + async getPrices(input: { + categoryId: number + tcgplayerSetId: number + tcgplayerProductId: number + }): Promise { + const setPricing = await this.loadSet( + input.categoryId as TCGTrackingCategoryId, + input.tcgplayerSetId, + ) + + if (!setPricing) { + return [] + } + + const productPricing = setPricing.prices[String(input.tcgplayerProductId)] + + if (!productPricing?.tcg) { + return [] + } + + return Object.entries(productPricing.tcg) + .map(([priceType, price]) => ({ + source: 'tcgtracking' as const, + productId: input.tcgplayerProductId, + priceType, + low: price.low, + market: price.market, + })) + .filter( + (price) => + typeof price.low === 'number' || typeof price.market === 'number', + ) + } + + private async loadSet( + categoryId: TCGTrackingCategoryId, + tcgplayerSetId: number, + ): Promise { + const cacheKey = `${categoryId}:${tcgplayerSetId}` + const cached = this.pricingCache.get(cacheKey) + + if (cached !== undefined) { + return cached + } + + let pending = this.inflight.get(cacheKey) + + if (!pending) { + pending = this.fetchSet(categoryId, tcgplayerSetId, cacheKey) + this.inflight.set(cacheKey, pending) + } + + try { + return await pending + } finally { + this.inflight.delete(cacheKey) + } + } + + private async fetchSet( + categoryId: TCGTrackingCategoryId, + tcgplayerSetId: number, + cacheKey: string, + ): Promise { + const fetchSetId = TCGTRACKING_SET_ID_OVERRIDES[tcgplayerSetId] ?? tcgplayerSetId + + try { + const data = (await fetchTCGTrackingSetPricing( + categoryId, + fetchSetId, + )) as SetPricingResponse + + this.pricingCache.set(cacheKey, data, PRICING_TTL_MS) + + return data + } catch (error) { + console.warn( + `TCGTracking: failed to load pricing for category ${categoryId}, set ${fetchSetId}: ${getErrorMessage(error)}`, + ) + + return null + } + } +} + +function getErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} diff --git a/scripts/pricing/types.ts b/scripts/pricing/types.ts new file mode 100644 index 0000000000..f0493d8a3f --- /dev/null +++ b/scripts/pricing/types.ts @@ -0,0 +1,18 @@ +export type PricingSourceName = 'tcgtracking' | 'tcgplayer' + +export interface ResolvedPrice { + source: PricingSourceName + productId: number + priceType: string + low?: number + market?: number +} + +export interface PricingProvider { + name: PricingSourceName + getPrices(input: { + categoryId: number + tcgplayerSetId: number + tcgplayerProductId: number + }): Promise +} diff --git a/scripts/updatePricing.ts b/scripts/updatePricing.ts new file mode 100644 index 0000000000..b18a8cbde3 --- /dev/null +++ b/scripts/updatePricing.ts @@ -0,0 +1,592 @@ +import fs from 'node:fs/promises' +import path from 'node:path' + +import { getCategoryIdForSetFile, toImportUrl } from './utils-data/tcgtracking' +import { getPricingProvidersFromEnv } from './pricing/sources' +import type { PricingProvider, PricingSourceName, ResolvedPrice } from './pricing/types' + +// --------------------------------------------------------------------------- +// Card / set shapes +// --------------------------------------------------------------------------- + +interface TCGDexSet { + id?: string + name?: Record + thirdParty?: { tcgplayer?: number } +} + +interface TCGDexVariant { + type?: string + subtype?: string + size?: string + stamp?: string[] + foil?: string + thirdParty?: { tcgplayer?: number } +} + +interface TCGDexCard { + name?: Record + set?: TCGDexSet + thirdParty?: { tcgplayer?: number } + variants?: Record | TCGDexVariant[] +} + +interface TCGDexCardModule { + default?: TCGDexCard +} + +// --------------------------------------------------------------------------- +// Report shapes +// --------------------------------------------------------------------------- + +interface ResolvedCardPriceEntry { + target: 'card' | 'variant' + variantIndex?: number + variant?: { + type?: string + subtype?: string + size?: string + stamp?: string[] + foil?: string + } + tcgplayerProductId: number + source: PricingSourceName + priceType: string + low?: number + market?: number +} + +interface ResolvedCardPrice { + cardId: string + cardFile: string + cardName?: string + setId?: string + setName?: string + categoryId: number + tcgplayerSetId: number + prices: ResolvedCardPriceEntry[] +} + +interface MissingPrice { + cardId: string + cardFile: string + cardName?: string + setId?: string + setName?: string + categoryId?: number + tcgplayerSetId?: number + tcgplayerProductId?: number + target: 'card' | 'variant' | 'set' + variantIndex?: number + reason: + | 'missing-card-tcgplayer-id' + | 'missing-variant-tcgplayer-id' + | 'missing-set-tcgplayer-id' + | 'no-pricing-data' +} + +interface PricingReport { + createdAt: string + providers: PricingSourceName[] + summary: { + cardFilesScanned: number + cardFilesWithPrices: number + priceEntriesResolved: number + missing: number + debugArtifactWritten: boolean + } + resolvedBySource: Array<{ source: PricingSourceName; count: number }> + resolvedByTarget: Array<{ target: 'card' | 'variant'; count: number }> + missingByReason: Array<{ reason: MissingPrice['reason']; count: number }> + resolved: ResolvedCardPrice[] + missing: MissingPrice[] +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main(): Promise { + const providers = getPricingProvidersFromEnv() + const writeDebugArtifact = process.argv.includes('--write-debug-artifact') + + console.log(`Providers: ${providers.map((p) => p.name).join(', ')}`) + + const cardFiles = await findCardFiles([ + path.join(process.cwd(), 'data'), + path.join(process.cwd(), 'data-asia'), + ]) + + const resolved: ResolvedCardPrice[] = [] + const missing: MissingPrice[] = [] + let priceEntriesResolved = 0 + + for (const cardFile of cardFiles) { + const cardId = getCardIdFromFile(cardFile) + const cardModule = (await import(toImportUrl(cardFile))) as TCGDexCardModule + const card = cardModule.default + + if (!card) { + continue + } + + const categoryId = getCategoryIdForSetFile(cardFile) + const tcgplayerSetId = card.set?.thirdParty?.tcgplayer + const cardName = getEnglishName(card.name) + const setId = card.set?.id + const setName = getEnglishName(card.set?.name) + + if (typeof tcgplayerSetId !== 'number') { + missing.push({ + cardId, + cardFile: rel(cardFile), + cardName, + setId, + setName, + categoryId, + target: 'set', + reason: 'missing-set-tcgplayer-id', + }) + continue + } + + const cardPriceEntries = await resolveCardPrices({ + card, + cardId, + cardFile, + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + providers, + missing, + }) + + if (cardPriceEntries.length === 0) { + continue + } + + priceEntriesResolved += cardPriceEntries.length + + resolved.push({ + cardId, + cardFile: rel(cardFile), + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + prices: cardPriceEntries, + }) + } + + const report: PricingReport = { + createdAt: new Date().toISOString(), + providers: providers.map((p) => p.name), + summary: { + cardFilesScanned: cardFiles.length, + cardFilesWithPrices: resolved.length, + priceEntriesResolved, + missing: missing.length, + debugArtifactWritten: writeDebugArtifact, + }, + resolvedBySource: groupBySource(resolved), + resolvedByTarget: groupByTarget(resolved), + missingByReason: groupMissingByReason(missing), + resolved, + missing, + } + + await writeReport(report) + + if (writeDebugArtifact) { + await writeResolvedArtifact(resolved) + } + + printSummary(report) +} + +// --------------------------------------------------------------------------- +// Price resolution +// --------------------------------------------------------------------------- + +async function resolveCardPrices({ + card, + cardId, + cardFile, + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + providers, + missing, +}: { + card: TCGDexCard + cardId: string + cardFile: string + cardName?: string + setId?: string + setName?: string + categoryId: number + tcgplayerSetId: number + providers: PricingProvider[] + missing: MissingPrice[] +}): Promise { + const entries: ResolvedCardPriceEntry[] = [] + + if (Array.isArray(card.variants)) { + for (const [variantIndex, variant] of card.variants.entries()) { + const tcgplayerProductId = variant.thirdParty?.tcgplayer + + if (typeof tcgplayerProductId !== 'number') { + missing.push({ + cardId, + cardFile: rel(cardFile), + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + target: 'variant', + variantIndex, + reason: 'missing-variant-tcgplayer-id', + }) + continue + } + + const prices = await getPricesFromProviders(providers, { + categoryId, + tcgplayerSetId, + tcgplayerProductId, + }) + + if (prices.length === 0) { + missing.push({ + cardId, + cardFile: rel(cardFile), + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + tcgplayerProductId, + target: 'variant', + variantIndex, + reason: 'no-pricing-data', + }) + continue + } + + for (const price of prices) { + entries.push({ + target: 'variant', + variantIndex, + variant: { + type: variant.type, + subtype: variant.subtype, + size: variant.size, + stamp: variant.stamp, + foil: variant.foil, + }, + tcgplayerProductId, + source: price.source, + priceType: price.priceType, + low: price.low, + market: price.market, + }) + } + } + + return entries + } + + const tcgplayerProductId = card.thirdParty?.tcgplayer + + if (typeof tcgplayerProductId !== 'number') { + missing.push({ + cardId, + cardFile: rel(cardFile), + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + target: 'card', + reason: 'missing-card-tcgplayer-id', + }) + return entries + } + + const prices = await getPricesFromProviders(providers, { + categoryId, + tcgplayerSetId, + tcgplayerProductId, + }) + + if (prices.length === 0) { + missing.push({ + cardId, + cardFile: rel(cardFile), + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + tcgplayerProductId, + target: 'card', + reason: 'no-pricing-data', + }) + return entries + } + + for (const price of prices) { + entries.push({ + target: 'card', + tcgplayerProductId, + source: price.source, + priceType: price.priceType, + low: price.low, + market: price.market, + }) + } + + return entries +} + +async function getPricesFromProviders( + providers: PricingProvider[], + input: { + categoryId: number + tcgplayerSetId: number + tcgplayerProductId: number + }, +): Promise { + for (const provider of providers) { + const prices = await provider.getPrices(input) + + if (prices.length > 0) { + return prices + } + } + + return [] +} + +// --------------------------------------------------------------------------- +// File scanning +// --------------------------------------------------------------------------- + +async function findCardFiles(rootFolders: string[]): Promise { + const files: string[] = [] + + for (const rootFolder of rootFolders) { + const exists = await pathExists(rootFolder) + + if (!exists) { + continue + } + + await walk(rootFolder, files) + } + + return files.filter((file) => { + if (!file.endsWith('.ts')) { + return false + } + + const relative = path.relative(process.cwd(), file).replace(/\\/g, '/') + const parts = relative.split('/') + + // data/Serie/Set/Card.ts or data-asia/Serie/Set/Card.ts + return ( + (parts[0] === 'data' || parts[0] === 'data-asia') && parts.length === 4 + ) + }) +} + +async function walk(folder: string, files: string[]): Promise { + const entries = await fs.readdir(folder, { withFileTypes: true }) + + for (const entry of entries) { + const fullPath = path.join(folder, entry.name) + + if (entry.isDirectory()) { + await walk(fullPath, files) + } else { + files.push(fullPath) + } + } +} + +// --------------------------------------------------------------------------- +// Report writing +// --------------------------------------------------------------------------- + +async function writeReport(report: PricingReport): Promise { + const reportFile = path.join( + process.cwd(), + 'var', + 'reports', + 'pricing-update.json', + ) + + await fs.mkdir(path.dirname(reportFile), { recursive: true }) + await fs.writeFile(reportFile, `${JSON.stringify(report, null, '\t')}\n`, 'utf8') +} + +async function writeResolvedArtifact(resolved: ResolvedCardPrice[]): Promise { + const outputFile = path.join( + process.cwd(), + 'var', + 'models', + 'tcgtracking', + 'resolved-pricing.json', + ) + + await fs.mkdir(path.dirname(outputFile), { recursive: true }) + await fs.writeFile( + outputFile, + `${JSON.stringify( + { + source: 'tcgtracking', + createdAt: new Date().toISOString(), + cards: resolved, + }, + null, + '\t', + )}\n`, + 'utf8', + ) +} + +// --------------------------------------------------------------------------- +// Grouping helpers +// --------------------------------------------------------------------------- + +function groupBySource( + resolved: ResolvedCardPrice[], +): Array<{ source: PricingSourceName; count: number }> { + const counts = new Map() + + for (const card of resolved) { + for (const price of card.prices) { + counts.set(price.source, (counts.get(price.source) ?? 0) + 1) + } + } + + return Array.from(counts.entries()) + .map(([source, count]) => ({ source, count })) + .sort((a, b) => b.count - a.count) +} + +function groupByTarget( + resolved: ResolvedCardPrice[], +): Array<{ target: 'card' | 'variant'; count: number }> { + const counts = new Map<'card' | 'variant', number>() + + for (const card of resolved) { + for (const price of card.prices) { + counts.set(price.target, (counts.get(price.target) ?? 0) + 1) + } + } + + return Array.from(counts.entries()) + .map(([target, count]) => ({ target, count })) + .sort((a, b) => b.count - a.count) +} + +function groupMissingByReason( + missing: MissingPrice[], +): Array<{ reason: MissingPrice['reason']; count: number }> { + const counts = new Map() + + for (const item of missing) { + counts.set(item.reason, (counts.get(item.reason) ?? 0) + 1) + } + + return Array.from(counts.entries()) + .map(([reason, count]) => ({ reason, count })) + .sort((a, b) => b.count - a.count) +} + +// --------------------------------------------------------------------------- +// Summary output +// --------------------------------------------------------------------------- + +function printSummary(report: PricingReport): void { + console.log('') + console.log('Pricing update complete.') + console.log(`Providers: ${report.providers.join(', ')}`) + console.log(`Card files scanned: ${report.summary.cardFilesScanned}`) + console.log(`Cards with prices: ${report.summary.cardFilesWithPrices}`) + console.log(`Price entries resolved: ${report.summary.priceEntriesResolved}`) + console.log(`Missing records: ${report.summary.missing}`) + console.log(`Debug artifact: ${report.summary.debugArtifactWritten ? 'written' : 'skipped'}`) + + if (report.resolvedBySource.length > 0) { + console.log('') + console.log('Resolved by source:') + + for (const item of report.resolvedBySource) { + console.log(` ${item.source}: ${item.count}`) + } + } + + if (report.resolvedByTarget.length > 0) { + console.log('') + console.log('Resolved by target:') + + for (const item of report.resolvedByTarget) { + console.log(` ${item.target}: ${item.count}`) + } + } + + if (report.missingByReason.length > 0) { + console.log('') + console.log('Missing by reason:') + + for (const item of report.missingByReason) { + console.log(` ${item.reason}: ${item.count}`) + } + } + + console.log('') + console.log('Report: var/reports/pricing-update.json') + + if (report.summary.debugArtifactWritten) { + console.log('Debug artifact: var/models/tcgtracking/resolved-pricing.json') + } +} + +// --------------------------------------------------------------------------- +// Utilities +// --------------------------------------------------------------------------- + +async function pathExists(filePath: string): Promise { + try { + await fs.access(filePath) + return true + } catch { + return false + } +} + +function getCardIdFromFile(filePath: string): string { + const filename = path.basename(filePath) + return filename.slice(0, filename.lastIndexOf('.')) +} + +function getEnglishName(value?: Record): string | undefined { + return value?.en ?? Object.values(value ?? {})[0] +} + +function rel(filePath: string): string { + return path.relative(process.cwd(), filePath).replace(/\\/g, '/') +} + +main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/scripts/updatePricingFromTCGTracking.ts b/scripts/updatePricingFromTCGTracking.ts deleted file mode 100644 index df95a5e047..0000000000 --- a/scripts/updatePricingFromTCGTracking.ts +++ /dev/null @@ -1,948 +0,0 @@ -import fs from 'node:fs/promises' -import path from 'node:path' - -import { - getCategoryIdForSetFile, - getTCGTrackingPricingCacheFile, - toImportUrl, - type TCGTrackingCategoryId, -} from './utils-data/tcgtracking' - -interface TCGDexSet { - id?: string - name?: Record - serie?: { - id?: string - name?: Record - } - thirdParty?: { - tcgplayer?: number - cardmarket?: number - } -} - -interface TCGDexVariant { - type?: string - subtype?: string - size?: string - stamp?: string[] - foil?: string - thirdParty?: { - tcgplayer?: number - cardmarket?: number - } -} - -interface TCGDexCard { - name?: Record - set?: TCGDexSet - thirdParty?: { - tcgplayer?: number - cardmarket?: number - } - variants?: Record | TCGDexVariant[] -} - -interface TCGDexCardModule { - default?: TCGDexCard -} - -interface CachedTCGTrackingPricingFile { - source: 'tcgtracking' - categoryId: TCGTrackingCategoryId - tcgplayerSetId: number - sourceFile: string - fetchedAt: string - data: { - set_id: number - updated?: string - prices?: Record - } -} - -interface TCGTrackingProductPricing { - tcg?: Record -} - -interface TCGTrackingPriceBucket { - low?: number - market?: number -} - -interface TCGCSVPricesResult { - productId: number - lowPrice: number | null - midPrice: number | null - highPrice: number | null - marketPrice: number | null - directLowPrice: number | null - subTypeName: string -} - -interface TCGCSVPricesFile { - totalItems: number - success: boolean - errors: unknown[] - results: TCGCSVPricesResult[] -} - -interface ResolvedPrice { - productId: number - priceType: string - low?: number - market?: number -} - -interface ResolvedCardPrice { - cardId: string - cardFile: string - cardName?: string - setId?: string - setName?: string - categoryId: number - tcgplayerSetId: number - sourceUpdated?: string - sourceFetchedAt: string - prices: ResolvedCardPriceEntry[] -} - -interface ResolvedCardPriceEntry { - target: 'card' | 'variant' - variantIndex?: number - variant?: { - type?: string - subtype?: string - size?: string - stamp?: string[] - foil?: string - } - tcgplayerProductId: number - priceType: string - low?: number - market?: number -} - -interface MissingPrice { - cardId: string - cardFile: string - cardName?: string - setId?: string - setName?: string - categoryId?: number - tcgplayerSetId?: number - tcgplayerProductId?: number - target: 'card' | 'variant' | 'set-cache' - variantIndex?: number - reason: - | 'missing-card-tcgplayer-id' - | 'missing-variant-tcgplayer-id' - | 'missing-set-tcgplayer-id' - | 'missing-cache' - | 'missing-product-price' - | 'missing-tcg-prices' -} - -interface CountByReason { - reason: MissingPrice['reason'] - count: number -} - -interface MissingCacheSummary { - categoryId?: number - tcgplayerSetId?: number - setId?: string - setName?: string - count: number -} - -interface CountByTarget { - target: ResolvedCardPriceEntry['target'] - count: number -} - -interface CountByPriceType { - priceType: string - count: number -} - -interface PricingUpdateReport { - createdAt: string - mode: 'dry-run' | 'apply' - summary: { - cardFilesScanned: number - cardFilesWithPrices: number - priceEntriesResolved: number - missing: number - - /** - * CHANGE: - * This is the number of card records that could not find a cache file. - * - * It is not the number of unique missing cache files. - */ - missingCacheRecords: number - - /** - * CHANGE: - * This is the actual number of unique set cache files missing. - */ - uniqueMissingCaches: number - - outputWritten: boolean - } - resolvedByTarget: CountByTarget[] - priceTypes: CountByPriceType[] - missingByReason: CountByReason[] - missingCaches: MissingCacheSummary[] - resolved: ResolvedCardPrice[] - missing: MissingPrice[] -} - -/** - * Controls which pricing source is primary. - * - * tcgtracking (default): TCGTracking is primary; TCGCSV cache is used as fallback - * when TCGTracking has no data for a set. - * tcgcsv: TCGCSV-only; skips TCGTracking cache entirely. - * tcgtracking-only: TCGTracking only; no TCGCSV fallback. - * - * Set via environment variable: - * PRICING_SOURCE=tcgtracking bun scripts/updatePricingFromTCGTracking.ts --apply - */ -const PRICING_SOURCE = (process.env.PRICING_SOURCE ?? 'tcgtracking') as - | 'tcgtracking' - | 'tcgcsv' - | 'tcgtracking-only' - -/** - * CHANGE: - * Added TCGTracking pricing resolver. - * - * This script: - * - reads cached TCGTracking pricing files created by preloadTCGTracking.ts - * - reads existing TCGDex card files - * - uses existing thirdParty.tcgplayer IDs - * - resolves prices from TCGTracking - * - writes a resolved pricing artifact when --apply is used - * - * This script does NOT: - * - create variants - * - remap variants - * - edit card source files - * - remove TCGCSV fallback - * - * Why: - * TCGTracking pricing cache is keyed by existing TCGPlayer product IDs: - * - * data.prices[tcgplayerProductId].tcg[priceType].market - * data.prices[tcgplayerProductId].tcg[priceType].low - */ -async function main(): Promise { - const mode = getMode() - - const cardFiles = await findCardFiles([ - path.join(process.cwd(), 'data'), - path.join(process.cwd(), 'data-asia'), - ]) - - const resolved: ResolvedCardPrice[] = [] - const missing: MissingPrice[] = [] - - let missingCacheRecords = 0 - let priceEntriesResolved = 0 - - const cacheByKey = new Map() - - for (const cardFile of cardFiles) { - const cardId = getCardIdFromFile(cardFile) - const cardModule = (await import(toImportUrl(cardFile))) as TCGDexCardModule - const card = cardModule.default - - if (!card) { - continue - } - - const categoryId = getCategoryIdForSetFile(cardFile) - const tcgplayerSetId = card.set?.thirdParty?.tcgplayer - const cardName = getEnglishName(card.name) - const setId = card.set?.id - const setName = getEnglishName(card.set?.name) - - if (typeof tcgplayerSetId !== 'number') { - missing.push({ - cardId, - cardFile: normalizePathForReport(cardFile), - cardName, - setId, - setName, - categoryId, - target: 'set-cache', - reason: 'missing-set-tcgplayer-id', - }) - continue - } - - const cache = await getCachedPricingFile({ - categoryId, - tcgplayerSetId, - cacheByKey, - }) - - if (!cache) { - missingCacheRecords += 1 - - missing.push({ - cardId, - cardFile: normalizePathForReport(cardFile), - cardName, - setId, - setName, - categoryId, - tcgplayerSetId, - target: 'set-cache', - reason: 'missing-cache', - }) - continue - } - - const cardPriceEntries = resolvePricesForCard({ - card, - cardId, - cardFile, - cardName, - setId, - setName, - categoryId, - tcgplayerSetId, - cache, - missing, - }) - - if (cardPriceEntries.length === 0) { - continue - } - - priceEntriesResolved += cardPriceEntries.length - - resolved.push({ - cardId, - cardFile: normalizePathForReport(cardFile), - cardName, - setId, - setName, - categoryId, - tcgplayerSetId, - sourceUpdated: cache.data.updated, - sourceFetchedAt: cache.fetchedAt, - prices: cardPriceEntries, - }) - } - - const missingByReason = groupMissingByReason(missing) - const missingCaches = groupMissingCaches(missing) - const resolvedByTarget = groupResolvedByTarget(resolved) - const priceTypes = groupPriceTypes(resolved) - - const report: PricingUpdateReport = { - createdAt: new Date().toISOString(), - mode, - summary: { - cardFilesScanned: cardFiles.length, - cardFilesWithPrices: resolved.length, - priceEntriesResolved, - missing: missing.length, - missingCacheRecords, - uniqueMissingCaches: missingCaches.length, - outputWritten: mode === 'apply', - }, - resolvedByTarget, - priceTypes, - missingByReason, - missingCaches, - resolved, - missing, - } - - await writeReport(report) - - if (mode === 'apply') { - await writeResolvedPricingArtifact(resolved) - } - - printSummary(report) -} - -/** - * Finds card files. - * - * Set files are shallow: - * - * data/Serie/Set.ts - * - * Card files are deeper: - * - * data/Serie/Set/Card.ts - */ -async function findCardFiles(rootFolders: string[]): Promise { - const files: string[] = [] - - for (const rootFolder of rootFolders) { - const exists = await pathExists(rootFolder) - - if (!exists) { - continue - } - - await walk(rootFolder, files) - } - - return files.filter((file) => { - if (!file.endsWith('.ts')) { - return false - } - - const relative = path.relative(process.cwd(), file).replace(/\\/g, '/') - const parts = relative.split('/') - - // data/Serie/Set/Card.ts - // data-asia/Serie/Set/Card.ts - return ( - (parts[0] === 'data' || parts[0] === 'data-asia') && - parts.length === 4 - ) - }) -} - -async function walk(folder: string, files: string[]): Promise { - const entries = await fs.readdir(folder, { - withFileTypes: true, - }) - - for (const entry of entries) { - const fullPath = path.join(folder, entry.name) - - if (entry.isDirectory()) { - await walk(fullPath, files) - continue - } - - files.push(fullPath) - } -} - -async function getCachedPricingFile({ - categoryId, - tcgplayerSetId, - cacheByKey, -}: { - categoryId: TCGTrackingCategoryId - tcgplayerSetId: number - cacheByKey: Map -}): Promise { - const key = `${categoryId}:${tcgplayerSetId}` - - if (cacheByKey.has(key)) { - return cacheByKey.get(key) ?? null - } - - if (PRICING_SOURCE !== 'tcgcsv') { - const cacheFile = getTCGTrackingPricingCacheFile(categoryId, tcgplayerSetId) - - try { - const raw = await fs.readFile(cacheFile, 'utf8') - const parsed = JSON.parse(raw) as CachedTCGTrackingPricingFile - - cacheByKey.set(key, parsed) - - return parsed - } catch (error) { - if (!isNodeError(error) || error.code !== 'ENOENT') { - throw error - } - } - } - - // Fall back to TCGCSV cache unless explicitly using tcgtracking-only. - if (PRICING_SOURCE !== 'tcgtracking-only') { - const fallback = await readTCGCSVPricingCache(categoryId, tcgplayerSetId) - - cacheByKey.set(key, fallback) - - return fallback - } - - cacheByKey.set(key, null) - - return null -} - -/** - * Reads a TCGCSV price cache file (var/models/tcgplayer/prices/{setId}.json) - * and converts it to the CachedTCGTrackingPricingFile shape so the same - * resolver can consume it without changes. - * - * Populated by: bun scripts/preloadTCGPlayer.ts (pricing:preload:backup) - */ -async function readTCGCSVPricingCache( - categoryId: TCGTrackingCategoryId, - tcgplayerSetId: number, -): Promise { - const cacheFile = path.join( - process.cwd(), - 'var', - 'models', - 'tcgplayer', - 'prices', - `${tcgplayerSetId}.json`, - ) - - try { - const raw = await fs.readFile(cacheFile, 'utf8') - const tcgcsv = JSON.parse(raw) as TCGCSVPricesFile - - if (!tcgcsv.success || !tcgcsv.results?.length) { - return null - } - - const prices: Record }> = {} - - for (const result of tcgcsv.results) { - const id = String(result.productId) - - if (!prices[id]) { - prices[id] = { tcg: {} } - } - - const subType = result.subTypeName || 'Normal' - - prices[id].tcg[subType] = { - low: result.lowPrice ?? undefined, - market: result.marketPrice ?? undefined, - } - } - - return { - source: 'tcgtracking', - categoryId, - tcgplayerSetId, - sourceFile: `tcgcsv-fallback:${tcgplayerSetId}`, - fetchedAt: new Date().toISOString(), - data: { - set_id: tcgplayerSetId, - prices, - }, - } - } catch (error) { - if (isNodeError(error) && error.code === 'ENOENT') { - return null - } - - throw error - } -} - -function resolvePricesForCard({ - card, - cardId, - cardFile, - cardName, - setId, - setName, - categoryId, - tcgplayerSetId, - cache, - missing, -}: { - card: TCGDexCard - cardId: string - cardFile: string - cardName?: string - setId?: string - setName?: string - categoryId: number - tcgplayerSetId: number - cache: CachedTCGTrackingPricingFile - missing: MissingPrice[] -}): ResolvedCardPriceEntry[] { - const entries: ResolvedCardPriceEntry[] = [] - - /** - * If card.variants is an array, the repo is using detailed variants. - * In that case, prefer variant-level thirdParty.tcgplayer IDs. - */ - if (Array.isArray(card.variants)) { - card.variants.forEach((variant, variantIndex) => { - const tcgplayerProductId = variant.thirdParty?.tcgplayer - - if (typeof tcgplayerProductId !== 'number') { - missing.push({ - cardId, - cardFile: normalizePathForReport(cardFile), - cardName, - setId, - setName, - categoryId, - tcgplayerSetId, - target: 'variant', - variantIndex, - reason: 'missing-variant-tcgplayer-id', - }) - - return - } - - const prices = getPricesForProduct(cache, tcgplayerProductId) - - if (prices.length === 0) { - missing.push({ - cardId, - cardFile: normalizePathForReport(cardFile), - cardName, - setId, - setName, - categoryId, - tcgplayerSetId, - tcgplayerProductId, - target: 'variant', - variantIndex, - reason: 'missing-product-price', - }) - - return - } - - for (const price of prices) { - entries.push({ - target: 'variant', - variantIndex, - variant: { - type: variant.type, - subtype: variant.subtype, - size: variant.size, - stamp: variant.stamp, - foil: variant.foil, - }, - tcgplayerProductId, - priceType: price.priceType, - low: price.low, - market: price.market, - }) - } - }) - - return entries - } - - /** - * If card.variants is not an array, use the card-level product ID. - * This keeps compatibility with older card files. - */ - const tcgplayerProductId = card.thirdParty?.tcgplayer - - if (typeof tcgplayerProductId !== 'number') { - missing.push({ - cardId, - cardFile: normalizePathForReport(cardFile), - cardName, - setId, - setName, - categoryId, - tcgplayerSetId, - target: 'card', - reason: 'missing-card-tcgplayer-id', - }) - - return entries - } - - const prices = getPricesForProduct(cache, tcgplayerProductId) - - if (prices.length === 0) { - missing.push({ - cardId, - cardFile: normalizePathForReport(cardFile), - cardName, - setId, - setName, - categoryId, - tcgplayerSetId, - tcgplayerProductId, - target: 'card', - reason: 'missing-product-price', - }) - - return entries - } - - for (const price of prices) { - entries.push({ - target: 'card', - tcgplayerProductId, - priceType: price.priceType, - low: price.low, - market: price.market, - }) - } - - return entries -} - -function getPricesForProduct( - cache: CachedTCGTrackingPricingFile, - tcgplayerProductId: number, -): ResolvedPrice[] { - const product = cache.data.prices?.[String(tcgplayerProductId)] - - if (!product) { - return [] - } - - if (!product.tcg) { - return [] - } - - return Object.entries(product.tcg) - .map(([priceType, price]) => ({ - productId: tcgplayerProductId, - priceType, - low: price.low, - market: price.market, - })) - .filter((price) => { - return typeof price.low === 'number' || typeof price.market === 'number' - }) -} - -function groupMissingByReason(missing: MissingPrice[]): CountByReason[] { - const counts = new Map() - - for (const item of missing) { - counts.set(item.reason, (counts.get(item.reason) ?? 0) + 1) - } - - return Array.from(counts.entries()) - .map(([reason, count]) => ({ - reason, - count, - })) - .sort((a, b) => b.count - a.count) -} - -function groupMissingCaches(missing: MissingPrice[]): MissingCacheSummary[] { - const groups = new Map() - - for (const item of missing) { - if (item.reason !== 'missing-cache') { - continue - } - - const key = `${item.categoryId ?? 'unknown'}:${item.tcgplayerSetId ?? 'unknown'}` - const existing = groups.get(key) - - if (existing) { - existing.count += 1 - continue - } - - groups.set(key, { - categoryId: item.categoryId, - tcgplayerSetId: item.tcgplayerSetId, - setId: item.setId, - setName: item.setName, - count: 1, - }) - } - - return Array.from(groups.values()).sort((a, b) => b.count - a.count) -} - -function groupResolvedByTarget(resolved: ResolvedCardPrice[]): CountByTarget[] { - const counts = new Map() - - for (const card of resolved) { - for (const price of card.prices) { - counts.set(price.target, (counts.get(price.target) ?? 0) + 1) - } - } - - return Array.from(counts.entries()) - .map(([target, count]) => ({ - target, - count, - })) - .sort((a, b) => b.count - a.count) -} - -function groupPriceTypes(resolved: ResolvedCardPrice[]): CountByPriceType[] { - const counts = new Map() - - for (const card of resolved) { - for (const price of card.prices) { - counts.set(price.priceType, (counts.get(price.priceType) ?? 0) + 1) - } - } - - return Array.from(counts.entries()) - .map(([priceType, count]) => ({ - priceType, - count, - })) - .sort((a, b) => b.count - a.count) -} - -async function writeReport(report: PricingUpdateReport): Promise { - const reportFile = path.join( - process.cwd(), - 'var', - 'reports', - 'tcgtracking-pricing-update.json', - ) - - await fs.mkdir(path.dirname(reportFile), { - recursive: true, - }) - - await fs.writeFile( - reportFile, - `${JSON.stringify(report, null, '\t')}\n`, - 'utf8', - ) -} - -/** - * CHANGE: - * This writes the resolved pricing artifact consumed by the next integration step. - * - * It intentionally does not mutate card source files because the current source files - * store marketplace IDs, while pricing should be generated from the cached provider data. - */ -async function writeResolvedPricingArtifact( - resolved: ResolvedCardPrice[], -): Promise { - const outputFile = path.join( - process.cwd(), - 'var', - 'models', - 'tcgtracking', - 'resolved-pricing.json', - ) - - await fs.mkdir(path.dirname(outputFile), { - recursive: true, - }) - - await fs.writeFile( - outputFile, - `${JSON.stringify( - { - source: 'tcgtracking', - createdAt: new Date().toISOString(), - cards: resolved, - }, - null, - '\t', - )}\n`, - 'utf8', - ) -} - -function printSummary(report: PricingUpdateReport): void { - console.log('') - console.log('TCGTracking pricing update complete.') - console.log(`Mode: ${report.mode}`) - console.log(`Pricing source: ${PRICING_SOURCE}`) - console.log(`Card files scanned: ${report.summary.cardFilesScanned}`) - console.log(`Cards with prices: ${report.summary.cardFilesWithPrices}`) - console.log(`Price entries resolved: ${report.summary.priceEntriesResolved}`) - console.log(`Missing records: ${report.summary.missing}`) - console.log(`Missing cache records: ${report.summary.missingCacheRecords}`) - console.log(`Unique missing caches: ${report.summary.uniqueMissingCaches}`) - console.log(`Output written: ${report.summary.outputWritten ? 'yes' : 'no'}`) - - if (report.resolvedByTarget.length > 0) { - console.log('') - console.log('Resolved by target:') - - for (const item of report.resolvedByTarget) { - console.log(`- ${item.target}: ${item.count}`) - } - } - - if (report.priceTypes.length > 0) { - console.log('') - console.log('Resolved price types:') - - for (const item of report.priceTypes.slice(0, 20)) { - console.log(`- ${item.priceType}: ${item.count}`) - } - } - - if (report.missingByReason.length > 0) { - console.log('') - console.log('Missing by reason:') - - for (const item of report.missingByReason) { - console.log(`- ${item.reason}: ${item.count}`) - } - } - - if (report.missingCaches.length > 0) { - console.log('') - console.log('Missing cache sets:') - - for (const item of report.missingCaches.slice(0, 20)) { - console.log( - `- ${item.setName ?? item.setId ?? 'Unknown set'} (${item.tcgplayerSetId ?? 'unknown'}): ${item.count}`, - ) - } - } - - console.log('') - console.log('Report:') - console.log('var/reports/tcgtracking-pricing-update.json') - - if (report.mode === 'apply') { - console.log('') - console.log('Resolved pricing artifact:') - console.log('var/models/tcgtracking/resolved-pricing.json') - } -} - -function getMode(): 'dry-run' | 'apply' { - if (process.argv.includes('--apply')) { - return 'apply' - } - - return 'dry-run' -} - -async function pathExists(filePath: string): Promise { - try { - await fs.access(filePath) - return true - } catch { - return false - } -} - -function getCardIdFromFile(filePath: string): string { - const filename = path.basename(filePath) - return filename.slice(0, filename.lastIndexOf('.')) -} - -function getEnglishName(value?: Record): string | undefined { - return value?.en ?? Object.values(value ?? {})[0] -} - -function normalizePathForReport(filePath: string): string { - return path.relative(process.cwd(), filePath).replace(/\\/g, '/') -} - -function isNodeError(error: unknown): error is NodeJS.ErrnoException { - return error instanceof Error && 'code' in error -} - -main().catch((error) => { - console.error(error) - process.exitCode = 1 -}) \ No newline at end of file diff --git a/scripts/utils-data/tcgtracking.ts b/scripts/utils-data/tcgtracking.ts index 07ba814ea0..bc7e1e63a7 100644 --- a/scripts/utils-data/tcgtracking.ts +++ b/scripts/utils-data/tcgtracking.ts @@ -1,12 +1,8 @@ -import fs from 'node:fs/promises' import path from 'node:path' import { pathToFileURL } from 'node:url' /** * TCGTracking Open TCG API base URL. - * - * This replaces the old TCGCSV fetch source for pricing data, - * but does not remove the old TCGCSV scripts yet. */ export const TCGTRACKING_API_BASE_URL = 'https://tcgtracking.com/tcgapi/v1' @@ -27,93 +23,8 @@ export const TCGTRACKING_CATEGORIES = { export type TCGTrackingCategoryId = (typeof TCGTRACKING_CATEGORIES)[keyof typeof TCGTRACKING_CATEGORIES] -/** - * The cached pricing record is intentionally loose because TCGTracking may expose - * price records with short keys, long keys, or nested response wrappers. - * - * The preload script stores the original raw JSON as-is. - * Normalisation happens when reading the cache later. - */ export type TCGTrackingRawPricingResponse = unknown -export interface TCGTrackingSetCacheTarget { - categoryId: TCGTrackingCategoryId - - /** - * Existing TCGDex / TCGPlayer set ID. - * - * This is used for cache file naming so the updater can still look up pricing - * by card.set.thirdParty.tcgplayer. - */ - tcgplayerSetId: number - - /** - * Optional TCGTracking set ID override. - * - * This is used only when fetching from TCGTracking if their set ID differs - * from the existing TCGDex / TCGPlayer set ID. - * - * Example: - * - TCGDex Black Bolt set ID: 22325 - * - TCGTracking Black Bolt set ID: 24325 - */ - tcgTrackingSetId?: number - - sourceFile: string -} - -/** - * Cached TCGTracking pricing file written by scripts/preloadTCGTracking.ts. - * - * Real cache shape: - * - * data.prices[tcgplayerProductId].tcg[priceType].low - * data.prices[tcgplayerProductId].tcg[priceType].market - * - * Example priceType values from the uploaded cache include: - * - * Normal - * Holofoil - * Reverse Holofoil - */ -export interface CachedTCGTrackingPricingFile { - source: 'tcgtracking' - categoryId: TCGTrackingCategoryId - - /** - * Existing TCGDex / TCGPlayer set ID. - * - * This should match card.set.thirdParty.tcgplayer. - */ - tcgplayerSetId: number - - /** - * Actual TCGTracking set ID used when fetching. - * - * Usually the same as tcgplayerSetId, but can differ for sets such as - * Black Bolt. - * - * Known missing TCGTracking pricing endpoints: - * - * - DP trainer Kit (Manaphy), TCGPlayer set ID 609 - * - DP trainer Kit (Lucario), TCGPlayer set ID 610 - * - * TCGTracking search currently returns no matching set for: - * "DP Trainer Kit Manaphy/Lucario" - * - * These should fall back to existing/TCGCSV pricing. - */ - tcgTrackingSetId?: number - - sourceFile: string - fetchedAt: string - data: { - set_id: number - updated?: string - prices?: Record - } -} - export interface TCGTrackingProductPricing { tcg?: Record } @@ -123,76 +34,6 @@ export interface TCGTrackingVariantPrice { market?: number } -export interface ResolvedTCGTrackingPrice { - productId: number - priceType: string - low?: number - market?: number -} - -/** - * CHANGE: - * Added a dedicated TCGTracking cache folder. - * - * This avoids changing the old TCGCSV cache: - * - * var/models/tcgplayer/... - * - * New cache: - * - * var/models/tcgtracking/category-3/pricing/{setId}.json - * var/models/tcgtracking/category-85/pricing/{setId}.json - * - * IMPORTANT: - * The {setId}.json filename should use the existing TCGDex / TCGPlayer set ID, - * not the override ID. This keeps the updater compatible with existing card data. - */ -export function getTCGTrackingPricingCacheFile( - categoryId: TCGTrackingCategoryId, - tcgplayerSetId: number, -): string { - return path.join( - process.cwd(), - 'var', - 'models', - 'tcgtracking', - `category-${categoryId}`, - 'pricing', - `${tcgplayerSetId}.json`, - ) -} - -export async function ensureParentFolder(filePath: string): Promise { - await fs.mkdir(path.dirname(filePath), { recursive: true }) -} - -export async function writeJsonFile(filePath: string, data: unknown): Promise { - await ensureParentFolder(filePath) - - await fs.writeFile( - filePath, - `${JSON.stringify(data, null, '\t')}\n`, - 'utf8', - ) -} - -export async function readJsonFile(filePath: string): Promise { - try { - const raw = await fs.readFile(filePath, 'utf8') - return JSON.parse(raw) as T - } catch (error) { - if (isNodeError(error) && error.code === 'ENOENT') { - return null - } - - throw error - } -} - -export function isNodeError(error: unknown): error is NodeJS.ErrnoException { - return error instanceof Error && 'code' in error -} - export async function fetchTCGTrackingSetPricing( categoryId: TCGTrackingCategoryId, tcgTrackingSetId: number, @@ -215,40 +56,18 @@ export async function fetchTCGTrackingSetPricing( } /** - * Small delay to avoid hammering the API when fetching many sets. - * - * TCGTracking is CDN-backed, but keeping this polite makes the script safer - * for repeated maintainer use and CI. - */ -export async function sleep(ms: number): Promise { - await new Promise((resolve) => { - setTimeout(resolve, ms) - }) -} - -/** - * CHANGE: * Converts a filesystem path into a valid file URL for dynamic import. * - * The previous manual file:/// conversion broke on paths with special characters. - * - * Example broken path: - * - * H:\cards-database\data\EX\Unseen Forces Unown Collection\?.ts - * - * A literal ? inside an import URL is treated as the start of a query string unless - * it is encoded. pathToFileURL handles that correctly. + * A literal ? inside an import URL is treated as the start of a query string + * unless it is encoded. pathToFileURL handles that correctly. */ export function toImportUrl(filePath: string): string { return pathToFileURL(path.resolve(filePath)).href } /** - * Decide which TCGTracking category to use based on the file location. - * - * CHANGE: - * This is only for selecting the pricing source category. - * It does not change card data or map variants. + * Decides which TCGTracking category to use based on the file location. + * Only used for selecting the pricing source category, not for card data. */ export function getCategoryIdForSetFile(filePath: string): TCGTrackingCategoryId { const normalized = filePath.replace(/\\/g, '/') @@ -259,69 +78,3 @@ export function getCategoryIdForSetFile(filePath: string): TCGTrackingCategoryId return TCGTRACKING_CATEGORIES.pokemon } - -/** - * Reads a cached TCGTracking pricing file. - */ -export async function readTCGTrackingPricingCache( - categoryId: TCGTrackingCategoryId, - tcgplayerSetId: number, -): Promise { - const cacheFile = getTCGTrackingPricingCacheFile(categoryId, tcgplayerSetId) - - return readJsonFile(cacheFile) -} - -/** - * Finds prices for an existing TCGPlayer product ID. - * - * This does not decide what variants exist. - * It only returns the price buckets TCGTracking has for that product. - */ -export function getPricesForTCGPlayerProduct( - cache: CachedTCGTrackingPricingFile, - tcgplayerProductId: number, -): ResolvedTCGTrackingPrice[] { - const productPricing = cache.data.prices?.[String(tcgplayerProductId)] - - if (!productPricing?.tcg) { - return [] - } - - return Object.entries(productPricing.tcg) - .map(([priceType, price]) => ({ - productId: tcgplayerProductId, - priceType, - low: price.low, - market: price.market, - })) - .filter((price) => { - return typeof price.low === 'number' || typeof price.market === 'number' - }) -} - -/** - * Gets the best single price for a product. - * - * Preference: - * 1. market - * 2. low - * - * This is useful if the current TCGDex pricing shape only stores one price value. - */ -export function getBestPriceForTCGPlayerProduct( - cache: CachedTCGTrackingPricingFile, - tcgplayerProductId: number, -): ResolvedTCGTrackingPrice | null { - const prices = getPricesForTCGPlayerProduct(cache, tcgplayerProductId) - - const withMarket = prices.find((price) => typeof price.market === 'number') - - if (withMarket) { - return withMarket - } - - const withLow = prices.find((price) => typeof price.low === 'number') - - return withLow ?? null -} \ No newline at end of file diff --git a/server/src/libs/providers/tcgplayer/tcgtracking.ts b/server/src/libs/providers/tcgplayer/tcgtracking.ts index 418bf42fb5..9d070e56df 100644 --- a/server/src/libs/providers/tcgplayer/tcgtracking.ts +++ b/server/src/libs/providers/tcgplayer/tcgtracking.ts @@ -1,25 +1,22 @@ -import fs from 'node:fs/promises' -import path from 'node:path' - -interface ResolvedPricingEntry { - target: 'card' | 'variant' - tcgplayerProductId: number - priceType: string - low?: number - market?: number -} +import { sets } from '../../../V2/Components/Set' + +const TCGTRACKING_API_BASE_URL = 'https://tcgtracking.com/tcgapi/v1' -interface ResolvedPricingCard { - cardId: string - sourceFetchedAt: string - sourceUpdated?: string - prices: ResolvedPricingEntry[] +/** + * TCGTracking sometimes exposes a set under a different ID than the + * existing TCGDex / TCGPlayer set ID. + */ +const TCGTRACKING_SET_ID_OVERRIDES: Record = { + // Black Bolt: TCGDex/TCGPlayer ID 22325, TCGTracking ID 24325 + 22325: 24325, } -interface ResolvedPricingFile { - source: 'tcgtracking' - createdAt: string - cards: ResolvedPricingCard[] +interface SetPricingResponse { + set_id: number + updated: string + prices: Record + }> } export interface PriceResult { @@ -31,72 +28,86 @@ export interface PriceResult { directLowPrice?: number } -// Index: productId → normalised-price-type → PriceResult +// productId → normalised-price-type → PriceResult let cache: Record> = {} -let lastLoaded: Date | undefined = undefined -let lastUpdated: string | undefined = undefined - -// Can be overridden via TCGTRACKING_PRICING_PATH env var for Docker / custom setups. -// Default: one level above the server directory (standard repo layout). -const RESOLVED_PRICING_PATH = process.env.TCGTRACKING_PRICING_PATH ?? path.join( - process.cwd(), - '..', - 'var', - 'models', - 'tcgtracking', - 'resolved-pricing.json', -) +let lastFetch: Date | undefined = undefined export async function updateTCGPlayerDatas(): Promise { - try { - const stat = await fs.stat(RESOLVED_PRICING_PATH) + // Refresh at most once per hour + if (lastFetch && Date.now() - lastFetch.getTime() < 3600000) { + return false + } - // Skip reload when file hasn't changed since last load - if (lastLoaded && stat.mtimeMs <= lastLoaded.getTime()) { - return false - } + const setIds = sets.en + .filter((it) => it?.thirdParty?.tcgplayer) + .map((it) => it!.thirdParty!.tcgplayer!) + + const newCache: Record> = {} + let loaded = 0 + let failed = 0 + + for (const setId of setIds) { + const fetchSetId = TCGTRACKING_SET_ID_OVERRIDES[setId] ?? setId + + try { + const res = await fetch( + `${TCGTRACKING_API_BASE_URL}/3/sets/${fetchSetId}/pricing`, + { headers: { Accept: 'application/json' } }, + ) + + if (!res.ok) { + failed++ + console.warn( + `TCGTracking: set ${fetchSetId} returned ${res.status} ${res.statusText}`, + ) + continue + } - const raw = await fs.readFile(RESOLVED_PRICING_PATH, 'utf8') - const data = JSON.parse(raw) as ResolvedPricingFile + const data = (await res.json()) as SetPricingResponse - if (data.source !== 'tcgtracking') { - console.warn('TCGTracking: unexpected source in resolved-pricing.json:', data.source) - return false - } + for (const [productIdStr, productPricing] of Object.entries(data.prices ?? {})) { + const productId = Number(productIdStr) + + if (!productPricing.tcg) { + continue + } - const newCache: Record> = {} + const productCache: Record = {} - for (const card of data.cards) { - for (const price of card.prices) { - const productId = price.tcgplayerProductId - const key = price.priceType.toLowerCase().replaceAll(' ', '-') + for (const [priceType, price] of Object.entries(productPricing.tcg)) { + const key = priceType.toLowerCase().replaceAll(' ', '-') - if (!newCache[productId]) { - newCache[productId] = {} + if (typeof price.low === 'number' || typeof price.market === 'number') { + productCache[key] = { + productId, + lowPrice: price.low ?? 0, + midPrice: 0, + highPrice: 0, + marketPrice: price.market, + } + } } - newCache[productId][key] = { - productId, - lowPrice: price.low ?? 0, - midPrice: 0, - highPrice: 0, - marketPrice: price.market, + if (Object.keys(productCache).length > 0) { + newCache[productId] = productCache } } - } - cache = newCache - lastLoaded = new Date() - lastUpdated = data.createdAt - - return true - } catch (error) { - if (isNodeError(error) && error.code === 'ENOENT') { - console.warn('TCGTracking: resolved-pricing.json not found:', RESOLVED_PRICING_PATH) - return false + loaded++ + } catch (error) { + failed++ + console.warn( + `TCGTracking: failed to load set ${fetchSetId}: ${error instanceof Error ? error.message : String(error)}`, + ) } - throw error } + + cache = newCache + lastFetch = new Date() + + console.log(`TCGTracking: loaded ${loaded} sets, ${failed} failed`) + + return true } export async function getTCGPlayerPrice(card: { thirdParty: { tcgplayer?: number } }): Promise<{ @@ -104,7 +115,7 @@ export async function getTCGPlayerPrice(card: { thirdParty: { tcgplayer?: number updated: string [key: string]: unknown } | null> { - if (!lastLoaded || typeof card.thirdParty?.tcgplayer !== 'number') { + if (!lastFetch || typeof card.thirdParty?.tcgplayer !== 'number') { return null } @@ -115,12 +126,8 @@ export async function getTCGPlayerPrice(card: { thirdParty: { tcgplayer?: number } return { - updated: lastUpdated ?? lastLoaded.toISOString(), + updated: lastFetch.toISOString(), unit: 'USD', ...variants, } } - -function isNodeError(error: unknown): error is NodeJS.ErrnoException { - return error instanceof Error && 'code' in error -}