diff --git a/package.json b/package.json index b9ec2112dd..15dc02db02 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,10 @@ "dex:audit:report": "bun scripts/pokedexIdFixer/audit-dex-ids.ts --report", "dex:fix:dry-run": "bun scripts/pokedexIdFixer/fix-dex-ids.ts --dry-run", "dex:fix:apply": "bun scripts/pokedexIdFixer/fix-dex-ids.ts --apply", - "dex:lint": "bun scripts/pokedexIdFixer/lint-dex-ids.ts" + "dex:lint": "bun scripts/pokedexIdFixer/lint-dex-ids.ts", + + "pricing:update": "bun scripts/updatePricing.ts", + "pricing:update:debug": "bun scripts/updatePricing.ts --write-debug-artifact", }, "devDependencies": { "@dzeio/object-util": "^1.9.2", @@ -22,4 +25,4 @@ "ts-node": "^10.0.0", "typescript": "^5.0.0" } -} +} \ No newline at end of file diff --git a/scripts/preloadTCGPlayer.ts b/scripts/preloadTCGPlayer.ts index affb658216..159b7c25e5 100644 --- a/scripts/preloadTCGPlayer.ts +++ b/scripts/preloadTCGPlayer.ts @@ -54,6 +54,20 @@ try { await fs.writeFile(`${folder}/${id}.json`, JSON.stringify(products)) } + // Load prices (used as TCGCSV fallback by updatePricingFromTCGTracking) + console.log('Loading prices...') + const pricesFolder = baseFolder + '/prices' + await fs.mkdir(pricesFolder, { recursive: true }) + + for (const id of ids) { + console.log('Loading prices', id) + const prices = await fetch(`https://tcgcsv.com/tcgplayer/3/${id}/prices`, { + headers: { 'User-Agent': userAgent } + }) + .then((it) => it.json()) + await fs.writeFile(`${pricesFolder}/${id}.json`, JSON.stringify(prices)) + } + console.log('done') } catch (error) { diff --git a/scripts/pricing/cache.ts b/scripts/pricing/cache.ts new file mode 100644 index 0000000000..e76a722838 --- /dev/null +++ b/scripts/pricing/cache.ts @@ -0,0 +1,35 @@ +interface CacheEntry { + value: V + expiresAt: number +} + +export class TTLCache { + private map = new Map>() + + set(key: K, value: V, ttlMs: number): void { + this.map.set(key, { value, expiresAt: Date.now() + ttlMs }) + } + + get(key: K): V | undefined { + const entry = this.map.get(key) + + if (!entry) { + return undefined + } + + if (Date.now() > entry.expiresAt) { + this.map.delete(key) + return undefined + } + + return entry.value + } + + has(key: K): boolean { + return this.get(key) !== undefined + } + + delete(key: K): void { + this.map.delete(key) + } +} diff --git a/scripts/pricing/sources.ts b/scripts/pricing/sources.ts new file mode 100644 index 0000000000..b926bee4d6 --- /dev/null +++ b/scripts/pricing/sources.ts @@ -0,0 +1,26 @@ +import { TCGTrackingProvider } from './tcgtracking' +import { TCGPlayerProvider } from './tcgplayer' +import type { PricingProvider, PricingSourceName } from './types' + +const PROVIDER_FACTORIES: Record PricingProvider> = { + tcgtracking: () => new TCGTrackingProvider(), + tcgplayer: () => new TCGPlayerProvider(), +} + +/** + * Returns pricing providers in the order defined by TCGDEX_PRICING_SOURCES. + * + * Default: tcgtracking,tcgplayer + * + * The first provider to return prices for a product wins. Providers that + * fail to load or have no data for a product are silently skipped. + */ +export function getPricingProvidersFromEnv(): PricingProvider[] { + const raw = process.env.TCGDEX_PRICING_SOURCES ?? 'tcgtracking,tcgplayer' + + return raw + .split(',') + .map((s) => s.trim() as PricingSourceName) + .filter((name) => name in PROVIDER_FACTORIES) + .map((name) => PROVIDER_FACTORIES[name]()) +} diff --git a/scripts/pricing/tcgplayer.ts b/scripts/pricing/tcgplayer.ts new file mode 100644 index 0000000000..51afb8f71f --- /dev/null +++ b/scripts/pricing/tcgplayer.ts @@ -0,0 +1,124 @@ +import { TTLCache } from './cache' +import type { PricingProvider, ResolvedPrice } from './types' + +// Same TTL as TCGTracking — prices update daily +const PRICING_TTL_MS = 12 * 60 * 60 * 1000 + +interface TCGCSVPricesResult { + productId: number + lowPrice: number | null + marketPrice: number | null + subTypeName: string +} + +interface TCGCSVPricesFile { + success: boolean + results: TCGCSVPricesResult[] +} + +type ProductPriceMap = Map + +export class TCGPlayerProvider implements PricingProvider { + readonly name = 'tcgplayer' as const + + // productId → normalised-price-type → prices + private cache = new TTLCache() + // Deduplicate concurrent fetches for the same set + private inflight = new Map>() + + async getPrices(input: { + categoryId: number + tcgplayerSetId: number + tcgplayerProductId: number + }): Promise { + const productPricing = await this.loadProduct( + input.tcgplayerSetId, + input.tcgplayerProductId, + ) + + if (!productPricing) { + return [] + } + + const result: ResolvedPrice[] = [] + + for (const [priceType, price] of productPricing) { + if (typeof price.low === 'number' || typeof price.market === 'number') { + result.push({ + source: 'tcgplayer', + productId: input.tcgplayerProductId, + priceType, + low: price.low, + market: price.market, + }) + } + } + + return result + } + + private async loadProduct( + tcgplayerSetId: number, + tcgplayerProductId: number, + ): Promise { + if (this.cache.has(tcgplayerProductId)) { + return this.cache.get(tcgplayerProductId) ?? null + } + + // Fetch the whole set so all products in it are cached together + let pending = this.inflight.get(tcgplayerSetId) + + if (!pending) { + pending = this.fetchSet(tcgplayerSetId) + this.inflight.set(tcgplayerSetId, pending) + } + + try { + await pending + } finally { + this.inflight.delete(tcgplayerSetId) + } + + return this.cache.get(tcgplayerProductId) ?? null + } + + private async fetchSet(tcgplayerSetId: number): Promise { + const userAgent = process.env.TCGCSV_USER_AGENT + + if (!userAgent) { + return + } + + try { + const res = await fetch( + `https://tcgcsv.com/tcgplayer/3/${tcgplayerSetId}/prices`, + { headers: { 'User-Agent': userAgent } }, + ) + + if (!res.ok) { + console.warn( + `TCGPlayer: set ${tcgplayerSetId} returned ${res.status} ${res.statusText}`, + ) + return + } + + const data = (await res.json()) as TCGCSVPricesFile + + for (const result of data.results ?? []) { + const priceType = result.subTypeName.toLowerCase().replaceAll(' ', '-') + const map: ProductPriceMap = this.cache.get(result.productId) ?? new Map() + + map.set(priceType, { + low: result.lowPrice ?? undefined, + market: result.marketPrice ?? undefined, + }) + + this.cache.set(result.productId, map, PRICING_TTL_MS) + } + } catch (error) { + console.warn( + `TCGPlayer: failed to load set ${tcgplayerSetId}: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } +} diff --git a/scripts/pricing/tcgtracking.ts b/scripts/pricing/tcgtracking.ts new file mode 100644 index 0000000000..33b6250820 --- /dev/null +++ b/scripts/pricing/tcgtracking.ts @@ -0,0 +1,126 @@ +import { + fetchTCGTrackingSetPricing, + type TCGTrackingCategoryId, + type TCGTrackingProductPricing, +} from '../utils-data/tcgtracking' +import { TTLCache } from './cache' +import type { PricingProvider, ResolvedPrice } from './types' + +// Prices update daily ~8 AM EST; 12-hour TTL means at most one stale window per day +const PRICING_TTL_MS = 12 * 60 * 60 * 1000 + +/** + * TCGTracking sometimes exposes a set under a different ID than the + * existing TCGDex / TCGPlayer set ID. + * + * Key: existing TCGDex / TCGPlayer set ID + * Value: TCGTracking set ID used for API fetching + * + * Lookups are still keyed by the original TCGDex ID so getPrices() + * stays compatible with existing card data. + */ +const TCGTRACKING_SET_ID_OVERRIDES: Record = { + // Black Bolt: TCGDex/TCGPlayer ID 22325, TCGTracking ID 24325 + 22325: 24325, +} + +interface SetPricingResponse { + set_id: number + updated: string + prices: Record +} + +export class TCGTrackingProvider implements PricingProvider { + readonly name = 'tcgtracking' as const + + private pricingCache = new TTLCache() + private inflight = new Map>() + + async getPrices(input: { + categoryId: number + tcgplayerSetId: number + tcgplayerProductId: number + }): Promise { + const setPricing = await this.loadSet( + input.categoryId as TCGTrackingCategoryId, + input.tcgplayerSetId, + ) + + if (!setPricing) { + return [] + } + + const productPricing = setPricing.prices[String(input.tcgplayerProductId)] + + if (!productPricing?.tcg) { + return [] + } + + return Object.entries(productPricing.tcg) + .map(([priceType, price]) => ({ + source: 'tcgtracking' as const, + productId: input.tcgplayerProductId, + priceType, + low: price.low, + market: price.market, + })) + .filter( + (price) => + typeof price.low === 'number' || typeof price.market === 'number', + ) + } + + private async loadSet( + categoryId: TCGTrackingCategoryId, + tcgplayerSetId: number, + ): Promise { + const cacheKey = `${categoryId}:${tcgplayerSetId}` + const cached = this.pricingCache.get(cacheKey) + + if (cached !== undefined) { + return cached + } + + let pending = this.inflight.get(cacheKey) + + if (!pending) { + pending = this.fetchSet(categoryId, tcgplayerSetId, cacheKey) + this.inflight.set(cacheKey, pending) + } + + try { + return await pending + } finally { + this.inflight.delete(cacheKey) + } + } + + private async fetchSet( + categoryId: TCGTrackingCategoryId, + tcgplayerSetId: number, + cacheKey: string, + ): Promise { + const fetchSetId = TCGTRACKING_SET_ID_OVERRIDES[tcgplayerSetId] ?? tcgplayerSetId + + try { + const data = (await fetchTCGTrackingSetPricing( + categoryId, + fetchSetId, + )) as SetPricingResponse + + this.pricingCache.set(cacheKey, data, PRICING_TTL_MS) + + return data + } catch (error) { + console.warn( + `TCGTracking: failed to load pricing for category ${categoryId}, set ${fetchSetId}: ${getErrorMessage(error)}`, + ) + + return null + } + } +} + +function getErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} diff --git a/scripts/pricing/types.ts b/scripts/pricing/types.ts new file mode 100644 index 0000000000..f0493d8a3f --- /dev/null +++ b/scripts/pricing/types.ts @@ -0,0 +1,18 @@ +export type PricingSourceName = 'tcgtracking' | 'tcgplayer' + +export interface ResolvedPrice { + source: PricingSourceName + productId: number + priceType: string + low?: number + market?: number +} + +export interface PricingProvider { + name: PricingSourceName + getPrices(input: { + categoryId: number + tcgplayerSetId: number + tcgplayerProductId: number + }): Promise +} diff --git a/scripts/updatePricing.ts b/scripts/updatePricing.ts new file mode 100644 index 0000000000..b18a8cbde3 --- /dev/null +++ b/scripts/updatePricing.ts @@ -0,0 +1,592 @@ +import fs from 'node:fs/promises' +import path from 'node:path' + +import { getCategoryIdForSetFile, toImportUrl } from './utils-data/tcgtracking' +import { getPricingProvidersFromEnv } from './pricing/sources' +import type { PricingProvider, PricingSourceName, ResolvedPrice } from './pricing/types' + +// --------------------------------------------------------------------------- +// Card / set shapes +// --------------------------------------------------------------------------- + +interface TCGDexSet { + id?: string + name?: Record + thirdParty?: { tcgplayer?: number } +} + +interface TCGDexVariant { + type?: string + subtype?: string + size?: string + stamp?: string[] + foil?: string + thirdParty?: { tcgplayer?: number } +} + +interface TCGDexCard { + name?: Record + set?: TCGDexSet + thirdParty?: { tcgplayer?: number } + variants?: Record | TCGDexVariant[] +} + +interface TCGDexCardModule { + default?: TCGDexCard +} + +// --------------------------------------------------------------------------- +// Report shapes +// --------------------------------------------------------------------------- + +interface ResolvedCardPriceEntry { + target: 'card' | 'variant' + variantIndex?: number + variant?: { + type?: string + subtype?: string + size?: string + stamp?: string[] + foil?: string + } + tcgplayerProductId: number + source: PricingSourceName + priceType: string + low?: number + market?: number +} + +interface ResolvedCardPrice { + cardId: string + cardFile: string + cardName?: string + setId?: string + setName?: string + categoryId: number + tcgplayerSetId: number + prices: ResolvedCardPriceEntry[] +} + +interface MissingPrice { + cardId: string + cardFile: string + cardName?: string + setId?: string + setName?: string + categoryId?: number + tcgplayerSetId?: number + tcgplayerProductId?: number + target: 'card' | 'variant' | 'set' + variantIndex?: number + reason: + | 'missing-card-tcgplayer-id' + | 'missing-variant-tcgplayer-id' + | 'missing-set-tcgplayer-id' + | 'no-pricing-data' +} + +interface PricingReport { + createdAt: string + providers: PricingSourceName[] + summary: { + cardFilesScanned: number + cardFilesWithPrices: number + priceEntriesResolved: number + missing: number + debugArtifactWritten: boolean + } + resolvedBySource: Array<{ source: PricingSourceName; count: number }> + resolvedByTarget: Array<{ target: 'card' | 'variant'; count: number }> + missingByReason: Array<{ reason: MissingPrice['reason']; count: number }> + resolved: ResolvedCardPrice[] + missing: MissingPrice[] +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main(): Promise { + const providers = getPricingProvidersFromEnv() + const writeDebugArtifact = process.argv.includes('--write-debug-artifact') + + console.log(`Providers: ${providers.map((p) => p.name).join(', ')}`) + + const cardFiles = await findCardFiles([ + path.join(process.cwd(), 'data'), + path.join(process.cwd(), 'data-asia'), + ]) + + const resolved: ResolvedCardPrice[] = [] + const missing: MissingPrice[] = [] + let priceEntriesResolved = 0 + + for (const cardFile of cardFiles) { + const cardId = getCardIdFromFile(cardFile) + const cardModule = (await import(toImportUrl(cardFile))) as TCGDexCardModule + const card = cardModule.default + + if (!card) { + continue + } + + const categoryId = getCategoryIdForSetFile(cardFile) + const tcgplayerSetId = card.set?.thirdParty?.tcgplayer + const cardName = getEnglishName(card.name) + const setId = card.set?.id + const setName = getEnglishName(card.set?.name) + + if (typeof tcgplayerSetId !== 'number') { + missing.push({ + cardId, + cardFile: rel(cardFile), + cardName, + setId, + setName, + categoryId, + target: 'set', + reason: 'missing-set-tcgplayer-id', + }) + continue + } + + const cardPriceEntries = await resolveCardPrices({ + card, + cardId, + cardFile, + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + providers, + missing, + }) + + if (cardPriceEntries.length === 0) { + continue + } + + priceEntriesResolved += cardPriceEntries.length + + resolved.push({ + cardId, + cardFile: rel(cardFile), + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + prices: cardPriceEntries, + }) + } + + const report: PricingReport = { + createdAt: new Date().toISOString(), + providers: providers.map((p) => p.name), + summary: { + cardFilesScanned: cardFiles.length, + cardFilesWithPrices: resolved.length, + priceEntriesResolved, + missing: missing.length, + debugArtifactWritten: writeDebugArtifact, + }, + resolvedBySource: groupBySource(resolved), + resolvedByTarget: groupByTarget(resolved), + missingByReason: groupMissingByReason(missing), + resolved, + missing, + } + + await writeReport(report) + + if (writeDebugArtifact) { + await writeResolvedArtifact(resolved) + } + + printSummary(report) +} + +// --------------------------------------------------------------------------- +// Price resolution +// --------------------------------------------------------------------------- + +async function resolveCardPrices({ + card, + cardId, + cardFile, + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + providers, + missing, +}: { + card: TCGDexCard + cardId: string + cardFile: string + cardName?: string + setId?: string + setName?: string + categoryId: number + tcgplayerSetId: number + providers: PricingProvider[] + missing: MissingPrice[] +}): Promise { + const entries: ResolvedCardPriceEntry[] = [] + + if (Array.isArray(card.variants)) { + for (const [variantIndex, variant] of card.variants.entries()) { + const tcgplayerProductId = variant.thirdParty?.tcgplayer + + if (typeof tcgplayerProductId !== 'number') { + missing.push({ + cardId, + cardFile: rel(cardFile), + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + target: 'variant', + variantIndex, + reason: 'missing-variant-tcgplayer-id', + }) + continue + } + + const prices = await getPricesFromProviders(providers, { + categoryId, + tcgplayerSetId, + tcgplayerProductId, + }) + + if (prices.length === 0) { + missing.push({ + cardId, + cardFile: rel(cardFile), + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + tcgplayerProductId, + target: 'variant', + variantIndex, + reason: 'no-pricing-data', + }) + continue + } + + for (const price of prices) { + entries.push({ + target: 'variant', + variantIndex, + variant: { + type: variant.type, + subtype: variant.subtype, + size: variant.size, + stamp: variant.stamp, + foil: variant.foil, + }, + tcgplayerProductId, + source: price.source, + priceType: price.priceType, + low: price.low, + market: price.market, + }) + } + } + + return entries + } + + const tcgplayerProductId = card.thirdParty?.tcgplayer + + if (typeof tcgplayerProductId !== 'number') { + missing.push({ + cardId, + cardFile: rel(cardFile), + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + target: 'card', + reason: 'missing-card-tcgplayer-id', + }) + return entries + } + + const prices = await getPricesFromProviders(providers, { + categoryId, + tcgplayerSetId, + tcgplayerProductId, + }) + + if (prices.length === 0) { + missing.push({ + cardId, + cardFile: rel(cardFile), + cardName, + setId, + setName, + categoryId, + tcgplayerSetId, + tcgplayerProductId, + target: 'card', + reason: 'no-pricing-data', + }) + return entries + } + + for (const price of prices) { + entries.push({ + target: 'card', + tcgplayerProductId, + source: price.source, + priceType: price.priceType, + low: price.low, + market: price.market, + }) + } + + return entries +} + +async function getPricesFromProviders( + providers: PricingProvider[], + input: { + categoryId: number + tcgplayerSetId: number + tcgplayerProductId: number + }, +): Promise { + for (const provider of providers) { + const prices = await provider.getPrices(input) + + if (prices.length > 0) { + return prices + } + } + + return [] +} + +// --------------------------------------------------------------------------- +// File scanning +// --------------------------------------------------------------------------- + +async function findCardFiles(rootFolders: string[]): Promise { + const files: string[] = [] + + for (const rootFolder of rootFolders) { + const exists = await pathExists(rootFolder) + + if (!exists) { + continue + } + + await walk(rootFolder, files) + } + + return files.filter((file) => { + if (!file.endsWith('.ts')) { + return false + } + + const relative = path.relative(process.cwd(), file).replace(/\\/g, '/') + const parts = relative.split('/') + + // data/Serie/Set/Card.ts or data-asia/Serie/Set/Card.ts + return ( + (parts[0] === 'data' || parts[0] === 'data-asia') && parts.length === 4 + ) + }) +} + +async function walk(folder: string, files: string[]): Promise { + const entries = await fs.readdir(folder, { withFileTypes: true }) + + for (const entry of entries) { + const fullPath = path.join(folder, entry.name) + + if (entry.isDirectory()) { + await walk(fullPath, files) + } else { + files.push(fullPath) + } + } +} + +// --------------------------------------------------------------------------- +// Report writing +// --------------------------------------------------------------------------- + +async function writeReport(report: PricingReport): Promise { + const reportFile = path.join( + process.cwd(), + 'var', + 'reports', + 'pricing-update.json', + ) + + await fs.mkdir(path.dirname(reportFile), { recursive: true }) + await fs.writeFile(reportFile, `${JSON.stringify(report, null, '\t')}\n`, 'utf8') +} + +async function writeResolvedArtifact(resolved: ResolvedCardPrice[]): Promise { + const outputFile = path.join( + process.cwd(), + 'var', + 'models', + 'tcgtracking', + 'resolved-pricing.json', + ) + + await fs.mkdir(path.dirname(outputFile), { recursive: true }) + await fs.writeFile( + outputFile, + `${JSON.stringify( + { + source: 'tcgtracking', + createdAt: new Date().toISOString(), + cards: resolved, + }, + null, + '\t', + )}\n`, + 'utf8', + ) +} + +// --------------------------------------------------------------------------- +// Grouping helpers +// --------------------------------------------------------------------------- + +function groupBySource( + resolved: ResolvedCardPrice[], +): Array<{ source: PricingSourceName; count: number }> { + const counts = new Map() + + for (const card of resolved) { + for (const price of card.prices) { + counts.set(price.source, (counts.get(price.source) ?? 0) + 1) + } + } + + return Array.from(counts.entries()) + .map(([source, count]) => ({ source, count })) + .sort((a, b) => b.count - a.count) +} + +function groupByTarget( + resolved: ResolvedCardPrice[], +): Array<{ target: 'card' | 'variant'; count: number }> { + const counts = new Map<'card' | 'variant', number>() + + for (const card of resolved) { + for (const price of card.prices) { + counts.set(price.target, (counts.get(price.target) ?? 0) + 1) + } + } + + return Array.from(counts.entries()) + .map(([target, count]) => ({ target, count })) + .sort((a, b) => b.count - a.count) +} + +function groupMissingByReason( + missing: MissingPrice[], +): Array<{ reason: MissingPrice['reason']; count: number }> { + const counts = new Map() + + for (const item of missing) { + counts.set(item.reason, (counts.get(item.reason) ?? 0) + 1) + } + + return Array.from(counts.entries()) + .map(([reason, count]) => ({ reason, count })) + .sort((a, b) => b.count - a.count) +} + +// --------------------------------------------------------------------------- +// Summary output +// --------------------------------------------------------------------------- + +function printSummary(report: PricingReport): void { + console.log('') + console.log('Pricing update complete.') + console.log(`Providers: ${report.providers.join(', ')}`) + console.log(`Card files scanned: ${report.summary.cardFilesScanned}`) + console.log(`Cards with prices: ${report.summary.cardFilesWithPrices}`) + console.log(`Price entries resolved: ${report.summary.priceEntriesResolved}`) + console.log(`Missing records: ${report.summary.missing}`) + console.log(`Debug artifact: ${report.summary.debugArtifactWritten ? 'written' : 'skipped'}`) + + if (report.resolvedBySource.length > 0) { + console.log('') + console.log('Resolved by source:') + + for (const item of report.resolvedBySource) { + console.log(` ${item.source}: ${item.count}`) + } + } + + if (report.resolvedByTarget.length > 0) { + console.log('') + console.log('Resolved by target:') + + for (const item of report.resolvedByTarget) { + console.log(` ${item.target}: ${item.count}`) + } + } + + if (report.missingByReason.length > 0) { + console.log('') + console.log('Missing by reason:') + + for (const item of report.missingByReason) { + console.log(` ${item.reason}: ${item.count}`) + } + } + + console.log('') + console.log('Report: var/reports/pricing-update.json') + + if (report.summary.debugArtifactWritten) { + console.log('Debug artifact: var/models/tcgtracking/resolved-pricing.json') + } +} + +// --------------------------------------------------------------------------- +// Utilities +// --------------------------------------------------------------------------- + +async function pathExists(filePath: string): Promise { + try { + await fs.access(filePath) + return true + } catch { + return false + } +} + +function getCardIdFromFile(filePath: string): string { + const filename = path.basename(filePath) + return filename.slice(0, filename.lastIndexOf('.')) +} + +function getEnglishName(value?: Record): string | undefined { + return value?.en ?? Object.values(value ?? {})[0] +} + +function rel(filePath: string): string { + return path.relative(process.cwd(), filePath).replace(/\\/g, '/') +} + +main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/scripts/utils-data/tcgtracking.ts b/scripts/utils-data/tcgtracking.ts new file mode 100644 index 0000000000..bc7e1e63a7 --- /dev/null +++ b/scripts/utils-data/tcgtracking.ts @@ -0,0 +1,80 @@ +import path from 'node:path' +import { pathToFileURL } from 'node:url' + +/** + * TCGTracking Open TCG API base URL. + */ +export const TCGTRACKING_API_BASE_URL = 'https://tcgtracking.com/tcgapi/v1' + +/** + * TCGTracking category IDs. + * + * 3 = Pokémon + * 85 = Pokémon Japan + * + * These are used only to fetch pricing data. + * They do not decide card variants, finishes, or canonical card data. + */ +export const TCGTRACKING_CATEGORIES = { + pokemon: 3, + pokemonJapan: 85, +} as const + +export type TCGTrackingCategoryId = + (typeof TCGTRACKING_CATEGORIES)[keyof typeof TCGTRACKING_CATEGORIES] + +export type TCGTrackingRawPricingResponse = unknown + +export interface TCGTrackingProductPricing { + tcg?: Record +} + +export interface TCGTrackingVariantPrice { + low?: number + market?: number +} + +export async function fetchTCGTrackingSetPricing( + categoryId: TCGTrackingCategoryId, + tcgTrackingSetId: number, +): Promise { + const url = `${TCGTRACKING_API_BASE_URL}/${categoryId}/sets/${tcgTrackingSetId}/pricing` + + const response = await fetch(url, { + headers: { + Accept: 'application/json', + }, + }) + + if (!response.ok) { + throw new Error( + `TCGTracking pricing fetch failed for category ${categoryId}, set ${tcgTrackingSetId}: ${response.status} ${response.statusText}`, + ) + } + + return response.json() +} + +/** + * Converts a filesystem path into a valid file URL for dynamic import. + * + * A literal ? inside an import URL is treated as the start of a query string + * unless it is encoded. pathToFileURL handles that correctly. + */ +export function toImportUrl(filePath: string): string { + return pathToFileURL(path.resolve(filePath)).href +} + +/** + * Decides which TCGTracking category to use based on the file location. + * Only used for selecting the pricing source category, not for card data. + */ +export function getCategoryIdForSetFile(filePath: string): TCGTrackingCategoryId { + const normalized = filePath.replace(/\\/g, '/') + + if (normalized.includes('/data-asia/')) { + return TCGTRACKING_CATEGORIES.pokemonJapan + } + + return TCGTRACKING_CATEGORIES.pokemon +} diff --git a/server/src/libs/providers/tcgplayer/index.ts b/server/src/libs/providers/tcgplayer/index.ts index 0ba969bc97..504ce35d31 100644 --- a/server/src/libs/providers/tcgplayer/index.ts +++ b/server/src/libs/providers/tcgplayer/index.ts @@ -1,8 +1,12 @@ import * as OfficialTCGPlayer from './official' import * as Fallback from './fallback' +import * as TCGTracking from './tcgtracking' import type RFC7807 from '../../RFCs/RFC7807' -let source: (typeof OfficialTCGPlayer) | (typeof Fallback) = Fallback +type Source = typeof OfficialTCGPlayer | typeof Fallback | typeof TCGTracking + +let source: Source + if ( process.env.TCGPLAYER_CLIENT_ID && process.env.TCGPLAYER_CLIENT_SECRET @@ -10,8 +14,14 @@ if ( ) { console.log('loading official TCGPlayer backend') source = OfficialTCGPlayer -} else { +} else if (process.env.PRICING_SOURCE === 'tcgcsv') { console.log('loading fallback TCGPlayer backend') + source = Fallback +} else { + // TCGTracking is the default — reads from var/models/tcgtracking/resolved-pricing.json + // Set PRICING_SOURCE=tcgcsv to revert to the old live TCGCSV fetch. + console.log('loading TCGTracking backend') + source = TCGTracking } diff --git a/server/src/libs/providers/tcgplayer/tcgtracking.ts b/server/src/libs/providers/tcgplayer/tcgtracking.ts new file mode 100644 index 0000000000..9d070e56df --- /dev/null +++ b/server/src/libs/providers/tcgplayer/tcgtracking.ts @@ -0,0 +1,133 @@ +import { sets } from '../../../V2/Components/Set' + +const TCGTRACKING_API_BASE_URL = 'https://tcgtracking.com/tcgapi/v1' + +/** + * TCGTracking sometimes exposes a set under a different ID than the + * existing TCGDex / TCGPlayer set ID. + */ +const TCGTRACKING_SET_ID_OVERRIDES: Record = { + // Black Bolt: TCGDex/TCGPlayer ID 22325, TCGTracking ID 24325 + 22325: 24325, +} + +interface SetPricingResponse { + set_id: number + updated: string + prices: Record + }> +} + +export interface PriceResult { + productId: number + lowPrice: number + midPrice: number + highPrice: number + marketPrice?: number + directLowPrice?: number +} + +// productId → normalised-price-type → PriceResult +let cache: Record> = {} +let lastFetch: Date | undefined = undefined + +export async function updateTCGPlayerDatas(): Promise { + // Refresh at most once per hour + if (lastFetch && Date.now() - lastFetch.getTime() < 3600000) { + return false + } + + const setIds = sets.en + .filter((it) => it?.thirdParty?.tcgplayer) + .map((it) => it!.thirdParty!.tcgplayer!) + + const newCache: Record> = {} + let loaded = 0 + let failed = 0 + + for (const setId of setIds) { + const fetchSetId = TCGTRACKING_SET_ID_OVERRIDES[setId] ?? setId + + try { + const res = await fetch( + `${TCGTRACKING_API_BASE_URL}/3/sets/${fetchSetId}/pricing`, + { headers: { Accept: 'application/json' } }, + ) + + if (!res.ok) { + failed++ + console.warn( + `TCGTracking: set ${fetchSetId} returned ${res.status} ${res.statusText}`, + ) + continue + } + + const data = (await res.json()) as SetPricingResponse + + for (const [productIdStr, productPricing] of Object.entries(data.prices ?? {})) { + const productId = Number(productIdStr) + + if (!productPricing.tcg) { + continue + } + + const productCache: Record = {} + + for (const [priceType, price] of Object.entries(productPricing.tcg)) { + const key = priceType.toLowerCase().replaceAll(' ', '-') + + if (typeof price.low === 'number' || typeof price.market === 'number') { + productCache[key] = { + productId, + lowPrice: price.low ?? 0, + midPrice: 0, + highPrice: 0, + marketPrice: price.market, + } + } + } + + if (Object.keys(productCache).length > 0) { + newCache[productId] = productCache + } + } + + loaded++ + } catch (error) { + failed++ + console.warn( + `TCGTracking: failed to load set ${fetchSetId}: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + + cache = newCache + lastFetch = new Date() + + console.log(`TCGTracking: loaded ${loaded} sets, ${failed} failed`) + + return true +} + +export async function getTCGPlayerPrice(card: { thirdParty: { tcgplayer?: number } }): Promise<{ + unit: 'USD' + updated: string + [key: string]: unknown +} | null> { + if (!lastFetch || typeof card.thirdParty?.tcgplayer !== 'number') { + return null + } + + const variants = cache[card.thirdParty.tcgplayer] + + if (!variants) { + return null + } + + return { + updated: lastFetch.toISOString(), + unit: 'USD', + ...variants, + } +}