diff --git a/server/compiler/index.ts b/server/compiler/index.ts index 582d190542..d99854cd1f 100644 --- a/server/compiler/index.ts +++ b/server/compiler/index.ts @@ -12,6 +12,10 @@ const LANGS: Array = [ const DIST_FOLDER = './generated' ;(async () => { + + const startDate = new Date() + console.log('Starting compilation at', startDate.toLocaleTimeString()) + const paths = (await fs.readdir('./compiler/endpoints')).filter((p) => p.endsWith('.ts')) // Prefetch the pictures at the start as it can bug because of bad connection @@ -71,4 +75,7 @@ const DIST_FOLDER = './generated' await fs.copyFile('../meta/definitions/' + file, './public/v2/' + file) } + const endDate = new Date() + console.log('\nFinished at', endDate.toLocaleTimeString(), 'after', ((endDate.getTime() - startDate.getTime()) / 1000).toFixed(2), 'seconds') + })() diff --git a/server/compiler/utils/cardUtil.ts b/server/compiler/utils/cardUtil.ts index 50a8e942ba..1c5e84d83f 100644 --- a/server/compiler/utils/cardUtil.ts +++ b/server/compiler/utils/cardUtil.ts @@ -186,6 +186,17 @@ export async function getCard(set: Set, id: string, lang: SupportedLanguages): P } } +/** + * Get the number of cards available in the set for a specific language + * @param lang the language of the cards + * @param set the set to filter in (optional) + * @returns the number of cards available in the set + */ +export async function getCardsLength(lang: SupportedLanguages, set?: Set): Promise { + let cards = await smartGlob(`${DB_PATH}/${getDataFolder(lang)}/${(set && (set.serie.name.en ?? set.serie.name[lang])) ?? '*'}/${(set && (set.name.en ?? set.name[lang])) ?? '*'}/*.ts`) + return cards.length +} + /** * Get cards filtered by the language they are available in * @param lang the language of the cards diff --git a/server/compiler/utils/setUtil.ts b/server/compiler/utils/setUtil.ts index 81856c8dc8..3427491769 100644 --- a/server/compiler/utils/setUtil.ts +++ b/server/compiler/utils/setUtil.ts @@ -1,7 +1,7 @@ import { objectKeys, objectMap } from '@dzeio/object-util' import { Card, Set, SupportedLanguages } from '../../../interfaces' import { SetResume, Set as SetSingle } from '../../../meta/definitions/api' -import { cardToCardSimple, getCards } from './cardUtil' +import { cardToCardSimple, getCards, getCardsLength } from './cardUtil' import { DB_PATH, fetchRemoteFile, getDataFolder, resolveText, setIsLegal, smartGlob } from './util' import path from 'node:path' @@ -11,6 +11,7 @@ interface t { } const setCache: t = {} +const setSimpleCache: Record = {} export function isSetAvailable(set: Set, lang: SupportedLanguages): boolean { return !!resolveText(set.name, lang) && !!resolveText(set.serie.name, lang) @@ -66,18 +67,21 @@ export async function getSetPictures(set: Set, lang: SupportedLanguages): Promis } export async function setToSetSimple(set: Set, lang: SupportedLanguages): Promise { - const cards = await getCards(lang, set) + if (setSimpleCache[set.id + lang]) return setSimpleCache[set.id + lang] + // const cards = await getCards(lang, set) const pics = await getSetPictures(set, lang) - return { + const res = { cardCount: { official: set.cardCount.official, - total: Math.max(set.cardCount.official, cards.length) + total: Math.max(set.cardCount.official, await getCardsLength(lang, set)) }, id: set.id, logo: pics[0], name: resolveText(set.name, lang), symbol: pics[1] } + setSimpleCache[set.id + lang] = res + return res } function getVariantCountForType(card: Card, type: 'normal' | 'reverse' | 'holo' | 'firstEdition'): number { diff --git a/server/compiler/utils/util.ts b/server/compiler/utils/util.ts index a3279e95b7..a18be37182 100644 --- a/server/compiler/utils/util.ts +++ b/server/compiler/utils/util.ts @@ -1,9 +1,9 @@ import { objectSize } from '@dzeio/object-util' -import Queue from '@dzeio/queue' import { glob } from 'glob' import { exec, spawn } from 'node:child_process' +import { createInterface } from 'node:readline' import { writeFileSync } from 'node:fs' -import { Card, Languages, Set, SupportedLanguages } from '../../../interfaces' +import type { Card, Languages, Set as CardSet, SupportedLanguages } from '../../../interfaces' import * as legals from '../../../meta/legals' interface fileCacheInterface { [key: string]: any @@ -75,7 +75,7 @@ export function cardIsLegal(type: 'standard' | 'expanded', card: Card, localId: * @param set the set to check * @returns {boolean} if the set is currently in the legal type */ -export function setIsLegal(type: 'standard' | 'expanded', set: Set): boolean { +export function setIsLegal(type: 'standard' | 'expanded', set: CardSet): boolean { const legal = legals[type] if ( legal.includes.series.includes(set.serie.id) || @@ -132,43 +132,62 @@ function runCommand(command: string, useSpawn = true): Promise { const lastEditsCache: Record = {} export async function loadLastEdits() { console.log('Loading Git File Tree...') - const firstCommand = 'git ls-tree -r --name-only HEAD ../data' - const files = (await runCommand(firstCommand)).split('\n') - const secondCommand = 'git ls-tree -r --name-only HEAD ../data-asia' - files.push(...(await runCommand(secondCommand)).split('\n')) - console.log('Loaded files tree', files.length, 'files') + const fileCommand = 'git -c core.quotepath=false ls-tree -r --name-only HEAD ../data ../data-asia' + const files = (await runCommand(fileCommand)) + .split('\n') + .map(f => f.trim()) + .filter(f => f.length > 0) + .map(f => f.replace(/^\.\.\//, '')) + + const remainingFiles = new Set() + for (const file of files) { + remainingFiles.add(file) + } + + console.log('Loaded files tree', remainingFiles.size, 'files') console.log('Loading their last edit time') - let processed = 0 - const concurrent = process.platform === 'win32' ? 10 : 1000 - const queue = new Queue(concurrent, 10) - queue.start() - - for await (let file of files) { - file = file.replace(/"/g, '').replace("\\303\\251", "é") - await queue.add(runCommand(`git log -1 --pretty="format:%cd" --date=iso-strict "${file}"`, false).then((res) => { - lastEditsCache[file] = res - }) - .catch(() => { - console.warn('could not load file', file, 'hope it does not break everything else lol') - }) - .finally(() => { - processed++ - if (processed % 1000 === 0) { - console.log('loaded', processed, 'out of', files.length, 'files', `(${(processed / files.length * 100).toFixed(0)}%)`) + + const logProcess = spawn('git', [ + '-c', 'core.quotepath=false', + 'log', + '--name-only', + '--format=COMMIT_DATE:%cI', + '../data', + '../data-asia' + ]) + + const rl = createInterface({ + input: logProcess.stdout, + crlfDelay: Infinity + }) + + let currentDate = new Date().toISOString() + let loadedCount = 0 + const totalFiles = remainingFiles.size + + for await (const line of rl) { + if (line.startsWith('COMMIT_DATE:')) { + currentDate = line.substring('COMMIT_DATE:'.length).trim() + } else if (line.trim().length > 0) { + const file = line.trim() + if (remainingFiles.has(file)) { + lastEditsCache["../" + file] = currentDate + remainingFiles.delete(file) + loadedCount++ + + if (loadedCount % 1000 === 0) { + console.log('loaded', loadedCount, 'out of', totalFiles, 'files', `(${(loadedCount / totalFiles * 100).toFixed(0)}%)`) + } + + if (remainingFiles.size === 0) { + logProcess.kill() + break + } } - })) - // try { - // // don't really know why but it does not correctly execute the command when using Spawn - // lastEditsCache[file] = await runCommand(`git log -1 --pretty="format:%cd" --date=iso-strict "${file}"`, false) - // } catch { - // console.warn('could not load file', file, 'hope it does not break everything else lol') - // } - // processed++ - // if (processed % 1000 === 0) { - // console.log('loaded', processed, 'out of', files.length, 'files', `(${(processed / files.length * 100).toFixed(0)}%)`) - // } + } } - await queue.waitEnd() + logProcess.kill() + console.log('done loading files', objectSize(lastEditsCache)) }