feat: Improve compile time

server/compiler/index.ts

@@ -12,6 +12,10 @@ const LANGS: Array<SupportedLanguages> = [
 const DIST_FOLDER = './generated'
 
 ;(async () => {
+
+	const startDate = new Date()
+	console.log('Starting compilation at', startDate.toLocaleTimeString())
+
 	const paths = (await fs.readdir('./compiler/endpoints')).filter((p) => p.endsWith('.ts'))
 
 	// Prefetch the pictures at the start as it can bug because of bad connection
@@ -71,4 +75,7 @@ const DIST_FOLDER = './generated'
 		await fs.copyFile('../meta/definitions/' + file, './public/v2/' + file)
 	}
 
+	const endDate = new Date()
+	console.log('\nFinished at', endDate.toLocaleTimeString(), 'after', ((endDate.getTime() - startDate.getTime()) / 1000).toFixed(2), 'seconds')
+
 })()

server/compiler/utils/cardUtil.ts

@@ -186,6 +186,17 @@ export async function getCard(set: Set, id: string, lang: SupportedLanguages): P
 	}
 }
 
+/**
+ * Get the number of cards available in the set for a specific language
+ * @param lang the language of the cards
+ * @param set the set to filter in (optional)
+ * @returns the number of cards available in the set
+ */
+export async function getCardsLength(lang: SupportedLanguages, set?: Set): Promise<number> {
+	let cards = await smartGlob(`${DB_PATH}/${getDataFolder(lang)}/${(set && (set.serie.name.en ?? set.serie.name[lang])) ?? '*'}/${(set && (set.name.en ?? set.name[lang])) ?? '*'}/*.ts`)
+	return cards.length
+}
+
 /**
  * Get cards filtered by the language they are available in
  * @param lang the language of the cards

server/compiler/utils/setUtil.ts

@@ -1,7 +1,7 @@
 import { objectKeys, objectMap } from '@dzeio/object-util'
 import { Card, Set, SupportedLanguages } from '../../../interfaces'
 import { SetResume, Set as SetSingle } from '../../../meta/definitions/api'
-import { cardToCardSimple, getCards } from './cardUtil'
+import { cardToCardSimple, getCards, getCardsLength } from './cardUtil'
 import { DB_PATH, fetchRemoteFile, getDataFolder, resolveText, setIsLegal, smartGlob } from './util'
 import path from 'node:path'
 
@@ -11,6 +11,7 @@ interface t {
 }
 
 const setCache: t = {}
+const setSimpleCache: Record<string, SetResume> = {}
 
 export function isSetAvailable(set: Set, lang: SupportedLanguages): boolean {
 	return !!resolveText(set.name, lang) && !!resolveText(set.serie.name, lang)
@@ -66,18 +67,21 @@ export async function getSetPictures(set: Set, lang: SupportedLanguages): Promis
 }
 
 export async function setToSetSimple(set: Set, lang: SupportedLanguages): Promise<SetResume> {
-	const cards = await getCards(lang, set)
+	if (setSimpleCache[set.id + lang]) return setSimpleCache[set.id + lang]
+	// const cards = await getCards(lang, set)
 	const pics = await getSetPictures(set, lang)
-	return {
+	const res = {
 		cardCount: {
 			official: set.cardCount.official,
-			total: Math.max(set.cardCount.official, cards.length)
+			total: Math.max(set.cardCount.official, await getCardsLength(lang, set))
 		},
 		id: set.id,
 		logo: pics[0],
 		name: resolveText(set.name, lang),
 		symbol: pics[1]
 	}
+	setSimpleCache[set.id + lang] = res
+	return res
 }
 
 function getVariantCountForType(card: Card, type: 'normal' | 'reverse' | 'holo' | 'firstEdition'): number {

server/compiler/utils/util.ts

@@ -1,9 +1,9 @@
 import { objectSize } from '@dzeio/object-util'
-import Queue from '@dzeio/queue'
 import { glob } from 'glob'
 import { exec, spawn } from 'node:child_process'
+import { createInterface } from 'node:readline'
 import { writeFileSync } from 'node:fs'
-import { Card, Languages, Set, SupportedLanguages } from '../../../interfaces'
+import type { Card, Languages, Set as CardSet, SupportedLanguages } from '../../../interfaces'
 import * as legals from '../../../meta/legals'
 interface fileCacheInterface {
 	[key: string]: any
@@ -75,7 +75,7 @@ export function cardIsLegal(type: 'standard' | 'expanded', card: Card, localId:
  * @param set the set to check
  * @returns {boolean} if the set is currently in the legal type
  */
-export function setIsLegal(type: 'standard' | 'expanded', set: Set): boolean {
+export function setIsLegal(type: 'standard' | 'expanded', set: CardSet): boolean {
 	const legal = legals[type]
 	if (
 		legal.includes.series.includes(set.serie.id) ||
@@ -132,43 +132,62 @@ function runCommand(command: string, useSpawn = true): Promise<string> {
 const lastEditsCache: Record<string, string> = {}
 export async function loadLastEdits() {
 	console.log('Loading Git File Tree...')
-	const firstCommand = 'git ls-tree -r --name-only HEAD ../data'
-	const files = (await runCommand(firstCommand)).split('\n')
-	const secondCommand = 'git ls-tree -r --name-only HEAD ../data-asia'
-	files.push(...(await runCommand(secondCommand)).split('\n'))
-	console.log('Loaded files tree', files.length, 'files')
+	const fileCommand = 'git -c core.quotepath=false ls-tree -r --name-only HEAD ../data ../data-asia'
+	const files = (await runCommand(fileCommand))
+		.split('\n')
+		.map(f => f.trim())
+		.filter(f => f.length > 0)
+		.map(f => f.replace(/^\.\.\//, ''))
+
+	const remainingFiles = new Set<string>()
+	for (const file of files) {
+		remainingFiles.add(file)
+	}
+
+	console.log('Loaded files tree', remainingFiles.size, 'files')
 	console.log('Loading their last edit time')
-	let processed = 0
-	const concurrent = process.platform === 'win32' ? 10 : 1000
-	const queue = new Queue(concurrent, 10)
-	queue.start()
-
-	for await (let file of files) {
-		file = file.replace(/"/g, '').replace("\\303\\251", "é")
-		await queue.add(runCommand(`git log -1 --pretty="format:%cd" --date=iso-strict "${file}"`, false).then((res) => {
-			lastEditsCache[file] = res
-		})
-		.catch(() => {
-			console.warn('could not load file', file, 'hope it does not break everything else lol')
-		})
-		.finally(() => {
-			processed++
-			if (processed % 1000 === 0) {
-				console.log('loaded', processed, 'out of', files.length, 'files', `(${(processed / files.length * 100).toFixed(0)}%)`)
+
+	const logProcess = spawn('git', [
+		'-c', 'core.quotepath=false',
+		'log',
+		'--name-only',
+		'--format=COMMIT_DATE:%cI',
+		'../data',
+		'../data-asia'
+	])
+
+	const rl = createInterface({
+		input: logProcess.stdout,
+		crlfDelay: Infinity
+	})
+
+	let currentDate = new Date().toISOString()
+	let loadedCount = 0
+	const totalFiles = remainingFiles.size
+
+	for await (const line of rl) {
+		if (line.startsWith('COMMIT_DATE:')) {
+			currentDate = line.substring('COMMIT_DATE:'.length).trim()
+		} else if (line.trim().length > 0) {
+			const file = line.trim()
+			if (remainingFiles.has(file)) {
+				lastEditsCache["../" + file] = currentDate
+				remainingFiles.delete(file)
+				loadedCount++
+
+				if (loadedCount % 1000 === 0) {
+					console.log('loaded', loadedCount, 'out of', totalFiles, 'files', `(${(loadedCount / totalFiles * 100).toFixed(0)}%)`)
+				}
+
+				if (remainingFiles.size === 0) {
+					logProcess.kill()
+					break
+				}
 			}
-		}))
-		// try {
-		// 	// don't really know why but it does not correctly execute the command when using Spawn
-		// 	lastEditsCache[file] = await runCommand(`git log -1 --pretty="format:%cd" --date=iso-strict "${file}"`, false)
-		// } catch {
-		// 	console.warn('could not load file', file, 'hope it does not break everything else lol')
-		// }
-		// processed++
-		// if (processed % 1000 === 0) {
-		// 	console.log('loaded', processed, 'out of', files.length, 'files', `(${(processed / files.length * 100).toFixed(0)}%)`)
-		// }
+		}
 	}
-	await queue.waitEnd()
+	logProcess.kill()
+
 	console.log('done loading files', objectSize(lastEditsCache))
 }