diff --git a/.dockerignore b/.dockerignore index e50d5df3d5..c7a104c625 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,7 +3,6 @@ # dependencies node_modules - # misc .DS_Store .env* @@ -16,6 +15,16 @@ dist/ /data/**/*.js - # production /server/dist + +# CI/CD and development files +.github/ +.cursor/ +.bruno/ +*.md +!README.md +*.test.ts +.git/ +.gitignore +.dockerignore diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 73666680fc..c29b6ec8da 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,6 +8,10 @@ on: branches: - "**" +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + env: REGISTRY_IMAGE: | ghcr.io/tcgdex/server @@ -17,6 +21,7 @@ jobs: build: runs-on: ubuntu-latest name: Build TCGdex Server + timeout-minutes: 60 steps: - name: Checkout @@ -65,6 +70,18 @@ jobs: with: bun-version: latest + - name: Cache Bun dependencies + uses: actions/cache@v4 + with: + path: | + ~/.bun/install/cache + ~/.bun/install/global + node_modules + server/node_modules + key: ${{ runner.os }}-bun-build-${{ hashFiles('bun.lockb', 'server/bun.lockb') }} + restore-keys: | + ${{ runner.os }}-bun-build- + - name: Pre build server run: | bun install --frozen-lockfile @@ -84,6 +101,6 @@ jobs: file: ./Dockerfile.github-actions tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha + cache-from: type=gha,scope=${{ github.ref_name }} push: ${{ !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }} - cache-to: type=gha,mode=max + cache-to: type=gha,mode=max,scope=${{ github.ref_name }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8664739ed2..324d3318de 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,66 +2,167 @@ name: Test the Data on: push: - branches: + branches: - master - pull_request_target: - branches: - - "**" + pull_request: + branches: + - '**' permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: + export-git-metadata: + runs-on: ubuntu-latest + permissions: {} + timeout-minutes: 30 + defaults: + run: + shell: bash + + steps: + - name: Checkout full history + uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + fetch-depth: 0 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + + - name: Cache Bun installs + uses: actions/cache@v4 + with: + path: | + ~/.bun/install/cache + ~/.bun/install/global + key: ${{ runner.os }}-bun-metadata-${{ hashFiles('bun.lockb', 'server/bun.lockb') }} + restore-keys: | + ${{ runner.os }}-bun-metadata- + + - name: Install server deps + run: | + cd server + bun install --frozen-lockfile + + - name: Export git metadata + run: | + cd server + bun run compile --export-git-metadata + + - name: Upload git metadata artifact + uses: actions/upload-artifact@v4 + with: + name: git-metadata + path: server/git-metadata.json + retention-days: 1 + if-no-files-found: error + compression-level: 9 + test: + needs: export-git-metadata runs-on: ${{ matrix.os }} permissions: {} + timeout-minutes: 120 + defaults: + run: + shell: bash strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] + fail-fast: false steps: - - name: Checkout - uses: actions/checkout@v6 - with: - ref: ${{ github.event.pull_request.head.sha }} - persist-credentials: false - - - name: Setup BunJS - uses: oven-sh/setup-bun@v2 - - - name: Install deps - run: | - bun install -g @usebruno/cli - bun install --frozen-lockfile - cd server - bun install --frozen-lockfile - bun run compile - - - name: Validate the data & the server - run: | - bun run validate - cd server - bun run --bun validate - - - name: Validate some requests - shell: bash - run: | - set -euo pipefail - cd server - bun run start & - SERVER_PID=$! - cd .. - - ATTEMPTS=0 - until curl -sSf http://127.0.0.1:3000/status > /dev/null; do - ATTEMPTS=$((ATTEMPTS + 1)) - if [ $ATTEMPTS -ge 60 ]; then - echo "Server did not become ready within 60 seconds" >&2 - kill $SERVER_PID + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + fetch-depth: 1 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + + - name: Cache Bun installs + uses: actions/cache@v4 + with: + path: | + ~/.bun/install/cache + ~/.bun/install/global + key: ${{ matrix.os }}-bun-${{ hashFiles('bun.lockb', 'server/bun.lockb') }} + restore-keys: | + ${{ matrix.os }}-bun- + + - name: Download git metadata + uses: actions/download-artifact@v4 + with: + name: git-metadata + path: server/ + + - name: Install CLI & root deps + run: | + bun install -g @usebruno/cli + bun install --frozen-lockfile + + - name: Install server deps + run: | + cd server + bun install --frozen-lockfile + + - name: Compile with imported metadata + run: | + cd server + set -o pipefail + bun run compile --import-git-metadata 2>&1 | tee ../compile.log + cd .. + if grep -q "could not load file" compile.log; then + echo "::error::Compiler reported missing card files" >&2 + cat compile.log exit 1 fi - sleep 1 - done + rm -f compile.log - cd .bruno - bru run --env Developpement + - name: Run TypeScript validation (root) + run: bun run validate - kill $SERVER_PID + - name: Run TypeScript validation (server) + run: | + cd server + bun run --bun validate + + - name: Start API server + run: | + cd server + MAX_WORKERS=1 bun run start > ../server.log 2>&1 & + echo $! > ../server.pid + + - name: Wait for server readiness + run: | + for i in {1..180}; do + if curl -sf http://127.0.0.1:3000/status > /dev/null; then + exit 0 + fi + sleep 2 + done + echo "Server failed to start" >&2 + cat server.log || true + exit 1 + + - name: Run Bruno integration suite + run: | + cd .bruno + bru run --env Developpement + + - name: Stop API server + if: always() + run: | + if [ -f server.pid ]; then + kill "$(cat server.pid)" || true + rm -f server.pid + fi + cat server.log || true + rm -f server.log diff --git a/.gitignore b/.gitignore index 773131be55..0156a1f405 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ scripts/pokedexIdFixer/pokemon-mapping.json scripts/pokedexIdFixer/fix-preview.txt scripts/pokedexIdFixer/fix-log.txt scripts/pokedexIdFixer/audit-report.txt + +# Git metadata cache (generated by compiler) +server/git-metadata.json diff --git a/server/compiler/index.ts b/server/compiler/index.ts index 582d190542..7edfad8988 100644 --- a/server/compiler/index.ts +++ b/server/compiler/index.ts @@ -26,6 +26,11 @@ const DIST_FOLDER = './generated' console.log('\n2. Loading informations from GIT') await loadLastEdits() + if (process.argv.includes('--export-git-metadata')) { + console.log('\nGit metadata export complete.') + process.exit(0) + } + console.log('\n3. Compiling Files') // Process each languages diff --git a/server/compiler/utils/util.ts b/server/compiler/utils/util.ts index a3279e95b7..d9a0897950 100644 --- a/server/compiler/utils/util.ts +++ b/server/compiler/utils/util.ts @@ -2,7 +2,7 @@ import { objectSize } from '@dzeio/object-util' import Queue from '@dzeio/queue' import { glob } from 'glob' import { exec, spawn } from 'node:child_process' -import { writeFileSync } from 'node:fs' +import { readFileSync, statSync, writeFileSync } from 'node:fs' import { Card, Languages, Set, SupportedLanguages } from '../../../interfaces' import * as legals from '../../../meta/legals' interface fileCacheInterface { @@ -11,6 +11,10 @@ interface fileCacheInterface { export const DB_PATH = "../" +const EXPORT_METADATA = process.argv.includes('--export-git-metadata') +const IMPORT_METADATA = process.argv.includes('--import-git-metadata') +const METADATA_FILE = './git-metadata.json' + const fileCache: fileCacheInterface = {} /** @@ -131,6 +135,22 @@ function runCommand(command: string, useSpawn = true): Promise { const lastEditsCache: Record = {} export async function loadLastEdits() { + if (IMPORT_METADATA) { + console.log('Importing git metadata from file...') + try { + const data = readFileSync(METADATA_FILE, 'utf-8') + const imported = JSON.parse(data) + Object.assign(lastEditsCache, imported) + const stats = statSync(METADATA_FILE) + console.log('Loaded', objectSize(lastEditsCache), 'file timestamps from cache') + console.log('Metadata file size:', (stats.size / 1024 / 1024).toFixed(2), 'MB') + return + } catch (error) { + console.error('Failed to import git metadata:', error) + throw new Error('Cannot import git metadata. File missing or corrupt.') + } + } + console.log('Loading Git File Tree...') const firstCommand = 'git ls-tree -r --name-only HEAD ../data' const files = (await runCommand(firstCommand)).split('\n') @@ -157,19 +177,17 @@ export async function loadLastEdits() { console.log('loaded', processed, 'out of', files.length, 'files', `(${(processed / files.length * 100).toFixed(0)}%)`) } })) - // try { - // // don't really know why but it does not correctly execute the command when using Spawn - // lastEditsCache[file] = await runCommand(`git log -1 --pretty="format:%cd" --date=iso-strict "${file}"`, false) - // } catch { - // console.warn('could not load file', file, 'hope it does not break everything else lol') - // } - // processed++ - // if (processed % 1000 === 0) { - // console.log('loaded', processed, 'out of', files.length, 'files', `(${(processed / files.length * 100).toFixed(0)}%)`) - // } } await queue.waitEnd() console.log('done loading files', objectSize(lastEditsCache)) + + if (EXPORT_METADATA) { + console.log('Exporting git metadata to file...') + writeFileSync(METADATA_FILE, JSON.stringify(lastEditsCache)) + const stats = statSync(METADATA_FILE) + console.log('Exported', objectSize(lastEditsCache), 'file timestamps') + console.log('Metadata file size:', (stats.size / 1024 / 1024).toFixed(2), 'MB') + } } export function getLastEdit(path: string): string {