From c871b87638122a6d72a08fdfd0579ee82bc7bc0a Mon Sep 17 00:00:00 2001 From: Jason Moggridge Date: Mon, 16 Mar 2026 08:48:49 -0400 Subject: [PATCH] BUILD: Migrate CI/CD to Nix, serve frontend + docs via SPA_DIR --- .github/workflows/build.yml | 311 ---------------- .github/workflows/ci.yml | 169 +-------- .github/workflows/deploy-preview.yml | 117 ++++++ .github/workflows/deploy-prod.yml | 22 ++ .github/workflows/deploy-staging.yml | 28 ++ .github/workflows/deploy-target.yml | 155 ++++++++ .github/workflows/deploy.yml | 285 --------------- dev-docs/default.nix | 35 ++ dev-docs/fixing-hash-mismatches.md | 12 +- flake.nix | 346 ++++-------------- infrastructure/biome.nix | 1 + infrastructure/checks/default.nix | 68 ++++ .../frontend-tests.nix} | 0 .../checks/generated-bindings-check.nix | 43 +++ infrastructure/checks/npm-checks.nix | 42 +++ infrastructure/checks/rust-clippy.nix | 14 + infrastructure/checks/rust-fmt.nix | 4 + infrastructure/checks/rust-tests.nix | 14 + infrastructure/devshell.nix | 82 +++++ infrastructure/generated-bindings-check.nix | 60 --- .../hosts/catcolab-next/default.nix | 3 + infrastructure/hosts/catcolab/default.nix | 3 + infrastructure/hosts/default.nix | 101 +++++ infrastructure/modules/catcolab/services.nix | 11 +- infrastructure/netlify-preview.nix | 21 ++ infrastructure/packages.nix | 133 +++++++ infrastructure/pnpm-deps.nix | 26 ++ infrastructure/rust-docs.nix | 41 +-- infrastructure/rust.nix | 89 +++++ infrastructure/site.nix | 39 ++ math-docs/default.nix | 51 +++ math-docs/shell.nix | 4 +- packages/frontend/default.nix | 26 +- packages/frontend/netlify.toml | 6 - packages/ui-components/default.nix | 34 ++ 35 files changed, 1265 insertions(+), 1131 deletions(-) delete mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/deploy-preview.yml create mode 100644 .github/workflows/deploy-prod.yml create mode 100644 .github/workflows/deploy-staging.yml create mode 100644 .github/workflows/deploy-target.yml delete mode 100644 .github/workflows/deploy.yml create mode 100644 dev-docs/default.nix create mode 100644 infrastructure/checks/default.nix rename infrastructure/{tests/frontend.nix => checks/frontend-tests.nix} (100%) create mode 100644 infrastructure/checks/generated-bindings-check.nix create mode 100644 infrastructure/checks/npm-checks.nix create mode 100644 infrastructure/checks/rust-clippy.nix create mode 100644 infrastructure/checks/rust-fmt.nix create mode 100644 infrastructure/checks/rust-tests.nix create mode 100644 infrastructure/devshell.nix delete mode 100644 infrastructure/generated-bindings-check.nix create mode 100644 infrastructure/hosts/default.nix create mode 100644 infrastructure/netlify-preview.nix create mode 100644 infrastructure/packages.nix create mode 100644 infrastructure/pnpm-deps.nix create mode 100644 infrastructure/rust.nix create mode 100644 infrastructure/site.nix create mode 100644 math-docs/default.nix delete mode 100644 packages/frontend/netlify.toml create mode 100644 packages/ui-components/default.nix diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 4fb137cf3..000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,311 +0,0 @@ -name: build - -on: - push: - branches: - - main - pull_request: - release: - types: [published] - -jobs: - build_dev_docs: - name: Build developer docs - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - # Need full history for `git log` for cache key - fetch-depth: 0 - - - name: Compute cache key - id: cache-key - run: | - hash=$(git log -1 --format="%H" -- dev-docs) - echo "git-hash=${hash}" >> $GITHUB_OUTPUT - echo "Dev docs hash: ${hash}" - - - name: Cache dev docs build - id: cache-dev-docs-build - uses: actions/cache@v4 - with: - path: dev-docs/output - key: dev-docs-build-${{ steps.cache-key.outputs.git-hash }} - - - name: Setup pnpm - if: steps.cache-dev-docs-build.outputs.cache-hit != 'true' - uses: pnpm/action-setup@v4 - - - name: Setup NodeJS - if: steps.cache-dev-docs-build.outputs.cache-hit != 'true' - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: "pnpm" - - - name: Build developer docs - if: steps.cache-dev-docs-build.outputs.cache-hit != 'true' - run: | - pnpm install - pnpm --filter ./dev-docs run doc - - - name: Report cache status - run: | - if [ "${{ steps.cache-dev-docs-build.outputs.cache-hit }}" = "true" ]; then - echo "βœ… Dev docs restored from cache" - else - echo "πŸ”¨ Dev docs built and cached" - fi - - - name: Upload developer docs - uses: actions/upload-artifact@v4 - with: - name: dev-docs - path: dev-docs/output - - build_frontend: - name: Build frontend - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - # Need full history for `git log` for cache key - fetch-depth: 0 - - - name: Compute cache key - id: cache-key - run: | - # Get git hash of the most recent commit affecting frontend or any of its dependencies - hash=$(git log -1 --format="%H" -- packages/frontend packages/backend/pkg packages/catlog packages/catlog-wasm packages/notebook-types packages/ui-components Cargo.toml Cargo.lock) - echo "git-hash=${hash}" >> $GITHUB_OUTPUT - echo "Frontend git hash: ${git_hash}" - - - name: Cache frontend build - id: cache-frontend-build - # Skip caching for production builds - if: github.event_name != 'release' - uses: actions/cache@v4 - with: - path: | - packages/frontend/dist - key: frontend-build-${{ steps.cache-key.outputs.git-hash }}-${{ github.event_name == 'release' && 'production' || 'staging' }} - - - name: Setup pnpm - if: github.event_name == 'release' || steps.cache-frontend-build.outputs.cache-hit != 'true' - uses: pnpm/action-setup@v4 - - - name: Setup NodeJS - if: github.event_name == 'release' || steps.cache-frontend-build.outputs.cache-hit != 'true' - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: "pnpm" - - - name: Install Rust toolchain from file - if: github.event_name == 'release' || steps.cache-frontend-build.outputs.cache-hit != 'true' - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - # Don't override flags in cargo config files. - rustflags: "" - - - name: Build for Staging - if: (github.event_name == 'push' || github.event_name == 'pull_request') && steps.cache-frontend-build.outputs.cache-hit != 'true' - run: | - pnpm install - # Uses env from packages/frontend/.env.staging - pnpm --filter ./packages/frontend run build -- --mode staging - - - name: Build for Production - if: github.event_name == 'release' - run: | - pnpm install - # Uses env from packages/frontend/.env.production - pnpm --filter ./packages/frontend run build - - - name: Report cache status - run: | - if [ "${{ github.event_name }}" = "release" ]; then - echo "πŸ”¨ Frontend production build completed (cache disabled)" - elif [ "${{ steps.cache-frontend-build.outputs.cache-hit }}" = "true" ]; then - echo "βœ… Frontend build restored from cache" - else - echo "πŸ”¨ Frontend build completed and cached" - fi - - - name: Upload - uses: actions/upload-artifact@v4 - with: - name: app - path: packages/frontend/dist - - - name: Cache frontend docs build - id: cache-frontend-docs - uses: actions/cache@v4 - with: - path: packages/frontend/docs - key: frontend-docs-${{ steps.cache-key.outputs.git-hash }} - - - name: Build frontend docs - if: steps.cache-frontend-docs.outputs.cache-hit != 'true' - run: | - pnpm --filter ./packages/frontend run doc - - - name: Report frontend docs cache status - run: | - if [ "${{ steps.cache-frontend-docs.outputs.cache-hit }}" = "true" ]; then - echo "βœ… Frontend docs restored from cache" - else - echo "πŸ”¨ Frontend docs built and cached" - fi - - - name: Upload frontend docs - uses: actions/upload-artifact@v4 - with: - name: frontend_docs - path: packages/frontend/docs - - build_rust_docs: - name: Build Rust docs - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install Nix - uses: cachix/install-nix-action@v25 - - - name: Configure Cachix - uses: cachix/cachix-action@v14 - with: - name: catcolab-jmoggr - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - - name: Build Rust docs - run: | - nix build .#rust-docs - - - name: Upload Rust docs - uses: actions/upload-artifact@v4 - with: - name: rust_docs - path: result/share/doc - - build_math-docs: - name: Build mathematical docs - runs-on: ubuntu-latest - steps: - - name: Repository Checkout - uses: actions/checkout@v4 - with: - # Need full history for `git log` for cache key - fetch-depth: 0 - - - name: Compute cache key - id: cache-key - run: | - hash=$(git log -1 --format="%H" -- math-docs) - echo "git-hash=${hash}" >> $GITHUB_OUTPUT - echo "Math docs hash: ${hash}" - - - name: Cache math docs build - id: cache-math-docs - uses: actions/cache@v4 - with: - path: math-docs/output - key: math-docs-${{ steps.cache-key.outputs.git-hash }} - - - name: Setup TinyTeX - if: steps.cache-math-docs.outputs.cache-hit != 'true' - uses: r-lib/actions/setup-tinytex@v2 - - - name: Install TeX Packages - if: steps.cache-math-docs.outputs.cache-hit != 'true' - run: | - tlmgr update --self - tlmgr install dvisvgm - tlmgr install standalone - tlmgr install pgf - tlmgr install tikz-cd - tlmgr install amsmath - tlmgr install quiver - tlmgr install spath3 - tlmgr install ebproof - - - name: Build mathematical docs - if: steps.cache-math-docs.outputs.cache-hit != 'true' - run: | - cd math-docs - ./forester build - - - name: Report cache status - run: | - if [ "${{ steps.cache-math-docs.outputs.cache-hit }}" = "true" ]; then - echo "βœ… Math docs restored from cache" - else - echo "πŸ”¨ Math docs built and cached" - fi - - - name: Upload mathematical docs - uses: actions/upload-artifact@v4 - with: - name: math-docs - path: math-docs/output - - build_ui_components: - name: Build ui-components Storybook - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - # Need full history for `git log` for cache key - fetch-depth: 0 - - - name: Compute cache key - id: cache-key - run: | - hash=$(git log -1 --format="%H" -- packages/ui-components) - echo "git-hash=${hash}" >> $GITHUB_OUTPUT - echo "UI Components hash: ${hash}" - - - name: Cache ui-components build - id: cache-ui-components - uses: actions/cache@v4 - with: - path: packages/ui-components/storybook-static - key: ui-components-${{ steps.cache-key.outputs.git-hash }} - - - name: Setup pnpm - if: steps.cache-ui-components.outputs.cache-hit != 'true' - uses: pnpm/action-setup@v4 - - - name: Setup NodeJS - if: steps.cache-ui-components.outputs.cache-hit != 'true' - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: "pnpm" - - - name: Install dependencies - if: steps.cache-ui-components.outputs.cache-hit != 'true' - run: pnpm install - - - name: Build ui-components Storybook - if: steps.cache-ui-components.outputs.cache-hit != 'true' - run: pnpm --filter ./packages/ui-components run build - - - name: Report cache status - run: | - if [ "${{ steps.cache-ui-components.outputs.cache-hit }}" = "true" ]; then - echo "βœ… UI Components Storybook restored from cache" - else - echo "πŸ”¨ UI Components Storybook built and cached" - fi - - - name: Upload ui-components Storybook - uses: actions/upload-artifact@v4 - with: - name: ui-components - path: packages/ui-components/storybook-static diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 36f113fbd..06b55fec5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,80 +6,27 @@ on: - main pull_request: -env: - CARGO_TERM_COLOR: always - jobs: - rust_tests: - name: rust tests + nix_checks: + name: nix checks runs-on: ubuntu-latest strategy: matrix: - toolchain: - - stable - - beta + check: [rustFmt, rustClippy, rustTests, rustDocsCheck, generatedBindingsCheck, npmChecks] steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Tests - run: | - rustup toolchain install ${{ matrix.toolchain }} - rustup default ${{ matrix.toolchain }} - - cargo build --all-features --verbose - cargo test --all-features --verbose - - rust_formatting: - name: rust formatting - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Install Rust toolchain from file - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - toolchain: nightly - components: rustfmt - - - name: Check formatting - run: | - cargo fmt --check - - rust_lints: - name: rust lints - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Install Rust toolchain from file - uses: actions-rust-lang/setup-rust-toolchain@v1 - - - name: Check clippy warnings - run: | - cargo clippy --all-targets --all-features -- --deny warnings - - check_rust_docs: - name: Build Rust docs - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install Nix - uses: cachix/install-nix-action@v25 - - - name: Configure Cachix - uses: cachix/cachix-action@v14 + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v25 + - uses: cachix/cachix-action@v14 with: name: catcolab-jmoggr authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - - name: Check Rust docs + - name: Run ${{ matrix.check }} + run: nix build .#checks.x86_64-linux.${{ matrix.check }} + - name: Show build logs on failure + if: failure() run: | - nix build .#rust-docs-check + nix log .#checks.x86_64-linux.${{ matrix.check }} + exit 1 ui_components_tests: name: ui-components tests @@ -106,95 +53,20 @@ jobs: - name: Run ui-components tests run: pnpm --filter ./packages/ui-components run test - npm_checks: - name: npm checks - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup pnpm - uses: pnpm/action-setup@v4 - - - name: Setup NodeJS - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: "pnpm" - - - name: Install NodeJS dependencies - run: | - pnpm install - - - name: Format/linting/import sorting - run: | - pnpm --filter "./packages/*" run ci - build_nixos_system: name: build NixOS system runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install Nix - uses: cachix/install-nix-action@v25 - - - name: Configure Cachix - uses: cachix/cachix-action@v14 - with: - name: catcolab-jmoggr - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - - name: Build the NixOS system for catcolab-next - run: nix build .#nixosConfigurations.catcolab-next.config.system.build.toplevel - - check_generated_bindings: - name: check generated bindings - runs-on: ubuntu-latest + strategy: + matrix: + config: [catcolab-next, catcolab] steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install Nix - uses: cachix/install-nix-action@v25 - - - name: Configure Cachix - uses: cachix/cachix-action@v14 + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v25 + - uses: cachix/cachix-action@v14 with: name: catcolab-jmoggr authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - - name: Check generated bindings - run: nix build .#generated-bindings-check - - backend_dev_setup: - name: backend dev setup - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Install Rust toolchain from file - uses: actions-rust-lang/setup-rust-toolchain@v1 - - - name: Setup pnpm - uses: pnpm/action-setup@v4 - - - name: Setup wasm-pack - uses: jetli/wasm-pack-action@v0.4.0 - with: - version: 'v0.13.1' - - - name: Run dev setup - run: | - docker run --name catcolab-postgres -e POSTGRES_USER=catcolab -e POSTGRES_PASSWORD=password -e POSTGRES_DB=catcolab -p 5432:5432 -d postgres:15 - cd packages/backend - cp .env.development .env && cp .env.development ../migrator/.env - # due to how we are detecting deploy vs dev we need to clear this env variable - unset INVOCATION_ID - cargo run -p migrator apply + - run: nix build .#nixosConfigurations.${{ matrix.config }}.config.system.build.toplevel frontend_tests_with_backend: name: frontend tests with backend @@ -206,8 +78,6 @@ jobs: - name: Install Nix uses: cachix/install-nix-action@v25 - with: - install_url: https://releases.nixos.org/nix/nix-2.29.2/install - name: Configure Cachix uses: cachix/cachix-action@v14 @@ -218,4 +88,3 @@ jobs: - name: Run frontend tests with VM run: | nix build .#checks.x86_64-linux.frontendTests -L --no-sandbox - diff --git a/.github/workflows/deploy-preview.yml b/.github/workflows/deploy-preview.yml new file mode 100644 index 000000000..758353a93 --- /dev/null +++ b/.github/workflows/deploy-preview.yml @@ -0,0 +1,117 @@ +name: deploy preview + +on: + pull_request: + types: [opened, synchronize, reopened, closed] + +permissions: + contents: read + deployments: write + +jobs: + deploy_preview: + name: deploy preview + runs-on: ubuntu-latest + if: >- + github.event.action != 'closed' && + github.event.pull_request.head.repo.full_name == github.repository + steps: + - name: Create GitHub Deployment + id: deployment + uses: chrnorm/deployment-action@v2 + with: + token: ${{ github.token }} + environment: netlify-preview + transient-environment: true + auto-inactive: true + ref: ${{ github.event.pull_request.head.ref }} + sha: ${{ github.event.pull_request.head.sha }} + initial-status: in_progress + log-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Nix + uses: cachix/install-nix-action@v25 + + - name: Configure Cachix + uses: cachix/cachix-action@v14 + with: + name: catcolab-jmoggr + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 24 + cache: "pnpm" + + - name: Install Netlify CLI + run: | + cd .netlify-env + pnpm install + + - name: Build site preview + run: nix build .#netlify-preview -L + + - name: Deploy to Netlify + id: netlify + run: | + BRANCH_NAME="${{ github.head_ref }}" + CLEAN_BRANCH=$(echo "$BRANCH_NAME" | sed 's/[^a-zA-Z0-9-]/-/g') + cd .netlify-env + npx netlify deploy \ + --dir ../result \ + --site ${{ secrets.NETLIFY_SITE_ID }} \ + --auth ${{ secrets.NETLIFY_API_TOKEN }} \ + --alias="branch-${CLEAN_BRANCH}" \ + --json > ../deploy_output.json + cd .. + echo "deploy_url=$(jq -r '.deploy_url' deploy_output.json)" >> "$GITHUB_OUTPUT" + + - name: Update deployment status (success) + if: success() + uses: chrnorm/deployment-status@v2 + with: + token: ${{ github.token }} + deployment-id: ${{ steps.deployment.outputs.deployment_id }} + state: success + environment-url: ${{ steps.netlify.outputs.deploy_url }} + log-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Update deployment status (failure) + if: failure() + uses: chrnorm/deployment-status@v2 + with: + token: ${{ github.token }} + deployment-id: ${{ steps.deployment.outputs.deployment_id }} + state: failure + log-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + deactivate_preview: + name: deactivate preview deployment + runs-on: ubuntu-latest + if: github.event.action == 'closed' + steps: + - name: Deactivate deployments + uses: actions/github-script@v7 + with: + script: | + const { data: deployments } = await github.rest.repos.listDeployments({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: context.payload.pull_request.head.ref, + environment: 'netlify-preview' + }); + for (const deployment of deployments) { + await github.rest.repos.createDeploymentStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + deployment_id: deployment.id, + state: 'inactive' + }); + } diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml new file mode 100644 index 000000000..63d1e8c09 --- /dev/null +++ b/.github/workflows/deploy-prod.yml @@ -0,0 +1,22 @@ +name: deploy prod + +on: + push: + tags: ['v*'] + +permissions: + deployments: write + +jobs: + deploy: + uses: ./.github/workflows/deploy-target.yml + with: + environment: backend + environment_url: https://catcolab.org + deploy_node: catcolab + hostname: backend.catcolab.org + ref: ${{ github.ref }} + sha: ${{ github.sha }} + secrets: + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + DEPLOY_SSH_KEY: ${{ secrets.CATCOLAB_PROD_DEPLOYUSER_KEY }} diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml new file mode 100644 index 000000000..aa49c4519 --- /dev/null +++ b/.github/workflows/deploy-staging.yml @@ -0,0 +1,28 @@ +name: deploy staging + +on: + workflow_run: + workflows: ["ci"] + types: [completed] + branches: [main] + +permissions: + deployments: write + +jobs: + deploy: + if: > + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event != 'pull_request' && + github.event.workflow_run.head_branch == 'main' + uses: ./.github/workflows/deploy-target.yml + with: + environment: backend-next + environment_url: https://backend-next.catcolab.org + deploy_node: catcolab-next + hostname: backend-next.catcolab.org + ref: ${{ github.event.workflow_run.head_branch }} + sha: ${{ github.event.workflow_run.head_sha }} + secrets: + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + DEPLOY_SSH_KEY: ${{ secrets.CATCOLAB_NEXT_DEPLOYUSER_KEY }} diff --git a/.github/workflows/deploy-target.yml b/.github/workflows/deploy-target.yml new file mode 100644 index 000000000..149a198a8 --- /dev/null +++ b/.github/workflows/deploy-target.yml @@ -0,0 +1,155 @@ +name: deploy target + +on: + workflow_call: + inputs: + environment: + description: GitHub deployment environment name + required: true + type: string + environment_url: + description: URL for deployment status + required: true + type: string + deploy_node: + description: deploy-rs node name + required: true + type: string + hostname: + description: SSH hostname + required: true + type: string + ssh_user: + description: SSH user for deploy and verify + required: false + type: string + default: catcolab + ref: + description: Git ref for the deployment + required: true + type: string + sha: + description: Git SHA for the deployment + required: true + type: string + secrets: + CACHIX_AUTH_TOKEN: + required: true + DEPLOY_SSH_KEY: + required: true + +# Permissions are inherited from the caller workflow. + +jobs: + create_deployment: + name: Create GitHub deployment + runs-on: ubuntu-latest + outputs: + deployment_id: ${{ steps.create_deployment.outputs.deployment_id }} + steps: + - name: Create new GitHub deployment + id: create_deployment + uses: chrnorm/deployment-action@v2 + with: + token: ${{ github.token }} + environment: ${{ inputs.environment }} + ref: ${{ inputs.ref }} + sha: ${{ inputs.sha }} + auto-inactive: true + log-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Update deployment status to in_progress + uses: chrnorm/deployment-status@v2 + with: + token: ${{ github.token }} + description: 'Deploying to ${{ inputs.hostname }}...' + state: 'in_progress' + log-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + deployment-id: ${{ steps.create_deployment.outputs.deployment_id }} + + deploy: + name: Deploy to ${{ inputs.hostname }} + runs-on: ubuntu-latest + needs: [create_deployment] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + + - name: Install Nix + uses: cachix/install-nix-action@v25 + + - name: Configure Cachix + uses: cachix/cachix-action@v14 + with: + name: catcolab-jmoggr + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + + - name: Set the SSH key for deployment + env: + DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} + DEPLOY_HOSTNAME: ${{ inputs.hostname }} + run: | + mkdir -p ~/.ssh + echo "$DEPLOY_SSH_KEY" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + ssh-keyscan "$DEPLOY_HOSTNAME" >> ~/.ssh/known_hosts + + - name: Deploy to ${{ inputs.deploy_node }} + env: + DEPLOY_NODE: ${{ inputs.deploy_node }} + run: | + nix run github:serokell/deploy-rs -- --skip-checks ".#$DEPLOY_NODE" + + - name: Verify deployment + env: + DEPLOY_NODE: ${{ inputs.deploy_node }} + SSH_USER: ${{ inputs.ssh_user }} + DEPLOY_HOSTNAME: ${{ inputs.hostname }} + run: | + EXPECTED=$(nix eval --raw ".#nixosConfigurations.$DEPLOY_NODE.config.system.build.toplevel") + ACTUAL=$(ssh "$SSH_USER@$DEPLOY_HOSTNAME" "readlink -f /run/current-system") + + if [[ "$ACTUAL" != "$EXPECTED" ]]; then + echo "ERROR: Deployment verification failed, the server is not running the expected nix profile" + echo "Expected: $EXPECTED" + echo "Actual: $ACTUAL" + exit 1 + fi + + echo "Deployment verified: $ACTUAL" + + - name: Rsync the repo to remote host + env: + SSH_USER: ${{ inputs.ssh_user }} + DEPLOY_HOSTNAME: ${{ inputs.hostname }} + run: | + rsync -az --delete ./ "$SSH_USER@$DEPLOY_HOSTNAME:~/catcolab" + + report_status: + name: Report deployment status + runs-on: ubuntu-latest + needs: [create_deployment, deploy] + if: always() && needs.create_deployment.result == 'success' + steps: + - name: Update deployment status to success + if: needs.deploy.result == 'success' + uses: chrnorm/deployment-status@v2 + with: + token: ${{ github.token }} + environment-url: ${{ inputs.environment_url }} + log-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + description: 'Deployed to ${{ inputs.hostname }}' + state: 'success' + deployment-id: ${{ needs.create_deployment.outputs.deployment_id }} + + - name: Update deployment status to failure + if: needs.deploy.result != 'success' + uses: chrnorm/deployment-status@v2 + with: + token: ${{ github.token }} + log-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + description: 'Deployment to ${{ inputs.hostname }} failed' + state: 'failure' + deployment-id: ${{ needs.create_deployment.outputs.deployment_id }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index 4e66b5099..000000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,285 +0,0 @@ -name: deploy - -on: - workflow_run: - workflows: ["build"] - types: - - completed - -env: - BRANCH_NAME: ${{ github.event.workflow_run.head_branch }} - -permissions: - deployments: write - -jobs: - create_deployment: - name: Create GitHub deployment - runs-on: ubuntu-latest - if: > - github.event.workflow_run.conclusion == 'success' && - github.event.workflow_run.event == 'pull_request' && - github.event.workflow_run.head_repository.full_name == 'ToposInstitute/CatColab' - outputs: - deployment_id: ${{ steps.create_deployment.outputs.deployment_id }} - steps: - - name: Create new GitHub deployment - id: create_deployment - uses: chrnorm/deployment-action@v2 - with: - token: ${{ github.token }} - environment: netlify-preview - ref: ${{ github.event.workflow_run.head_branch }} - sha: ${{ github.event.workflow_run.head_sha }} - transient-environment: true - auto-inactive: true - log-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: Update deployment status to in_progress - uses: chrnorm/deployment-status@v2 - with: - token: ${{ github.token }} - description: 'Building and deploying...' - state: 'in_progress' - log-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - deployment-id: ${{ steps.create_deployment.outputs.deployment_id }} - - deploy: - name: Deploy to Netlify - runs-on: ubuntu-latest - if: > - github.event.workflow_run.conclusion == 'success' && - (github.event.workflow_run.event != 'pull_request' || github.event.workflow_run.head_repository.full_name == 'ToposInstitute/CatColab') - outputs: - deploy_url: ${{ steps.url_preview.outputs.NETLIFY_PREVIEW_URL }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup pnpm - uses: pnpm/action-setup@v4 - - - name: Setup NodeJS - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: "pnpm" - - - name: Install Netlify - run: | - cd .netlify-env - pnpm install - - - name: Download artifacts - uses: actions/download-artifact@v4 - with: - run-id: ${{ github.event.workflow_run.id }} - github-token: ${{ github.token }} - - - name: Consolidate and deploy to Staging - if: github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'pull_request' - id: netlify_deploy - run: | - mv app site/ - mv dev-docs site/dev - mv rust_docs site/dev/rust - mv frontend_docs site/dev/frontend - mv ui-components site/dev/ui-components - mv math-docs site/math - echo '/dev/rust /dev/rust/catlog' >> site/_redirects - echo '/dev/core /dev/rust/catlog' >> site/_redirects - echo '/dev/catcolab_backend /dev/rust/backend' >> site/_redirects - echo '/math /math/index.xml' >> site/_redirects - echo '/maths /math/index.xml' >> site/_redirects - echo '/* /index.html 200' >> site/_redirects - cd .netlify-env - branch_flag="" - if [ "$BRANCH_NAME" = "main" ]; then branch_flag="--prod"; else branch_flag="--alias='branch-$BRANCH_NAME'"; fi - npx netlify deploy --dir ../site --site ${{ secrets.NETLIFY_SITE_ID }} --auth ${{ secrets.NETLIFY_API_TOKEN }} $branch_flag --json > ../deploy_output.json - - - name: Consolidate and deploy to Production - if: github.event.workflow_run.event == 'release' - run: | - mv app site/ - echo '/* /index.html 200' >> site/_redirects - cd .netlify-env - npx netlify deploy --dir ../site --site ${{ secrets.NETLIFY_PROD_SITE_ID }} --auth ${{ secrets.NETLIFY_API_TOKEN }} --prod - - - name: Generate URL preview - id: url_preview - if: env.BRANCH_NAME != 'main' && (github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'pull_request') - run: | - NETLIFY_PREVIEW_URL=$(jq -r '.deploy_url' deploy_output.json) - echo "NETLIFY_PREVIEW_URL=$NETLIFY_PREVIEW_URL" >> "$GITHUB_OUTPUT" - - report_deployment_status: - name: Report deployment status - runs-on: ubuntu-latest - needs: [create_deployment, deploy] - if: always() && github.event.workflow_run.event == 'pull_request' && needs.create_deployment.result == 'success' - steps: - - name: Delete old deployments - if: needs.deploy.result == 'success' - uses: actions/github-script@v7 - with: - script: | - const { data: deployments } = await github.rest.repos.listDeployments({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: '${{ github.event.workflow_run.head_branch }}', - environment: 'netlify-preview' - }); - - console.log(`Found ${deployments.length} existing deployment(s)`); - - const currentDeploymentId = parseInt('${{ needs.create_deployment.outputs.deployment_id }}'); - - for (const deployment of deployments) { - if (deployment.id !== currentDeploymentId) { - console.log(`Deleting old deployment ${deployment.id}`); - - // A deployment must be marked inactive before it can be deleted - await github.rest.repos.createDeploymentStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - deployment_id: deployment.id, - state: 'inactive', - description: 'Superseded by newer deployment' - }); - - await github.rest.repos.deleteDeployment({ - owner: context.repo.owner, - repo: context.repo.repo, - deployment_id: deployment.id - }); - - console.log(`Deleted deployment ${deployment.id}`); - } - } - - - name: Update deployment status to success - if: needs.deploy.result == 'success' - uses: chrnorm/deployment-status@v2 - with: - token: ${{ github.token }} - environment-url: ${{ needs.deploy.outputs.deploy_url }} - log-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - description: 'Netlify preview deployment' - state: 'success' - deployment-id: ${{ needs.create_deployment.outputs.deployment_id }} - - - name: Update deployment status to failure - if: needs.deploy.result != 'success' - uses: chrnorm/deployment-status@v2 - with: - token: ${{ github.token }} - log-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - description: 'Deployment failed' - state: 'failure' - deployment-id: ${{ needs.create_deployment.outputs.deployment_id }} - - create_backend_deployment: - name: Create backend GitHub deployment - runs-on: ubuntu-latest - if: > - github.event.workflow_run.conclusion == 'success' && - github.event.workflow_run.event != 'pull_request' && - github.event.workflow_run.head_branch == 'main' - outputs: - deployment_id: ${{ steps.create_deployment.outputs.deployment_id }} - steps: - - name: Create new GitHub deployment - id: create_deployment - uses: chrnorm/deployment-action@v2 - with: - token: ${{ github.token }} - environment: backend-next - ref: ${{ github.event.workflow_run.head_branch }} - sha: ${{ github.event.workflow_run.head_sha }} - auto-inactive: true - log-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: Update deployment status to in_progress - uses: chrnorm/deployment-status@v2 - with: - token: ${{ github.token }} - description: 'Deploying backend to AWS...' - state: 'in_progress' - log-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - deployment-id: ${{ steps.create_deployment.outputs.deployment_id }} - - deploy_backend: - name: Deploy backend to AWS - runs-on: ubuntu-latest - needs: [create_backend_deployment] - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install Nix - uses: cachix/install-nix-action@v25 - - - name: Configure Cachix - uses: cachix/cachix-action@v14 - with: - name: catcolab-jmoggr - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - - name: Set the SSH key for deployment to catcolab-next - run: | - mkdir -p ~/.ssh - echo "${{ secrets.CATCOLAB_NEXT_DEPLOYUSER_KEY }}" > ~/.ssh/id_ed25519 - chmod 600 ~/.ssh/id_ed25519 - ssh-keyscan backend-next.catcolab.org >> ~/.ssh/known_hosts - - - name: Deploy to catcolab-next - run: | - nix run github:serokell/deploy-rs -- --skip-checks .#catcolab-next - - - name: Verify deployment - run: | - EXPECTED=$(nix eval --raw .#nixosConfigurations.catcolab-next.config.system.build.toplevel) - ACTUAL=$(ssh catcolab@backend-next.catcolab.org "readlink -f /run/current-system") - - if [[ "$ACTUAL" != "$EXPECTED" ]]; then - echo "ERROR: Deployment verification failed, the server is not running the expected nix profile" - echo "Expected: $EXPECTED" - echo "Actual: $ACTUAL" - exit 1 - fi - - echo "Deployment verified: $ACTUAL" - - # Ensure that a copy of the deployed repository is on the machine that it was deployed to. This is - # a nice-to-have which enables checking the configuration of the currently deployed system and could - # make recovery slightly less aweful in the event nix commands need to be run on the remote. - - name: Rsync the repo to remote host - run: | - rsync -az --delete ./ catcolab@backend-next.catcolab.org:~/catcolab - - report_backend_deployment_status: - name: Report backend deployment status - runs-on: ubuntu-latest - needs: [create_backend_deployment, deploy_backend] - if: always() && needs.create_backend_deployment.result == 'success' - steps: - - name: Update deployment status to success - if: needs.deploy_backend.result == 'success' - uses: chrnorm/deployment-status@v2 - with: - token: ${{ github.token }} - environment-url: https://backend-next.catcolab.org - log-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - description: 'Backend deployed to AWS' - state: 'success' - deployment-id: ${{ needs.create_backend_deployment.outputs.deployment_id }} - - - name: Update deployment status to failure - if: needs.deploy_backend.result != 'success' - uses: chrnorm/deployment-status@v2 - with: - token: ${{ github.token }} - log-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - description: 'Backend deployment failed' - state: 'failure' - deployment-id: ${{ needs.create_backend_deployment.outputs.deployment_id }} diff --git a/dev-docs/default.nix b/dev-docs/default.nix new file mode 100644 index 000000000..f61918e84 --- /dev/null +++ b/dev-docs/default.nix @@ -0,0 +1,35 @@ +{ + pkgs, + lib, + pnpmDeps, +}: +pkgs.stdenv.mkDerivation { + pname = "catcolab-dev-docs"; + version = "0.0.1"; + + src = lib.fileset.toSource { + root = ../.; + fileset = lib.fileset.unions [ + ../.npmrc + ../pnpm-workspace.yaml + ../pnpm-lock.yaml + ../dev-docs + ../CONTRIBUTING.md + ]; + }; + + nativeBuildInputs = [ pkgs.pnpm.configHook ]; + buildInputs = [ pkgs.nodejs_24 ]; + + inherit pnpmDeps; + + buildPhase = '' + cd dev-docs + npm run doc + ''; + + installPhase = '' + mkdir -p $out + cp -r output/* $out/ + ''; +} diff --git a/dev-docs/fixing-hash-mismatches.md b/dev-docs/fixing-hash-mismatches.md index 43e806c83..7f7a0a639 100644 --- a/dev-docs/fixing-hash-mismatches.md +++ b/dev-docs/fixing-hash-mismatches.md @@ -18,7 +18,7 @@ the external source changed unexpectedly. #### pnpm Dependencies -This only applies to the `frontend` package. +This applies to the `frontend`, `dev-docs`, and `ui-components-storybook` packages. The following error occurs when a dependency has changed but the Nix hash has not: ``` @@ -34,11 +34,13 @@ In this case you can follow the instructions given below the error message: > 3. Copy the 'got: sha256-' value back into the pnpmDeps.hash field ``` -The hash is located in `packages/frontend/default.nix` within the `pkgs.fetchPnpmDeps` block. -You can search for the text "hash" to find it quickly. +The hash is located in the `pkgs.fetchPnpmDeps` block within each package's `default.nix`: +- `packages/frontend/default.nix` (build with `nix build .#frontend`) +- `dev-docs/default.nix` (build with `nix build .#dev-docs`) +- `packages/ui-components/default.nix` (build with `nix build .#ui-components-storybook`) -The frontend package can be built by running the command `nix build .#frontend` in the repository root. -This will build the minimum needed to print the hash mismatch described in the instructions. +You can search for the text "hash" to find it quickly. Building any of these packages will print +the hash mismatch with the correct value to use. #### Other Dependencies diff --git a/flake.nix b/flake.nix index 85a88821d..6895ca4e6 100644 --- a/flake.nix +++ b/flake.nix @@ -9,11 +9,9 @@ url = "github:nix-community/fenix"; inputs.nixpkgs.follows = "nixpkgs"; }; - crane = { url = "github:ipetkov/crane"; }; - nixos-generators.url = "github:nix-community/nixos-generators"; }; @@ -28,7 +26,6 @@ ... }@inputs: let - # Linux-specific outputs (NixOS configurations and deploy) linuxSystem = "x86_64-linux"; devShellSystems = [ @@ -37,6 +34,7 @@ "aarch64-darwin" ]; + # Kept here to avoid circular deps between rust.nix and devshell.nix nixpkgsFor = system: import nixpkgs { @@ -61,308 +59,98 @@ ]; }; - rustToolchainFor = - system: - inputs.fenix.packages.${system}.fromToolchainFile { - file = ./rust-toolchain.toml; - sha256 = "sha256-vra6TkHITpwRyA5oBKAHSX0Mi6CBDNQD+ryPSpxFsfg="; - }; - pkgsLinux = nixpkgsFor linuxSystem; - rustToolchainLinux = rustToolchainFor linuxSystem; - craneLib = (crane.mkLib pkgsLinux).overrideToolchain rustToolchainLinux; - - cargoArtifacts = craneLib.buildDepsOnly { - src = craneLib.cleanCargoSource ./.; - strictDeps = true; - nativeBuildInputs = [ - pkgsLinux.pkg-config - ]; - - buildInputs = [ - pkgsLinux.openssl - ]; + rust = import ./infrastructure/rust.nix { + inherit + pkgsLinux + fenix + crane + linuxSystem + ; }; - # Generate devShells for each system - devShellForSystem = - system: - let - pkgs = nixpkgsFor system; - rustToolchain = rustToolchainFor system; - - # macOS-specific configurations for libraries - darwinDeps = - if pkgs.stdenv.isDarwin then - [ - pkgs.libiconv - ] - else - [ ]; - - nightlyRustfmt = inputs.fenix.packages.${system}.latest.rustfmt; - in - pkgs.mkShell { - name = "catcolab-devshell"; - RUSTFMT = "${nightlyRustfmt}/bin/rustfmt"; - buildInputs = - with pkgs; - [ - darkhttpd - esbuild - lld - netcat - nodejs_24 - nix - openssl - pkg-config - pnpm - postgresql - python3 - python312Packages.ipykernel - python312Packages.jupyter-core - python312Packages.jupyter-server - python312Packages.requests - python312Packages.websocket-client - rustToolchain - nightlyRustfmt - sqlx-cli - vscode-langservers-extracted - wasm-bindgen-cli - wasm-pack - ] - ++ darwinDeps - ++ [ - inputs.agenix.packages.${system}.agenix - inputs.deploy-rs.packages.${system}.default - (import ./infrastructure/biome.nix { inherit pkgs system; }) - ]; - - # macOS-specific environment variables for OpenSSL and pkg-config - shellHook = '' - ${ - if pkgs.stdenv.isDarwin then - '' - export OPENSSL_DIR=${pkgs.openssl.dev} - export OPENSSL_LIB_DIR=${pkgs.openssl.out}/lib - export OPENSSL_INCLUDE_DIR=${pkgs.openssl.dev}/include - export PKG_CONFIG_PATH=${pkgs.openssl.dev}/lib/pkgconfig:$PKG_CONFIG_PATH - '' - else - "" - } - - export PATH=$PWD/infrastructure/scripts:$PATH + pnpmDeps = import ./infrastructure/pnpm-deps.nix { inherit pkgsLinux; }; - # Load DATABASE_URL into the environment - if [ -f packages/backend/.env ]; then - export $(grep -v '^#' packages/backend/.env | xargs) - fi - ''; - }; - - # NOTE: this is not currently used, but was painful to build and might be useful in the future. - # Wraps the typical `deploy-rs.lib.${linuxSystem}.activate.nixos` activation function - # with a custom script that can run additional health checks. The script runs on the remote host - # and if it fails the whole deployment will fail. - # use: - # `deploy.nodes.${host}.profiles.system.path = healthcheckWrapper self.nixosConfigurations.host;` - healthcheckWrapper = - nixosConfiguration: - let - defaultNixos = deploy-rs.lib.${linuxSystem}.activate.nixos nixosConfiguration; + devshell = import ./infrastructure/devshell.nix { + inherit nixpkgsFor inputs; + rustToolchainFor = rust.rustToolchainFor; + }; - healthcheckWrapperScript = pkgsLinux.writeShellScriptBin "healthcheck-wrapper-script" '' - PROFILE=${defaultNixos} - # insert healthchecks - ${defaultNixos}/deploy-rs-activate - ''; - in - deploy-rs.lib.${linuxSystem}.activate.custom healthcheckWrapperScript - "./bin/healthcheck-wrapper-script"; + hosts = import ./infrastructure/hosts { + inherit + nixpkgs + deploy-rs + inputs + self + pkgsLinux + linuxSystem + ; + inherit (rust) rustToolchainLinux; + }; in { - # Create devShells for all supported systems devShells = builtins.listToAttrs ( map (system: { name = system; value = { - default = devShellForSystem system; + default = devshell.devShellForSystem system; }; }) devShellSystems ); - # Example of how to build and test individual package built by nix: - # nix build .#packages.x86_64-linux.automerge - # node ./result/main.cjs - packages = { - x86_64-linux = { - catcolabApi = pkgsLinux.stdenv.mkDerivation { - pname = "catcolab-api"; - version = "0.1.0"; - - src = ./packages/backend/pkg; - - installPhase = '' - mkdir -p $out - cp -r * $out/ - ''; - }; - - backend = pkgsLinux.callPackage ./packages/backend/default.nix { - inherit craneLib cargoArtifacts; - pkgs = pkgsLinux; - }; - - migrator = pkgsLinux.callPackage ./packages/migrator/default.nix { - inherit craneLib cargoArtifacts; - pkgs = pkgsLinux; - }; - - catlog-wasm-browser = pkgsLinux.callPackage ./packages/catlog-wasm/default.nix { - inherit craneLib cargoArtifacts; - pkgs = pkgsLinux; - }; - - frontend = - (pkgsLinux.callPackage ./packages/frontend/default.nix { - inherit inputs rustToolchainLinux self; - }).package; - - frontend-tests = - (pkgsLinux.callPackage ./packages/frontend/default.nix { - inherit inputs rustToolchainLinux self; - }).tests; - - rust-docs = pkgsLinux.callPackage ./infrastructure/rust-docs.nix { - inherit craneLib cargoArtifacts; - pkgs = pkgsLinux; - }; - - rust-docs-check = pkgsLinux.callPackage ./infrastructure/rust-docs.nix { - inherit craneLib cargoArtifacts; - pkgs = pkgsLinux; - checkMode = true; - }; - - generated-bindings-check = pkgsLinux.callPackage ./infrastructure/generated-bindings-check.nix { - inherit craneLib cargoArtifacts; - pkgs = pkgsLinux; - }; - - # VMs built with `nixos-rebuild build-vm` (like `nix build - # .#nixosConfigurations.catcolab-vm.config.system.build.vm`) are not the same - # as "traditional" VMs, which causes deploy-rs to fail when deploying to them. - # https://github.com/serokell/deploy-rs/issues/85#issuecomment-885782350 - # - # This is worked around by creating a full featured VM image. - # - # use: - # nix build .#catcolab-vm - # cp result/catcolab-vm.qcow2 catcolab-vm.qcow2 - # db-utils vm start - # deploy -s .#catcolab-vm - catcolab-vm = pkgsLinux.stdenv.mkDerivation { - name = "catcolab-vm"; - src = nixos-generators.nixosGenerate { - system = "x86_64-linux"; - format = "qcow"; - - modules = [ - ./infrastructure/hosts/catcolab-vm - ]; - - specialArgs = { - inherit inputs self; - rustToolchain = rustToolchainLinux; - }; - }; - installPhase = '' - mkdir -p $out - cp $src/nixos.qcow2 $out/catcolab-vm.qcow2 - ''; - }; - }; + packages.x86_64-linux = import ./infrastructure/packages.nix { + inherit + pkgsLinux + pnpmDeps + inputs + self + nixos-generators + ; + inherit (rust) + craneLib + cargoArtifacts + rustSrc + rustBuildInputs + rustToolchainLinux + ; }; - # Create a NixOS configuration for each host - nixosConfigurations = { - catcolab = nixpkgs.lib.nixosSystem { - specialArgs = { - inherit inputs self; - rustToolchain = rustToolchainLinux; - }; - system = linuxSystem; - modules = [ - ./infrastructure/hosts/catcolab - ]; - pkgs = pkgsLinux; - }; - catcolab-next = nixpkgs.lib.nixosSystem { - specialArgs = { - inherit inputs self; - rustToolchain = rustToolchainLinux; - }; - system = linuxSystem; - modules = [ - ./infrastructure/hosts/catcolab-next - ]; - pkgs = pkgsLinux; - }; - - catcolab-vm = nixpkgs.lib.nixosSystem { - system = linuxSystem; - modules = [ - ./infrastructure/hosts/catcolab-vm - ]; - specialArgs = { - inherit inputs self; - rustToolchain = rustToolchainLinux; - }; - }; - }; + inherit (hosts) nixosConfigurations; + deploy.nodes = hosts.deployNodes; - deploy.nodes = { - catcolab = { - hostname = "backend.catcolab.org"; - profiles.system = { - sshUser = "catcolab"; - user = "root"; - path = deploy-rs.lib.${linuxSystem}.activate.nixos self.nixosConfigurations.catcolab; - }; - }; - catcolab-next = { - hostname = "backend-next.catcolab.org"; - profiles.system = { - sshUser = "catcolab"; - user = "root"; - path = deploy-rs.lib.${linuxSystem}.activate.nixos self.nixosConfigurations.catcolab-next; - }; - }; - catcolab-vm = { - hostname = "localhost"; - fastConnection = true; - profiles.system = { - sshOpts = [ - "-p" - "2221" - ]; - sshUser = "catcolab"; - path = deploy-rs.lib.${linuxSystem}.activate.nixos self.nixosConfigurations.catcolab-vm; - user = "root"; - }; - }; - }; + formatter = builtins.listToAttrs ( + map (system: { + name = system; + value = + let + pkgs = nixpkgsFor system; + in + pkgs.writeShellScriptBin "nixfmt-all" '' + find "$@" -name '*.nix' -print0 | xargs -0 ${pkgs.nixfmt-rfc-style}/bin/nixfmt + ''; + }) devShellSystems + ); - checks.x86_64-linux.frontendTests = import ./infrastructure/tests/frontend.nix { + checks.x86_64-linux = import ./infrastructure/checks { inherit + pkgsLinux nixpkgs inputs self linuxSystem + pnpmDeps + ; + inherit (rust) + craneLib + craneLibNightly + cargoArtifacts + rustSrc + rustSrcWithExamples + rustSrcBindings + rustBuildInputs + rustToolchainLinux ; - rustToolchain = rustToolchainLinux; }; }; } diff --git a/infrastructure/biome.nix b/infrastructure/biome.nix index a04cf6036..8557ca1e9 100644 --- a/infrastructure/biome.nix +++ b/infrastructure/biome.nix @@ -38,4 +38,5 @@ pkgs.stdenv.mkDerivation { cp $src $out/bin/biome chmod +x $out/bin/biome ''; + meta.mainProgram = "biome"; } diff --git a/infrastructure/checks/default.nix b/infrastructure/checks/default.nix new file mode 100644 index 000000000..fd3893aa9 --- /dev/null +++ b/infrastructure/checks/default.nix @@ -0,0 +1,68 @@ +{ + pkgsLinux, + nixpkgs, + inputs, + self, + linuxSystem, + craneLib, + craneLibNightly, + cargoArtifacts, + rustSrc, + rustSrcWithExamples, + rustSrcBindings, + rustBuildInputs, + rustToolchainLinux, + pnpmDeps, +}: +{ + frontendTests = import ./frontend-tests.nix { + inherit + nixpkgs + inputs + self + linuxSystem + ; + rustToolchain = rustToolchainLinux; + }; + + rustFmt = import ./rust-fmt.nix { + inherit craneLibNightly; + }; + + rustClippy = import ./rust-clippy.nix { + inherit + craneLib + cargoArtifacts + rustSrc + rustBuildInputs + ; + }; + + rustTests = import ./rust-tests.nix { + inherit craneLib cargoArtifacts rustBuildInputs; + rustSrc = rustSrcWithExamples; + }; + + rustDocsCheck = import ../rust-docs.nix { + inherit + craneLib + cargoArtifacts + rustSrc + rustBuildInputs + ; + checkMode = true; + }; + + generatedBindingsCheck = import ./generated-bindings-check.nix { + inherit craneLib cargoArtifacts rustBuildInputs; + rustSrc = rustSrcBindings; + }; + + npmChecks = pkgsLinux.callPackage ./npm-checks.nix { + inherit pnpmDeps; + biome = import ../biome.nix { + pkgs = pkgsLinux; + system = linuxSystem; + }; + }; +} diff --git a/infrastructure/tests/frontend.nix b/infrastructure/checks/frontend-tests.nix similarity index 100% rename from infrastructure/tests/frontend.nix rename to infrastructure/checks/frontend-tests.nix diff --git a/infrastructure/checks/generated-bindings-check.nix b/infrastructure/checks/generated-bindings-check.nix new file mode 100644 index 000000000..e4303e474 --- /dev/null +++ b/infrastructure/checks/generated-bindings-check.nix @@ -0,0 +1,43 @@ +{ + craneLib, + cargoArtifacts, + rustSrc, + rustBuildInputs, +}: +craneLib.mkCargoDerivation ( + { + inherit cargoArtifacts; + pname = "generated-bindings-check"; + version = "0.1.0"; + src = rustSrc; + + SQLX_OFFLINE = "true"; + # Override crane's default of "false" to match the behavior developers get + # in their shell. TypeId ordering depends on incremental compilation state. + CARGO_BUILD_INCREMENTAL = "true"; + + buildPhaseCargoCommand = '' + cargo run -p backend -- generate-bindings + ''; + + checkPhaseCargoCommand = '' + if ! diff -ru packages/backend/pkg.orig packages/backend/pkg --exclude node_modules; then + echo "generate-bindings produced changes to packages/backend/pkg/." + echo "Please run 'cargo run -p backend -- generate-bindings' and commit the result." + exit 1 + fi + ''; + + installPhaseCommand = '' + mkdir -p $out + ''; + + doCheck = true; + + # Save a copy of the original before building so we can diff afterwards + preBuild = '' + cp -r packages/backend/pkg packages/backend/pkg.orig + ''; + } + // rustBuildInputs +) diff --git a/infrastructure/checks/npm-checks.nix b/infrastructure/checks/npm-checks.nix new file mode 100644 index 000000000..ee82846a0 --- /dev/null +++ b/infrastructure/checks/npm-checks.nix @@ -0,0 +1,42 @@ +{ + pkgs, + lib, + pnpmDeps, + biome, +}: +pkgs.stdenv.mkDerivation { + pname = "catcolab-npm-checks"; + version = "0.0.0"; + + src = lib.fileset.toSource { + root = ../..; + fileset = lib.fileset.unions [ + ../../.gitignore + ../../.npmrc + ../../biome.json + ../../pnpm-workspace.yaml + ../../pnpm-lock.yaml + ../../packages/frontend + ../../packages/ui-components + ]; + }; + + nativeBuildInputs = [ + pkgs.pnpm.configHook + biome + ]; + buildInputs = [ pkgs.nodejs_24 ]; + + inherit pnpmDeps; + + BIOME_BINARY = lib.getExe biome; + + buildPhase = '' + pnpm --filter "./packages/*" run ci + ''; + + installPhase = '' + mkdir -p $out + echo "npm checks passed" > $out/result.txt + ''; +} diff --git a/infrastructure/checks/rust-clippy.nix b/infrastructure/checks/rust-clippy.nix new file mode 100644 index 000000000..2b2b7fd38 --- /dev/null +++ b/infrastructure/checks/rust-clippy.nix @@ -0,0 +1,14 @@ +{ + craneLib, + cargoArtifacts, + rustSrc, + rustBuildInputs, +}: +craneLib.cargoClippy ( + { + inherit cargoArtifacts; + src = rustSrc; + cargoClippyExtraArgs = "--all-targets --all-features -- --deny warnings"; + } + // rustBuildInputs +) diff --git a/infrastructure/checks/rust-fmt.nix b/infrastructure/checks/rust-fmt.nix new file mode 100644 index 000000000..6b75ec075 --- /dev/null +++ b/infrastructure/checks/rust-fmt.nix @@ -0,0 +1,4 @@ +{ craneLibNightly }: +craneLibNightly.cargoFmt { + src = craneLibNightly.cleanCargoSource ../..; +} diff --git a/infrastructure/checks/rust-tests.nix b/infrastructure/checks/rust-tests.nix new file mode 100644 index 000000000..367c62b3f --- /dev/null +++ b/infrastructure/checks/rust-tests.nix @@ -0,0 +1,14 @@ +{ + craneLib, + cargoArtifacts, + rustSrc, + rustBuildInputs, +}: +craneLib.cargoTest ( + { + inherit cargoArtifacts; + src = rustSrc; + cargoTestExtraArgs = "--all-features"; + } + // rustBuildInputs +) diff --git a/infrastructure/devshell.nix b/infrastructure/devshell.nix new file mode 100644 index 000000000..753f8d53f --- /dev/null +++ b/infrastructure/devshell.nix @@ -0,0 +1,82 @@ +{ + nixpkgsFor, + inputs, + rustToolchainFor, +}: +{ + devShellForSystem = + system: + let + pkgs = nixpkgsFor system; + rustToolchain = rustToolchainFor system; + + # macOS-specific configurations for libraries + darwinDeps = + if pkgs.stdenv.isDarwin then + [ + pkgs.libiconv + ] + else + [ ]; + + nightlyRustfmt = inputs.fenix.packages.${system}.latest.rustfmt; + in + pkgs.mkShell { + name = "catcolab-devshell"; + RUSTFMT = "${nightlyRustfmt}/bin/rustfmt"; + buildInputs = + with pkgs; + [ + darkhttpd + esbuild + lld + netcat + nodejs_24 + nix + openssl + pkg-config + pnpm + postgresql + python3 + python312Packages.ipykernel + python312Packages.jupyter-core + python312Packages.jupyter-server + python312Packages.requests + python312Packages.websocket-client + rustToolchain + nightlyRustfmt + sqlx-cli + vscode-langservers-extracted + wasm-bindgen-cli + wasm-pack + ] + ++ darwinDeps + ++ [ + inputs.agenix.packages.${system}.agenix + inputs.deploy-rs.packages.${system}.default + (import ./biome.nix { inherit pkgs system; }) + ]; + + # macOS-specific environment variables for OpenSSL and pkg-config + shellHook = '' + ${ + if pkgs.stdenv.isDarwin then + '' + export OPENSSL_DIR=${pkgs.openssl.dev} + export OPENSSL_LIB_DIR=${pkgs.openssl.out}/lib + export OPENSSL_INCLUDE_DIR=${pkgs.openssl.dev}/include + export PKG_CONFIG_PATH=${pkgs.openssl.dev}/lib/pkgconfig:$PKG_CONFIG_PATH + '' + else + "" + } + + export PATH=$PWD/infrastructure/scripts:$PATH + + # Load DATABASE_URL into the environment + if [ -f packages/backend/.env ]; then + export $(grep -v '^#' packages/backend/.env | xargs) + fi + ''; + }; +} diff --git a/infrastructure/generated-bindings-check.nix b/infrastructure/generated-bindings-check.nix deleted file mode 100644 index 17942f521..000000000 --- a/infrastructure/generated-bindings-check.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - craneLib, - cargoArtifacts, - pkgs, -}: -craneLib.mkCargoDerivation { - inherit cargoArtifacts; - - pname = "generated-bindings-check"; - version = "0.1.0"; - - nativeBuildInputs = [ - pkgs.pkg-config - ]; - - buildInputs = [ - pkgs.openssl - ]; - - src = pkgs.lib.fileset.toSource { - root = ../.; - fileset = pkgs.lib.fileset.unions [ - ../Cargo.toml - ../Cargo.lock - (craneLib.fileset.commonCargoSources ../packages/backend) - (craneLib.fileset.commonCargoSources ../packages/migrator) - (craneLib.fileset.commonCargoSources ../packages/notebook-types) - ../packages/backend/.sqlx - ../packages/backend/pkg - ]; - }; - - SQLX_OFFLINE = "true"; - # Override crane's default of "false" to match the behavior developers get - # in their shell. TypeId ordering depends on incremental compilation state. - CARGO_BUILD_INCREMENTAL = "true"; - - buildPhaseCargoCommand = '' - cargo run -p backend -- generate-bindings - ''; - - checkPhaseCargoCommand = '' - if ! diff -ru packages/backend/pkg.orig packages/backend/pkg --exclude node_modules; then - echo "generate-bindings produced changes to packages/backend/pkg/." - echo "Please run 'cargo run -p backend -- generate-bindings' and commit the result." - exit 1 - fi - ''; - - installPhaseCommand = '' - mkdir -p $out - ''; - - doCheck = true; - - # Save a copy of the original before building so we can diff afterwards - preBuild = '' - cp -r packages/backend/pkg packages/backend/pkg.orig - ''; -} diff --git a/infrastructure/hosts/catcolab-next/default.nix b/infrastructure/hosts/catcolab-next/default.nix index 6ebb8c868..716051752 100644 --- a/infrastructure/hosts/catcolab-next/default.nix +++ b/infrastructure/hosts/catcolab-next/default.nix @@ -33,6 +33,9 @@ in backend = { port = 8000; hostname = "backend-next.catcolab.org"; + serveFrontend = true; + serveSite = true; + siteMode = "staging"; }; environmentFile = config.age.secrets.catcolabSecrets.path; host = { diff --git a/infrastructure/hosts/catcolab/default.nix b/infrastructure/hosts/catcolab/default.nix index d691e52e8..5b7f414a5 100644 --- a/infrastructure/hosts/catcolab/default.nix +++ b/infrastructure/hosts/catcolab/default.nix @@ -31,6 +31,9 @@ in backend = { port = 8000; hostname = "backend.catcolab.org"; + serveFrontend = true; + serveSite = true; + siteMode = "production"; }; environmentFile = config.age.secrets.catcolabSecrets.path; host = { diff --git a/infrastructure/hosts/default.nix b/infrastructure/hosts/default.nix new file mode 100644 index 000000000..a10942272 --- /dev/null +++ b/infrastructure/hosts/default.nix @@ -0,0 +1,101 @@ +{ + nixpkgs, + deploy-rs, + inputs, + self, + pkgsLinux, + linuxSystem, + rustToolchainLinux, +}: +let + # NOTE: this is not currently used, but was painful to build and might be useful in the future. + # Wraps the typical `deploy-rs.lib.${linuxSystem}.activate.nixos` activation function + # with a custom script that can run additional health checks. The script runs on the remote host + # and if it fails the whole deployment will fail. + # use: + # `deploy.nodes.${host}.profiles.system.path = healthcheckWrapper self.nixosConfigurations.host;` + healthcheckWrapper = + nixosConfiguration: + let + defaultNixos = deploy-rs.lib.${linuxSystem}.activate.nixos nixosConfiguration; + + healthcheckWrapperScript = pkgsLinux.writeShellScriptBin "healthcheck-wrapper-script" '' + PROFILE=${defaultNixos} + # insert healthchecks + ${defaultNixos}/deploy-rs-activate + ''; + in + deploy-rs.lib.${linuxSystem}.activate.custom healthcheckWrapperScript + "./bin/healthcheck-wrapper-script"; + + nixosConfigurations = { + catcolab = nixpkgs.lib.nixosSystem { + specialArgs = { + inherit inputs self; + rustToolchain = rustToolchainLinux; + }; + system = linuxSystem; + modules = [ + ./catcolab + ]; + pkgs = pkgsLinux; + }; + catcolab-next = nixpkgs.lib.nixosSystem { + specialArgs = { + inherit inputs self; + rustToolchain = rustToolchainLinux; + }; + system = linuxSystem; + modules = [ + ./catcolab-next + ]; + pkgs = pkgsLinux; + }; + + catcolab-vm = nixpkgs.lib.nixosSystem { + system = linuxSystem; + modules = [ + ./catcolab-vm + ]; + specialArgs = { + inherit inputs self; + rustToolchain = rustToolchainLinux; + }; + }; + }; +in +{ + inherit nixosConfigurations; + + deployNodes = { + catcolab = { + hostname = "backend.catcolab.org"; + profiles.system = { + sshUser = "catcolab"; + user = "root"; + path = deploy-rs.lib.${linuxSystem}.activate.nixos nixosConfigurations.catcolab; + }; + }; + catcolab-next = { + hostname = "backend-next.catcolab.org"; + profiles.system = { + sshUser = "catcolab"; + user = "root"; + path = deploy-rs.lib.${linuxSystem}.activate.nixos nixosConfigurations.catcolab-next; + }; + }; + catcolab-vm = { + hostname = "localhost"; + fastConnection = true; + profiles.system = { + sshOpts = [ + "-p" + "2221" + ]; + sshUser = "catcolab"; + path = deploy-rs.lib.${linuxSystem}.activate.nixos nixosConfigurations.catcolab-vm; + user = "root"; + }; + }; + }; +} diff --git a/infrastructure/modules/catcolab/services.nix b/infrastructure/modules/catcolab/services.nix index bce48be01..71d117981 100644 --- a/infrastructure/modules/catcolab/services.nix +++ b/infrastructure/modules/catcolab/services.nix @@ -9,6 +9,7 @@ let cfg = config.catcolab; frontendPkg = self.packages.${pkgs.system}.frontend; + sitePkg = mode: self.packages.${pkgs.system}."site-${mode}"; backendPkg = self.packages.${pkgs.system}.backend; backendPortStr = builtins.toString cfg.backend.port; @@ -58,6 +59,12 @@ with lib; description = "Hostname for the backend reverse proxy."; }; serveFrontend = lib.mkEnableOption "serving the frontend."; + serveSite = lib.mkEnableOption "serving the full site (frontend + docs). Requires serveFrontend."; + siteMode = mkOption { + type = types.str; + default = "staging"; + description = "Vite build mode for the site (development, staging, production)"; + }; }; environmentFile = mkOption { type = types.path; @@ -122,7 +129,9 @@ with lib; environment = lib.mkMerge [ { PORT = backendPortStr; } - (lib.mkIf cfg.backend.serveFrontend { SPA_DIR = "${frontendPkg}"; }) + (lib.mkIf cfg.backend.serveFrontend { + SPA_DIR = if cfg.backend.serveSite then "${sitePkg cfg.backend.siteMode}" else "${frontendPkg}"; + }) ]; serviceConfig = { diff --git a/infrastructure/netlify-preview.nix b/infrastructure/netlify-preview.nix new file mode 100644 index 000000000..b9edcfa7b --- /dev/null +++ b/infrastructure/netlify-preview.nix @@ -0,0 +1,21 @@ +# Wraps site-staging with Netlify-specific _redirects for PR preview deploys. +{ + pkgs, + self, +}: +let + site = self.packages.${pkgs.system}.site-staging; +in +pkgs.runCommand "catcolab-netlify-preview" { } '' + cp -rL ${site} $out + chmod -R u+w $out + + cat > $out/_redirects << 'EOF' +/dev/rust /dev/rust/catlog +/dev/core /dev/rust/catlog +/dev/catcolab_backend /dev/rust/backend +/math /math/index.xml +/maths /math/index.xml +/* /index.html 200 +EOF +'' diff --git a/infrastructure/packages.nix b/infrastructure/packages.nix new file mode 100644 index 000000000..f3d692bae --- /dev/null +++ b/infrastructure/packages.nix @@ -0,0 +1,133 @@ +{ + pkgsLinux, + craneLib, + cargoArtifacts, + rustSrc, + rustBuildInputs, + rustToolchainLinux, + pnpmDeps, + inputs, + self, + nixos-generators, +}: +let + # Helpers for frontend/site mode variants + mkFrontend = + mode: + (pkgsLinux.callPackage ../packages/frontend/default.nix { + inherit + inputs + rustToolchainLinux + self + mode + ; + }).package; + + mkSite = + mode: + pkgsLinux.callPackage ./site.nix { + inherit self mode; + pkgs = pkgsLinux; + }; +in +{ + catcolab-api = pkgsLinux.stdenv.mkDerivation { + pname = "catcolab-api"; + version = "0.1.0"; + + src = ../packages/backend/pkg; + + installPhase = '' + mkdir -p $out + cp -r * $out/ + ''; + }; + + backend = pkgsLinux.callPackage ../packages/backend/default.nix { + inherit craneLib cargoArtifacts; + pkgs = pkgsLinux; + }; + + migrator = pkgsLinux.callPackage ../packages/migrator/default.nix { + inherit craneLib cargoArtifacts; + pkgs = pkgsLinux; + }; + + catlog-wasm-browser = pkgsLinux.callPackage ../packages/catlog-wasm/default.nix { + inherit craneLib cargoArtifacts; + pkgs = pkgsLinux; + }; + + frontend = mkFrontend "development"; + + frontend-tests = + (pkgsLinux.callPackage ../packages/frontend/default.nix { + inherit inputs rustToolchainLinux self; + }).tests; + + rust-docs = import ./rust-docs.nix { + inherit + craneLib + cargoArtifacts + rustSrc + rustBuildInputs + ; + }; + + dev-docs = pkgsLinux.callPackage ../dev-docs/default.nix { inherit pnpmDeps; }; + math-docs = pkgsLinux.callPackage ../math-docs/default.nix { }; + ui-components-storybook = pkgsLinux.callPackage ../packages/ui-components/default.nix { + inherit pnpmDeps; + }; + + frontend-docs = + (pkgsLinux.callPackage ../packages/frontend/default.nix { + inherit inputs rustToolchainLinux self; + }).docs; + + frontend-development = self.packages.x86_64-linux.frontend; + frontend-staging = mkFrontend "staging"; + frontend-production = mkFrontend "production"; + + site-development = mkSite "development"; + site-staging = mkSite "staging"; + site-production = mkSite "production"; + + netlify-preview = pkgsLinux.callPackage ./netlify-preview.nix { + inherit self; + pkgs = pkgsLinux; + }; + + # VMs built with `nixos-rebuild build-vm` (like `nix build + # .#nixosConfigurations.catcolab-vm.config.system.build.vm`) are not the same + # as "traditional" VMs, which causes deploy-rs to fail when deploying to them. + # https://github.com/serokell/deploy-rs/issues/85#issuecomment-885782350 + # + # This is worked around by creating a full featured VM image. + # + # use: + # nix build .#catcolab-vm + # cp result/catcolab-vm.qcow2 catcolab-vm.qcow2 + # db-utils vm start + # deploy -s .#catcolab-vm + catcolab-vm = pkgsLinux.stdenv.mkDerivation { + name = "catcolab-vm"; + src = nixos-generators.nixosGenerate { + system = "x86_64-linux"; + format = "qcow"; + + modules = [ + ./hosts/catcolab-vm + ]; + + specialArgs = { + inherit inputs self; + rustToolchain = rustToolchainLinux; + }; + }; + installPhase = '' + mkdir -p $out + cp $src/nixos.qcow2 $out/catcolab-vm.qcow2 + ''; + }; +} diff --git a/infrastructure/pnpm-deps.nix b/infrastructure/pnpm-deps.nix new file mode 100644 index 000000000..93e40ac31 --- /dev/null +++ b/infrastructure/pnpm-deps.nix @@ -0,0 +1,26 @@ +{ pkgsLinux }: +pkgsLinux.fetchPnpmDeps { + hash = "sha256-GmumQtLYHqYci2UtrZQ5VK0X398R09b0MgnYkHsgiKw="; + pname = "catcolab-pnpm-deps"; + fetcherVersion = 2; + src = pkgsLinux.lib.fileset.toSource { + root = ../.; + fileset = pkgsLinux.lib.fileset.unions [ + ../.npmrc + ../pnpm-workspace.yaml + ../pnpm-lock.yaml + ../dev-docs/package.json + ../dev-docs/pnpm-lock.yaml + ../packages/frontend/package.json + ../packages/frontend/pnpm-lock.yaml + ../packages/ui-components/package.json + ../packages/ui-components/pnpm-lock.yaml + ../packages/notebook-types/package.json + ../packages/notebook-types/pnpm-lock.yaml + ../packages/backend/pkg/package.json + ../packages/backend/pkg/pnpm-lock.yaml + ../packages/patchwork/package.json + ../packages/patchwork/pnpm-lock.yaml + ]; + }; +} diff --git a/infrastructure/rust-docs.nix b/infrastructure/rust-docs.nix index 9d20341d1..97cddae1d 100644 --- a/infrastructure/rust-docs.nix +++ b/infrastructure/rust-docs.nix @@ -1,35 +1,16 @@ { craneLib, cargoArtifacts, - pkgs, + rustSrc, + rustBuildInputs, checkMode ? false, }: -craneLib.cargoDoc { - inherit cargoArtifacts; - - cargoExtraArgs = "--all-features --workspace --exclude migrator"; - - RUSTDOCFLAGS = if checkMode then "--deny warnings" else ""; - - nativeBuildInputs = [ - pkgs.pkg-config - ]; - - buildInputs = [ - pkgs.openssl - ]; - - src = pkgs.lib.fileset.toSource { - root = ../.; - fileset = pkgs.lib.fileset.unions [ - ../Cargo.toml - ../Cargo.lock - (craneLib.fileset.commonCargoSources ../packages/backend) - (craneLib.fileset.commonCargoSources ../packages/catlog) - (craneLib.fileset.commonCargoSources ../packages/catlog-wasm) - (craneLib.fileset.commonCargoSources ../packages/migrator) - (craneLib.fileset.commonCargoSources ../packages/notebook-types) - ../packages/backend/.sqlx - ]; - }; -} +craneLib.cargoDoc ( + { + inherit cargoArtifacts; + src = rustSrc; + cargoExtraArgs = "--all-features --workspace --exclude migrator"; + RUSTDOCFLAGS = if checkMode then "--deny warnings" else ""; + } + // rustBuildInputs +) diff --git a/infrastructure/rust.nix b/infrastructure/rust.nix new file mode 100644 index 000000000..869c47739 --- /dev/null +++ b/infrastructure/rust.nix @@ -0,0 +1,89 @@ +{ + pkgsLinux, + fenix, + crane, + linuxSystem, +}: +let + rustToolchainFor = + system: + fenix.packages.${system}.fromToolchainFile { + file = ../rust-toolchain.toml; + sha256 = "sha256-vra6TkHITpwRyA5oBKAHSX0Mi6CBDNQD+ryPSpxFsfg="; + }; + + rustToolchainLinux = rustToolchainFor linuxSystem; + + craneLib = (crane.mkLib pkgsLinux).overrideToolchain rustToolchainLinux; + + rustToolchainNightly = fenix.packages.${linuxSystem}.latest.withComponents [ + "cargo" + "rustfmt" + ]; + craneLibNightly = (crane.mkLib pkgsLinux).overrideToolchain rustToolchainNightly; + + rustBuildInputs = { + nativeBuildInputs = [ pkgsLinux.pkg-config ]; + buildInputs = [ pkgsLinux.openssl ]; + }; + + cargoArtifacts = craneLib.buildDepsOnly ( + { + src = craneLib.cleanCargoSource ../.; + strictDeps = true; + } + // rustBuildInputs + ); + + # Shared Rust source filesets + rustSrcBase = pkgsLinux.lib.fileset.unions [ + ../Cargo.toml + ../Cargo.lock + (craneLib.fileset.commonCargoSources ../packages/backend) + (craneLib.fileset.commonCargoSources ../packages/catlog) + (craneLib.fileset.commonCargoSources ../packages/catlog-wasm) + (craneLib.fileset.commonCargoSources ../packages/migrator) + (craneLib.fileset.commonCargoSources ../packages/notebook-types) + ../packages/backend/.sqlx + ]; + + rustSrc = pkgsLinux.lib.fileset.toSource { + root = ../.; + fileset = rustSrcBase; + }; + + rustSrcWithExamples = pkgsLinux.lib.fileset.toSource { + root = ../.; + fileset = pkgsLinux.lib.fileset.unions [ + rustSrcBase + ../packages/catlog/examples + ../packages/notebook-types/examples + ]; + }; + + rustSrcBindings = pkgsLinux.lib.fileset.toSource { + root = ../.; + fileset = pkgsLinux.lib.fileset.unions [ + ../Cargo.toml + ../Cargo.lock + (craneLib.fileset.commonCargoSources ../packages/backend) + (craneLib.fileset.commonCargoSources ../packages/migrator) + (craneLib.fileset.commonCargoSources ../packages/notebook-types) + ../packages/backend/.sqlx + ../packages/backend/pkg + ]; + }; +in +{ + inherit + rustToolchainFor + rustToolchainLinux + craneLib + craneLibNightly + cargoArtifacts + rustSrc + rustSrcWithExamples + rustSrcBindings + rustBuildInputs + ; +} diff --git a/infrastructure/site.nix b/infrastructure/site.nix new file mode 100644 index 000000000..c91261e50 --- /dev/null +++ b/infrastructure/site.nix @@ -0,0 +1,39 @@ +{ + pkgs, + self, + mode ? "staging", +}: +let + frontend = self.packages.${pkgs.system}."frontend-${mode}"; + devDocs = self.packages.${pkgs.system}.dev-docs; + rustDocs = self.packages.${pkgs.system}.rust-docs; + frontendDocs = self.packages.${pkgs.system}.frontend-docs; + mathDocs = self.packages.${pkgs.system}.math-docs; + uiComponents = self.packages.${pkgs.system}.ui-components-storybook; +in +pkgs.stdenv.mkDerivation { + pname = "catcolab-site-${mode}"; + version = "0.1.0"; + dontUnpack = true; + + installPhase = '' + mkdir -p $out + + # Frontend as root + cp -r ${frontend}/* $out/ + + # Dev docs + mkdir -p $out/dev + cp -r ${devDocs}/* $out/dev/ + mkdir -p $out/dev/rust + cp -r ${rustDocs}/share/doc/* $out/dev/rust/ + mkdir -p $out/dev/frontend + cp -r ${frontendDocs}/* $out/dev/frontend/ + mkdir -p $out/dev/ui-components + cp -r ${uiComponents}/* $out/dev/ui-components/ + + # Math docs + mkdir -p $out/math + cp -r ${mathDocs}/* $out/math/ + ''; +} diff --git a/math-docs/default.nix b/math-docs/default.nix new file mode 100644 index 000000000..6db67a262 --- /dev/null +++ b/math-docs/default.nix @@ -0,0 +1,51 @@ +{ + pkgs, + lib, +}: +let + forester = pkgs.fetchurl { + url = "http://forester-builds.s3-website.us-east-2.amazonaws.com/forester-4.3.1-linux-x86_64.tar.gz"; + sha256 = "8e03600808a36c35ee70e2522c63566021accff27d6053897d2ca020664e5b57"; + }; + texEnv = pkgs.texlive.combine { + inherit (pkgs.texlive) + scheme-small + dvisvgm + standalone + pgf + tikz-cd + amsmath + quiver + spath3 + ebproof + ; + }; +in +pkgs.stdenv.mkDerivation { + pname = "catcolab-math-docs"; + version = "0.1.0"; + + src = lib.fileset.toSource { + root = ./.; + fileset = lib.fileset.unions [ + ./trees + ./assets + ./theme + ./forest.toml + ]; + }; + + nativeBuildInputs = [ texEnv ]; + + buildPhase = '' + mkdir -p forester-bin + tar -xf ${forester} -C forester-bin + chmod +x forester-bin/forester + ./forester-bin/forester build + ''; + + installPhase = '' + mkdir -p $out + cp -r output/* $out/ + ''; +} diff --git a/math-docs/shell.nix b/math-docs/shell.nix index 47599f273..a098244e4 100644 --- a/math-docs/shell.nix +++ b/math-docs/shell.nix @@ -1,5 +1,7 @@ # shell.nix -{ pkgs ? import {} }: +{ + pkgs ? import { }, +}: pkgs.mkShell { # Swap β€˜git’ for whatever you actually need. diff --git a/packages/frontend/default.nix b/packages/frontend/default.nix index 54c74f6b8..cb05964d9 100644 --- a/packages/frontend/default.nix +++ b/packages/frontend/default.nix @@ -2,6 +2,7 @@ pkgs, self, lib, + mode ? "development", ... }: let @@ -71,8 +72,7 @@ let cp -r ${self.packages.x86_64-linux.catlog-wasm-browser}/* packages/catlog-wasm/dist/pkg-browser/ cd packages/frontend - # Build with development mode to use .env.development configuration - npm run build:nix -- --mode development + npm run build:nix -- --mode ${mode} cd - ''; @@ -165,7 +165,27 @@ let meta.mainProgram = "${name}-tests"; } ); + docs = pkgs.stdenv.mkDerivation ( + commonAttrs + // { + pname = "${name}-docs"; + + buildPhase = '' + mkdir -p packages/catlog-wasm/dist/pkg-browser + cp -r ${self.packages.x86_64-linux.catlog-wasm-browser}/* packages/catlog-wasm/dist/pkg-browser/ + + cd packages/frontend + npx typedoc --entryPointStrategy expand ./src + cd - + ''; + + installPhase = '' + mkdir -p $out + cp -r packages/frontend/docs/* $out/ + ''; + } + ); in { - inherit package tests; + inherit package tests docs; } diff --git a/packages/frontend/netlify.toml b/packages/frontend/netlify.toml deleted file mode 100644 index 96738f3c0..000000000 --- a/packages/frontend/netlify.toml +++ /dev/null @@ -1,6 +0,0 @@ -[build] -publish = "dist/" -command = "rustup toolchain install stable && rustup target add wasm32-unknown-unknown && npm run build" - -[build.environment] -RUST_VERSION = "1.81.0" diff --git a/packages/ui-components/default.nix b/packages/ui-components/default.nix new file mode 100644 index 000000000..c890a995b --- /dev/null +++ b/packages/ui-components/default.nix @@ -0,0 +1,34 @@ +{ + pkgs, + lib, + pnpmDeps, +}: +pkgs.stdenv.mkDerivation { + pname = "catcolab-ui-components-storybook"; + version = "0.0.0"; + + src = lib.fileset.toSource { + root = ../../.; + fileset = lib.fileset.unions [ + ../../.npmrc + ../../pnpm-workspace.yaml + ../../pnpm-lock.yaml + ../../packages/ui-components + ]; + }; + + nativeBuildInputs = [ pkgs.pnpm.configHook ]; + buildInputs = [ pkgs.nodejs_24 ]; + + inherit pnpmDeps; + + buildPhase = '' + cd packages/ui-components + npm run build + ''; + + installPhase = '' + mkdir -p $out + cp -r storybook-static/* $out/ + ''; +}