tests.yml

name: Tests
on:
  push:
    branches:
      - main
      - stable-*
  pull_request:

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  code-tests:
    name: Code
    runs-on: ubuntu-24.04
    strategy:
      fail-fast: false
      matrix:
        go:
          - oldstable
          - stable
          - tip
    steps:
      - name: Checkout
        uses: actions/checkout@v7
        with:
          # Differential ShellCheck requires full git history
          fetch-depth: 0

      - name: Dependency Review
        uses: actions/dependency-review-action@v5
        if: github.event_name == 'pull_request'
        with:
          allow-ghsas: GHSA-4p9m-8gc4-rw2h

      - id: ShellCheck
        name: Differential ShellCheck
        uses: redhat-plumbers-in-action/differential-shellcheck@v5
        env:
          SHELLCHECK_OPTS: --shell sh
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          exclude-path: internal/server/instance/drivers/agent-loader/rc.d/incus-agent
        if: github.event_name == 'pull_request' && matrix.go == 'stable'

      - name: Upload artifact with ShellCheck defects in SARIF format
        uses: actions/upload-artifact@v7
        with:
          name: Differential ShellCheck SARIF
          path: ${{ steps.ShellCheck.outputs.sarif }}
        if: github.event_name == 'pull_request' && matrix.go == 'stable'

      - name: Install Go (${{ matrix.go }})
        uses: actions/setup-go@v7
        with:
          go-version: ${{ matrix.go }}
        if: matrix.go != 'tip'

      - name: Install Go (stable)
        uses: actions/setup-go@v7
        with:
          go-version: stable
        if: matrix.go == 'tip'

      - name: Install Go (tip)
        run: |
          go install golang.org/dl/gotip@latest
          gotip download
          ~/sdk/gotip/bin/go version
          echo "PATH=$HOME/go/bin:$HOME/sdk/gotip/bin/:$PATH" >> $GITHUB_ENV
        if: matrix.go == 'tip'

      - name: Install dependencies
        run: |
          sudo apt-get update
          sudo apt-get install --no-install-recommends -y \
            curl \
            gettext \
            git \
            libacl1-dev \
            libcap-dev \
            libdbus-1-dev \
            libcowsql-dev \
            liblxc-dev \
            lxc-templates \
            libseccomp-dev \
            libselinux-dev \
            libsqlite3-dev \
            libtool \
            libudev-dev \
            make \
            pipx \
            pkg-config \
            shellcheck

          # With pipx >= 1.5.0, we could use pipx --global instead.
          PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin \
            pipx install codespell flake8

      - name: Fix repository permissions
        run: |
          sudo chown -R runner:docker .

      - name: Check compatible min Go version
        run: |
          go mod tidy

      - name: Download go dependencies
        run: |
          go mod download

      - name: Run Incus build
        run: |
          make

      - name: Run static analysis
        env:
          GITHUB_BEFORE: ${{ github.event.before }}
        run: |
          make static-analysis

      - name: Unit tests (all)
        run: |
          sudo --preserve-env=CGO_CFLAGS,CGO_LDFLAGS,CGO_LDFLAGS_ALLOW,LD_LIBRARY_PATH LD_LIBRARY_PATH=${LD_LIBRARY_PATH} env "PATH=${PATH}" go test ./...

  system-tests:
    name: System
    strategy:
      fail-fast: false
      matrix:
        go:
          - oldstable
          - stable
          - tip
        suite:
          - cluster
          - standalone_core
          - standalone_container
          - standalone_network
          - standalone_storage
        backend:
          - dir
        os:
          - ubuntu-24.04
          - ubuntu-24.04-arm
        include:
          # Run standalone storage tests on all storage drivers but only on Ubuntu 24.04 with stable Go
          - os: ubuntu-24.04
            backend: btrfs
            go: stable
            suite: standalone_storage
          - os: ubuntu-24.04
            backend: ceph
            go: stable
            suite: standalone_storage
          - os: ubuntu-24.04
            backend: linstor
            go: stable
            suite: standalone_storage
          - os: ubuntu-24.04
            backend: lvm
            go: stable
            suite: standalone_storage
          - os: ubuntu-24.04
            backend: random
            go: stable
            suite: standalone_storage
          - os: ubuntu-24.04
            backend: zfs
            go: stable
            suite: standalone_storage

          # Run cluster tests on all storage drivers but only on Ubuntu 24.04 with stable Go
          - os: ubuntu-24.04
            backend: btrfs
            go: stable
            suite: cluster
          - os: ubuntu-24.04
            backend: ceph
            go: stable
            suite: cluster
          - os: ubuntu-24.04
            backend: linstor
            go: stable
            suite: cluster
          - os: ubuntu-24.04
            backend: lvm
            go: stable
            suite: cluster
          - os: ubuntu-24.04
            backend: random
            go: stable
            suite: cluster
          - os: ubuntu-24.04
            backend: zfs
            go: stable
            suite: cluster
    runs-on: ${{ matrix.os }}

    steps:
      - name: Performance tuning
        run: |
          set -eux
          # optimize ext4 FSes for performance, not reliability
          for fs in $(findmnt --noheading --type ext4 --list --uniq | awk '{print $1}'); do
            # nombcache and data=writeback cannot be changed on remount
            sudo mount -o remount,noatime,barrier=0,commit=6000 "${fs}" || true
          done

          # disable dpkg from calling sync()
          echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io

      - name: Reclaim some space
        run: |
          set -eux

          sudo snap remove lxd --purge
          # Purge older snap revisions that are disabled/superseded by newer revisions of the same snap
          snap list --all | while read -r name _ rev _ _ notes _; do
            [ "${notes}" = "disabled" ] && snap remove "${name}" --revision "${rev}" --purge
          done || true

          # This was inspired from https://github.com/easimon/maximize-build-space
          df -h /
          # dotnet
          sudo rm -rf /usr/share/dotnet
          # android
          sudo rm -rf /usr/local/lib/android
          # haskell
          sudo rm -rf /opt/ghc
          df -h /

      - name: Remove docker
        run: |
          set -eux
          sudo apt-get autopurge -y moby-containerd docker uidmap
          sudo ip link delete docker0
          sudo nft flush ruleset

      - name: Remove pre-installed Java
        run: |
          set -eux
          sudo apt-get update
          sudo apt-get remove --yes --purge temurin.*

      - name: Checkout
        uses: actions/checkout@v7

      - name: Install Go (${{ matrix.go }})
        uses: actions/setup-go@v7
        with:
          go-version: ${{ matrix.go }}
        if: matrix.go != 'tip'

      - name: Install Go (stable)
        uses: actions/setup-go@v7
        with:
          go-version: stable
        if: matrix.go == 'tip'

      - name: Install Go (tip)
        run: |
          go install golang.org/dl/gotip@latest
          gotip download
          ~/sdk/gotip/bin/go version
          echo "PATH=$HOME/go/bin:$HOME/sdk/gotip/bin/:$PATH" >> $GITHUB_ENV
        if: matrix.go == 'tip'

      - name: Install dependencies
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          set -x

          # Configure ppa:ubuntu-lxc/daily directly
          # (apt-add-repository relies on the Launchpad API which is unreliable).
          sudo install -d -m 0755 /etc/apt/keyrings
          codename="$(lsb_release -cs)"

          curl -fsSL "https://keyserver.ubuntu.com/pks/lookup?op=get&options=mr&search=0xE9C00C1B1B59A86C2CFEE6990CE27B8C4122B4B7" | sudo tee /etc/apt/keyrings/ubuntu-lxc-daily.asc > /dev/null
          sudo tee /etc/apt/sources.list.d/ubuntu-lxc-daily.sources > /dev/null <<EOF
          Types: deb
          URIs: https://ppa.launchpadcontent.net/ubuntu-lxc/daily/ubuntu
          Suites: ${codename}
          Components: main
          Signed-By: /etc/apt/keyrings/ubuntu-lxc-daily.asc
          EOF

          sudo apt-get update

          sudo systemctl mask lxc.service lxc-net.service

          sudo apt-get install --no-install-recommends -y \
            apparmor \
            autoconf \
            automake \
            bsdextrautils \
            bzip2 \
            curl \
            dosfstools \
            git \
            libacl1-dev \
            libcap-dev \
            libdbus-1-dev \
            libelf-dev \
            liblxc-dev \
            liblz4-dev \
            libseccomp-dev \
            libselinux-dev \
            libsqlite3-dev \
            libtool \
            libudev-dev \
            libuv1-dev \
            linux-modules-extra-$(uname -r) \
            llvm \
            make \
            pkg-config\
            acl \
            attr \
            bind9-dnsutils \
            btrfs-progs \
            busybox-static \
            dnsmasq-base \
            easy-rsa \
            gettext \
            jq \
            lxc-utils \
            lvm2 \
            nftables \
            quota \
            rsync \
            s3cmd \
            socat \
            sqlite3 \
            squashfs-tools \
            tar \
            tcl \
            thin-provisioning-tools \
            uuid-runtime \
            xfsprogs \
            xz-utils \
            zfsutils-linux

          # Make sure all AppArmor profiles are loaded.
          sudo systemctl start apparmor

          # Reclaim some space
          sudo apt-get clean

          github_api_curl() {
            curl -sSfL \
              -H "Accept: application/vnd.github+json" \
              -H "Authorization: Bearer ${GITHUB_TOKEN}" \
              -H "X-GitHub-Api-Version: 2026-03-10" \
              "$@"
          }

          # Download latest release of openfga server.
          mkdir -p "$(go env GOPATH)/bin/"
          github_api_curl https://api.github.com/repos/openfga/openfga/releases/latest | jq -r ".assets | .[] | .browser_download_url | select(. | test(\"_linux_$(dpkg --print-architecture).tar.gz$\"))" | xargs -I {} curl -sSfL {} -o openfga.tar.gz
          tar -xzf openfga.tar.gz -C "$(go env GOPATH)/bin/"

          # Download latest release of openfga cli.
          github_api_curl https://api.github.com/repos/openfga/cli/releases/latest | jq -r ".assets | .[] | .browser_download_url | select(. | test(\"_linux_$(dpkg --print-architecture).tar.gz$\"))" | xargs -I {} curl -sSfL {} -o fga.tar.gz
          tar -xzf fga.tar.gz -C "$(go env GOPATH)/bin/"

      - name: Download go dependencies
        run: |
          go mod download

      - name: Build cowsql and raft
        run: |
          set -x
          make deps

          raft_path="$(go env GOPATH)/deps/raft"
          cowsql_path="$(go env GOPATH)/deps/cowsql"
          {
            echo "CGO_CFLAGS=-I${raft_path}/include/ -I${cowsql_path}/include/"
            echo "CGO_LDFLAGS=-L${raft_path}/.libs -L${cowsql_path}/.libs/"
            echo "LD_LIBRARY_PATH=${raft_path}/.libs/:${cowsql_path}/.libs/"
            echo "CGO_LDFLAGS_ALLOW=(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"
          } >> "$GITHUB_ENV"

      - name: Run Incus build
        run: |
          make

      - name: Setup scratch space
        if: "matrix.backend == 'ceph' || matrix.backend == 'linstor'"
        run: |
          set -eux

          if mountpoint -q /mnt; then
              [ -e /mnt/swapfile ] && sudo swapoff /mnt/swapfile
              block_path="$(findmnt --noheadings --output SOURCE --target /mnt | sed 's/[0-9]\+$//')"
              sudo umount /mnt
              sudo wipefs -a "${block_path}"
              sudo ln -s "${block_path}" "/dev/scratch"
          else
              sudo truncate -s 20G /scratch.img
              block_path="$(sudo losetup --show -f /scratch.img)"
              sudo ln -s "${block_path}" "/dev/scratch"
          fi

      - name: Setup MicroCeph
        if: matrix.backend == 'ceph'
        run: |
          set -x

          sudo apt-get install --no-install-recommends -y snapd
          sudo snap install microceph --channel=quincy/stable
          sudo apt-get install --no-install-recommends -y ceph-common
          sudo microceph cluster bootstrap
          sudo microceph.ceph config set global osd_pool_default_size 1
          sudo microceph.ceph config set global mon_allow_pool_delete true
          sudo microceph.ceph config set global osd_memory_target 939524096
          sudo microceph.ceph osd crush rule rm replicated_rule
          sudo microceph.ceph osd crush rule create-replicated replicated default osd
          for flag in nosnaptrim noscrub nobackfill norebalance norecover noscrub nodeep-scrub; do
              sudo microceph.ceph osd set $flag
          done

          # Repurpose the ephemeral disk for ceph OSD.
          sudo microceph disk add --wipe "$(readlink -f /dev/scratch)"
          sudo rm -rf /etc/ceph
          sudo ln -s /var/snap/microceph/current/conf/ /etc/ceph
          sudo microceph enable rgw
          sudo microceph.ceph osd pool create cephfs_meta 32
          sudo microceph.ceph osd pool create cephfs_data 32
          sudo microceph.ceph fs new cephfs cephfs_meta cephfs_data
          sudo microceph.ceph fs ls
          sleep 30
          sudo microceph.ceph status

          # Wait until there are no more "unkowns" pgs
          for _ in $(seq 60); do
            if sudo microceph.ceph pg stat | grep -wF unknown; then
              sleep 1
            else
              break
            fi
          done
          sudo microceph.ceph status
          sudo rm -f /snap/bin/rbd

      - name: Setup LINSTOR
        if: matrix.backend == 'linstor'
        run: |
          set -x

          # Configure ppa:linbit/linbit-drbd9-stack directly
          # (apt-add-repository relies on the Launchpad API which is unreliable).
          sudo install -d -m 0755 /etc/apt/keyrings
          curl -fsSL "https://keyserver.ubuntu.com/pks/lookup?op=get&options=mr&search=0xCC1B5A793C04BB3905AD837734893610CEAA9512" | sudo tee /etc/apt/keyrings/linbit-drbd9-stack.asc > /dev/null
          sudo tee /etc/apt/sources.list.d/linbit-drbd9-stack.sources > /dev/null <<EOF
          Types: deb
          URIs: https://ppa.launchpadcontent.net/linbit/linbit-drbd9-stack/ubuntu
          Suites: $(lsb_release -cs)
          Components: main
          Signed-By: /etc/apt/keyrings/linbit-drbd9-stack.asc
          EOF
          sudo apt-get update

          # Install everything required to compile DRBD and run LINSTOR tools.
          sudo apt-get install --no-install-recommends -y \
            drbd-dkms \
            linstor-client \
            linstor-controller \
            linstor-satellite \
            linux-headers-generic \
            python3-setuptools

          # Enable DRBD.
          sudo modprobe -r drbd
          sudo modprobe drbd

          # Get the runner IP.
          runner_ip="$(hostname -I | cut -d' ' -f1)"

          # Create a single local node.
          sudo linstor node create local "${runner_ip}" --node-type combined

          # Repurpose the ephemeral disk for LINSTOR physical storage.
          sudo linstor physical-storage create-device-pool --storage-pool incus --pool-name linstor-incus zfsthin local "$(readlink -f /dev/scratch)"

          # Update the runner env.
          echo "INCUS_LINSTOR_CLUSTER=${runner_ip}" >> "$GITHUB_ENV"

      - name: "Ensure offline mode (block image server)"
        run: |
          sudo nft add table inet filter
          sudo nft add chain 'inet filter output { type filter hook output priority 10 ; }'
          sudo nft add rule inet filter output ip daddr 45.45.148.8 reject
          sudo nft add rule inet filter output ip6 daddr 2602:fc62:a:1::8 reject

      - name: "Run system tests (${{ matrix.go }}, ${{ matrix.suite }}, ${{ matrix.backend }})"
        env:
          CGO_LDFLAGS_ALLOW: "(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"
          INCUS_CEPH_CLUSTER: "ceph"
          INCUS_CEPH_CEPHFS: "cephfs"
          INCUS_CEPH_CEPHOBJECT_RADOSGW: "http://127.0.0.1"
          INCUS_LINSTOR_LOCAL_SATELLITE: "local"
          INCUS_CONCURRENT: "1"
          INCUS_VERBOSE: "1"
          INCUS_OFFLINE: "1"
          INCUS_TMPFS: "1"
          INCUS_REQUIRED_TESTS: "test_storage_buckets"
        run: |
          chmod +x ~
          echo "root:1000000:1000000000" | sudo tee /etc/subuid /etc/subgid
          cd test
          sudo --preserve-env=PATH,GOPATH,GITHUB_ACTIONS,INCUS_VERBOSE,INCUS_BACKEND,INCUS_CEPH_CLUSTER,INCUS_CEPH_CEPHFS,INCUS_CEPH_CEPHOBJECT_RADOSGW,INCUS_LINSTOR_LOCAL_SATELLITE,INCUS_LINSTOR_CLUSTER,INCUS_OFFLINE,INCUS_SKIP_TESTS,INCUS_REQUIRED_TESTS, INCUS_BACKEND=${{ matrix.backend }} env LD_LIBRARY_PATH=${LD_LIBRARY_PATH} JAVA_HOME= ./main.sh ${{ matrix.suite }}

  ovn-tests:
    name: OVN
    runs-on: ubuntu-24.04
    strategy:
      fail-fast: false
      matrix:
        test:
          - basic
          - forward
          - load_balancer
          - peering
          - dhcp_reservation
          - nested_vlan
          - acl
          - l3only

    steps:
      - name: Performance tuning
        run: |
          set -eux
          # optimize ext4 FSes for performance, not reliability
          for fs in $(findmnt --noheading --type ext4 --list --uniq | awk '{print $1}'); do
            # nombcache and data=writeback cannot be changed on remount
            sudo mount -o remount,noatime,barrier=0,commit=6000 "${fs}" || true
          done

          # disable dpkg from calling sync()
          echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io

      - name: Remove LXD
        run: |
          set -eux
          sudo snap remove lxd --purge

      - name: Remove docker
        run: |
          set -eux
          sudo apt-get autopurge -y moby-containerd docker uidmap
          sudo ip link delete docker0
          sudo nft flush ruleset

      - name: Checkout
        uses: actions/checkout@v7

      - name: Install Go (stable)
        uses: actions/setup-go@v7
        with:
          go-version: stable

      - name: Install dependencies
        run: |
          set -x

          sudo install -d -m 0755 /etc/apt/keyrings
          codename="$(lsb_release -cs)"

          # Configure ppa:ubuntu-lxc/daily directly
          # (apt-add-repository relies on the Launchpad API which is unreliable).
          curl -fsSL "https://keyserver.ubuntu.com/pks/lookup?op=get&options=mr&search=0xE9C00C1B1B59A86C2CFEE6990CE27B8C4122B4B7" | sudo tee /etc/apt/keyrings/ubuntu-lxc-daily.asc > /dev/null
          sudo tee /etc/apt/sources.list.d/ubuntu-lxc-daily.sources > /dev/null <<EOF
          Types: deb
          URIs: https://ppa.launchpadcontent.net/ubuntu-lxc/daily/ubuntu
          Suites: ${codename}
          Components: main
          Signed-By: /etc/apt/keyrings/ubuntu-lxc-daily.asc
          EOF

          # Configure the Zabbly OVN stable repository.
          curl -fsSL https://pkgs.zabbly.com/key.asc | sudo tee /etc/apt/keyrings/zabbly.asc > /dev/null
          sudo tee /etc/apt/sources.list.d/zabbly-ovn-stable.sources > /dev/null <<EOF
          Types: deb
          URIs: https://pkgs.zabbly.com/ovn/stable
          Suites: ${codename}
          Components: main
          Architectures: $(dpkg --print-architecture)
          Signed-By: /etc/apt/keyrings/zabbly.asc
          EOF

          sudo apt-get update

          sudo systemctl mask lxc.service lxc-net.service

          sudo apt-get install --no-install-recommends -y \
            acl \
            apparmor \
            attr \
            autoconf \
            automake \
            bind9-dnsutils \
            curl \
            dnsmasq-base \
            gettext \
            git \
            jq \
            libacl1-dev \
            libcap-dev \
            libdbus-1-dev \
            liblxc-dev \
            liblz4-dev \
            libseccomp-dev \
            libselinux-dev \
            libsqlite3-dev \
            libtool \
            libudev-dev \
            libuv1-dev \
            linux-modules-extra-$(uname -r) \
            make \
            nftables \
            openvswitch-switch \
            ovn-central \
            ovn-host \
            pkg-config \
            socat \
            sqlite3 \
            squashfs-tools \
            uuid-runtime \
            xz-utils

          # Make sure all AppArmor profiles are loaded.
          sudo systemctl start apparmor

      - name: Configure OVN
        run: |
          set -eux
          sudo ovs-vsctl set open_vswitch . \
            external_ids:ovn-remote=unix:/var/run/ovn/ovnsb_db.sock \
            external_ids:ovn-encap-type=geneve \
            external_ids:ovn-encap-ip=127.0.0.1

      - name: Download go dependencies
        run: |
          go mod download

      - name: Build cowsql and raft
        run: |
          set -x
          make deps

          raft_path="$(go env GOPATH)/deps/raft"
          cowsql_path="$(go env GOPATH)/deps/cowsql"
          {
            echo "CGO_CFLAGS=-I${raft_path}/include/ -I${cowsql_path}/include/"
            echo "CGO_LDFLAGS=-L${raft_path}/.libs -L${cowsql_path}/.libs/"
            echo "LD_LIBRARY_PATH=${raft_path}/.libs/:${cowsql_path}/.libs/"
            echo "CGO_LDFLAGS_ALLOW=(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"
          } >> "$GITHUB_ENV"

      - name: Run Incus build
        run: |
          make

      - name: "Run OVN tests (${{ matrix.test }})"
        env:
          CGO_LDFLAGS_ALLOW: "(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"
          INCUS_VERBOSE: "1"
          INCUS_TMPFS: "1"
          INCUS_REQUIRED_TESTS: "network_ovn_${{ matrix.test }}"
        run: |
          chmod +x ~
          echo "root:1000000:1000000000" | sudo tee /etc/subuid /etc/subgid
          cd test
          sudo --preserve-env=PATH,GOPATH,GITHUB_ACTIONS,INCUS_VERBOSE,INCUS_TMPFS,INCUS_REQUIRED_TESTS env LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./main.sh network_ovn_${{ matrix.test }}

  extended-tests:
    name: Extended
    strategy:
      fail-fast: false
      matrix:
        test:
          - cgroup
          - interception
          - network_bridge_firewall
        os:
          - ubuntu-24.04
          - ubuntu-24.04-arm
    runs-on: ${{ matrix.os }}

    steps:
      - name: Performance tuning
        run: |
          set -eux
          # optimize ext4 FSes for performance, not reliability
          for fs in $(findmnt --noheading --type ext4 --list --uniq | awk '{print $1}'); do
            # nombcache and data=writeback cannot be changed on remount
            sudo mount -o remount,noatime,barrier=0,commit=6000 "${fs}" || true
          done

          # disable dpkg from calling sync()
          echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io

      - name: Remove LXD
        run: |
          set -eux
          sudo snap remove lxd --purge

      - name: Remove docker
        run: |
          set -eux
          sudo apt-get autopurge -y moby-containerd docker uidmap
          sudo ip link delete docker0
          sudo nft flush ruleset

      - name: Setup swap
        if: matrix.test == 'cgroup'
        run: |
          set -eux
          if [ "$(wc -l < /proc/swaps)" = "1" ]; then
            sudo fallocate -l 4G /SWAP.img
            sudo chmod 600 /SWAP.img
            sudo mkswap /SWAP.img
            sudo swapon /SWAP.img
          fi

      - name: Checkout
        uses: actions/checkout@v7

      - name: Install Go (stable)
        uses: actions/setup-go@v7
        with:
          go-version: stable

      - name: Install dependencies
        run: |
          set -x

          sudo install -d -m 0755 /etc/apt/keyrings
          codename="$(lsb_release -cs)"

          # Configure ppa:ubuntu-lxc/daily directly
          # (apt-add-repository relies on the Launchpad API which is unreliable).
          curl -fsSL "https://keyserver.ubuntu.com/pks/lookup?op=get&options=mr&search=0xE9C00C1B1B59A86C2CFEE6990CE27B8C4122B4B7" | sudo tee /etc/apt/keyrings/ubuntu-lxc-daily.asc > /dev/null
          sudo tee /etc/apt/sources.list.d/ubuntu-lxc-daily.sources > /dev/null <<EOF
          Types: deb
          URIs: https://ppa.launchpadcontent.net/ubuntu-lxc/daily/ubuntu
          Suites: ${codename}
          Components: main
          Signed-By: /etc/apt/keyrings/ubuntu-lxc-daily.asc
          EOF

          sudo apt-get update

          sudo systemctl mask lxc.service lxc-net.service

          sudo apt-get install --no-install-recommends -y \
            acl \
            apparmor \
            attr \
            autoconf \
            automake \
            bind9-dnsutils \
            btrfs-progs \
            curl \
            dnsmasq-base \
            gettext \
            git \
            iperf3 \
            jq \
            libacl1-dev \
            libcap-dev \
            libdbus-1-dev \
            liblxc-dev \
            liblz4-dev \
            libseccomp-dev \
            libselinux-dev \
            libsqlite3-dev \
            libtool \
            libudev-dev \
            libuv1-dev \
            linux-modules-extra-$(uname -r) \
            lvm2 \
            lxcfs \
            make \
            nftables \
            pkg-config \
            socat \
            sqlite3 \
            squashfs-tools \
            thin-provisioning-tools \
            uuid-runtime \
            xz-utils \
            zfsutils-linux

          # Make sure all AppArmor profiles are loaded.
          sudo systemctl start apparmor

      - name: Build LXCFS from main
        if: matrix.test == 'cgroup'
        run: |
          set -eux

          sudo apt-get install --no-install-recommends -y \
            libfuse3-dev \
            meson \
            ninja-build \
            python3-jinja2 \
            systemd-dev

          git clone --depth=1 https://github.com/lxc/lxcfs ~/lxcfs
          cd ~/lxcfs
          meson setup build --prefix=/usr -Ddocs=false
          ninja -C build
          sudo ninja -C build install

          sudo systemctl daemon-reload
          sudo systemctl restart lxcfs
          lxcfs --version

      - name: Download go dependencies
        run: |
          go mod download

      - name: Build cowsql and raft
        run: |
          set -x
          make deps

          raft_path="$(go env GOPATH)/deps/raft"
          cowsql_path="$(go env GOPATH)/deps/cowsql"
          {
            echo "CGO_CFLAGS=-I${raft_path}/include/ -I${cowsql_path}/include/"
            echo "CGO_LDFLAGS=-L${raft_path}/.libs -L${cowsql_path}/.libs/"
            echo "LD_LIBRARY_PATH=${raft_path}/.libs/:${cowsql_path}/.libs/"
            echo "CGO_LDFLAGS_ALLOW=(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"
          } >> "$GITHUB_ENV"

      - name: Run Incus build
        run: |
          make

      - name: "Run extended tests (${{ matrix.test }}, ${{ matrix.os }})"
        env:
          CGO_LDFLAGS_ALLOW: "(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"
          INCUS_VERBOSE: "1"
          INCUS_REQUIRED_TESTS: "${{ matrix.test }}"
        run: |
          chmod +x ~
          echo "root:1000000:1000000000" | sudo tee /etc/subuid /etc/subgid
          cd test
          sudo --preserve-env=PATH,GOPATH,GITHUB_ACTIONS,INCUS_VERBOSE,INCUS_REQUIRED_TESTS env LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./main.sh ${{ matrix.test }}

  vm-tests:
    name: VM
    strategy:
      fail-fast: false
      matrix:
        include:
          - test: cpu_vm
          - test: guestapi_vm
          - test: network_routed
          - test: nvram_vm
          - test: storage_disks_vm
          - test: storage_vm
            driver: dir
          - test: storage_vm
            driver: btrfs
          - test: storage_vm
            driver: lvm
          - test: storage_vm
            driver: lvm-thin
          - test: storage_vm
            driver: zfs
          - test: storage_vm
            driver: ceph
          - test: storage_vm
            driver: linstor
          - test: storage_volumes_vm
            driver: dir
          - test: storage_volumes_vm
            driver: btrfs
          - test: storage_volumes_vm
            driver: lvm
          - test: storage_volumes_vm
            driver: lvm-thin
          - test: storage_volumes_vm
            driver: zfs
          - test: storage_volumes_vm
            driver: ceph
          - test: storage_volumes_vm
            driver: linstor
    # Runs on ephemeral Debian VMs with nested virtualization (amd64 only).
    runs-on:
      - self-hosted
      - lxc-incus-test

    steps:
      - name: Checkout
        uses: actions/checkout@v7

      - name: Install Go (stable)
        uses: actions/setup-go@v7
        with:
          go-version: stable

      - name: Install dependencies
        run: |
          set -x

          sudo apt-get update
          sudo DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
            acl \
            attr \
            autoconf \
            automake \
            bind9-dnsutils \
            btrfs-progs \
            build-essential \
            busybox-static \
            curl \
            dnsmasq-base \
            dosfstools \
            genisoimage \
            gettext \
            git \
            iputils-ping \
            jq \
            libacl1-dev \
            libcap-dev \
            libdbus-1-dev \
            lxc-dev \
            liblz4-dev \
            libseccomp-dev \
            libselinux-dev \
            libsqlite3-dev \
            libtool \
            libudev-dev \
            libuv1-dev \
            lvm2 \
            make \
            nftables \
            ovmf \
            pkg-config \
            procps \
            qemu-system-modules-spice \
            qemu-system-x86 \
            qemu-utils \
            rsync \
            socat \
            sqlite3 \
            squashfs-tools \
            thin-provisioning-tools \
            uuid-runtime \
            virtiofsd \
            xfsprogs \
            xz-utils

          # Install ZFS
          sudo DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y "linux-headers-$(uname -r)"
          curl -sL https://pkgs.zabbly.com/get/zfs-stable | sudo DEBIAN_FRONTEND=noninteractive sh

      - name: Setup MicroCeph
        if: matrix.driver == 'ceph'
        run: |
          set -x

          # Setup scratch space
          sudo truncate -s 20G /scratch.img
          block_path="$(sudo losetup --show -f /scratch.img)"

          sudo DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y ceph-common qemu-block-extra snapd
          sudo snap wait system seed.loaded
          sudo snap install microceph --channel=quincy/stable

          # /snap/bin isn't part of sudo's secure_path on Debian.
          sudo ln -s /snap/bin/microceph /snap/bin/microceph.ceph /usr/local/bin/

          sudo microceph cluster bootstrap
          sudo microceph.ceph config set global osd_pool_default_size 1
          sudo microceph.ceph config set global mon_allow_pool_delete true
          sudo microceph.ceph config set global osd_memory_target 939524096
          sudo microceph.ceph osd crush rule rm replicated_rule
          sudo microceph.ceph osd crush rule create-replicated replicated default osd
          for flag in nosnaptrim noscrub nobackfill norebalance norecover noscrub nodeep-scrub; do
              sudo microceph.ceph osd set $flag
          done

          # Repurpose the scratch disk for ceph OSD.
          sudo microceph disk add --wipe "${block_path}"
          sudo rm -rf /etc/ceph
          sudo ln -s /var/snap/microceph/current/conf/ /etc/ceph
          sleep 30
          sudo microceph.ceph status

          # Wait until there are no more "unkowns" pgs
          for _ in $(seq 60); do
            if sudo microceph.ceph pg stat | grep -wF unknown; then
              sleep 1
            else
              break
            fi
          done
          sudo microceph.ceph status
          sudo rm -f /snap/bin/rbd

      - name: Setup LINSTOR
        if: matrix.driver == 'linstor'
        run: |
          set -x

          # Setup scratch space
          sudo truncate -s 20G /scratch.img
          block_path="$(sudo losetup --show -f /scratch.img)"

          # Configure the LINBIT public repository (drbd-dkms only comes with the Proxmox suite).
          sudo install -d -m 0755 /etc/apt/keyrings
          curl -fsSL https://packages.linbit.com/package-signing-pubkey.asc | sudo tee /etc/apt/keyrings/linbit.asc > /dev/null
          sudo tee /etc/apt/sources.list.d/linbit.sources > /dev/null <<EOF
          Types: deb
          URIs: https://packages.linbit.com/public
          Suites: $(. /etc/os-release && echo ${VERSION_CODENAME})
          Components: misc
          Signed-By: /etc/apt/keyrings/linbit.asc

          Types: deb
          URIs: https://packages.linbit.com/public
          Suites: proxmox-9
          Components: drbd-9
          Signed-By: /etc/apt/keyrings/linbit.asc
          EOF
          sudo apt-get update

          # Install everything required to compile DRBD and run LINSTOR tools.
          sudo DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
            drbd-dkms \
            linstor-client \
            linstor-controller \
            linstor-satellite \
            python3-setuptools

          # Enable DRBD.
          sudo modprobe -r drbd || true
          sudo modprobe drbd

          # Get the runner IP.
          runner_ip="$(hostname -I | cut -d' ' -f1)"

          # Create a single local node.
          sudo linstor node create local "${runner_ip}" --node-type combined

          # Repurpose the scratch disk for LINSTOR physical storage.
          sudo linstor physical-storage create-device-pool --storage-pool incus --pool-name linstor-incus zfsthin local "${block_path}"

          # Update the runner env.
          echo "INCUS_LINSTOR_CLUSTER=${runner_ip}" >> "$GITHUB_ENV"

      - name: Download go dependencies
        run: |
          go mod download

      - name: Build cowsql and raft
        run: |
          set -x
          make deps

          raft_path="$(go env GOPATH)/deps/raft"
          cowsql_path="$(go env GOPATH)/deps/cowsql"
          {
            echo "CGO_CFLAGS=-I${raft_path}/include/ -I${cowsql_path}/include/"
            echo "CGO_LDFLAGS=-L${raft_path}/.libs -L${cowsql_path}/.libs/"
            echo "LD_LIBRARY_PATH=${raft_path}/.libs/:${cowsql_path}/.libs/"
            echo "CGO_LDFLAGS_ALLOW=(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"
          } >> "$GITHUB_ENV"

      - name: Run Incus build
        run: |
          make

      - name: "Run VM tests (${{ matrix.test }}, ${{ matrix.driver }})"
        env:
          CGO_LDFLAGS_ALLOW: "(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"
          INCUS_CEPH_CLUSTER: "ceph"
          INCUS_CLI_TIMEOUT: "600"
          INCUS_LINSTOR_LOCAL_SATELLITE: "local"
          INCUS_VERBOSE: "1"
          INCUS_REQUIRED_TESTS: "${{ matrix.test }}"
          INCUS_VM_STORAGE_DRIVERS: "${{ matrix.driver }}"
        run: |
          chmod +x ~
          echo "root:1000000:1000000000" | sudo tee /etc/subuid /etc/subgid
          cd test
          sudo --preserve-env=PATH,GOPATH,GITHUB_ACTIONS,INCUS_VERBOSE,INCUS_CEPH_CLUSTER,INCUS_CLI_TIMEOUT,INCUS_LINSTOR_CLUSTER,INCUS_LINSTOR_LOCAL_SATELLITE,INCUS_REQUIRED_TESTS,INCUS_VM_STORAGE_DRIVERS env LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./main.sh ${{ matrix.test }}

  client:
    name: Client
    strategy:
      fail-fast: false
      matrix:
        go:
          - oldstable
          - stable
        os:
          - ubuntu-latest
          - macos-latest
          - windows-latest
    runs-on: ${{ matrix.os }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v7

      - name: Install Go
        uses: actions/setup-go@v7
        with:
          go-version: ${{ matrix.go }}

      - name: Create build directory
        run: |
            mkdir bin

      - name: Build static incus (x86_64)
        env:
          CGO_ENABLED: 0
          GOARCH: amd64
        run: |
          go build -o bin/incus.x86_64 ./cmd/incus

      - name: Build static incus (aarch64)
        env:
          CGO_ENABLED: 0
          GOARCH: arm64
        run: |
          go build -o bin/incus.aarch64 ./cmd/incus

      - name: Build static incus-agent (x86_64)
        env:
          CGO_ENABLED: 0
          GOARCH: amd64
        run: |
          go build -o bin/incus-agent.x86_64 ./cmd/incus-agent

      - name: Build static incus-agent (aarch64)
        env:
          CGO_ENABLED: 0
          GOARCH: arm64
        run: |
          go build -o bin/incus-agent.aarch64 ./cmd/incus-agent

      - name: Build static incus-migrate
        if: runner.os == 'Linux'
        env:
          CGO_ENABLED: 0
        run: |
          GOARCH=amd64 go build -o bin/incus-migrate.x86_64 ./cmd/incus-migrate
          GOARCH=arm64 go build -o bin/incus-migrate.aarch64 ./cmd/incus-migrate

      - name: Unit tests (client)
        env:
          CGO_ENABLED: 0
        run: go test -v ./client/...

      - name: Unit tests (incus)
        env:
          CGO_ENABLED: 0
        run: go test -v ./cmd/incus/...

      - name: Unit tests (shared)
        env:
          CGO_ENABLED: 0
        run: go test -v ./shared/...

      - name: Upload incus client artifacts
        if: matrix.go == 'stable'
        uses: actions/upload-artifact@v7
        continue-on-error: true
        with:
          name: ${{ runner.os }}
          path: bin/

  documentation:
    name: Documentation
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout
        uses: actions/checkout@v7

      - name: Install Go
        uses: actions/setup-go@v7
        with:
          go-version: stable

      - name: Install dependencies
        run: |
          sudo apt-get install -y aspell aspell-en ruby
          sudo gem install --no-document mdl

      - name: Run markdown linter
        run: |
          make doc-lint

      - name: Run spell checker
        run: |
          make doc-spellcheck

      - name: Run inclusive naming checker
        uses: get-woke/woke-action@v0
        with:
          fail-on-error: true
          woke-args: "*.md **/*.md -c https://github.com/canonical/Inclusive-naming/raw/main/config.yml"

      - name: Run link checker
        run: |
          make doc-linkcheck

      - name: Build docs (Sphinx)
        run: make doc

      - name: Print warnings (Sphinx)
        run: if [ -s doc/.sphinx/warnings.txt ]; then cat doc/.sphinx/warnings.txt; exit 1; fi

      - name: Upload documentation artifacts
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: documentation
          path: doc/html