diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 6eff8894..734b9bf5 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -64,6 +64,5 @@ steps: CLEANROOM_KERNEL_IMAGE: /var/lib/buildkite-agent/.local/share/cleanroom/images/vmlinux.bin CLEANROOM_FIRECRACKER_BINARY: /usr/local/bin/firecracker CLEANROOM_PRIVILEGED_MODE: helper - CLEANROOM_PRIVILEGED_HELPER_PATH: /usr/local/sbin/cleanroom-root-helper agents: queue: cleanroom diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ba09fa0b..d557752b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,7 +40,8 @@ jobs: fi done - mkdir -p "release-extra/darwin_${{ matrix.arch }}" + helper_output_dir="release-extra/darwin_${{ matrix.arch }}/libexec/cleanroom" + mkdir -p "${helper_output_dir}" CERT_PATH="${RUNNER_TEMP}/cleanroom-darwin-vz-release.p12" PROFILE_PATH="${RUNNER_TEMP}/Cleanroom_Darwin_VZ_Backend.provisionprofile" KEYCHAIN_PATH="${RUNNER_TEMP}/cleanroom-release-signing.keychain-db" @@ -81,9 +82,9 @@ jobs: CLEANROOM_DARWIN_VZ_HELPER_SIGN_IDENTIFIER="com.buildkite.cleanroom.darwin-vz" \ CLEANROOM_DARWIN_VZ_HELPER_PROVISION_PROFILE="${PROFILE_PATH}" \ CLEANROOM_DARWIN_VZ_HELPER_BUNDLE=1 \ - scripts/build-darwin-vz-helper.sh "release-extra/darwin_${{ matrix.arch }}/cleanroom-darwin-vz.app" - cp cmd/cleanroom-darwin-vz/entitlements.plist "release-extra/darwin_${{ matrix.arch }}/entitlements.plist" - cp cmd/cleanroom-darwin-vz/entitlements-vmnet.plist "release-extra/darwin_${{ matrix.arch }}/entitlements-vmnet.plist" + scripts/build-darwin-vz-helper.sh "${helper_output_dir}/cleanroom-darwin-vz.app" + cp cmd/cleanroom-darwin-vz/entitlements.plist "${helper_output_dir}/entitlements.plist" + cp cmd/cleanroom-darwin-vz/entitlements-vmnet.plist "${helper_output_dir}/entitlements-vmnet.plist" - uses: actions/upload-artifact@v4 with: @@ -114,11 +115,11 @@ jobs: - name: Build release extra binaries run: | - mkdir -p release-extra/linux_amd64 release-extra/linux_arm64 + mkdir -p release-extra/linux_amd64/libexec/cleanroom release-extra/linux_arm64/libexec/cleanroom GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags "-s -w" \ - -o release-extra/linux_amd64/cleanroom-guest-agent ./cmd/cleanroom-guest-agent + -o release-extra/linux_amd64/libexec/cleanroom/cleanroom-guest-agent-linux-amd64 ./cmd/cleanroom-guest-agent GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -trimpath -ldflags "-s -w" \ - -o release-extra/linux_arm64/cleanroom-guest-agent ./cmd/cleanroom-guest-agent + -o release-extra/linux_arm64/libexec/cleanroom/cleanroom-guest-agent-linux-arm64 ./cmd/cleanroom-guest-agent - uses: goreleaser/goreleaser-action@v6 with: diff --git a/.goreleaser.yml b/.goreleaser.yml index 42904679..56f3d4c8 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -3,7 +3,7 @@ version: 2 builds: - id: cleanroom-linux main: ./cmd/cleanroom - binary: cleanroom + binary: bin/cleanroom env: - CGO_ENABLED=0 goos: [linux] @@ -13,7 +13,7 @@ builds: - id: cleanroom-darwin main: ./cmd/cleanroom - binary: cleanroom + binary: bin/cleanroom env: - CGO_ENABLED=0 goos: [darwin] @@ -26,30 +26,35 @@ archives: ids: [cleanroom-linux] formats: [tar.gz] name_template: >- - {{ .Binary }}_ + {{ .ProjectName }}_ {{- title .Os }}_ {{- if eq .Arch "amd64" }}x86_64 {{- else }}{{ .Arch }}{{ end }} files: - - src: release-extra/linux_{{ .Arch }}/cleanroom-guest-agent + - src: release-extra/linux_{{ .Arch }}/libexec/cleanroom/cleanroom-guest-agent-linux-{{ .Arch }} + dst: libexec/cleanroom strip_parent: true - id: cleanroom-darwin ids: [cleanroom-darwin] formats: [tar.gz] name_template: >- - {{ .Binary }}_ + {{ .ProjectName }}_ {{- title .Os }}_ {{- if eq .Arch "amd64" }}x86_64 {{- else }}{{ .Arch }}{{ end }} files: - - src: release-extra/linux_{{ .Arch }}/cleanroom-guest-agent + - src: release-extra/linux_{{ .Arch }}/libexec/cleanroom/cleanroom-guest-agent-linux-{{ .Arch }} + dst: libexec/cleanroom strip_parent: true - - src: release-extra/darwin_{{ .Arch }}/cleanroom-darwin-vz.app + - src: release-extra/darwin_{{ .Arch }}/libexec/cleanroom/cleanroom-darwin-vz.app + dst: libexec/cleanroom strip_parent: true - - src: release-extra/darwin_{{ .Arch }}/entitlements-vmnet.plist + - src: release-extra/darwin_{{ .Arch }}/libexec/cleanroom/entitlements-vmnet.plist + dst: libexec/cleanroom strip_parent: true - - src: release-extra/darwin_{{ .Arch }}/entitlements.plist + - src: release-extra/darwin_{{ .Arch }}/libexec/cleanroom/entitlements.plist + dst: libexec/cleanroom strip_parent: true checksum: diff --git a/.mise.toml b/.mise.toml index 0f116d2f..3baa6525 100644 --- a/.mise.toml +++ b/.mise.toml @@ -21,12 +21,12 @@ description = "Run full Go test suite" run = "go test ./..." [tasks."build:go"] -description = "Build Go binaries into dist/" +description = "Build staged host binaries into dist/-/" run = "scripts/build-go.sh" [tasks."build:darwin"] -description = "Build macOS darwin-vz helper into dist/" -run = "{% if os() == \"macos\" %}scripts/build-darwin-vz-helper.sh dist/cleanroom-darwin-vz.app{% else %}true{% endif %}" +description = "Build macOS darwin-vz helper into the staged dist layout" +run = "{% if os() == \"macos\" %}scripts/build-darwin-vz-helper.sh{% else %}true{% endif %}" [tasks."build:image:alpine"] description = "Build local alpine base image" @@ -46,28 +46,23 @@ depends = ["build:image:alpine", "build:image:alpine-docker", "build:image:alpin run = "true" [tasks.build] -description = "Build binaries into dist/" +description = "Build staged binaries into dist/-/" depends = ["build:go", "build:darwin"] run = "true" [tasks."install:go"] -description = "Install Go binaries into GOBIN (or GOPATH/bin)" +description = "Install the cleanroom CLI into GOBIN (or GOPATH/bin)" run = "scripts/install-go.sh" -[tasks."install:darwin"] -description = "Install macOS darwin-vz helper into GOBIN (or GOPATH/bin)" -depends = ["build:darwin"] -run = "{% if os() == \"macos\" %}BIN_DIR=\"$(go env GOBIN)\"; if [ -z \"$BIN_DIR\" ]; then BIN_DIR=\"$(go env GOPATH)/bin\"; fi; mkdir -p \"$BIN_DIR\" && scripts/build-darwin-vz-helper.sh \"$BIN_DIR/cleanroom-darwin-vz.app\"{% else %}true{% endif %}" - [tasks.install] -description = "Install all binaries into GOBIN (or GOPATH/bin)" -depends = ["install:go", "install:darwin"] -run = "true" +description = "Install the staged runtime into ~/.local" +depends = ["build"] +run = "CLEANROOM_GLOBAL_PREFIX=\"$HOME/.local\" scripts/install-global.sh" [tasks."install:global"] -description = "Install all runtime binaries into /usr/local/bin" +description = "Install the staged runtime into /usr/local" depends = ["build"] -run = "scripts/install-global.sh" +run = "CLEANROOM_GLOBAL_PREFIX=\"/usr/local\" scripts/install-global.sh" [tasks.doctor] description = "Run cleanroom diagnostics" diff --git a/README.md b/README.md index dd870c1f..ccf2332d 100644 --- a/README.md +++ b/README.md @@ -35,11 +35,12 @@ curl -fsSL https://raw.githubusercontent.com/buildkite/cleanroom/main/scripts/in bash -s -- --version vX.Y.Z ``` -By default this installs to `/usr/local/bin`. Override with `--install-dir` or `CLEANROOM_INSTALL_DIR`. +By default the install prefix is `~/.local` for non-root installs and `/usr/local` for root installs. `cleanroom` is installed to `${prefix}/bin/cleanroom` and runtime assets are installed under `${prefix}/libexec/cleanroom`. Override the prefix with `--prefix` or `CLEANROOM_PREFIX`. -Install the locally built binaries from this checkout into `/usr/local/bin`: +Install the locally built staged runtime from this checkout: ```bash +mise run install # installs into ~/.local mise run install:global ``` @@ -363,7 +364,7 @@ backends: firecracker: binary_path: firecracker kernel_image: "" # auto-managed when unset - privileged_helper_path: /usr/local/sbin/cleanroom-root-helper + privileged_helper_path: /usr/local/libexec/cleanroom/cleanroom-root-helper vcpus: 2 memory_mib: 1024 launch_seconds: 30 @@ -388,7 +389,7 @@ When `rootfs` is unset, Cleanroom derives one from `sandbox.image.ref` and injec - Firecracker binary installed - `mkfs.ext4` for OCI-to-ext4 materialization - `debugfs` for runtime rootfs preparation -- `sudo -n` access to `/usr/local/sbin/cleanroom-root-helper` for host networking +- `sudo -n` access to `/usr/local/libexec/cleanroom/cleanroom-root-helper` **macOS ([darwin-vz](docs/backend/darwin-vz.md)):** - `cleanroom-darwin-vz` helper signed with `com.apple.security.virtualization` entitlement diff --git a/docs/backend/darwin-vz.md b/docs/backend/darwin-vz.md index b9c8f469..33b6d317 100644 --- a/docs/backend/darwin-vz.md +++ b/docs/backend/darwin-vz.md @@ -181,6 +181,8 @@ default it emits a signed `cleanroom-darwin-vz.app` bundle. When `CLEANROOM_DARWIN_VZ_HELPER_PROVISION_PROFILE` is set it embeds that profile in the bundle so the helper can carry restricted entitlements. Set `CLEANROOM_DARWIN_VZ_HELPER_BUNDLE=0` to emit a loose helper binary instead. +Without an explicit output path, the helper is staged at +`dist/-/libexec/cleanroom/cleanroom-darwin-vz.app`. When a prebuilt helper `.app` bundle is available, the install script preserves that bundle as-is and only re-signs it when the caller explicitly provides @@ -207,7 +209,7 @@ CLEANROOM_DARWIN_VZ_HELPER_ENTITLEMENTS=cmd/cleanroom-darwin-vz/entitlements-vmn CLEANROOM_DARWIN_VZ_HELPER_SIGN_IDENTITY='Apple Development: ()' \ CLEANROOM_DARWIN_VZ_HELPER_SIGN_IDENTIFIER='com.buildkite.cleanroom.darwin-vz' \ CLEANROOM_DARWIN_VZ_HELPER_PROVISION_PROFILE="$HOME/Downloads/Cleanroom_Darwin_VZ_Backend.provisionprofile" \ -scripts/build-darwin-vz-helper.sh dist/cleanroom-darwin-vz.app +scripts/build-darwin-vz-helper.sh ``` ## Runtime Discovery @@ -215,21 +217,24 @@ scripts/build-darwin-vz-helper.sh dist/cleanroom-darwin-vz.app The helper path is resolved in this order: 1. `CLEANROOM_DARWIN_VZ_HELPER` -2. sibling binary next to `cleanroom` -3. sibling `cleanroom-darwin-vz.app` bundle next to `cleanroom` -4. `dist/` under the current working directory or one of its ancestors -5. `dist/cleanroom-darwin-vz.app` under the current working directory or one of its ancestors -6. `PATH` +2. installed `../libexec/cleanroom/cleanroom-darwin-vz.app` relative to `cleanroom` +3. sibling binary next to `cleanroom` +4. sibling `cleanroom-darwin-vz.app` bundle next to `cleanroom` +5. `dist/-/libexec/cleanroom/cleanroom-darwin-vz.app` under the current working directory or one of its ancestors +6. legacy `dist/cleanroom-darwin-vz.app` or `dist/cleanroom-darwin-vz` under the current working directory or one of its ancestors +7. `PATH` If missing, runtime fails with an actionable error. The Linux guest agent follows the same general pattern: -1. sibling binary next to `cleanroom` -2. `dist/cleanroom-guest-agent-linux-$GOARCH` under the current working directory or one of its ancestors -3. `PATH` +1. installed `../libexec/cleanroom/cleanroom-guest-agent-linux-$GOARCH` relative to `cleanroom` +2. sibling binary next to `cleanroom` +3. `dist/-/libexec/cleanroom/cleanroom-guest-agent-linux-$GOARCH` under the current working directory or one of its ancestors +4. legacy `dist/cleanroom-guest-agent-linux-$GOARCH` under the current working directory or one of its ancestors +5. `PATH` -`mise run build` now produces the matching prebuilt set in `dist/` for macOS development. +`mise run build` now produces the matching staged runtime under `dist/-/` for macOS development. ## Testing @@ -261,7 +266,7 @@ Supported e2e overrides: Focused vmnet spike path: ```bash -CLEANROOM_DARWIN_VZ_HELPER="$PWD/dist/cleanroom-darwin-vz.app" \ +CLEANROOM_DARWIN_VZ_HELPER="$PWD/dist/$(go env GOOS)-$(go env GOARCH)/libexec/cleanroom/cleanroom-darwin-vz.app" \ CLEANROOM_DARWIN_VZ_VMNET_E2E=1 \ mise exec -- go test ./internal/backend/darwinvz -run TestVMNetSharedE2E -v ``` diff --git a/docs/ci.md b/docs/ci.md index 7385aca7..19167466 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -73,9 +73,9 @@ The `:apple: E2E (darwin-vz)` step runs launched execution checks on macOS using Notes: -- `scripts/ci-darwin-vz-e2e.sh` builds `dist/cleanroom` and `dist/cleanroom-darwin-vz.app`, exports `CLEANROOM_DARWIN_VZ_HELPER` to the built helper, and isolates XDG runtime paths. +- `scripts/ci-darwin-vz-e2e.sh` builds the staged runtime under `dist/-/`, exports `CLEANROOM_DARWIN_VZ_HELPER` to the staged helper bundle under `libexec/cleanroom`, and isolates XDG runtime paths. - the CI script writes `backends.darwin-vz.network.mode: nat` into its temporary config because Buildkite only ad-hoc signs the helper; `vmnet-shared` needs a vmnet-capable provisioning profile and identifier. -- the script also builds `dist/cleanroom-guest-agent-linux-` so CI can self-bootstrap the Linux guest agent dependency without a separate install step. +- the staged build also includes `dist/-/libexec/cleanroom/cleanroom-guest-agent-linux-` so CI can self-bootstrap the Linux guest agent dependency without a separate install step. - Set `CLEANROOM_DARWIN_VZ_KERNEL_IMAGE` on the worker if you want an explicit kernel path; otherwise the script uses managed-kernel fallback. ### 3.2 Upload vmnet signing secrets @@ -106,7 +106,7 @@ downloads the Apple WWDR G3 intermediate, builds a signed helper bundle with `cmd/cleanroom-darwin-vz/entitlements-vmnet.plist`, and runs: ```bash -CLEANROOM_DARWIN_VZ_HELPER="$PWD/dist/cleanroom-darwin-vz.app" \ +CLEANROOM_DARWIN_VZ_HELPER="$PWD/dist/$(go env GOOS)-$(go env GOARCH)/libexec/cleanroom/cleanroom-darwin-vz.app" \ CLEANROOM_DARWIN_VZ_VMNET_E2E=1 \ go test ./internal/backend/darwinvz -run TestVMNetSharedE2E -v ``` @@ -191,14 +191,15 @@ For CI script usage, you can also set: Install the helper from this repository and only grant sudo access to that helper: ```bash -sudo install -o root -g root -m 0755 scripts/cleanroom-root-helper.sh /usr/local/sbin/cleanroom-root-helper +sudo install -d -o root -g root -m 0755 /usr/local/libexec/cleanroom +sudo install -o root -g root -m 0755 scripts/cleanroom-root-helper.sh /usr/local/libexec/cleanroom/cleanroom-root-helper ``` ```sudoers -buildkite-agent ALL=(root) NOPASSWD: /usr/local/sbin/cleanroom-root-helper * +buildkite-agent ALL=(root) NOPASSWD: /usr/local/libexec/cleanroom/cleanroom-root-helper * ``` -Then set `CLEANROOM_PRIVILEGED_HELPER_PATH=/usr/local/sbin/cleanroom-root-helper` if you need to override the runtime config. +Then set `CLEANROOM_PRIVILEGED_HELPER_PATH=/usr/local/libexec/cleanroom/cleanroom-root-helper` if you need to override the runtime config. `scripts/ci-cleanroom-e2e.sh` probes the installed helper with `capabilities`, and `cleanroom doctor` also records the helper `version`, before running Firecracker checks. They do not compare helper file hashes and they do not self-update the helper from the checkout. diff --git a/docs/plans/darwin-vz-vmnet-mode.md b/docs/plans/darwin-vz-vmnet-mode.md index 7e41bb11..4713118d 100644 --- a/docs/plans/darwin-vz-vmnet-mode.md +++ b/docs/plans/darwin-vz-vmnet-mode.md @@ -140,7 +140,7 @@ CLEANROOM_DARWIN_VZ_HELPER_ENTITLEMENTS=cmd/cleanroom-darwin-vz/entitlements-vmn CLEANROOM_DARWIN_VZ_HELPER_SIGN_IDENTITY='Apple Development: ()' \ CLEANROOM_DARWIN_VZ_HELPER_SIGN_IDENTIFIER='com.buildkite.cleanroom.darwin-vz' \ CLEANROOM_DARWIN_VZ_HELPER_PROVISION_PROFILE="$HOME/Downloads/Cleanroom_Darwin_VZ_Backend.provisionprofile" \ -scripts/build-darwin-vz-helper.sh dist/cleanroom-darwin-vz +scripts/build-darwin-vz-helper.sh ``` The helper should carry: @@ -221,7 +221,7 @@ binary, not packages expected to exist in the base image. Run it with a signed helper bundle: ```bash -CLEANROOM_DARWIN_VZ_HELPER="$PWD/dist/cleanroom-darwin-vz.app" \ +CLEANROOM_DARWIN_VZ_HELPER="$PWD/dist/$(go env GOOS)-$(go env GOARCH)/libexec/cleanroom/cleanroom-darwin-vz.app" \ CLEANROOM_DARWIN_VZ_VMNET_E2E=1 \ mise exec -- go test ./internal/backend/darwinvz -run TestVMNetSharedE2E -v ``` diff --git a/infra/terraform/envs/prod/README.md b/infra/terraform/envs/prod/README.md index 97035437..f27e021b 100644 --- a/infra/terraform/envs/prod/README.md +++ b/infra/terraform/envs/prod/README.md @@ -13,7 +13,7 @@ Default host behaviour: - defaults to `m8i.4xlarge` with a 500 GiB root volume - installs `scripts/bootstrap-cleanroom-host.sh`, which installs the pinned `cleanroom` GitHub release, installs the matching - `/usr/local/sbin/cleanroom-root-helper`, installs Firecracker, writes runtime + `/usr/local/libexec/cleanroom/cleanroom-root-helper`, installs Firecracker, writes runtime config, installs a rerunnable host bootstrap command, and installs the system daemon Set `cleanroom_release_repo` when `repo_url` is not a GitHub remote. diff --git a/internal/backend/darwinvz/backend_darwin.go b/internal/backend/darwinvz/backend_darwin.go index ed30f3f8..20ec217b 100644 --- a/internal/backend/darwinvz/backend_darwin.go +++ b/internal/backend/darwinvz/backend_darwin.go @@ -6,7 +6,6 @@ import ( "context" cryptorand "crypto/rand" "crypto/sha256" - "debug/elf" "encoding/hex" "errors" "fmt" @@ -30,6 +29,7 @@ import ( "github.com/buildkite/cleanroom/internal/imagemgr" "github.com/buildkite/cleanroom/internal/paths" "github.com/buildkite/cleanroom/internal/policy" + "github.com/buildkite/cleanroom/internal/runtimeassets" "github.com/buildkite/cleanroom/internal/volumestore" "github.com/buildkite/cleanroom/internal/vsockexec" "github.com/charmbracelet/log" @@ -1761,7 +1761,7 @@ func discoverGuestAgentBinary() (string, error) { os.Executable, os.Getwd, os.Stat, - isLinuxGuestAgentBinary, + nil, ) } @@ -1773,72 +1773,7 @@ func discoverGuestAgentBinaryWith( stat func(string) (os.FileInfo, error), validate func(string) (bool, error), ) (string, error) { - linuxName := fmt.Sprintf("cleanroom-guest-agent-linux-%s", goarch) - candidates := []string{} - if self, err := executable(); err == nil { - candidates = append(candidates, filepath.Join(filepath.Dir(self), linuxName)) - candidates = append(candidates, filepath.Join(filepath.Dir(self), "cleanroom-guest-agent")) - } - if getwd != nil { - if cwd, err := getwd(); err == nil { - if path, err := resolvePrebuiltBinaryPathFromWorkdir(cwd, linuxName, stat); err == nil { - candidates = append(candidates, path) - } - } - } - candidates = append(candidates, linuxName, "cleanroom-guest-agent") - - for _, candidate := range candidates { - if strings.TrimSpace(candidate) == "" { - continue - } - resolved := candidate - if !filepath.IsAbs(candidate) { - p, lookErr := lookPath(candidate) - if lookErr != nil { - continue - } - resolved = p - } - info, statErr := stat(resolved) - if statErr != nil || info.IsDir() { - continue - } - ok, validateErr := validate(resolved) - if validateErr != nil { - return "", fmt.Errorf("validate guest agent binary %q: %w", resolved, validateErr) - } - if ok { - return resolved, nil - } - } - return "", fmt.Errorf("linux guest-agent binary not found for architecture %s; run `mise run build` or `mise run install` to make cleanroom-guest-agent-linux-%s discoverable", goarch, goarch) -} - -func isLinuxGuestAgentBinary(path string) (bool, error) { - f, err := elf.Open(path) - if err != nil { - // Non-ELF binaries are not valid guest binaries. - return false, nil - } - defer f.Close() - - expectedMachine, ok := expectedGuestAgentELFMachine(runtime.GOARCH) - if !ok { - return false, fmt.Errorf("unsupported host architecture %q", runtime.GOARCH) - } - return f.FileHeader.Machine == expectedMachine, nil -} - -func expectedGuestAgentELFMachine(goarch string) (elf.Machine, bool) { - switch goarch { - case "arm64": - return elf.EM_AARCH64, true - case "amd64": - return elf.EM_X86_64, true - default: - return 0, false - } + return runtimeassets.ResolveLinuxGuestAgentBinaryWith(goarch, lookPath, executable, getwd, stat, validate) } func hashFileSHA256(path string) (string, error) { diff --git a/internal/backend/darwinvz/guest_agent_path_test.go b/internal/backend/darwinvz/guest_agent_path_test.go index 7205c5ba..e5d4f179 100644 --- a/internal/backend/darwinvz/guest_agent_path_test.go +++ b/internal/backend/darwinvz/guest_agent_path_test.go @@ -5,7 +5,10 @@ package darwinvz import ( "errors" "os" + "path/filepath" "testing" + + "github.com/buildkite/cleanroom/internal/runtimeassets" ) func TestDiscoverGuestAgentBinaryUsesAncestorDistBeforePATH(t *testing.T) { @@ -41,6 +44,43 @@ func TestDiscoverGuestAgentBinaryUsesAncestorDistBeforePATH(t *testing.T) { } } +func TestDiscoverGuestAgentBinaryUsesStagedDistBeforeLegacyDistAndPATH(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + repoRoot := filepath.Join(tmp, "repo") + cwd := filepath.Join(repoRoot, "nested", "workdir") + staged := filepath.Join(repoRoot, "dist", runtimeassets.HostStageDirName("darwin", "arm64"), "libexec", "cleanroom", "cleanroom-guest-agent-linux-arm64") + legacy := filepath.Join(repoRoot, "dist", "cleanroom-guest-agent-linux-arm64") + if err := os.MkdirAll(cwd, 0o755); err != nil { + t.Fatalf("mkdir workdir: %v", err) + } + if err := os.MkdirAll(filepath.Dir(staged), 0o755); err != nil { + t.Fatalf("mkdir staged dist dir: %v", err) + } + if err := os.WriteFile(staged, []byte("binary"), 0o755); err != nil { + t.Fatalf("write staged guest agent: %v", err) + } + if err := os.WriteFile(legacy, []byte("binary"), 0o755); err != nil { + t.Fatalf("write legacy guest agent: %v", err) + } + + got, err := discoverGuestAgentBinaryWith( + "arm64", + func(string) (string, error) { return "/usr/local/bin/cleanroom-guest-agent-linux-arm64", nil }, + func() (string, error) { return "", errors.New("no executable") }, + func() (string, error) { return cwd, nil }, + os.Stat, + func(path string) (bool, error) { return path == staged, nil }, + ) + if err != nil { + t.Fatalf("discoverGuestAgentBinaryWith returned error: %v", err) + } + if got != staged { + t.Fatalf("unexpected guest agent path: got %q want %q", got, staged) + } +} + func TestDiscoverGuestAgentBinaryUsesSiblingBeforePATH(t *testing.T) { t.Parallel() @@ -70,6 +110,46 @@ func TestDiscoverGuestAgentBinaryUsesSiblingBeforePATH(t *testing.T) { } } +func TestDiscoverGuestAgentBinaryPrefersInstalledLibexecBeforeSibling(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + selfDir := filepath.Join(tmp, "prefix", "bin") + self := filepath.Join(selfDir, "cleanroom") + sibling := filepath.Join(selfDir, "cleanroom-guest-agent-linux-arm64") + libexec := filepath.Join(tmp, "prefix", "libexec", "cleanroom", "cleanroom-guest-agent-linux-arm64") + if err := os.MkdirAll(selfDir, 0o755); err != nil { + t.Fatalf("mkdir self dir: %v", err) + } + if err := os.MkdirAll(filepath.Dir(libexec), 0o755); err != nil { + t.Fatalf("mkdir libexec dir: %v", err) + } + if err := os.WriteFile(self, []byte("binary"), 0o755); err != nil { + t.Fatalf("write self binary: %v", err) + } + if err := os.WriteFile(sibling, []byte("binary"), 0o755); err != nil { + t.Fatalf("write sibling guest agent: %v", err) + } + if err := os.WriteFile(libexec, []byte("binary"), 0o755); err != nil { + t.Fatalf("write libexec guest agent: %v", err) + } + + got, err := discoverGuestAgentBinaryWith( + "arm64", + func(string) (string, error) { return "/usr/local/bin/cleanroom-guest-agent-linux-arm64", nil }, + func() (string, error) { return self, nil }, + func() (string, error) { return "", errors.New("no working directory") }, + os.Stat, + func(path string) (bool, error) { return path == libexec, nil }, + ) + if err != nil { + t.Fatalf("discoverGuestAgentBinaryWith returned error: %v", err) + } + if got != libexec { + t.Fatalf("unexpected guest agent path: got %q want %q", got, libexec) + } +} + func TestDiscoverGuestAgentBinaryPrefersSiblingBeforeAncestorDist(t *testing.T) { t.Parallel() diff --git a/internal/backend/darwinvz/helper_path.go b/internal/backend/darwinvz/helper_path.go index 02dd6125..06e0948c 100644 --- a/internal/backend/darwinvz/helper_path.go +++ b/internal/backend/darwinvz/helper_path.go @@ -1,12 +1,13 @@ package darwinvz import ( - "errors" "fmt" "os" "os/exec" - "path/filepath" + "runtime" "strings" + + "github.com/buildkite/cleanroom/internal/runtimeassets" ) const ( @@ -26,105 +27,30 @@ func resolveHelperBinaryPathWith( stat func(string) (os.FileInfo, error), ) (string, error) { if override := strings.TrimSpace(envOverride); override != "" { - path, err := resolveHelperCandidatePath(override, stat) + path, err := runtimeassets.ResolveFileOrAppBundleExecutable(override, helperBinaryName, stat) if err != nil { return "", fmt.Errorf("resolve darwin-vz helper from %s=%q: %w", helperEnvVar, override, err) } return path, nil } - if self, err := executable(); err == nil { - sibling := filepath.Join(filepath.Dir(self), helperBinaryName) - siblingAppBundle := sibling + ".app" - if path, err := resolveHelperCandidatePath(siblingAppBundle, stat); err == nil { - return path, nil - } - if path, err := resolveHelperCandidatePath(sibling, stat); err == nil { + candidates := append(runtimeassets.InstalledLibexecCandidates(executable, runtimeassets.DarwinHelperNames()...), runtimeassets.InstalledSiblingCandidates(executable, runtimeassets.DarwinHelperNames()...)...) + candidates = append(candidates, runtimeassets.StagedDistCandidates(getwd, runtime.GOOS, runtime.GOARCH, "libexec/cleanroom/cleanroom-darwin-vz.app", "libexec/cleanroom/cleanroom-darwin-vz")...) + candidates = append(candidates, runtimeassets.DistCandidates(getwd, runtimeassets.DarwinHelperNames()...)...) + for _, candidate := range candidates { + if path, err := runtimeassets.ResolveFileOrAppBundleExecutable(candidate, helperBinaryName, stat); err == nil { return path, nil } } - if getwd != nil { - if cwd, err := getwd(); err == nil { - if path, err := resolvePrebuiltBinaryPathFromWorkdir(cwd, helperBinaryName, stat); err == nil { - return path, nil - } - } - } - if path, err := lookPath(helperBinaryName); err == nil { return path, nil } return "", fmt.Errorf( - "%s helper binary was not found (set %s, build prebuilt binaries with `mise run build`, or install %s in PATH)", + "%s helper binary was not found (set %s, install cleanroom with runtime assets, build prebuilt binaries with `mise run build`, or install %s in PATH)", helperBinaryName, helperEnvVar, helperBinaryName, ) } - -func resolvePrebuiltBinaryPathFromWorkdir(startDir, binaryName string, stat func(string) (os.FileInfo, error)) (string, error) { - trimmedDir := strings.TrimSpace(startDir) - if trimmedDir == "" { - return "", errors.New("working directory is empty") - } - absStartDir, err := filepath.Abs(trimmedDir) - if err != nil { - return "", fmt.Errorf("resolve working directory: %w", err) - } - trimmedName := strings.TrimSpace(binaryName) - if trimmedName == "" { - return "", errors.New("binary name is empty") - } - - for dir := absStartDir; ; dir = filepath.Dir(dir) { - candidate := filepath.Join(dir, "dist", trimmedName) - if path, err := resolveHelperCandidatePath(candidate+".app", stat); err == nil { - return path, nil - } - if path, err := resolveHelperCandidatePath(candidate, stat); err == nil { - return path, nil - } - parent := filepath.Dir(dir) - if parent == dir { - break - } - } - return "", fmt.Errorf("prebuilt binary %q not found under dist/ from %s", trimmedName, absStartDir) -} - -func resolveHelperCandidatePath(path string, stat func(string) (os.FileInfo, error)) (string, error) { - trimmed := strings.TrimSpace(path) - if trimmed == "" { - return "", errors.New("path is empty") - } - - absPath, err := filepath.Abs(trimmed) - if err != nil { - return "", fmt.Errorf("resolve absolute path: %w", err) - } - info, err := stat(absPath) - if err != nil { - return "", err - } - if info.IsDir() { - if strings.EqualFold(filepath.Ext(absPath), ".app") { - return resolveHelperBundleExecutablePath(absPath, stat) - } - return "", fmt.Errorf("%s is a directory", absPath) - } - return absPath, nil -} - -func resolveHelperBundleExecutablePath(appPath string, stat func(string) (os.FileInfo, error)) (string, error) { - bundleExecutablePath := filepath.Join(appPath, "Contents", "MacOS", helperBinaryName) - info, err := stat(bundleExecutablePath) - if err != nil { - return "", err - } - if info.IsDir() { - return "", fmt.Errorf("%s is a directory", bundleExecutablePath) - } - return bundleExecutablePath, nil -} diff --git a/internal/backend/darwinvz/helper_path_test.go b/internal/backend/darwinvz/helper_path_test.go index bc767d76..577ad41b 100644 --- a/internal/backend/darwinvz/helper_path_test.go +++ b/internal/backend/darwinvz/helper_path_test.go @@ -4,8 +4,11 @@ import ( "errors" "os" "path/filepath" + "runtime" "strings" "testing" + + "github.com/buildkite/cleanroom/internal/runtimeassets" ) func TestResolveHelperBinaryPathPrefersEnvOverride(t *testing.T) { @@ -60,6 +63,46 @@ func TestResolveHelperBinaryPathUsesSiblingBeforePath(t *testing.T) { } } +func TestResolveHelperBinaryPathPrefersInstalledLibexecAppBundleBeforeSibling(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + selfDir := filepath.Join(tmp, "prefix", "bin") + self := filepath.Join(selfDir, "cleanroom") + sibling := filepath.Join(selfDir, "cleanroom-darwin-vz") + appBundle := filepath.Join(tmp, "prefix", "libexec", "cleanroom", "cleanroom-darwin-vz.app") + appExecutable := filepath.Join(appBundle, "Contents", "MacOS", "cleanroom-darwin-vz") + if err := os.MkdirAll(selfDir, 0o755); err != nil { + t.Fatalf("mkdir self dir: %v", err) + } + if err := os.MkdirAll(filepath.Dir(appExecutable), 0o755); err != nil { + t.Fatalf("mkdir app bundle: %v", err) + } + if err := os.WriteFile(self, []byte("binary"), 0o755); err != nil { + t.Fatalf("write self binary: %v", err) + } + if err := os.WriteFile(sibling, []byte("binary"), 0o755); err != nil { + t.Fatalf("write sibling helper: %v", err) + } + if err := os.WriteFile(appExecutable, []byte("binary"), 0o755); err != nil { + t.Fatalf("write app bundle helper: %v", err) + } + + got, err := resolveHelperBinaryPathWith( + "", + func(string) (string, error) { return "/usr/local/bin/cleanroom-darwin-vz", nil }, + func() (string, error) { return self, nil }, + func() (string, error) { return "", errors.New("no working directory") }, + os.Stat, + ) + if err != nil { + t.Fatalf("resolveHelperBinaryPathWith returned error: %v", err) + } + if got != appExecutable { + t.Fatalf("unexpected helper path: got %q want %q", got, appExecutable) + } +} + func TestResolveHelperBinaryPathPrefersSiblingAppBundleOverLooseBinary(t *testing.T) { t.Parallel() @@ -156,6 +199,43 @@ func TestResolveHelperBinaryPathUsesAncestorDistBeforePATH(t *testing.T) { } } +func TestResolveHelperBinaryPathUsesStagedDistBeforeLegacyDistAndPATH(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + repoRoot := filepath.Join(tmp, "repo") + cwd := filepath.Join(repoRoot, "nested", "workdir") + stageDir := filepath.Join(repoRoot, "dist", runtimeassets.HostStageDirName(runtime.GOOS, runtime.GOARCH), "libexec", "cleanroom", "cleanroom-darwin-vz.app") + stagedExecutable := filepath.Join(stageDir, "Contents", "MacOS", "cleanroom-darwin-vz") + legacyExecutable := filepath.Join(repoRoot, "dist", "cleanroom-darwin-vz") + if err := os.MkdirAll(cwd, 0o755); err != nil { + t.Fatalf("mkdir workdir: %v", err) + } + if err := os.MkdirAll(filepath.Dir(stagedExecutable), 0o755); err != nil { + t.Fatalf("mkdir staged app bundle: %v", err) + } + if err := os.WriteFile(stagedExecutable, []byte("binary"), 0o755); err != nil { + t.Fatalf("write staged helper: %v", err) + } + if err := os.WriteFile(legacyExecutable, []byte("binary"), 0o755); err != nil { + t.Fatalf("write legacy helper: %v", err) + } + + got, err := resolveHelperBinaryPathWith( + "", + func(string) (string, error) { return "/usr/local/bin/cleanroom-darwin-vz", nil }, + func() (string, error) { return "", errors.New("no executable") }, + func() (string, error) { return cwd, nil }, + os.Stat, + ) + if err != nil { + t.Fatalf("resolveHelperBinaryPathWith returned error: %v", err) + } + if got != stagedExecutable { + t.Fatalf("unexpected helper path: got %q want %q", got, stagedExecutable) + } +} + func TestResolveHelperBinaryPathUsesAncestorDistAppBundleBeforePATH(t *testing.T) { t.Parallel() @@ -306,7 +386,7 @@ func TestResolveHelperBinaryPathReturnsActionableError(t *testing.T) { } } -func TestResolvePrebuiltBinaryPathFromWorkdirUsesAncestorDist(t *testing.T) { +func TestDistCandidatesUsesAncestorDist(t *testing.T) { t.Parallel() tmp := t.TempDir() @@ -323,9 +403,10 @@ func TestResolvePrebuiltBinaryPathFromWorkdirUsesAncestorDist(t *testing.T) { t.Fatalf("write repo dist helper: %v", err) } - got, err := resolvePrebuiltBinaryPathFromWorkdir(cwd, helperBinaryName, os.Stat) + candidates := runtimeassets.DistCandidates(func() (string, error) { return cwd, nil }, helperBinaryName) + got, err := runtimeassets.ResolveFirstCandidate(candidates, os.Stat, nil) if err != nil { - t.Fatalf("resolvePrebuiltBinaryPathFromWorkdir returned error: %v", err) + t.Fatalf("ResolveFirstCandidate returned error: %v", err) } if got != prebuilt { t.Fatalf("unexpected prebuilt path: got %q want %q", got, prebuilt) diff --git a/internal/backend/firecracker/backend.go b/internal/backend/firecracker/backend.go index f6c59d5f..582caad9 100644 --- a/internal/backend/firecracker/backend.go +++ b/internal/backend/firecracker/backend.go @@ -33,6 +33,7 @@ import ( "github.com/buildkite/cleanroom/internal/imagemgr" "github.com/buildkite/cleanroom/internal/paths" "github.com/buildkite/cleanroom/internal/policy" + "github.com/buildkite/cleanroom/internal/runtimeassets" "github.com/buildkite/cleanroom/internal/volumestore" "github.com/buildkite/cleanroom/internal/vsockexec" fcvsock "github.com/firecracker-microvm/firecracker-go-sdk/vsock" @@ -100,7 +101,7 @@ type sandboxInstance struct { const runObservabilityFile = "execution-observability.json" const vsockDialRetryInterval = 50 * time.Millisecond const preparedRuntimeRootFSVersion = "v2-debugfs" -const defaultPrivilegedHelperPath = "/usr/local/sbin/cleanroom-root-helper" +const defaultPrivilegedHelperPath = "/usr/local/libexec/cleanroom/cleanroom-root-helper" const helperCapabilityFirecrackerNetwork = "firecracker-network" const helperCapabilityFirecrackerZFS = "firecracker-zfs" const defaultDownloadMaxBytes int64 = 10 * 1024 * 1024 @@ -1398,6 +1399,60 @@ func (a *Adapter) installGuestRuntimeIntoRootFS(rootFSPath, guestAgentPath strin return nil } +func legacyCompatibleGuestAgentInstallSource(path string, stat func(string) (os.FileInfo, error)) string { + legacyPaths := stagedDistGuestAgentCompatibilityPaths(path) + if len(legacyPaths) == 0 { + return path + } + if stat == nil { + stat = os.Stat + } + for _, legacyPath := range legacyPaths { + info, err := stat(legacyPath) + if err != nil || info.IsDir() { + continue + } + return legacyPath + } + return path +} + +func stagedDistGuestAgentCompatibilityPaths(path string) []string { + legacyPath, ok := stagedDistGuestAgentCompatibilityPath(path) + if !ok { + return nil + } + + distDir := filepath.Dir(legacyPath) + return []string{ + filepath.Join(distDir, runtimeassets.GuestAgentName), + legacyPath, + } +} + +func stagedDistGuestAgentCompatibilityPath(path string) (string, bool) { + trimmed := strings.TrimSpace(path) + if trimmed == "" { + return "", false + } + + cleaned := filepath.Clean(trimmed) + if filepath.Base(filepath.Dir(cleaned)) != "cleanroom" { + return "", false + } + if filepath.Base(filepath.Dir(filepath.Dir(cleaned))) != "libexec" { + return "", false + } + + stageDir := filepath.Dir(filepath.Dir(filepath.Dir(cleaned))) + distDir := filepath.Dir(stageDir) + if filepath.Base(distDir) != "dist" { + return "", false + } + + return filepath.Join(distDir, filepath.Base(cleaned)), true +} + func createGuestInitScript() (string, error) { f, err := os.CreateTemp("", "cleanroom-init-*.sh") if err != nil { @@ -1438,17 +1493,25 @@ func (a *Adapter) getGuestAgentBinary() (string, string, error) { } func discoverGuestAgentBinary() (string, error) { - if p, err := exec.LookPath("cleanroom-guest-agent"); err == nil { - return p, nil - } - self, err := os.Executable() - if err == nil { - candidate := filepath.Join(filepath.Dir(self), "cleanroom-guest-agent") - if info, statErr := os.Stat(candidate); statErr == nil && !info.IsDir() { - return candidate, nil - } - } - return "", errors.New("cleanroom-guest-agent binary not found in PATH; run `mise run install` first") + return discoverGuestAgentBinaryWith( + runtime.GOARCH, + exec.LookPath, + os.Executable, + os.Getwd, + os.Stat, + nil, + ) +} + +func discoverGuestAgentBinaryWith( + goarch string, + lookPath func(string) (string, error), + executable func() (string, error), + getwd func() (string, error), + stat func(string) (os.FileInfo, error), + validate func(string) (bool, error), +) (string, error) { + return runtimeassets.ResolveLinuxGuestAgentBinaryWith(goarch, lookPath, executable, getwd, stat, validate) } func hashFileSHA256(path string) (string, error) { @@ -2352,11 +2415,23 @@ func resolveForwardRulesWithLookup(ctx context.Context, allow []policy.AllowRule } func resolvePrivilegedHelperPath(cfg backend.FirecrackerConfig) string { + return resolvePrivilegedHelperPathWith(cfg, os.Executable, os.Stat) +} + +func resolvePrivilegedHelperPathWith( + cfg backend.FirecrackerConfig, + executable func() (string, error), + stat func(string) (os.FileInfo, error), +) string { helperPath := strings.TrimSpace(cfg.PrivilegedHelperPath) - if helperPath == "" { - helperPath = defaultPrivilegedHelperPath + if helperPath != "" { + return helperPath + } + candidates := runtimeassets.InstalledLibexecCandidates(executable, runtimeassets.RootHelperName) + if path, err := runtimeassets.ResolveFirstCandidate(candidates, stat, nil); err == nil { + return path } - return helperPath + return defaultPrivilegedHelperPath } func helperRequiredCapabilities(cfg backend.FirecrackerConfig) []string { diff --git a/internal/backend/firecracker/guest_agent_install_source_test.go b/internal/backend/firecracker/guest_agent_install_source_test.go new file mode 100644 index 00000000..cb31cedc --- /dev/null +++ b/internal/backend/firecracker/guest_agent_install_source_test.go @@ -0,0 +1,68 @@ +package firecracker + +import ( + "os" + "path/filepath" + "testing" +) + +func TestLegacyCompatibleGuestAgentInstallSourceUsesLegacyDistCopy(t *testing.T) { + t.Parallel() + + repoRoot := t.TempDir() + staged := filepath.Join(repoRoot, "dist", "linux-amd64", "libexec", "cleanroom", "cleanroom-guest-agent-linux-amd64") + legacy := filepath.Join(repoRoot, "dist", "cleanroom-guest-agent") + if err := os.MkdirAll(filepath.Dir(staged), 0o755); err != nil { + t.Fatalf("mkdir staged guest agent dir: %v", err) + } + if err := os.WriteFile(staged, []byte("staged"), 0o755); err != nil { + t.Fatalf("write staged guest agent: %v", err) + } + if err := os.WriteFile(legacy, []byte("legacy"), 0o755); err != nil { + t.Fatalf("write legacy guest agent: %v", err) + } + + got := legacyCompatibleGuestAgentInstallSource(staged, os.Stat) + if got != legacy { + t.Fatalf("unexpected guest agent install source: got %q want %q", got, legacy) + } +} + +func TestLegacyCompatibleGuestAgentInstallSourceFallsBackToLegacyArchSpecificCopy(t *testing.T) { + t.Parallel() + + repoRoot := t.TempDir() + staged := filepath.Join(repoRoot, "dist", "linux-amd64", "libexec", "cleanroom", "cleanroom-guest-agent-linux-amd64") + legacy := filepath.Join(repoRoot, "dist", "cleanroom-guest-agent-linux-amd64") + if err := os.MkdirAll(filepath.Dir(staged), 0o755); err != nil { + t.Fatalf("mkdir staged guest agent dir: %v", err) + } + if err := os.WriteFile(staged, []byte("staged"), 0o755); err != nil { + t.Fatalf("write staged guest agent: %v", err) + } + if err := os.WriteFile(legacy, []byte("legacy"), 0o755); err != nil { + t.Fatalf("write legacy guest agent: %v", err) + } + + got := legacyCompatibleGuestAgentInstallSource(staged, os.Stat) + if got != legacy { + t.Fatalf("unexpected guest agent install source: got %q want %q", got, legacy) + } +} + +func TestLegacyCompatibleGuestAgentInstallSourceKeepsInstalledLibexecPath(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "prefix", "libexec", "cleanroom", "cleanroom-guest-agent-linux-amd64") + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir installed guest agent dir: %v", err) + } + if err := os.WriteFile(path, []byte("installed"), 0o755); err != nil { + t.Fatalf("write installed guest agent: %v", err) + } + + got := legacyCompatibleGuestAgentInstallSource(path, os.Stat) + if got != path { + t.Fatalf("unexpected guest agent install source: got %q want %q", got, path) + } +} diff --git a/internal/backend/firecracker/guest_agent_path_test.go b/internal/backend/firecracker/guest_agent_path_test.go new file mode 100644 index 00000000..e919b755 --- /dev/null +++ b/internal/backend/firecracker/guest_agent_path_test.go @@ -0,0 +1,121 @@ +package firecracker + +import ( + "errors" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/buildkite/cleanroom/internal/runtimeassets" +) + +func TestDiscoverGuestAgentBinaryPrefersInstalledLibexecBeforeSibling(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + selfDir := filepath.Join(tmp, "prefix", "bin") + self := filepath.Join(selfDir, "cleanroom") + sibling := filepath.Join(selfDir, "cleanroom-guest-agent-linux-amd64") + libexec := filepath.Join(tmp, "prefix", "libexec", "cleanroom", "cleanroom-guest-agent-linux-amd64") + if err := os.MkdirAll(selfDir, 0o755); err != nil { + t.Fatalf("mkdir self dir: %v", err) + } + if err := os.MkdirAll(filepath.Dir(libexec), 0o755); err != nil { + t.Fatalf("mkdir libexec dir: %v", err) + } + if err := os.WriteFile(self, []byte("binary"), 0o755); err != nil { + t.Fatalf("write self binary: %v", err) + } + if err := os.WriteFile(sibling, []byte("binary"), 0o755); err != nil { + t.Fatalf("write sibling guest agent: %v", err) + } + if err := os.WriteFile(libexec, []byte("binary"), 0o755); err != nil { + t.Fatalf("write libexec guest agent: %v", err) + } + + got, err := discoverGuestAgentBinaryWith( + "amd64", + func(string) (string, error) { return "/usr/local/bin/cleanroom-guest-agent-linux-amd64", nil }, + func() (string, error) { return self, nil }, + func() (string, error) { return "", errors.New("no working directory") }, + os.Stat, + func(path string) (bool, error) { return path == libexec, nil }, + ) + if err != nil { + t.Fatalf("discoverGuestAgentBinaryWith returned error: %v", err) + } + if got != libexec { + t.Fatalf("unexpected guest agent path: got %q want %q", got, libexec) + } +} + +func TestDiscoverGuestAgentBinaryUsesAncestorDistBeforePATH(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + repoRoot := filepath.Join(tmp, "repo") + cwd := filepath.Join(repoRoot, "nested", "workdir") + prebuilt := filepath.Join(repoRoot, "dist", "cleanroom-guest-agent-linux-amd64") + if err := os.MkdirAll(cwd, 0o755); err != nil { + t.Fatalf("mkdir workdir: %v", err) + } + if err := os.MkdirAll(filepath.Dir(prebuilt), 0o755); err != nil { + t.Fatalf("mkdir dist dir: %v", err) + } + if err := os.WriteFile(prebuilt, []byte("binary"), 0o755); err != nil { + t.Fatalf("write prebuilt guest agent: %v", err) + } + + got, err := discoverGuestAgentBinaryWith( + "amd64", + func(string) (string, error) { return "/usr/local/bin/cleanroom-guest-agent-linux-amd64", nil }, + func() (string, error) { return "", errors.New("no executable") }, + func() (string, error) { return cwd, nil }, + os.Stat, + func(path string) (bool, error) { return path == prebuilt, nil }, + ) + if err != nil { + t.Fatalf("discoverGuestAgentBinaryWith returned error: %v", err) + } + if got != prebuilt { + t.Fatalf("unexpected guest agent path: got %q want %q", got, prebuilt) + } +} + +func TestDiscoverGuestAgentBinaryUsesStagedDistBeforeLegacyDistAndPATH(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + repoRoot := filepath.Join(tmp, "repo") + cwd := filepath.Join(repoRoot, "nested", "workdir") + staged := filepath.Join(repoRoot, "dist", runtimeassets.HostStageDirName(runtime.GOOS, "amd64"), "libexec", "cleanroom", "cleanroom-guest-agent-linux-amd64") + legacy := filepath.Join(repoRoot, "dist", "cleanroom-guest-agent-linux-amd64") + if err := os.MkdirAll(cwd, 0o755); err != nil { + t.Fatalf("mkdir workdir: %v", err) + } + if err := os.MkdirAll(filepath.Dir(staged), 0o755); err != nil { + t.Fatalf("mkdir staged dist dir: %v", err) + } + if err := os.WriteFile(staged, []byte("binary"), 0o755); err != nil { + t.Fatalf("write staged guest agent: %v", err) + } + if err := os.WriteFile(legacy, []byte("binary"), 0o755); err != nil { + t.Fatalf("write legacy guest agent: %v", err) + } + + got, err := discoverGuestAgentBinaryWith( + "amd64", + func(string) (string, error) { return "/usr/local/bin/cleanroom-guest-agent-linux-amd64", nil }, + func() (string, error) { return "", errors.New("no executable") }, + func() (string, error) { return cwd, nil }, + os.Stat, + func(path string) (bool, error) { return path == staged, nil }, + ) + if err != nil { + t.Fatalf("discoverGuestAgentBinaryWith returned error: %v", err) + } + if got != staged { + t.Fatalf("unexpected guest agent path: got %q want %q", got, staged) + } +} diff --git a/internal/backend/firecracker/privileged_test.go b/internal/backend/firecracker/privileged_test.go index 756d6577..9f5c266e 100644 --- a/internal/backend/firecracker/privileged_test.go +++ b/internal/backend/firecracker/privileged_test.go @@ -380,6 +380,35 @@ func TestResolvePrivilegedHelperPathDefaultsToInstalledPath(t *testing.T) { } } +func TestResolvePrivilegedHelperPathPrefersInstalledLibexecPath(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + self := filepath.Join(tmp, "prefix", "bin", "cleanroom") + libexec := filepath.Join(tmp, "prefix", "libexec", "cleanroom", "cleanroom-root-helper") + if err := os.MkdirAll(filepath.Dir(self), 0o755); err != nil { + t.Fatalf("mkdir self dir: %v", err) + } + if err := os.MkdirAll(filepath.Dir(libexec), 0o755); err != nil { + t.Fatalf("mkdir libexec dir: %v", err) + } + if err := os.WriteFile(self, []byte("binary"), 0o755); err != nil { + t.Fatalf("write self binary: %v", err) + } + if err := os.WriteFile(libexec, []byte("helper"), 0o755); err != nil { + t.Fatalf("write helper binary: %v", err) + } + + got := resolvePrivilegedHelperPathWith( + backend.FirecrackerConfig{}, + func() (string, error) { return self, nil }, + os.Stat, + ) + if got != libexec { + t.Fatalf("unexpected helper path: got %q want %q", got, libexec) + } +} + func TestHelperRequiredCapabilitiesIncludesZFSWhenConfigured(t *testing.T) { t.Parallel() diff --git a/internal/cli/policy_config.go b/internal/cli/policy_config.go index 0a39dd78..25fa6cd2 100644 --- a/internal/cli/policy_config.go +++ b/internal/cli/policy_config.go @@ -129,7 +129,7 @@ func defaultRuntimeConfig(defaultBackend string) runtimeconfig.Config { Enabled: false, Driver: "file", }, - PrivilegedHelperPath: "/usr/local/sbin/cleanroom-root-helper", + PrivilegedHelperPath: "/usr/local/libexec/cleanroom/cleanroom-root-helper", VCPUs: 2, MemoryMiB: 1024, GuestCID: 3, diff --git a/internal/runtimeassets/runtimeassets.go b/internal/runtimeassets/runtimeassets.go new file mode 100644 index 00000000..c6fbfc08 --- /dev/null +++ b/internal/runtimeassets/runtimeassets.go @@ -0,0 +1,367 @@ +package runtimeassets + +import ( + "debug/elf" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" +) + +const ( + libexecDirName = "cleanroom" + GuestAgentName = "cleanroom-guest-agent" + RootHelperName = "cleanroom-root-helper" + darwinHelperBinary = "cleanroom-darwin-vz" +) + +type ( + LookPathFunc func(string) (string, error) + ExecutableFunc func() (string, error) + GetwdFunc func() (string, error) + StatFunc func(string) (os.FileInfo, error) + ValidateFunc func(string) (bool, error) +) + +func DarwinHelperNames() []string { + return []string{darwinHelperBinary + ".app", darwinHelperBinary} +} + +func LinuxGuestAgentName(goarch string) string { + return fmt.Sprintf("cleanroom-guest-agent-linux-%s", strings.TrimSpace(goarch)) +} + +func HostStageDirName(goos, goarch string) string { + return fmt.Sprintf("%s-%s", strings.ToLower(strings.TrimSpace(goos)), strings.TrimSpace(goarch)) +} + +func InstalledLibexecCandidates(executable ExecutableFunc, names ...string) []string { + rels := make([]string, 0, len(names)) + for _, name := range names { + trimmed := strings.TrimSpace(name) + if trimmed == "" { + continue + } + rels = append(rels, filepath.Join("..", "libexec", libexecDirName, trimmed)) + } + return ExecutableRelativeCandidates(executable, rels...) +} + +func InstalledSiblingCandidates(executable ExecutableFunc, names ...string) []string { + return ExecutableRelativeCandidates(executable, names...) +} + +func ExecutableRelativeCandidates(executable ExecutableFunc, relPaths ...string) []string { + if executable == nil { + return nil + } + self, err := executable() + if err != nil { + return nil + } + base, err := absoluteBaseDir(self) + if err != nil { + return nil + } + return baseRelativeCandidates(base, relPaths...) +} + +func DistCandidates(getwd GetwdFunc, names ...string) []string { + return distCandidates(getwd, names...) +} + +func StagedDistCandidates(getwd GetwdFunc, goos, goarch string, relPaths ...string) []string { + stageDir := HostStageDirName(goos, goarch) + if strings.TrimSpace(stageDir) == "-" { + return nil + } + prefixed := make([]string, 0, len(relPaths)) + for _, relPath := range relPaths { + trimmed := strings.TrimSpace(relPath) + if trimmed == "" { + continue + } + prefixed = append(prefixed, filepath.Join(stageDir, trimmed)) + } + return distCandidates(getwd, prefixed...) +} + +func distCandidates(getwd GetwdFunc, relPaths ...string) []string { + if getwd == nil { + return nil + } + startDir, err := getwd() + if err != nil { + return nil + } + absStartDir, err := filepath.Abs(strings.TrimSpace(startDir)) + if err != nil || strings.TrimSpace(absStartDir) == "" { + return nil + } + + rels := make([]string, 0, len(relPaths)) + for _, relPath := range relPaths { + trimmed := strings.TrimSpace(relPath) + if trimmed == "" { + continue + } + rels = append(rels, filepath.Join("dist", trimmed)) + } + + out := []string{} + seen := map[string]struct{}{} + for dir := absStartDir; ; dir = filepath.Dir(dir) { + for _, rel := range rels { + appendUniquePath(&out, seen, filepath.Join(dir, rel)) + } + parent := filepath.Dir(dir) + if parent == dir { + break + } + } + return out +} + +func ResolveFirstCandidate(candidates []string, stat StatFunc, validate ValidateFunc) (string, error) { + if stat == nil { + stat = os.Stat + } + + seen := map[string]struct{}{} + for _, candidate := range candidates { + trimmed := strings.TrimSpace(candidate) + if trimmed == "" { + continue + } + resolved := trimmed + if !filepath.IsAbs(resolved) { + abs, err := filepath.Abs(resolved) + if err != nil { + continue + } + resolved = abs + } + if _, ok := seen[resolved]; ok { + continue + } + seen[resolved] = struct{}{} + + info, err := stat(resolved) + if err != nil || info.IsDir() { + continue + } + if validate != nil { + ok, err := validate(resolved) + if err != nil { + return "", fmt.Errorf("validate %q: %w", resolved, err) + } + if !ok { + continue + } + } + return resolved, nil + } + + return "", os.ErrNotExist +} + +func ResolveLookPath(names []string, lookPath LookPathFunc, validate ValidateFunc) (string, error) { + if lookPath == nil { + return "", os.ErrNotExist + } + + seen := map[string]struct{}{} + for _, name := range names { + trimmed := strings.TrimSpace(name) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + + path, err := lookPath(trimmed) + if err != nil { + continue + } + if validate != nil { + ok, err := validate(path) + if err != nil { + return "", fmt.Errorf("validate %q: %w", path, err) + } + if !ok { + continue + } + } + return path, nil + } + + return "", os.ErrNotExist +} + +func ResolveFileOrAppBundleExecutable(path, executableName string, stat StatFunc) (string, error) { + if stat == nil { + stat = os.Stat + } + + trimmed := strings.TrimSpace(path) + if trimmed == "" { + return "", errors.New("path is empty") + } + absPath, err := filepath.Abs(trimmed) + if err != nil { + return "", fmt.Errorf("resolve absolute path: %w", err) + } + info, err := stat(absPath) + if err != nil { + return "", err + } + if !info.IsDir() { + return absPath, nil + } + if !strings.EqualFold(filepath.Ext(absPath), ".app") { + return "", fmt.Errorf("%s is a directory", absPath) + } + + bundleExecutablePath := filepath.Join(absPath, "Contents", "MacOS", strings.TrimSpace(executableName)) + info, err = stat(bundleExecutablePath) + if err != nil { + return "", err + } + if info.IsDir() { + return "", fmt.Errorf("%s is a directory", bundleExecutablePath) + } + return bundleExecutablePath, nil +} + +func ResolveLinuxGuestAgentBinary(goarch string) (string, error) { + return ResolveLinuxGuestAgentBinaryWith( + goarch, + nil, + nil, + nil, + nil, + nil, + ) +} + +func ResolveLinuxGuestAgentBinaryWith( + goarch string, + lookPath LookPathFunc, + executable ExecutableFunc, + getwd GetwdFunc, + stat StatFunc, + validate ValidateFunc, +) (string, error) { + if lookPath == nil { + lookPath = exec.LookPath + } + if executable == nil { + executable = os.Executable + } + if getwd == nil { + getwd = os.Getwd + } + if stat == nil { + stat = os.Stat + } + if validate == nil { + validate = func(path string) (bool, error) { + return IsLinuxGuestAgentBinary(path, goarch) + } + } + + linuxName := LinuxGuestAgentName(goarch) + names := []string{linuxName, GuestAgentName} + candidates := append(InstalledLibexecCandidates(executable, names...), InstalledSiblingCandidates(executable, names...)...) + candidates = append(candidates, StagedDistCandidates(getwd, runtime.GOOS, goarch, filepath.Join("libexec", libexecDirName, linuxName), filepath.Join("libexec", libexecDirName, GuestAgentName))...) + candidates = append(candidates, DistCandidates(getwd, names...)...) + + path, err := ResolveFirstCandidate(candidates, stat, validate) + if err == nil { + return path, nil + } + if !errors.Is(err, os.ErrNotExist) { + return "", err + } + + path, err = ResolveLookPath(names, lookPath, validate) + if err == nil { + return path, nil + } + if !errors.Is(err, os.ErrNotExist) { + return "", err + } + + return "", fmt.Errorf( + "linux guest-agent binary not found for architecture %s; make %s available via ../libexec/cleanroom, dist/, or PATH (for local development run `mise run build`)", + goarch, + linuxName, + ) +} + +func IsLinuxGuestAgentBinary(path, goarch string) (bool, error) { + f, err := elf.Open(path) + if err != nil { + return false, nil + } + defer f.Close() + + expectedMachine, ok := expectedGuestAgentELFMachine(goarch) + if !ok { + return false, fmt.Errorf("unsupported host architecture %q", goarch) + } + return f.FileHeader.Machine == expectedMachine, nil +} + +func expectedGuestAgentELFMachine(goarch string) (elf.Machine, bool) { + switch strings.TrimSpace(goarch) { + case "arm64": + return elf.EM_AARCH64, true + case "amd64": + return elf.EM_X86_64, true + default: + return 0, false + } +} + +func absoluteBaseDir(path string) (string, error) { + trimmed := strings.TrimSpace(path) + if trimmed == "" { + return "", errors.New("path is empty") + } + absPath, err := filepath.Abs(trimmed) + if err != nil { + return "", err + } + return filepath.Dir(absPath), nil +} + +func baseRelativeCandidates(base string, relPaths ...string) []string { + out := []string{} + seen := map[string]struct{}{} + for _, rel := range relPaths { + trimmed := strings.TrimSpace(rel) + if trimmed == "" { + continue + } + appendUniquePath(&out, seen, filepath.Clean(filepath.Join(base, trimmed))) + } + return out +} + +func appendUniquePath(out *[]string, seen map[string]struct{}, path string) { + trimmed := strings.TrimSpace(path) + if trimmed == "" { + return + } + if _, ok := seen[trimmed]; ok { + return + } + seen[trimmed] = struct{}{} + *out = append(*out, trimmed) +} diff --git a/scripts/benchmark-sandbox-workloads.sh b/scripts/benchmark-sandbox-workloads.sh index 435c1375..7113cbff 100755 --- a/scripts/benchmark-sandbox-workloads.sh +++ b/scripts/benchmark-sandbox-workloads.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash set -euo pipefail +# shellcheck source=scripts/dist-layout.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/dist-layout.sh" + usage() { cat <<'EOF' Benchmark sandbox workloads in a single reused sandbox. @@ -20,7 +23,7 @@ Options: -c, --chdir Repository/policy directory (default: current directory) -n, --iterations Iterations per workload (default: 5) --output-dir Output directory (default: benchmarks/results) - --cleanroom-bin cleanroom binary path (default: cleanroom from PATH, then ./dist/cleanroom) + --cleanroom-bin cleanroom binary path (default: cleanroom from PATH, then the staged dist binary) --repo-url Git repo for clone benchmark (default: https://github.com/kubernetes/kubernetes.git) --repo-depth Git clone depth (default: 1) --iops-block-size Block size for IOPS benchmark (default: 4096) @@ -46,8 +49,8 @@ fi if command -v cleanroom >/dev/null 2>&1; then cleanroom_bin="$(command -v cleanroom)" -elif [[ -x "./dist/cleanroom" ]]; then - cleanroom_bin="./dist/cleanroom" +elif [[ -x "./$(cleanroom_stage_bin_path cleanroom)" ]]; then + cleanroom_bin="./$(cleanroom_stage_bin_path cleanroom)" else cleanroom_bin="cleanroom" fi diff --git a/scripts/benchmark-tti.sh b/scripts/benchmark-tti.sh index ca1c7efd..2f7f7aa0 100755 --- a/scripts/benchmark-tti.sh +++ b/scripts/benchmark-tti.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash set -euo pipefail +# shellcheck source=scripts/dist-layout.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/dist-layout.sh" + usage() { cat <<'EOF' Benchmark cleanroom TTI (sandbox create -> first successful command) with hyperfine. @@ -15,7 +18,7 @@ Options: --backend Optional backend override for cleanroom exec -c, --chdir Repository/policy directory (default: current directory) --output-dir JSON output directory (default: benchmarks/results) - --cleanroom-bin cleanroom binary path (default: cleanroom from PATH, then ./dist/cleanroom) + --cleanroom-bin cleanroom binary path (default: cleanroom from PATH, then the staged dist binary) -h, --help Show this help Environment: @@ -36,8 +39,8 @@ fi if command -v cleanroom >/dev/null 2>&1; then cleanroom_bin="$(command -v cleanroom)" -elif [[ -x "./dist/cleanroom" ]]; then - cleanroom_bin="./dist/cleanroom" +elif [[ -x "./$(cleanroom_stage_bin_path cleanroom)" ]]; then + cleanroom_bin="./$(cleanroom_stage_bin_path cleanroom)" else cleanroom_bin="cleanroom" fi diff --git a/scripts/bootstrap-buildkite-agent.sh b/scripts/bootstrap-buildkite-agent.sh index 17044613..5fee41c5 100755 --- a/scripts/bootstrap-buildkite-agent.sh +++ b/scripts/bootstrap-buildkite-agent.sh @@ -457,7 +457,7 @@ AGENT_VERSION="${CLEANROOM_BUILDKITE_AGENT_VERSION:-3.119.1}" INSTALL_FIRECRACKER="${CLEANROOM_INSTALL_FIRECRACKER:-true}" FIRECRACKER_VERSION="${CLEANROOM_FIRECRACKER_VERSION:-1.14.2}" KERNEL_IMAGE_URL="${CLEANROOM_KERNEL_IMAGE_URL:-https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin}" -HELPER_INSTALL_PATH="${CLEANROOM_HELPER_INSTALL_PATH:-/usr/local/sbin/cleanroom-root-helper}" +HELPER_INSTALL_PATH="${CLEANROOM_HELPER_INSTALL_PATH:-/usr/local/libexec/cleanroom/cleanroom-root-helper}" SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd -- "$SCRIPT_DIR/.." && pwd)" @@ -536,6 +536,7 @@ chmod 0640 "/etc/buildkite-agent/${QUEUE_NAME}.cfg" if [ ! -f "$HELPER_SOURCE_PATH" ]; then die "cleanroom helper script not found at ${HELPER_SOURCE_PATH}" fi +install -d -o root -g root -m 0755 "$(dirname "$HELPER_INSTALL_PATH")" install -o root -g root -m 0755 "$HELPER_SOURCE_PATH" "$HELPER_INSTALL_PATH" install_linux_bootstrap_runner diff --git a/scripts/bootstrap-cleanroom-host.sh b/scripts/bootstrap-cleanroom-host.sh index 320083ee..9dcd360d 100755 --- a/scripts/bootstrap-cleanroom-host.sh +++ b/scripts/bootstrap-cleanroom-host.sh @@ -583,7 +583,7 @@ install_cleanroom_release() { if [ "$release_version" = "latest" ]; then retry 5 5 env \ CLEANROOM_REPO="$release_repo" \ - CLEANROOM_INSTALL_DIR="$CLEANROOM_BINARY_INSTALL_DIR" \ + CLEANROOM_PREFIX="$CLEANROOM_INSTALL_PREFIX" \ bash "$install_script_path" return fi @@ -591,7 +591,7 @@ install_cleanroom_release() { retry 5 5 env \ CLEANROOM_REPO="$release_repo" \ CLEANROOM_VERSION="$release_version" \ - CLEANROOM_INSTALL_DIR="$CLEANROOM_BINARY_INSTALL_DIR" \ + CLEANROOM_PREFIX="$CLEANROOM_INSTALL_PREFIX" \ bash "$install_script_path" } @@ -628,7 +628,15 @@ fi NAME_PREFIX="${CLEANROOM_BOOTSTRAP_NAME_PREFIX:-cleanroom-prod}" INSTALL_FIRECRACKER="${CLEANROOM_INSTALL_FIRECRACKER:-true}" FIRECRACKER_VERSION="${CLEANROOM_FIRECRACKER_VERSION:-1.14.2}" -CLEANROOM_BINARY_INSTALL_DIR="${CLEANROOM_BINARY_INSTALL_DIR:-/usr/local/bin}" +CLEANROOM_INSTALL_PREFIX="${CLEANROOM_INSTALL_PREFIX:-}" +if [ -z "$CLEANROOM_INSTALL_PREFIX" ] && [ -n "${CLEANROOM_BINARY_INSTALL_DIR:-}" ]; then + case "$CLEANROOM_BINARY_INSTALL_DIR" in + */bin) CLEANROOM_INSTALL_PREFIX="${CLEANROOM_BINARY_INSTALL_DIR%/bin}" ;; + *) die "CLEANROOM_BINARY_INSTALL_DIR must end with /bin when CLEANROOM_INSTALL_PREFIX is not set" ;; + esac +fi +CLEANROOM_INSTALL_PREFIX="${CLEANROOM_INSTALL_PREFIX:-/usr/local}" +CLEANROOM_BINARY_INSTALL_DIR="${CLEANROOM_BINARY_INSTALL_DIR:-$CLEANROOM_INSTALL_PREFIX/bin}" CLEANROOM_CONFIG_DIR="${CLEANROOM_CONFIG_DIR:-/root/.config/cleanroom}" CLEANROOM_VERSION="${CLEANROOM_VERSION:-v0.3.0}" CLEANROOM_INSTALL_SCRIPT_REF="${CLEANROOM_INSTALL_SCRIPT_REF:-main}" @@ -636,7 +644,7 @@ CLEANROOM_RELEASE_REPO="${CLEANROOM_RELEASE_REPO:-}" CLEANROOM_FIRECRACKER_VCPUS="${CLEANROOM_FIRECRACKER_VCPUS:-4}" CLEANROOM_FIRECRACKER_MEMORY_MIB="${CLEANROOM_FIRECRACKER_MEMORY_MIB:-8192}" CLEANROOM_FIRECRACKER_LAUNCH_SECONDS="${CLEANROOM_FIRECRACKER_LAUNCH_SECONDS:-90}" -HELPER_INSTALL_PATH="${CLEANROOM_HELPER_INSTALL_PATH:-/usr/local/sbin/cleanroom-root-helper}" +HELPER_INSTALL_PATH="${CLEANROOM_HELPER_INSTALL_PATH:-$CLEANROOM_INSTALL_PREFIX/libexec/cleanroom/cleanroom-root-helper}" AWS_REGION="${AWS_REGION:-${CLEANROOM_BOOTSTRAP_REGION:-}}" BOOTSTRAP_REPO_URL="${CLEANROOM_BOOTSTRAP_REPO_URL:-git@github.com:buildkite/cleanroom.git}" TAILSCALE_AUTH_KEY_PARAMETER_NAME="${TAILSCALE_AUTH_KEY_PARAMETER_NAME:-${CLEANROOM_TAILSCALE_AUTH_KEY_PARAMETER_NAME:-}}" diff --git a/scripts/bootstrap_tailscale_test.go b/scripts/bootstrap_tailscale_test.go index d72690b6..c32451ce 100644 --- a/scripts/bootstrap_tailscale_test.go +++ b/scripts/bootstrap_tailscale_test.go @@ -42,3 +42,33 @@ func TestBootstrapScriptsSkipTailscaleWhenAuthKeyLookupFails(t *testing.T) { }) } } + +func TestBootstrapBuildkiteAgentCreatesHelperParentDir(t *testing.T) { + t.Helper() + + content, err := os.ReadFile("bootstrap-buildkite-agent.sh") + if err != nil { + t.Fatalf("read bootstrap-buildkite-agent.sh: %v", err) + } + + script := string(content) + needle := "install -d -o root -g root -m 0755 \"$(dirname \"$HELPER_INSTALL_PATH\")\"\ninstall -o root -g root -m 0755 \"$HELPER_SOURCE_PATH\" \"$HELPER_INSTALL_PATH\"" + if !strings.Contains(script, needle) { + t.Fatal("expected bootstrap-buildkite-agent.sh to create the helper parent directory before installing the helper") + } +} + +func TestBootstrapCleanroomHostRejectsLegacyBinaryDirWithoutBinSuffix(t *testing.T) { + t.Helper() + + content, err := os.ReadFile("bootstrap-cleanroom-host.sh") + if err != nil { + t.Fatalf("read bootstrap-cleanroom-host.sh: %v", err) + } + + script := string(content) + needle := "*) die \"CLEANROOM_BINARY_INSTALL_DIR must end with /bin when CLEANROOM_INSTALL_PREFIX is not set\" ;;" + if !strings.Contains(script, needle) { + t.Fatal("expected bootstrap-cleanroom-host.sh to reject CLEANROOM_BINARY_INSTALL_DIR values that do not end with /bin when inferring the install prefix") + } +} diff --git a/scripts/build-darwin-vz-helper.sh b/scripts/build-darwin-vz-helper.sh index 4ae6e373..0b59c1e3 100755 --- a/scripts/build-darwin-vz-helper.sh +++ b/scripts/build-darwin-vz-helper.sh @@ -3,8 +3,10 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +# shellcheck source=scripts/dist-layout.sh +source "${SCRIPT_DIR}/dist-layout.sh" -OUTPUT_PATH="${1:-${REPO_ROOT}/dist/cleanroom-darwin-vz.app}" +OUTPUT_PATH="${1:-${REPO_ROOT}/$(cleanroom_stage_libexec_path cleanroom-darwin-vz.app)}" SWIFT_TARGET="${CLEANROOM_DARWIN_VZ_HELPER_SWIFT_TARGET:-}" ENTITLEMENTS_PATH="${CLEANROOM_DARWIN_VZ_HELPER_ENTITLEMENTS:-${REPO_ROOT}/cmd/cleanroom-darwin-vz/entitlements.plist}" PROVISION_PROFILE="${CLEANROOM_DARWIN_VZ_HELPER_PROVISION_PROFILE:-}" @@ -41,7 +43,11 @@ swiftc_args+=( -o "${build_output_path}" ) +echo "[build-darwin-vz-helper] compiling helper to ${build_output_path}" +compile_start="$(date +%s)" xcrun swiftc "${swiftc_args[@]}" +compile_end="$(date +%s)" +echo "[build-darwin-vz-helper] swiftc completed in $((compile_end - compile_start))s" package_env=( "CLEANROOM_DARWIN_VZ_HELPER_ENTITLEMENTS=${ENTITLEMENTS_PATH}" @@ -54,4 +60,8 @@ if [[ -n "${PROVISION_PROFILE}" || "${BUNDLE_MODE}" != "0" ]]; then package_env+=("CLEANROOM_DARWIN_VZ_HELPER_BUNDLE=1") fi +echo "[build-darwin-vz-helper] packaging helper to ${OUTPUT_PATH}" +package_start="$(date +%s)" env "${package_env[@]}" "${SCRIPT_DIR}/package-darwin-vz-helper.sh" "${build_output_path}" "${OUTPUT_PATH}" +package_end="$(date +%s)" +echo "[build-darwin-vz-helper] packaging completed in $((package_end - package_start))s" diff --git a/scripts/build-go.sh b/scripts/build-go.sh index f8507088..0a87b6d4 100755 --- a/scripts/build-go.sh +++ b/scripts/build-go.sh @@ -1,13 +1,23 @@ #!/usr/bin/env bash set -euo pipefail +# shellcheck source=scripts/dist-layout.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/dist-layout.sh" + VERSION=$(git describe --tags --always --dirty 2>/dev/null || echo dev) HOST_OS=$(go env GOOS) HOST_ARCH=$(go env GOARCH) -mkdir -p dist -go build -ldflags "-X main.version=$VERSION" -o dist/cleanroom ./cmd/cleanroom -go build -o dist/download-sandbox-file ./scripts/download_sandbox_file -GOOS=linux GOARCH="$HOST_ARCH" CGO_ENABLED=0 go build -trimpath -o "dist/cleanroom-guest-agent-linux-$HOST_ARCH" ./cmd/cleanroom-guest-agent +DIST_DIR="$(cleanroom_dist_root)" +BIN_DIR="$(cleanroom_stage_bin_dir "$HOST_OS" "$HOST_ARCH")" +LIBEXEC_DIR="$(cleanroom_stage_libexec_dir "$HOST_OS" "$HOST_ARCH")" +GUEST_AGENT_NAME="cleanroom-guest-agent-linux-$HOST_ARCH" +LEGACY_GUEST_AGENT_PATH="$DIST_DIR/$GUEST_AGENT_NAME" +LEGACY_GENERIC_GUEST_AGENT_PATH="$DIST_DIR/cleanroom-guest-agent" +mkdir -p "$DIST_DIR" "$BIN_DIR" "$LIBEXEC_DIR" +go build -ldflags "-X main.version=$VERSION" -o "$BIN_DIR/cleanroom" ./cmd/cleanroom +go build -o "$BIN_DIR/download-sandbox-file" ./scripts/download_sandbox_file +GOOS=linux GOARCH="$HOST_ARCH" CGO_ENABLED=0 go build -trimpath -o "$LIBEXEC_DIR/$GUEST_AGENT_NAME" ./cmd/cleanroom-guest-agent +install -m 0755 "$LIBEXEC_DIR/$GUEST_AGENT_NAME" "$LEGACY_GUEST_AGENT_PATH" if [[ "$HOST_OS" == "linux" ]]; then - cp "dist/cleanroom-guest-agent-linux-$HOST_ARCH" dist/cleanroom-guest-agent + install -m 0755 "$LIBEXEC_DIR/$GUEST_AGENT_NAME" "$LEGACY_GENERIC_GUEST_AGENT_PATH" fi diff --git a/scripts/build_go_test.go b/scripts/build_go_test.go index 6934691c..58d8c384 100644 --- a/scripts/build_go_test.go +++ b/scripts/build_go_test.go @@ -15,13 +15,22 @@ func TestBuildGoBootstrapsLinuxGuestAgentBinary(t *testing.T) { } script := string(content) + if !strings.Contains(script, `source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/dist-layout.sh"`) { + t.Fatalf("expected build-go.sh to source the staged dist layout helpers") + } if !strings.Contains(script, "HOST_ARCH=$(go env GOARCH)") { t.Fatalf("expected build-go.sh to resolve the host architecture for linux guest-agent cross-compiles") } - if !strings.Contains(script, "go build -o dist/download-sandbox-file ./scripts/download_sandbox_file") { - t.Fatalf("expected build-go.sh to bootstrap dist/download-sandbox-file") + if !strings.Contains(script, `go build -o "$BIN_DIR/download-sandbox-file" ./scripts/download_sandbox_file`) { + t.Fatalf("expected build-go.sh to stage download-sandbox-file under dist/-/bin") + } + if !strings.Contains(script, `GOOS=linux GOARCH="$HOST_ARCH" CGO_ENABLED=0 go build -trimpath -o "$LIBEXEC_DIR/$GUEST_AGENT_NAME" ./cmd/cleanroom-guest-agent`) { + t.Fatalf("expected build-go.sh to stage cleanroom-guest-agent-linux-$HOST_ARCH under dist/-/libexec/cleanroom") + } + if !strings.Contains(script, `install -m 0755 "$LIBEXEC_DIR/$GUEST_AGENT_NAME" "$LEGACY_GUEST_AGENT_PATH"`) { + t.Fatalf("expected build-go.sh to keep a legacy dist/cleanroom-guest-agent-linux-$HOST_ARCH compatibility copy") } - if !strings.Contains(script, "GOOS=linux GOARCH=\"$HOST_ARCH\" CGO_ENABLED=0 go build -trimpath -o \"dist/cleanroom-guest-agent-linux-$HOST_ARCH\" ./cmd/cleanroom-guest-agent") { - t.Fatalf("expected build-go.sh to bootstrap cleanroom-guest-agent-linux-$HOST_ARCH") + if !strings.Contains(script, `if [[ "$HOST_OS" == "linux" ]]; then`) || !strings.Contains(script, `install -m 0755 "$LIBEXEC_DIR/$GUEST_AGENT_NAME" "$LEGACY_GENERIC_GUEST_AGENT_PATH"`) { + t.Fatalf("expected build-go.sh to keep a legacy dist/cleanroom-guest-agent compatibility copy on linux hosts") } } diff --git a/scripts/buildkite_pipeline_test.go b/scripts/buildkite_pipeline_test.go index 32fa74ca..9769f0cb 100644 --- a/scripts/buildkite_pipeline_test.go +++ b/scripts/buildkite_pipeline_test.go @@ -34,6 +34,9 @@ func TestBuildkitePipelineUsesMisePlugin(t *testing.T) { if !strings.Contains(pipeline, "CLEANROOM_DARWIN_VZ_HELPER_SIGN_IDENTIFIER: com.buildkite.cleanroom.darwin-vz") { t.Fatalf("expected .buildkite/pipeline.yml to set the darwin-vz vmnet helper bundle identifier") } + if strings.Contains(pipeline, "CLEANROOM_PRIVILEGED_HELPER_PATH:") { + t.Fatalf("expected .buildkite/pipeline.yml to let ci-cleanroom-e2e.sh auto-detect a compatible helper path") + } } func TestBuildkiteCommandHookIsRemoved(t *testing.T) { diff --git a/scripts/ci-cleanroom-e2e.sh b/scripts/ci-cleanroom-e2e.sh index 3e172cf0..8e5e10a2 100755 --- a/scripts/ci-cleanroom-e2e.sh +++ b/scripts/ci-cleanroom-e2e.sh @@ -1,6 +1,18 @@ #!/usr/bin/env bash set -euo pipefail +# shellcheck source=scripts/dist-layout.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/dist-layout.sh" + +KERNEL_IMAGE="${CLEANROOM_KERNEL_IMAGE:-}" +FIRECRACKER_BINARY="${CLEANROOM_FIRECRACKER_BINARY:-firecracker}" +PREFERRED_PRIVILEGED_HELPER_PATH="${CLEANROOM_PRIVILEGED_HELPER_PATH:-}" +DEFAULT_PRIVILEGED_HELPER_PATH="/usr/local/libexec/cleanroom/cleanroom-root-helper" +LEGACY_PRIVILEGED_HELPER_PATH="/usr/local/sbin/cleanroom-root-helper" +PRIVILEGED_HELPER_PATH="" +CLEANROOM_BIN="./$(cleanroom_stage_bin_path cleanroom)" +DOWNLOAD_HELPER_BIN="./$(cleanroom_stage_bin_path download-sandbox-file)" + ROOT_HELPER_REQUIRED_CAPABILITIES=( firecracker-network ) @@ -14,6 +26,47 @@ run_privileged() { sudo -n "$PRIVILEGED_HELPER_PATH" "$@" } +helper_supports_capability_probe() { + local helper_path="$1" + + [[ -n "$helper_path" ]] || return 1 + [[ -x "$helper_path" ]] || return 1 + sudo -n "$helper_path" capabilities >/dev/null 2>&1 +} + +resolve_privileged_helper_path() { + local candidates=() + local candidate + local seen="" + + if [[ -n "${PREFERRED_PRIVILEGED_HELPER_PATH:-}" ]]; then + candidates+=("$PREFERRED_PRIVILEGED_HELPER_PATH") + fi + candidates+=("$DEFAULT_PRIVILEGED_HELPER_PATH" "$LEGACY_PRIVILEGED_HELPER_PATH") + + for candidate in "${candidates[@]}"; do + [[ -n "$candidate" ]] || continue + if [[ " $seen " == *" $candidate "* ]]; then + continue + fi + seen+=" $candidate" + + if helper_supports_capability_probe "$candidate"; then + if [[ -n "${PREFERRED_PRIVILEGED_HELPER_PATH:-}" && "$candidate" != "$PREFERRED_PRIVILEGED_HELPER_PATH" ]]; then + echo "falling back from $PREFERRED_PRIVILEGED_HELPER_PATH to $candidate for non-interactive helper access" + fi + printf '%s\n' "$candidate" + return 0 + fi + done + + if [[ -n "${PREFERRED_PRIVILEGED_HELPER_PATH:-}" ]]; then + printf '%s\n' "$PREFERRED_PRIVILEGED_HELPER_PATH" + return 0 + fi + printf '%s\n' "$DEFAULT_PRIVILEGED_HELPER_PATH" +} + annotate_root_helper_problem() { local heading="$1" local message="$2" @@ -107,28 +160,14 @@ purge_stale_cleanroom_resources() { done <<< "$nat_rules" } -cleanup() { - if [[ -n "${srv_pid:-}" ]]; then - kill "$srv_pid" >/dev/null 2>&1 || true - wait "$srv_pid" >/dev/null 2>&1 || true - fi - # Give the server a moment to clean up sandboxes (TAPs, iptables, VMs). - sleep 1 - # Best-effort cleanup of any resources the server didn't tear down. - purge_stale_cleanroom_resources 2>/dev/null || true - rm -rf "$tmpdir" -} - main() { - KERNEL_IMAGE="${CLEANROOM_KERNEL_IMAGE:-}" - FIRECRACKER_BINARY="${CLEANROOM_FIRECRACKER_BINARY:-firecracker}" - PRIVILEGED_HELPER_PATH="${CLEANROOM_PRIVILEGED_HELPER_PATH:-/usr/local/sbin/cleanroom-root-helper}" - if [[ -z "$KERNEL_IMAGE" ]]; then echo "CLEANROOM_KERNEL_IMAGE is required for Firecracker e2e CI" >&2 exit 1 fi + PRIVILEGED_HELPER_PATH="$(resolve_privileged_helper_path)" + verify_helper_capabilities echo "--- :broom: Pre-build cleanup" @@ -138,6 +177,17 @@ main() { scripts/build-go.sh tmpdir="$(mktemp -d)" + cleanup() { + if [[ -n "${srv_pid:-}" ]]; then + kill "$srv_pid" >/dev/null 2>&1 || true + wait "$srv_pid" >/dev/null 2>&1 || true + fi + # Give the server a moment to clean up sandboxes (TAPs, iptables, VMs). + sleep 1 + # Best-effort cleanup of any resources the server didn't tear down. + purge_stale_cleanroom_resources 2>/dev/null || true + rm -rf "$tmpdir" + } trap cleanup EXIT export XDG_CONFIG_HOME="$tmpdir/config" @@ -161,7 +211,7 @@ EOF echo " privileged_helper_path: $PRIVILEGED_HELPER_PATH" >> "$XDG_CONFIG_HOME/cleanroom/config.yaml" echo "--- :stethoscope: Doctor" - ./dist/cleanroom doctor --json | tee "$tmpdir/doctor.json" + "$CLEANROOM_BIN" doctor --json | tee "$tmpdir/doctor.json" if grep -q '"status": "fail"' "$tmpdir/doctor.json"; then echo "doctor checks reported failures" >&2 exit 1 @@ -170,29 +220,29 @@ EOF socket_path="$tmpdir/cleanroom.sock" listen_endpoint="unix://$socket_path" -dump_runtime_diagnostics() { - local server_lines="${1:-40}" - if [[ -f "$tmpdir/server.log" ]]; then - echo "--- server log tail ---" >&2 - tail -n "$server_lines" "$tmpdir/server.log" >&2 || true - fi + dump_runtime_diagnostics() { + local server_lines="${1:-40}" + if [[ -f "$tmpdir/server.log" ]]; then + echo "--- server log tail ---" >&2 + tail -n "$server_lines" "$tmpdir/server.log" >&2 || true + fi - # Surface recent Firecracker process logs when provisioning/agent readiness - # flakes occur so failures are diagnosable from CI output alone. - local fc_logs - fc_logs="$(find "$XDG_STATE_HOME"/cleanroom/sandboxes -maxdepth 3 -type f \( -name 'firecracker.stdout.log' -o -name 'firecracker.stderr.log' \) 2>/dev/null | sort | tail -n 6 || true)" - if [[ -n "$fc_logs" ]]; then - echo "--- firecracker log tails ---" >&2 - while IFS= read -r log_file; do - [[ -n "$log_file" ]] || continue - echo "[$log_file]" >&2 - tail -n 30 "$log_file" >&2 || true - done <<< "$fc_logs" - fi -} + # Surface recent Firecracker process logs when provisioning/agent readiness + # flakes occur so failures are diagnosable from CI output alone. + local fc_logs + fc_logs="$(find "$XDG_STATE_HOME"/cleanroom/sandboxes -maxdepth 3 -type f \( -name 'firecracker.stdout.log' -o -name 'firecracker.stderr.log' \) 2>/dev/null | sort | tail -n 6 || true)" + if [[ -n "$fc_logs" ]]; then + echo "--- firecracker log tails ---" >&2 + while IFS= read -r log_file; do + [[ -n "$log_file" ]] || continue + echo "[$log_file]" >&2 + tail -n 30 "$log_file" >&2 || true + done <<< "$fc_logs" + fi + } echo "--- :rocket: Start cleanroom control-plane" - ./dist/cleanroom serve --listen "$listen_endpoint" --gateway-listen ":0" >"$tmpdir/server.log" 2>&1 & + "$CLEANROOM_BIN" serve --listen "$listen_endpoint" --gateway-listen ":0" >"$tmpdir/server.log" 2>&1 & srv_pid=$! for _ in $(seq 1 40); do @@ -213,7 +263,7 @@ smoke_attempt=1 smoke_max_attempts=3 while true; do set +e - ./dist/cleanroom exec --host "$listen_endpoint" -c "$PWD" -- sh -lc 'echo cleanroom-e2e' >"$tmpdir/exec.out" 2>"$tmpdir/exec.err" + "$CLEANROOM_BIN" exec --host "$listen_endpoint" -c "$PWD" -- sh -lc 'echo cleanroom-e2e' >"$tmpdir/exec.out" 2>"$tmpdir/exec.err" smoke_status=$? set -e @@ -239,21 +289,21 @@ while true; do done echo "--- :recycle: Persistent sandbox lifecycle test" -sandbox_id="$(./dist/cleanroom create --host "$listen_endpoint" -c "$PWD" | tr -d '\n')" +sandbox_id="$("$CLEANROOM_BIN" sandbox create --host "$listen_endpoint" | tr -d '\n')" if [[ -z "$sandbox_id" ]]; then - echo "cleanroom create did not return an id" >&2 + echo "sandbox create did not return an id" >&2 exit 1 fi echo "sandbox id: $sandbox_id" -./dist/cleanroom exec --host "$listen_endpoint" -c "$PWD" --in "$sandbox_id" -- sh -lc 'printf persisted-data >/tmp/persist.txt' -./dist/cleanroom exec --host "$listen_endpoint" -c "$PWD" --in "$sandbox_id" -- sh -lc 'cat /tmp/persist.txt' | tee "$tmpdir/persist-read.out" +"$CLEANROOM_BIN" exec --host "$listen_endpoint" -c "$PWD" --in "$sandbox_id" -- sh -lc 'printf persisted-data >/tmp/persist.txt' +"$CLEANROOM_BIN" exec --host "$listen_endpoint" -c "$PWD" --in "$sandbox_id" -- sh -lc 'cat /tmp/persist.txt' | tee "$tmpdir/persist-read.out" if ! grep -q '^persisted-data$' "$tmpdir/persist-read.out"; then echo "expected persisted sandbox file contents from second execution" >&2 exit 1 fi -./dist/download-sandbox-file \ +"$DOWNLOAD_HELPER_BIN" \ --host "$listen_endpoint" \ --sandbox-id "$sandbox_id" \ --path /tmp/persist.txt \ @@ -266,14 +316,14 @@ if ! grep -q '^persisted-data$' "$tmpdir/persist-download.out"; then exit 1 fi -./dist/cleanroom sandbox rm --host "$listen_endpoint" "$sandbox_id" | tee "$tmpdir/sandbox-rm.out" +"$CLEANROOM_BIN" sandbox rm --host "$listen_endpoint" "$sandbox_id" | tee "$tmpdir/sandbox-rm.out" if ! grep -q 'sandbox terminated' "$tmpdir/sandbox-rm.out"; then echo "expected sandbox terminate acknowledgement" >&2 exit 1 fi set +e -./dist/cleanroom exec --host "$listen_endpoint" -c "$PWD" --in "$sandbox_id" -- sh -lc 'echo should-not-run' >"$tmpdir/terminated.out" 2>"$tmpdir/terminated.err" +"$CLEANROOM_BIN" exec --host "$listen_endpoint" -c "$PWD" --in "$sandbox_id" -- sh -lc 'echo should-not-run' >"$tmpdir/terminated.out" 2>"$tmpdir/terminated.err" terminated_status=$? set -e if [[ "$terminated_status" -eq 0 ]]; then @@ -292,7 +342,7 @@ git_gateway_max_attempts=3 while true; do set +e # shellcheck disable=SC2016 - ./dist/cleanroom exec --host "$listen_endpoint" -c "$PWD" -- sh -lc ' + "$CLEANROOM_BIN" exec --host "$listen_endpoint" -c "$PWD" -- sh -lc ' set -eu key="$(env | awk -F= '"'"'/^GIT_CONFIG_KEY_[0-9]+=url\.http:\/\/.+\/git\/github\.com\/\.insteadOf$/ {print $2; exit}'"'"')" @@ -429,7 +479,7 @@ exit_max_attempts=3 status=1 while true; do set +e - ./dist/cleanroom exec --host "$listen_endpoint" -c "$PWD" -- sh -lc 'exit 7' >"$tmpdir/exit7.out" 2>"$tmpdir/exit7.err" + "$CLEANROOM_BIN" exec --host "$listen_endpoint" -c "$PWD" -- sh -lc 'exit 7' >"$tmpdir/exit7.out" 2>"$tmpdir/exit7.err" status=$? set -e if [[ "$status" -eq 7 ]]; then @@ -485,7 +535,7 @@ else fi echo "--- :bar_chart: Execution observability present" -./dist/cleanroom status --last | tee "$tmpdir/status.out" +"$CLEANROOM_BIN" status --last | tee "$tmpdir/status.out" if ! grep -q 'execution-observability.json' "$tmpdir/status.out"; then echo "expected execution-observability.json reference in status output" >&2 exit 1 @@ -548,7 +598,7 @@ EOF fi fi - echo "Firecracker e2e checks passed" +echo "Firecracker e2e checks passed" } if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then diff --git a/scripts/ci-darwin-vz-e2e.sh b/scripts/ci-darwin-vz-e2e.sh index 41ec3a73..c6fb001c 100755 --- a/scripts/ci-darwin-vz-e2e.sh +++ b/scripts/ci-darwin-vz-e2e.sh @@ -1,19 +1,18 @@ #!/usr/bin/env bash set -euo pipefail +# shellcheck source=scripts/dist-layout.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/dist-layout.sh" + DARWIN_VZ_KERNEL_IMAGE="${CLEANROOM_DARWIN_VZ_KERNEL_IMAGE:-}" +CLEANROOM_BIN="./$(cleanroom_stage_bin_path cleanroom)" echo "--- :hammer: Building binaries" scripts/build-go.sh -scripts/build-darwin-vz-helper.sh dist/cleanroom-darwin-vz.app - -# `scripts/build-go.sh` produces host binaries in dist/, but darwin-vz doctor also -# requires a Linux guest agent binary named cleanroom-guest-agent-linux-. -host_arch="$(go env GOARCH)" -GOOS=linux GOARCH="$host_arch" CGO_ENABLED=0 go build -trimpath -o "dist/cleanroom-guest-agent-linux-$host_arch" ./cmd/cleanroom-guest-agent +scripts/build-darwin-vz-helper.sh -helper_path="${CLEANROOM_DARWIN_VZ_HELPER:-$PWD/dist/cleanroom-darwin-vz.app}" +helper_path="${CLEANROOM_DARWIN_VZ_HELPER:-$PWD/$(cleanroom_stage_libexec_path cleanroom-darwin-vz.app)}" if [[ -d "$helper_path" ]]; then helper_executable="$helper_path/Contents/MacOS/cleanroom-darwin-vz" if [[ ! -x "$helper_executable" ]]; then @@ -60,7 +59,7 @@ if [[ -n "$DARWIN_VZ_KERNEL_IMAGE" ]]; then fi echo "--- :stethoscope: Doctor" -./dist/cleanroom doctor --backend darwin-vz --json | tee "$tmpdir/doctor.json" +"$CLEANROOM_BIN" doctor --backend darwin-vz --json | tee "$tmpdir/doctor.json" if grep -q '"status": "fail"' "$tmpdir/doctor.json"; then echo "darwin-vz doctor checks reported failures" >&2 exit 1 @@ -70,7 +69,7 @@ socket_path="$tmpdir/cleanroom.sock" listen_endpoint="unix://$socket_path" echo "--- :rocket: Start cleanroom control-plane" -./dist/cleanroom serve --listen "$listen_endpoint" --gateway-listen ":0" >"$tmpdir/server.log" 2>&1 & +"$CLEANROOM_BIN" serve --listen "$listen_endpoint" --gateway-listen ":0" >"$tmpdir/server.log" 2>&1 & srv_pid=$! for _ in $(seq 1 40); do @@ -87,7 +86,7 @@ if [[ ! -S "$socket_path" ]]; then fi echo "--- :white_check_mark: Launched execution smoke test" -./dist/cleanroom exec --host "$listen_endpoint" --backend darwin-vz -c "$PWD" -- sh -lc 'echo darwin-vz-e2e' | tee "$tmpdir/exec.out" +"$CLEANROOM_BIN" exec --host "$listen_endpoint" --backend darwin-vz -c "$PWD" -- sh -lc 'echo darwin-vz-e2e' | tee "$tmpdir/exec.out" if ! grep -q '^darwin-vz-e2e$' "$tmpdir/exec.out"; then echo "expected darwin-vz smoke-test output missing" >&2 exit 1 @@ -95,7 +94,7 @@ fi echo "--- :warning: Exit code propagation test" set +e -./dist/cleanroom exec --host "$listen_endpoint" --backend darwin-vz -c "$PWD" -- sh -lc 'exit 9' >"$tmpdir/exit9.out" 2>"$tmpdir/exit9.err" +"$CLEANROOM_BIN" exec --host "$listen_endpoint" --backend darwin-vz -c "$PWD" -- sh -lc 'exit 9' >"$tmpdir/exit9.out" 2>"$tmpdir/exit9.err" status=$? set -e if [[ "$status" -ne 9 ]]; then @@ -122,7 +121,7 @@ sandbox: EOF set +e -./dist/cleanroom exec --host "$listen_endpoint" --backend darwin-vz -c "$invalid_policy_dir" -- sh -lc 'echo should-not-run' >"$tmpdir/policy.out" 2>"$tmpdir/policy.err" +"$CLEANROOM_BIN" exec --host "$listen_endpoint" --backend darwin-vz -c "$invalid_policy_dir" -- sh -lc 'echo should-not-run' >"$tmpdir/policy.out" 2>"$tmpdir/policy.err" policy_status=$? set -e if [[ "$policy_status" -eq 0 ]]; then diff --git a/scripts/ci-darwin-vz-vmnet-e2e.sh b/scripts/ci-darwin-vz-vmnet-e2e.sh index 0a72c73f..62d2367a 100755 --- a/scripts/ci-darwin-vz-vmnet-e2e.sh +++ b/scripts/ci-darwin-vz-vmnet-e2e.sh @@ -3,6 +3,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +# shellcheck source=scripts/dist-layout.sh +source "${SCRIPT_DIR}/dist-layout.sh" require_command() { local name="$1" @@ -76,11 +78,18 @@ assert_profile_allows_current_device() { resolve_local_helper_path() { local helper="${CLEANROOM_DARWIN_VZ_HELPER:-}" + local staged_helper legacy_helper if [[ -n "$helper" ]]; then printf '%s\n' "$helper" return 0 fi - printf '%s\n' "$REPO_ROOT/dist/cleanroom-darwin-vz.app" + staged_helper="$REPO_ROOT/$(cleanroom_stage_libexec_path cleanroom-darwin-vz.app)" + legacy_helper="$REPO_ROOT/dist/cleanroom-darwin-vz.app" + if [[ -d "$staged_helper" ]]; then + printf '%s\n' "$staged_helper" + return 0 + fi + printf '%s\n' "$legacy_helper" } setup_buildkite_signing_assets() { @@ -207,14 +216,17 @@ main() { } echo "--- :key: Building vmnet-signed helper" - run_with_macos_user_home env \ + if ! run_with_macos_user_home env \ CLEANROOM_DARWIN_VZ_HELPER_ENTITLEMENTS=cmd/cleanroom-darwin-vz/entitlements-vmnet.plist \ CLEANROOM_DARWIN_VZ_HELPER_SIGN_IDENTITY="$sign_identity" \ CLEANROOM_DARWIN_VZ_HELPER_SIGN_KEYCHAIN="$sign_keychain" \ CLEANROOM_DARWIN_VZ_HELPER_SIGN_IDENTIFIER="${CLEANROOM_DARWIN_VZ_HELPER_SIGN_IDENTIFIER:-com.buildkite.cleanroom.darwin-vz}" \ CLEANROOM_DARWIN_VZ_HELPER_PROVISION_PROFILE="$profile_path" \ CLEANROOM_DARWIN_VZ_HELPER_BUNDLE=1 \ - scripts/build-darwin-vz-helper.sh dist/cleanroom-darwin-vz.app + bash -x scripts/build-darwin-vz-helper.sh dist/cleanroom-darwin-vz.app; then + echo "vmnet helper build failed or timed out; helper target: $helper_path" >&2 + exit 1 + fi fi if [[ ! -d "$helper_path" ]]; then diff --git a/scripts/ci_cleanroom_e2e_behavior_test.go b/scripts/ci_cleanroom_e2e_behavior_test.go index 7aef59c0..37263036 100644 --- a/scripts/ci_cleanroom_e2e_behavior_test.go +++ b/scripts/ci_cleanroom_e2e_behavior_test.go @@ -59,7 +59,7 @@ func TestCiCleanroomE2EVerifyHelperCapabilitiesDetectsMissingCapabilities(t *tes annotationPath := filepath.Join(workDir, "annotation.md") helperPath := filepath.Join(workDir, "helper.sh") - writeExecutable(t, helperPath, "#!/usr/bin/env bash\nprintf 'firecracker-network\\n'\n") + writeExecutable(t, helperPath, "#!/usr/bin/env bash\nprintf 'firecracker-zfs\\n'\n") writeExecutable(t, filepath.Join(binDir, "sudo"), "#!/usr/bin/env bash\n[[ \"$1\" == \"-n\" ]] && shift\nexec \"$@\"\n") writeExecutable(t, filepath.Join(binDir, "buildkite-agent"), "#!/usr/bin/env bash\ncat >\"$ANNOTATION_FILE\"\n") @@ -83,7 +83,7 @@ fi if !strings.Contains(stderr, "missing required capabilities") { t.Fatalf("expected missing capabilities in stderr, got %q", stderr) } - if !strings.Contains(stderr, "firecracker-rootfs") { + if !strings.Contains(stderr, "firecracker-network") { t.Fatalf("expected missing capability name in stderr, got %q", stderr) } diff --git a/scripts/ci_cleanroom_e2e_test.go b/scripts/ci_cleanroom_e2e_test.go index c2eac213..86811f1b 100644 --- a/scripts/ci_cleanroom_e2e_test.go +++ b/scripts/ci_cleanroom_e2e_test.go @@ -16,7 +16,10 @@ func TestCiCleanroomE2EUsesHelperViaNonInteractiveSudo(t *testing.T) { script := string(content) for _, needle := range []string{ - "PRIVILEGED_HELPER_PATH=\"${CLEANROOM_PRIVILEGED_HELPER_PATH:-/usr/local/sbin/cleanroom-root-helper}\"", + "PREFERRED_PRIVILEGED_HELPER_PATH=\"${CLEANROOM_PRIVILEGED_HELPER_PATH:-}\"", + "DEFAULT_PRIVILEGED_HELPER_PATH=\"/usr/local/libexec/cleanroom/cleanroom-root-helper\"", + "LEGACY_PRIVILEGED_HELPER_PATH=\"/usr/local/sbin/cleanroom-root-helper\"", + "PRIVILEGED_HELPER_PATH=\"$(resolve_privileged_helper_path)\"", "sudo -n \"$PRIVILEGED_HELPER_PATH\" \"$@\"", } { if !strings.Contains(script, needle) { @@ -58,6 +61,9 @@ func TestCiCleanroomE2EProbesHelperCapabilitiesInsteadOfHelperDrift(t *testing.T script := string(content) for _, needle := range []string{ "ROOT_HELPER_REQUIRED_CAPABILITIES=(", + "resolve_privileged_helper_path()", + "helper_supports_capability_probe()", + "falling back from $PREFERRED_PRIVILEGED_HELPER_PATH to $candidate", "sudo -n \"$PRIVILEGED_HELPER_PATH\" capabilities", "verify_helper_capabilities", "Roll out the latest helper on the CI host", diff --git a/scripts/ci_darwin_vz_e2e_test.go b/scripts/ci_darwin_vz_e2e_test.go index c7e53845..b331bbdd 100644 --- a/scripts/ci_darwin_vz_e2e_test.go +++ b/scripts/ci_darwin_vz_e2e_test.go @@ -6,7 +6,7 @@ import ( "testing" ) -func TestCiDarwinVZE2EBootstrapsLinuxGuestAgentBinary(t *testing.T) { +func TestCiDarwinVZE2EUsesStagedBuildLayout(t *testing.T) { t.Helper() content, err := os.ReadFile("ci-darwin-vz-e2e.sh") @@ -14,8 +14,16 @@ func TestCiDarwinVZE2EBootstrapsLinuxGuestAgentBinary(t *testing.T) { t.Fatalf("read ci-darwin-vz-e2e.sh: %v", err) } - if !strings.Contains(string(content), "GOOS=linux GOARCH=\"$host_arch\" CGO_ENABLED=0 go build -trimpath -o \"dist/cleanroom-guest-agent-linux-$host_arch\" ./cmd/cleanroom-guest-agent") { - t.Fatalf("expected ci-darwin-vz-e2e.sh to bootstrap cleanroom-guest-agent-linux-$host_arch") + script := string(content) + for _, needle := range []string{ + `source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/dist-layout.sh"`, + `CLEANROOM_BIN="./$(cleanroom_stage_bin_path cleanroom)"`, + `helper_path="${CLEANROOM_DARWIN_VZ_HELPER:-$PWD/$(cleanroom_stage_libexec_path cleanroom-darwin-vz.app)}"`, + `scripts/build-go.sh`, + } { + if !strings.Contains(script, needle) { + t.Fatalf("expected ci-darwin-vz-e2e.sh to contain %q", needle) + } } } @@ -30,8 +38,8 @@ func TestCiDarwinVZE2EForcesNATNetworkMode(t *testing.T) { if !strings.Contains(string(content), "mode: nat") { t.Fatalf("expected ci-darwin-vz-e2e.sh to pin darwin-vz CI to nat mode") } - if !strings.Contains(string(content), `helper_path="${CLEANROOM_DARWIN_VZ_HELPER:-$PWD/dist/cleanroom-darwin-vz.app}"`) { - t.Fatalf("expected ci-darwin-vz-e2e.sh to default to the prebuilt helper app bundle") + if !strings.Contains(string(content), `helper_path="${CLEANROOM_DARWIN_VZ_HELPER:-$PWD/$(cleanroom_stage_libexec_path cleanroom-darwin-vz.app)}"`) { + t.Fatalf("expected ci-darwin-vz-e2e.sh to default to the staged helper app bundle") } } @@ -45,6 +53,7 @@ func TestBuildDarwinVZHelperUsesSharedPackager(t *testing.T) { script := string(content) for _, needle := range []string{ + `OUTPUT_PATH="${1:-${REPO_ROOT}/$(cleanroom_stage_libexec_path cleanroom-darwin-vz.app)}"`, `tmpdir="$(mktemp -d /tmp/cleanroom-darwin-vz-build.XXXXXX)"`, `build_output_path="${tmpdir}/cleanroom-darwin-vz"`, `BUNDLE_MODE="${CLEANROOM_DARWIN_VZ_HELPER_BUNDLE:-1}"`, @@ -106,14 +115,18 @@ func TestCiDarwinVZVMNetE2EUsesBuildkiteSecretsAndVMNetEntitlements(t *testing.T `requested_sign_identity`, `CLEANROOM_DARWIN_VZ_HELPER_SIGN_KEYCHAIN="$sign_keychain"`, `imported signing identity not found in ${system_keychain_path}`, - `run_with_macos_user_home env`, - `run_with_macos_user_home codesign --verify --strict --verbose=2 "$helper_path"`, - "CLEANROOM_DARWIN_VZ_HELPER_ENTITLEMENTS=cmd/cleanroom-darwin-vz/entitlements-vmnet.plist", - "CLEANROOM_DARWIN_VZ_HELPER_BUNDLE=1", - `scripts/build-darwin-vz-helper.sh dist/cleanroom-darwin-vz.app`, - "CLEANROOM_DARWIN_VZ_VMNET_E2E=1", - `go test ./internal/backend/darwinvz -run TestVMNetSharedE2E -v`, - } { + `staged_helper="$REPO_ROOT/$(cleanroom_stage_libexec_path cleanroom-darwin-vz.app)"`, + `legacy_helper="$REPO_ROOT/dist/cleanroom-darwin-vz.app"`, + `if [[ -d "$staged_helper" ]]; then`, + `run_with_macos_user_home env`, + `run_with_macos_user_home codesign --verify --strict --verbose=2 "$helper_path"`, + "CLEANROOM_DARWIN_VZ_HELPER_ENTITLEMENTS=cmd/cleanroom-darwin-vz/entitlements-vmnet.plist", + "CLEANROOM_DARWIN_VZ_HELPER_BUNDLE=1", + `printf '%s\n' "$legacy_helper"`, + `bash -x scripts/build-darwin-vz-helper.sh dist/cleanroom-darwin-vz.app`, + "CLEANROOM_DARWIN_VZ_VMNET_E2E=1", + `go test ./internal/backend/darwinvz -run TestVMNetSharedE2E -v`, + } { if !strings.Contains(script, needle) { t.Fatalf("expected ci-darwin-vz-vmnet-e2e.sh to contain %q", needle) } diff --git a/scripts/cleanroom-root-helper.sh b/scripts/cleanroom-root-helper.sh index 667b5045..5b5719ae 100644 --- a/scripts/cleanroom-root-helper.sh +++ b/scripts/cleanroom-root-helper.sh @@ -3,7 +3,7 @@ set -euo pipefail # Security model: # - This script is a root-owned allowlist for privileged cleanroom operations. -# - Unprivileged callers reach it via `sudo -n /usr/local/sbin/cleanroom-root-helper ...`. +# - Unprivileged callers reach it via `sudo -n /usr/local/libexec/cleanroom/cleanroom-root-helper ...`. # - Roll it out from trusted host administration paths such as bootstrap or SSM, not PR CI. # # Sharp edges: @@ -65,7 +65,6 @@ is_cidr() { local v="$1" [[ "$v" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}/[0-9]{1,2}$ ]] } - is_zfs_dataset() { local v="$1" [[ "$v" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]*(/[A-Za-z0-9][A-Za-z0-9._:-]*)*$ ]] @@ -259,7 +258,6 @@ run_sysctl() { fi die "sysctl: unsupported arguments" } - run_zfs() { [[ "$#" -ge 1 ]] || die "zfs: missing arguments" local bin diff --git a/scripts/dist-layout.sh b/scripts/dist-layout.sh new file mode 100644 index 00000000..fb066dd2 --- /dev/null +++ b/scripts/dist-layout.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +cleanroom_dist_root() { + printf '%s\n' "${CLEANROOM_DIST_DIR:-dist}" +} + +cleanroom_host_goos() { + go env GOOS +} + +cleanroom_host_goarch() { + go env GOARCH +} + +cleanroom_stage_dir() { + local goos="${1:-$(cleanroom_host_goos)}" + local goarch="${2:-$(cleanroom_host_goarch)}" + printf '%s/%s-%s\n' "$(cleanroom_dist_root)" "$goos" "$goarch" +} + +cleanroom_stage_bin_dir() { + printf '%s/bin\n' "$(cleanroom_stage_dir "$@")" +} + +cleanroom_stage_libexec_dir() { + printf '%s/libexec/cleanroom\n' "$(cleanroom_stage_dir "$@")" +} + +cleanroom_stage_bin_path() { + local name="$1" + shift || true + printf '%s/%s\n' "$(cleanroom_stage_bin_dir "$@")" "$name" +} + +cleanroom_stage_libexec_path() { + local name="$1" + shift || true + printf '%s/%s\n' "$(cleanroom_stage_libexec_dir "$@")" "$name" +} + +cleanroom_prefix_bin_dir() { + local prefix="$1" + printf '%s/bin\n' "$prefix" +} + +cleanroom_prefix_libexec_dir() { + local prefix="$1" + printf '%s/libexec/cleanroom\n' "$prefix" +} diff --git a/scripts/install-global.sh b/scripts/install-global.sh index 833b04f5..d2aa45eb 100755 --- a/scripts/install-global.sh +++ b/scripts/install-global.sh @@ -1,6 +1,10 @@ #!/usr/bin/env bash set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/dist-layout.sh +source "${SCRIPT_DIR}/dist-layout.sh" + log() { printf '[install:global] %s\n' "$*" } @@ -15,11 +19,31 @@ require_cmd() { command -v "$cmd" >/dev/null 2>&1 || die "required command not found: ${cmd}" } -INSTALL_DIR="${CLEANROOM_GLOBAL_INSTALL_DIR:-/usr/local/bin}" -DIST_DIR="${CLEANROOM_GLOBAL_DIST_DIR:-dist}" +default_prefix() { + if [[ "$(id -u)" -eq 0 ]]; then + printf '/usr/local\n' + return + fi + printf '%s/.local\n' "$HOME" +} + +PREFIX="${CLEANROOM_GLOBAL_PREFIX:-}" +if [[ -z "$PREFIX" && -n "${CLEANROOM_GLOBAL_INSTALL_DIR:-}" ]]; then + case "${CLEANROOM_GLOBAL_INSTALL_DIR}" in + */bin) PREFIX="${CLEANROOM_GLOBAL_INSTALL_DIR%/bin}" ;; + *) die "CLEANROOM_GLOBAL_INSTALL_DIR must end with /bin; use CLEANROOM_GLOBAL_PREFIX instead" ;; + esac +fi +PREFIX="${PREFIX:-$(default_prefix)}" +DIST_DIR="${CLEANROOM_GLOBAL_DIST_DIR:-}" +if [[ -z "$DIST_DIR" ]]; then + DIST_DIR="$(cleanroom_stage_dir)" +fi +BIN_DIR="$(cleanroom_prefix_bin_dir "$PREFIX")" +LIBEXEC_DIR="$(cleanroom_prefix_libexec_dir "$PREFIX")" + HOST_OS="$(go env GOOS)" HOST_ARCH="$(go env GOARCH)" -ENTITLEMENTS="cmd/cleanroom-darwin-vz/entitlements.plist" declare -a SUDO_CMD=() @@ -31,22 +55,24 @@ run_cmd() { fi } -prepare_install_dir() { - if [ ! -d "$INSTALL_DIR" ]; then - if [ "$(id -u)" -eq 0 ]; then - mkdir -p "$INSTALL_DIR" - else - if mkdir -p "$INSTALL_DIR" 2>/dev/null; then - : +prepare_prefix_dirs() { + for dir in "$BIN_DIR" "$LIBEXEC_DIR"; do + if [ ! -d "$dir" ]; then + if [ "$(id -u)" -eq 0 ]; then + mkdir -p "$dir" else - require_cmd sudo - SUDO_CMD=(sudo) - run_cmd mkdir -p "$INSTALL_DIR" + if mkdir -p "$dir" 2>/dev/null; then + : + else + require_cmd sudo + SUDO_CMD=(sudo) + run_cmd mkdir -p "$dir" + fi fi fi - fi + done - if [ "$(id -u)" -ne 0 ] && [ ! -w "$INSTALL_DIR" ]; then + if [ "$(id -u)" -ne 0 ] && { [ ! -w "$BIN_DIR" ] || [ ! -w "$LIBEXEC_DIR" ]; }; then require_cmd sudo SUDO_CMD=(sudo) fi @@ -61,26 +87,62 @@ install_binary() { log "installed $(basename "$dst") to $dst" } -require_cmd go -prepare_install_dir +install_optional_binary() { + local src="$1" + local dst="$2" -CLEANROOM_BIN="${DIST_DIR}/cleanroom" -GUEST_AGENT_LINUX_BIN="${DIST_DIR}/cleanroom-guest-agent-linux-${HOST_ARCH}" -GUEST_AGENT_BIN="${DIST_DIR}/cleanroom-guest-agent" + [ -f "$src" ] || return 0 + install_binary "$src" "$dst" +} -if [ ! -f "$GUEST_AGENT_BIN" ]; then - GUEST_AGENT_BIN="$GUEST_AGENT_LINUX_BIN" -fi +install_file() { + local src="$1" + local dst="$2" + + [ -f "$src" ] || die "missing file artifact: ${src}" + run_cmd install -m 0644 "$src" "$dst" + log "installed $(basename "$dst") to $dst" +} -install_binary "$CLEANROOM_BIN" "${INSTALL_DIR}/cleanroom" -install_binary "$GUEST_AGENT_LINUX_BIN" "${INSTALL_DIR}/cleanroom-guest-agent-linux-${HOST_ARCH}" -install_binary "$GUEST_AGENT_BIN" "${INSTALL_DIR}/cleanroom-guest-agent" +install_optional_file() { + local src="$1" + local dst="$2" + + [ -f "$src" ] || return 0 + install_file "$src" "$dst" +} + +install_app_bundle() { + local src="$1" + local dst="$2" + + [ -d "$src" ] || die "missing app bundle: ${src}" + run_cmd rm -rf "$dst" + if [ "${#SUDO_CMD[@]}" -gt 0 ]; then + run_cmd ditto "$src" "$dst" + else + ditto "$src" "$dst" + fi + log "installed $(basename "$dst") to $dst" +} + +require_cmd go +prepare_prefix_dirs + +CLEANROOM_BIN="${DIST_DIR}/bin/cleanroom" +DOWNLOAD_HELPER_BIN="${DIST_DIR}/bin/download-sandbox-file" +GUEST_AGENT_LINUX_BIN="${DIST_DIR}/libexec/cleanroom/cleanroom-guest-agent-linux-${HOST_ARCH}" + +install_binary "$CLEANROOM_BIN" "${BIN_DIR}/cleanroom" +install_optional_binary "$DOWNLOAD_HELPER_BIN" "${BIN_DIR}/download-sandbox-file" +install_binary "$GUEST_AGENT_LINUX_BIN" "${LIBEXEC_DIR}/cleanroom-guest-agent-linux-${HOST_ARCH}" if [ "$HOST_OS" = "darwin" ]; then - require_cmd codesign - HELPER_BIN="${DIST_DIR}/cleanroom-darwin-vz" - install_binary "$HELPER_BIN" "${INSTALL_DIR}/cleanroom-darwin-vz" - [ -f "$ENTITLEMENTS" ] || die "missing entitlements file: ${ENTITLEMENTS}" - run_cmd codesign --force --sign - --entitlements "$ENTITLEMENTS" "${INSTALL_DIR}/cleanroom-darwin-vz" - log "signed ${INSTALL_DIR}/cleanroom-darwin-vz" + require_cmd ditto + HELPER_APP="${DIST_DIR}/libexec/cleanroom/cleanroom-darwin-vz.app" + if [ -d "$HELPER_APP" ]; then + install_app_bundle "$HELPER_APP" "${LIBEXEC_DIR}/cleanroom-darwin-vz.app" + fi + install_optional_file "${DIST_DIR}/libexec/cleanroom/entitlements.plist" "${LIBEXEC_DIR}/entitlements.plist" + install_optional_file "${DIST_DIR}/libexec/cleanroom/entitlements-vmnet.plist" "${LIBEXEC_DIR}/entitlements-vmnet.plist" fi diff --git a/scripts/install-go.sh b/scripts/install-go.sh index 196c0820..442be12e 100755 --- a/scripts/install-go.sh +++ b/scripts/install-go.sh @@ -4,7 +4,5 @@ set -euo pipefail VERSION=$(git describe --tags --always --dirty 2>/dev/null || echo dev) BIN_DIR="$(go env GOBIN)" if [ -z "$BIN_DIR" ]; then BIN_DIR="$(go env GOPATH)/bin"; fi -HOST_ARCH="$(go env GOARCH)" mkdir -p "$BIN_DIR" -go install -ldflags "-X main.version=$VERSION" ./cmd/cleanroom ./cmd/cleanroom-guest-agent -GOOS=linux GOARCH="$HOST_ARCH" CGO_ENABLED=0 go build -trimpath -o "$BIN_DIR/cleanroom-guest-agent-linux-$HOST_ARCH" ./cmd/cleanroom-guest-agent +go install -ldflags "-X main.version=$VERSION" ./cmd/cleanroom diff --git a/scripts/install.sh b/scripts/install.sh index 692efcb7..cdc80f6e 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -21,16 +21,16 @@ usage() { Install cleanroom from GitHub releases. Usage: - install.sh [--version ] [--install-dir ] [--repo ] [--no-darwin-helper] + install.sh [--version ] [--prefix ] [--repo ] [--no-darwin-helper] Examples: curl -fsSL https://raw.githubusercontent.com/buildkite/cleanroom/main/scripts/install.sh | bash curl -fsSL https://raw.githubusercontent.com/buildkite/cleanroom/main/scripts/install.sh | \ - bash -s -- --version vX.Y.Z + bash -s -- --version vX.Y.Z --prefix "$HOME/.local" Environment variables: CLEANROOM_VERSION Optional release version (example: vX.Y.Z) - CLEANROOM_INSTALL_DIR Install destination (default: /usr/local/bin) + CLEANROOM_PREFIX Install prefix (default: ~/.local for non-root, /usr/local for root) CLEANROOM_REPO GitHub repo in owner/repo format (default: buildkite/cleanroom) CLEANROOM_INSTALL_DARWIN_HELPER Set to 0 to skip cleanroom-darwin-vz install on macOS CLEANROOM_DARWIN_VZ_HELPER_ENTITLEMENTS Optional entitlements plist for cleanroom-darwin-vz signing or re-signing @@ -38,6 +38,9 @@ Environment variables: CLEANROOM_DARWIN_VZ_HELPER_SIGN_IDENTIFIER Optional codesign identifier for cleanroom-darwin-vz (default: com.buildkite.cleanroom.darwin-vz when embedding a provisioning profile) CLEANROOM_DARWIN_VZ_HELPER_PROVISION_PROFILE Optional provisioning profile to embed when building or re-signing a helper bundle CLEANROOM_DARWIN_VZ_HELPER_SIGN_KEYCHAIN Optional keychain path when using the repo helper packager + +Deprecated compatibility: + CLEANROOM_INSTALL_DIR Legacy alias for a bin dir such as /usr/local/bin; inferred into CLEANROOM_PREFIX USAGE } @@ -83,6 +86,22 @@ normalize_version() { esac } +default_prefix() { + if [ "$(id -u)" -eq 0 ]; then + printf '/usr/local' + return + fi + printf '%s/.local' "$HOME" +} + +infer_prefix_from_install_dir() { + local dir="$1" + case "$dir" in + */bin) printf '%s' "${dir%/bin}" ;; + *) die "legacy install dir must end with /bin: ${dir}" ;; + esac +} + lookup_checksum() { local asset="$1" local checksums_file="$2" @@ -110,7 +129,7 @@ verify_asset_against_checksums() { fi } -extract_binary() { +extract_archive() { local archive="$1" local output_dir="$2" mkdir -p "$output_dir" @@ -119,23 +138,25 @@ extract_binary() { declare -a SUDO_CMD=() -prepare_install_dir() { - if [ ! -d "$INSTALL_DIR" ]; then - if [ "$(id -u)" -eq 0 ]; then - mkdir -p "$INSTALL_DIR" - else - if mkdir -p "$INSTALL_DIR" 2>/dev/null; then - : +prepare_prefix_dirs() { + for dir in "$BIN_DIR" "$LIBEXEC_DIR"; do + if [ ! -d "$dir" ]; then + if [ "$(id -u)" -eq 0 ]; then + mkdir -p "$dir" else - command -v sudo >/dev/null 2>&1 || die "${INSTALL_DIR} does not exist and sudo is unavailable" - SUDO_CMD=(sudo) - "${SUDO_CMD[@]}" mkdir -p "$INSTALL_DIR" + if mkdir -p "$dir" 2>/dev/null; then + : + else + command -v sudo >/dev/null 2>&1 || die "${dir} does not exist and sudo is unavailable" + SUDO_CMD=(sudo) + "${SUDO_CMD[@]}" mkdir -p "$dir" + fi fi fi - fi + done - if [ "$(id -u)" -ne 0 ] && [ ! -w "$INSTALL_DIR" ]; then - command -v sudo >/dev/null 2>&1 || die "${INSTALL_DIR} is not writable and sudo is unavailable" + if [ "$(id -u)" -ne 0 ] && { [ ! -w "$BIN_DIR" ] || [ ! -w "$LIBEXEC_DIR" ]; }; then + command -v sudo >/dev/null 2>&1 || die "${PREFIX} is not writable and sudo is unavailable" SUDO_CMD=(sudo) fi } @@ -146,6 +167,12 @@ install_binary() { "${SUDO_CMD[@]}" install -m 0755 "$src" "$dst" } +install_file() { + local src="$1" + local dst="$2" + "${SUDO_CMD[@]}" install -m 0644 "$src" "$dst" +} + install_app_bundle() { local src="$1" local dst="$2" @@ -179,6 +206,20 @@ package_darwin_helper_with_repo_script() { "${cmd[@]}" } +resolve_archive_path() { + local staged="$1" + local legacy="$2" + if [ -e "$staged" ]; then + printf '%s' "$staged" + return + fi + if [ -e "$legacy" ]; then + printf '%s' "$legacy" + return + fi + printf '%s' "$staged" +} + HOST_OS_RAW="$(uname -s)" HOST_ARCH_RAW="$(uname -m)" @@ -189,13 +230,17 @@ case "$HOST_OS_RAW" in esac case "$HOST_ARCH_RAW" in - x86_64|amd64) HOST_ARCH="x86_64" ;; - arm64|aarch64) HOST_ARCH="arm64" ;; + x86_64|amd64) HOST_ARCH="x86_64"; HOST_GOARCH="amd64" ;; + arm64|aarch64) HOST_ARCH="arm64"; HOST_GOARCH="arm64" ;; *) die "unsupported architecture: ${HOST_ARCH_RAW}" ;; esac VERSION="${CLEANROOM_VERSION:-}" -INSTALL_DIR="${CLEANROOM_INSTALL_DIR:-/usr/local/bin}" +PREFIX="${CLEANROOM_PREFIX:-}" +if [ -z "$PREFIX" ] && [ -n "${CLEANROOM_INSTALL_DIR:-}" ]; then + PREFIX="$(infer_prefix_from_install_dir "$CLEANROOM_INSTALL_DIR")" +fi +PREFIX="${PREFIX:-$(default_prefix)}" REPO="${CLEANROOM_REPO:-buildkite/cleanroom}" INSTALL_DARWIN_HELPER="${CLEANROOM_INSTALL_DARWIN_HELPER:-1}" @@ -206,9 +251,15 @@ while [ "$#" -gt 0 ]; do VERSION="$2" shift 2 ;; + --prefix) + [ "$#" -ge 2 ] || die "--prefix requires a value" + PREFIX="$2" + shift 2 + ;; --install-dir) [ "$#" -ge 2 ] || die "--install-dir requires a value" - INSTALL_DIR="$2" + PREFIX="$(infer_prefix_from_install_dir "$2")" + warn "--install-dir is deprecated; use --prefix instead" shift 2 ;; --repo) @@ -234,6 +285,10 @@ require_cmd curl require_cmd tar require_cmd awk +PREFIX="${PREFIX%/}" +BIN_DIR="${PREFIX}/bin" +LIBEXEC_DIR="${PREFIX}/libexec/cleanroom" + VERSION="$(normalize_version "$VERSION")" if [ "$VERSION" = "latest" ]; then RELEASE_BASE="https://github.com/${REPO}/releases/latest/download" @@ -247,7 +302,7 @@ WORK_DIR="$(mktemp -d)" trap 'rm -rf "$WORK_DIR"' EXIT DARWIN_HELPER_INSTALLED=0 -log "Installing cleanroom from ${REPO} (${RELEASE_LABEL}) for ${HOST_OS}/${HOST_ARCH}" +log "Installing cleanroom from ${REPO} (${RELEASE_LABEL}) for ${HOST_OS}/${HOST_ARCH} into ${PREFIX}" CHECKSUMS_PATH="${WORK_DIR}/checksums.txt" download "${RELEASE_BASE}/checksums.txt" "$CHECKSUMS_PATH" @@ -259,19 +314,23 @@ download "${RELEASE_BASE}/${CLEANROOM_ASSET}" "$CLEANROOM_ARCHIVE_PATH" verify_asset_against_checksums "$CLEANROOM_ASSET" "$CLEANROOM_ARCHIVE_PATH" "$CHECKSUMS_PATH" CLEANROOM_EXTRACT_DIR="${WORK_DIR}/cleanroom" -extract_binary "$CLEANROOM_ARCHIVE_PATH" "$CLEANROOM_EXTRACT_DIR" -[ -f "${CLEANROOM_EXTRACT_DIR}/cleanroom" ] || die "cleanroom binary missing in ${CLEANROOM_ASSET}" -[ -f "${CLEANROOM_EXTRACT_DIR}/cleanroom-guest-agent" ] || die "cleanroom-guest-agent missing in ${CLEANROOM_ASSET}" +extract_archive "$CLEANROOM_ARCHIVE_PATH" "$CLEANROOM_EXTRACT_DIR" + +CLEANROOM_BIN_SRC="$(resolve_archive_path "${CLEANROOM_EXTRACT_DIR}/bin/cleanroom" "${CLEANROOM_EXTRACT_DIR}/cleanroom")" +GUEST_AGENT_LINUX_SRC="$(resolve_archive_path "${CLEANROOM_EXTRACT_DIR}/libexec/cleanroom/cleanroom-guest-agent-linux-${HOST_GOARCH}" "${CLEANROOM_EXTRACT_DIR}/cleanroom-guest-agent")" -prepare_install_dir -install_binary "${CLEANROOM_EXTRACT_DIR}/cleanroom" "${INSTALL_DIR}/cleanroom" -install_binary "${CLEANROOM_EXTRACT_DIR}/cleanroom-guest-agent" "${INSTALL_DIR}/cleanroom-guest-agent" +[ -f "${CLEANROOM_BIN_SRC}" ] || die "cleanroom binary missing in ${CLEANROOM_ASSET}" +[ -f "${GUEST_AGENT_LINUX_SRC}" ] || die "cleanroom-guest-agent-linux-${HOST_GOARCH} missing in ${CLEANROOM_ASSET}" + +prepare_prefix_dirs +install_binary "${CLEANROOM_BIN_SRC}" "${BIN_DIR}/cleanroom" +install_binary "${GUEST_AGENT_LINUX_SRC}" "${LIBEXEC_DIR}/cleanroom-guest-agent-linux-${HOST_GOARCH}" if [ "$HOST_OS" = "Darwin" ] && [ "$INSTALL_DARWIN_HELPER" != "0" ]; then - HELPER_BUNDLE_SRC="${CLEANROOM_EXTRACT_DIR}/cleanroom-darwin-vz.app" - HELPER_BINARY_SRC="${CLEANROOM_EXTRACT_DIR}/cleanroom-darwin-vz" - HELPER_ENTITLEMENTS_VMNET_PATH="${CLEANROOM_EXTRACT_DIR}/entitlements-vmnet.plist" - HELPER_ENTITLEMENTS_DEFAULT_PATH="${CLEANROOM_EXTRACT_DIR}/entitlements.plist" + HELPER_BUNDLE_SRC="$(resolve_archive_path "${CLEANROOM_EXTRACT_DIR}/libexec/cleanroom/cleanroom-darwin-vz.app" "${CLEANROOM_EXTRACT_DIR}/cleanroom-darwin-vz.app")" + HELPER_BINARY_SRC="$(resolve_archive_path "${CLEANROOM_EXTRACT_DIR}/libexec/cleanroom/cleanroom-darwin-vz" "${CLEANROOM_EXTRACT_DIR}/cleanroom-darwin-vz")" + HELPER_ENTITLEMENTS_VMNET_PATH="$(resolve_archive_path "${CLEANROOM_EXTRACT_DIR}/libexec/cleanroom/entitlements-vmnet.plist" "${CLEANROOM_EXTRACT_DIR}/entitlements-vmnet.plist")" + HELPER_ENTITLEMENTS_DEFAULT_PATH="$(resolve_archive_path "${CLEANROOM_EXTRACT_DIR}/libexec/cleanroom/entitlements.plist" "${CLEANROOM_EXTRACT_DIR}/entitlements.plist")" HELPER_SIGN_IDENTITY="${CLEANROOM_DARWIN_VZ_HELPER_SIGN_IDENTITY:-}" HELPER_SIGN_IDENTIFIER="${CLEANROOM_DARWIN_VZ_HELPER_SIGN_IDENTIFIER:-}" HELPER_SIGN_KEYCHAIN="${CLEANROOM_DARWIN_VZ_HELPER_SIGN_KEYCHAIN:-}" @@ -306,7 +365,7 @@ if [ "$HOST_OS" = "Darwin" ] && [ "$INSTALL_DARWIN_HELPER" != "0" ]; then if [ -d "${HELPER_BUNDLE_SRC}" ]; then require_cmd ditto - HELPER_BUNDLE_DIR="${INSTALL_DIR}/cleanroom-darwin-vz.app" + HELPER_BUNDLE_DIR="${LIBEXEC_DIR}/cleanroom-darwin-vz.app" install_app_bundle "${HELPER_BUNDLE_SRC}" "${HELPER_BUNDLE_DIR}" HELPER_INSTALL_LOG_PATH="${HELPER_BUNDLE_DIR}" @@ -317,7 +376,7 @@ if [ "$HOST_OS" = "Darwin" ] && [ "$INSTALL_DARWIN_HELPER" != "0" ]; then [ -f "${HELPER_ENTITLEMENTS_PATH}" ] || die "entitlements plist missing: ${HELPER_ENTITLEMENTS_PATH}" HELPER_BUNDLE_PROFILE_DEST="${HELPER_BUNDLE_DIR}/Contents/embedded.provisionprofile" if [ -n "${HELPER_PROVISION_PROFILE}" ]; then - install_binary "${HELPER_PROVISION_PROFILE}" "${HELPER_BUNDLE_PROFILE_DEST}" + install_file "${HELPER_PROVISION_PROFILE}" "${HELPER_BUNDLE_PROFILE_DEST}" else "${SUDO_CMD[@]}" rm -f "${HELPER_BUNDLE_PROFILE_DEST}" fi @@ -333,22 +392,28 @@ if [ "$HOST_OS" = "Darwin" ] && [ "$INSTALL_DARWIN_HELPER" != "0" ]; then fi fi + if [ -f "${HELPER_ENTITLEMENTS_DEFAULT_PATH}" ]; then + install_file "${HELPER_ENTITLEMENTS_DEFAULT_PATH}" "${LIBEXEC_DIR}/entitlements.plist" + fi + if [ -f "${HELPER_ENTITLEMENTS_VMNET_PATH}" ]; then + install_file "${HELPER_ENTITLEMENTS_VMNET_PATH}" "${LIBEXEC_DIR}/entitlements-vmnet.plist" + fi DARWIN_HELPER_INSTALLED=1 else [ -f "${HELPER_BINARY_SRC}" ] || die "cleanroom-darwin-vz missing in ${CLEANROOM_ASSET}" HELPER_SIGN_IDENTITY="${HELPER_SIGN_IDENTITY:--}" if [ -n "${HELPER_PROVISION_PROFILE}" ]; then - HELPER_SIGN_TARGET="${INSTALL_DIR}/cleanroom-darwin-vz.app" + HELPER_SIGN_TARGET="${LIBEXEC_DIR}/cleanroom-darwin-vz.app" else - HELPER_SIGN_TARGET="${INSTALL_DIR}/cleanroom-darwin-vz" + HELPER_SIGN_TARGET="${LIBEXEC_DIR}/cleanroom-darwin-vz" fi HELPER_INSTALL_LOG_PATH="${HELPER_SIGN_TARGET}" if ! package_darwin_helper_with_repo_script "${HELPER_BINARY_SRC}" "${HELPER_SIGN_TARGET}"; then require_cmd codesign [ -f "${HELPER_ENTITLEMENTS_PATH}" ] || die "entitlements plist missing: ${HELPER_ENTITLEMENTS_PATH}" if [ -n "${HELPER_PROVISION_PROFILE}" ]; then - HELPER_BUNDLE_DIR="${INSTALL_DIR}/cleanroom-darwin-vz.app" + HELPER_BUNDLE_DIR="${LIBEXEC_DIR}/cleanroom-darwin-vz.app" HELPER_EXECUTABLE_PATH="${HELPER_BUNDLE_DIR}/Contents/MacOS/cleanroom-darwin-vz" HELPER_INFO_PLIST_PATH="${HELPER_BUNDLE_DIR}/Contents/Info.plist" "${SUDO_CMD[@]}" mkdir -p "$(dirname "${HELPER_EXECUTABLE_PATH}")" @@ -373,11 +438,11 @@ if [ "$HOST_OS" = "Darwin" ] && [ "$INSTALL_DARWIN_HELPER" != "0" ]; then EOF" install_binary "${HELPER_BINARY_SRC}" "${HELPER_EXECUTABLE_PATH}" - install_binary "${HELPER_PROVISION_PROFILE}" "${HELPER_BUNDLE_DIR}/Contents/embedded.provisionprofile" + install_file "${HELPER_PROVISION_PROFILE}" "${HELPER_BUNDLE_DIR}/Contents/embedded.provisionprofile" HELPER_SIGN_TARGET="${HELPER_BUNDLE_DIR}" HELPER_INSTALL_LOG_PATH="${HELPER_BUNDLE_DIR}" else - install_binary "${HELPER_BINARY_SRC}" "${INSTALL_DIR}/cleanroom-darwin-vz" + install_binary "${HELPER_BINARY_SRC}" "${LIBEXEC_DIR}/cleanroom-darwin-vz" fi codesign_cmd=("${SUDO_CMD[@]}" codesign --force --sign "${HELPER_SIGN_IDENTITY}" --entitlements "${HELPER_ENTITLEMENTS_PATH}") if [ -n "${HELPER_SIGN_KEYCHAIN}" ]; then @@ -393,13 +458,13 @@ EOF" fi fi -log "Installed cleanroom to ${INSTALL_DIR}/cleanroom" -log "Installed cleanroom-guest-agent to ${INSTALL_DIR}/cleanroom-guest-agent" +log "Installed cleanroom to ${BIN_DIR}/cleanroom" +log "Installed cleanroom runtime assets to ${LIBEXEC_DIR}" if [ "$DARWIN_HELPER_INSTALLED" = "1" ]; then log "Installed cleanroom-darwin-vz to ${HELPER_INSTALL_LOG_PATH}" fi case ":${PATH}:" in - *":${INSTALL_DIR}:"*) ;; - *) warn "${INSTALL_DIR} is not in PATH" ;; + *":${BIN_DIR}:"*) ;; + *) warn "${BIN_DIR} is not in PATH" ;; esac diff --git a/scripts/package-darwin-vz-helper.sh b/scripts/package-darwin-vz-helper.sh index 5f99064d..9006a3f4 100755 --- a/scripts/package-darwin-vz-helper.sh +++ b/scripts/package-darwin-vz-helper.sh @@ -97,6 +97,7 @@ codesign_target() { args+=(-i "${SIGN_IDENTIFIER}") fi args+=("${target}") + echo "[package-darwin-vz-helper] codesigning ${target}" codesign "${args[@]}" } @@ -163,6 +164,7 @@ if [[ -n "${BUNDLE_MODE}" || "${OUTPUT_PATH}" == *.app ]]; then else rm -f "${PROFILE_DEST}" fi + echo "[package-darwin-vz-helper] prepared app bundle at ${APP_PATH}" codesign_target "${APP_PATH}" exit 0 fi @@ -172,6 +174,7 @@ PROFILE_DEST="${OUTPUT_PATH}.provisionprofile" rm -rf "${APP_PATH}" mkdir -p "$(dirname "${OUTPUT_PATH}")" copy_file "${SOURCE_EXECUTABLE_PATH}" "${OUTPUT_PATH}" +echo "[package-darwin-vz-helper] prepared binary at ${OUTPUT_PATH}" codesign_target "${OUTPUT_PATH}" if [[ -n "${PROVISION_PROFILE}" ]]; then install -m 0644 "${PROVISION_PROFILE}" "${PROFILE_DEST}" diff --git a/scripts/prepare-firecracker-image.sh b/scripts/prepare-firecracker-image.sh index 22ed548d..c335dcc0 100755 --- a/scripts/prepare-firecracker-image.sh +++ b/scripts/prepare-firecracker-image.sh @@ -1,6 +1,10 @@ #!/usr/bin/env bash set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/dist-layout.sh +source "${SCRIPT_DIR}/dist-layout.sh" + usage() { cat < 65535 exit 1 fi -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" if [[ -z "$AGENT_BINARY" ]]; then - AGENT_BINARY="$REPO_ROOT/dist/cleanroom-guest-agent" + host_arch="$(cleanroom_host_goarch)" + AGENT_BINARY="$REPO_ROOT/$(cleanroom_stage_libexec_path "cleanroom-guest-agent-linux-$host_arch")" fi if [[ ! -x "$AGENT_BINARY" ]]; then