feat(panoramic): add native macOS runtime for integration tests

.githooks/pre-commit

@@ -7,6 +7,7 @@ make check-fmt
 make check-clippy
 make check-licenses
 make check-deny
+make check-docs
 make generate-api-docs
 
 echo "[*] Pre-commit checks passed."

.gitlab-ci.yml

@@ -14,6 +14,15 @@ stages:
 #
 # This mostly controls how we tag our ADP container images and set various bits of metadata.
 workflow:
+  # Auto-cancel any interruptible jobs from a previous pipeline when a new commit lands on the
+  # same ref. The default ('conservative') only cancels the pipeline if no non-interruptible
+  # job has started yet — since our Linux jobs are non-interruptible and start immediately,
+  # that mode effectively never cancels anything. 'interruptible' cancels just the jobs marked
+  # interruptible: true (currently the macOS unit + integration jobs, which run on scarce
+  # bare-metal runner capacity); non-interruptible jobs continue to completion as before.
+  # https://docs.gitlab.com/ci/yaml/#workflowauto_cancelon_new_commit
+  auto_cancel:
+    on_new_commit: interruptible
   rules:
     - if: $CI_COMMIT_TAG == null
       variables:
@@ -151,8 +160,23 @@ default:
     KUBERNETES_MEMORY_REQUEST: "8Gi"
     KUBERNETES_MEMORY_LIMIT: "12Gi"
 
+# Shared mixins for macOS runner jobs.
+#
+# arm64 jobs use the shared virtualized macOS Tart runner pool (`macos:tart`). Each job runs
+# in a fresh VM, which gives clean isolation between pipelines, decouples the host toolchain,
+# and lets two jobs share a single `mac2.metal` host. amd64 still uses the dedicated bare-metal
+# pool (`macos:sonoma-amd64`); migrating it requires equivalent infra that isn't yet in place.
+#
+# `interruptible: true` is set here so every macOS job inherits it. macOS runner capacity in
+# the GitLab fleet is finite; auto-cancelling superseded pipelines (for example, after a quick
+# fixup push to an open PR) frees the runner immediately instead of holding the slot for the
+# duration of the now-stale run. See:
+# https://docs.gitlab.com/ci/yaml/#interruptible
 .macos-amd64-test-job:
   tags: ["macos:sonoma-amd64", "specific:true"]
+  interruptible: true
 
 .macos-arm64-test-job:
-  tags: ["macos:sonoma-arm64", "specific:true"]
+  tags: ["macos:tart"]
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/ci-platform-machine-images/tart-vm:saluki-sonoma-latest
+  interruptible: true

.gitlab/e2e.yml

@@ -116,3 +116,59 @@ test-integration:
     - docker pull ${SALUKI_IMAGE_REPO_BASE}/bundled-agent-adp:${CI_COMMIT_SHA}
     - docker tag ${SALUKI_IMAGE_REPO_BASE}/bundled-agent-adp:${CI_COMMIT_SHA} saluki-images/datadog-agent:testing-devel
     - make test-integration-quick
+
+# Runs the subset of integration tests that have opted in to the `mac` runtime
+# directly on a bare-metal macOS runner. No Docker, no virtualization: panoramic spawns ADP
+# (and the Core Agent for converged tests) as real macOS processes against a per-test temp
+# state directory. The Datadog Agent is installed into a sandbox under /tmp/saluki-dda by
+# the Makefile target (idempotent: re-uses the install across runs if the pinned version is
+# already present). The sandbox install never touches /opt/datadog-agent, so any system
+# install on the runner is left alone.
+.test-integration-macos-base:
+  stage: e2e
+  needs: []
+  retry: 2
+  timeout: 30m
+  artifacts:
+    expire_in: 1 week
+    paths:
+      - integration-logs/
+    when: always
+  variables:
+    PANORAMIC_LOG_DIR: integration-logs
+  before_script:
+    # Defensive: clean up any leftover Agent/ADP processes from prior runs on this shared
+    # runner. All test-Agent ports are shifted out of the canonical range (see
+    # panoramic::test_env::port_isolation_env), so we don't need to touch a system
+    # install at /opt/datadog-agent. We do need to sweep:
+    #   - our own Core Agent sandbox under /tmp/saluki-dda (trace-agent / process-agent
+    #     children that survived a non-graceful job termination still hold our shifted ports)
+    #   - any stranded agent-data-plane process from a prior pipeline (built into
+    #     $CI_PROJECT_DIR/target/release/, holds UDP 58125 / TCP 5100–5102 etc. across runs)
+    - sudo pkill -9 -f /tmp/saluki-dda/ || true
+    - sudo pkill -9 -f /target/release/agent-data-plane || true
+  script:
+    - make test-integration-macos-ci
+  after_script:
+    # Collect host-level diagnostics into the artifact so we have something to debug from
+    # when something fails outside panoramic's per-test log capture (bootstrap-Agent failures,
+    # system state, stranded processes from this run). Runs whether the test step passed or
+    # failed.
+    - mkdir -p integration-logs/host-diag
+    - sw_vers > integration-logs/host-diag/sw_vers.txt 2>&1 || true
+    - uname -a > integration-logs/host-diag/uname.txt 2>&1 || true
+    - mount > integration-logs/host-diag/mount.txt 2>&1 || true
+    - df -h > integration-logs/host-diag/df.txt 2>&1 || true
+    - ps -axo pid,ppid,user,command > integration-logs/host-diag/ps.txt 2>&1 || true
+    - cp /tmp/saluki-agent-bootstrap.log integration-logs/host-diag/saluki-agent-bootstrap.log 2>/dev/null || true
+    - ls -la /tmp/saluki-dda/datadog-agent/etc/ > integration-logs/host-diag/sandbox-etc.txt 2>&1 || true
+
+test-integration-macos-arm64:
+  extends:
+    - .macos-arm64-test-job
+    - .test-integration-macos-base
+
+test-integration-macos-amd64:
+  extends:
+    - .macos-amd64-test-job
+    - .test-integration-macos-base

.vale/styles/config/vocabularies/technical/accept.txt

@@ -228,3 +228,4 @@ libtest
 mpmc
 dhat
 profiler
+launchd

Makefile

@@ -22,6 +22,12 @@ export ADP_APP_BUILD_TIME := $(APP_BUILD_TIME)
 # ADP-specific settings used when running.
 export ADP_STANDALONE_IPC_CERT_FILE := /tmp/adp-ipc-cert.pem
 
+# macOS integration-test settings.
+MACOS_TEST_AGENT_VERSION ?= 7.78.0
+MACOS_TEST_AGENT_DMG_DIR ?= /tmp/saluki-dda-dmg-cache
+MACOS_TEST_AGENT_DMG_URL ?= https://s3.amazonaws.com/dd-agent/datadog-agent-$(MACOS_TEST_AGENT_VERSION)-1.$(shell uname -m).dmg
+MACOS_TEST_AGENT_INSTALL_DIR ?= /tmp/saluki-dda/datadog-agent
+
 # General build settings used for tooling, etc.
 export GO_BUILD_IMAGE ?= golang:1.23-bullseye
 export GO_APP_IMAGE ?= ubuntu:24.04
@@ -569,6 +575,96 @@ list-integration-tests: build-panoramic
 list-integration-tests: ## Lists available ADP integration tests
 	@target/release/panoramic list -d $(shell pwd)/test/integration/cases
 
+.PHONY: build-adp-host
+build-adp-host: check-rust-build-tools
+build-adp-host: ## Builds the agent-data-plane binary for the current host (release profile)
+	@echo "[*] Building agent-data-plane (release, host target)..."
+	@APP_FULL_NAME="$(ADP_APP_FULL_NAME)" \
+		APP_SHORT_NAME="$(ADP_APP_SHORT_NAME)" \
+		APP_IDENTIFIER="$(ADP_APP_IDENTIFIER)" \
+		APP_GIT_HASH="$(ADP_APP_GIT_HASH)" \
+		APP_VERSION="$(ADP_APP_VERSION)" \
+		APP_BUILD_DATE="$(ADP_APP_BUILD_DATE)" \
+		cargo build --release --bin agent-data-plane
+
+.PHONY: test-integration-macos-run
+test-integration-macos-run: ## Runs the macOS host-process integration tests using already-built binaries (assumes target/release/{panoramic,agent-data-plane} exist). Defaults to all `mac`-runtime-eligible tests; narrow with CASE=<name>.
+	@echo "[*] Running macOS host-process integration tests..."
+	@ADP_BINARY_PATH="$(CURDIR)/target/release/agent-data-plane" \
+		CORE_AGENT_BINARY_PATH="$(MACOS_TEST_AGENT_INSTALL_DIR)/bin/agent/agent" \
+		target/release/panoramic run -d "$(CURDIR)/test/integration/cases" \
+		$(if $(CASE),-t $(CASE)) --no-tui -p 1 \
+		$(if $(PANORAMIC_LOG_DIR),-l $(PANORAMIC_LOG_DIR))
+
+.PHONY: provision-macos-test-env
+provision-macos-test-env: ## Installs the pinned Datadog Agent ($(MACOS_TEST_AGENT_VERSION)) into $(MACOS_TEST_AGENT_INSTALL_DIR) (a sandbox under /tmp) and bootstraps the IPC cert. Idempotent: re-uses the install if it already matches the pinned version.
+	@echo "[*] Provisioning macOS test environment..."
+	@if [ "$(shell uname -s)" != "Darwin" ]; then \
+		echo "provision-macos-test-env only runs on macOS hosts" >&2; exit 1; \
+	fi
+	@if [ -x $(MACOS_TEST_AGENT_INSTALL_DIR)/bin/agent/agent ] && \
+	   [ "$$($(MACOS_TEST_AGENT_INSTALL_DIR)/bin/agent/agent version 2>/dev/null | awk '{print $$2}')" = "$(MACOS_TEST_AGENT_VERSION)" ]; then \
+		echo "[*] Datadog Agent $(MACOS_TEST_AGENT_VERSION) already extracted to $(MACOS_TEST_AGENT_INSTALL_DIR)"; \
+	else \
+		echo "[*] Installing Datadog Agent $(MACOS_TEST_AGENT_VERSION) into $(MACOS_TEST_AGENT_INSTALL_DIR)..."; \
+		mkdir -p $(MACOS_TEST_AGENT_DMG_DIR); \
+		DMG_PATH=$(MACOS_TEST_AGENT_DMG_DIR)/datadog-agent-$(MACOS_TEST_AGENT_VERSION).dmg; \
+		if [ ! -f "$$DMG_PATH" ]; then \
+			curl -fL "$(MACOS_TEST_AGENT_DMG_URL)" -o "$$DMG_PATH"; \
+		fi; \
+		MOUNT_DIR=$$(mktemp -d /tmp/saluki-dda-mount-XXXXXX); \
+		hdiutil attach "$$DMG_PATH" -mountpoint "$$MOUNT_DIR" -nobrowse >/dev/null; \
+		PKG=$$(find "$$MOUNT_DIR" -name '*.pkg' | head -1); \
+		EXPAND_DIR=$$(mktemp -d /tmp/saluki-dda-expand-XXXXXX) && rm -rf "$$EXPAND_DIR"; \
+		pkgutil --expand-full "$$PKG" "$$EXPAND_DIR" >/dev/null; \
+		hdiutil detach "$$MOUNT_DIR" >/dev/null; \
+		rmdir "$$MOUNT_DIR" 2>/dev/null || true; \
+		PAYLOAD_DIR=$$(find "$$EXPAND_DIR" -type d -name Payload | head -1); \
+		if [ -z "$$PAYLOAD_DIR" ] || [ ! -x "$$PAYLOAD_DIR/bin/agent/agent" ]; then \
+			echo "ERROR: pkg payload did not contain bin/agent/agent. Expanded layout:" >&2; \
+			find "$$EXPAND_DIR" -maxdepth 3 -type d >&2; \
+			exit 1; \
+		fi; \
+		rm -rf $(MACOS_TEST_AGENT_INSTALL_DIR); \
+		mkdir -p $$(dirname $(MACOS_TEST_AGENT_INSTALL_DIR)); \
+		mv "$$PAYLOAD_DIR" $(MACOS_TEST_AGENT_INSTALL_DIR); \
+		rm -rf "$$EXPAND_DIR"; \
+		test -x $(MACOS_TEST_AGENT_INSTALL_DIR)/bin/agent/agent; \
+	fi
+	@if [ ! -f $(MACOS_TEST_AGENT_INSTALL_DIR)/etc/ipc_cert.pem ] || [ ! -f $(MACOS_TEST_AGENT_INSTALL_DIR)/etc/auth_token ]; then \
+		echo "[*] Bootstrapping IPC cert + auth_token by running the Agent briefly..."; \
+		mkdir -p $(MACOS_TEST_AGENT_INSTALL_DIR)/etc $(MACOS_TEST_AGENT_INSTALL_DIR)/run; \
+		touch $(MACOS_TEST_AGENT_INSTALL_DIR)/etc/datadog.yaml; \
+		DD_API_KEY=bootstrap DD_HOSTNAME=bootstrap \
+			DD_RUN_PATH=$(MACOS_TEST_AGENT_INSTALL_DIR)/run \
+			DD_AUTH_TOKEN_FILE_PATH=$(MACOS_TEST_AGENT_INSTALL_DIR)/etc/auth_token \
+			DD_IPC_CERT_FILE_PATH=$(MACOS_TEST_AGENT_INSTALL_DIR)/etc/ipc_cert.pem \
+			DD_CMD_PORT=55001 DD_GUI_PORT=-1 \
+			DD_EXPVAR_PORT=55000 DD_APM_RECEIVER_PORT=58126 \
+			DD_PROCESS_CONFIG_CMD_PORT=56062 DD_AGENT_IPC_PORT=55004 \
+			DD_DOGSTATSD_PORT=58125 \
+			$(MACOS_TEST_AGENT_INSTALL_DIR)/bin/agent/agent run -c $(MACOS_TEST_AGENT_INSTALL_DIR)/etc >/tmp/saluki-agent-bootstrap.log 2>&1 & \
+		AGENT_PID=$$!; \
+		for i in $$(seq 1 30); do \
+			sleep 1; \
+			if [ -f $(MACOS_TEST_AGENT_INSTALL_DIR)/etc/ipc_cert.pem ] && [ -f $(MACOS_TEST_AGENT_INSTALL_DIR)/etc/auth_token ]; then break; fi; \
+		done; \
+		kill $$AGENT_PID 2>/dev/null || true; \
+		wait $$AGENT_PID 2>/dev/null || true; \
+		if [ ! -f $(MACOS_TEST_AGENT_INSTALL_DIR)/etc/ipc_cert.pem ]; then \
+			echo "ERROR: bootstrap Agent did not write the IPC cert. Bootstrap log:" >&2; \
+			cat /tmp/saluki-agent-bootstrap.log >&2 2>/dev/null || true; \
+			exit 1; \
+		fi; \
+	else \
+		echo "[*] IPC cert already present at $(MACOS_TEST_AGENT_INSTALL_DIR)/etc/ipc_cert.pem"; \
+	fi
+	@echo "[*] macOS test environment ready."
+	@echo "[*] Agent binary: $(MACOS_TEST_AGENT_INSTALL_DIR)/bin/agent/agent"
+
+.PHONY: test-integration-macos-ci
+test-integration-macos-ci: build-panoramic build-adp-host provision-macos-test-env test-integration-macos-run ## CI entry point: builds binaries, ensures Agent + cert are provisioned, then runs the `mac`-runtime integration tests
+
 .PHONY: ensure-rust-miri
 ensure-rust-miri:
 ifeq ($(shell command -v rustup >/dev/null || echo not-found), not-found)

bin/correctness/airlock/Cargo.toml

@@ -15,8 +15,17 @@ home = { workspace = true }
 saluki-error = { workspace = true }
 tokio = { workspace = true, features = [
   "fs",
+  "io-util",
   "macros",
+  "process",
   "rt",
   "rt-multi-thread",
 ] }
+tokio-util = { workspace = true }
 tracing = { workspace = true }
+
+# Unix process driver uses libc's killpg/SIGTERM/SIGKILL inside #[cfg(unix)] blocks. The wider
+# correctness/integration test suite is only operated on Linux/Docker today, so this gate exists
+# more as a forward-looking marker than as something a Windows build actually depends on.
+[target.'cfg(unix)'.dependencies]
+libc = { workspace = true }

bin/correctness/airlock/src/lib.rs

@@ -1,3 +1,4 @@
 pub mod config;
 pub mod docker;
 pub mod driver;
+pub mod unix;