Vary datadog.yaml in test/antithesis, assert aliveness

This PR introduces variation in the datadog.yaml we use under test in
the antithesis rig. The goal here is to explore variation in buffer
sizes etc and also startup panics on truly weird configs.

ADP aliveness on bootup via the way it is rigged into the compose cluster
and we assert a 'sometimes' check to forwarding in datadog/io.rs. This
later 'sometimes' acts as a checkpoint for antithesis, allowing it to
figure that ADP has reached a nominally functional state and can be explored
from that point. The antithesis setup checkpoint is done before datadog.yaml
is sampled.

Notable things:

* first_sample_config runs after setup-checkpoint and before ADP boots,
  is responsible for creating datadog.yaml and other configs in the future
* eventually_adp_alive is a weak check and we may drop it in the future
  as our coverage improves, but it doesn't hurt anything now
* I introduced a harness::rand to encode antithesis-friendly sampling of
  large domains, this will expand over time
* Skill `antithesis-research` has updated its 'scratchbook' but this is a
  mechanical domain for now, will later convert it to a human-hybrid material

Author: blt

Cargo.lock

@@ -11,7 +11,7 @@ workspace = true
 [features]
 default = []
 fips = ["saluki-app/tls-fips", "saluki-components/fips"]
-antithesis = ["dep:antithesis_sdk", "antithesis_sdk/full", "dep:antithesis-instrumentation"]
+antithesis = ["dep:antithesis_sdk", "antithesis_sdk/full", "dep:antithesis-instrumentation", "saluki-components/antithesis"]
 
 [dependencies]
 antithesis-instrumentation = { workspace = true, optional = true }

bin/agent-data-plane/Cargo.toml

@@ -12,8 +12,10 @@ workspace = true
 default = []
 config-test-support = []
 fips = ["saluki-io/fips"]
+antithesis = ["dep:antithesis_sdk", "antithesis_sdk/full"]
 
 [dependencies]
+antithesis_sdk = { workspace = true, optional = true }
 arc-swap = { workspace = true }
 async-trait = { workspace = true }
 axum = { workspace = true }

lib/saluki-components/Cargo.toml

@@ -546,6 +546,19 @@ async fn process_http_response(
     if status.is_success() {
         debug!(endpoint_url, %status, "Request completed.");
 
+        // Reaching a successful intake response means the whole pipeline
+        // ran. This is a useful signal for process health but also
+        // acts as a checkpoint anchor for Antithesis replay: at this point
+        // there is a nominally functional system.
+        //
+        // No-op outside the `antithesis` feature build.
+        #[cfg(feature = "antithesis")]
+        antithesis_sdk::assert_sometimes!(
+            true,
+            "ADP forwarded a payload to the intake",
+            &serde_json::json!({ "domain": domain })
+        );
+
         telemetry.track_successful_transaction(&metadata, domain);
     } else {
         telemetry.track_permanently_failed_transaction(&metadata, Some(status), domain);

lib/saluki-components/src/common/datadog/io.rs

@@ -32,11 +32,14 @@ RUN --mount=type=bind,source=rust-toolchain.toml,target=/tmp/rust-toolchain.toml
 # ---------------------------------------------------------------------------
 # Build the instrumented Agent Data Plane.
 #
-# Coverage instrumentation uses the modern Antithesis Rust flow (post-2026-05-22): the
-# `antithesis-instrumentation` crate (referenced once in main.rs behind the `antithesis` feature)
-# provides the runtime shim, and these RUSTFLAGS enable LLVM sancov coverage. `--build-id` is
-# required for symbolization; the release profile sets `debug = true`, so the binary keeps DWARF
-# for /symbols. LTO is disabled to keep sancov instrumentation predictable.
+# Coverage instrumentation uses the modern Antithesis Rust flow
+# (post-2026-05-22): the `antithesis-instrumentation` crate (referenced once in
+# main.rs behind the `antithesis` feature) provides the runtime shim, and these
+# RUSTFLAGS enable LLVM sancov coverage. `--build-id` is required for
+# symbolization; the release profile sets `debug = true`, so the binary keeps
+# DWARF for /symbols. LTO is disabled to keep sancov instrumentation
+# predictable. `panic = "abort"` (antithesis build only) turns any ADP panic
+# into SIGABRT, caught as a hard crash.
 # ---------------------------------------------------------------------------
 FROM build-base AS adp-builder
 ENV APP_FULL_NAME="Agent Data Plane" \
@@ -55,6 +58,7 @@ RUN --mount=type=cache,target=/adp/target,id=antithesis-adp-target \
     --mount=type=cache,target=/root/.cargo/git,id=cargo-git \
     cargo build --release --package agent-data-plane --features antithesis \
         --target x86_64-unknown-linux-gnu \
+        --config 'profile.release.panic="abort"' \
         --config 'target.x86_64-unknown-linux-gnu.rustflags=["--cfg","tokio_unstable","-Ccodegen-units=1","-Cpasses=sancov-module","-Cllvm-args=-sanitizer-coverage-level=3","-Cllvm-args=-sanitizer-coverage-trace-pc-guard","-Clink-args=-Wl,--build-id"]' && \
     cp /adp/target/x86_64-unknown-linux-gnu/release/agent-data-plane /usr/local/bin/agent-data-plane && \
     echo "Validating Antithesis instrumentation symbols..." && \
@@ -74,11 +78,14 @@ RUN --mount=type=cache,target=/tools/target,id=antithesis-tools-target \
     --mount=type=cache,target=/root/.cargo/git,id=cargo-git \
     cargo build --release \
         --bin datadog-intake --bin millstone \
-        --bin parallel_driver_send_dogstatsd --bin finally_verify_delivery && \
+        --bin parallel_driver_send_dogstatsd --bin finally_verify_delivery --bin eventually_adp_alive \
+        --bin first_sample_config && \
     cp /tools/target/release/datadog-intake /usr/local/bin/datadog-intake && \
     cp /tools/target/release/millstone /usr/local/bin/millstone && \
     cp /tools/target/release/parallel_driver_send_dogstatsd /usr/local/bin/parallel_driver_send_dogstatsd && \
-    cp /tools/target/release/finally_verify_delivery /usr/local/bin/finally_verify_delivery
+    cp /tools/target/release/finally_verify_delivery /usr/local/bin/finally_verify_delivery && \
+    cp /tools/target/release/eventually_adp_alive /usr/local/bin/eventually_adp_alive && \
+    cp /tools/target/release/first_sample_config /usr/local/bin/first_sample_config
 
 # ---------------------------------------------------------------------------
 # Runtime: Agent Data Plane (SUT).
@@ -92,8 +99,12 @@ RUN apt-get update && \
 COPY --from=adp-builder /usr/local/bin/agent-data-plane /usr/local/bin/agent-data-plane
 # Expose DWARF/build-id symbols to Antithesis for symbolization (one-hop symlink to the unstripped binary).
 RUN mkdir -p /symbols && ln -s /usr/local/bin/agent-data-plane /symbols/agent-data-plane
-# main.rs requires the bootstrap config file to exist at the default path; ship a minimal standalone config.
+# main.rs requires the bootstrap config file to exist at the default path; ship a minimal standalone
+# config as a fallback. The boot wrapper overwrites it with the per-replay config written by the
+# `first_sample_config` workload command onto the shared `agent-config` volume.
 COPY test/antithesis/deploy/adp/datadog.yaml /etc/datadog-agent/datadog.yaml
+# Boot wrapper: waits for the drawn config sentinel, copies the config into place, then execs ADP.
+COPY --chmod=755 test/antithesis/deploy/adp/entrypoint.sh /entrypoint.sh
 # ADP's control-plane secure API requires an IPC TLS cert (a single PEM holding both certificate and
 # private key) that the Core Agent normally generates. In standalone mode there is no Core Agent, so
 # generate a self-signed cert+key. An empty auth_token satisfies the IPC auth config at startup.
@@ -103,7 +114,7 @@ RUN openssl req -x509 -newkey rsa:2048 -nodes -days 3650 \
     cat /tmp/ipc_cert.pem /tmp/ipc_key.pem > /etc/datadog-agent/ipc_cert.pem && \
     rm -f /tmp/ipc_cert.pem /tmp/ipc_key.pem && \
     touch /etc/datadog-agent/auth_token
-ENTRYPOINT ["/usr/local/bin/agent-data-plane"]
+ENTRYPOINT ["/entrypoint.sh"]
 CMD ["run"]
 
 # ---------------------------------------------------------------------------
@@ -128,7 +139,9 @@ COPY --from=tools-builder /usr/local/bin/millstone /usr/local/bin/millstone
 COPY --chmod=755 test/antithesis/deploy/workload/setup-complete.sh /opt/antithesis/setup-complete.sh
 COPY test/antithesis/deploy/workload/test/ /opt/antithesis/test/
 # Inject the compiled test-command binaries into the "main" test template.
+COPY --from=tools-builder --chmod=755 /usr/local/bin/first_sample_config /opt/antithesis/test/v1/main/first_sample_config
 COPY --from=tools-builder --chmod=755 /usr/local/bin/parallel_driver_send_dogstatsd /opt/antithesis/test/v1/main/parallel_driver_send_dogstatsd
 COPY --from=tools-builder --chmod=755 /usr/local/bin/finally_verify_delivery /opt/antithesis/test/v1/main/finally_verify_delivery
+COPY --from=tools-builder --chmod=755 /usr/local/bin/eventually_adp_alive /opt/antithesis/test/v1/main/eventually_adp_alive
 COPY --chmod=755 test/antithesis/deploy/workload/entrypoint.sh /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]

test/antithesis/deploy/Dockerfile

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Agent Data Plane boot wrapper.
+#
+# first_sample_config writes this timeline's datadog.yaml + a `ready` sentinel to
+# the shared volume; we block on it, copy the config, then `exec` one stable ADP.
+# We block indefinitely rather than timing out and exiting non-zero, which would
+# be read as an ADP crash. The startup log below makes the wait visible in triage,
+# so a missing release shows as "waiting…" with no boot rather than a silent hang.
+
+CONFIG_DIR="${AGENT_CONFIG_DIR:-/agent-config}"
+
+echo "adp: waiting for ${CONFIG_DIR}/ready (released by first_sample_config)" >&2
+while [ ! -f "${CONFIG_DIR}/ready" ]; do
+  sleep 1
+done
+
+cp "${CONFIG_DIR}/datadog.yaml" /etc/datadog-agent/datadog.yaml
+
+exec /usr/local/bin/agent-data-plane "$@"

test/antithesis/deploy/adp/entrypoint.sh

@@ -40,15 +40,11 @@ services:
       DD_DATA_PLANE_DOGSTATSD_ENABLED: "true"
     volumes:
       - dogstatsd-socket:/var/run/datadog
+      # first_sample_config (workload) writes this timeline's datadog.yaml + ready sentinel here.
+      - agent-config:/agent-config:ro
     depends_on:
       intake:
         condition: service_healthy
-    healthcheck:
-      # ADP's unprivileged API listens on TCP :5100 once the internal supervisor is up.
-      test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/localhost/5100'"]
-      interval: 2s
-      timeout: 2s
-      retries: 60
 
   workload:
     container_name: workload
@@ -62,19 +58,15 @@ services:
     image: workload:latest
     environment:
       NO_COLOR: "1"
-      ADP_HOST: "adp"
-      ADP_API_PORT: "5100"
       DSD_SOCKET: "/var/run/datadog/dsd.socket"
       INTAKE_ADDR: "intake:2049"
-      INTAKE_HOST: "intake"
-      INTAKE_PORT: "2049"
     volumes:
       - dogstatsd-socket:/var/run/datadog
+      - agent-config:/agent-config
     depends_on:
-      adp:
-        condition: service_healthy
       intake:
         condition: service_healthy
 
 volumes:
   dogstatsd-socket:
+  agent-config:

test/antithesis/deploy/docker-compose.yaml

@@ -3,51 +3,9 @@ set -euo pipefail
 
 # Workload client entrypoint.
 #
-# By the time this runs, docker-compose has gated startup on the `adp` and `intake` services being
-# healthy (depends_on: condition: service_healthy). We re-confirm reachability defensively, emit the
-# Antithesis `setup_complete` signal, then idle so Antithesis can run test commands from the test
-# template at /opt/antithesis/test/v1/.
+# Gated on intake-healthy (compose `depends_on`). Emit `setup_complete`, then
+# idle so Antithesis runs the test commands.
 
-ADP_HOST="${ADP_HOST:-adp}"
-ADP_API_PORT="${ADP_API_PORT:-5100}"
-DSD_SOCKET="${DSD_SOCKET:-/var/run/datadog/dsd.socket}"
-INTAKE_HOST="${INTAKE_HOST:-intake}"
-INTAKE_PORT="${INTAKE_PORT:-2049}"
-
-wait_for_tcp() {
-  local host="$1" port="$2" name="$3" tries=60
-  echo "Waiting for ${name} (${host}:${port})..."
-  while (( tries-- > 0 )); do
-    if (exec 3<>"/dev/tcp/${host}/${port}") 2>/dev/null; then
-      echo "${name} is reachable."
-      return 0
-    fi
-    sleep 1
-  done
-  echo "Timed out waiting for ${name} (${host}:${port})." >&2
-  return 1
-}
-
-wait_for_socket() {
-  local path="$1" name="$2" tries=60
-  echo "Waiting for ${name} (${path})..."
-  while (( tries-- > 0 )); do
-    if [[ -S "${path}" ]]; then
-      echo "${name} is reachable."
-      return 0
-    fi
-    sleep 1
-  done
-  echo "Timed out waiting for ${name} (${path})." >&2
-  return 1
-}
-
-wait_for_tcp "${ADP_HOST}" "${ADP_API_PORT}" "agent-data-plane API"
-wait_for_socket "${DSD_SOCKET}" "agent-data-plane DogStatsD socket"
-wait_for_tcp "${INTAKE_HOST}" "${INTAKE_PORT}" "datadog-intake"
-
-echo "System is ready. Emitting setup_complete."
 /opt/antithesis/setup-complete.sh
-
-echo "Workload client idle; awaiting Antithesis test commands."
+echo "setup_complete emitted; workload idle, awaiting Antithesis test commands."
 exec tail -f /dev/null

test/antithesis/deploy/workload/entrypoint.sh

@@ -16,8 +16,12 @@ clap = { workspace = true, features = [
   "std",
   "usage",
 ] }
+num-traits = { workspace = true }
 rand = { workspace = true }
+rand_distr = { workspace = true }
+serde = { workspace = true }
 serde_json = { workspace = true }
+serde_yaml = { workspace = true }
 
 [lints.clippy]
 all = "deny"

test/antithesis/harness/Cargo.toml

@@ -0,0 +1,75 @@
+//! Antithesis `eventually_` liveness check: ADP booted and became reachable
+//! within a bounded window.
+//!
+//! `eventually_` commands run in a fault-quiet period, so a node-fault induced
+//! kill of ADP does not trip this check but a self-inflicted process exit
+//! does. This triggers on ADP's own bugs, rather than antithesis fault
+//! injection.
+//!
+//! We check two signals. First that ADP is reachable on :5100 and second that
+//! it created a `DogStatsD` listener socket.
+
+use std::net::{TcpStream, ToSocketAddrs};
+use std::os::unix::fs::FileTypeExt;
+use std::path::PathBuf;
+use std::thread::sleep;
+use std::time::Duration;
+
+use antithesis_sdk::prelude::*;
+use clap::{builder::NonEmptyStringValueParser, Parser};
+use serde_json::json;
+
+#[derive(Debug, Parser)]
+#[command(name = "eventually_adp_alive")]
+struct Config {
+    #[arg(
+        long = "adp-api-addr",
+        env = "ADP_API_ADDR",
+        default_value = "adp:5100",
+        value_parser = NonEmptyStringValueParser::new()
+    )]
+    adp_api_addr: String,
+    #[arg(
+        long = "dsd-socket",
+        env = "DSD_SOCKET",
+        default_value = "/var/run/datadog/dsd.socket"
+    )]
+    dsd_socket: PathBuf,
+}
+
+fn main() -> anyhow::Result<()> {
+    antithesis_init();
+    let config = Config::try_parse()?;
+
+    let mut api_reachable = false;
+    let mut socket_present = false;
+    // Check that the adp-api is reachable and the DogStatsD socket exists for
+    // about 60 seconds. A 1s connect timeout keeps the poll cadence bounded
+    // even when the API host is unresponsive.
+    for _ in 0..60 {
+        api_reachable = config
+            .adp_api_addr
+            .to_socket_addrs()
+            .ok()
+            .and_then(|mut addrs| addrs.next())
+            .is_some_and(|addr| TcpStream::connect_timeout(&addr, Duration::from_secs(1)).is_ok());
+        socket_present = config.dsd_socket.metadata().is_ok_and(|m| m.file_type().is_socket());
+        if api_reachable && socket_present {
+            break;
+        }
+        sleep(Duration::from_secs(1));
+    }
+
+    assert_always!(
+        api_reachable && socket_present,
+        "ADP booted: API reachable and DogStatsD socket present",
+        &json!({
+            "adp_api_addr": config.adp_api_addr,
+            "dsd_socket": config.dsd_socket.display().to_string(),
+            "api_reachable": api_reachable,
+            "socket_present": socket_present,
+        })
+    );
+
+    Ok(())
+}