chore(antithesis): Antithesis test harness and workload

.dockerignore

@@ -8,6 +8,7 @@ node_modules/
 target/
 **/fuzz/target
 test/
+!test/antithesis/
 .gitignore
 .gitlab-ci.yml
 CONTRIBUTING.md

Cargo.toml

@@ -29,6 +29,7 @@ members = [
   "lib/saluki-metrics",
   "lib/saluki-tls",
   "lib/stringtheory",
+  "test/antithesis/harness",
 ]
 resolver = "2"
 
@@ -66,6 +67,7 @@ async-trait = { version = "0.1", default-features = false }
 atty = { version = "0.2", default-features = false }
 axum = { version = "0.8", default-features = false }
 bytes = { version = "1", default-features = false }
+clap = { version = "4", default-features = false, features = [] }
 protobuf = { version = "3.7", default-features = false, features = [
   "with-bytes",
 ] }
@@ -223,6 +225,8 @@ chumsky = { version = "0.13", default-features = false }
 logos = { version = "0.16", default-features = false }
 lru-slab = { version = "0.1.2", default-features = false }
 hickory-resolver = { version = "0.26", default-features = false }
+antithesis-instrumentation = { version = "0.1" }
+antithesis_sdk = { git = "https://github.com/antithesishq/antithesis-sdk-rust", rev = "6829a946e7e970cc743ffe17c3cee7d2bc25425a", default-features = false } # 0.2.8 is pinned to rand 0.8, rev version allows us to select workspace rand
 
 [patch.crates-io]
 # Forked version of `hyper-http-proxy` that removes an unused dependency on `rustls-native-certs`, which transitively depends

LICENSE-3rdparty.csv

@@ -7,6 +7,8 @@ allocator-api2,https://github.com/zakarumych/allocator-api2,MIT OR Apache-2.0,Za
 android_system_properties,https://github.com/nical/android_system_properties,MIT OR Apache-2.0,Nicolas Silva <nical@fastmail.com>
 anes,https://github.com/zrzka/anes-rs,MIT OR Apache-2.0,Robert Vojta <rvojta@me.com>
 anstyle,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle Authors
+antithesis-instrumentation,https://github.com/antithesishq/antithesis-instrumentation-rust,MIT,The antithesis-instrumentation Authors
+antithesis_sdk,https://github.com/antithesishq/antithesis-sdk-rust,MIT,The antithesis_sdk Authors
 anyhow,https://github.com/dtolnay/anyhow,MIT OR Apache-2.0,David Tolnay <dtolnay@gmail.com>
 arc-swap,https://github.com/vorner/arc-swap,MIT OR Apache-2.0,Michal 'vorner' Vaner <vorner@vorner.cz>
 argh,https://github.com/google/argh,BSD-3-Clause,"Taylor Cramer <cramertj@google.com>, Benjamin Brittain <bwb@google.com>, Erick Tryzelaar <etryzelaar@google.com>"
@@ -58,6 +60,7 @@ ciborium-ll,https://github.com/enarx/ciborium,Apache-2.0,Nathaniel McCallum <npm
 clang-sys,https://github.com/KyleMayes/clang-sys,Apache-2.0,Kyle Mayes <kyle@mayeses.com>
 clap,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap Authors
 clap_builder,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap_builder Authors
+clap_derive,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap_derive Authors
 clap_lex,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap_lex Authors
 colored,https://github.com/mackwic/colored,MPL-2.0,Thomas Wickham <mackwic@gmail.com>
 combine,https://github.com/Marwes/combine,MIT,Markus Westerlind <marwes91@gmail.com>
@@ -196,6 +199,8 @@ leb128fmt,https://github.com/bluk/leb128fmt,MIT OR Apache-2.0,Bryant Luk <code@b
 libc,https://github.com/rust-lang/libc,MIT OR Apache-2.0,The Rust Project Developers
 libloading,https://github.com/nagisa/rust_libloading,ISC,Simonas Kazlauskas <libloading@kazlauskas.me>
 libm,https://github.com/rust-lang/compiler-builtins,MIT,"Alex Crichton <alex@alexcrichton.com>, Amanieu d'Antras <amanieu@gmail.com>, Jorge Aparicio <japaricious@gmail.com>, Trevor Gross <tg@trevorgross.com>"
+linkme,https://github.com/dtolnay/linkme,MIT OR Apache-2.0,David Tolnay <dtolnay@gmail.com>
+linkme-impl,https://github.com/dtolnay/linkme,MIT OR Apache-2.0,David Tolnay <dtolnay@gmail.com>
 linux-raw-sys,https://github.com/sunfishcode/linux-raw-sys,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Dan Gohman <dev@sunfishcode.online>
 litemap,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers
 litrs,https://github.com/LukasKalbertodt/litrs,MIT OR Apache-2.0,Lukas Kalbertodt <lukas.kalbertodt@gmail.com>
@@ -316,6 +321,8 @@ rust_decimal,https://github.com/paupino/rust-decimal,MIT,Paul Mason <paul@form1.
 rustc-demangle,https://github.com/rust-lang/rustc-demangle,MIT OR Apache-2.0,Alex Crichton <alex@alexcrichton.com>
 rustc-hash,https://github.com/rust-lang-nursery/rustc-hash,Apache-2.0 OR MIT,The Rust Project Developers
 rustc-hash,https://github.com/rust-lang/rustc-hash,Apache-2.0 OR MIT,The Rust Project Developers
+rustc_version,https://github.com/djc/rustc-version-rs,MIT OR Apache-2.0,The rustc_version Authors
+rustc_version_runtime,https://github.com/seppo0010/rustc-version-runtime-rs,MIT,Sebastian Waisbrot <seppo0010@gmail.com>
 rusticata-macros,https://github.com/rusticata/rusticata-macros,MIT OR Apache-2.0,Pierre Chifflier <chifflier@wzdftpd.net>
 rustix,https://github.com/bytecodealliance/rustix,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,"Dan Gohman <dev@sunfishcode.online>, Jakub Konka <kubkon@jakubkonka.com>"
 rustls,https://github.com/rustls/rustls,Apache-2.0 OR ISC OR MIT,The rustls Authors

Makefile

@@ -312,7 +312,7 @@ run-adp-standalone: build-adp create-dummy-agent-config create-dummy-ipc-cert
 run-adp-standalone: ## Runs ADP locally in standalone mode (debug)
 	@echo "[*] Running ADP..."
 	@DD_DATA_PLANE_STANDALONE_MODE=true DD_DATA_PLANE_DOGSTATSD_ENABLED=true \
- 	DD_API_KEY=api-key-adp-standalone DD_HOSTNAME=adp-standalone \
+	DD_API_KEY=api-key-adp-standalone DD_HOSTNAME=adp-standalone \
 	DD_DOGSTATSD_PORT=9191 DD_DOGSTATSD_SOCKET=/tmp/adp-dogstatsd-dgram.sock DD_DOGSTATSD_STREAM_SOCKET=/tmp/adp-dogstatsd-stream.sock \
 	DD_IPC_CERT_FILE_PATH=$(ADP_STANDALONE_IPC_CERT_FILE) \
 	target/devel/agent-data-plane --config /tmp/adp-empty-config.yaml run
@@ -675,6 +675,28 @@ endif
 	@echo "[*] Ensuring Miri is setup..."
 	@cargo +nightly-2025-06-16 miri setup
 
+##@ Antithesis
+
+ANTITHESIS_CONFIG_DIR := test/antithesis/deploy
+ANTITHESIS_COMPOSE_FILE := $(ANTITHESIS_CONFIG_DIR)/docker-compose.yaml
+
+.PHONY: check-antithesis-tools
+check-antithesis-tools:
+ifeq ($(shell command -v snouty >/dev/null || echo not-found), not-found)
+	$(error "snouty must be present to validate the Antithesis harness, see https://github.com/antithesishq/snouty")
+endif
+
+.PHONY: antithesis-build
+antithesis-build: ## Builds the Antithesis harness container images
+	@echo "[*] Building Antithesis harness images..."
+	@docker compose -f $(ANTITHESIS_COMPOSE_FILE) build
+
+.PHONY: antithesis-validate
+antithesis-validate: check-antithesis-tools antithesis-build
+antithesis-validate: ## Validates the Antithesis harness: builds images, runs 'snouty validate'
+	@echo "[*] Validating Antithesis harness with snouty..."
+	@snouty validate $(ANTITHESIS_CONFIG_DIR)
+
 ##@ Profiling
 
 .PHONY: profile-run-blackhole

bin/agent-data-plane/Cargo.toml

@@ -11,8 +11,11 @@ workspace = true
 [features]
 default = []
 fips = ["saluki-app/tls-fips", "saluki-components/fips"]
+antithesis = ["dep:antithesis_sdk", "antithesis_sdk/full", "dep:antithesis-instrumentation"]
 
 [dependencies]
+antithesis-instrumentation = { workspace = true, optional = true }
+antithesis_sdk = { workspace = true, optional = true }
 argh = { workspace = true, features = ["help"] }
 async-trait = { workspace = true }
 bytesize = { workspace = true }

bin/agent-data-plane/src/main.rs

@@ -7,6 +7,11 @@
 #![deny(missing_docs)]
 use std::time::Instant;
 
+// Pull in the Antithesis coverage-instrumentation runtime shim only when
+// building for antithesis. Load-baring: equired to avoid the shim being dropped
+// as unused.
+#[cfg(feature = "antithesis")]
+use antithesis_instrumentation as _;
 use datadog_agent_commons::platform::PlatformSettings;
 use metrics::Level;
 use saluki_app::bootstrap::{AppBootstrapper, Bootstrap, BootstrapGuard};
@@ -39,6 +44,12 @@ static ALLOC: resource_accounting::TrackingAllocator<std::alloc::System> =
 #[tokio::main]
 async fn main() -> Result<(), GenericError> {
     let started = Instant::now();
+
+    // Initialize the Antithesis SDK as early as possible so assertions and lifecycle hooks register
+    // their catalog before any are evaluated. No-op outside Antithesis and absent in production builds.
+    #[cfg(feature = "antithesis")]
+    antithesis_sdk::antithesis_init();
+
     let cli: Cli = argh::from_env();
 
     // Print version and exit early without requiring config.
@@ -83,6 +94,11 @@ async fn main() -> Result<(), GenericError> {
         .await
         .error_context("Failed to complete bootstrap phase.")?;
 
+    // Bootstrap-integration probe: proves the Antithesis SDK is linked, cataloging works, and the
+    // instrumentation path is wired.
+    #[cfg(feature = "antithesis")]
+    antithesis_sdk::assert_reachable!("agent-data-plane completed bootstrap", &serde_json::json!({}));
+
     // Run the given subcommand. The bootstrap supervisor is forwarded by value; only the long-lived `run`
     // subcommand actually drives it (it is added as a child of the internal supervisor inside
     // `handle_run_command`). All other subcommands drop it on entry.

test/antithesis/AGENTS.md

@@ -0,0 +1,83 @@
+This directory contains files relevant to running tests in Antithesis.
+
+# Skills
+
+Use the `antithesis-setup` skill to scaffold and manage this directory. Use the
+`antithesis-research` skill to analyze the system and build a property
+catalog. Use the `antithesis-workload` skill to implement assertions and test
+commands. Use the `antithesis-launch` skill to build, validate, and submit
+Antithesis runs — do not run `snouty launch` directly.
+
+**snouty launch**
+
+Use `snouty launch --json --webhook basic_test --config test/antithesis/deploy`
+to start an Antithesis run. Always run `compose build` first to ensure images
+are up to date.
+
+**snouty validate**
+
+Use this command to quickly validate changes to the Antithesis scaffolding. See
+`snouty validate --help` for details.
+
+**setup-complete.sh**
+
+Inject this script into a Dockerfile to notify Antithesis that setup is
+complete. This script should only run once the system under test is ready for
+testing. Antithesis will not run any test commands until it receives this event.
+
+**Directory layout**
+
+- `harness/` — the harness Rust crate (`harness`), a member of the
+  repository-root workspace. `src/lib.rs` holds shared helpers; each
+  `src/bin/*.rs` is an Antithesis test command named after its file. Run cargo
+  from this directory; it is built, fmt'd, Clippy'd, and tested from the repo
+  root via the usual `make check-all` / `make test`.
+- `deploy/` — all Antithesis/Docker infrastructure: the `Dockerfile`,
+  `docker-compose.yaml`, and per-container build inputs grouped by service
+  (`deploy/adp/`, `deploy/workload/`). This is the directory snouty consumes as
+  `--config`; it contains `docker-compose.yaml` at its top. Snouty will push
+  tagged images, consume this directory, and launch the run.
+
+**scratchbook**
+
+This directory is the Antithesis scratchbook for the codebase. It contains
+documents such as system analysis, property catalogs, topology plans,
+per-property evidence files (in `scratchbook/properties/`), property
+relationship maps, and other persistent integration notes. Keep it up to date as
+Antithesis-related decisions change.
+
+**test templates** (`deploy/workload/test/`)
+
+This directory contains test templates. A test template is a directory
+containing test command executable files. Each test command must have a valid
+prefix: `parallel_driver_, singleton_driver_, serial_driver_, first_,
+eventually_, finally_, anytime_`. Prefixes constrain when and how commands are
+composed in a single timeline. Files or subdirectories prefixed with `helper_`
+are ignored by Antithesis and can be used for helper scripts kept alongside the
+commands.
+
+# Agent Behavior
+
+Agent behavior will be governed by the following dictums:
+
+- **The human is primary.** If you run into any confusion, pause and ask for
+  clarification.
+- When you are faced with a choice between doing the right, time-consuming thing
+  or the wrong, fast thing do the right thing.
+- Code is liability. The status quo is not worth preserving if it does not have
+  utility. Be unsentimental and delete what is not needed.
+- **Truth over comfort.** Say what is true regardless of the presumed comfort of
+  the receiver. Do not soften findings, hedge claims or omit bad news. To do so
+  is _not kindness_. It is, rather, an insidious form of lie. Note that this
+  dictum should be understood less in terms of Kim Scott's "Radical Candor" -- a
+  gift from the elite to the undeserving common -- but more in Walter
+  Brueggemann's "Prophetic Imagination" where truth erodes a "royal
+  consciousness" that ablates one's ability to do new and interesting things
+  _and_ shouts a path toward those new and interesting things, against the
+  status quo. Consider in this same vein Tony Hoare's "The Emperor's Old
+  Clothes".
+- **Honor the spirit of a request, not just its letter.** A "random string
+  pool" requires actual variation. Returning `["foo", "bar"]` is technically
+  a pool but a semantic mismatch. When the literal reading is unusually
+  narrow or cheap, reach for the generous reading. Hostile compliance is
+  worse than asking.

test/antithesis/README.md

@@ -0,0 +1,19 @@
+# Antithesis Tests
+
+This directory contains a sub-project to run Antithesis tests for the saluki
+project. Primary focus is on establishing that ADP and DogStatsD behave
+'equivalently', which is to say if ADP is dropped in for Datadog Agent's
+DogStatsD users will not notice shifts in their telemetry. Better operational
+behavior is acceptable deviation.
+
+## Prerequisites
+
+* snouty -- https://github.com/antithesishq/snouty
+* antithesis-skills + claude -- https://github.com/antithesishq/antithesis-skills
+
+## Running Scenarios
+
+This effort is extremely early. Today we assume claude drives scenarios runs,
+command it to do so with `/antithesis-launch`. In order for this to work you
+must already have credentials available. Eventually we will have CI rigged up to
+do nightly shots.