Add CVE-Bench benchmark adapter

README.md

@@ -52,6 +52,24 @@ local` only inside a disposable outer environment such as Daytona, an ephemeral
 CI worker, or a throwaway VM; local mode runs `pi` directly and does not provide
 PITHOS-managed process isolation.
 
+## CVE-Bench
+
+PITHOS can run the official CVE-Bench exploit-style benchmark by delegating
+target setup and scoring to CVE-Bench while swapping in a Pi-backed Inspect
+solver:
+
+```bash
+pithos cvebench doctor --provider openai --model gpt-5.5
+pithos cvebench smoke --provider openai --model gpt-5.5
+pithos cvebench eval --provider openai --model gpt-5.5 --pull
+```
+
+Use `pithos cvebench smoke` first. The full `eval` command runs every default
+CVE-Bench challenge and variant, pulls/builds Docker images, and can be long and
+API-expensive. Results are written under `results/cve-bench/<timestamp>-eval/`
+with `benchmark-summary.json`, `BENCHMARK.md`, raw CVE-Bench stdout/stderr logs,
+and Inspect logs.
+
 ## Live Verification
 
 Live verification is opt-in. PITHOS first needs a runtime profile that describes

pithos/cli.py

@@ -14,6 +14,16 @@
 from pathlib import Path
 from urllib.parse import urlparse
 
+from .cvebench_runner import (
+    DEFAULT_CVEBENCH_CACHE,
+    DEFAULT_CVEBENCH_RESULTS,
+    DEFAULT_CVEBENCH_VERSION,
+    DEFAULT_INSPECT_MODEL,
+    DEFAULT_KALI_SIZE,
+    DEFAULT_SMOKE_CHALLENGE,
+    CveBenchRunner,
+    CveBenchRunnerError,
+)
 from .events import event_mode_from_env, make_event_sink
 from .exec_backend import SANDBOX_MODES
 from .env_discovery import env_hint_report_from_agent_audit
@@ -239,6 +249,65 @@ def main(argv: list[str] | None = None) -> int:
         help="Print the generated profile instead of writing it",
     )
 
+    p_cvebench = sub.add_parser("cvebench", help="Run PITHOS against CVE-Bench")
+    cvebench_sub = p_cvebench.add_subparsers(dest="cvebench_command")
+    p_cvebench_doctor = cvebench_sub.add_parser(
+        "doctor",
+        help="Check CVE-Bench/Pi benchmark prerequisites",
+    )
+    _add_cvebench_common_args(p_cvebench_doctor, require_results=False)
+
+    p_cvebench_pull = cvebench_sub.add_parser(
+        "pull",
+        help="Clone/update CVE-Bench and pull benchmark images",
+    )
+    _add_cvebench_common_args(p_cvebench_pull)
+
+    p_cvebench_smoke = cvebench_sub.add_parser(
+        "smoke",
+        help="Run a one-challenge CVE-Bench smoke evaluation",
+    )
+    _add_cvebench_common_args(p_cvebench_smoke)
+    p_cvebench_smoke.add_argument("--challenge", default=DEFAULT_SMOKE_CHALLENGE)
+    p_cvebench_smoke.add_argument("--pull", action="store_true", help="Pull images first")
+    p_cvebench_smoke.add_argument(
+        "--no-build-agent",
+        action="store_true",
+        help="Do not build the Pi-enabled CVE-Bench agent image",
+    )
+
+    p_cvebench_eval = cvebench_sub.add_parser(
+        "eval",
+        help="Run the full or filtered CVE-Bench evaluation",
+    )
+    _add_cvebench_common_args(p_cvebench_eval)
+    p_cvebench_eval.add_argument(
+        "--challenge",
+        dest="challenges",
+        action="append",
+        help="CVE challenge to include; repeat or omit for all",
+    )
+    p_cvebench_eval.add_argument(
+        "--variant",
+        dest="variants",
+        action="append",
+        choices=("zero_day", "one_day"),
+        help="Variant to include; repeat or omit for all",
+    )
+    p_cvebench_eval.add_argument("--pull", action="store_true", help="Pull images first")
+    p_cvebench_eval.add_argument(
+        "--no-build-agent",
+        action="store_true",
+        help="Do not build the Pi-enabled CVE-Bench agent image",
+    )
+    p_cvebench_eval.add_argument(
+        "--inspect-arg",
+        dest="inspect_args",
+        action="append",
+        default=[],
+        help="Additional raw argument to pass through to `./run eval`; repeat as needed",
+    )
+
     args = parser.parse_args(argv)
     if args.version:
         from importlib.metadata import PackageNotFoundError, version
@@ -257,10 +326,127 @@ def main(argv: list[str] | None = None) -> int:
             return _cmd_runtime_init(args)
         p_runtime.print_help()
         return 0
+    if args.command == "cvebench":
+        return _cmd_cvebench(args, p_cvebench)
     parser.print_help()
     return 0
 
 
+def _add_cvebench_common_args(
+    parser: argparse.ArgumentParser, *, require_results: bool = True
+) -> None:
+    parser.add_argument("--provider", default=os.environ.get("PITHOS_PROVIDER", DEFAULT_PROVIDER))
+    parser.add_argument("--model", default=os.environ.get("PITHOS_MODEL"))
+    parser.add_argument(
+        "--pi-config-dir",
+        type=Path,
+        default=sandbox.resolve_pi_config_dir(),
+        help="Directory containing Pi auth.json/models.json to mount read-only",
+    )
+    parser.add_argument("--checkout", type=Path, default=None, help="Existing CVE-Bench checkout")
+    parser.add_argument("--cache-dir", type=Path, default=DEFAULT_CVEBENCH_CACHE)
+    parser.add_argument("--ref", default=None, help="CVE-Bench branch, tag, or commit")
+    parser.add_argument("--results-dir", type=Path, default=DEFAULT_CVEBENCH_RESULTS)
+    parser.add_argument("--kali-size", choices=("core", "large"), default=DEFAULT_KALI_SIZE)
+    parser.add_argument("--cvebench-version", default=DEFAULT_CVEBENCH_VERSION)
+    parser.add_argument("--cvebench-tag", default=os.environ.get("CVEBENCH_TAG"))
+    parser.add_argument("--inspect-model", default=DEFAULT_INSPECT_MODEL)
+    parser.add_argument("--max-messages", type=int, default=30)
+    parser.add_argument("--max-resume-attempts", type=int, default=3)
+    if not require_results:
+        parser.set_defaults(results_dir=DEFAULT_CVEBENCH_RESULTS)
+
+
+def _cmd_cvebench(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int:
+    if not args.cvebench_command:
+        parser.print_help()
+        return 0
+
+    needs_agent = args.cvebench_command in {"smoke", "eval"}
+    pi_config_dir = sandbox.resolve_pi_config_dir(args.pi_config_dir)
+    agent_env, auth_error = _resolve_agent_env(args.provider, pi_config_dir)
+    provider_auth = _provider_auth_status(args.provider, pi_config_dir)
+    if needs_agent:
+        if not args.model:
+            print("error: --model required (or set PITHOS_MODEL)", file=sys.stderr)
+            return 1
+        if auth_error:
+            print(auth_error, file=sys.stderr)
+            return 1
+        if not provider_auth:
+            print(
+                f"error: no Pi auth found for provider {args.provider!r}. "
+                "Set the provider API key env var or pass --pi-config-dir with matching auth.json/models.json.",
+                file=sys.stderr,
+            )
+            return 1
+
+    runner = CveBenchRunner(
+        provider=args.provider,
+        model=args.model or "",
+        agent_env=agent_env,
+        pi_config_dir=pi_config_dir,
+        checkout=args.checkout,
+        cache_dir=args.cache_dir,
+        ref=args.ref,
+        results_dir=args.results_dir,
+        kali_size=args.kali_size,
+        cvebench_version=args.cvebench_version,
+        cvebench_tag=args.cvebench_tag,
+        inspect_model=args.inspect_model,
+        max_messages=args.max_messages,
+        max_resume_attempts=args.max_resume_attempts,
+    )
+
+    try:
+        if args.cvebench_command == "doctor":
+            return _cmd_cvebench_doctor(runner, provider_auth=provider_auth)
+        if args.cvebench_command == "pull":
+            result = runner.pull()
+        elif args.cvebench_command == "smoke":
+            result = runner.eval(
+                challenges=[args.challenge],
+                variants=["one_day"],
+                label="smoke",
+                pull=args.pull,
+                build_agent=not args.no_build_agent,
+            )
+        elif args.cvebench_command == "eval":
+            result = runner.eval(
+                challenges=args.challenges,
+                variants=args.variants,
+                label="eval",
+                pull=args.pull,
+                build_agent=not args.no_build_agent,
+                extra_args=args.inspect_args,
+            )
+        else:
+            parser.print_help()
+            return 0
+    except (CveBenchRunnerError, subprocess.CalledProcessError, OSError) as e:
+        print(f"error: CVE-Bench run failed: {redact_secrets(str(e))}", file=sys.stderr)
+        return 1
+
+    print("PITHOS CVE-Bench summary")
+    print(f"  status:     {result.status}")
+    print(f"  artifacts:  {result.out_dir}/")
+    print(f"  command rc: {result.command_result.returncode}")
+    print(f"  elapsed:    {result.command_result.elapsed_s:.1f}s")
+    return 0 if result.status == "completed" else 2
+
+
+def _cmd_cvebench_doctor(runner: CveBenchRunner, *, provider_auth: bool) -> int:
+    result = runner.doctor(provider_auth=provider_auth)
+    print("PITHOS CVE-Bench doctor")
+    for name, ok in result.checks.items():
+        print(f"  {name:16s} {'ok' if ok else 'missing'}")
+    if result.warnings:
+        print("  warnings:")
+        for warning in result.warnings:
+            print(f"    - {warning}")
+    return 0 if result.ok else 1
+
+
 def _cmd_run(args: argparse.Namespace) -> int:
     try:
         event_sink = make_event_sink(args.events)