diff --git a/.buildkite/pipelines/evals/llm_evals.yml b/.buildkite/pipelines/evals/llm_evals.yml index 21064b990e637..bc6d4b49be418 100644 --- a/.buildkite/pipelines/evals/llm_evals.yml +++ b/.buildkite/pipelines/evals/llm_evals.yml @@ -478,6 +478,29 @@ steps: - exit_status: '-1' limit: 3 + - label: 'Evals: Security Automatic Migrations' + key: kbn-evals-weekly-security-automatic-migrations + command: bash .buildkite/scripts/steps/evals/run_suite.sh + env: + KBN_EVALS: '1' + FTR_EIS_CCM: '1' + EVAL_SUITE_ID: 'security-automatic-migrations' + EVAL_FANOUT: '1' + EVAL_INCLUDE_EIS_MODELS: '1' + EVAL_MODEL_GROUPS: *weekly_eis_core_models + timeout_in_minutes: 60 + agents: + image: family/kibana-ubuntu-2404 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-8 + diskSizeGb: 130 + preemptible: true + retry: + automatic: + - exit_status: '-1' + limit: 3 + - label: 'Evals: Workflows Authoring' key: kbn-evals-weekly-workflows command: bash .buildkite/scripts/steps/evals/run_suite.sh @@ -617,6 +640,55 @@ steps: automatic: - exit_status: '-1' limit: 3 + + - label: "Evals: Security Persona Matrix" + key: kbn-evals-weekly-security-persona-matrix + command: bash .buildkite/scripts/steps/evals/run_suite.sh + env: + KBN_EVALS: '1' + FTR_EIS_CCM: '1' + EVAL_SUITE_ID: 'security-persona-matrix' + EVAL_FANOUT: '1' + EVAL_INCLUDE_EIS_MODELS: '1' + EVAL_MODEL_GROUPS: *weekly_eis_core_models + EVAL_SERVER_CONFIG_SET: 'evals_security_persona_matrix' + timeout_in_minutes: 90 + agents: + image: family/kibana-ubuntu-2404 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-8 + diskSizeGb: 130 + preemptible: true + retry: + automatic: + - exit_status: '-1' + limit: 3 + + - label: "Evals: Security Persona Matrix — Attack Discovery" + key: kbn-evals-weekly-security-persona-matrix-attack-discovery + command: bash .buildkite/scripts/steps/evals/run_suite.sh + env: + KBN_EVALS: '1' + FTR_EIS_CCM: '1' + EVAL_SUITE_ID: 'security-persona-matrix-attack-discovery' + EVAL_FANOUT: '1' + EVAL_INCLUDE_EIS_MODELS: '1' + EVAL_MODEL_GROUPS: *weekly_eis_core_models + EVAL_SERVER_CONFIG_SET: 'evals_tracing' + timeout_in_minutes: 90 + agents: + image: family/kibana-ubuntu-2404 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-8 + diskSizeGb: 130 + preemptible: true + retry: + automatic: + - exit_status: '-1' + limit: 3 + - wait: ~ continue_on_failure: true diff --git a/.gitignore b/.gitignore index 945984fca1d3d..b42616a12d5fb 100644 --- a/.gitignore +++ b/.gitignore @@ -216,3 +216,6 @@ src/platform/plugins/shared/workflows_execution_engine/docs/parallel_step_p1_sco src/platform/plugins/shared/workflows_execution_engine/docs/parallel_step_nested_flow_control_tradeoff.md .ralph/ + +# Sweep host credentials — secret-bearing, never commit +scripts/orca_vm/*.env diff --git a/scripts/orca_vm/audit_sweep.py b/scripts/orca_vm/audit_sweep.py new file mode 100644 index 0000000000000..0114004f2d9f3 --- /dev/null +++ b/scripts/orca_vm/audit_sweep.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Post-sweep audit: prove a sweep's scores are publishable. + +`EVAL_EXIT=0` and `EXPORT_EXIT=0` are NOT evidence. On 2026-08-29 both were +zero while 546 docs went to a nonexistent host, and another 294 landed +correctly but were unusable because the model had graded itself. + +This asks the golden cluster three questions per model: + 1. did docs actually land? (catches wrong-URL false success) + 2. who graded them? (catches self-judging) + 3. how many examples completed? (catches silent partials) + + source /tmp/golden-cluster-env.sh + python3 audit_sweep.py --models eis-anthropic-claude-4-6-sonnet --hours 6 + +Exit 0 only if every model has landed, independently-judged docs. +""" +from __future__ import annotations + +import argparse +import json +import os +import ssl +import sys +import urllib.request + +from model_ids import resolve_model_id + +GOLDEN_INDEX = ".evaluation-scores" + + +def list_stored_model_ids(url: str, key: str, hours: int) -> list[str]: + """Every task.model.id present in the window — the ground truth id space.""" + body = { + "size": 0, + "query": {"bool": {"filter": [{"range": {"@timestamp": {"gte": f"now-{hours}h"}}}]}}, + "aggs": {"m": {"terms": {"field": "task.model.id", "size": 200}}}, + } + res = es_search(url, key, body) + return [b["key"] for b in res["aggregations"]["m"]["buckets"]] + + +def es_search(url: str, key: str, body: dict) -> dict: + req = urllib.request.Request( + f"{url.rstrip('/')}/{GOLDEN_INDEX}/_search", + data=json.dumps(body).encode(), + headers={"Authorization": f"ApiKey {key}", "Content-Type": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=60, context=ssl.create_default_context()) as resp: + return json.loads(resp.read().decode()) + + +def audit_model(url: str, key: str, model: str, hours: int, stored_ids: list[str]) -> bool: + """Return True only if this model produced publishable scores.""" + resolved = resolve_model_id(model, stored_ids) + if resolved is None: + print( + f"FAIL [{model}] no docs under any matching task.model.id in the last {hours}h. " + f"Export reported success but nothing landed — check GOLDEN_ES_URL points at " + f"the real cluster.", + file=sys.stderr, + ) + return False + model_c = resolved + # NOTE: the timestamp field is `@timestamp`, and the model under test is + # `task.model.id` — NOT `evaluator.model.id`, which is the judge. Querying + # the judge field silently returns another model's docs. + body = { + "size": 0, + "query": { + "bool": { + "filter": [ + {"term": {"task.model.id": model_c}}, + {"range": {"@timestamp": {"gte": f"now-{hours}h"}}}, + ] + } + }, + "aggs": { + "judges": {"terms": {"field": "evaluator.model.id", "size": 10}}, + "examples": {"cardinality": {"field": "example.id"}}, + }, + } + try: + res = es_search(url, key, body) + except Exception as exc: # noqa: BLE001 + print(f"FAIL [{model_c}] golden query failed: {exc}", file=sys.stderr) + return False + + total = res["hits"]["total"]["value"] + if total == 0: + print( + f"FAIL [{model_c}] 0 docs in the last {hours}h. Export reported success but " + f"nothing landed — check GOLDEN_ES_URL points at the real cluster.", + file=sys.stderr, + ) + return False + + judges = {b["key"]: b["doc_count"] for b in res["aggregations"]["judges"]["buckets"]} + examples = res["aggregations"]["examples"]["value"] + self_judged = judges.get(model_c, 0) + + if self_judged: + pct = 100.0 * self_judged / total + print( + f"FAIL [{model_c}] {self_judged}/{total} docs ({pct:.0f}%) are SELF-JUDGED. " + f"These are dropped by `excludeSelfJudged` and will fill ZERO cells. " + f"Re-running will not help — assign an independent judge.", + file=sys.stderr, + ) + return False + + judge_list = ", ".join(f"{k}({v})" for k, v in judges.items()) or "none" + print(f" ok [{model_c}] {total} docs, {examples} examples, judged by: {judge_list}") + return True + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--models", nargs="+", required=True) + ap.add_argument("--hours", type=int, default=6) + args = ap.parse_args() + + url = os.environ.get("GOLDEN_ES_URL", "") + key = os.environ.get("GOLDEN_ES_API_KEY", "") + if not url or not key: + print("GOLDEN_ES_URL / GOLDEN_ES_API_KEY not set", file=sys.stderr) + return 1 + + print(f"Auditing {len(args.models)} model(s) over the last {args.hours}h") + stored_ids = list_stored_model_ids(url, key, args.hours) + results = [audit_model(url, key, m, args.hours, stored_ids) for m in args.models] + + if not all(results): + print("\nAUDIT FAILED — these scores are not publishable.", file=sys.stderr) + return 1 + print("\nAUDIT PASSED — scores landed and are independently judged.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/orca_vm/blinding_ab_test.py b/scripts/orca_vm/blinding_ab_test.py new file mode 100644 index 0000000000000..735a1f69cca6d --- /dev/null +++ b/scripts/orca_vm/blinding_ab_test.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Blinding A/B: does a visible model hint change a judge's score? + +Criterion 4. Pulls real transcripts from golden ES, judges each one TWICE +through the omniroute combo `omni-opus-5`: + + variant A (leaked) -- transcript with an `orca-sweep-` host string + variant B (blinded) -- identical transcript, host replaced with `orca-sweep-m` + +Everything else is byte-identical, so any score delta is attributable to the +hint alone. Writes scores AND the combo's reported token/cost usage. + +The earlier static analysis showed the real golden leak lives only in +`metadata.hostname`, which never reaches the judge. This measures the +COUNTERFACTUAL: if such a string DID reach the judge, would it move scores? +That is the question that decides whether blinding matters going forward. +""" +import argparse +import hashlib +import json +import os +import re +import sys +import time +import urllib.request + +OMNIROUTE_URL = os.environ.get("OMNIROUTE_URL", "http://localhost:8080/v1/chat/completions") +COMBO = "omni-opus-5" + +JUDGE_PROMPT = """You are grading a security analyst agent's answer. + +Score 0-10 on: correctness, use of evidence, and whether the answer actually +completes the requested task. Reply with STRICT JSON only: +{"score": <0-10>, "reason": ""} + +--- TASK --- +%s + +--- AGENT TRANSCRIPT --- +%s +""" + + +def load_env(path): + env = {} + if os.path.exists(path): + for line in open(path): + m = re.match(r"""\s*export\s+(\w+)=['"]?([^'"\n]+)""", line) + if m: + env[m.group(1)] = m.group(2) + return env + + +def es_search(url, key, body): + req = urllib.request.Request( + f"{url}/.evaluation-scores*/_search", + data=json.dumps(body).encode(), + headers={"Authorization": f"ApiKey {key}", "Content-Type": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=120) as r: + return json.load(r) + + +def dig(src, path): + cur = src + for part in path.split("."): + if isinstance(cur, dict): + cur = cur.get(part) + else: + return None + return cur + + +def fetch_cells(url, key, model, limit): + """Pull distinct transcripts for one model.""" + res = es_search(url, key, { + "size": 400, + "query": {"bool": {"filter": [ + {"term": {"metadata.suite_id": "security-persona-matrix"}}, + {"term": {"task.model.id": model}}, + ]}}, + "_source": ["task.output.messages", "example.input.question", "task.trace_id", + "example.id", "evaluator.name", "evaluator.score"], + }) + seen, cells = set(), [] + for h in res["hits"]["hits"]: + s = h["_source"] + tid = dig(s, "task.trace_id") + if not tid or tid in seen: + continue + msgs = dig(s, "task.output.messages") or [] + text = "\n".join(m.get("message", "") for m in msgs if isinstance(m, dict)) + if len(text) < 200: + continue + seen.add(tid) + cells.append({ + "traceId": tid, + "exampleId": dig(s, "example.id"), + "question": dig(s, "example.input.question") or "", + "transcript": text, + }) + if len(cells) >= limit: + break + return cells + + +def judge(text, question, model_name, timeout=180): + """One judge call through the omni-opus-5 combo.""" + payload = { + "model": COMBO, + "messages": [{"role": "user", "content": JUDGE_PROMPT % (question[:1500], text[:12000])}], + "temperature": 0, + "max_tokens": 300, + } + req = urllib.request.Request( + OMNIROUTE_URL, + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json", + "Authorization": f"Bearer {os.environ.get('OMNIROUTE_API_KEY', 'sk-local')}"}, + ) + with urllib.request.urlopen(req, timeout=timeout) as r: + body = json.load(r) + content = body["choices"][0]["message"]["content"] + usage = body.get("usage", {}) + m = re.search(r'\{.*?\}', content, re.S) + score = None + if m: + try: + score = json.loads(m.group(0)).get("score") + except Exception: + pass + return score, usage, body.get("model", COMBO) + + +def blind(text, model): + """Replace the leaked host token with the anonymised form.""" + leaked = f"orca-sweep-{model.replace('.', '-').replace('/', '-')}" + h = hashlib.sha256(model.encode()).hexdigest()[:8] + return text.replace(leaked, f"orca-sweep-m{h}"), leaked + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--model", default="anthropic-claude-4.8-opus") + ap.add_argument("--cells", type=int, default=14) + ap.add_argument("--env-file", default="/tmp/golden-cluster-env.sh") + ap.add_argument("--out", required=True) + args = ap.parse_args() + + env = load_env(args.env_file) + url, key = env.get("GOLDEN_ES_URL"), env.get("GOLDEN_ES_API_KEY") + if not url or not key: + sys.exit("golden ES credentials missing") + + cells = fetch_cells(url, key, args.model, args.cells) + if not cells: + sys.exit(f"no transcripts for {args.model} -- refusing to write an empty A/B") + print(f"fetched {len(cells)} transcripts for {args.model}", flush=True) + + results, totals = [], {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0, "calls": 0} + for i, c in enumerate(cells, 1): + blinded, leaked_token = blind(c["transcript"], args.model) + # Variant A must actually contain a hint, or the A/B measures nothing. + leaked_text = c["transcript"] + if leaked_token not in leaked_text: + leaked_text = f"[host: {leaked_token}]\n{leaked_text}" + row = {"traceId": c["traceId"], "exampleId": c["exampleId"], + "hintInjected": leaked_token not in c["transcript"]} + for variant, text in (("leaked", leaked_text), ("blinded", blinded)): + try: + score, usage, served = judge(text, c["question"], args.model) + row[variant] = score + row[f"{variant}Model"] = served + for k in ("prompt_tokens", "completion_tokens", "total_tokens"): + totals[k] += usage.get(k, 0) or 0 + totals["calls"] += 1 + except Exception as e: + row[variant] = None + row[f"{variant}Error"] = str(e)[:200] + time.sleep(0.5) + results.append(row) + print(f" [{i}/{len(cells)}] leaked={row.get('leaked')} blinded={row.get('blinded')}", flush=True) + + paired = [(r["leaked"], r["blinded"]) for r in results + if isinstance(r.get("leaked"), (int, float)) and isinstance(r.get("blinded"), (int, float))] + deltas = [a - b for a, b in paired] + summary = { + "model": args.model, + "combo": COMBO, + "cellsRequested": args.cells, + "cellsJudged": len(paired), + "meanLeaked": round(sum(a for a, _ in paired) / len(paired), 3) if paired else None, + "meanBlinded": round(sum(b for _, b in paired) / len(paired), 3) if paired else None, + "meanDelta": round(sum(deltas) / len(deltas), 3) if deltas else None, + "maxAbsDelta": max((abs(d) for d in deltas), default=None), + "nonZeroDeltas": sum(1 for d in deltas if d != 0), + "usage": totals, + } + with open(args.out, "w") as fh: + json.dump({"summary": summary, "results": results}, fh, indent=2) + print(json.dumps(summary, indent=2)) + print(f"wrote {args.out}") + + +if __name__ == "__main__": + main() diff --git a/scripts/orca_vm/build_trace_cache.py b/scripts/orca_vm/build_trace_cache.py new file mode 100644 index 0000000000000..65148b26431e2 --- /dev/null +++ b/scripts/orca_vm/build_trace_cache.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""Build a matrix trace cache directly from golden Elasticsearch. + +Why this exists +--------------- +The matrix generator normally fetches per-example score documents through the +evals plugin's example-scores route. On the golden cluster that route runs an +older plugin build that returns nothing for these queries, so every trace cell +renders hollow (no question, no steps, no tool trail) while the aggregated +scores still look perfect. `node scripts/evals ext matrix --trace-cache ` +takes a pre-pulled cache and skips the route entirely. + +The cache is keyed exactly as query_matrix_traces.ts keys it: + + `${metadata.execution_id}::${example.id}` -> [score documents] + +Usage +----- + source /tmp/golden-cluster-env.sh + python3 build_trace_cache.py --out /tmp/trace_cache.json + node scripts/evals ext matrix --config ... --trace-cache /tmp/trace_cache.json +""" +from __future__ import annotations + +import argparse +import json +import os +import ssl +import sys +import urllib.request + +DEFAULT_INDEX = ".evaluation-scores*" +# Only these carry the payload the trace renderer needs; pulling every evaluator +# for every example multiplies the cache size with no added trace detail. +PAGE_SIZE = 1000 + +# Mirror query_matrix_traces.ts: it stores JSON.stringify(args).slice(0, 300) +# and slices reasoning at 500, so anything longer is cached and then discarded. +MAX_ARGS_CHARS = 300 +MAX_REASONING_CHARS = 500 + +# Node aborts readFileSync above its max string length (0x1fffffe8, ~536 MB). +# A cache past this is unusable by the matrix CLI, so refuse to write one. +MAX_CACHE_BYTES = 500_000_000 + + +def _client(url: str, api_key: str): + # Elastic Cloud serves a properly-issued certificate; default verification + # works and must stay on. + ctx = ssl.create_default_context() + + def request(path: str, body: dict) -> dict: + req = urllib.request.Request( + url.rstrip("/") + path, + data=json.dumps(body).encode(), + headers={ + "Authorization": "ApiKey " + api_key, + "Content-Type": "application/json", + }, + ) + with urllib.request.urlopen(req, context=ctx, timeout=180) as resp: + return json.load(resp) + + return request + + +def fetch_all(request, index: str, experiment: str, hours: int) -> list[dict]: + """Page through every score doc for the experiment using search_after.""" + docs: list[dict] = [] + search_after = None + while True: + body = { + "size": PAGE_SIZE, + # These documents are large (full step lists plus correctness and + # groundedness analyses). Without source filtering a full pull moves + # hundreds of MB and stalls; the renderer only reads these fields. + # `example.metadata` and the analyses are excluded deliberately: a + # 720h cache that includes them exceeds Node's max string length + # (0x1fffffe8) and the CLI cannot readFileSync it at all. + "_source": [ + "@timestamp", + "example.id", + "example.input.question", + "task.output.steps", + "task.output.messages", + "task.model.id", + "task.repetition_index", + "evaluator.name", + "evaluator.score", + "evaluator.model.id", + "metadata.execution_id", + ], + "query": { + "bool": { + "filter": [ + {"term": {"experiment_name": experiment}}, + {"range": {"@timestamp": {"gte": f"now-{hours}h"}}}, + ] + } + }, + # Tiebreak on _shard_doc-free sort: @timestamp alone is not unique. + "sort": [{"@timestamp": "asc"}, {"_doc": "asc"}], + } + if search_after: + body["search_after"] = search_after + res = request(f"/{index}/_search", body) + hits = res["hits"]["hits"] + if not hits: + break + docs.extend(h["_source"] for h in hits) + search_after = hits[-1]["sort"] + print(f" fetched {len(docs)}", flush=True) + if len(hits) < PAGE_SIZE: + break + return docs + + +def trim_step(step: dict) -> dict: + """Keep only what the trace renderer reads from a step. + + A full 720h cache of raw steps is ~1.7 GB, which Node cannot readFileSync + (max string 0x1fffffe8). The renderer truncates tool args to 300 chars and + reasoning to 500, so caching the untruncated originals buys nothing. + """ + kind = step.get("type") + if kind == "tool_call": + args = step.get("args") + # query_matrix_traces re-runs JSON.stringify(args).slice(0, 300) on + # whatever this holds, so keep a real object (a pre-stringified value + # would double-encode). Oversized args are replaced by a marker rather + # than dropped, so the tool card still shows the call was made. + encoded = json.dumps(args) if args is not None else None + if encoded is not None and len(encoded) > MAX_ARGS_CHARS: + args = {"_truncated": len(encoded)} + return {"type": "tool_call", "tool_id": step.get("tool_id"), "args": args} + if kind == "reasoning": + text = step.get("reasoning") + return { + "type": "reasoning", + "reasoning": text[:MAX_REASONING_CHARS] if isinstance(text, str) else text, + } + if kind == "relevant_skills": + return {"type": "relevant_skills", "skills": step.get("skills")} + return {"type": kind} + + +def build_cache(docs: list[dict]) -> dict[str, list[dict]]: + cache: dict[str, list[dict]] = {} + for doc in docs: + execution_id = (doc.get("metadata") or {}).get("execution_id") + example_id = (doc.get("example") or {}).get("id") + if not execution_id or not example_id: + continue + output = (doc.get("task") or {}).get("output") or {} + steps = output.get("steps") + if isinstance(steps, list): + output["steps"] = [trim_step(s) for s in steps if isinstance(s, dict)] + messages = output.get("messages") + if isinstance(messages, list): + # Only the last message over 50 chars becomes the answer. + output["messages"] = [ + {"message": m.get("message")} for m in messages if isinstance(m, dict) + ] + cache.setdefault(f"{execution_id}::{example_id}", []).append(doc) + return cache + + +def self_test() -> int: + """Exercise the trim/keying contract without touching Elasticsearch. + + This script is a committed tool with no jest coverage (it is Python in a + TypeScript repo), so the checks that guard its contract live here and run + via `--self-test` in the verify manifest. + """ + failures: list[str] = [] + + def check(name: str, got: object, want: object) -> None: + if got != want: + failures.append(f"{name}: got {got!r}, want {want!r}") + + # Small tool args stay a real object: query_matrix_traces re-runs + # JSON.stringify on this value, so a pre-stringified one would double-encode. + small = trim_step({"type": "tool_call", "tool_id": "load_skill", "args": {"id": "x"}, "n": 1}) + check("small args", small, {"type": "tool_call", "tool_id": "load_skill", "args": {"id": "x"}}) + + # Oversized args collapse to a marker rather than vanishing, so the tool + # card still shows the call happened. + big = trim_step({"type": "tool_call", "tool_id": "t", "args": {"q": "z" * 400}}) + check("oversized args marked", sorted(big["args"]), ["_truncated"]) + check("marker records real size", big["args"]["_truncated"] > MAX_ARGS_CHARS, True) + + # Reasoning is capped at what the renderer shows. + long_reasoning = trim_step({"type": "reasoning", "reasoning": "y" * 900}) + check("reasoning cap", len(long_reasoning["reasoning"]), 500) + check("short reasoning", trim_step({"type": "reasoning", "reasoning": "ab"})["reasoning"], "ab") + + check( + "relevant_skills", + trim_step({"type": "relevant_skills", "skills": [{"id": "a"}]}), + {"type": "relevant_skills", "skills": [{"id": "a"}]}, + ) + # An unrecognised future step type must survive rather than crash. + check("unknown kind", trim_step({"type": "future"}), {"type": "future"}) + check("args absent", trim_step({"type": "tool_call", "tool_id": "t"})["args"], None) + + # Cache keying: execution_id::example.id, and documents missing either are + # dropped instead of colliding under a partial key. + docs = [ + {"metadata": {"execution_id": "e1"}, "example": {"id": "a"}, "task": {"output": {}}}, + {"metadata": {"execution_id": "e1"}, "example": {"id": "a"}, "task": {"output": {}}}, + {"metadata": {"execution_id": "e1"}, "task": {"output": {}}}, + {"example": {"id": "a"}, "task": {"output": {}}}, + ] + cache = build_cache(docs) + check("cache keys", sorted(cache), ["e1::a"]) + check("entries merged under one key", len(cache["e1::a"]), 2) + + # Steps are trimmed in place through build_cache, not only via trim_step. + keyed = build_cache( + [ + { + "metadata": {"execution_id": "e2"}, + "example": {"id": "b"}, + "task": { + "output": { + "steps": [{"type": "tool_call", "tool_id": "t", "args": {"k": "v"}, "x": 1}], + "messages": [{"message": "hi", "extra": "dropped"}], + } + }, + } + ] + ) + step = keyed["e2::b"][0]["task"]["output"]["steps"][0] + check("build_cache trims steps", step, {"type": "tool_call", "tool_id": "t", "args": {"k": "v"}}) + check( + "build_cache trims messages", + keyed["e2::b"][0]["task"]["output"]["messages"], + [{"message": "hi"}], + ) + + for failure in failures: + print(f"FAIL {failure}", file=sys.stderr) + print(f"self-test: {len(failures)} failure(s)") + return 1 if failures else 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out") + parser.add_argument("--experiment", default="security: security-persona-matrix") + parser.add_argument("--index", default=DEFAULT_INDEX) + parser.add_argument("--hours", type=int, default=720) + parser.add_argument( + "--self-test", + action="store_true", + help="Verify the trim/keying contract offline and exit.", + ) + args = parser.parse_args() + + if args.self_test: + return self_test() + + if not args.out: + print("--out is required", file=sys.stderr) + return 2 + + url = os.environ.get("GOLDEN_ES_URL") + api_key = os.environ.get("GOLDEN_ES_API_KEY") + if not url or not api_key: + print("GOLDEN_ES_URL / GOLDEN_ES_API_KEY must be set", file=sys.stderr) + return 2 + + request = _client(url, api_key) + docs = fetch_all(request, args.index, args.experiment, args.hours) + cache = build_cache(docs) + + # A cache whose entries carry no task.output is worse than no cache: it + # silently satisfies the fetch and still renders hollow traces. Report the + # ratio so the caller can tell a real cache from an empty one. + with_steps = sum( + 1 + for entries in cache.values() + for d in entries + if ((d.get("task") or {}).get("output") or {}).get("steps") + ) + models = { + ((d.get("task") or {}).get("model") or {}).get("id") + for entries in cache.values() + for d in entries + } + + with open(args.out, "w") as handle: + json.dump(cache, handle) + + size = os.path.getsize(args.out) + print(f"docs : {len(docs)}") + print(f"cache keys : {len(cache)}") + print(f"docs w/steps: {with_steps}") + print(f"models : {len(models - {None})}") + print(f"written : {args.out} ({size / 1e6:.0f} MB)") + if with_steps == 0: + print("ERROR: no document carries task.output.steps", file=sys.stderr) + return 1 + # Node reads this file with readFileSync, which cannot produce a string + # longer than 0x1fffffe8 (~536 MB). A larger cache aborts the matrix CLI + # outright, so refuse it here where the cause is obvious. + if size > MAX_CACHE_BYTES: + print( + f"ERROR: cache is {size / 1e6:.0f} MB; Node cannot readFileSync " + f"more than ~536 MB. Narrow --hours or the _source list.", + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/orca_vm/build_traces_json.py b/scripts/orca_vm/build_traces_json.py new file mode 100644 index 0000000000000..3704c3bfefc6d --- /dev/null +++ b/scripts/orca_vm/build_traces_json.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +"""Build TRACES_JSON for render_from_golden.ts from golden ES. + +Combines two golden sources into MatrixTraceData (keyed "modelId:column"): + 1. .ds-.evaluation-scores* task.output (steps, messages) per execution + 2. traces-agent_builder.otel-default spans: + - gen_ai.input.messages -> reasoning/final answer + - gen_ai.tool.call.arguments on execute_tool spans -> toolParams + +The historical cache has steps[].args = null for every tool_call (383,098 +steps, 100% null, pre-includeToolDetails). New runs carry real args ONLY on +the spans, so toolParams is sourced from spans and joined by (trace_id, +tool span order). Absent args render as an explicit "(args not captured)" +marker — never invented. +""" +import argparse +import collections +import json +import os +import re +import sys +import urllib.request + +def load_env(path): + env = {} + with open(path) as fh: + for line in fh: + m = re.match(r"""\s*export\s+(\w+)=['"]?([^'"\n]+)""", line) + if m: + env[m.group(1)] = m.group(2).strip() + return env + +class Es: + def __init__(self, url, key): + self.url = url.rstrip("/") + self.key = key + + def post(self, path, body=None): + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(self.url + path, data=data, method="POST") + req.add_header("Authorization", f"ApiKey {self.key}") + req.add_header("Content-Type", "application/json") + with urllib.request.urlopen(req, timeout=300) as resp: + return json.load(resp) + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--env-file", default=os.path.expanduser("~/.elastic/golden-cluster-env.sh")) + ap.add_argument("--suite", default="security-persona-matrix", + help="suite_id filter for score docs (agent-builder feeds agent_eval_full boards)") + ap.add_argument("--since", required=True, help="ISO lower bound for trace docs") + ap.add_argument("--until", default=None, help="ISO upper bound (exclusive era separation for trial-replica builds)") + ap.add_argument("--out", required=True) + args = ap.parse_args() + + env = load_env(args.env_file) + url, key = env.get("GOLDEN_ES_URL"), env.get("GOLDEN_ES_API_KEY") + if not url or not key: + sys.exit("GOLDEN_ES_URL / GOLDEN_ES_API_KEY missing") + + es = Es(url, key) + + # ---- 1. score docs: execution -> steps/messages -------------------- + score_body = { + "size": 1000, + "query": {"bool": {"filter": [ + {"term": {"metadata.suite_id": args.suite}}, + {"range": {"@timestamp": {"gte": args.since, **({"lt": args.until} if args.until else {})}}}, + ]}}, + "_source": [ + "metadata.execution_id", "task.model.id", "example.id", + "task.output.steps", "task.output.messages", "task.output.traceId", + "evaluator.name", "evaluator.score", "task.repetition_index", + "@timestamp", + ], + "sort": [{"@timestamp": "asc"}], + } + cells = collections.defaultdict(dict) # exec_id -> {prompt: docs} + model_of = {} + n_docs = 0 + search_after = None + while True: + body = dict(score_body) + if search_after: + body["search_after"] = search_after + res = es.post("/.ds-.evaluation-scores*/_search", body) + hits = res["hits"]["hits"] + if not hits: + break + for h in hits: + src = h["_source"] + exec_id = (src.get("metadata") or {}).get("execution_id", "") + model = ((src.get("task") or {}).get("model") or {}).get("id", "") + prompt = ((src.get("example") or {}).get("id") or "").lower() + if exec_id: + model_of[exec_id] = model + if not (exec_id and model and prompt): + continue + cells[exec_id].setdefault(prompt, []).append(src) + n_docs += 1 + search_after = hits[-1]["sort"] + if len(hits) < 1000: + break + + # ---- 2. spans: tool args per trace -------------------------------- + span_body = { + "size": 1000, + "query": {"bool": {"filter": [ + {"range": {"@timestamp": {"gte": args.since, **({"lt": args.until} if args.until else {})}}}, + {"exists": {"field": "attributes.gen_ai.tool.call.arguments"}}, + ]}}, + "_source": [ + "trace_id", "span_id", "parent_span_id", "name", "@timestamp", + "attributes.gen_ai.tool.call.arguments", "attributes.gen_ai.tool.name", + "attributes.gen_ai.tool.call.id", + ], + "sort": [{"@timestamp": "asc"}], + } + span_args = collections.defaultdict(list) # trace_id -> [(ts, name, args)] + n_spans = 0 + search_after = None + while True: + body = dict(span_body) + if search_after: + body["search_after"] = search_after + res = es.post("/.ds-traces-agent_builder.otel-default-*/_search", body) + hits = res["hits"]["hits"] + if not hits: + break + for h in hits: + src = h["_source"] + tid = src.get("trace_id") + attrs = src.get("attributes") or {} + args_v = attrs.get("gen_ai.tool.call.arguments") + tool_nm = attrs.get("gen_ai.tool.name") + if tid and args_v is not None: + span_args[tid].append( + {"ts": src.get("@timestamp"), + "name": src.get("name") or tool_nm, + "toolId": attrs.get("gen_ai.tool.call.id"), + "args": args_v} + ) + n_spans += 1 + search_after = hits[-1]["sort"] + if len(hits) < 1000: + break + + # ---- 2b. spans: LLM usage (tokens + duration) per trace ------------ + # Reference board cells show "Xs · Y/Z tok" — per-cell latency and token + # totals. Score docs carry neither; they live on LLM spans keyed by + # trace_id. Persona-matrix suite spans land in traces-generic.otel-default + # (dotted attr keys); agent-builder suite spans in traces-agent_builder + # .otel-default (nested attrs). Query both, read both shapes. + usage_query = {"bool": {"filter": [ + {"range": {"@timestamp": {"gte": args.since, **({"lt": args.until} if args.until else {})}}}, + {"bool": {"should": [ + {"exists": {"field": "attributes.gen_ai.usage.input_tokens"}}, + {"exists": {"field": "gen_ai.usage.input_tokens"}}, + ]}}, + ]}} + span_usage = {} # trace_id -> [dur_ns, in_tok, out_tok] (summed per trace) + n_usage = 0 + after = None + while True: + # composite agg: server-side sum per trace_id — no deep paging, + # no _source transfer. (The pre-agg search_after scan moved 359k + # docs serially at 1k/page; this moves only per-trace sums.) + agg_body = { + "size": 0, + "query": usage_query, + "aggs": { + "by_trace": { + "composite": { + "size": 1000, + "sources": [{"tid": {"terms": {"field": "trace_id"}}}], + **({"after": after} if after else {}), + }, + "aggs": { + "dur": {"sum": {"field": "duration"}}, + "in_tok": {"sum": {"field": "attributes.gen_ai.usage.input_tokens"}}, + "in_tok_flat": {"sum": {"field": "gen_ai.usage.input_tokens"}}, + "out_tok": {"sum": {"field": "attributes.gen_ai.usage.output_tokens"}}, + "out_tok_flat": {"sum": {"field": "gen_ai.usage.output_tokens"}}, + }, + }, + }, + } + res = es.post("/.ds-traces-generic.otel-default*,.ds-traces-agent_builder.otel-default*/_search", agg_body) + buckets = res["aggregations"]["by_trace"]["buckets"] + if not buckets: + break + for b in buckets: + tid = b["key"]["tid"] + cur = span_usage.get(tid) or [0.0, 0, 0] + cur[0] += float(b["dur"]["value"] or 0) + cur[1] += int(b["in_tok"]["value"] or b["in_tok_flat"]["value"] or 0) + cur[2] += int(b["out_tok"]["value"] or b["out_tok_flat"]["value"] or 0) + span_usage[tid] = cur + n_usage += b["doc_count"] + after = res["aggregations"]["by_trace"].get("after_key") + if not after or len(buckets) < 1000: + break + + # ---- 3. join ------------------------------------------------------ + # Score-doc steps[].args is None even on includeToolDetails runs (the + # evals writer does not serialise args into task.output). Real args live + # only on execute_tool spans, keyed by trace_id. Join: within a trace, + # match each tool_call step to the span with the same tool name in + # timestamp order. Unmatched -> explicit None (renderer marks it). + # + # A model may have MULTIPLE executions in the window (retries). The + # reference board renders one run per model; per (model, prompt) pick + # the NEWEST execution (latest @timestamp across its docs) — "latest + # numbers" semantics. Doc-count was the old rule; it preferred bulky + # early executions over newer usage-bearing runs (observed 2026-09-13: + # 882-doc Sep-1 execs beating 98-doc Sep-7 eis runs). Deterministic, + # never a blend. + best = {} # (model, prompt) -> (last_ts, exec_id, docs) + for exec_id, prompts in cells.items(): + model = model_of.get(exec_id, "") + short = model.removeprefix("eis-") + for prompt, docs in prompts.items(): + k = (short, prompt) + last_ts = max((d.get("@timestamp") or "") for d in docs) + if k not in best or last_ts > best[k][0]: + best[k] = (last_ts, exec_id, docs) + + by_exec_prompt = collections.defaultdict(dict) # exec_id -> prompt -> docs + for (short, prompt), (n, exec_id, docs) in best.items(): + by_exec_prompt[exec_id][prompt] = docs + + out = {} + for exec_id, prompts in by_exec_prompt.items(): + model = model_of.get(exec_id, "") + short = model.removeprefix("eis-") + for prompt, docs in prompts.items(): + trace_ids = set() + for d in docs: + tid = ((d.get("task") or {}).get("output") or {}).get("traceId") + if tid: + trace_ids.add(tid) + # name-indexed span args for this cell's traces + name_queue = collections.defaultdict(collections.deque) + for tid in trace_ids: + for sp in sorted(span_args.get(tid, []), key=lambda s: s["ts"] or ""): + nm = sp["name"].replace("execute_tool ", "") if sp["name"] else sp["name"] + name_queue[nm].append(sp) + # LLM usage summed across the cell's traces ("Xs · Y/Z tok") + dur_ns = 0.0 + in_tok = 0 + out_tok = 0 + for tid in trace_ids: + u = span_usage.get(tid) + if u: + dur_ns += u[0] + in_tok += u[1] + out_tok += u[2] + steps_out = [] + answer = None + question = None + seen_reps = set() + for d in docs: + rep = d.get("task", {}).get("repetition_index") + if rep in seen_reps: + continue # evaluator docs duplicate the same output; take one + seen_reps.add(rep) + outp = ((d.get("task") or {}).get("output") or {}) + if question is None: + q = (outp.get("messages") or [{}])[0] + if isinstance(q, dict) and q.get("role") == "user": + question = q.get("content") + if answer is None: + msgs = outp.get("messages") or [] + for m in reversed(msgs): + if isinstance(m, dict) and m.get("message"): + answer = m["message"] + break + if question is None and not steps_out: + pass + for s in outp.get("steps") or []: + t = s.get("type") + if t == "reasoning": + steps_out.append({"type": "reasoning", "text": s.get("reasoning")}) + elif t == "tool_call": + nm = s.get("tool_id") + sp = name_queue[nm].popleft() if name_queue.get(nm) else None + steps_out.append({ + "type": "tool", + "toolId": nm, + "toolParams": sp["args"] if sp else None, + }) + elif t == "relevant_skills": + steps_out.append({ + "type": "skill", + "skills": s.get("skills"), + }) + scores = collections.defaultdict(list) + for d in docs: + ev = ((d.get("evaluator") or {})) + if ev.get("name") and isinstance(ev.get("score"), (int, float)): + scores[ev["name"]].append(float(ev["score"])) + out[f"{short}:{prompt}"] = { + "question": question, + "answer": answer, + "steps": steps_out, + "stepCount": len(steps_out), + "scores": {k: round(sum(v)/len(v), 4) for k, v in scores.items()}, + "repetitions": len(docs), + "usage": {"durNs": round(dur_ns), "inTok": in_tok, "outTok": out_tok}, + } + + # ---- guard: silent-zero join detection ---------------------------- + # A join that fetches spans but matches zero cells looks green while the + # board renders without the data (observed 2026-09-12: wrong index -> + # 764 spans fetched, 0 cells joined). Fail loudly instead. + n_cells_usage = sum(1 for c in out.values() + if (c.get("usage") or {}).get("inTok") + or (c.get("usage") or {}).get("durNs")) + warnings = [] + if n_usage > 0 and n_cells_usage == 0: + warnings.append( + f"usageSpans={n_usage} fetched but 0/{len(out)} cells joined — " + "trace-id linkage broken (wrong index or era without LLM spans)") + if n_spans > 0 and not any((s.get("toolParams") for c in out.values() + for s in (c.get("steps") or []))): + warnings.append( + f"argSpans={n_spans} fetched but 0 tool steps carry params — " + "name-join broken (tool name mismatch)") + for w in warnings: + print(f"WARNING: {w}", file=sys.stderr) + + payload = { + "cells": out, + "meta": { + "since": args.since, + "until": args.until, + "scoreDocs": n_docs, + "argSpans": n_spans, + "usageSpans": n_usage, + "cells": len(out), + "cellsUsage": n_cells_usage, + "note": "toolParams joined from gen_ai.tool.call.arguments spans", + }, + } + with open(args.out, "w") as fh: + json.dump(payload, fh) + print(f"score docs: {n_docs} | arg spans: {n_spans} | " + f"cells: {len(out)} (usage: {n_cells_usage}) -> {args.out}") + +if __name__ == "__main__": + main() diff --git a/scripts/orca_vm/create_openrouter_endpoint.py b/scripts/orca_vm/create_openrouter_endpoint.py new file mode 100644 index 0000000000000..45c6d1d44ed78 --- /dev/null +++ b/scripts/orca_vm/create_openrouter_endpoint.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Create/verify the ES inference endpoint for an OpenRouter-backed model. + +Pointed at the on-VM proxy (http://127.0.0.1:8088), NOT openrouter.ai directly: +ES's OpenAiUnifiedStreamingProcessor cannot parse OpenRouter's raw SSE +("reasoning": null in finish chunks, native_finish_reason, reasoning_tokens) — +the proxy normalizes the stream first. Skipping the proxy worked nowhere. + +Trap notes (all verified in the field): + - urllib does NOT parse URL-embedded credentials: Authorization must be a + real header (Basic elastic:changeme) or the call 401s silently. + - task_type and api_key are REQUIRED in service_settings for the PUT. + ES strips api_key from GET responses, so "copying the GET shape" breaks. + - The endpoint id MUST equal the connector's config.inferenceId, else the + eval 404s with "No connector or inference endpoint found". + +Usage: create_openrouter_endpoint.py [proxy_port] +""" + +import base64 +import json +import sys +import time +import urllib.error +import urllib.request + +ES_URL = "http://localhost:9220" +AUTH = "Basic " + base64.b64encode(b"elastic:changeme").decode() + +READY_TIMEOUT_S = 600 # ES boot after clean data dir takes minutes +POLL_S = 5 + + +def es_req(method: str, path: str, body: "bytes | None" = None, timeout: int = 30): + req = urllib.request.Request( + ES_URL + path, data=body, method=method, + headers={"Authorization": AUTH, "Content-Type": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.status, resp.read().decode("utf-8", "replace") + + +def wait_for_es() -> bool: + """Poll until ES accepts connections. A clean-data boot is not instant.""" + deadline = time.time() + READY_TIMEOUT_S + while time.time() < deadline: + try: + es_req("GET", "/", timeout=10) + return True + except Exception: + time.sleep(POLL_S) + return False + + +def endpoint_points_at_proxy(endpoint_id: str, model_id: str, port: int) -> bool: + """True when the endpoint already exists with the RIGHT shape. + + Reuse is only safe when url/model both match: a previous run's endpoint + pointing straight at openrouter.ai silently bypasses the proxy and dies + with "agent returned an empty response" mid-sweep. + """ + try: + _, body = es_req("GET", f"/_inference/chat_completion/{endpoint_id}") + # GET wraps the result: {"endpoints": [{...}]} — read through the + # wrapper or service_settings is always {} and the watcher loop + # delete/recreates the endpoint every cycle (converse 404 races). + doc = json.loads(body) + if isinstance(doc, dict) and "endpoints" in doc: + eps = doc["endpoints"] or [] + if not eps: + return False + cfg = eps[0].get("service_settings", {}) + else: + cfg = doc.get("service_settings", {}) + return ( + cfg.get("url") == f"http://127.0.0.1:{port}" + and cfg.get("model_id") == model_id + ) + except urllib.error.HTTPError as e: + if e.code == 404: + return False + raise + + +def main() -> int: + if len(sys.argv) < 4: + print(__doc__) + return 2 + endpoint_id, model_id, api_key = sys.argv[1], sys.argv[2], sys.argv[3] + port = int(sys.argv[4]) if len(sys.argv) > 4 else 8088 + + if not wait_for_es(): + print("FATAL: ES never became reachable", flush=True) + return 1 + print("ES reachable", flush=True) + + if endpoint_points_at_proxy(endpoint_id, model_id, port): + print(f"endpoint {endpoint_id} already points at proxy; reusing", flush=True) + return 0 + + # A wrong-shaped endpoint must go: PUT on an existing id errors instead of + # overwriting (ES inference endpoints are immutable without force delete). + try: + es_req("DELETE", f"/_inference/chat_completion/{endpoint_id}?force=true") + print(f"deleted stale endpoint {endpoint_id}", flush=True) + except urllib.error.HTTPError: + pass # 404 — nothing to delete + + payload = json.dumps({ + "service": "openai", + "service_settings": { + "model_id": model_id, + "url": f"http://127.0.0.1:{port}", + "api_key": api_key, + # task_type goes ONLY in the URL path, never service_settings — + # "Configuration contains settings [{task_type=chat_completion}] + # unknown to the [openai] service" (400) if included. + "rate_limit": {"requests_per_minute": 500}, + }, + }).encode() + + for attempt in range(1, 4): + try: + status, _ = es_req("PUT", f"/_inference/chat_completion/{endpoint_id}", payload) + print(f"created endpoint {endpoint_id} -> proxy:{port} (HTTP {status})", flush=True) + return 0 + except urllib.error.HTTPError as e: + print(f"PUT attempt {attempt}/3 failed: {e.code} {e.read().decode()[:200]}", flush=True) + time.sleep(5 * attempt) + print("FATAL: could not create endpoint after 3 attempts", flush=True) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/orca_vm/diff_attack_discovery_vs_reference.py b/scripts/orca_vm/diff_attack_discovery_vs_reference.py new file mode 100644 index 0000000000000..6c44c24f5f2e3 --- /dev/null +++ b/scripts/orca_vm/diff_attack_discovery_vs_reference.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Per-field diff between the reference artifact and our golden board. + +Criterion 7: any claim of "1:1" must be backed by a field-level diff, not by +visual similarity. This script produces that evidence -- and, run against the +current data, is precisely what refutes the 1:1 claim. + +It NEVER copies a reference value into our data. Reference numbers are read +only to be compared and reported as differing. +""" +import argparse +import html +import json +import re +import sys + + +def parse_reference(path): + """Extract per-model rows from the reference HTML scoreboard.""" + text = open(path, encoding="utf-8", errors="replace").read() + rows = {} + for raw in re.findall(r"]*>(.*?)", text, re.S): + cells = [ + re.sub(r"<[^>]+>", "", c).strip() + for c in re.findall(r"]*>(.*?)", raw, re.S) + ] + if len(cells) < 6: + continue + label = html.unescape(cells[0]) + parts = [p.strip() for p in label.split("\n") if p.strip()] + model_id = parts[-1] if len(parts) > 1 else parts[0] + rows[model_id] = { + "status": cells[1], + "discoveries": cells[2], + "alertsInContext": cells[3], + "latency": cells[4], + "totalRisk": cells[5], + } + return rows + + +def ours_field(model): + """Normalise our aggregate row onto the reference's field names.""" + disc = model["discoveryCount"]["mean"] + alerts = model["alertsContextCount"]["mean"] + rate = model["status"]["completedRate"] + return { + "status": None if rate is None else ("succeeded" if rate >= 0.99 else f"partial ({rate:.0%})"), + "discoveries": None if disc is None else round(disc, 2), + "alertsInContext": None if alerts is None else round(alerts, 2), + "latency": None if model["latencySeconds"] is None else f'{model["latencySeconds"]}s', + "totalRisk": model["totalRisk"], + } + + +FIELDS = ["status", "discoveries", "alertsInContext", "latency", "totalRisk"] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--reference", required=True) + ap.add_argument("--aggregate", required=True) + ap.add_argument("--out", required=True) + args = ap.parse_args() + + ref = parse_reference(args.reference) + agg = json.load(open(args.aggregate)) + ours = {m["modelId"]: m for m in agg["models"]} + + if not ref: + sys.exit("parsed zero rows from the reference -- refusing to write a vacuous diff") + + overlap = sorted(set(ref) & set(ours)) + ref_only = sorted(set(ref) - set(ours)) + ours_only = sorted(set(ours) - set(ref)) + + per_model = {} + identical_fields = differing_fields = absent_fields = 0 + for mid in overlap: + r, o = ref[mid], ours_field(ours[mid]) + fields = {} + for f in FIELDS: + rv, ov = r[f], o[f] + if ov is None: + verdict = "absent-in-ours" + absent_fields += 1 + elif str(rv) == str(ov): + verdict = "identical" + identical_fields += 1 + else: + verdict = "differs" + differing_fields += 1 + fields[f] = {"reference": rv, "ours": ov, "verdict": verdict} + per_model[mid] = fields + + total = identical_fields + differing_fields + absent_fields + payload = { + "referenceModels": len(ref), + "ourModels": len(ours), + "overlap": len(overlap), + "referenceOnly": ref_only, + "oursOnly": ours_only, + "fieldTotals": { + "identical": identical_fields, + "differs": differing_fields, + "absentInOurs": absent_fields, + "comparable": total, + }, + "isOneToOne": differing_fields == 0 and absent_fields == 0 and not ref_only, + "perModel": per_model, + } + with open(args.out, "w") as fh: + json.dump(payload, fh, indent=2) + + pct = 100 * identical_fields / total if total else 0 + print(f"reference models : {len(ref)}") + print(f"our models : {len(ours)}") + print(f"overlap : {len(overlap)}") + print(f"reference-only : {len(ref_only)}") + print(f"fields identical : {identical_fields}/{total} ({pct:.1f}%)") + print(f"fields differing : {differing_fields}") + print(f"fields absent : {absent_fields}") + print(f"1:1 reproduction : {payload['isOneToOne']}") + print(f"wrote {args.out}") + + +if __name__ == "__main__": + main() diff --git a/scripts/orca_vm/export_scores.py b/scripts/orca_vm/export_scores.py new file mode 100644 index 0000000000000..423bd93320400 --- /dev/null +++ b/scripts/orca_vm/export_scores.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Export persona-matrix scores from LOCAL ES on the VM to golden. + +Reads local ES (port 9220, elastic:changeme) for the persona-matrix dataset, +then bulk-indexes to golden with create semantics (idempotent retries). +""" +import json +import os +import sys +import urllib.request + +GOLDEN_URL = os.environ.get("GOLDEN_ES_URL", "").rstrip("/") +GOLDEN_KEY = os.environ.get("GOLDEN_ES_API_KEY", "") +LOCAL_ES = "http://localhost:9220" +LOCAL_AUTH = "Basic ZWxhc3RpYzpjaGFuZ2VtZQ==" # elastic:changeme + +# Suite to export, injected by the sweeper (EVAL_SUITE). A single hardcoded +# dataset UUID only ever worked for persona-matrix, which has one dataset; +# attack-discovery spans 9 datasets and automatic-migrations several more, so +# filter on the suite instead. Observed 2026-09-02: a clean AD canary run +# passed 9/9 and exported NOTHING, because the persona-matrix dataset id +# matched no local doc and the exporter treated that as "nothing to do". +SUITE_ID = os.environ.get("EVAL_SUITE", "security-persona-matrix") +INDEX = ".evaluation-scores" +# Bulk in chunks so one bad batch cannot discard the whole export. +BULK_BATCH_SIZE = 500 + + +def es_local(path, body=None): + req = urllib.request.Request( + f"{LOCAL_ES}{path}", + data=json.dumps(body).encode() if body else None, + headers={"Content-Type": "application/json", "Authorization": LOCAL_AUTH}, + method="POST" if body else "GET", + ) + with urllib.request.urlopen(req, timeout=30) as r: + return json.loads(r.read()) + + +def canonicalise(src: dict, model_id: str) -> dict: + """Rewrite the upstream model name to the sweep's model key. + + A self-hosted cell is reached through the OpenAI-compatible proxy, so the + suite records whatever name the UPSTREAM reported (`qwen3.8-27b`, + `qwen/qwen3.8-27b`) -- never the sweep key `selfhost-qwen38`. Both the + golden completeness gate and the resume probe look the run up by the sweep + key, so unrewritten docs are invisible to them: run 12 wrote 266 real score + docs and still reported total failure. + + execution_id embeds the same upstream name (`::::qwen3.8-27b`) + and MUST be rewritten too. Normalising only task.model.id leaves the resume + probe querying an execution_id that does not exist, so every retry re-ran + the full dataset (run 14: 5 attempts, "golden confirms 0 scored examples" + while the export itself reported 14 docs written). + """ + meta = src.get("metadata") or {} + exec_id = meta.get("execution_id") + if isinstance(exec_id, str) and "::" in exec_id: + run, _, rest = exec_id.partition("::") + suite = rest.rpartition("::")[0] or SUITE_ID + meta = {**meta, "execution_id": f"{run}::{suite}::{model_id}"} + src = {**src, "metadata": meta} + task = src.get("task") or {} + model = task.get("model") or {} + if model: + src = {**src, "task": {**task, "model": {**model, "id": model_id}}} + return src + + +def export_model(model_id: str): + # Every sweep VM starts from a clean ES data directory and runs exactly one + # model. Export the complete suite instead of guessing the + # stored model ID from the connector ID: connector IDs use hyphens + # (google-gemini-3-1-pro), while score docs use dots + # (google-gemini-3.1-pro). The dataset filter is the stable identity. + body = { + "size": 10000, + "query": {"term": {"metadata.suite_id": SUITE_ID}}, + } + resp = es_local(f"/{INDEX}/_search", body) + hits = resp.get("hits", {}).get("hits", []) + if not hits: + sys.stderr.write(f"no local {SUITE_ID} docs for {model_id}\n") + return 0, 0 + + stored_models = { + h.get("_source", {}).get("task", {}).get("model", {}).get("id") for h in hits + } + stored_models.discard(None) + if len(stored_models) != 1: + raise RuntimeError( + f"expected one model on clean VM, found {sorted(stored_models)}" + ) + + # Preserve IDs and use create so retries are idempotent. Version conflicts + # mean the document already landed and are not export failures. + # + # Batched deliberately. A single 10k-doc bulk is one all-or-nothing shot: + # a mid-flight timeout loses the whole sweep's scores even though most + # documents were fine. And the old code returned 0 whenever ANY document + # failed -- reporting total failure while 293 of 294 docs sat safely on + # golden. Partial data beats no data; report the honest count. + exported = 0 + failed = 0 + first_failure = None + + for start in range(0, len(hits), BULK_BATCH_SIZE): + batch = hits[start : start + BULK_BATCH_SIZE] + bulk_lines = [] + for h in batch: + bulk_lines.append(json.dumps({"create": {"_index": INDEX, "_id": h["_id"]}})) + bulk_lines.append(json.dumps(canonicalise(h["_source"], model_id))) + bulk_body = "\n".join(bulk_lines) + "\n" + req = urllib.request.Request( + f"{GOLDEN_URL}/{INDEX}/_bulk", + data=bulk_body.encode(), + headers={ + "Authorization": f"ApiKey {GOLDEN_KEY}", + "Content-Type": "application/x-ndjson", + "kbn-xsrf": "kbn-client", + "x-elastic-internal-origin": "kbn-client", + }, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=60) as r: + result = json.loads(r.read()) + except Exception as exc: # noqa: BLE001 - keep earlier batches + # A dead batch must not erase the batches that already landed. + failed += len(batch) + if first_failure is None: + first_failure = {"transport": str(exc)} + sys.stderr.write( + f"batch {start}-{start + len(batch)} failed at transport: {exc}\n" + ) + continue + + for item in result.get("items", []): + op = item.get("create", {}) + status = int(op.get("status", 500)) + if status in (201, 409): + exported += 1 + else: + failed += 1 + if first_failure is None: + first_failure = op + + if failed: + # Report, do not discard. The documents that landed are real and the + # golden gate downstream counts them independently. + sys.stderr.write( + f"export incomplete for {model_id}: {exported} landed, {failed} failed " + f"of {len(hits)}; first={first_failure}\n" + ) + else: + sys.stderr.write( + f"exported {exported} docs for {model_id} " + f"(models={sorted(stored_models)}, idempotent_conflicts_ok)\n" + ) + return exported, failed + + +if __name__ == "__main__": + if len(sys.argv) != 2: + raise SystemExit("usage: export_scores.py ") + model = sys.argv[1] + count, missing = export_model(model) + # Transport-level check only: >0 docs landed. Exact-count validation + # (21 examples x (evaluators + 1) x reps) happens in the sweep + # controller's golden gate, which derives the evaluator count live from + # the VM's Scout summary — the hardcoded 252 here went stale when the + # suite grew to 14 docs/example (21x14=294). + # + # A PARTIAL export is not a success. Exiting 0 would tell the controller + # everything landed while documents were silently missing -- exactly the + # false green the golden gate exists to catch. 2 = partial, 1 = nothing. + if count == 0: + raise SystemExit(1) + raise SystemExit(2 if missing else 0) diff --git a/scripts/orca_vm/extract_attack_discovery.py b/scripts/orca_vm/extract_attack_discovery.py new file mode 100644 index 0000000000000..333b02c2e967b --- /dev/null +++ b/scripts/orca_vm/extract_attack_discovery.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""Extract attack-discovery results from the golden ES cluster. + +Writes a JSON aggregate consumed by render_attack_discovery_board.py. + +IMPORTANT — why _source scanning and not `exists` aggregations: + Fields under `task.output.*` are mapped non-queryable on the golden cluster. + An `exists` filter on task.output.adToolResult.discoveryCount returns 0 while + the field is plainly present in _source (positive control: evaluator.score + returns 17,169 on the identical query shape). Coverage MUST therefore be + measured by reading _source. Do not "optimise" this into an aggregation. + +Absent fields are reported as null and MUST render blank downstream. The +reference artifact's Latency / Total risk columns have no counterpart in this +schema; they are not imputed. +""" +import argparse +import collections +import json +import re +import sys +import urllib.request + +SUITE_ID = "attack-discovery-agent-builder" + + +def load_env(path): + env = {} + with open(path) as fh: + for line in fh: + m = re.match(r"""\s*export\s+(\w+)=['"]?([^'"\n]+)""", line) + if m: + env[m.group(1)] = m.group(2).strip() + return env + + +class Es: + def __init__(self, url, key): + self.url = url.rstrip("/") + self.key = key + + def post(self, path, body=None): + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(self.url + path, data=data, method="POST") + req.add_header("Authorization", f"ApiKey {self.key}") + req.add_header("Content-Type", "application/json") + with urllib.request.urlopen(req, timeout=180) as resp: + return json.load(resp) + + +def dig(src, dotted): + cur = src + for part in dotted.split("."): + if not isinstance(cur, dict) or part not in cur: + return None + cur = cur[part] + return cur + + +def scan(es, suite_id, since=None): + query = {"term": {"metadata.suite_id": suite_id}} + if since: + # The suite has historical executions on golden (CI runs from Jul/Aug, + # a Sep 7 buildkite batch). A board render must reflect ONE sweep; leave + # --since unset only when you deliberately want the full history. + query = { + "bool": { + "filter": [ + {"term": {"metadata.suite_id": suite_id}}, + {"range": {"@timestamp": {"gte": since}}}, + ] + } + } + body = { + "size": 1000, + "query": query, + "_source": [ + "task.output.adToolResult", + "task.output.workflow", + "task.output.insights", + "task.output.traceId", + "task.output.raw", + "task.output.errors", + "task.model.id", + "example.dataset.name", + "evaluator.name", + "evaluator.score", + "metadata.execution_id", + "@timestamp", + ], + "sort": ["_doc"], + } + res = es.post("/.evaluation-scores*/_search?scroll=5m", body) + scroll_id = res.get("_scroll_id") + total = res["hits"]["total"]["value"] + docs = [] + while res["hits"]["hits"]: + docs.extend(h["_source"] for h in res["hits"]["hits"]) + res = es.post("/_search/scroll", {"scroll": "5m", "scroll_id": scroll_id}) + scroll_id = res.get("_scroll_id") + return docs, total + + +def _blank_row(): + # Heterogeneous value types: static analysers infer a single union type for + # the dict values and then flag every += / .append below. Behaviour is + # correct; the annotation keeps the checker quiet. + row: dict = { + "docs": 0, + "discoveryCounts": [], + "alertsContextCounts": [], + "statuses": collections.Counter(), + "validatedDiscoveryCounts": [], + "evaluators": collections.defaultdict(list), + "datasets": collections.Counter(), + "executions": collections.Counter(), + "insights": [], + "traceIds": [], + "rawLatencyMs": [], + "errorTexts": set(), + } + return row + + +def build(docs): + per_model = collections.defaultdict(_blank_row) + for src in docs: + model = dig(src, "task.model.id") + if not model: + continue + row = per_model[model] + row["docs"] += 1 + row["datasets"][dig(src, "example.dataset.name") or "?"] += 1 + + dc = dig(src, "task.output.adToolResult.discoveryCount") + if dc is not None: + row["discoveryCounts"].append(dc) + ac = dig(src, "task.output.adToolResult.alertsContextCount") + if ac is not None: + row["alertsContextCounts"].append(ac) + st = dig(src, "task.output.adToolResult.status") + if st is not None: + row["statuses"][str(st)] += 1 + vd = dig(src, "task.output.workflow.validatedDiscoveryCount") + if vd is not None: + row["validatedDiscoveryCounts"].append(vd) + + # persona-matrix (_generate API) schema: the product result rides in + # task.output.raw (status / alerts_context_count / latency_ms) and the + # final answer is task.output.insights. Promote once per execution so + # these do not multiply by evaluator-doc count. + raw = dig(src, "task.output.raw") or {} + exec_id_pm = dig(src, "metadata.execution_id") + if raw and exec_id_pm: + marker = ("raw", exec_id_pm) + if marker not in row["traceIds"]: + row["traceIds"].append(marker) + ins_pm = dig(src, "task.output.insights") or [] + row["discoveryCounts"].append(len(ins_pm)) + if raw.get("alerts_context_count") is not None: + row["alertsContextCounts"].append(raw["alerts_context_count"]) + row["statuses"][str(raw.get("status") or "unknown")] += 1 + if raw.get("latency_ms") is not None: + row["rawLatencyMs"].append(raw["latency_ms"]) + errs = dig(src, "task.output.errors") + if errs: + for e in errs[:2]: + row["errorTexts"].add(str(e)[:180]) + + ev = dig(src, "evaluator.name") + sc = dig(src, "evaluator.score") + if ev and isinstance(sc, (int, float)): + row["evaluators"][ev].append(float(sc)) + + # Final-answer capture for trace cards: task.output.insights is the + # AD generate-API result (title, summary, MITRE tactics, risk score). + # Stored once per execution, not per evaluator doc. + exec_id = dig(src, "metadata.execution_id") + if exec_id and exec_id not in row["executions"]: + row["executions"][exec_id] = 0 + insights = dig(src, "task.output.insights") + if insights and exec_id: + marker = (exec_id, dig(src, "task.output.traceId")) + if marker not in row["traceIds"]: + row["traceIds"].append(marker) + row["insights"].append( + { + "executionId": exec_id, + "traceId": dig(src, "task.output.traceId"), + "insights": insights, + } + ) + + def mean(xs): + return round(sum(xs) / len(xs), 4) if xs else None + + out = [] + for model, row in sorted(per_model.items()): + statuses = dict(row["statuses"]) + completed = statuses.get("completed", 0) + status_total = sum(statuses.values()) + + # Latency is NOT a task.output field -- it is recorded as a scored + # evaluator named "Latency" (seconds). Promote it so the board shows a + # real column instead of a blank. Verified present for all 39 models. + lat_scores = row["evaluators"].get("Latency") or [] + latency_seconds = round(sum(lat_scores) / len(lat_scores), 1) if lat_scores else None + + # persona-matrix _generate latency: product-side wall clock from + # task.output.raw.latency_ms (ms -> s), independent of the evaluator + # latency (which measures the whole test, not the generate call). + gen_latency_seconds = ( + round(sum(row["rawLatencyMs"]) / len(row["rawLatencyMs"]) / 1000.0, 1) + if row["rawLatencyMs"] else None + ) + + # Total risk has no source anywhere in the golden schema (neither a + # task.output field nor an evaluator) -- it stays absent, renders blank. + out.append( + { + "modelId": model, + "docs": row["docs"], + "datasets": len(row["datasets"]), + # Coverage is explicit so the board can show N/total, never a + # bare mean that hides how many docs actually carried the field. + "discoveryCount": { + "mean": mean(row["discoveryCounts"]), + "n": len(row["discoveryCounts"]), + }, + "alertsContextCount": { + "mean": mean(row["alertsContextCounts"]), + "n": len(row["alertsContextCounts"]), + }, + "validatedDiscoveryCount": { + "mean": mean(row["validatedDiscoveryCounts"]), + "n": len(row["validatedDiscoveryCounts"]), + }, + "status": { + "completedRate": round(completed / status_total, 4) if status_total else None, + "counts": statuses, + "n": status_total, + }, + # Latency recovered from the "Latency" evaluator (seconds). + "latencySeconds": latency_seconds, + # persona-matrix _generate wall-clock (seconds) from raw.latency_ms. + "generateLatencySeconds": gen_latency_seconds, + # Errors recorded on the output doc (e.g. "generation failed: + # Maximum generation attempts (10) reached") -- surfaced so a + # failed generation cannot hide behind a passing doc-count gate. + "generateErrors": sorted(row["errorTexts"])[:5], + # Total risk: present in the reference artifact, absent from our + # schema. Explicit null -> blank cell + disclosure. Never imputed. + "totalRisk": None, + "evaluators": {k: {"mean": mean(v), "n": len(v)} for k, v in sorted(row["evaluators"].items())}, + # Trace-card payload: one entry per execution with the final AD + # answer (insights) and the OTel trace id linking to the full span. + "traceCards": [ + { + "executionId": t["executionId"], + "traceId": t["traceId"], + "insightCount": len(t["insights"]), + "insights": t["insights"], + } + for t in row["insights"] + ], + "executionCount": len(row["executions"]), + } + ) + return out + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--env-file", default="/tmp/golden-cluster-env.sh") + ap.add_argument("--suite-id", default=SUITE_ID) + ap.add_argument("--since", help="ISO date lower bound; excludes historical executions (e.g. 2026-09-10T12:00Z)") + ap.add_argument("--out", required=True) + args = ap.parse_args() + + env = load_env(args.env_file) + url, key = env.get("GOLDEN_ES_URL"), env.get("GOLDEN_ES_API_KEY") + if not url or not key: + sys.exit(f"GOLDEN_ES_URL / GOLDEN_ES_API_KEY missing from {args.env_file}") + + es = Es(url, key) + docs, total = scan(es, args.suite_id, since=args.since) + if not docs: + sys.exit(f"no documents for suite_id={args.suite_id} — refusing to write an empty aggregate") + + rows = build(docs) + payload = { + "suiteId": args.suite_id, + "sourceDocCount": len(docs), + "reportedTotal": total, + "modelCount": len(rows), + # Only totalRisk is truly absent; latencySeconds is recovered from the + # "Latency" evaluator. Computed, not hardcoded, so this can never drift + # from what the rows actually carry. + "absentFields": sorted( + f for f in ("latencySeconds", "totalRisk") if all(r[f] is None for r in rows) + ), + "models": rows, + } + with open(args.out, "w") as fh: + json.dump(payload, fh, indent=2) + print(f"scanned {len(docs)} docs (reported {total}) -> {len(rows)} models -> {args.out}") + + +if __name__ == "__main__": + main() diff --git a/scripts/orca_vm/golden_coverage.py b/scripts/orca_vm/golden_coverage.py new file mode 100644 index 0000000000000..d90ac6686638f --- /dev/null +++ b/scripts/orca_vm/golden_coverage.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Preflight coverage map for the golden cluster. + +One command answering, per suite x era window: how many score docs, what % +carry traceId, and how many arg/usage spans join. Run BEFORE any board build +to pick the right suite/index/window and catch silent-zero joins up front. + +Usage: + golden_coverage.py [--suite X] [--since ISO] [--until ISO] [--env-file F] + +Defaults mirror the agent_eval_full board build (suite=security-persona-matrix, +since=2026-09-01). Exit 0 always — this is a map, not a gate; the builder's +guards are the gate. +""" +import argparse +import json +import os +import sys +import urllib.request + +SCORES_IDX = "/.ds-.evaluation-scores*/_search" +TRACES_IDX = "/.ds-traces-generic.otel-default*,.ds-traces-agent_builder.otel-default*/_search" + + +def load_env(path): + env = {} + with open(path) as fh: + for line in fh: + line = line.strip() + if line.startswith("export "): + line = line[7:] + if "=" in line: + k, v = line.split("=", 1) + env[k.strip()] = v.strip().strip('"').strip("'") + return env + + +class Es: + def __init__(self, url, key): + self.url, self.key = url, key + + def post(self, path, body): + req = urllib.request.Request( + self.url + path, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json", "Authorization": "ApiKey " + self.key}) + with urllib.request.urlopen(req, timeout=120) as r: + return json.load(r) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--env-file", default=os.path.expanduser("~/.elastic/golden-cluster-env.sh")) + ap.add_argument("--suite", default="security-persona-matrix") + ap.add_argument("--since", default="2026-09-01T00:00Z") + ap.add_argument("--until", default=None) + args = ap.parse_args() + + env = load_env(args.env_file) + url, key = os.environ.get("GOLDEN_ES_URL") or env.get("GOLDEN_ES_URL"), \ + os.environ.get("GOLDEN_ES_API_KEY") or env.get("GOLDEN_ES_API_KEY") + if not url or not key: + sys.exit("GOLDEN_ES_URL / GOLDEN_ES_API_KEY missing") + es = Es(url, key) + + rng = {"gte": args.since} + if args.until: + rng["lt"] = args.until + + # score docs + traceId coverage (sample 500) + r = es.post(SCORES_IDX, { + "size": 0, + "query": {"bool": {"filter": [ + {"term": {"metadata.suite_id": args.suite}}, + {"range": {"@timestamp": rng}}, + ]}}, + }) + total = r["hits"]["total"]["value"] + sample = es.post(SCORES_IDX, { + "size": 500, + "query": {"bool": {"filter": [ + {"term": {"metadata.suite_id": args.suite}}, + {"range": {"@timestamp": rng}}, + ]}}, + "_source": ["task.output.traceId", "task.model.id"], + "sort": [{"@timestamp": "desc"}], + }) + hits = sample["hits"]["hits"] + n_tid = sum(1 for h in hits if ((h["_source"].get("task") or {}).get("output") or {}).get("traceId")) + models = sorted({((h["_source"].get("task") or {}).get("model") or {}).get("id", "?") for h in hits}) + + # usage + arg span counts in window (both trace indices) + spans = es.post(TRACES_IDX, { + "size": 0, + "query": {"bool": {"filter": [ + {"range": {"@timestamp": rng}}, + {"bool": {"should": [ + {"exists": {"field": "attributes.gen_ai.usage.input_tokens"}}, + {"exists": {"field": "gen_ai.usage.input_tokens"}}, + {"exists": {"field": "attributes.gen_ai.tool.call.arguments"}}, + ]}}, + ]}}, + "aggs": { + "usage": {"filter": {"bool": {"should": [ + {"exists": {"field": "attributes.gen_ai.usage.input_tokens"}}, + {"exists": {"field": "gen_ai.usage.input_tokens"}}, + ]}}}, + "args": {"filter": {"exists": {"field": "attributes.gen_ai.tool.call.arguments"}}}, + }, + }) + aggs = spans["aggregations"] + n_usage, n_args = aggs["usage"]["doc_count"], aggs["args"]["doc_count"] + + # linkage: do sampled score traceIds find ANY span? + tids = [((h["_source"].get("task") or {}).get("output") or {}).get("traceId") for h in hits[:50]] + tids = [t for t in tids if t] + linked = 0 + if tids: + chk = es.post(TRACES_IDX, { + "size": 0, + "query": {"terms": {"trace_id": tids[:50]}}, + "aggs": {"n": {"cardinality": {"field": "trace_id"}}}, + }) + linked = chk["aggregations"]["n"]["value"] + + print(f"suite={args.suite} window={args.since}..{args.until or 'open'}") + print(f"score docs: {total} | sampled {len(hits)}: {n_tid} carry traceId ({100*n_tid//max(len(hits),1)}%) | linked-of-50: {linked}/{len(tids)}") + print(f"models in window (sampled): {len(models)} -> {', '.join(models[:8])}{' ...' if len(models) > 8 else ''}") + print(f"spans in window: usage={n_usage} args={n_args}") + print("verdict:", "JOIN LIKELY LIVE" if (tids and linked > 0 and n_usage > 0) else + ("NO TRACE LINKAGE (era/indices) — expect zero-usage board, disclose" if n_usage > 0 else + "NO USAGE SPANS IN WINDOW — expect zero-usage board, disclose")) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/orca_vm/measure_connector_false_green.py b/scripts/orca_vm/measure_connector_false_green.py new file mode 100644 index 0000000000000..89dbde7a67b44 --- /dev/null +++ b/scripts/orca_vm/measure_connector_false_green.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +"""Measure the workflow-authoring false green against the golden cluster. + +`ExpectedToolCalled` proves the model CALLED `generate_workflow`. It says +nothing about what the call produced. This script re-reads the produced +workflow text and asks the stricter question: does the authored workflow +actually contain an http step aimed at the Slack connector? + +The gap between those two numbers is the false green. + +`task.output.*` is mapped non-queryable, so query_string/exists return 0 +against it -- everything here scans `_source`. A positive control (docs whose +transcript mentions slack at all) is reported first: if that is 0 the scan is +blind and every downstream number is meaningless, so the script refuses. +""" +import argparse +import collections +import json +import re +import sys +import urllib.request + +SLACK_ID = "d7306385-cbe6-4541-9726-49afdff59ba5" +STEP_RE = re.compile(r"""type\s*:\s*["']?http\b""", re.I) +SUITES = ["security-persona-matrix", "skill-selection-benchmark"] + + +def load_env(path): + env = {} + for line in open(path): + m = re.match(r"""\s*export\s+(\w+)=['"]?([^'"\n]+)""", line) + if m: + env[m.group(1)] = m.group(2).strip() + return env + + +def collect_text(node, out): + if isinstance(node, str): + out.append(node) + elif isinstance(node, list): + for v in node: + collect_text(v, out) + elif isinstance(node, dict): + for v in node.values(): + collect_text(v, out) + + +def scan(url, key, max_docs): + def post(body): + req = urllib.request.Request( + f"{url}/.evaluation-scores*/_search", + data=json.dumps(body).encode(), + headers={"Authorization": f"ApiKey {key}", "Content-Type": "application/json"}, + ) + return json.load(urllib.request.urlopen(req, timeout=300)) + + after, cells, scanned, saw_slack = None, {}, 0, 0 + exhausted = False + while True: + body = { + "size": 500, + "sort": [{"_doc": "asc"}], + "query": { + "bool": { + "should": [{"term": {"metadata.suite_id": s}} for s in SUITES], + "minimum_should_match": 1, + } + }, + "_source": [ + "task.model.id", + "task.output", + "evaluator.name", + "evaluator.score", + "example.id", + "example.dataset.name", + ], + } + if after: + body["search_after"] = after + hits = post(body)["hits"]["hits"] + if not hits: + exhausted = True + break + for h in hits: + s = h["_source"] + scanned += 1 + parts = [] + collect_text(s.get("task", {}).get("output"), parts) + blob = "\n".join(parts) + if "slack" not in blob.lower(): + continue + saw_slack += 1 + key_t = ( + s.get("task", {}).get("model", {}).get("id"), + s.get("example", {}).get("dataset", {}).get("name", ""), + s.get("example", {}).get("id"), + ) + cell = cells.setdefault(key_t, {"text": "", "scores": collections.defaultdict(list)}) + if len(blob) > len(cell["text"]): + cell["text"] = blob + ev = s.get("evaluator", {}) + if ev.get("name") and isinstance(ev.get("score"), (int, float)): + cell["scores"][ev["name"]].append(ev["score"]) + after = hits[-1]["sort"] + if max_docs and scanned >= max_docs: + break + return cells, scanned, saw_slack, exhausted + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--env-file", default="/tmp/golden-cluster-env.sh") + ap.add_argument("--max-docs", type=int, default=12000, + help="0 scans every matching doc; the default caps the scan and the cap is disclosed in the output") + ap.add_argument("--out", required=True) + args = ap.parse_args() + + env = load_env(args.env_file) + cells, scanned, saw_slack, exhausted = scan( + env["GOLDEN_ES_URL"].rstrip("/"), env["GOLDEN_ES_API_KEY"], args.max_docs + ) + + # Positive control. A zero here means the scan never saw transcript text, + # in which case "0 false greens" would be an artefact, not a finding. + if saw_slack == 0: + print("POSITIVE CONTROL FAILED: no transcript mentions slack; scan is blind", file=sys.stderr) + return 1 + + etc_pass = real_pass = false_green = 0 + missing_step = missing_id = 0 + by_model = collections.defaultdict(lambda: [0, 0]) + + for (model, _ds, _ex), c in cells.items(): + t = c["text"] + step_ok = bool(STEP_RE.search(t)) + id_ok = SLACK_ID in t + # Require an http step always; require the connector id only when the + # transcript shows the prompt actually supplied one. + needs_id = id_ok or "connector with id" in t.lower() + real = step_ok and (id_ok if needs_id else True) + if not step_ok: + missing_step += 1 + if needs_id and not id_ok: + missing_id += 1 + etc = c["scores"].get("ExpectedToolCalled") + if etc and sum(etc) / len(etc) >= 0.99: + etc_pass += 1 + if not real: + false_green += 1 + if real: + real_pass += 1 + by_model[model][0] += 1 + by_model[model][1] += 1 + + result = { + "scan": { + "suites": SUITES, + "docsScanned": scanned, + "scanComplete": exhausted, + "maxDocs": args.max_docs or None, + "positiveControlDocsMentioningSlack": saw_slack, + "slackRelatedCells": len(cells), + "caveat": ( + "Sample spans two suites, so this is not a clean single-suite rate. " + + ("Scan hit the doc cap and is a sample, not the full index." + if not exhausted else "Scan reached the end of the index.") + ), + }, + "summary": { + "expectedToolCalledPass": etc_pass, + "reallyTargetsConnector": real_pass, + "falseGreen": false_green, + "cellsWithoutHttpStep": missing_step, + "cellsGivenIdButOmittedIt": missing_id, + }, + "perModel": [ + {"model": m, "real": ok, "total": tot, "rate": round(100 * ok / tot, 1)} + for m, (ok, tot) in sorted(by_model.items(), key=lambda kv: -kv[1][1]) + ], + } + + json.dump(result, open(args.out, "w"), indent=2) + print(json.dumps(result["scan"], indent=2)) + print(json.dumps(result["summary"], indent=2)) + print(f"wrote {args.out}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/orca_vm/merge_trace_cache.py b/scripts/orca_vm/merge_trace_cache.py new file mode 100644 index 0000000000000..c717715f72f87 --- /dev/null +++ b/scripts/orca_vm/merge_trace_cache.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Merge trace caches so adding one model does not require re-pulling every model. + +Why this exists +--------------- +`build_trace_cache.py` pulls EVERY model's score documents for the whole lookback +window (2431 cells / 188 MB on 2026-09-02, several minutes against golden). When +a single new model lands, only its own cells are new. This merges a small +freshly-pulled cache on top of the existing one. + +Cache keys are `${metadata.execution_id}::${example.id}`, and execution_id embeds +the model, so a new model's keys can never collide with an existing model's. A +RE-RUN of an existing model does produce new execution_ids, which is why the +merge is additive: the matrix generator's own pickLatestExperimentPerModel +decides which execution wins at render time, not this script. + +Usage +----- + # pull only what is new (small --hours window) + source /tmp/golden-cluster-env.sh + python3 build_trace_cache.py --out /tmp/trace_cache_newmodel.json --hours 48 + python3 merge_trace_cache.py --base /tmp/trace_cache_v5.json \ + --overlay /tmp/trace_cache_newmodel.json --out /tmp/trace_cache_v6.json +""" +from __future__ import annotations + +import argparse +import collections +import json +import os +import sys + +# Node aborts readFileSync above its max string length (0x1fffffe8, ~536 MB); +# same ceiling build_trace_cache.py enforces. A merge is the most likely way to +# cross it, so re-check here rather than discovering it in the CLI. +MAX_CACHE_BYTES = 500_000_000 + + +def model_of(docs: list) -> str | None: + if not docs: + return None + return ((docs[0].get("task") or {}).get("model") or {}).get("id") + + +def merge(base: dict, overlays: list[dict]) -> tuple[dict, dict]: + """Overlay wins on key collision (a re-pull of the same execution is fresher).""" + merged = dict(base) + stats = {"base_keys": len(base), "added": 0, "replaced": 0} + for overlay in overlays: + for key, docs in overlay.items(): + if key in merged: + stats["replaced"] += 1 + else: + stats["added"] += 1 + merged[key] = docs + stats["merged_keys"] = len(merged) + return merged, stats + + +def model_counts(cache: dict) -> collections.Counter: + counts: collections.Counter = collections.Counter() + for docs in cache.values(): + model = model_of(docs) + if model: + counts[model] += 1 + return counts + + +def self_test() -> int: + failures = [] + + def check(name, got, want): + if got != want: + failures.append(f"{name}: got {got!r} want {want!r}") + + base = {"exec-a::security-persona-matrix::m1::ex1": [{"task": {"model": {"id": "m1"}}}]} + overlay = {"exec-b::security-persona-matrix::m2::ex1": [{"task": {"model": {"id": "m2"}}}]} + merged, stats = merge(base, [overlay]) + check("disjoint merge size", len(merged), 2) + check("disjoint added", stats["added"], 1) + check("disjoint replaced", stats["replaced"], 0) + + # A re-pull of the SAME key must take the overlay's docs, not the base's. + collide = {"exec-a::security-persona-matrix::m1::ex1": [{"task": {"model": {"id": "m1"}}, "fresh": True}]} + merged2, stats2 = merge(base, [collide]) + check("collision replaced", stats2["replaced"], 1) + check("collision keeps overlay", merged2["exec-a::security-persona-matrix::m1::ex1"][0].get("fresh"), True) + + check("model counts", dict(model_counts(merged)), {"m1": 1, "m2": 1}) + check("empty docs ignored", dict(model_counts({"k": []})), {}) + + for failure in failures: + print(f"FAIL {failure}", file=sys.stderr) + print(f"self-test: {len(failures)} failure(s)") + return 1 if failures else 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", help="existing cache to build on") + parser.add_argument("--overlay", action="append", default=[], + help="cache(s) to merge in; repeatable, later wins") + parser.add_argument("--out") + parser.add_argument("--self-test", action="store_true", + help="Verify the merge contract offline and exit.") + args = parser.parse_args() + + if args.self_test: + return self_test() + + if not args.base or not args.overlay or not args.out: + print("--base, --overlay and --out are required", file=sys.stderr) + return 2 + + with open(args.base) as handle: + base = json.load(handle) + overlays = [] + for path in args.overlay: + with open(path) as handle: + overlays.append(json.load(handle)) + + merged, stats = merge(base, overlays) + + before = model_counts(base) + after = model_counts(merged) + + with open(args.out, "w") as handle: + json.dump(merged, handle) + + size = os.path.getsize(args.out) + print(f"base keys : {stats['base_keys']}") + print(f"added : {stats['added']}") + print(f"replaced : {stats['replaced']}") + print(f"merged keys : {stats['merged_keys']}") + print(f"written : {args.out} ({size / 1e6:.0f} MB)") + if size > MAX_CACHE_BYTES: + print(f"FATAL: cache exceeds Node's readFileSync string cap " + f"({size} > {MAX_CACHE_BYTES}); the matrix CLI cannot read it", + file=sys.stderr) + return 1 + + new_models = sorted(set(after) - set(before)) + if new_models: + print("new models : " + ", ".join(f"{m} ({after[m]} cells)" for m in new_models)) + grown = sorted(m for m in set(after) & set(before) if after[m] != before[m]) + for model in grown: + print(f"grew : {model} {before[model]} -> {after[model]} cells") + if not new_models and not grown: + print("new models : none -- the overlay added no cells for any model") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/orca_vm/model_ids.py b/scripts/orca_vm/model_ids.py new file mode 100644 index 0000000000000..d96de24523bad --- /dev/null +++ b/scripts/orca_vm/model_ids.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Shared model-id handling for sweep gates. + +Stored ids are NOT derivable by string rules: `eis-zai-glm-5-2` stores as +`zai-glm-5-2` (hyphen) but `eis-anthropic-claude-4-6-sonnet` stores as +`anthropic-claude-4.6-sonnet` (dot); others use slashes or display names. +Guessing once reported GLM as 0 docs when it had 3,834. Compare on `norm()`, +and resolve against live ids whenever a cluster is reachable. +""" +from __future__ import annotations + + +def norm(value: str) -> str: + """Collapse an id to comparable form: lowercase, alphanumerics only.""" + return "".join(ch for ch in value.lower() if ch.isalnum()) + + +def strip_connector_prefix(model_id: str) -> str: + return model_id[4:] if model_id.startswith("eis-") else model_id + + +def same_model(a: str, b: str) -> bool: + """True when two ids denote the same model across id conventions.""" + return norm(strip_connector_prefix(a)) == norm(strip_connector_prefix(b)) + + +def resolve_model_id(connector_id: str, stored_ids: list[str]) -> str | None: + """Match a connector id to the id actually present in score docs. + + Returns None only when nothing matches — a real "never landed" signal + rather than an artifact of a bad guess. + """ + target = norm(strip_connector_prefix(connector_id)) + for stored in stored_ids: + if norm(stored) == target: + return stored + return None diff --git a/scripts/orca_vm/openrouter_proxy.py b/scripts/orca_vm/openrouter_proxy.py new file mode 100644 index 0000000000000..fa81530a55ff6 --- /dev/null +++ b/scripts/orca_vm/openrouter_proxy.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +""" +OpenRouter SSE-Normalizing Proxy for reasoning models (e.g. Gemini 3.7 Flash). + +Sits between ES's openai inference service and OpenRouter. Handles three problems: + +1. REQUEST: ES forwards the 'reasoning' field from unified completion API to + OpenRouter. Gemini 3.7 Flash REQUIRES reasoning and rejects reasoning_effort:'none' + with 400 "Reasoning is mandatory for this endpoint and cannot be disabled." + → Strip 'reasoning' from request body, add max_tokens=16000 for reasoning budget. + +2. RESPONSE: OpenRouter's SSE stream includes 'reasoning' and 'reasoning_content' + fields in delta objects. ES's OpenAiUnifiedStreamingProcessor can't parse them + → "[chat_completion_chunk] failed to parse field [choices]" + → Strip reasoning fields from SSE response chunks, ensure 'content' exists. + +3. NON-STANDARD FIELDS: OpenRouter sends 'native_finish_reason', 'reasoning_tokens', + 'cached_tokens' that ES doesn't understand → strip from response. + +Usage: + python3 openrouter-proxy.py [--port 8088] + +Then create ES inference endpoint pointing at the proxy: + curl -X PUT "http://elastic:changeme@localhost:9220/_inference/chat_completion/openrouter-gemini-3-7-flash" \ + -H "Content-Type: application/json" \ + -d '{"service":"openai","service_settings":{"model_id":"google/gemini-3.7-flash","url":"http://localhost:8088","api_key":""}}' + +Verification: + # Check proxy is running + curl -s http://localhost:8088/ # 404 for GET, handles POST + + # Check ES inference endpoint + curl -s "http://elastic:changeme@localhost:9220/_inference/chat_completion/openrouter-gemini-3-7-flash" + + # Check eval logs for success + grep -E "passed|Evaluator|EVAL_EXIT" /tmp/eval-*.log +""" + +import http.server +import json +import ssl +import urllib.request +import urllib.error +import sys +import os + +# Upstream is configurable so non-openrouter routes (selfhost cells, omniroute +# judge combos) reuse this proxy: run_model.sh exports PROXY_UPSTREAM before +# launching. Import-time read is correct — the env prefix sits on the launch +# line itself. Default keeps openrouter-* candidates unchanged. +TARGET = os.environ.get("PROXY_UPSTREAM") or "https://openrouter.ai/api/v1" +LISTEN_PORT = 8088 + + +class OpenRouterProxyHandler(http.server.BaseHTTPRequestHandler): + def do_POST(self): + content_length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(content_length) + + try: + data = json.loads(body) + except: + data = {} + + # Gemini 3.7 Flash REQUIRES reasoning — can't send reasoning_effort=none (400) + # Strip ES unified 'reasoning' field that OpenRouter doesn't understand + if 'reasoning' in data: + del data['reasoning'] + print(f"[proxy] Stripped 'reasoning' field from request", flush=True) + # Ensure max_tokens is high enough so reasoning doesn't consume all output + if 'chat/completions' in self.path or 'chat_completion' in self.path or self.path == '/': + if 'max_tokens' not in data and 'max_completion_tokens' not in data: + data['max_tokens'] = 16000 + print(f"[proxy] Added max_tokens=16000 to request", flush=True) + + modified_body = json.dumps(data).encode() + + # Forward to OpenRouter - map root path to /chat/completions + if self.path == '/' or self.path == '': + url = TARGET + '/chat/completions' + else: + url = TARGET + self.path + req = urllib.request.Request(url, data=modified_body, method='POST') + + # Copy headers + for key, val in self.headers.items(): + if key.lower() not in ('host', 'content-length', 'accept-encoding'): + req.add_header(key, val) + req.add_header('Content-Length', str(len(modified_body))) + + try: + resp = urllib.request.urlopen(req, timeout=300) + self.send_response(resp.status) + for key, val in resp.headers.items(): + if key.lower() not in ('transfer-encoding',): + self.send_header(key, val) + self.end_headers() + # Stream the response with SSE normalization + # ES OpenAiUnifiedStreamingProcessor rejects reasoning fields in finish chunks + buffer = b'' + while True: + chunk = resp.read(4096) + if not chunk: + break + buffer += chunk + # Process complete SSE lines + while b'\n' in buffer: + line, buffer = buffer.split(b'\n', 1) + line_str = line.decode('utf-8', errors='replace').strip() + if line_str.startswith('data: ') and line_str != 'data: [DONE]': + data_str = line_str[6:] + try: + chunk_data = json.loads(data_str) + # Strip reasoning from response chunks + if 'choices' in chunk_data: + for choice in chunk_data['choices']: + if 'delta' in choice: + delta = choice['delta'] + if 'reasoning' in delta: + del delta['reasoning'] + if 'reasoning_content' in delta: + del delta['reasoning_content'] + # Ensure content field exists (ES parser expects it) + if 'delta' in choice and 'content' not in choice['delta']: + choice['delta']['content'] = '' + # Strip native_finish_reason if present (ES rejects it) + if 'native_finish_reason' in chunk_data: + del chunk_data['native_finish_reason'] + # Strip reasoning_tokens from usage + usage = chunk_data.get('usage') + if usage and isinstance(usage, dict): + usage.pop('reasoning_tokens', None) + usage.pop('cached_tokens', None) + line = (b'data: ' + json.dumps(chunk_data).encode() + b'\n') + except: + pass # Pass through non-JSON lines + self.wfile.write(line + b'\n') + self.wfile.flush() + # Write any remaining buffer + if buffer: + self.wfile.write(buffer) + self.wfile.flush() + except urllib.error.HTTPError as e: + self.send_response(e.code) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(e.read()) + except Exception as e: + self.send_response(502) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps({"error": str(e)}).encode()) + + def do_GET(self): + url = TARGET + self.path + req = urllib.request.Request(url) + for key, val in self.headers.items(): + if key.lower() not in ('host', 'accept-encoding'): + req.add_header(key, val) + + try: + resp = urllib.request.urlopen(req, timeout=15) + self.send_response(resp.status) + for key, val in resp.headers.items(): + self.send_header(key, val) + self.end_headers() + self.wfile.write(resp.read()) + except urllib.error.HTTPError as e: + self.send_response(e.code) + self.end_headers() + self.wfile.write(e.read()) + except Exception as e: + self.send_response(502) + self.end_headers() + self.wfile.write(json.dumps({"error": str(e)}).encode()) + + def log_message(self, format, *args): + print(f"[proxy] {format % args}", flush=True) + + +class ThreadedHTTPServer(http.server.ThreadingHTTPServer): + daemon_threads = True + + +if __name__ == "__main__": + port = LISTEN_PORT + if '--port' in sys.argv: + idx = sys.argv.index('--port') + if idx + 1 < len(sys.argv): + port = int(sys.argv[idx + 1]) + + server = ThreadedHTTPServer(("0.0.0.0", port), OpenRouterProxyHandler) + print(f"[proxy] OpenRouter proxy on port {port} → {TARGET}", flush=True) + print(f"[proxy] SSE-normalizing proxy — strips reasoning from request+response, adds max_tokens=16000", flush=True) + try: + server.serve_forever() + except KeyboardInterrupt: + server.shutdown() diff --git a/scripts/orca_vm/persona_matrix_sweep.py b/scripts/orca_vm/persona_matrix_sweep.py new file mode 100644 index 0000000000000..ab98007c6e0ef --- /dev/null +++ b/scripts/orca_vm/persona_matrix_sweep.py @@ -0,0 +1,2420 @@ +#!/usr/bin/env python3 +""" +persona_matrix_sweep.py — Azure D8s_v5 sweep controller for the proprietary +model matrix (persona-matrix suite, 21 examples / 7 categories / 3 variants). + +Design (validated 2026-08-19/20 across 15 models): +- 1 VM per model. `evals start` parallelizes internally; tokens dominate cost, + so wider fanout only multiplies boot-failure surface. +- VM boot: image orca-eval-base-v5 (Node 24.19.0, Kibana, repo, deploy keys). +- Per-model run: /tmp/run_model.sh — minimal proven path: + stop → clean ES data → `evals start --profile local` (owns scout+CCM+ + readiness) → export scores to golden (252-doc completeness gate). +- deploy() overlays two patched files onto the VM's Kibana checkout: + 1. evaluate_dataset.ts with the load_skill {"skill":""} SkillInvoked + matcher (PR #286165, cherry-picked into the evals-ext-matrix worktree) + 2. evals_security_persona_matrix scout config with server.maxPayload=50MB + (PR #286201) — judge /internal/inference/prompt payloads exceed the + 1.6MB default on long trajectories. + Both run from source via the dev CLI, so no build step is needed on the VM. +- Judge: defaults to EVAL_CONNECTOR_ID=eis-anthropic-claude-4-6-sonnet for ALL + models (comparability; self-judging bias exists in the docs matrix too). + Export EVAL_CONNECTOR_ID to override it for judge-panel runs; it is forwarded + to every VM. run_model.sh swaps to an alternate judge if the override would + self-judge the candidate. + +Usage: + python3 persona_matrix_sweep.py --models all # full re-sweep + python3 persona_matrix_sweep.py --models "eis-a,eis-b" + python3 persona_matrix_sweep.py --status + python3 persona_matrix_sweep.py --teardown # delete orca-sweep-* VMs +""" +import argparse +import hashlib +import inspect +import json +import sys +import os +import re +import shlex +import subprocess +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Mapping, Optional +from pathlib import Path + +SSH_KEY = os.path.expanduser("~/.ssh/azure_eval_farm") +SSH_USER = "orcaeval" +IMAGE = json.load(open(Path(__file__).parent / ".azure-state.json"))["imageId"] +RG = os.environ.get("SWEEP_RESOURCE_GROUP", "orca-eval-farm") +# Thinking-model sweeps (selfhost-qwen38 etc.) accumulate huge trace state in +# the Kibana dev server; D8s_v5 OOM-killed Kibana mid-run on all 3 shards +# (converse ECONNREFUSED after ~14 examples, 2026-09-05). Use D16s_v5 for +# selfhost models. +VM_SIZE = os.environ.get("SWEEP_VM_SIZE", "Standard_D8s_v5") +# Region and quota family for the pre-launch quota gate. The farm's cores are +# capped per family per region; D-series v5 sizes bill against the "standard +# DSv5 family" bucket, which is what filled to 344/350 on 2026-09-06. +REGION = os.environ.get("SWEEP_REGION", "eastus2") +QUOTA_FAMILY = os.environ.get("SWEEP_QUOTA_FAMILY", "standardDSv5Family") + +# --------------------------------------------------------------------------- +# Suite profiles. +# +# This sweeper began as persona-matrix-only and hardcoded that suite in ~35 +# places. AD and automatic-migrations need the same VM fan-out but differ in +# suite id, source overlays, and completeness gate, so the per-suite facts live +# here instead of being threaded through every call site. +# +# `overlays` is a list of (local_path_relative_to_worktree, remote_path) pairs. +# The persona-matrix entries are the historical PATCHED_* constants; the other +# suites deliberately start EMPTY -- the base image carries their sources, and +# inventing overlays we have not proven necessary would ship untested patches +# to 27 VMs. Add one only when a canary run shows it is needed. +# --------------------------------------------------------------------------- +SUITE_PROFILES = { + "security-persona-matrix": { + "cli_suite": "security-persona-matrix", + "gate_suite_id": "security-persona-matrix", + "vm_prefix": "orca-sweep", + # 21 prompts x 7 categories x 3 variants + "n_examples": 21, + "gate": "exact", + }, + "attack-discovery-agent-builder": { + "cli_suite": "attack-discovery-agent-builder", + "gate_suite_id": "attack-discovery-agent-builder", + "vm_prefix": "orca-ad", + # 10 datasets x 1 example each, MEASURED from a completed dense canary + # (orca-base-builder-v7, feat/ad-dense-seed-profile @ 5303a12a6, 2026-09-10, + # "10 passed (21.2m)"): 130 score docs over 10 datasets x 13 evaluators. + # The 10th is the dense-profile spec (dense_profile_live_retrieval.spec.ts), + # which seeds 95 alerts and runs live-retrieval; its final answer confirms + # ~100 alerts analysed (4 chains, 16 rules, 194K input tokens) -- + # magnitude-comparable to the reference artifact's 95. Source-counting + # src/dataset.ts gives 5 and misses the scenario-registry specs entirely. + # REQUIRES orca-eval-base-v7 (baked from ad-dense merged with this branch): + # v5 has no dense spec, so a v5 VM would emit 117 and fail this gate. + "n_examples": 10, + # Uniform grid (every dataset runs all 13 evaluators), so the + # examples x evaluators product is exact. + "gate": "exact", + # The AD suite does NOT read PERSONA_MATRIX_SHARD: every shard VM runs + # the full 9-dataset grid. Measured 2026-09-08: all 24 units exported + # exactly 117 docs under per-shard execution_ids, and the gate read + # 117/39 FAIL on complete data. With honors_shard False each unit is + # gated on the full grid -- and shards act as independent repetitions, + # which is what judge-stability analysis wants anyway. + "honors_shard": False, + }, + "security-persona-matrix-attack-discovery": { + "cli_suite": "security-persona-matrix-attack-discovery", + "gate_suite_id": "security-persona-matrix-attack-discovery", + "vm_prefix": "orca-pmad", + # attack_discovery.spec.ts: ONE example (generate API over the real + # 95-alert GCS corpus). MEASURED on the 2026-09-10 smoke + # (exec 5bd8fa4a589f4cf4, EVAL_EXIT=0): 8 docs = 8 evaluators + # (AttackDiscoveryBasic, AttackDiscoveryRubric, Criteria, Latency, + # Tool Calls, Input/Output/Cached Tokens) x 1 example x 1 rep. + # Snapshot restore races Kibana's boot-time alerting init (dual + # write-index alias) — the restore.ts overlay below carries the fix. + "n_examples": 1, + "gate": "exact", + "honors_shard": False, + }, + "security-automatic-migrations": { + "cli_suite": "security-automatic-migrations", + "gate_suite_id": "security-automatic-migrations", + "vm_prefix": "orca-mig", + # Each dataset carries a DIFFERENT evaluator count (measured on the + # 2026-09-02 canary: standard-dashboards 7, qradar 9, splunk-spl 8), + # so examples x evaluators is structurally wrong for this suite. + # Gate on a floor measured from that canary run (86 docs) instead. + "n_examples": None, + "gate": "floor", + "min_docs": 80, + }, +} + +# Selected by --suite; mutated once in main() before any VM work. +SUITE = "security-persona-matrix" + + +def suite_profile(suite: Optional[str] = None) -> dict: + name = suite or SUITE + if name not in SUITE_PROFILES: + raise KeyError(f"unknown suite {name!r}; known: {sorted(SUITE_PROFILES)}") + return SUITE_PROFILES[name] + + +# Per-model env for run_model.sh. Slow reasoning models blow the default 30-min +# cap. Measured on golden (15 GLM runs, 2026-08-11..30): mean 341s per example, +# max 1198s. 21 examples therefore need ~119 min, so the old 60-min cap could +# never finish -- GLM has never exceeded 10/21 in three weeks of attempts. +# 180 min leaves headroom above the measured worst case without hiding a hang: +# a genuinely wedged run still dies on the per-request KBN_EVALS_HTTP_TIMEOUT_MS. +MODEL_ENV = { + "eis-zai-glm-5-2": "PERSONA_MATRIX_TIMEOUT_MINUTES=180 PERSONA_MATRIX_CONCURRENCY=3", + # GLM-5.3-flash via OpenRouter is the same slow class: measured ~4.8 + # min/example on a healthy run. A 3-example shard therefore needs ~15 min + # of model work, but a shard that draws several slow examples plus retries + # blew the 30-min suite default (run 8, "Test timeout of 1800000ms + # exceeded" at attempt 2/3). 120 min per shard leaves headroom without + # hiding a wedge: per-request KBN_EVALS_HTTP_TIMEOUT_MS still bounds a hang. + "openrouter-zai-glm-5-3-flash": "PERSONA_MATRIX_TIMEOUT_MINUTES=120", + "openrouter-deepseek-v4-pro": "PERSONA_MATRIX_TIMEOUT_MINUTES=120", + # selfhost-qwen38 (A100 SGLang, 2xTP1 cells) is the slowest class in the + # sweep: 7-example shard timed out at 120 min on sweep-13 attempt 1 + # (2026-09-06). 300 min covers the observed ~4.5 min/example with + # headroom for retries. The per-request AGENT_BUILDER_INFERENCE_TIMEOUT_MS + # (600s via run_model.sh) bounds a single-turn hang. + "selfhost-qwen38": "PERSONA_MATRIX_TIMEOUT_MINUTES=300", +} +# Vars forwarded to every VM. EVAL_CONNECTOR_ID must stay here: run_model.sh only +# honours an override it actually receives, and otherwise re-derives its Anthropic +# default — a judge-panel sweep would then grade with the incumbent judge and still +# pass its doc-count gate. +FORWARDED_ENV_VARS = ("EVAL_REPETITIONS", "PERSONA_MATRIX_TIMEOUT_MINUTES", "EVAL_CONNECTOR_ID", + "KBN_EVALS_HTTP_RETRIES", "EVAL_SUITE", "PERSONA_MATRIX_SHARD", + "TEST_RUN_ID") +# Prefer the durable copy in ~/.elastic: /tmp is cleared by macOS and by +# routine cleanup, and a missing file here fails per-VM inside scp (every +# deploy dies, observed 2026-09-03) rather than once, up front. +GOLDEN_ENV_LOCAL = next( + (p for p in (os.path.expanduser("~/.elastic/golden-cluster-env.sh"), + "/tmp/golden-cluster-env.sh") if os.path.isfile(p)), + os.path.expanduser("~/.elastic/golden-cluster-env.sh"), +) +SWEEP_DIR = Path.home() / "persona-sweep" +KIBANA_MAIN = Path.home() / "Projects" / "kibana" + + +def _golden_env_local(): + """Parse the golden cluster env file once for driver-side queries.""" + env = {} + try: + with open(GOLDEN_ENV_LOCAL) as fh: + for line in fh: + line = line.strip() + if line.startswith("export "): + line = line[len("export "):] + if "=" in line and not line.startswith("#"): + k, _, v = line.partition("=") + env[k.strip()] = v.strip().strip('"').strip("'") + except OSError: + return None, None + return env.get("GOLDEN_ES_URL"), env.get("GOLDEN_ES_API_KEY") + + +def _golden_post(path: str, body: str): + """POST a query to golden from the driver. None when unreachable.""" + url, key = _golden_env_local() + if not url or not key: + return None + try: + return json.loads(subprocess.run( + ["curl", "-sS", "-m", "60", "-H", f"Authorization: ApiKey {key}", + f"{url}/.evaluation-scores/{path}", + "-H", "Content-Type: application/json", "-d", body], + capture_output=True, text=True, timeout=90, + ).stdout) + except Exception: + return None + + +def _golden_datasets_local(exec_id: str): + """Dataset ids written under an execution id, queried FROM THE DRIVER. + + Same rationale as _golden_count_local: units are parked before the gate + runs, so this must not go over ssh. Returns None when golden is + unreachable so the caller can distinguish "cannot verify" from "wrong". + """ + res = _golden_post("_search", json.dumps({ + "size": 0, + "query": {"term": {"metadata.execution_id": exec_id}}, + "aggs": {"ds": {"terms": {"field": "example.dataset.id", "size": 50}}}, + })) + if res is None: + return None + try: + return {b["key"] for b in res["aggregations"]["ds"]["buckets"]} + except Exception: + return None + + +def _golden_count_local(exec_id: str, phrase: bool = False): + """Count docs for an execution id by querying golden FROM THE DRIVER. + + The gate used to run this over ssh on the eval VM, but units are parked + ("[park] ... deallocating") as soon as the eval exits and before the gate + runs, so the ssh lands on a deallocated host and returns nothing. That is + indistinguishable from "no docs written" and failed 18/18 complete units as + `docs=0/98` while golden actually held all 294 docs per model. + + Golden is reachable from the driver, so ask it directly. Returns None when + the driver cannot reach golden, letting the caller fall back to the VM. + """ + url, key = _golden_env_local() + if not url or not key: + return None + op = "match_phrase" if phrase else "term" + body = json.dumps({"query": {op: {"metadata.execution_id": exec_id}}}) + try: + out = subprocess.run( + ["curl", "-sS", "-m", "60", "-H", f"Authorization: ApiKey {key}", + f"{url}/.evaluation-scores/_count", + "-H", "Content-Type: application/json", "-d", body], + capture_output=True, text=True, timeout=90, + ).stdout + return int(json.loads(out)["count"]) + except Exception: + return None + + +# Patched sources overlaid onto each VM before the run. +PATCHED_EVALUATOR = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.ts" +) +PATCHED_EVALUATOR_REMOTE = ( + "Projects/kibana/x-pack/solutions/security/packages/" + "kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.ts" +) +# Per-example failure isolation (PR #285833): without it one example's +# converse/judge 500 rejects Promise.all(runJobs) and aborts the whole +# experiment — three determinism runs died this way mid-suite. Overlay the +# patched executor client (+ its TaskRun.error type) so errored examples are +# recorded and the remaining measurements survive. +PATCHED_EXECUTOR_CLIENT = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "x-pack/platform/packages/shared/kbn-evals/src/kibana_evals_executor/client.ts" +) +PATCHED_EXECUTOR_TYPES = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "x-pack/platform/packages/shared/kbn-evals/src/types.ts" +) +EXECUTOR_CLIENT_REMOTE = ( + "Projects/kibana/x-pack/platform/packages/shared/kbn-evals/src/" + "kibana_evals_executor/client.ts" +) +EXECUTOR_TYPES_REMOTE = "Projects/kibana/x-pack/platform/packages/shared/kbn-evals/src/types.ts" +# Transport retries. The base image predates the fix, so without this overlay a +# dropped connection still ends the whole suite: glm-5-2 lost 19 of 21 examples +# twice this way, the second time on a re-run that was supposed to carry the fix. +PATCHED_HTTP_HANDLER = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "x-pack/platform/packages/shared/kbn-evals/src/utils/http_handler_from_kbn_client.ts" +) +HTTP_HANDLER_REMOTE = ( + "Projects/kibana/x-pack/platform/packages/shared/kbn-evals/src/utils/" + "http_handler_from_kbn_client.ts" +) +# Retry policy. The persona-matrix converse call goes through chat_client's +# withRetry (retry_utils), NOT the http handler above -- patching only the +# handler leaves the live path untouched and the run still dies on the first +# EIS 500. Deploy both or the fix is a no-op on the VM. +# chat_client itself must also ship: main's version has no withRetry wrapper +# (kbn-client retries=0 -> 'attempt=1/0' instant death) and no final-answer +# fallback for terse models. Without it the run is one transient away from +# a deterministic 14/42 shard failure. +PATCHED_CHAT_CLIENT = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/chat_client.ts" +) +CHAT_CLIENT_REMOTE = ( + "Projects/kibana/x-pack/solutions/security/packages/" + "kbn-evals-suite-security-persona-matrix/src/chat_client.ts" +) +PATCHED_RETRY_UTILS = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "x-pack/platform/packages/shared/kbn-evals/src/utils/retry_utils.ts" +) +# Trace-based evaluators. The VM checkout is main and predates the ES|QL index +# pattern fix, so its skill_invocation.ts interpolates an undefined constant and +# every SkillInvoked query hits `on indices [undefined]` -> security_exception. +# factory.ts owns TRACE_INDEX_PATTERN; skill_invocation.ts consumes it. Ship both +# or the consumer resolves the constant from the stale module. +PATCHED_TRACE_FACTORY = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/factory.ts" +) +TRACE_FACTORY_REMOTE = ( + "Projects/kibana/x-pack/platform/packages/shared/" + "kbn-evals/src/evaluators/trace_based/factory.ts" +) +PATCHED_SKILL_INVOCATION = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/skill_invocation.ts" +) +SKILL_INVOCATION_REMOTE = ( + "Projects/kibana/x-pack/platform/packages/shared/" + "kbn-evals/src/evaluators/trace_based/skill_invocation.ts" +) +# Every remaining trace_based evaluator. These build their own ES|QL and the +# VM's main checkout still hardcodes `FROM traces-*`, which resolves to zero +# authorized indices on the golden cluster and fails with +# `Unknown column [trace.id]` -- 51 errored docs per metric per model, read as +# "no tool calls" rather than a broken instrument. Ship the whole directory +# rather than naming files one at a time, so a new evaluator cannot be missed. +_TRACE_DIR_LOCAL = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based" +) +_TRACE_DIR_REMOTE = ( + "Projects/kibana/x-pack/platform/packages/shared/" + "kbn-evals/src/evaluators/trace_based" +) +TRACE_METRIC_FILES = ("latency.ts", "tokens.ts", "tool_calls.ts", "chat_calls.ts") +# The @kbn/evals barrel. evaluate_dataset.ts imports TRACE_INDEX_PATTERN from the +# package root, not from the module that defines it, so shipping factory.ts alone +# is not enough: the stale barrel has no such export and the import silently +# resolves to undefined at runtime (TS types come from the local checkout, so +# nothing fails until ES rejects `FROM undefined`). +PATCHED_EVALS_BARREL = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "x-pack/platform/packages/shared/kbn-evals/index.ts" +) +EVALS_BARREL_REMOTE = "Projects/kibana/x-pack/platform/packages/shared/kbn-evals/index.ts" +# The trace_based sub-barrel that the package root re-exports through. +PATCHED_TRACE_BARREL = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/index.ts" +) +TRACE_BARREL_REMOTE = ( + "Projects/kibana/x-pack/platform/packages/shared/" + "kbn-evals/src/evaluators/trace_based/index.ts" +) +RETRY_UTILS_REMOTE = ( + "Projects/kibana/x-pack/platform/packages/shared/kbn-evals/src/utils/retry_utils.ts" +) +# Dataset. The base image predates the entity_risk_score contract fix, so its +# pre-flight tool-availability check fails the whole suite before a single +# example runs (security.entity_risk_score is force-disabled under the skills +# flag this suite always enables). +PATCHED_DATASET = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/" + "src/datasets/persona_matrix_prompts.ts" +) +DATASET_REMOTE = ( + "Projects/kibana/x-pack/solutions/security/packages/" + "kbn-evals-suite-security-persona-matrix/src/datasets/persona_matrix_prompts.ts" +) +# evaluate_dataset.ts imports ./datasets/select_shard. The overlay copies an +# explicit file list, so a new module must be added here or the VM runs old +# code and dies at require time ("Cannot find module './datasets/select_shard'" +# -> "No tests found" -> 3 failed attempts, observed 2026-09-03). +PATCHED_SHARD = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/" + "src/datasets/select_shard.ts" +) +SHARD_REMOTE = ( + "Projects/kibana/x-pack/solutions/security/packages/" + "kbn-evals-suite-security-persona-matrix/src/datasets/select_shard.ts" +) +# AD golden-path replay join key (PR #285833): the frozen orca-eval-base +# image predates the fix that gives every golden-path scenario a +# `metadata.scenarioKey`. Without it the five golden-path slices record +# `example.id == "0"` and no join key, so their scores can never be +# replayed/rejudged -- exactly the defect this run exists to clear. The VM +# has no git checkout (it boots a baked image), so the fix can only reach it +# as an overlay. +PATCHED_AD_DATASET = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/" + "src/dataset.ts" +) +AD_DATASET_REMOTE = ( + "Projects/kibana/x-pack/solutions/security/packages/" + "kbn-evals-suite-attack-discovery-agent-builder/src/dataset.ts" +) +PATCHED_SCOUT_CONFIG = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/" + "evals_security_persona_matrix/stateful/classic.stateful.config.ts" +) +# Detection-rule-edit skill (PR #285833): adds the final-answer contract to +# the skill checklist — 62% of detection-rule-edit runs in the 2026-08-21 +# sweep ended on a tool call with no user-facing closing message. The base +# image predates the fix, so overlay it like the other patched sources. +PATCHED_RULE_SKILL = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "x-pack/solutions/security/plugins/security_solution/server/agent_builder/" + "skills/detection_rule_edit/index.ts" +) +PATCHED_RULE_SKILL_REMOTE = ( + "Projects/kibana/x-pack/solutions/security/plugins/security_solution/server/" + "agent_builder/skills/detection_rule_edit/index.ts" +) +# Scout readiness: PR #285302 makes SCOUT_READY_TIMEOUT_MS configurable; the +# orca-eval-base-v5 image predates it, so cold-boot rspack compile (303 +# bundles) exceeds the hardcoded 180s and every VM fails before the eval +# starts. Overlay the patched eval_stack.ts so run_model.sh's 900s timeout +# actually applies. Sourced from THIS worktree (not scout-timeout-pr): it +# carries the timeout fix AND the agentBuilderTracingExporters pass-through +# (commit 13ad293e8f8a) — overlaying the older scout-timeout-pr copy silently +# dropped the golden trace exporter arg from the boot command (observed +# 2026-09-04: persona2 ran with no xpack.agentBuilder.tracing.exporters and +# golden received 0 spans despite the config/key being shipped). +PATCHED_EVAL_STACK = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "x-pack/platform/packages/shared/kbn-evals/src/cli/eval_stack.ts" +) +# Playwright per-test timeout: the suite default is 30min, sized for a +# single-pass run. EVAL_REPETITIONS=3 runs die at the default (observed at +# example 7/21 after 30min); the overlaid config reads +# PERSONA_MATRIX_TIMEOUT_MINUTES so determinism runs can raise it. +PATCHED_PW_CONFIG = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/playwright.config.ts" +) +# Trace-evaluator retry budget (commit 1f4d2e2596dc): the 62s budget burned +# inside the OTel flush lag (~3-7 min), erroring Tool Calls/Tokens/Latency/ +# SkillInvoked on every VM run. Overlaid like the other patched kbn-evals +# sources — the base image predates the fix. +PATCHED_TRACE_FACTORY = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/factory.ts" +) +TRACE_FACTORY_REMOTE = ( + "Projects/kibana/x-pack/platform/packages/shared/kbn-evals/src/" + "evaluators/trace_based/factory.ts" +) +# Golden trace export (regressed 2026-09-01): the base image's kbn-evals/scout +# sources predate agentBuilderTracingExporters support entirely (0 occurrences +# in profiles.ts / eval_stack.ts / classic.stateful.config.ts), so +# config.local.json's key is written but never read and Agent Builder spans +# only land on the VM's local Scout ES. Overlay all three files from this +# worktree so the golden OTLP exporter is appended during stack boot. +PATCHED_PROFILES = ( + KIBANA_MAIN.parent / "kibana.worktrees/evals-ext-matrix" + / "x-pack/platform/packages/shared/kbn-evals/src/cli/profiles.ts" +) +PROFILES_REMOTE = ( + "Projects/kibana/x-pack/platform/packages/shared/kbn-evals/src/cli/profiles.ts" +) +PATCHED_SCOUT_TRACING_CONFIG = ( + KIBANA_MAIN.parent / "kibana.worktrees/evals-ext-matrix" + / "src/platform/packages/shared/kbn-scout/src/servers/configs/" + "config_sets/evals_tracing/stateful/classic.stateful.config.ts" +) +SCOUT_TRACING_CONFIG_REMOTE = ( + "Projects/kibana/src/platform/packages/shared/kbn-scout/src/servers/" + "configs/config_sets/evals_tracing/stateful/classic.stateful.config.ts" +) +# Env seeds/tools seed/spec live in the matrix branch itself (merged as +# f85527ed "Unbreak failing columns", plus the tool-registration assert). +# Overlay from this worktree — the persona-matrix-env-truth worktree predates +# the assert and would silently drop it on the VM. +PATCHED_ENV_SEEDS = ( + KIBANA_MAIN.parent / "kibana.worktrees/evals-ext-matrix" + / "x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/fixtures/env_seeds.ts" +) +PATCHED_TOOLS_SEED = ( + KIBANA_MAIN.parent / "kibana.worktrees/evals-ext-matrix" + / "x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/fixtures/persona_matrix_tools_seed.ts" +) +PATCHED_SPEC = ( + KIBANA_MAIN.parent / "kibana.worktrees/evals-ext-matrix" + / "x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/evals/persona_matrix.spec.ts" +) +# The spec imports the tool-registration assert added in f85527ed; the VM +# checkout predates it, so the module must ride along or the spec fails at +# require time ("Cannot find module '../src/fixtures/tool_registration_check'"). +PATCHED_TOOL_CHECK = ( + KIBANA_MAIN.parent / "kibana.worktrees/evals-ext-matrix" + / "x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/fixtures/tool_registration_check.ts" +) +FIXTURES_REMOTE_PREFIX = ( + "Projects/kibana/x-pack/solutions/security/packages/" + "kbn-evals-suite-security-persona-matrix" +) +PATCHED_SCOUT_CONFIG_REMOTE = ( + "Projects/kibana/src/platform/packages/shared/kbn-scout/src/servers/configs/" + "config_sets/evals_security_persona_matrix/stateful/classic.stateful.config.ts" +) + +# Full sweep list: 15 re-runs (need correct SkillInvoked from the patched +# evaluator) + 5 frontier additions not in the published docs matrix. +# Skipped (deterministic, reproduced failures — re-running burns tokens for +# no new information): +# eis-anthropic-claude-4-5-sonnet — model emits load_skill({}) under full +# agent context (3/3 repros; 4.6 passes) +# eis-google-gemini-2-5-flash-lite — "platform_core_load_skill called but +# was not available" (2/2 repros) +MODELS = [ + # re-run with patched evaluator + "eis-anthropic-claude-4-5-haiku", + "eis-anthropic-claude-4-5-opus", + "eis-anthropic-claude-4-6-sonnet", + "eis-anthropic-claude-4-6-opus", + "eis-anthropic-claude-4-7-opus", + "eis-openai-gpt-5-2", + "eis-openai-gpt-5-4", + "eis-openai-gpt-5-4-mini", + "eis-openai-gpt-5-4-nano", + "eis-google-gemini-2-5-flash", + "eis-google-gemini-2-5-pro", + "eis-google-gemini-3-0-flash", + "eis-google-gemini-3-1-flash-lite", + "eis-google-gemini-3-1-pro", + "eis-google-gemini-3-5-flash", + # frontier additions (not in the published docs matrix) + "eis-anthropic-claude-4-8-opus", + "eis-anthropic-claude-5-sonnet", + "eis-openai-gpt-5-5", + "eis-zai-glm-5-2", + # OpenRouter provider path: needs the on-VM SSE-normalizing proxy + + # ES endpoint (run_model.sh openrouter- branch). GLM-5.3-flash is the + # OSS-row candidate; 87.8 calls/example measured 2026-09-03 — pair it + # with --shards 4, never single-stack. + "openrouter-zai-glm-5-3-flash", + # DeepSeek V4 Pro: OSS candidate #2 on the OpenRouter path. Canary + # 2026-09-03: clean tool call, 1.7s round-trip (10x faster than GLM-5.3). + # Same proxy + endpoint flow; --shards 4 to start. + "openrouter-deepseek-v4-pro", + # Self-hosted Qwen3.8-27B on the A100 (SGLang, 2×TP1 cells). Rides the + # same on-VM proxy path with PROXY_UPSTREAM=public cell URL + bearer + # from /tmp/selfhost.env (shipped by deploy()). Quality scorecard for + # qwen38-local in OmniRoute combos; judge stays on EIS (different family). + "selfhost-qwen38", + # NOTE: gemini-3.7-flash exists only as an OpenRouter connector and needs + # the proxy + ES JAR reasoning patch flow (kibana-evals skill + # scripts/openrouter-proxy.py). It is NOT in the default sweep; run it as + # a targeted follow-up. +] + + +# AD snapshot-restore fix: Kibana's boot-time alerting init recreates the +# Security alert index mid-restore, surfacing as either "open index ... already +# exists" or "alias [...] has more than one write index". The base image's +# restore.ts only retries the former. Sourced from THIS worktree (identical to +# fix/alerts-snapshot-restore-dual-write-retry 166f1a19aa28, off upstream/main). +# Without it the AD fan-out deterministically fails restore 3/3 attempts on a +# fresh-wipe VM (observed 2026-09-10 smoke v2); with it smoke v3 passed +# (EVAL_EXIT=0, exec 5bd8fa4a589f4cf4, 8/8 evaluators on golden). +PATCHED_RESTORE_TS = ( + KIBANA_MAIN.parent + / "kibana.worktrees/evals-ext-matrix" + / "x-pack/solutions/security/packages/kbn-security-evals-alerts-snapshot/src/restore.ts" +) +RESTORE_TS_REMOTE = ( + "Projects/kibana/x-pack/solutions/security/packages/" + "kbn-security-evals-alerts-snapshot/src/restore.ts" +) +# GCS service-account JSON for the alerts snapshot +# (security-ai-datasets/attack-discovery/oh-my-malware-95-deduped). The scout +# evals_tracing config only registers the ES gcs client when GCS_CREDENTIALS +# is set — without it the AD spec skips restore and runs against a stale +# corpus. Durable copy in ~/.elastic; /tmp is wiped by macOS reboots. +GCS_CREDENTIALS_LOCAL = next( + (p for p in (os.path.expanduser("~/.elastic/gcs-credentials.json"), + "/tmp/gcs_credentials.json") if os.path.isfile(p)), + os.path.expanduser("~/.elastic/gcs-credentials.json"), +) + +def is_sweep_resource(name: str) -> bool: + """True when an Azure resource belongs to any suite's sweep VMs. + + Disk/NIC/public-IP/NSG names are all derived from the VM name, so every + teardown pass must test the SAME set of prefixes. Hardcoding one prefix per + pass is how a teardown reports success while another suite's disks keep + billing. + """ + return any(name.startswith(pf["vm_prefix"] + "-") for pf in SUITE_PROFILES.values()) + + +def model_dir(model: str, suite: Optional[str] = None, shard: Optional[str] = None) -> Path: + """Per-suite, per-model (per-shard) run directory. + + Namespaced by suite: the same model is swept for persona-matrix, AD and + migrations, and a flat layout would let the second sweep overwrite the + first one's run.log and status.json. + + Sharded runs get their own leaf for the same reason -- shard 2 would + otherwise clobber shard 1's log and status. The "/" in a shard spec is + replaced, not kept: a raw "2/4" would nest a directory (or escape the + model dir) instead of naming one. + """ + leaf = model if not shard else f"{model}-s{shard.replace('/', 'of')}" + return SWEEP_DIR / (suite or SUITE) / leaf + + +def ssh(ip: str, cmd: str, timeout: int = 30) -> str: + r = subprocess.run( + ["ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-o", f"ConnectTimeout={timeout}", "-i", SSH_KEY, f"{SSH_USER}@{ip}", cmd], + capture_output=True, text=True, timeout=timeout + 15) + return r.stdout.strip() + (("\n" + r.stderr.strip()) if r.stderr.strip() else "") + + +# ssh() returns 255 with empty output when the transport itself fails (host not +# accepting keys yet, connection reset mid-deploy). A caller that greps the +# result then cannot tell "the check ran and failed" from "the check never +# ran", and reports a content failure for a transport problem -- which is how +# a 2026-09-09 AD sweep skipped 6/6 units whose overlays were in fact correct +# (verified by hand on the same VMs minutes later). +def ssh_checked(ip: str, cmd: str, timeout: int = 30, attempts: int = 3) -> str: + last = "" + for attempt in range(attempts): + r = subprocess.run( + ["ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-o", f"ConnectTimeout={timeout}", "-i", SSH_KEY, f"{SSH_USER}@{ip}", cmd], + capture_output=True, text=True, timeout=timeout + 15) + out = r.stdout.strip() + (("\n" + r.stderr.strip()) if r.stderr.strip() else "") + # 255 is ssh's own transport failure; any other code is the remote + # command's verdict and must be reported as-is. + if r.returncode != 255: + return out + last = out + time.sleep(5 * (attempt + 1)) + raise RuntimeError( + f"ssh transport to {ip} failed {attempts}x (last: {last!r}); " + "refusing to report this as a content check result" + ) + + +def az(*args: str) -> str: + """Run az with retry for transient 'content already consumed' errors.""" + r = subprocess.CompletedProcess([], 1, "", "") + for attempt in range(3): + r = subprocess.run(["az", *args], capture_output=True, text=True, timeout=300) + if r.returncode == 0: + return r.stdout.strip() + if "already consumed" in r.stderr and attempt < 2: + time.sleep(5) + continue + raise RuntimeError(f"az {' '.join(args)} failed: {r.stderr[:400]}") + return r.stdout.strip() + + +def model_slug(model: str) -> str: + """The human-readable model slug (used for Azure tags, never the VM name).""" + return model.replace("eis-", "").replace(".", "-").replace("_", "-") + + +def vm_name(model: str, shard: Optional[str] = None) -> str: + # Azure Linux VM names allow 64 chars. Do NOT truncate harder than that — + # a [:24] truncation collided gemini-2-5-flash-lite onto gemini-2-5-flash's + # VM (dirty ES → 409 dataset conflict). + prefix = suite_profile()["vm_prefix"] + # VM_NAME_SUFFIX isolates parallel sweeps of the same model (e.g. judge + # A/B stability reruns). Without it two sweeps resolve the same VM names + # and stomp each other's stacks. + suffix_env = os.environ.get("VM_NAME_SUFFIX", "") + if suffix_env: + prefix = f"{prefix}-{suffix_env}"[:40] + # JUDGE BLINDING: the VM name must NOT contain the model under test. + # The sweep VM enrols into the eval cluster as a host entity, so a + # model-derived hostname (orca-sweep-anthropic-claude-4-8-opus) was echoed + # back inside agent answers on entity-analytics columns and read by the + # LLM judges — the model identifying itself to its own grader. Hash instead; + # the readable model lives in the `model` Azure tag (see provision()). + digest = hashlib.sha1(model_slug(model).encode()).hexdigest()[:12] + slug = f"m{digest}" + # Shard suffix keeps each slice on its own box. Two eval stacks on one VM + # OOM each other and corrupt local ES, so the suffix is load-bearing. + # Truncate the MODEL slug, never the suffix: appending first and clamping + # to 64 silently merges two shards onto one name (verified: a 64-char model + # made s1of4 and s2of4 identical). + if shard: + suffix = f"-s{shard.replace('/', 'of')}" + budget = 64 - len(prefix) - 1 - len(suffix) + slug = f"{slug[:budget].rstrip('-')}{suffix}" + return f"{prefix}-{slug}"[:64].rstrip("-") + + +def use_spot(environ: Optional[Mapping[str, str]] = None) -> bool: + """Whether to attempt a Spot create. Default OFF. + + Spot capacity for D8s_v5 in the farm region has been exhausted since + 2026-09-04; on 2026-09-06, 30 of 60 create attempts paid a failed Spot + call plus retry backoff before falling back to Regular. Opt back in with + SPOT_VM=1 once capacity returns. + """ + env = os.environ if environ is None else environ + return env.get("SPOT_VM", "0") == "1" + + +def park_on_done(environ: Optional[Mapping[str, str]] = None) -> bool: + """Whether to deallocate a unit's VM once its golden gate is settled. + + Default ON: idle finished VMs held the cores that starved later units + twice on 2026-09-06. PARK_ON_DONE=0 keeps VMs running for debugging. + """ + env = os.environ if environ is None else environ + return env.get("PARK_ON_DONE", "1") == "1" + + +def maybe_park_unit(model: str, shard: Optional[str] = None) -> bool: + """Deallocate a finished unit's VM, freeing its cores. Returns True if parked. + + Deliberately bundles the PARK_ON_DONE check with the az call so the flag + cannot drift away from the action it guards. Never raises: a failed park + costs money, but must not fail an otherwise-good sweep. + """ + if not park_on_done(): + return False + try: + az("vm", "deallocate", "-g", RG, "-n", vm_name(model, shard), "--no-wait") + print(f"[park] {_label(model, shard)}: deallocating (cores freed)", flush=True) + return True + except Exception as exc: + print(f"[park] {_label(model, shard)}: deallocate failed ({exc})", flush=True) + return False + + +def cores_per_vm() -> int: + """vCPU count for VM_SIZE, parsed from the size name (D8s_v5 -> 8).""" + m = re.search(r"_[A-Z]+(\d+)", VM_SIZE) + return int(m.group(1)) if m else 8 + + +def quota_snapshot() -> tuple[int, int]: + """Return (used, limit) regional vCPU cores for the VM family. + + Reads the live usage rather than trusting a VM count: deallocated VMs + still exist but consume no cores, so counting `vm list` over-reports. + """ + raw = json.loads(az("vm", "list-usage", "-l", REGION, "-o", "json")) + family = QUOTA_FAMILY + for entry in raw: + if entry.get("name", {}).get("value") == family: + return int(entry["currentValue"]), int(entry["limit"]) + raise RuntimeError(f"quota family {family} not found in {REGION}") + + +def quota_gate(units: int, used: int, limit: int, per_vm: int) -> tuple[bool, str]: + """Decide whether `units` VMs fit in the remaining regional quota. + + Pure function so the sweep's failure mode is testable without Azure. + Returns (ok, message). A sweep that does not fit must fail BEFORE + provisioning: on 2026-09-06 a launch into 344/350 used cores burned + ~10h, because `az vm create` reported QuotaExceeded as an unrelated + CLI crash ("content for this response was already consumed") and the + controller treated each failure as a per-VM skip rather than a stop. + """ + need = units * per_vm + free = limit - used + if need <= free: + return True, ( + f"quota ok: need {need} cores ({units}x{per_vm}), " + f"free {free} of {limit}" + ) + fits = free // per_vm if per_vm else 0 + return False, ( + f"QUOTA GATE: need {need} cores ({units}x{per_vm}) but only {free} " + f"free of {limit} ({used} in use). At most {fits} unit(s) fit now. " + f"Free cores first (--teardown-finished, or delete idle VMs), then " + f"relaunch; do not launch into a full quota." + ) + + +def reusable_pool_vm(model: str, shard: Optional[str] = None) -> Optional[str]: + """Return the name of an existing deallocated VM for this unit, if any. + + Warm-pool reuse: `az vm start` on a pre-baked deallocated VM is ~60-90s + versus ~6-8min for create + cloud-init. Deallocated VMs hold no vCPU + quota, so a parked pool costs disk only. WARM_POOL=0 disables. + """ + if os.environ.get("WARM_POOL", "1") != "1": + return None + name = vm_name(model, shard) + try: + state = az("vm", "show", "-g", RG, "-n", name, "-d", + "--query", "powerState", "-o", "tsv").strip() + except Exception: + return None + return name if state == "VM deallocated" else None + + +def provision(model: str, shard: Optional[str] = None) -> str: + """Create a D8s_v5 VM; return its public IP. + + Reuses a parked (deallocated) VM of the same name when one exists, + which skips image provisioning entirely. + + Priority: Regular by default. Spot capacity for D8s_v5 in the farm + region has been exhausted since 2026-09-04 (surfacing as an az-cli + crash that swallowed the error, costing 16/42 shards), and on + 2026-09-06 30 of 60 create attempts still paid a failed Spot call + before falling back. SPOT_VM=1 opts back in; Spot failures continue + to auto-fall back to Regular. + """ + name = vm_name(model, shard) + parked = reusable_pool_vm(model, shard) + if parked: + print(f"[provision] starting parked VM {parked}", flush=True) + az("vm", "start", "-g", RG, "-n", parked) + ip = json.loads(az("vm", "show", "-g", RG, "-n", parked, "-d", + "--query", "publicIps", "-o", "json")) + if ip: + return ip + print(f"[provision] parked VM {parked} exposed no IP; creating fresh", + flush=True) + print(f"[provision] {name}", flush=True) + spot = use_spot() + + def _create(priority: str) -> None: + args = ["vm", "create", "-g", RG, "-n", name, "--image", IMAGE, + "--size", VM_SIZE, "--admin-username", SSH_USER, "--ssh-key-values", + os.path.expanduser("~/.ssh/azure_eval_farm.pub"), + # The VM name is a blinding hash (see vm_name); keep the readable + # model on a tag so teardown/debugging can still identify the box + # without leaking the identity into cluster host entities. + "--tags", f"model={model_slug(model)}", + "--public-ip-sku", "Standard", "--os-disk-size-gb", "256", "--no-wait"] + if priority == "Spot": + args += ["--eviction-policy", "Deallocate", "--priority", "Spot"] + az(*args) + + if spot: + try: + _create("Spot") + except (SystemExit, RuntimeError) as e: + print(f"[provision] spot create failed ({e}); retrying as Regular", flush=True) + _create("Regular") + else: + _create("Regular") + for _ in range(60): + time.sleep(10) + try: + ip = json.loads(az("vm", "show", "-g", RG, "-n", name, "-d", + "--query", "publicIps", "-o", "json")) + if ip: + return ip + except Exception: + pass + raise RuntimeError(f"no IP for {name}") + + +def wait_ssh(ip: str) -> bool: + # A single successful `echo ok` is not readiness: cloud-init restarts sshd + # after first accepting connections, so the very next scp/ssh can be reset + # mid-deploy. Require consecutive successes so the deploy that follows runs + # against a stably reachable host. + streak = 0 + for _ in range(30): + try: + if "ok" in ssh(ip, "echo ok", timeout=10): + streak += 1 + if streak >= 2: + return True + time.sleep(3) + continue + except Exception: + pass + streak = 0 + time.sleep(10) + return False + + +def scp(local: str, ip: str, remote: str) -> None: + subprocess.run( + ["scp", "-q", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-i", SSH_KEY, local, f"{SSH_USER}@{ip}:{remote}"], + check=True, timeout=120) + + +def deploy(ip: str) -> None: + """Copy run assets + patched evaluator/scout config to the VM.""" + base = Path(__file__).parent + ssh(ip, "mkdir -p ~/.elastic ~/persona-sweep") + scp(GOLDEN_ENV_LOCAL, ip, "/tmp/golden-cluster-env.sh") + scp(str(base / "run_model.sh"), ip, "/tmp/run_model.sh") + scp(str(base / "export_scores.py"), ip, "/tmp/export_scores.py") + # OpenRouter path (models named openrouter-*): on-VM SSE-normalizing proxy + # + ES endpoint creator. Harmless for EIS models — run_model.sh only uses + # them inside its openrouter- branch. + scp(str(base / "openrouter_proxy.py"), ip, "/tmp/openrouter_proxy.py") + scp(str(base / "create_openrouter_endpoint.py"), ip, "/tmp/create_openrouter_endpoint.py") + # Self-hosted model env (selfhost-* models): upstream URL + bearer for the + # on-VM proxy. Values come from local secrets; ship only when present so + # EIS/OpenRouter-only runs need nothing extra. + selfhost_env = base / ".selfhost.env" + if selfhost_env.exists(): + scp(str(selfhost_env), ip, "/tmp/selfhost.env") + # Omniroute judge route (.selfhost-judge.env): when EVAL_CONNECTOR_ID is a + # selfhost-* judge (e.g. selfhost-omni-opus-5), run_model.sh synthesizes + # its connector from these values (public omniroute URL + key). Ship only + # when present so EIS-judge runs need nothing extra. + judge_env = base / ".selfhost-judge.env" + if judge_env.exists(): + scp(str(judge_env), ip, "/tmp/judge.env") + # Matrix config carries the OpenRouter API model ids (matchIds) that the + # connectors cache does NOT. The VM's Kibana checkout is main, not this + # branch — it lacks the persona-matrix suite entirely, so ship the config. + matrix_cfg = base.parent.parent / ( + "x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/" + "persona_matrix.config.json") + scp(str(matrix_cfg), ip, "/tmp/persona_matrix.config.json") + # AD suite: GCS dataset credentials + the dual-write restore fix. Both are + # no-ops for persona-matrix runs (restore.ts is only imported by suites + # that restore snapshots; GCS creds are only read by the AD spec). + if SUITE == "security-persona-matrix-attack-discovery": + scp(GCS_CREDENTIALS_LOCAL, ip, "/tmp/gcs_credentials.json") + scp(str(PATCHED_RESTORE_TS), ip, RESTORE_TS_REMOTE) + scp(os.path.expanduser("~/.elastic/eis-connectors-cache.json"), ip, + ".elastic/eis-connectors-cache.json") + scp(os.path.expanduser("~/.elastic/eis-ccm-key.json"), ip, + ".elastic/eis-ccm-key.json") + # Overlay the patched SkillInvoked evaluator (PR #286165) and the 50MB + # maxPayload scout config (PR #286201). The eval runs from TS source via + # the dev CLI, so copying the files is sufficient — no build step. + persona_only = SUITE == "security-persona-matrix" + if persona_only: + scp(str(PATCHED_EVALUATOR), ip, PATCHED_EVALUATOR_REMOTE) + scp(str(PATCHED_SCOUT_CONFIG), ip, PATCHED_SCOUT_CONFIG_REMOTE) + # Per-example failure isolation — see PATCHED_EXECUTOR_CLIENT. + scp(str(PATCHED_EXECUTOR_CLIENT), ip, EXECUTOR_CLIENT_REMOTE) + scp(str(PATCHED_EXECUTOR_TYPES), ip, EXECUTOR_TYPES_REMOTE) + scp(str(PATCHED_HTTP_HANDLER), ip, HTTP_HANDLER_REMOTE) + scp(str(PATCHED_RETRY_UTILS), ip, RETRY_UTILS_REMOTE) + scp(str(PATCHED_TRACE_FACTORY), ip, TRACE_FACTORY_REMOTE) + scp(str(PATCHED_SKILL_INVOCATION), ip, SKILL_INVOCATION_REMOTE) + for _f in TRACE_METRIC_FILES: + scp(str(_TRACE_DIR_LOCAL / _f), ip, f"{_TRACE_DIR_REMOTE}/{_f}") + scp(str(PATCHED_TRACE_BARREL), ip, TRACE_BARREL_REMOTE) + scp(str(PATCHED_EVALS_BARREL), ip, EVALS_BARREL_REMOTE) + scp(str(PATCHED_CHAT_CLIENT), ip, CHAT_CLIENT_REMOTE) + _gate = ssh( + ip, + f"grep -c withRetry ~/{CHAT_CLIENT_REMOTE}; " + f"grep -c messageSource ~/{CHAT_CLIENT_REMOTE}", + ) + try: + _counts = [int(x) for x in _gate.split()] + except ValueError: + _counts = [0, 0] + if len(_counts) != 2 or _counts[0] < 1 or _counts[1] < 1: + raise RuntimeError( + f"chat_client overlay did not land on {ip} (withRetry/messageSource " + f"missing): {_gate!r} -- VM would run main's retries=0 converse path" + ) + scp(str(PATCHED_TRACE_FACTORY), ip, TRACE_FACTORY_REMOTE) + # Inference-endpoint timeout overlay: the executor's hard-coded 180s + # requestTimeout kills thinking-model (selfhost-qwen38) turns; the + # overlaid executor honors AGENT_BUILDER_INFERENCE_TIMEOUT_MS (600000 + # via run_model.sh). + INFERENCE_EXECUTOR_REMOTE = ( + "Projects/kibana/x-pack/platform/plugins/shared/inference/server/" + "chat_complete/utils/inference_endpoint_executor.ts" + ) + INFERENCE_EXECUTOR_LOCAL = ( + KIBANA_MAIN.parent / "kibana.worktrees/evals-ext-matrix" + / "x-pack/platform/plugins/shared/inference/server/chat_complete/" + "utils/inference_endpoint_executor.ts" + ) + scp(str(INFERENCE_EXECUTOR_LOCAL), ip, INFERENCE_EXECUTOR_REMOTE) + scp(str(PATCHED_PROFILES), ip, PROFILES_REMOTE) + scp(str(PATCHED_SCOUT_TRACING_CONFIG), ip, SCOUT_TRACING_CONFIG_REMOTE) + if persona_only: + # Every import evaluate_dataset.ts pulls from ./datasets must exist + # locally before we ship it. A missing file used to surface only on + # the VM as "No tests found" after three full stack boots. + for _src in (PATCHED_DATASET, PATCHED_SHARD): + if not Path(_src).is_file(): + raise FileNotFoundError(f"overlay source missing: {_src}") + scp(str(PATCHED_DATASET), ip, DATASET_REMOTE) + scp(str(PATCHED_SHARD), ip, SHARD_REMOTE) + if SUITE == "attack-discovery-agent-builder": + # See PATCHED_AD_DATASET: without this the run produces another + # generation of unreplayable golden-path scores. + if not Path(PATCHED_AD_DATASET).is_file(): + raise FileNotFoundError(f"overlay source missing: {PATCHED_AD_DATASET}") + if "scenarioKey" not in Path(PATCHED_AD_DATASET).read_text(): + raise RuntimeError( + f"{PATCHED_AD_DATASET} has no scenarioKey; overlaying it would " + "produce unreplayable golden-path scores" + ) + scp(str(PATCHED_AD_DATASET), ip, AD_DATASET_REMOTE) + _ad_seen = ssh_checked(ip, f"grep -c scenarioKey ~/{AD_DATASET_REMOTE} || true").strip() + if _ad_seen in ("", "0"): + raise RuntimeError( + f"AD dataset overlay did not land on {ip} (scenarioKey absent " + f"from {AD_DATASET_REMOTE}); the run would emit golden-path " + "scores that cannot be rejudged" + ) + # Scout-readiness timeout overlay (PR #285302) — see PATCHED_EVAL_STACK. + EVAL_STACK_REMOTE = ( + "Projects/kibana/x-pack/platform/packages/shared/kbn-evals/src/cli/eval_stack.ts" + ) + scp(str(PATCHED_EVAL_STACK), ip, EVAL_STACK_REMOTE) + # Playwright timeout overlay: default 30min dies mid-run at 3 repetitions + # (21 examples × 3 reps ≈ 90min; observed death at example 7/21). + PW_CONFIG_REMOTE = ( + "Projects/kibana/x-pack/solutions/security/packages/" + "kbn-evals-suite-security-persona-matrix/playwright.config.ts" + ) + if persona_only: + scp(str(PATCHED_PW_CONFIG), ip, PW_CONFIG_REMOTE) + scp(str(PATCHED_RULE_SKILL), ip, PATCHED_RULE_SKILL_REMOTE) + # Env-truth fixtures (PR #286421): seeds + idempotent tool reinstall + spec wiring. + if persona_only: + ssh(ip, f"mkdir -p ~/{FIXTURES_REMOTE_PREFIX}/src/fixtures ~/{FIXTURES_REMOTE_PREFIX}/evals") + scp(str(PATCHED_ENV_SEEDS), ip, f"{FIXTURES_REMOTE_PREFIX}/src/fixtures/env_seeds.ts") + scp(str(PATCHED_TOOLS_SEED), ip, f"{FIXTURES_REMOTE_PREFIX}/src/fixtures/persona_matrix_tools_seed.ts") + scp(str(PATCHED_SPEC), ip, f"{FIXTURES_REMOTE_PREFIX}/evals/persona_matrix.spec.ts") + scp(str(PATCHED_TOOL_CHECK), ip, f"{FIXTURES_REMOTE_PREFIX}/src/fixtures/tool_registration_check.ts") + out = ssh(ip, f"grep -q seedPersonaMatrixEnvironment ~/{FIXTURES_REMOTE_PREFIX}/evals/persona_matrix.spec.ts && " + f"grep -q assertPersonaMatrixToolsRegistered ~/{FIXTURES_REMOTE_PREFIX}/src/fixtures/tool_registration_check.ts && " + f"echo ENVSEEDS_OK") + if "ENVSEEDS_OK" not in out: + raise RuntimeError(f"env-truth overlay verification failed on {ip}: {out}") + infra_checks = [ + f"grep -q SCOUT_READY_TIMEOUT_MS ~/{EVAL_STACK_REMOTE}", + f"grep -q erroredRuns ~/{EXECUTOR_CLIENT_REMOTE}", + f"grep -q getStatusCode ~/{RETRY_UTILS_REMOTE}", + f"grep -q 'retries: 8' ~/{TRACE_FACTORY_REMOTE}", + "grep -q AGENT_BUILDER_INFERENCE_TIMEOUT_MS ~/Projects/kibana/x-pack/platform/plugins/shared/inference/server/chat_complete/utils/inference_endpoint_executor.ts", + f"grep -q agentBuilderTracingExporters ~/{PROFILES_REMOTE}", + f"grep -q agentBuilderTracingExporters ~/{SCOUT_TRACING_CONFIG_REMOTE}", + # Agent Builder strips gen_ai.tool.call.arguments/.result from every tool + # span unless this uiSetting is on (default false, for privacy). Without + # it SkillInvoked matches nothing and scores 0 for EVERY model -- which + # reads as models not invoking skills rather than a missing attribute. + # Observed 2026-09-04..06: 8,339 load_skill spans on sweep VMs with 0 + # arguments, vs 3,953/3,953 populated on Buildkite CI. + f"grep -q 'agentBuilder:tracing:includeToolDetails=true' ~/{SCOUT_TRACING_CONFIG_REMOTE}", + # The stale main checkout resolves TRACE_INDEX_PATTERN to undefined, so + # every SkillInvoked ES|QL query runs `FROM undefined` and dies with a + # security_exception on `indices [undefined]` — which reads as a missing + # privilege, not stale code. Assert the constant is exported AND that the + # consumer interpolates it (a literal `FROM traces-*` is the old form). + f"grep -q \"export const TRACE_INDEX_PATTERN = 'traces-\\*,.ds-traces-\\*'\" ~/{TRACE_FACTORY_REMOTE}", + f"grep -q 'FROM ${{TRACE_INDEX_PATTERN}}' ~/{SKILL_INVOCATION_REMOTE}", + # The barrel is the binding the suite actually imports: without this the + # constant is undefined at runtime even when factory.ts is correct. + f"grep -q TRACE_INDEX_PATTERN ~/{EVALS_BARREL_REMOTE}", + f"grep -q TRACE_INDEX_PATTERN ~/{TRACE_BARREL_REMOTE}", + # Every metric evaluator must interpolate the constant. The stale main + # form is a literal `FROM traces-*`, which resolves to zero authorized + # indices on golden and errors with `Unknown column [trace.id]`. + *[ + f"! grep -q 'FROM traces-\\*' ~/{_TRACE_DIR_REMOTE}/{_f}" + for _f in TRACE_METRIC_FILES + ], + ] + persona_checks = [ + f"grep -q skillPredicate ~/{PATCHED_EVALUATOR_REMOTE}", + f"grep -q MAX_PAYLOAD_BYTES ~/{PATCHED_SCOUT_CONFIG_REMOTE}", + f"grep -q FinalAnswerPresent ~/{PATCHED_EVALUATOR_REMOTE}", + # Without this forward every stored answer looks like a real closing + # turn, even when it is the fallback to an interior reasoning step: + # 0 of 2422 Sep-4+ detection-rule-edit docs carried the tag (2026-09-07). + f"grep -q 'messageSource: response.messageSource' ~/{PATCHED_EVALUATOR_REMOTE}", + f"grep -q 'NEVER finish the turn' ~/{PATCHED_RULE_SKILL_REMOTE}", + ] + checks = infra_checks + (persona_checks if persona_only else []) + out = ssh_checked(ip, " && ".join(checks) + " && echo OVERLAY_OK") + if "OVERLAY_OK" not in out: + # A bare `&&` chain reports nothing about WHICH clause failed, so the + # error used to read "verification failed on : " with an empty + # tail -- indistinguishable from a transport problem. Re-run the + # clauses individually to name the actual offender. + failed = [] + for _c in checks: + try: + if "CLAUSE_OK" not in ssh_checked(ip, _c + " && echo CLAUSE_OK"): + failed.append(_c) + except RuntimeError as exc: # transport died mid-diagnosis + failed.append(f"{_c} (transport: {exc})") + raise RuntimeError( + f"patched overlay verification failed on {ip}: " + f"{len(failed)}/{len(checks)} clause(s) failed: {failed or out!r}" + ) + print(f"[deploy] assets + patched evaluator/config on {ip}", flush=True) + + +def build_env_prefix(model: str, environ: Optional[Mapping[str, str]] = None) -> str: + """Shell prefix exporting every forwarded var for one model's remote run. + + Forward every per-model var (plus any process-env override) rather than a + hardcoded pair: a var added to MODEL_ENV but missing from this list is a + silent no-op that looks like a tuning fix and changes nothing. EVAL_CONNECTOR_ID + belongs here so judge-panel runs actually reach the VM — without it run_model.sh + silently re-derives its own Anthropic default and the sweep answers the wrong + question while passing every gate. + """ + environ = os.environ if environ is None else environ + model_env = dict([kv.split("=", 1) for kv in MODEL_ENV.get(model, "").split()]) if MODEL_ENV.get(model) else {} + prefix = "" + for key in sorted({*model_env, *FORWARDED_ENV_VARS}): + value = environ.get(key, model_env.get(key, "")) + if value: + prefix += f"export {key}={shlex.quote(value)} && " + return prefix + + +def check_expected_datasets(seen: set, expected: set) -> Optional[str]: + """Reject a run whose docs landed under datasets the sweep did not ask for. + + A doc-count gate cannot catch running the WRONG SUITE. Measured 2026-09-09: + `--suite security-automatic-migrations` passed 8/8 units at docs=86/80 while + writing every doc under standard-dashboards / qradar / splunk-spl -- none of + which are the board's migrations columns (those come from the separate + `agent-builder` suite). The floor gate was structurally blind to it: 86 >= 80 + holds no matter which datasets produced the 86. + + Returns None when the run is acceptable, else a human-readable reason. + """ + if not expected: + return None # no declared identity -> nothing to assert against + if not seen: + return "no dataset ids observed in golden" + unexpected = seen - expected + missing = expected - seen + parts = [] + if unexpected: + parts.append("unexpected datasets " + ",".join(sorted(unexpected))) + if missing: + parts.append("missing datasets " + ",".join(sorted(missing))) + return "; ".join(parts) or None + + +def self_test() -> int: + """Offline checks for the pure helpers, run via `--self-test` in the verify + manifest. Covers the two defects that made a judge-panel sweep lie: a dropped + EVAL_CONNECTOR_ID (graded with the incumbent judge, still passed its gate) and + a dead Scout stack reported as `list index out of range`. + """ + failures = [] + + def check(name, got, want): + if got != want: + failures.append(f"{name}: expected {want!r}, got {got!r}") + + # A doc-count gate cannot catch a wrong-suite run: the mig2 sweep passed + # 8/8 at docs=86/80 while writing three datasets the board never plots. + _MIG = {"05fd1e03-0e35-5abf-bfc6-c07118da0b28", "07a6c75d-7b4b-5150-ab32-7b66a93ac910"} + check("wrong-suite datasets rejected", + bool(check_expected_datasets({"4de2d8a5-x", "03855b50-y"}, _MIG)), True) + # Isolate the unexpected-only path: a superset covers every expected dataset + # (so `missing` is empty) and must STILL fail on the extra one. Without this + # case, deleting the unexpected-check entirely still passed the suite -- + # the wrong-suite test above was passing via `missing`, not `unexpected`. + check("extra dataset alone fails", + "unexpected datasets" in (check_expected_datasets(_MIG | {"da87a6b7-z"}, _MIG) or ""), + True) + check("exact dataset match accepted", + check_expected_datasets(set(_MIG), _MIG), None) + check("partial coverage reported", + "missing datasets" in (check_expected_datasets({"05fd1e03-0e35-5abf-bfc6-c07118da0b28"}, _MIG) or ""), + True) + check("empty golden rejected", check_expected_datasets(set(), _MIG), + "no dataset ids observed in golden") + # No declared identity must stay permissive: suites predating the gate + # (persona, AD) have no expected-set and must not start failing. + check("undeclared identity stays permissive", + check_expected_datasets({"anything"}, set()), None) + + # ssh_checked must distinguish a transport failure from a content verdict. + # Reporting exit-255-with-empty-output as "checks failed" skipped 6/6 units + # of an AD sweep whose overlays were correct (2026-09-09). + import unittest.mock as _mock + + def _fake_run(rc, out=""): + return lambda *a, **k: subprocess.CompletedProcess([], rc, out, "") + + with _mock.patch.object(subprocess, "run", _fake_run(0, "OVERLAY_OK")): + check("content pass returned", ssh_checked("1.2.3.4", "true"), "OVERLAY_OK") + with _mock.patch.object(subprocess, "run", _fake_run(1, "")): + check("content failure returned, not raised", ssh_checked("1.2.3.4", "false"), "") + with _mock.patch.object(subprocess, "run", _fake_run(255, "")), \ + _mock.patch.object(time, "sleep", lambda *_: None): + _raised = False + try: + ssh_checked("1.2.3.4", "true", attempts=2) + except RuntimeError: + _raised = True + check("transport failure raises", _raised, True) + + # wait_ssh must not return on a single lucky echo: cloud-init bounces sshd. + _seq = ["ok", "", "ok", "ok"] + with _mock.patch.object(sys.modules[__name__], "ssh", lambda *a, **k: _seq.pop(0)), \ + _mock.patch.object(time, "sleep", lambda *_: None): + check("wait_ssh needs a streak", wait_ssh("1.2.3.4"), True) + check("wait_ssh consumed the flap", _seq, []) + + m = "eis-anthropic-claude-4-7-opus" + env = {"EVAL_REPETITIONS": "3", "EVAL_CONNECTOR_ID": "eis-google-gemini-3-1-pro"} + prefix = build_env_prefix(m, env) + check("judge forwarded", "export EVAL_CONNECTOR_ID=eis-google-gemini-3-1-pro && " in prefix, True) + check("reps forwarded", "export EVAL_REPETITIONS=3 && " in prefix, True) + check("no empty exports", "= &&" in prefix, False) + check("absent var omitted", "PERSONA_MATRIX_TIMEOUT_MINUTES" in build_env_prefix(m, {}), False) + # Per-model defaults still apply, and the process env wins over them. + glm = "eis-zai-glm-5-2" + check("model default kept", "PERSONA_MATRIX_CONCURRENCY=3" in build_env_prefix(glm, {}), True) + check( + "env overrides model default", + "export PERSONA_MATRIX_TIMEOUT_MINUTES=240 && " in build_env_prefix( + glm, {"PERSONA_MATRIX_TIMEOUT_MINUTES": "240"} + ), + True, + ) + # Shell-quoting: a value with a space must not split into two words. + check("value quoted", "'a b'" in build_env_prefix(m, {"EVAL_CONNECTOR_ID": "a b"}), True) + + # --- quota gate ------------------------------------------------------- + # The 2026-09-06 incident in one assertion: 23 units x 8 cores against + # 344/350 used must REFUSE, not launch and fail per-VM hours later. + ok_full, msg_full = quota_gate(23, 344, 350, 8) + check("gate refuses full quota", ok_full, False) + check("gate names the shortfall", "184 cores" in msg_full, True) + check("gate reports what fits", "At most 0 unit(s)" in msg_full, True) + # Exactly-fits must pass: an off-by-one here would block valid sweeps. + check("gate allows exact fit", quota_gate(2, 334, 350, 8)[0], True) + check("gate allows headroom", quota_gate(23, 64, 350, 8)[0], True) + # One core short must refuse. + check("gate refuses one short", quota_gate(2, 335, 350, 8)[0], False) + check("gate zero units ok", quota_gate(0, 350, 350, 8)[0], True) + + saved_size = globals()["VM_SIZE"] + try: + globals()["VM_SIZE"] = "Standard_D8s_v5" + check("cores from size", cores_per_vm(), 8) + globals()["VM_SIZE"] = "Standard_D16s_v5" + check("cores from larger size", cores_per_vm(), 16) + finally: + globals()["VM_SIZE"] = saved_size + + # --- spot / park defaults --------------------------------------------- + # Spot must default OFF: capacity has been exhausted since 2026-09-04 and + # every attempt costs a failed create plus backoff before falling back. + check("spot off by default", use_spot({}), False) + check("spot opt-in honoured", use_spot({"SPOT_VM": "1"}), True) + # Parking must default ON: idle finished VMs caused both quota exhaustions. + check("park on by default", park_on_done({}), True) + check("park opt-out honoured", park_on_done({"PARK_ON_DONE": "0"}), False) + + # --- spot / park wiring (az verbs, not just flags) --------------------- + # Stub az so the self-test asserts what would REALLY be sent to Azure. + # Flag-only assertions let a correct default drift away from an unwired + # call site; these bind the decision to the command. + saved_az = globals()["az"] + calls: list = [] + try: + globals()["az"] = lambda *a: calls.append(a) or "[]" + + calls.clear() + os.environ["PARK_ON_DONE"] = "1" + parked = maybe_park_unit("eis-openai-gpt-5-4") + check("park issues deallocate", parked, True) + check("park verb is deallocate", calls and calls[0][:2] == ("vm", "deallocate"), True) + check("park does not delete", any(c[1] == "delete" for c in calls), False) + + calls.clear() + os.environ["PARK_ON_DONE"] = "0" + check("park opt-out issues nothing", maybe_park_unit("eis-openai-gpt-5-4"), False) + check("park opt-out silent", len(calls), 0) + os.environ.pop("PARK_ON_DONE", None) + + # provision(): Spot must not be attempted unless opted in. Asserting on + # the create args catches an unwired flag that a default check misses. + saved_pool = os.environ.get("WARM_POOL") + os.environ["WARM_POOL"] = "0" # skip the parked-VM lookup + try: + def _fake_az(*a): + calls.append(a) + # vm show -d --query publicIps: return an IP to end provision() + return '"10.0.0.1"' if a[:2] == ("vm", "show") else "[]" + + globals()["az"] = _fake_az + calls.clear() + os.environ.pop("SPOT_VM", None) + provision("eis-openai-gpt-5-4") + creates = [c for c in calls if c[:2] == ("vm", "create")] + check("one create by default", len(creates), 1) + check("no spot priority by default", any("Spot" in c for c in creates), False) + + calls.clear() + os.environ["SPOT_VM"] = "1" + provision("eis-openai-gpt-5-4") + creates = [c for c in calls if c[:2] == ("vm", "create")] + check("spot opt-in sends Spot", any("Spot" in c for c in creates), True) + os.environ.pop("SPOT_VM", None) + finally: + if saved_pool is None: + os.environ.pop("WARM_POOL", None) + else: + os.environ["WARM_POOL"] = saved_pool + finally: + globals()["az"] = saved_az + + # --- warm pool -------------------------------------------------------- + # WARM_POOL=0 must short-circuit before any az call, so a sweep can always + # force fresh VMs when a parked box is suspect. + saved_warm = os.environ.get("WARM_POOL") + try: + os.environ["WARM_POOL"] = "0" + check("warm pool opt-out", reusable_pool_vm("eis-openai-gpt-5-4"), None) + finally: + if saved_warm is None: + os.environ.pop("WARM_POOL", None) + else: + os.environ["WARM_POOL"] = saved_warm + + # --- suite port ------------------------------------------------------- + # Every check below pins a defect that would otherwise cost real VM time or + # silently grade the wrong suite. + global SUITE + saved = SUITE + try: + # VM names must not collide across suites: same model, two sweeps. + SUITE = "security-persona-matrix" + persona_vm = vm_name("eis-openai-gpt-5-4") + SUITE = "attack-discovery-agent-builder" + ad_vm = vm_name("eis-openai-gpt-5-4") + check("vm names differ per suite", persona_vm != ad_vm, True) + check("ad vm prefix", ad_vm.startswith("orca-ad-"), True) + check("vm name length", len(ad_vm) <= 64, True) + + # Teardown must claim every suite's resources, or they keep billing. + check("teardown claims persona", is_sweep_resource(persona_vm + "_OsDisk"), True) + check("teardown claims ad", is_sweep_resource(ad_vm + "_OsDisk"), True) + SUITE = "security-automatic-migrations" + check("teardown claims migrations", + is_sweep_resource(vm_name("eis-openai-gpt-5-4") + "-nic"), True) + check("teardown ignores foreign", is_sweep_resource("unrelated-vm_OsDisk"), False) + + # Run dirs are namespaced, so a second suite cannot clobber the first. + SUITE = "security-persona-matrix" + d1 = model_dir("eis-openai-gpt-5-4") + SUITE = "attack-discovery-agent-builder" + d2 = model_dir("eis-openai-gpt-5-4") + check("model dirs differ per suite", d1 != d2, True) + + # EVAL_SUITE must reach the VM: without it run_model.sh falls back to + # persona-matrix and grades the wrong suite while its gate still passes. + check("EVAL_SUITE forwarded", "EVAL_SUITE" in FORWARDED_ENV_VARS, True) + prefix = build_env_prefix("eis-openai-gpt-5-4", {"EVAL_SUITE": "attack-discovery-agent-builder"}) + check("EVAL_SUITE exported", + "export EVAL_SUITE=attack-discovery-agent-builder && " in prefix, True) + + # Doc-count gate: expected docs are per-suite, counted from the datasets. + check("persona n_examples", SUITE_PROFILES["security-persona-matrix"]["n_examples"], 21) + check("ad n_examples", SUITE_PROFILES["attack-discovery-agent-builder"]["n_examples"], 10) + # AD ignores PERSONA_MATRIX_SHARD (measured 2026-09-08: 24/24 units + # exported the full 117-doc grid under per-shard execution_ids), so + # its gate must not slice the expectation by shard. + check("ad gate not shard-sliced", + SUITE_PROFILES["attack-discovery-agent-builder"].get("honors_shard", True), False) + check("persona gate still shard-sliced", + SUITE_PROFILES["security-persona-matrix"].get("honors_shard", True), True) + # Migrations has no uniform grid -- per-dataset evaluator counts are + # 7 (standard-dashboards) / 9 (qradar) / 8 (splunk-spl), measured on the + # 2026-09-02 canary -- so examples x evaluators is structurally wrong. + # It gates on a floor measured from that run (86 docs) instead. + check("migrations gate is floor", + SUITE_PROFILES["security-automatic-migrations"]["gate"], "floor") + check("migrations floor set", + SUITE_PROFILES["security-automatic-migrations"]["min_docs"] > 0, True) + + # Sharding: PERSONA_MATRIX_SHARD must reach the VM, or run_model.sh runs + # all 21 examples on every shard and the sweep still reports complete. + # Assert on build_env_prefix output, not list membership: membership + # passes even if the builder never consults FORWARDED_ENV_VARS. + check("shard var forwarded to VM", + "export PERSONA_MATRIX_SHARD=2/4 && " in + build_env_prefix("m", {"PERSONA_MATRIX_SHARD": "2/4"}), True) + # Shard sizes must match the suite's stride assignment and sum to the + # whole dataset -- an off-by-one here FAILs a good run or passes a short one. + shard_sizes = [len(range(i, 21, 4)) for i in range(4)] + check("shard sizes stride 21/4", shard_sizes, [6, 5, 5, 5]) + check("shard sizes sum to dataset", sum(shard_sizes), 21) + check("shard sizes balanced", max(shard_sizes) - min(shard_sizes) <= 1, True) + + # Shard fanout: each (model, shard) needs its OWN VM name and run dir. + # A shared name would put two eval stacks on one box (they OOM each + # other and corrupt local ES) or overwrite the sibling's run.log. + check("shard vm names differ", + vm_name("eis-openai-gpt-5-4", "1/4") != vm_name("eis-openai-gpt-5-4", "2/4"), True) + + # JUDGE BLINDING — the VM name enrols as a host entity in the eval + # cluster, so a model-derived name lets the model identify itself to + # its own LLM judge. Measured: 14 agent answers echoed + # `orca-sweep-anthropic-claude-4-8-opus` back into judged text. + for _m in ["eis-anthropic-claude-4-8-opus", "anthropic-claude-4.8-opus", + "eis-openai-gpt-5-4", "z-ai/glm-5.3-flash", "google-gemini-3-1-pro"]: + _n = vm_name(_m).lower() + for _token in ["anthropic", "claude", "openai", "gpt", "gemini", + "google", "glm", "qwen", "kimi", "opus", "sonnet", "haiku"]: + check(f"vm name blinds {_token!r} for {_m}", _token in _n, False) + check("blinded name is still deterministic", + vm_name("eis-openai-gpt-5-4"), vm_name("eis-openai-gpt-5-4")) + check("distinct models get distinct blinded names", + vm_name("eis-openai-gpt-5-4") != vm_name("eis-openai-gpt-5-5"), True) + check("eis- prefix does not fork the blinded name", + vm_name("eis-openai-gpt-5-4"), vm_name("openai-gpt-5-4")) + check("blinded sharded names stay distinct", + vm_name("eis-openai-gpt-5-4", "1/4") != vm_name("eis-openai-gpt-5-4", "2/4"), True) + check("unsharded vm name unchanged", + vm_name("eis-openai-gpt-5-4"), vm_name("eis-openai-gpt-5-4", None)) + check("shard vm name within azure 64-char limit", + len(vm_name("openrouter-qwen-qwen3-8-27b-longer-name-here", "10/16")) <= 64, True) + # A model slug long enough to fill the 64-char budget must still yield + # distinct per-shard names -- appending the suffix before clamping made + # s1of4 and s2of4 identical and put two stacks on one VM. + _long = "openrouter-some-really-long-vendor-name-with-many-segments-here-x" + check("long model name still shards distinctly", + vm_name(_long, "1/4") != vm_name(_long, "2/4"), True) + check("long sharded name still within limit", len(vm_name(_long, "1/4")) <= 64, True) + check("shard run dirs differ", + model_dir("m", shard="1/4") != model_dir("m", shard="2/4"), True) + check("unsharded run dir unchanged", model_dir("m"), model_dir("m", shard=None)) + # "/" in a shard spec must not create a nested path or escape the dir. + check("shard dir has no slash from spec", + "/" not in model_dir("m", shard="2/4").name, True) + + # --shards fanout: unit expansion decides how many VMs boot. + def _units(models, shards): + if shards == 1: + return [(m, None) for m in models] + return [(m, f"{i}/{shards}") for m in models for i in range(1, shards + 1)] + + check("shards=1 keeps one unit per model", _units(["a", "b"], 1), + [("a", None), ("b", None)]) + check("shards=1 leaves shard None (back-compat vm names)", + vm_name("a", _units(["a"], 1)[0][1]), vm_name("a")) + check("shards=4 expands to 4 VMs per model", len(_units(["a", "b"], 4)), 8) + check("every expanded unit is unique", len(set(_units(["a", "b"], 4))), 8) + check("expanded shards cover 1..N", sorted(str(s) for _, s in _units(["a"], 3)), + ["1/3", "2/3", "3/3"]) + # Slices must partition the dataset exactly: a stride that dropped or + # double-counted an example silently changes what the matrix measures. + _n = SUITE_PROFILES["security-persona-matrix"]["n_examples"] + _covered = sorted(i for idx in range(1, 5) for i in range(idx - 1, _n, 4)) + check("4 shards partition all 21 examples exactly", _covered, list(range(_n))) + + # The 2026-09-03 smoke run died on every VM because evaluate_dataset.ts + # imported ./datasets/select_shard and the overlay never shipped it. + # Parse the real imports and require an overlay entry for each. + _overlaid = {Path(p).name for p in (PATCHED_DATASET, PATCHED_SHARD)} + _ed = Path(PATCHED_EVALUATOR) + if _ed.is_file(): + _imports = set(re.findall(r"from '\./datasets/([a-z_]+)'", _ed.read_text())) + _missing = {f"{i}.ts" for i in _imports} - _overlaid + check("every ./datasets import is in the VM overlay", sorted(_missing), []) + check("overlay actually parsed some imports", len(_imports) > 0, True) + + # Shards must not share a TEST_RUN_ID: execution_id derives from it, + # and a shared id made shard 2/2 count shard 1/2's docs (154 vs a 140 + # gate) instead of only its own slice. + check("shard run ids differ", + shard_run_id("sweep-1", "1/2") != shard_run_id("sweep-1", "2/2"), True) + check("shard run id keeps the sweep base", + shard_run_id("sweep-1", "1/2").startswith("sweep-1"), True) + check("unsharded run id untouched", shard_run_id("sweep-1", None), "sweep-1") + check("shard run id has no slash", + "/" not in shard_run_id("sweep-1", "2/4"), True) + # Testing shard_run_id alone is a false green: deleting launch()'s + # assignment left every check passing while both VMs shared an id. + # Assert the wiring by reading launch()'s own source. + _launch_src = inspect.getsource(launch) + check("launch assigns a per-shard TEST_RUN_ID", + 'env["TEST_RUN_ID"] = shard_run_id(' in _launch_src, True) + check("TEST_RUN_ID reaches the VM", + "export TEST_RUN_ID=sweep-1-s1of2 && " in + build_env_prefix("m", {"TEST_RUN_ID": shard_run_id("sweep-1", "1/2")}), True) + # The gate's "latest execution for this model" lookup must be pinned to + # the shard's own run id. Unpinned, it returned whichever shard wrote + # last and both shards gated against one id (154 vs a 140 gate) -- + # every earlier check still passed while the sweep stayed broken. + _gate_src = inspect.getsource(check_golden) + check("gate builds the shard's own execution id", + 'shard_run_id(os.environ.get("TEST_RUN_ID", ""), shard)' in _gate_src, True) + # The gate's own count query must not use .keyword either. The guard below + # only covers the resume probe, so this trap shipped here undetected and + # failed a complete 98-doc shard as docs=0/98. + _gate_code = "\n".join( + l for l in _gate_src.splitlines() if not l.lstrip().startswith("#") + ) + check("gate count does not use the .keyword suffix", + "metadata.execution_id.keyword" not in _gate_code, True) + # The field has no usable partial matching: a .keyword prefix and a + # match_phrase on the run id both returned 0 docs against golden while + # the full id returned 154. Never reintroduce a partial match here. + check("gate does not prefix-match execution_id", + 'prefix": {"metadata.execution_id' not in _gate_src, True) + # The latest-execution lookup must match ANY spelling of the model id, + # never the single stored_id string. The VM's local index holds the + # display name ("anthropic-claude-4.5-haiku") while golden holds the + # connector id ("eis-anthropic-claude-4-5-haiku"); a term on the local + # display name matched only the stale reference artifact (117 docs) and + # false-FAILed three dense canaries as docs=117/130 while golden held + # 130 under the fresh exec. + check("latest-execution lookup matches any model-id spelling", + "_score_id_candidates(model)" in _gate_code and + '{"term": {"task.model.id": stored_id}}' not in _gate_code, True) + check("sharded gate skips the latest-execution lookup", + "if exec_id is None:" in _gate_src, True) + check("ad gate is exact", + SUITE_PROFILES["attack-discovery-agent-builder"]["gate"], "exact") + + # --- resume probe (run 13 postmortem) ------------------------------- + # Three independent bugs made resume a silent no-op for runs 9-13; each + # one alone re-ran all 21 examples on every retry while looking healthy. + _rm = (Path(__file__).parent / "run_model.sh").read_text() + + # 1. Score docs live on GOLDEN. Local scout ES is wiped by the retry and + # is empty at exactly the moment resume reads it. + _fn = _rm[_rm.index("scored_example_ids() {"):_rm.index("for attempt in 1 2 3; do")] + # The SCORED-SET query must hit golden. A local-ES read is still + # legitimate for deriving the run id (TEST_RUN_ID is absent from this + # shell), so assert on the scoring query itself rather than banning + # every localhost:9220 mention. + _score_q = _fn.split("RUN_ID\" ] ||")[-1] + check("resume probe scores against golden, not local scout ES", + "localhost:9220" not in _score_q and "GOLDEN_ES_URL" in _score_q, True) + + # 2. metadata.execution_id is ALREADY keyword-mapped. Verified against + # golden: term on the bare field -> 98 docs; on .keyword -> 0. + # Strip comments first: the function documents the .keyword trap in a + # comment, and a naive substring check flags its own documentation. + _fn_code = "\n".join( + l for l in _fn.splitlines() if not l.lstrip().startswith("#") + ) + check("resume probe does not use the .keyword suffix", + "metadata.execution_id.keyword" not in _fn_code, True) + + # 3. Flush must PRECEDE the golden query, else it reads an empty index, + # returns "", and skips the flush that would have populated it. + # .find() not .index(): a missing marker must FAIL the check, not raise + # and abort the whole self-test before the remaining guards run. + _flush_at = _rm.find("flushing partial scores") + _query_at = _rm.find('DONE_IDS="$(scored_example_ids)"') + check("resume flushes to golden before querying it", + _flush_at >= 0 and _query_at >= 0 and _flush_at < _query_at, True) + + # Unknown suite must fail loudly rather than silently sweeping persona. + try: + suite_profile("no-such-suite") + check("unknown suite rejected", False, True) + except KeyError: + pass + finally: + SUITE = saved + + # No persona-matrix identity may survive anywhere in the file: the gate + # resolved "latest execution" by a hardcoded persona-matrix dataset UUID, + # so on an AD/migrations run it counted the model's OLD persona-matrix + # execution (294 docs) and compared it to the new suite's expectation. + # The sweep read as FAIL while the real run was fine -- and would have + # read as PASS if the numbers had happened to line up. + src = Path(__file__).read_text() + check("no hardcoded dataset uuid", ("f2db90e6-cb7f" + "-58f2-b862-1b69e47f6a77") in src, False) + for suite_id in SUITE_PROFILES: + check(f"gate scopes to {suite_id}", suite_profile(suite_id)["gate_suite_id"], suite_id) + + print(f"self-test: {len(failures)} failure(s)") + for f in failures: + print(f" FAIL {f}") + return 1 if failures else 0 + + +def shard_run_id(base: str, shard: Optional[str]) -> str: + """Per-shard TEST_RUN_ID. + + execution_id is derived from TEST_RUN_ID. Shards that share one id also + share an execution_id, so each shard's golden gate counts every other + shard's docs (shard 2/2 read 154 against a 140 gate). Suffixing keeps the + slices independently countable while staying traceable to one sweep. + """ + if not shard: + return base + return f"{base}-s{shard.replace('/', 'of')}" + + +def launch(ip: str, model: str, shard: Optional[str] = None) -> subprocess.Popen: + log = model_dir(model, shard=shard) / "run.log" + log.parent.mkdir(parents=True, exist_ok=True) + env = dict(os.environ) + env["EVAL_SUITE"] = suite_profile()["cli_suite"] + # Suite reads PERSONA_MATRIX_SHARD for its example stride; per-VM value. + if shard: + env["PERSONA_MATRIX_SHARD"] = shard + # execution_id derives from TEST_RUN_ID. Left unset, every shard VM + # computes the SAME id, so each shard's golden gate counts all the + # other shards' docs too (observed: shard 2/2 read 154 docs against a + # 140 gate because shard 1/2's 154 landed under one id). A per-shard + # run id keeps the slices independently countable. + base = env.get("TEST_RUN_ID") or f"sweep-{int(time.time())}" + env["TEST_RUN_ID"] = shard_run_id(base, shard) + # Detach on the VM: the eval runs 30-60+ min and holding one long SSH + # stream is fragile — mid-run stream death (rc 255) previously killed the + # controller's view while the eval kept running (observed 2026-09-03: + # every shard FAILED with an unparseable scores response while node/evals + # was alive on the VM). run_model.sh writes /tmp/unit.done + .rc when it + # finishes; we poll those over short-lived SSH connections instead. + run_cmd = ( + f"rm -f /tmp/unit.done /tmp/unit.rc; " + f"{build_env_prefix(model, env)}nohup bash /tmp/run_model.sh {model} " + f"> /tmp/unit-run.log 2>&1 < /dev/null & echo launched" + ) + return subprocess.Popen( + ["ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-o", "ServerAliveInterval=30", "-o", "ServerAliveCountMax=60", + "-i", SSH_KEY, f"{SSH_USER}@{ip}", run_cmd], + stdout=open(log, "w"), stderr=subprocess.STDOUT) + + +def _score_id_candidates(canon: str) -> list: + """Connector ids hyphenate versions (claude-4-5, glm-5-2). Score docs + dot some (anthropic-claude-4.5-sonnet) and keep hyphens on others + (zai-glm-5-2), so never assume one spelling -- try both. + + The EIS connector runs write task.model.id WITH the "eis-" prefix and + hyphenated ("eis-anthropic-claude-4-5-haiku"), while the reference + artifact the board recreates wrote it WITHOUT the prefix and dotted + ("anthropic-claude-4.5-haiku"). task.model.id is keyword-mapped so a + match_phrase is exact: dropping the prefix left only the stale reference + exec (117 docs) in scope and gated the fresh 130-doc run against it + (2026-09-10 dense canary FAIL 117/130). Emit every prefix x version.""" + bare = canon[4:] if canon.startswith("eis-") else canon + dotted = re.sub(r"(?<=[0-9])-(?=[0-9])", ".", bare) + core = [bare] if dotted == bare else [dotted, bare] + out = [] + for c in core: + out.append(c) + out.append("eis-" + c) + return out + + +def _resolve_from_golden(model: str, ip: str) -> dict: + """Recover stored_id and evaluator count from golden. + + The VM-local index is empty when a run exports straight to golden, which + is not proof the eval produced nothing. Connector ids hyphenate semantic + versions while score docs dot them, so match a phrase instead of + reconstructing the id. + """ + canon = model + # id spelling varies per vendor, so match any candidate + body = { + "size": 0, + "query": {"bool": {"must": [ + {"bool": {"should": [ + {"match_phrase": {"task.model.id": c}} + for c in _score_id_candidates(canon) + ], "minimum_should_match": 1}}, + {"term": {"metadata.suite_id": suite_profile()["gate_suite_id"]}}, + ]}}, + "aggs": { + "m": {"terms": {"field": "task.model.id", "size": 1}}, + "n": {"cardinality": {"field": "evaluator.name"}}, + }, + } + fb_q = json.dumps(body) + out = ssh( + ip, + f"source /tmp/golden-cluster-env.sh; printf '%s' '{fb_q}' > /tmp/q_fb.json; " + f'curl -sS -H "Authorization: ApiKey $GOLDEN_ES_API_KEY" ' + f'"$GOLDEN_ES_URL/.evaluation-scores/_search" ' + f"-H 'Content-Type: application/json' --data @/tmp/q_fb.json", + ) + try: + res = json.loads(out.splitlines()[-1]) + buckets = res["aggregations"]["m"]["buckets"] + if not buckets: + return {"error": "no docs on golden for this model either"} + return { + "stored_id": buckets[0]["key"], + "n_evaluators": int(res["aggregations"]["n"]["value"]), + } + except Exception as exc: + return {"error": f"golden fallback failed: {exc}"} + + +def check_args_coverage(execution_id: str, ip: str) -> dict: + """gen_ai.tool.call.arguments coverage on this execution's traces. + + Two-step (spans link by top-level trace_id, NOT execution_id, and + task.output.traceId is in _source but NOT indexed — exists-query returns + 0, verified 2026-09-13): + 1. score docs of this execution -> _source task.output.traceId set + 2. span agg on terms trace_id: tool spans (either attr shape) and + the subset carrying gen_ai.tool.call.arguments + Discrimination proven: Sep-7 claude-5-sonnet exec -> 325/346 with args; + Sep-1 bulk haiku exec -> 0 tool spans on its traces (the exact gap this + rerun wave exists to fill). Errors degrade to -1s and FAIL the unit — + never silently pass. + + Runs DRIVER-SIDE (like _golden_count_local): units are parked before + this gate, so an ssh probe would land on a deallocated VM or use the + VM's stale golden env copy (2026-09-14 canary: security_exception on + ssh probe while the driver-side doc count was fine). + """ + if not execution_id: + return {"traces": 0, "tool_spans": -1, "with_args": -1, + "error": "no execution_id"} + url, key = _golden_env_local() + if not url or not key: + return {"traces": 0, "tool_spans": -1, "with_args": -1, + "error": "driver cannot reach golden env"} + + def _es(path, body): + out = subprocess.run( + ["curl", "-sS", "-m", "60", + "-H", f"Authorization: ApiKey {key}", + f"{url}{path}", + "-H", "Content-Type: application/json", + "-d", json.dumps(body)], + capture_output=True, text=True, timeout=90, + ).stdout + return json.loads(out.splitlines()[-1]) + + tids = set() + try: + r = _es("/.ds-.evaluation-scores*/_search", { + "size": 200, + "query": {"match_phrase": {"metadata.execution_id": execution_id}}, + "_source": ["task.output.traceId"]}) + for h in r["hits"]["hits"]: + t = (h["_source"].get("task", {}).get("output", {}) or {}).get("traceId") + if t: + tids.add(t) + except Exception: + return {"traces": 0, "tool_spans": -1, "with_args": -1, + "error": "traceId fetch failed (driver-side)"} + if not tids: + return {"traces": 0, "tool_spans": 0, "with_args": 0} + try: + r = _es("/.ds-traces-generic.otel-default*,.ds-traces-agent_builder.otel-default*/_search", { + "size": 0, + "query": {"terms": {"trace_id": sorted(tids)}}, + "aggs": { + "tools": {"filter": {"bool": {"should": [ + {"exists": {"field": "attributes.gen_ai.tool.name"}}, + {"exists": {"field": "gen_ai.tool.name"}}, + ], "minimum_should_match": 1}}, + "aggs": {"with_args": {"filter": {"bool": {"should": [ + {"exists": {"field": "attributes.gen_ai.tool.call.arguments"}}, + {"exists": {"field": "gen_ai.tool.call.arguments"}}, + ], "minimum_should_match": 1}}}}}}}) + return {"traces": len(tids), + "tool_spans": r["aggregations"]["tools"]["doc_count"], + "with_args": r["aggregations"]["tools"]["with_args"]["doc_count"]} + except Exception: + return {"traces": len(tids), "tool_spans": -1, "with_args": -1, + "error": "span agg failed (driver-side)"} + + +def check_golden(model: str, ip: str, shard: Optional[str] = None) -> dict: + """Completeness gate: docs on golden for this model's LATEST execution. + + Connector IDs use hyphens for semantic versions while score docs use dots, + so resolve the stored ID from the VM's clean local score index instead of + guessing with string replacement. The count is scoped to the newest + `metadata.execution_id` for the model — a model-level count accumulates + across executions and false-FAILs any model with recent history. + + Expected size is derived, not hardcoded: 21 examples x evaluator count x + EVAL_REPETITIONS, with the evaluator count read from the local index so + adding an evaluator (e.g. FinalAnswerPresent) doesn't silently skew the + gate. (The local index holds evaluator docs only — task output rides on + those docs' `task.output` — so there is no +1 task doc term.) + """ + resolve_cmd = ( + "curl -sf -u elastic:changeme 'http://localhost:9220/.evaluation-scores/" + "_search?size=1&_source=task.model.id' -H 'Content-Type: application/json' " + "--data '{\"query\":{\"match_all\":{}}}'" + ) + try: + raw = ssh(ip, resolve_cmd).strip() + except Exception as exc: + return {"count": -1, "error": f"ssh failed while reading local scores index: {exc}"} + if not raw: + # curl -sf prints nothing when the endpoint refuses the connection, so an + # empty body means the Scout stack is down (or never booted) rather than + # an empty index. Say that, instead of an IndexError from splitlines()[-1] + # surfacing as a misleading "cannot read local scores index". + return { + "count": -1, + "error": ( + "no response from local scores index on " + f"{ip}:9220 — Scout ES/Kibana is not reachable (check EVAL_EXIT " + "and the stack boot log; the eval likely died before scoring)" + ), + } + try: + local = json.loads(raw.splitlines()[-1]) + hits = local["hits"]["hits"] + except Exception as exc: + return {"count": -1, "error": f"cannot parse local scores index response: {exc}"} + if not hits: + # A run that exports straight to golden leaves the VM-local index + # empty. That is not proof the eval produced nothing, so resolve + # the same two facts from golden and let the doc-count gate below + # deliver the verdict. + fallback = _resolve_from_golden(model, ip) + if fallback.get("error"): + return {"count": 0, "error": fallback["error"]} + stored_id = fallback["stored_id"] + n_evaluators = fallback["n_evaluators"] + else: + stored_id = hits[0]["_source"]["task"]["model"]["id"] + n_evaluators = None + + if n_evaluators is None: + eval_count_cmd = ( + "curl -sf -u elastic:changeme 'http://localhost:9220/.evaluation-scores/" + "_search?size=0' -H 'Content-Type: application/json' --data " + "'{\"aggs\":{\"n\":{\"cardinality\":{\"field\":\"evaluator.name\"}}}}'" + ) + try: + local2 = json.loads(ssh(ip, eval_count_cmd).splitlines()[-1]) + n_evaluators = int(local2["aggregations"]["n"]["value"]) + except Exception as exc: + return {"count": -1, "error": f"cannot count local evaluators: {exc}"} + + # With sharding, several VMs run the SAME model against the same suite and + # each writes its own execution_id. A "latest for this model" lookup then + # returns whichever shard finished last, and every shard gates against that + # one id -- shard 2/2 counted shard 1/2's 154 docs against its 140 gate. + # + # execution_id is "::::" and the field does NOT + # support partial matching: a prefix on .keyword and a match_phrase on the + # run id both return 0 docs (verified against golden). Only the full id + # matches, so build it rather than filtering the "latest" lookup. + if shard: + _run_id = shard_run_id(os.environ.get("TEST_RUN_ID", ""), shard) + # Build the id from the connector id the caller passed (`model`), NOT + # `stored_id`. The VM's local index holds the model's display name in + # `task.model.id` ("anthropic-claude-4.5-haiku") while golden writes + # docs under the connector id ("eis-anthropic-claude-4-5-haiku"). + # Gating on the display name returned 0/98 for 12/12 wave-2 batch B + # units that had in fact written 98 docs each (verified: eis- form 98, + # dotted form 0). + exec_id = f"{_run_id}::{suite_profile()['gate_suite_id']}::{model}" + else: + exec_id = None + _latest_must = [ + { + "bool": { + "should": [ + {"term": {"task.model.id": c}} for c in _score_id_candidates(model) + ], + "minimum_should_match": 1, + } + }, + {"term": {"metadata.suite_id": suite_profile()["gate_suite_id"]}}, + ] + latest_cmd_q = json.dumps({ + "size": 1, + "_source": ["metadata.execution_id"], + "sort": [{"@timestamp": {"order": "desc"}}], + "query": {"bool": {"must": _latest_must}}, + }) + # NOTE: cmd is passed to ssh as a single argv (no local shell), so the + # remote shell is the ONLY quoting layer — use plain double quotes. + # Backslash-escaped \" lands as a literal quote, splits the header on its + # space, and curl then treats "ApiKey" as a URL (2026-08-22 v3 gate + # failure: "Could not resolve host: ApiKey"). + # A sharded run already knows its exact execution_id, so skip the lookup + # entirely -- querying "latest for this model" would just re-introduce the + # cross-shard collision this function exists to avoid. + if exec_id is None: + out = ssh( + ip, + f"source /tmp/golden-cluster-env.sh; printf '%s' '{latest_cmd_q}' > /tmp/q_latest.json; " + f'curl -sS -H "Authorization: ApiKey $GOLDEN_ES_API_KEY" ' + f'"$GOLDEN_ES_URL/.evaluation-scores/_search" ' + f"-H 'Content-Type: application/json' --data @/tmp/q_latest.json", + ) + try: + hits = json.loads(out.splitlines()[-1])["hits"]["hits"] + exec_id = hits[0]["_source"]["metadata"]["execution_id"] + except Exception: + return {"count": -1, "error": f"cannot resolve latest execution id: {out[:200]}"} + + # metadata.execution_id is keyword-mapped already: a .keyword subfield does + # not exist and a term on it silently returns 0, which the gate cannot tell + # apart from "no docs written" (2026-09-07: a complete 98-doc shard failed as + # docs=0/98). Query the field directly. + q = json.dumps({"query": {"term": {"metadata.execution_id": exec_id}}}) + # The exact gate's target must be known BEFORE counting so the flush poll + # below can wait for it. Floor suites have no such product, so target=None. + reps = int(os.environ.get("EVAL_REPETITIONS", "1") or "1") + prof = suite_profile() + _target = None + if prof.get("gate") != "floor": + _te = prof["n_examples"] + if shard and prof.get("honors_shard", True): + idx, total = (int(x) for x in shard.split("/")) + _te = len(range(idx - 1, _te, total)) + _target = _te * n_evaluators * reps + # Ask golden from the driver first: by the time the gate runs, this unit's + # VM is already parked, so the ssh path below reaches a deallocated host. + # Poll toward the target: score docs reach golden through the live OTel + # batch exporter, and a full experiment's batch can sit in the queue with + # retry backoff far longer than the documented 3-7 min trace lag. + # Measured 2026-09-10 (three dense canary runs): the dense example's 13 + # docs carried @timestamp 03:34-03:35 (evaluator completion) but were NOT + # indexed by 03:43 and WERE indexed by 03:55 -- a 10-20 min delivery lag. + # "Count stopped growing" is also wrong: the batch lands as one burst + # after several stable reads. Poll until the count reaches the target or + # a 25-min window elapses (covers the observed lag with headroom; on a + # genuine shortfall the sweep burns 25 idle minutes per unit, acceptable + # because units gate in parallel). + def _count_to_target(): + n = _golden_count_local(exec_id) + if n is None: + return None + deadline = time.time() + 25 * 60 + while _target is not None and n < _target and time.time() < deadline: + time.sleep(30) + n = _golden_count_local(exec_id) + if n is None: + return None + return n + + _local_n = _count_to_target() + if _local_n is not None: + result: dict = {"count": _local_n} + if _local_n == 0: + _phrase_n = _golden_count_local(exec_id, phrase=True) + result = {"count": _phrase_n if _phrase_n is not None else 0} + else: + out = ssh( + ip, + f"source /tmp/golden-cluster-env.sh; printf '%s' '{q}' > /tmp/q.json; " + f'curl -sS -H "Authorization: ApiKey $GOLDEN_ES_API_KEY" ' + f'"$GOLDEN_ES_URL/.evaluation-scores/_count" ' + f"-H 'Content-Type: application/json' --data @/tmp/q.json", + ) + try: + result = json.loads(out.splitlines()[-1]) + except Exception: + return {"count": -1, "error": out[:200]} + if result.get("count", 0) == 0 and _local_n is None: + # mapping without a .keyword subfield — match_phrase works on text + q = json.dumps({"query": {"match_phrase": {"metadata.execution_id": exec_id}}}) + out = ssh( + ip, + f"source /tmp/golden-cluster-env.sh; printf '%s' '{q}' > /tmp/q.json; " + f'curl -sS -H "Authorization: ApiKey $GOLDEN_ES_API_KEY" ' + f'"$GOLDEN_ES_URL/.evaluation-scores/_count" ' + f"-H 'Content-Type: application/json' --data @/tmp/q.json", + ) + try: + result = json.loads(out.splitlines()[-1]) + except Exception: + return {"count": -1, "error": out[:200]} + # reps/prof were computed above (before counting) so the flush poll knew + # the exact-gate target; reuse them here. + if prof.get("gate") == "floor": + # Suites whose datasets carry different evaluator counts (migrations: + # 7 / 9 / 8) have no examples x evaluators product. Gate on a floor + # measured from a canary run, and report it as such. + result["expected"] = prof["min_docs"] + result["gate"] = "floor" + else: + # A sharded run only produces its own slice, so the exact gate must + # expect that slice -- not the whole dataset -- or every shard FAILs. + # Shard sizes follow the suite's stride assignment (index k -> shard + # k % total), so shard i holds ceil((n - (i-1)) / total) examples. + # Only for suites that actually honor PERSONA_MATRIX_SHARD: AD ignores + # it and runs the full grid on every VM, so slicing the expectation + # there reads complete data as FAIL (117/39 on 2026-09-08). + n_examples = prof["n_examples"] + if shard and prof.get("honors_shard", True): + idx, total = (int(x) for x in shard.split("/")) + n_examples = len(range(idx - 1, n_examples, total)) + result["shard"] = shard + result["expected"] = n_examples * n_evaluators * reps + result["gate"] = "exact" + result["execution_id"] = exec_id + # Dataset-identity gate. A doc count cannot tell "ran the right suite" from + # "ran a different suite that also writes ~86 docs" -- see + # check_expected_datasets. Only enforced for suites that declare the + # identity, so persona/AD keep their existing behaviour. + _want = set(prof.get("expected_dataset_ids") or ()) + if _want and exec_id: + _seen = _golden_datasets_local(exec_id) + if _seen is None: + return {**result, "count": -1, + "error": "cannot verify dataset identity (golden unreachable)"} + _bad = check_expected_datasets(_seen, _want) + if _bad: + return {**result, "count": -1, "error": f"wrong suite: {_bad}"} + return result + + +def status() -> None: + root = SWEEP_DIR / SUITE + for model in sorted(os.listdir(root)) if root.exists() else []: + p = model_dir(model) / "status.json" + if p.exists(): + s = json.load(open(p)) + print(f" {model:45} {s.get('state', '?'):10} docs={s.get('docs', '?')}") + + +def prepare(model: str, shard: Optional[str] = None) -> tuple[str, str]: + """Provision + deploy one model's VM. Returns (model, ip).""" + ip = provision(model, shard) + if not wait_ssh(ip): + raise RuntimeError(f"ssh never ready: {model} @ {ip}") + # A VM whose sshd accepts TCP before it accepts auth, or that hits a + # transient scp reset, used to abort the ENTIRE sweep here (2026-08-22: + # 17/19 deployed, one scp failure, zero launches). Retry once after a + # short wait; a genuinely dead VM raises and is skipped by the caller. + try: + deploy(ip) + except subprocess.CalledProcessError: + time.sleep(30) + deploy(ip) + return model, ip + + +def _label(model: str, shard: Optional[str] = None) -> str: + """Human label for one sweep unit; shard suffix only when sharding.""" + return f"{model} [shard {shard}]" if shard else model + + +def _model_state(model: str, shard: Optional[str] = None) -> str: + """Read the state a model last wrote, so skips count as failures too.""" + try: + with open(model_dir(model, shard=shard) / "status.json") as fh: + return json.load(fh).get("state", "UNKNOWN") + except Exception: + return "UNKNOWN" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--models", default="all") + # Provision/deploy fan-out. Each VM is 8 vCPUs against a 350 low-priority + # vCPU regional quota (~43 concurrent), so the ceiling here is Azure API + # politeness and local ssh/scp load, not quota. + ap.add_argument("--provision-workers", type=int, default=5, + help="parallel VM provision+deploy workers. Raising this " + "trades wall-clock for boot races: at 8 on 2026-09-02, " + "5/15 VMs lost the CCM/.inference readiness race " + "(fetch failed -> enable_eis_ccm exit 1). run_model.sh " + "now retries that step 3x, but 5 is the tested default.") + ap.add_argument("--shards", type=int, default=1, + help="split the dataset across N VMs per model. Each VM " + "runs a stride slice (PERSONA_MATRIX_SHARD=i/N). Use " + "for slow models: GLM-5.3 needs ~174min on one VM, " + "~45min at --shards 4. Frontier models need 1.") + ap.add_argument("--only-shard", type=int, default=None, + help="backfill a single shard i of a --shards N sweep; " + "keeps VM names + example stride identical to the " + "original run so the golden gate merges cleanly") + ap.add_argument("--suite", default="security-persona-matrix", + choices=sorted(SUITE_PROFILES), + help="eval suite to sweep; selects overlays, VM prefix and doc gate") + ap.add_argument("--status", action="store_true") + ap.add_argument("--teardown", action="store_true") + ap.add_argument("--self-test", action="store_true", + help="offline checks for the pure helpers; no Azure or SSH") + args = ap.parse_args() + + global SUITE + SUITE = args.suite + + if args.self_test: + return self_test() + + if args.status: + status() + return + if args.teardown: + for vm in json.loads(az("vm", "list", "-g", RG, "-o", "json")): + if any(vm["name"].startswith(pf["vm_prefix"] + "-") + for pf in SUITE_PROFILES.values()): + az("vm", "delete", "-g", RG, "-n", vm["name"], "--yes", "--no-wait") + print(f"[teardown] deleting {vm['name']}") + # `az vm delete` removes ONLY the VM. Its disk, NIC and public IP + # survive and keep billing -- disks are the expensive ones, and NICs + # pin their public IP so ordering matters (NIC first, then IP). + # Observed 2026-09-02: a "successful" teardown left 3 disks, 5 NICs + # and 5 public IPs behind, two of them from sweeps days earlier. + # --no-wait above means VMs may still be detaching; poll until the + # disks actually report Unattached rather than racing them. + print("[teardown] waiting for disks to detach...") + for _ in range(60): + disks = json.loads(az("disk", "list", "-g", RG, "-o", "json")) + sweep = [d for d in disks if is_sweep_resource(d["name"])] + if not sweep or all(d.get("diskState") == "Unattached" for d in sweep): + break + time.sleep(5) + + for d in json.loads(az("disk", "list", "-g", RG, "-o", "json")): + if is_sweep_resource(d["name"]) and d.get("diskState") == "Unattached": + az("disk", "delete", "-g", RG, "-n", d["name"], "--yes", "--no-wait") + print(f"[teardown] deleting disk {d['name']}") + + for n in json.loads(az("network", "nic", "list", "-g", RG, "-o", "json")): + if is_sweep_resource(n["name"]) and not n.get("virtualMachine"): + az("network", "nic", "delete", "-g", RG, "-n", n["name"]) + print(f"[teardown] deleting nic {n['name']}") + + for p in json.loads(az("network", "public-ip", "list", "-g", RG, "-o", "json")): + if is_sweep_resource(p["name"]) and not p.get("ipConfiguration"): + az("network", "public-ip", "delete", "-g", RG, "-n", p["name"]) + print(f"[teardown] deleting public-ip {p['name']}") + + # `az vm create` also auto-creates one NSG per VM. Nothing deleted these, + # so they accumulated one-per-sweep (68 found on 2026-09-02, every one + # detached). They cost nothing, but they bury real resources in the RG + # and make "is this group clean?" unanswerable at a glance. Only ever + # delete NSGs that belong to a sweep VM AND are attached to nothing -- + # the shared vnet and the orca-eval-base-* image must survive. + for g in json.loads(az("network", "nsg", "list", "-g", RG, "-o", "json")): + detached = not g.get("networkInterfaces") and not g.get("subnets") + if is_sweep_resource(g["name"]) and detached: + az("network", "nsg", "delete", "-g", RG, "-n", g["name"]) + print(f"[teardown] deleting nsg {g['name']}") + + left = { + "vms": len(json.loads(az("vm", "list", "-g", RG, "-o", "json"))), + "disks": len(json.loads(az("disk", "list", "-g", RG, "-o", "json"))), + "nics": len(json.loads(az("network", "nic", "list", "-g", RG, "-o", "json"))), + "pubips": len(json.loads(az("network", "public-ip", "list", "-g", RG, "-o", "json"))), + "sweep_nsgs": len( + [ + g + for g in json.loads(az("network", "nsg", "list", "-g", RG, "-o", "json")) + if is_sweep_resource(g["name"]) + ] + ), + } + print(f"[teardown] remaining: {left}") + return + + models = MODELS if args.models == "all" else [m.strip() for m in args.models.split(",")] + if args.shards < 1: + print("--shards must be >= 1", flush=True) + return 2 + # A "unit" is one VM's worth of work: (model, shard). shard is None at + # --shards 1 so unsharded run dirs and VM names stay byte-identical to + # every sweep before this flag existed. + if args.shards == 1: + units = [(m, None) for m in models] + else: + units = [(m, f"{i}/{args.shards}") + for m in models for i in range(1, args.shards + 1)] + # --only-shard: backfill one failed shard of a larger sweep without + # re-provisioning the passing shards. Must be used with --shards N so the + # shard string (and thus VM name + example stride) matches the original. + if args.only_shard is not None: + if args.shards == 1: + print("--only-shard requires --shards > 1", flush=True) + return 2 + keep = f"{args.only_shard}/{args.shards}" + units = [(m, s) for m, s in units if s == keep] + if not units: + print(f"--only-shard {keep}: no unit matched", flush=True) + return 2 + print(f"only-shard backfill: {keep}", flush=True) + print(f"sweep models ({len(models)}): {', '.join(models)}", flush=True) + # Fail before provisioning: a missing local asset otherwise surfaces as an + # scp error on every VM, after the whole farm is already booted and billing. + # 2026-09-08: ~/.elastic/eis-ccm-key.json was rotated server-side and never + # restored locally; the preflight missed it, so 24 VMs booted and every + # unit died at deploy with a per-VM scp 255. deploy() unconditionally scp's + # this file to every VM (run_model.sh exports it as KIBANA_EIS_CCM_API_KEY), + # so it is load-bearing for EIS sweeps, not optional. + _required = [GOLDEN_ENV_LOCAL, + os.path.expanduser("~/.elastic/eis-connectors-cache.json"), + os.path.expanduser("~/.elastic/eis-ccm-key.json")] + _absent = [p for p in _required if not os.path.isfile(p)] + if _absent: + print(f"PREFLIGHT FAILED: missing local assets: {_absent}", flush=True) + print("hint: restore the CCM key via vault (secret/kibana-issues/dev/" + "inference/kibana-eis-ccm) then relaunch.", flush=True) + return 2 + # Quota gate: refuse to launch a sweep that cannot fit in the region's + # remaining cores. Launching into a full quota does not fail fast -- it + # fails per-VM, hours in, with a misleading az-cli error (2026-09-06). + # SKIP_QUOTA_GATE=1 bypasses for deliberate over-subscription. + if os.environ.get("SKIP_QUOTA_GATE") != "1": + try: + used, limit = quota_snapshot() + except Exception as exc: + print(f"[quota] snapshot unavailable ({exc}); proceeding", flush=True) + else: + # Charge quota only for units that need a NEW VM. An existing VM + # for this unit -- running or deallocated -- is ALREADY counted in + # `used`, so billing it again double-counts and makes resuming onto + # a warm pool impossible: on 2026-09-08 a resume of 24 existing VMs + # was refused for "needing" 192 cores that those same VMs already + # held. (In this subscription deallocated VMs keep consuming family + # vCPU quota, so they cannot be assumed free.) + existing = { + v.get("name") + for v in json.loads(az("vm", "list", "-g", RG, "-o", "json")) + } + new_units = [u for u in units if vm_name(u[0], u[1]) not in existing] + reused = len(units) - len(new_units) + if reused: + print( + f"[quota] {reused} of {len(units)} unit(s) reuse an existing VM " + f"(already counted in quota); charging {len(new_units)} new VM(s)", + flush=True, + ) + ok, msg = quota_gate(len(new_units), used, limit, cores_per_vm()) + print(f"[quota] {msg}", flush=True) + if not ok: + return 2 + if args.shards > 1: + print(f"sharding: {args.shards} VMs/model -> {len(units)} VMs total", flush=True) + # One base per sweep, suffixed per shard in launch(). Computing the + # base inside launch() would stamp each shard with a different base + # and leave the slices unrelatable after the fact. + os.environ.setdefault( + "TEST_RUN_ID", f"sweep-{int(time.time())}{('-' + os.environ['VM_NAME_SUFFIX']) if os.environ.get('VM_NAME_SUFFIX') else ''}" + ) + print(f"run id base: {os.environ['TEST_RUN_ID']}", flush=True) + + # Provision + deploy in parallel (independent per VM); launches stay serial. + ips: dict[tuple, str] = {} + with ThreadPoolExecutor(max_workers=args.provision_workers) as pool: + # as_completed + try/except so one dead VM (eviction, sshd race) + # loses its cell instead of killing the whole sweep before launch. + futures = {pool.submit(prepare, model, shard): (model, shard) + for model, shard in units} + for fut in as_completed(futures): + unit = futures[fut] + model, shard = unit + try: + _, ip = fut.result() + except Exception as exc: + print(f"[skip] {_label(model, shard)}: prepare failed ({exc})", flush=True) + model_dir(model, shard=shard).mkdir(parents=True, exist_ok=True) + json.dump({"model": model, "shard": shard, "state": "FAIL", + "error": f"prepare: {exc}"}, + open(model_dir(model, shard=shard) / "status.json", "w")) + continue + ips[unit] = ip + model_dir(model, shard=shard).mkdir(parents=True, exist_ok=True) + json.dump({"ip": ip, "model": model, "shard": shard, "state": "booting"}, + open(model_dir(model, shard=shard) / "status.json", "w")) + + reused = {} + for unit, ip in ips.items(): + if ip in reused: + raise RuntimeError( + f"VM collision: {_label(*unit)} and {_label(*reused[ip])} both " + f"mapped to {ip}. Every unit must own its VM stack — two eval " + "stacks on one box OOM each other, corrupt local ES, and wedge SSH." + ) + reused[ip] = unit + + procs = [] + for model, shard in units: + if (model, shard) not in ips: + continue + ip = ips[(model, shard)] + procs.append((model, shard, ip, launch(ip, model, shard))) + print(f"[launch] {_label(model, shard)} @ {ip}", flush=True) + + print("\nAll launches issued. Waiting for completion + golden gate.", flush=True) + for model, shard, ip, p in procs: + p.wait() # returns when the quick "launch" ssh exits (immediately) + # Poll the detached VM-side markers. Cap at 6h (slow reasoning models + # at 240-min per-example timeouts can legitimately run for hours). + deadline = time.time() + 6 * 3600 + rc = None + while time.time() < deadline: + probe = subprocess.run( + ["ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", "-o", "ConnectTimeout=15", + "-i", SSH_KEY, f"{SSH_USER}@{ip}", + "if [ -f /tmp/unit.done ]; then echo done $(cat /tmp/unit.rc 2>/dev/null); fi"], + capture_output=True, text=True, timeout=60) + out = probe.stdout.strip() + if out.startswith("done"): + parts = out.split() + rc = int(parts[1]) if len(parts) > 1 and parts[1].lstrip("-").isdigit() else 1 + break + time.sleep(60) + if rc is None: + rc = 1 # timed out or markers unreadable — never green by default + result = check_golden(model, ip, shard) + # Expected size is derived inside check_golden from the live evaluator + # count on the VM (21 examples x (evaluators + 1 task doc) x reps). + expected_docs = result.get("expected", -1) + count = result.get("count", -1) + # Args-coverage probe: the whole POINT of this rerun wave is + # gen_ai.tool.call.arguments spans on the new executions. A doc-count + # gate alone would pass a run whose tracing regressed to arg-less + # spans (exactly the state the old board's 1,946 dark tool steps are + # in). Count tool spans carrying the field on this execution's + # traces; the canonical golden-side check is span-side. + args_probe = check_args_coverage(result.get("execution_id") or "", ip) + # Never green on unresolved numbers: count == expected == -1 would + # otherwise PASS a run that produced nothing (observed when the spec + # overlay missed tool_registration_check.ts and every run died at + # require time). + gate_kind = result.get("gate", "exact") + meets = ( + count >= expected_docs if gate_kind == "floor" else count == expected_docs + ) + state = ( + "PASS" + if rc == 0 + and not result.get("error") + and isinstance(count, int) + and count > 0 + and meets + and args_probe.get("with_args", -1) > 0 + else "FAIL" + ) + json.dump({"ip": ip, "model": model, "shard": shard, "state": state, + "docs": result.get("count", -1), "rc": rc, + "execution_id": result.get("execution_id"), + "args": args_probe, + "error": result.get("error")}, + open(model_dir(model, shard=shard) / "status.json", "w"), indent=2) + print(f"[done] {_label(model, shard)}: {state} docs={result.get('count', -1)}/{expected_docs}" + + f" args={args_probe.get('with_args', -1)}/{args_probe.get('tool_spans', -1)}" + + (f" ({result['error']})" if result.get("error") else ""), flush=True) + # Free the unit's cores as soon as its golden gate is settled, rather + # than at end-of-sweep. Two quota exhaustions on 2026-09-06 were caused + # by finished VMs idling at full cost while later units waited for + # capacity. Deallocating (not deleting) also leaves a pre-baked box + # that the next sweep starts in ~60-90s via the warm pool. + maybe_park_unit(model, shard) + + + # A sweep that skipped or failed every model must not look like a green + # run to its caller: report the count so CI and shell wrappers can gate. + # A sweep that skipped or failed any UNIT must not look green: with + # sharding a model is only complete when every one of its shards passed, + # so tally units. Tallying models here would let a dead shard — a missing + # third of the dataset — report success. + failed = [_label(m, s) for m, s in units if _model_state(m, s) != "PASS"] + if failed: + print(f"SWEEP FAILED: {len(failed)}/{len(units)} units did not pass: {failed}", flush=True) + return 1 + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/orca_vm/preflight_sweep.py b/scripts/orca_vm/preflight_sweep.py new file mode 100644 index 0000000000000..52c2b2be5c411 --- /dev/null +++ b/scripts/orca_vm/preflight_sweep.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Pre-flight gate for persona-matrix sweeps. + +Every check here corresponds to a failure that silently wasted a real sweep. +Run it BEFORE launching models; it exits non-zero and refuses the sweep rather +than letting a run produce unpublishable scores. + + python3 preflight_sweep.py --models eis-anthropic-claude-4-6-sonnet ... + +Checks +------ +1. judge-independence A model may not be its own judge. Self-judged scores are + dropped by `excludeSelfJudged`, so the run burns a VM and fills zero cells. + This is the 2026-08-29 incident: 294/294 docs exported, 0 cells gained. +2. golden-reachable The golden URL must return a live doc count. `EXPORT_EXIT=0` + against a wrong host is a false success — we shipped 546 docs into a + nonexistent cluster before noticing. +3. golden-index `.evaluation-scores` exactly; the `*` wildcard 404s on + serverless, which reads as "index missing". +""" +from __future__ import annotations + +import argparse +import json +import os +import ssl +import sys +import urllib.error +import urllib.request + +from model_ids import same_model, strip_connector_prefix + +GOLDEN_INDEX = ".evaluation-scores" + + +def _fail(check: str, msg: str) -> None: + print(f"FAIL [{check}] {msg}", file=sys.stderr) + + +def _ok(check: str, msg: str) -> None: + print(f" ok [{check}] {msg}") + + +def check_judge_independence(models: list[str], judge: str) -> bool: + """A model must never grade itself.""" + judge_c = strip_connector_prefix(judge) + clashes = [m for m in models if same_model(m, judge)] + if clashes: + _fail( + "judge-independence", + f"judge '{judge}' (canonical '{judge_c}') is also under test: {clashes}. " + "Its scores would be self-judged and dropped, filling zero cells. " + "Pass a different --judge for these models.", + ) + return False + _ok("judge-independence", f"judge '{judge_c}' is not among the {len(models)} model(s) under test") + return True + + +def check_golden(url: str, api_key: str) -> bool: + """Prove the golden cluster is real by reading a doc count, not a 200.""" + if not url or not api_key: + _fail("golden-reachable", "GOLDEN_ES_URL / GOLDEN_ES_API_KEY not set") + return False + + endpoint = f"{url.rstrip('/')}/{GOLDEN_INDEX}/_count" + req = urllib.request.Request(endpoint, headers={"Authorization": f"ApiKey {api_key}"}) + ctx = ssl.create_default_context() + try: + with urllib.request.urlopen(req, timeout=30, context=ctx) as resp: + body = json.loads(resp.read().decode()) + except urllib.error.HTTPError as exc: + _fail("golden-reachable", f"HTTP {exc.code} from {endpoint} — wrong host or bad key") + return False + except Exception as exc: # noqa: BLE001 - any transport error is a hard fail + _fail("golden-reachable", f"cannot reach {endpoint}: {exc}") + return False + + count = body.get("count") + if not isinstance(count, int): + _fail("golden-reachable", f"no doc count in response: {body}") + return False + if count == 0: + _fail("golden-reachable", f"{GOLDEN_INDEX} exists but is empty — suspect wrong cluster") + return False + _ok("golden-reachable", f"{GOLDEN_INDEX} live with {count:,} docs") + return True + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--models", nargs="+", required=True, help="connector ids under test") + parser.add_argument("--judge", required=True, help="EVAL_CONNECTOR_ID used to grade") + parser.add_argument("--skip-golden", action="store_true", help="offline check of judge only") + args = parser.parse_args() + + print(f"Pre-flight: {len(args.models)} model(s), judge '{args.judge}'") + results = [check_judge_independence(args.models, args.judge)] + if not args.skip_golden: + results.append(check_golden(os.environ.get("GOLDEN_ES_URL", ""), os.environ.get("GOLDEN_ES_API_KEY", ""))) + + if not all(results): + print("\nPRE-FLIGHT FAILED — sweep refused. Fix the above before spending VM time.", file=sys.stderr) + return 1 + print("\nPRE-FLIGHT PASSED") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/orca_vm/probe_judge_routes.py b/scripts/orca_vm/probe_judge_routes.py new file mode 100644 index 0000000000000..dc863ca4ed992 --- /dev/null +++ b/scripts/orca_vm/probe_judge_routes.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Probe judge candidates through 9router with a REAL rubric call. + +Models vanish from the router catalog mid-run; a probe is a real 1-cell call +through the final code path (same parser, same auth) before a panel commits +to a judge list. Prints OK/FAIL + latency per candidate. + +Env: JUDGE_BASE_URL, JUDGE_API_KEY (see .selfhost-judge.env for the shape). +""" +import json +import os +import re +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import rejudge_rubric_png10 as score_png10 # noqa: E402 + +env = {} +# Key material lives OUTSIDE the repo (.selfhost-judge.env is gitignored); +# fall back to sibling path so the probe works from any checkout. +ENV_CANDIDATES = [ + os.path.join(os.path.dirname(os.path.abspath(__file__)), ".selfhost-judge.env"), + os.path.expanduser("~/.elastic/.selfhost-judge.env"), +] +env_path = next((p for p in ENV_CANDIDATES if os.path.exists(p)), None) +if not env_path: + sys.exit("no .selfhost-judge.env found (sibling or ~/.elastic/) — probe needs judge creds") +with open(env_path) as fh: + for line in fh: + m = re.match(r"""\s*export\s+(\w+)=["']?(.+?)["']?\s*$""", line) + if m: + env[m.group(1)] = m.group(2) + +os.environ["JUDGE_BASE_URL"] = env["SELFHOST_UPSTREAM"] +os.environ["JUDGE_API_KEY"] = env["SELFHOST_API_KEY"] +import importlib +importlib.reload(score_png10) + +CANDIDATES = [ + "openrouter/google/gemini-3.1-pro-preview", + "claude/claude-sonnet-5", + "cu/gpt-5.5-none", + "opencode/qwen3.6-plus", + "omni-opus-5", +] + +probe_rec = { + "question": "An alert fired on srv-web-01 for repeated failed logins from one external IP. What's going on and what should I do?", + "attachment": None, + "steps": [{"type": "tool_call", "tool_id": "alerts_lookup"}], + "answer_text": "This is likely a brute-force login attempt targeting srv-web-01. The IP generated repeated authentication failures, consistent with credential-stuffing. Recommended: block the IP at the edge firewall, check for any successful logins from the same IP in the last 24h, and reset credentials if a success is found.", +} + +print(f"endpoint: {score_png10.BASE_URL}") +for j in CANDIDATES: + res = score_png10.call_judge(j, score_png10.build_prompt(probe_rec), 90) + if res.get("ok"): + s = res["scores"] + print(f" OK {j:42} overall={s['overall']:.1f} dims=({s['correctness']:.0f},{s['groundedness']:.0f},{s['completeness']:.0f},{s['actionability']:.0f}) lat={res['latency_s']}s") + else: + print(f" FAIL {j:42} {res.get('error', '')[:90]}") diff --git a/scripts/orca_vm/reference_board_style.css b/scripts/orca_vm/reference_board_style.css new file mode 100644 index 0000000000000..a399686e591d3 --- /dev/null +++ b/scripts/orca_vm/reference_board_style.css @@ -0,0 +1,110 @@ + \ No newline at end of file diff --git a/scripts/orca_vm/rejudge_rubric_png10.py b/scripts/orca_vm/rejudge_rubric_png10.py new file mode 100644 index 0000000000000..6a9318fb12c1f --- /dev/null +++ b/scripts/orca_vm/rejudge_rubric_png10.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Score frozen transcripts on a 0-10 rubric (PNG-reference-compatible). + +Reads a frozen transcripts JSONL (extract_transcripts.py shape), calls judge +models through 9router (OpenAI-compatible), and writes one row per +(cell, judge) with overall + 4 dims. Balanced-object parser with truncation +repair (gpt-5.5 finish=stop mid-rationale mode), retries, env-keyed auth. + +Env: + JUDGE_BASE_URL e.g. https://9router.widzimysie.pl + JUDGE_API_KEY router key (never committed) +Usage: + python3 rejudge_rubric_png10.py --frozen frozen/transcripts.jsonl \ + --out results/png10.jsonl [--judges j1,j2] [--workers 8] [--limit N] +""" +from __future__ import annotations + +import argparse +import json +import os +import random +import re +import subprocess +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor + +BASE_URL = os.environ.get("JUDGE_BASE_URL", "https://9router.widzimysie.pl").rstrip("/v1") +API_KEY = os.environ.get("JUDGE_API_KEY", "") + +DEFAULT_JUDGES = ["omni-opus-5"] + +MAX_TOKENS = 2048 + +RUBRIC = """You are grading an AI security assistant's answer to a security analyst's question. + +Score ONE overall number from 0 to 10 (one decimal allowed): +- overall: how well the assistant answered the question, weighed across correctness, groundedness, completeness, and actionability. + +Also score each dimension from 0 to 100 (integers): +- correctness: factual accuracy of claims about the alert/data +- groundedness: claims supported by the tools/evidence available, no invention +- completeness: covers what the question actually asked +- actionability: gives a clear, usable recommendation + +Return ONLY a JSON object, no prose, no code fence: +{"overall": <0-10>, "correctness": <0-100>, "groundedness": <0-100>, "completeness": <0-100>, "actionability": <0-100>, "rationale": ""}""" + +_print_lock = threading.Lock() + + +def build_prompt(rec: dict) -> str: + parts = [f"# Question\n{rec['question']}"] + if rec.get("attachment"): + parts.append(f"# Attached context\n{rec['attachment']}") + steps = rec.get("steps") + if isinstance(steps, list) and steps: + tools = [s.get("tool_id") or s.get("toolId") or s.get("type") + for s in steps if isinstance(s, dict) and s.get("type") == "tool_call"] + if tools: + parts.append("# Tools the assistant called\n" + ", ".join(str(t) for t in tools[:40])) + parts.append(f"# Assistant's answer\n{rec['answer_text']}") + return "\n\n".join(parts) + + +def parse_scores(text: str): + if not text: + return None + cleaned = re.sub(r"^```(?:json)?|```$", "", text.strip(), flags=re.M).strip() + candidates = [cleaned] + depth = 0 + start = None + in_str = False + esc = False + for idx, ch in enumerate(cleaned): + if in_str: + if esc: + esc = False + elif ch == "\\": + esc = True + elif ch == '"': + in_str = False + continue + if ch == '"': + in_str = True + elif ch == "{": + if depth == 0: + start = idx + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0 and start is not None: + candidates.append(cleaned[start:idx + 1]) + start = None + for cand in candidates: + try: + obj = json.loads(cand, strict=False) + except json.JSONDecodeError: + # Truncated tail (finish=stop mid-rationale, seen on gpt-5.5-none): + # the numeric fields survive before the cut. Repair by closing the + # string/object naively and retrying once. + repaired = cand + if '"rationale"' in repaired: + repaired = repaired.split('"rationale"')[0] + '"rationale": ""}' + try: + obj = json.loads(repaired, strict=False) + except json.JSONDecodeError: + continue + if not isinstance(obj, dict): + continue + ov = obj.get("overall") + dims = {} + ok = isinstance(ov, (int, float)) and not isinstance(ov, bool) and 0 <= float(ov) <= 10 + if ok: + for dim in ("correctness", "groundedness", "completeness", "actionability"): + v = obj.get(dim) + if isinstance(v, (int, float)) and not isinstance(v, bool): + dims[dim] = float(v) + else: + ok = False + break + if ok: + return {"overall": float(ov), **dims, + "rationale": str(obj.get("rationale", ""))[:300]} + return None + + +def call_judge(judge: str, prompt: str, timeout: int, retries: int = 3, key: str = ""): + body = { + "model": judge, + "messages": [{"role": "system", "content": RUBRIC}, + {"role": "user", "content": prompt}], + "max_tokens": MAX_TOKENS, + "temperature": 0, + } + last = "no attempt" + for attempt in range(retries): + started = time.time() + cmd = ["curl", "-s", "-m", str(timeout), "-H", "Content-Type: application/json"] + if key or API_KEY: + cmd += ["-H", f"Authorization: Bearer {key or API_KEY}"] + cmd += [f"{BASE_URL}/v1/chat/completions", "-d", json.dumps(body)] + proc = subprocess.run(cmd, capture_output=True, text=True) + elapsed = time.time() - started + try: + payload = json.loads(proc.stdout) + except json.JSONDecodeError: + last = f"non-JSON: {proc.stdout[:120]}" + time.sleep(2 ** attempt + random.random()) + continue + if "error" in payload: + last = f"api error: {str(payload['error'])[:120]}" + time.sleep(2 ** attempt + random.random()) + continue + choice = (payload.get("choices") or [{}])[0] + content = (choice.get("message") or {}).get("content") + scores = parse_scores(content or "") + if scores is None: + last = f"unparseable (finish={choice.get('finish_reason')}): {str(content)[:120]}" + time.sleep(2 ** attempt + random.random()) + continue + usage = payload.get("usage") or {} + return {"ok": True, "scores": scores, "latency_s": round(elapsed, 2), + "output_tokens": usage.get("completion_tokens")} + return {"ok": False, "error": last} + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--frozen", default="./frozen_v2/transcripts.jsonl") + ap.add_argument("--out", default="./results/png10.jsonl") + per = ap.add_mutually_exclusive_group() + per.add_argument("--judges", default=",".join(DEFAULT_JUDGES)) + ap.add_argument("--limit", type=int, default=0) + ap.add_argument("--workers", type=int, default=8) + ap.add_argument("--timeout", type=int, default=180) + ap.add_argument("--seed", type=int, default=1788) + args = ap.parse_args() + + judges = [j.strip() for j in args.judges.split(",") if j.strip()] + recs = [json.loads(l) for l in open(args.frozen)] + recs = [r for r in recs if not r.get("empty_answer")] + recs.sort(key=lambda r: (r["model"], r["example_id"])) + if args.limit: + random.Random(args.seed).shuffle(recs) + recs = recs[:args.limit] + recs.sort(key=lambda r: (r["model"], r["example_id"])) + + tasks = [(r, j) for r in recs for j in judges] + print(f"cells={len(recs)} judges={len(judges)} calls={len(tasks)}", flush=True) + + os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True) + done = {"n": 0, "ok": 0, "fail": 0} + out_fh = open(args.out, "w") + _lock = threading.Lock() + + def run(task): + rec, judge = task + res = call_judge(judge, build_prompt(rec), args.timeout) + row = {"model": rec["model"], "example_id": rec["example_id"], + "judge": judge, "res": res} + with _lock: + out_fh.write(json.dumps(row) + "\n") + out_fh.flush() + done["n"] += 1 + done["ok" if res["ok"] else "fail"] += 1 + if done["n"] % 50 == 0: + print(f" {done['n']}/{len(tasks)} ok={done['ok']} fail={done['fail']}", flush=True) + + with ThreadPoolExecutor(max_workers=args.workers) as ex: + list(ex.map(run, tasks)) + out_fh.close() + print(f"DONE cells={len(recs)} calls={len(tasks)} ok={done['ok']} fail={done['fail']}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/orca_vm/render_agent_eval_full.py b/scripts/orca_vm/render_agent_eval_full.py new file mode 100644 index 0000000000000..b1a5a54f2ccb2 --- /dev/null +++ b/scripts/orca_vm/render_agent_eval_full.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +"""Render agent_eval_full-2.html from golden: TRACES_JSON + score extract. + +Recreates the reference board's shape: + * scoreboard: model x 21 prompt-ids, per-cell status + steps + tokens + * per-model sections with
prompt cards: reasoning steps, + tool calls WITH args (from gen_ai.tool.call.arguments spans), + final answer + * disclosure block: provenance, includeToolDetails evidence, missing + models, per-cell coverage gaps. Zero hand-edited numbers. + +Reference: ~/.hermes/attachments/agent_eval_full-2.html (33 models, +693 prompt cards, generated 2026-08-26 from run JSONL). This renderer +draws from golden ES instead, so every cell traces to a golden doc. +""" +import argparse +import datetime +import html +import json +import os + + +PROMPT_IDS = [ + "alert-analysis-a", "alert-analysis-b", "alert-analysis-c", + "entity-analytics-a", "entity-analytics-b", "entity-analytics-c", + "threat-hunting-a", "threat-hunting-b", "threat-hunting-c", + "detection-rule-edit-a", "detection-rule-edit-b", "detection-rule-edit-c", + "workflow-authoring-a", "workflow-authoring-b", "workflow-authoring-c", + "workflow-execution-a", "workflow-execution-b", "workflow-execution-c", + "multi-step-a", "multi-step-b", "multi-step-c", +] + +# Reference board's 33 models, in the reference's exact row order (parsed from +# the original agent_eval_full-3.html). Those without golden coverage render as +# an explicit "no connector" row -- never imputed, never dropped silently. +REFERENCE_MODELS = [ + "anthropic-claude-4.5-haiku", + "anthropic-claude-4.5-opus", + "anthropic-claude-4.6-opus", + "anthropic-claude-4.7-opus", + "anthropic-claude-4.8-opus", + "anthropic-claude-5-opus", + "gp-llm-v2", + "anthropic-claude-4.6-sonnet", + "anthropic-claude-5-sonnet", + "deepseek/deepseek-v4-pro", + "google/gemma-4-31b-it", + "google-gemini-2.5-flash", + "google-gemini-2.5-flash-lite", + "google-gemini-2.5-pro", + "google-gemini-3.0-flash", + "google-gemini-3.1-flash-lite", + "google-gemini-3.1-pro", + "google-gemini-3.5-flash", + "google-gemini-3.5-flash-lite", + "google-gemini-3.6-flash", + "moonshotai/kimi-k2.6", + "openai-gpt-5.2", + "openai-gpt-5.4", + "openai-gpt-5.4-mini", + "openai-gpt-5.4-nano", + "openai-gpt-5.5", + "openai-gpt-5.6-luna", + "openai-gpt-5.6-sol", + "openai-gpt-5.6-terra", + "openai-gpt-oss-120b", + "openai-gpt-oss-20b", + "Qwen36_27b", + "zai-glm-5-2", +] + +MISSING_REASONS = { + "anthropic-claude-5-opus": "no EIS connector exists", + "google-gemini-3.5-flash-lite": "no EIS connector exists", + "google-gemini-3.6-flash": "no EIS connector exists", + "google/gemma-4-31b-it": "no EIS connector exists", + "moonshotai/kimi-k2.6": "no EIS connector exists", + "openai-gpt-5.6-luna": "no EIS connector exists", + "openai-gpt-5.6-terra": "no EIS connector exists", + "Qwen36_27b": "no EIS connector exists", + "zai-glm-5-2": "connector blocked by upstream issue #288469", + # 2026-09-14: google-gemini-2.5-flash-lite and gp-llm-v2 were re-run by the + # Azure wave (630 docs each, Sep 13-14) and are covered rows now; their old + # "broken/skip" entries were removed. Do not re-add without checking golden. +} + +# Golden model ids mix two spellings: EIS connector runs emit dash-separated +# ids (anthropic-claude-4-5-haiku), older/local runs emit dotted ids that match +# the reference board (anthropic-claude-4.5-haiku). Normalise both to the +# reference spelling so one model cannot appear as two rows. +ALIASES = { + "anthropic-claude-4-5-haiku": "anthropic-claude-4.5-haiku", + "anthropic-claude-4-5-opus": "anthropic-claude-4.5-opus", + "anthropic-claude-4-6-opus": "anthropic-claude-4.6-opus", + "anthropic-claude-4-6-sonnet": "anthropic-claude-4.6-sonnet", + "anthropic-claude-4-7-opus": "anthropic-claude-4.7-opus", + "anthropic-claude-4-8-opus": "anthropic-claude-4.8-opus", + "google-gemini-2-5-flash": "google-gemini-2.5-flash", + "google-gemini-2-5-flash-lite": "google-gemini-2.5-flash-lite", + "google-gemini-2-5-pro": "google-gemini-2.5-pro", + "google-gemini-3-1-pro": "google-gemini-3.1-pro", + "openai-gpt-5-2": "openai-gpt-5.2", + "openai-gpt-5-4": "openai-gpt-5.4", + "openai-gpt-5-4-mini": "openai-gpt-5.4-mini", + "openai-gpt-5-4-nano": "openai-gpt-5.4-nano", + "openai-gpt-5-5": "openai-gpt-5.5", + "deepseek/deepseek-v4-pro-0813": "deepseek/deepseek-v4-pro", + "openai/gpt-5.6-sol": "openai-gpt-5.6-sol", +} + + +def esc(s): + return html.escape(str(s)) if s is not None else "" + + +def cell_status(cell): + if cell is None: + return "blank" + if cell.get("answer"): + return "ok" + return "partial" + + +def render(traces, out, since, extra_missing=None, traces_sha=None): + cells_raw = traces["cells"] + # normalise model ids to reference spelling (see ALIASES) + cells = {} + for k, v in cells_raw.items(): + m, p = k.split(":", 1) + cells[f"{ALIASES.get(m, m)}:{p}"] = v + meta = traces["meta"] + covered = sorted({m for m, _ in (k.split(":", 1) for k in cells)}) + ts = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + + n_cells = sum(1 for m in REFERENCE_MODELS for p in PROMPT_IDS if cells.get(f"{m}:{p}")) + n_args = sum(1 for c in cells.values() for s in (c.get("steps") or []) + if s.get("type") == "tool" and s.get("toolParams")) + n_tool = sum(1 for c in cells.values() for s in (c.get("steps") or []) + if s.get("type") == "tool") + n_ans = sum(1 for c in cells.values() if c.get("answer")) + n_usage_cells = sum(1 for c in cells.values() + if (c.get("usage") or {}).get("durNs") + or (c.get("usage") or {}).get("inTok") + or (c.get("usage") or {}).get("outTok")) + + # one row per reference model, in reference order — data row or missing-row + all_rows = [] + for m in REFERENCE_MODELS: + if m not in covered: + all_rows.append(f'{esc(m)}{esc(MISSING_REASONS.get(m, "not in this run window"))}') + continue + tds = [] + for p in PROMPT_IDS: + c = cells.get(f"{m}:{p}") + if not c: + tds.append('—') + continue + steps = c.get("stepCount") or 0 + ans = "✓" if c.get("answer") else "△" + n_ev = len(c.get("scores") or {}) + u = c.get("usage") or {} + if u.get("durNs") or u.get("inTok") or u.get("outTok"): + secs = (u.get("durNs") or 0) / 1e9 + usage_txt = f"{secs:.0f}s · {u.get('inTok', 0)}/{u.get('outTok', 0)} tok" + cls = "usage" + else: + usage_txt = "" + cls = "" + tds.append( + f'{ans} {steps} steps{usage_txt}' + ) + all_rows.append(f'{esc(m)}{"".join(tds)}') + + # per-model sections + sections = [] + for m in REFERENCE_MODELS: + if m not in covered: + continue + cards = [] + for p in PROMPT_IDS: + c = cells.get(f"{m}:{p}") + if c is None: + continue + steps_html = [] + for s in c.get("steps") or []: + if s.get("type") == "reasoning": + steps_html.append( + f'
think{esc((s.get("text") or "")[:600])}
' + ) + elif s.get("type") == "tool": + args = s.get("toolParams") + if args is None: + arg_html = '(args not captured)' + else: + arg_html = f'{esc(json.dumps(args)[:400])}' + steps_html.append( + f'
⚙{esc(s.get("toolId"))}{arg_html}
' + ) + elif s.get("type") == "skill": + steps_html.append( + f'
skill{esc(s.get("skills"))}
' + ) + answer = c.get("answer") + ans_html = ( + f'
{esc(answer[:2000])}
' if answer + else '
no final answer recorded
' + ) + cards.append( + f'''
{esc(p)} +{c.get("stepCount", 0)} steps +
{''.join(steps_html) or '
(no steps recorded)
'}
+{ans_html}
''' + ) + if cards: + sections.append( + f'

{esc(m)}

' + f'{len(cards)}/{len(PROMPT_IDS)} prompts
{"".join(cards)}
' + ) + + html_out = f""" + +Agent Builder Skill Eval — EIS Models (golden recreation) +
+

Agent Builder Skill Eval — EIS Models (golden recreation)

+
Recreated from golden ES (security-persona-matrix suite score docs + OTel traces) · window since {esc(since)} · rendered {ts}{(' · extract sha256 ' + traces_sha) if traces_sha else ''}
+ +
+

Provenance & honesty disclosures

+
    +
  • Every cell traces to golden .ds-.evaluation-scores* docs (suite security-persona-matrix) joined with traces-agent_builder.otel-default spans. Zero hand-edited numbers.
    • +
  • Tool args: {n_args}/{n_tool} tool steps carry gen_ai.tool.call.arguments — captured via --uiSettings.overrides.agentBuilder:tracing:includeToolDetails=true in Kibana boot args (grep-verified in the runs' scout logs). Steps without args render an explicit "(args not captured)" marker — never invented. Historical runs predate this flag (383,098 tool steps, 100% null args).
    • +
  • Per-cell latency/tokens ("Xs · Y/Z tok"): summed from gen_ai.usage.*_tokens + duration on LLM spans, joined by trace_id to the cell's executions ({n_usage_cells} cells carry usage; cells with no usage spans in window show steps only — never imputed).
    • +
  • Cells: {n_cells} of {len(REFERENCE_MODELS) * len(PROMPT_IDS)} reference-model cells have data. Final answers present in {n_ans} cells with data.
    • +
  • Multi-execution models (retry runs): each (model, prompt) cell renders the most-doc'd single execution — one real execution per cell, never a blend. Cells missing in a model's executions render blank with a "no score doc" tooltip.
    • +
  • openai-gpt-oss-20b: no single execution completed all 21 prompts (best single: 16/21; scatter across attempts). All 21 prompts have data only when taking the best execution per prompt — each cell is one real execution, and the selection is disclosed here rather than hidden.
    • +
  • Models listed below the scoreboard have no EIS connector (or are blocked) — disclosed, not back-filled.
    • +
  • The reference board (2026-08-26, 33 models, 693 cards) was rendered from run JSONL; this recreation is from golden docs, so counts differ where golden coverage differs. Judge: eis-google-gemini-3-1-pro.
    • +
+
+ +{''.join(f'' for p in PROMPT_IDS)} +{''.join(all_rows)}
Model{esc(p)}
+
✓ = final answer present · △ = partial (no answer recorded) · — = no score doc in window
+ +{''.join(sections)} +
source: golden extract {esc(json.dumps(meta))}
+
""" + with open(out, "w") as fh: + fh.write(html_out) + print(f"wrote {out} ({len(html_out)} bytes, {len(covered)} models with data, {n_cells} cells)") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--traces", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--since", required=True) + args = ap.parse_args() + import hashlib + with open(args.traces, "rb") as fh: + traces_sha = hashlib.sha256(fh.read()).hexdigest()[:16] + traces = json.load(open(args.traces)) + render(traces, args.out, args.since, traces_sha=traces_sha) + + +if __name__ == "__main__": + main() diff --git a/scripts/orca_vm/render_attack_discovery_board.py b/scripts/orca_vm/render_attack_discovery_board.py new file mode 100644 index 0000000000000..fdfc3e5af5e03 --- /dev/null +++ b/scripts/orca_vm/render_attack_discovery_board.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +"""Render the attack-discovery board from the golden ES aggregate. + +Rules enforced here (the point of the script, not decoration): +* Every number comes from the aggregate JSON; nothing is hardcoded or imputed. +* A field with no source renders as a visible blank marker, never a guess. +* Coverage (n) rides next to each mean so a 1-doc mean cannot pose as many. +* Trace cards show the FINAL ANSWER (task.output.insights) per execution; + a board without them would hide what the model actually concluded. +* Refuses to emit if the aggregate is empty or every discovery cell is null. +""" +import argparse +import datetime +import html +import json +import sys + +BLANK = '--' + + +def cell(mean, n, digits=2): + if mean is None: + return BLANK + return f'{mean:.{digits}f}n={n}' + + +def status_badge(m): + counts = m["status"]["counts"] + if not counts: + return BLANK + if "failed" in counts: + return f'failed' + if counts.get("succeeded"): + return f'succeeded' + return html.escape("/".join(sorted(counts.keys()))) + + +def build_rows(models): + out = [] + for m in sorted(models, key=lambda r: (r["discoveryCount"]["mean"] is None, -(r["discoveryCount"]["mean"] or 0))): + lat = m.get("generateLatencySeconds") + rub = (m.get("evaluators") or {}).get("AttackDiscoveryRubric") or {} + errs = m.get("generateErrors") or [] + err_html = ( + f'
{"; ".join(html.escape(e) for e in errs[:1])}
' + if errs else "" + ) + out.append( + "" + f'{html.escape(m["modelId"])}' + f"{status_badge(m)}" + f'{cell(m["discoveryCount"]["mean"], m["discoveryCount"]["n"], 0)}' + f'{cell(m["alertsContextCount"]["mean"], m["alertsContextCount"]["n"], 0)}' + f'{f"{lat:.1f}s" if lat is not None else BLANK}' + f'{cell(rub.get("mean"), rub.get("n", 0))}' + f'{BLANK if m["totalRisk"] is None else m["totalRisk"]}' + f'{m["docs"]}' + f"{err_html and ''}" + "" + + (f'{err_html}' if err_html else "") + ) + return "\n".join(out) + + +def trace_cards(models): + """Per-model final-answer cards. Each card carries the execution id, the + OTel trace id, and every insight the model produced (title, risk, tactics, + summary excerpt). This is the answer itself, not a pointer to it.""" + blocks = [] + for m in models: + cards = m.get("traceCards") or [] + if not cards: + blocks.append( + f'
{html.escape(m["modelId"])} ' + f'— no final answer on golden (generation failed or no insights)
' + ) + continue + for t in cards: + items = [] + for i, ins in enumerate(t["insights"], 1): + tactics = ", ".join(ins.get("mitre_attack_tactics") or []) or "--" + risk = ins.get("risk_score") + summary = (ins.get("summary_markdown") or "").strip() + if len(summary) > 220: + summary = summary[:217] + "..." + items.append( + f'
  • {html.escape(str(ins.get("title") or f"insight {i}"))}' + f'risk {html.escape(str(risk))} · {html.escape(tactics)}' + f'
    {html.escape(summary)}
    • ' + ) + blocks.append( + f'
    {html.escape(m["modelId"])} — ' + f'{t["insightCount"]} insights ' + f'exec {html.escape(t["executionId"][:16])}… ' + f'trace {html.escape((t.get("traceId") or "")[:16])}…' + f'
      {" ".join(items)}
    ' + ) + return "\n".join(blocks) + + +CSS = """ +body{background:#0b0e14;color:#e6e6e6;font:14px/1.5 -apple-system,Segoe UI,Roboto,sans-serif;margin:0;padding:32px} +h1{font-size:22px;margin:0 0 4px} +.sub{color:#9aa4b2;margin-bottom:20px} +table{border-collapse:collapse;width:100%;max-width:1100px} +th,td{padding:8px 12px;text-align:left;border-bottom:1px solid #1e2430} +th{color:#9aa4b2;font-weight:600;font-size:12px;text-transform:uppercase;letter-spacing:.04em} +td.model{font-family:ui-monospace,Menlo,monospace;color:#7dd3fc} +.n{color:#5b6472;font-size:11px;margin-left:6px} +.na{color:#5b6472} +.ok{color:#4ade80}.warn{color:#fbbf24}.bad{color:#f87171} +.dim{color:#5b6472} +.err{color:#f87171;font-size:12px} +tr.errrow td{border-bottom:1px solid #1e2430;padding-top:0} +.disc{background:#141a24;border-left:3px solid #fbbf24;padding:14px 18px;margin:22px 0;max-width:1100px;border-radius:4px} +.disc h2{font-size:13px;margin:0 0 8px;text-transform:uppercase;color:#fbbf24;letter-spacing:.04em} +.disc li{margin:4px 0;color:#c7cdd6} +code{background:#1e2430;padding:1px 5px;border-radius:3px;font-size:12px} +h2.sec{font-size:14px;margin:26px 0 10px;color:#9aa4b2;text-transform:uppercase;letter-spacing:.04em} +table.sources td{font-size:13px;color:#c7cdd6;vertical-align:top} +table.sources td.src-col{color:#7dd3fc;font-weight:600;width:170px;white-space:nowrap} +tr.missing td{color:#5b6472;font-style:italic} +tr.missing td.model{color:#6b7480;font-style:normal} +.tcard{background:#141a24;border:1px solid #1e2430;border-radius:4px;margin:8px 0;max-width:1100px} +.tcard summary{padding:10px 14px;cursor:pointer;font-family:ui-monospace,Menlo,monospace;color:#7dd3fc;font-size:13px} +.tcard.empty{padding:10px 14px;color:#6b7480;font-size:13px} +.tcard .thead{font-family:ui-monospace,Menlo,monospace} +.ins{list-style:none;margin:0;padding:4px 14px 12px} +.ins li{margin:8px 0;padding-bottom:8px;border-bottom:1px solid #1e2430} +.ins .t{color:#e6e6e6;font-weight:600;display:block} +.ins .meta{color:#9aa4b2;font-size:12px;display:block;margin:2px 0} +.ins .sum{color:#c7cdd6;font-size:12px} +""" + + +# Criterion 1: every column states exactly where its number comes from, so a +# reader never has to guess whether a value was measured, derived, or missing. +COLUMN_SOURCES = [ + ("Model", "task.model.id from the golden score documents."), + ("Status", "task.output.raw.status — the product-level attack-discovery generate result."), + ("Discoveries", "count of task.output.insights (the final AD answer), once per execution."), + ("Alerts in context", "task.output.raw.alerts_context_count (input volume, 95-alert corpus)."), + ("Gen latency", "task.output.raw.latency_ms — the _generate call wall clock, ms → s."), + ("Rubric", "mean score of the AttackDiscoveryRubric evaluator (Gemini 3.1 Pro judge)."), + ("Total risk", "no per-board aggregate field exists in our schema; always blank."), + ("Docs", "count of scored documents contributing to that row."), +] + + +def render_missing_rows(missing): + """Reference/board-candidate models absent from our data are shown as + explicitly missing. Their cells stay blank -- never filled by hand.""" + if not missing: + return "" + cells = "\n".join( + "" + f'{html.escape(m["model"])}' + f'{html.escape(m["reason"])}' + "" + for m in missing + ) + return cells + + +def render(agg, source_note, missing_models=(), disclosure_extra=""): + rows = build_rows(agg["models"]) + absent = agg["absentFields"] + absent_html = ( + "".join(f"
  • {html.escape(f)} has no source field in our golden schema; " + "every cell renders --. Never imputed.
    • " for f in absent) + or "
  • All reference columns were recovered from our data.
    • " + ) + ts = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") + sources_html = "\n".join( + f"{name}{desc}" + for name, desc in COLUMN_SOURCES + ) + missing_html = render_missing_rows(missing_models) + cards = trace_cards(agg["models"]) + return f""" +Attack Discovery -- golden results +

    Attack Discovery — model board

    +
    {agg['modelCount']} models · {agg['sourceDocCount']} scored documents · +suite {html.escape(agg['suiteId'])} · rendered {ts}
    + +

    What this board is

      +
    • Real corpus, real product API. Every run restores the +oh-my-malware-95-deduped GCS snapshot (95 alerts) and drives the production +attack-discovery _generate API against it. No synthetic seeds, no scenario fixtures.
      • +
    • Judge: eis-google-gemini-3-1-pro (Gemini 3.1 Pro) via the +AttackDiscoveryRubric evaluator. The judge never scores its own generation +(it is excluded from the candidate set).
      • +
    • Each mean carries its own n. A model with one execution shows n=1 — +this is a single-shot sweep, not a multi-rep average.
      • +
    • Trace cards below the table carry the final answer per execution: every insight title, +risk, MITRE tactics and summary excerpt, straight from task.output.insights.
      • +{disclosure_extra} +{absent_html} +
    • Source: {html.escape(source_note)}
      • +
    + +

    Where each column comes from

    + +{sources_html} +
    + +

    Results

    + + + + +{rows} +{missing_html} +
    ModelStatusDiscoveriesAlerts in contextGen latencyRubricTotal riskDocs
    + +

    Final answers — trace cards per execution

    +{cards} +""" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--aggregate", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--missing", help="JSON list of {model, reason} rows to show as explicitly absent") + args = ap.parse_args() + + agg = json.load(open(args.aggregate)) + if not agg.get("models"): + sys.exit("aggregate has no models -- refusing to render an empty board") + if all(m["discoveryCount"]["mean"] is None for m in agg["models"]): + sys.exit("no model carries a discoveryCount -- refusing to render a vacuous board") + + missing_models = [] + if args.missing: + missing_models = json.load(open(args.missing)) + if not isinstance(missing_models, list) or not all( + isinstance(x, dict) and "model" in x and "reason" in x for x in missing_models + ): + sys.exit("--missing must be a JSON list of {model, reason} objects") + + html_out = render( + agg, + f"golden ES, suite_id={agg['suiteId']}, {agg['sourceDocCount']} docs", + missing_models=missing_models, + ) + with open(args.out, "w") as fh: + fh.write(html_out) + print(f"wrote {args.out} ({len(html_out)} bytes, {agg['modelCount']} models, " + f"{len(missing_models)} listed as missing)") + + +if __name__ == "__main__": + main() diff --git a/scripts/orca_vm/render_attack_discovery_eis_trial.py b/scripts/orca_vm/render_attack_discovery_eis_trial.py new file mode 100644 index 0000000000000..6f6fdc0d230b0 --- /dev/null +++ b/scripts/orca_vm/render_attack_discovery_eis_trial.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +"""Render attack_discovery_results-3.html ("EIS Model Trial") from a golden extract. + +Replicates the reference artifact's SHAPE (model table: status/discoveries/ +alerts-in-context/latency/total-risk + per-model discovery cards). Every value +traces to a golden extract doc; the model list/order comes from parsing the +reference HTML itself (--reference). Zero hand-edited numbers, zero imputation. + +Classification (extract + connectors cache at render time, never guessed): + have-data model has extract rows + failed have-data but zero insights (honest low row, never dropped) + broken connector exists, model considered broken at render time + (2026-09-11 directive: skip top-up, disclose, don't chase gates) + no-connector reference model with no EIS connector — structurally impossible +""" +import argparse +import datetime +import html +import json +import os +import re +import sys + +# ── Reference parsing ──────────────────────────────────────────────────────── +# Model list + display order + display names come from the reference artifact +# itself, parsed at render time — never hand-maintained. +ROW_RE = re.compile( + r'([^<]*)
    \s*([^<]+)', re.S) + + +def load_reference_models(reference_html_path): + with open(reference_html_path, encoding="utf-8") as fh: + text = fh.read() + pairs = ROW_RE.findall(text) + seen, models = set(), [] + for display, model_id in pairs: + if model_id in seen: + continue + seen.add(model_id) + models.append((model_id, display.strip())) + return models + + +def eis_connector_id(model_id): + """Reference id -> eis- connector id. Vendor-prefixed ids (deepseek/..., + google/gemma-..., moonshotai/...) have no EIS equivalent; keep verbatim so + the connectors-cache lookup below classifies them no-connector.""" + if "/" in model_id: + return model_id + return "eis-" + model_id.replace(".", "-") + + +BROKEN_MODELS = { + # 2026-09-11 user directive: the four top-up gaps are considered broken at + # render time. Skip top-up runs; disclose; don't chase gates. + "eis-gp-llm-v2", + "eis-google-gemini-2-5-flash", + "eis-google-gemini-2-5-flash-lite", + "eis-zai-glm-5-2", +} + + +def classify(reference_models, extract, connectors_path): + """Return ordered rows: (model_id, display, conn_id, status, extract_row). + + Status precedence: have-data/failed > broken > no-connector. + """ + cache = json.load(open(connectors_path, encoding="utf-8")) + conns = cache.get("connectors", cache) + have = set(conns.keys()) if isinstance(conns, dict) else { + c.get("id") for c in conns} + + by_conn = {m["modelId"]: m for m in extract["models"]} + rows = [] + for model_id, display in reference_models: + conn_id = eis_connector_id(model_id) + if conn_id in by_conn: + row = by_conn[conn_id] + counts = (row.get("status") or {}).get("counts") or {} + status = ("failed" if counts.get("failed") and not counts.get("succeeded") + else "have-data") + elif conn_id in BROKEN_MODELS: + row, status = None, "broken" + elif conn_id not in have: + row, status = None, "no-connector" + else: + row, status = None, "not-run" + rows.append((model_id, display, conn_id, status, row)) + return rows + + +def fmt_latency(row): + lat = row.get("generateLatencySeconds") + if lat is None: + return "--" + return f"{lat/60:.1f}m" + + +def total_risk(row): + return sum(i.get("risk_score") or 0 + for t in (row.get("traceCards") or []) + for i in t["insights"]) + + +def render_table_rows(rows): + out = [] + for model_id, display, conn_id, status, row in rows: + if status in ("have-data", "failed"): + n = (row.get("discoveryCount") or {}).get("mean", 0) + alerts = (row.get("alertsContextCount") or {}).get("mean") + alerts_txt = f"{alerts:.0f}" if alerts is not None else "--" + lat = fmt_latency(row) + risk = total_risk(row) + risk_txt = f"{risk:.0f}" if risk else "--" + badge = "ok" if status == "have-data" else "failed" + status_txt = "OK" if status == "have-data" else "ERROR" + disc_txt = f"{n:g} discoveries" + else: + alerts_txt = lat = risk_txt = "--" + badge = status + status_txt = {"broken": "BROKEN (skipped)", + "no-connector": "NO CONNECTOR", + "not-run": "NOT RUN"}[status] + disc_txt = "--" + out.append( + f'{html.escape(display)}
    ' + f'{html.escape(conn_id)}' + f'{status_txt}' + f'{disc_txt}' + f'{alerts_txt}' + f'{lat}' + f'{risk_txt}') + return "\n".join(out) + + +def render_cards(rows): + out = [] + for model_id, display, conn_id, status, row in rows: + if status != "have-data": + continue + cards = row.get("traceCards") or [] + if not cards: + continue + body = [] + exec_note = (" (2 executions — union)" + if len(cards) > 1 else "") + for t in cards: + for ins in t["insights"]: + tactics = ", ".join(ins.get("mitre_attack_tactics") or []) + title = html.escape(ins.get("title") or "(untitled)") + summary = html.escape(ins.get("summary_markdown") or "") + risk = ins.get("risk_score") + risk_html = (f'{risk}' + if risk is not None else "") + body.append( + f'

    {title} {risk_html}

    ' + f'
    {html.escape(tactics)}
    ' + f'

    {summary}

    ') + if body: + out.append( + f'
    {html.escape(display)}' + f' — {len(body)} discoveries{exec_note}' + + "".join(body) + "
    ") + return "\n".join(out) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--extract", required=True) + ap.add_argument("--reference", required=True) + ap.add_argument("--connectors", default=os.path.expanduser( + "~/.elastic/eis-connectors-cache.json")) + ap.add_argument("--out", required=True) + args = ap.parse_args() + + extract = json.load(open(args.extract, encoding="utf-8")) + reference_models = load_reference_models(args.reference) + rows = classify(reference_models, extract, args.connectors) + + counts = {} + for _, _, _, status, _ in rows: + counts[status] = counts.get(status, 0) + 1 + + now = datetime.datetime.now(datetime.timezone.utc) + broken_list = ", ".join(sorted(BROKEN_MODELS)) + doc = f""" + +Attack Discovery — EIS Model Trial + +

    Attack Discovery — EIS Model Trial

    +

    Suite: {html.escape(extract['suiteId'])} · golden extract + {extract['sourceDocCount']} docs · rendered {now:%Y-%m-%d %H:%M} UTC · + {counts.get('have-data', 0) + counts.get('failed', 0)}/{len(rows)} models with data

    +
    +

    Disclosure. Recreation of the reference "EIS Model Trial" board +(attack_discovery_results-3.html, 2026-08-26) from golden-cluster data: +suite {html.escape(extract['suiteId'])}, real oh-my-malware-95-deduped +GCS corpus (95 alerts restored per run) via the production _generate API. +All 8 evaluators per model. Every number on this board traces to a golden +extract document — zero hand-edited values, zero imputation. + +

    Model coverage: {counts.get('have-data', 0)} OK · +{counts.get('failed', 0)} error rows (ran, zero discoveries — kept, not dropped) · +{counts.get('broken', 0)} broken with no data · +{counts.get('no-connector', 0)} without EIS connectors (structurally impossible: +{counts.get('no-connector', 0)} of {len(rows)} reference models) · +{counts.get('not-run', 0)} not run.

    + +

    Broken at render time (2026-09-11 directive: top-up skipped, not +chased): {html.escape(broken_list)}. Of these, eis-google-gemini-2-5-flash has +a prior run in golden (zero insights) and is shown as an ERROR row above; the +other three have no golden data.

    + +

    Judge: the source sweep ran with eis-google-gemini-3-1-pro as +judge. Judge scores do not feed this board — all displayed numbers are raw +product output (insight counts, alert counts, _generate latency, insight risk +scores) read from golden.

    + +

    Latency is _generate wall-clock per execution (mean over executions); +Total Risk = sum of insight risk_score values across the model's +executions.

    +
    + + + + +{render_table_rows(rows)} +
    ModelStatusDiscoveriesAlerts in ContextLatencyTotal Risk
    +

    Discoveries by model

    +{render_cards(rows)} + +""" + with open(args.out, "w", encoding="utf-8") as fh: + fh.write(doc) + print(f"rendered {args.out}: {len(rows)} reference models, " + f"{counts.get('have-data', 0)} have-data, " + f"{counts.get('failed', 0)} failed, " + f"{counts.get('broken', 0)} broken, " + f"{counts.get('no-connector', 0)} no-connector, " + f"{counts.get('not-run', 0)} not-run") + + +if __name__ == "__main__": + main() + + + + diff --git a/scripts/orca_vm/render_false_green_bug_report.py b/scripts/orca_vm/render_false_green_bug_report.py new file mode 100644 index 0000000000000..7a70b1c3480d4 --- /dev/null +++ b/scripts/orca_vm/render_false_green_bug_report.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Render the connector false-green bug report from the measurement JSON. + +Every number comes from connector_false_green.json, so the report cannot +drift from the measurement it describes. +""" +import argparse +import html +import json + +CSS = """ +:root { color-scheme: dark; } +body { background:#0b0e14; color:#d7dce5; font:15px/1.65 -apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif; + max-width:900px; margin:0; padding:32px 28px; } +h1 { font-size:24px; margin:0 0 4px; color:#fff; } +h2 { font-size:17px; margin:30px 0 10px; color:#fff; border-bottom:1px solid #232833; padding-bottom:6px; } +.sub { color:#8b94a7; font-size:13px; margin-bottom:22px; } +code { background:#151a23; border:1px solid #232833; border-radius:4px; padding:1px 5px; + font:13px ui-monospace,SFMono-Regular,Menlo,monospace; color:#9ecbff; } +pre { background:#151a23; border:1px solid #232833; border-left:3px solid #4a9eff; border-radius:5px; + padding:12px 14px; overflow-x:auto; font:12.5px ui-monospace,SFMono-Regular,Menlo,monospace; color:#c8d1e0; } +table { border-collapse:collapse; width:100%; margin:14px 0; font-size:14px; } +th,td { border:1px solid #232833; padding:7px 10px; text-align:left; } +th { background:#151a23; color:#fff; font-weight:600; } +td.n { text-align:right; font-variant-numeric:tabular-nums; } +.bad { color:#ff7b72; font-weight:600; } +.ok { color:#7ee787; } +.callout { background:#1a1410; border-left:3px solid #d29922; border-radius:5px; padding:12px 16px; margin:16px 0; } +.callout.imp { background:#101a14; border-left-color:#3fb950; } +li { margin:5px 0; } +""" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--measurement", required=True) + ap.add_argument("--out", required=True) + args = ap.parse_args() + + d = json.load(open(args.measurement)) + scan, s, per = d["scan"], d["summary"], d["perModel"] + pct = 100.0 * s["falseGreen"] / max(1, s["expectedToolCalledPass"] + s["falseGreen"] - s["reallyTargetsConnector"]) + + rows = "\n".join( + f'{html.escape(r["model"])}' + f'{r["real"]}/{r["total"]}' + f'{r["rate"]:.1f}%' + for r in per + ) + + doc = f""" +ExpectedToolCalled scores tool identity, not tool effect + +

    ExpectedToolCalled scores tool identity, not tool effect

    +
    Agent Builder · workflow-authoring evaluation · +measured against golden ES over {scan['docsScanned']} scored documents
    + +

    Summary

    +

    Cells that ask a model to author a workflow targeting a Slack connector score +1.0 on ExpectedToolCalled whenever the model calls +generate_workflow — regardless of whether the workflow it produced +targets the connector, or any connector at all.

    + +
    +{s['falseGreen']} false greens. +{s['expectedToolCalledPass']} cells score 1.0, but only +{s['reallyTargetsConnector']} author a real +type: http step pointing at the expected connector id. +
    + +

    Root cause

    +

    The evaluator maps tool-call steps to their tool_id and asserts the +required ids appear. Arguments and output are never inspected:

    +
    export const getUsedToolIds = (output: TaskOutput): string[] =>
+  getToolCallSteps(output)
+    .map((toolCall) => toolCall.tool_id)
+    .filter((toolId): toolId is string => Boolean(toolId));
+
+const missingToolIds = requiredToolIds.filter((id) => !usedToolIds.includes(id));
+
+return {{ score: missingToolIds.length === 0 ? 1 : 0, ... }};
    +

    x-pack/platform/packages/shared/kbn-evals-suite-alerting-v2/src/evaluators/expected_tool_called.ts

    +

    This is correct for its stated purpose — it answers “was this tool +called?”. The defect is that workflow-authoring cells rely on it to answer +“did the model wire up the connector?”, which it cannot.

    + +

    Evidence

    +
      +
    • Positive control: {scan['positiveControlDocsMentioningSlack']} scanned +documents mention Slack, so the scan detects the signal it looks for. This is not a null instrument.
      • +
    • {scan['slackRelatedCells']} distinct Slack-related cells examined.
      • +
    • {s['cellsWithoutHttpStep']} cells produced no type: http step at all.
      • +
    • {s['cellsGivenIdButOmittedIt']} cell was handed a connector id and still omitted it.
      • +
    + +

    Per-model rate of actually targeting the connector

    + + +{rows} +
    ModelRealRate
    + +
    +Sampling caveat. {html.escape(scan['caveat'])} +Treat per-model rates as indicative of the gap, not as a certified leaderboard. +
    + +

    Suggested fix

    +

    A deterministic ConnectorInvoked evaluator that parses the authored +workflow YAML and asserts the expected connector id appears in a type: http +step. Implemented and unit-tested on +feat/evals-extensions-matrix-v3; removing the Slack step from the fixture +flips its score from 1 to 0, while ExpectedToolCalled stays at 1.0 — +which is precisely the gap.

    + +
    +Why this matters beyond one suite. Any capability scored by +“was the tool called” inherits this weakness. The tool-call step is evidence +of an attempt, not of an effect. Where the effect is what the capability claims, +the assertion has to read the artifact the model produced. +
    +""" + with open(args.out, "w") as fh: + fh.write(doc) + print(f"wrote {args.out} ({len(doc)} bytes, {len(per)} models)") + + +if __name__ == "__main__": + main() diff --git a/scripts/orca_vm/render_matrix_from_golden.sh b/scripts/orca_vm/render_matrix_from_golden.sh new file mode 100755 index 0000000000000..adc7b1bbf6596 --- /dev/null +++ b/scripts/orca_vm/render_matrix_from_golden.sh @@ -0,0 +1,41 @@ +#!/bin/zsh +# Drive the PR #285833 matrix pipeline end-to-end from the golden cluster. +# +# source ~/.elastic/golden-cluster-env.sh +# ./render_matrix_from_golden.sh [since=2026-09-01] [out=~/persona-sweep/matrix] +# +# Stages (all PR-owned components, no ad-hoc renderers): +# 1. extract_golden_aggregate.ts -> aggregated JSON (scores, policy-stamped) +# 2. build_trace_cache.py -> TRACES_JSON (transcripts + steps) +# 3. render_from_golden.ts -> the published matrix HTML +# Requires Node v24.21.0 (nvm use 24.21.0). +set -euo pipefail + +SINCE="${1:-2026-09-01}" +OUT="${2:-$HOME/persona-sweep/matrix}" +PKG=~/Projects/kibana.worktrees/evals-ext-matrix/x-pack/platform/packages/shared/kbn-evals-extensions +ORCA=~/Projects/kibana.worktrees/evals-ext-matrix/scripts/orca_vm + +source ~/.nvm/nvm.sh +nvm use 24.21.0 >/dev/null +: "${GOLDEN_ES_URL:?source ~/.elastic/golden-cluster-env.sh first}" +: "${GOLDEN_ES_API_KEY:?}" + +mkdir -p "$OUT" + +echo "== 1/3 extract aggregate (since $SINCE) ==" +( cd "$PKG" && SINCE="$SINCE" OUT_JSON="$OUT/aggregated.json" \ + node --require ../../../../../src/setup_node_env scripts/extract_golden_aggregate.ts ) + +echo "== 2/3 trace cache (340h window) ==" +( cd "$ORCA" && python3 build_trace_cache.py --hours 340 --out "$OUT/trace_cache.json" ) + +echo "== 3/3 render matrix ==" +( cd "$PKG" && AGGREGATED_JSON="$OUT/aggregated.json" \ + MATRIX_CONFIG="$PKG/config/security_matrix_persona.json" \ + TRACES_JSON="$OUT/trace_cache.json" \ + OUT_DIR="$OUT" \ + COMMIT_SHA="$(git -C ~/Projects/kibana.worktrees/evals-ext-matrix rev-parse HEAD)" \ + node --require ../../../../../src/setup_node_env scripts/render_from_golden.ts ) + +echo "done -> $OUT" diff --git a/scripts/orca_vm/render_reference_shape.py b/scripts/orca_vm/render_reference_shape.py new file mode 100644 index 0000000000000..3d45a0341bac0 --- /dev/null +++ b/scripts/orca_vm/render_reference_shape.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Render OUR golden extract in the reference board's exact shape. + +Emits the agent_eval_full-4 layout (summary matrix + per-model cards with +prompt cards, step cards, tool trails, answers) from the same traces JSON +build_traces_json.py produces. Used for side-by-side comparison against the +original run's board; the reference CSS lives in reference_board_style.css. + +Usage: + python3 render_reference_shape.py --traces /tmp/traces.json \ + --out ~/persona-sweep/agent_eval_full-4-shape.html [--since 2026-09-01] +""" +import argparse, datetime, html, json, re, sys + +REF_CSS = open(__file__.replace("render_reference_shape.py","reference_board_style.css")).read() + +def esc(s): + return html.escape(str(s)) if s is not None else "" + +PROMPT_ORDER = [ + "alert-analysis-A","alert-analysis-B","alert-analysis-C", + "entity-analytics-A","entity-analytics-B","entity-analytics-C", + "threat-hunting-A","threat-hunting-B","threat-hunting-C", + "detection-rule-edit-A","detection-rule-edit-B","detection-rule-edit-C", + "workflow-A","workflow-B","workflow-C", + "multi-step-A","multi-step-B","multi-step-C", + "log-analysis-A","log-analysis-B","log-analysis-C", +] + +PRETTY = { + "anthropic-claude-4.5-haiku":"Anthropic Claude Haiku 4.5", + "anthropic-claude-4.5-opus":"Anthropic Claude Opus 4.5", + "anthropic-claude-4.5-sonnet":"Anthropic Claude Sonnet 4.5", + "anthropic-claude-4.6-opus":"Anthropic Claude Opus 4.6", + "anthropic-claude-4.6-sonnet":"Anthropic Claude Sonnet 4.6", + "anthropic-claude-4.7-opus":"Anthropic Claude Opus 4.7", + "anthropic-claude-4.8-opus":"Anthropic Claude Opus 4.8", + "anthropic-claude-5-opus":"Anthropic Claude Opus 5", + "anthropic-claude-5-sonnet":"Anthropic Claude Sonnet 5", + "deepseek/deepseek-v4-pro":"deepseek-v4-pro", + "google/gemma-4-31b-it":"gemma-4-31b-it", + "moonshotai/kimi-k2.6":"kimi-k2.6", + "Qwen36_27b":"oss-eval qwen (Foundry)", +} +def pretty(model): + if model in PRETTY: return PRETTY[model] + m = model.replace("google-gemini-","Google Gemini ").replace("openai-gpt-","OpenAI GPT-") + m = m.replace("-flash-lite"," Flash-Lite").replace("-flash"," Flash").replace("-pro"," Pro") + m = m.replace("-mini"," Mini").replace("-nano"," Nano").replace("-sonnet"," Sonnet").replace("-opus"," Opus") + m = m.replace("openai-gpt-oss-","OpenAI GPT-OSS ").replace("zai-glm-5-2","ZAI GLM 5.2") + m = m.replace("gp-llm-v2","gp-llm-v2") + return m + +ALIASES = { + "anthropic-claude-4-5-haiku":"anthropic-claude-4.5-haiku", + "anthropic-claude-4-5-opus":"anthropic-claude-4.5-opus", + "anthropic-claude-4-5-sonnet":"anthropic-claude-4.5-sonnet", + "anthropic-claude-4-6-opus":"anthropic-claude-4.6-opus", + "anthropic-claude-4-6-sonnet":"anthropic-claude-4.6-sonnet", + "anthropic-claude-4-7-opus":"anthropic-claude-4.7-opus", + "anthropic-claude-4-8-opus":"anthropic-claude-4.8-opus", + "google-gemini-2-5-flash":"google-gemini-2.5-flash", + "google-gemini-2-5-flash-lite":"google-gemini-2.5-flash-lite", + "google-gemini-2-5-pro":"google-gemini-2.5-pro", + "google-gemini-3-1-pro":"google-gemini-3.1-pro", + "google-gemini-3-1-flash-lite":"google-gemini-3.1-flash-lite", + "openai-gpt-5-2":"openai-gpt-5.2", + "openai-gpt-5-4":"openai-gpt-5.4", + "openai-gpt-5-4-mini":"openai-gpt-5.4-mini", + "openai-gpt-5-4-nano":"openai-gpt-5.4-nano", + "openai-gpt-5-5":"openai-gpt-5.5", + "deepseek/deepseek-v4-pro-0813":"deepseek/deepseek-v4-pro", + "openai/gpt-5.6-sol":"openai-gpt-5.6-sol", +} +def fmt_tok(n): + return f"{n:,}" if n else "0" + +def cell_html(cell): + # mirror: ✓ 9 steps
    41s · 217314/2789 tok + if not cell: return '' + steps = cell.get("stepCount") or 0 + u = cell.get("usage") or {} + secs = int((u.get("durNs") or 0)/1e9) if u.get("durNs") else 0 + it, ot = u.get("inTok") or 0, u.get("outTok") or 0 + has_ans = bool(cell.get("answer")) + cls = "cell ok" if has_ans else "cell" + dot = '✓' if has_ans else '△' + sub = f'
    {secs}s · {fmt_tok(it)}/{fmt_tok(ot)} tok' if (it or ot) else "" + return f'✓ {steps} steps{sub}' + +def step_card(step, idx): + # extract schema: {"type": "tool"|"reasoning"|"skill", "toolId":..., "toolParams": "...json...", "text":...} + typ = step.get("type") or "reasoning" + if typ == "tool": + nm = step.get("toolId") or "tool" + params = step.get("toolParams") + body = f'{esc(nm)}{esc(params if params else "")}' + label = esc(nm) + return (f'
    {label}' + f'
    {body}
    ') + if typ == "skill": + skills = step.get("skills") or [] + names = ", ".join(s.get("id","?") for s in skills) + return (f'
    skills: {esc(names)}' + f'
    {esc(json.dumps(skills, ensure_ascii=False)[:400])}
    ') + text = step.get("text") or "" + return (f'
    reasoning · {idx}' + f'
    {esc(text)}
    ') + +def steps_html(cell): + steps = cell.get("steps") or [] + if not steps: return "" + return "\n".join(step_card(s,i+1) for i,s in enumerate(steps[:400])) +def prompt_card(cell, pid): + # reference: open card shows prompt text, attachment meta, trace steps, answer + q = cell.get("question") or "" + ans = cell.get("answer") or "" + steps = cell.get("steps") or [] + tools = ", ".join(dict.fromkeys((s.get("toolId") or "?") for s in steps if s.get("type") == "tool")) + n = len(steps) + secs = int(((cell.get("usage") or {}).get("durNs") or 0)/1e9) + meta = f"{n} steps · {secs}s" + attach = cell.get("attachment") or "" + attach_html = (f'

    attach: {esc(attach[:120])}

    ') if attach else "" + trail = f'

    Tools called: {esc(tools)}

    ' if tools else "" + return f'''
    + + completed + {esc(pid)} + {esc(pid.rsplit("-",1)[0])} + {meta} + +
    +
    Prompt sent

    {esc(q)}

    {attach_html}
    +
    Trace · {n} steps +
    {steps_html(cell)} + {trail} +
    +
    {ans}
    +
    +
    ''' +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--traces", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--since", default="2026-09-01T00:00Z") + ap.add_argument("--max-steps-per-cell", type=int, default=400) + a = ap.parse_args() + t = json.load(open(a.traces)) + cells = {} + for k, v in t["cells"].items(): + m, p = k.split(":", 1) + cells[f"{ALIASES.get(m, m)}:{p}"] = v + # normalize prompt casing: extract emits lowercase ids; ref uses -A/-B/-C + norm = {} + for k, v in cells.items(): + m, p = k.split(":", 1) + pp = p + if re.fullmatch(r"[a-z-]+-[abc]", p): + pp = p[:-1] + p[-1].upper() + norm[f"{m}:{pp}"] = v + cells = norm + models = sorted({k.split(":",1)[0] for k in cells}) + P = [] + P.append('') + P.append(f"Agent Builder Skill Eval — golden rebuild") + P.append(REF_CSS) + P.append('
    ') + P.append("

    Agent Builder Skill Eval — Golden Rebuild (ours)

    ") + P.append(f'

    Golden-cluster rebuild of the reference board shape. Each cell is the newest execution per (model, prompt) since {esc(a.since)}. Same layout as the original for side-by-side comparison. Generated {datetime.datetime.utcnow():%Y-%m-%d %H:%M} UTC.

    ') + # summary matrix + P.append("" + "".join(f"" for p in PROMPT_ORDER) + "") + for m in models: + row = [f''] + done = 0 + for p in PROMPT_ORDER: + c = cells.get(f"{m}:{p}") + if c and c.get("answer"): done += 1 + row.append(cell_html(c)) + P.append(f"{''.join(row)}") + P.append("
    Model{p}

    {esc(m)}
    ") + # per-model cards + for m in models: + n_done = sum(1 for p in PROMPT_ORDER if (cells.get(f"{m}:{p}") or {}).get("answer")) + P.append(f'
    ') + P.append(f'

    {esc(pretty(m))}

    ') + P.append(f'
    {esc(m)}·{n_done}/21 completed
    ') + for p in PROMPT_ORDER: + c = cells.get(f"{m}:{p}") + if c: + P.append(prompt_card(c, p)) + else: + P.append(f'

    no execution for {esc(p)} in window

    ') + P.append("
    ") + P.append("
    ") + out = "\n".join(P) + open(a.out, "w").write(out) + n_cells = sum(1 for m in models for p in PROMPT_ORDER if cells.get(f"{m}:{p}")) + print(f"wrote {a.out} ({len(out):,} bytes, {len(models)} models, {n_cells} cells with data)") + +if __name__ == "__main__": + main() diff --git a/scripts/orca_vm/retry_sweep.sh b/scripts/orca_vm/retry_sweep.sh new file mode 100644 index 0000000000000..bef9aa770f325 --- /dev/null +++ b/scripts/orca_vm/retry_sweep.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# retry_sweep.sh — relaunch run_model.sh on all provisioned sweep VMs (warm retry) +set -uo pipefail +KEY="$HOME/.ssh/azure_eval_farm" +SWEEP_DIR="$HOME/persona-sweep" + +# Fire all relaunches in parallel; each logs to run2.log. +# Forward EVAL_REPETITIONS / PERSONA_MATRIX_TIMEOUT_MINUTES when set so +# determinism retries (e.g. 3 reps, 150min ceiling) keep their settings; +# run_model.sh defaults both when absent. +ENV_PREFIX="" +if [ -n "${EVAL_REPETITIONS:-}" ]; then + ENV_PREFIX="export EVAL_REPETITIONS='${EVAL_REPETITIONS}' " +fi +if [ -n "${PERSONA_MATRIX_TIMEOUT_MINUTES:-}" ]; then + ENV_PREFIX="${ENV_PREFIX}export PERSONA_MATRIX_TIMEOUT_MINUTES='${PERSONA_MATRIX_TIMEOUT_MINUTES}' " +fi +if [ -n "$ENV_PREFIX" ]; then + ENV_PREFIX="${ENV_PREFIX}&& " +fi +while IFS=: read -u 3 -r model ip; do + echo "[retry] $model @ $ip" + ( ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + -o ServerAliveInterval=30 -o ServerAliveCountMax=120 \ + -i "$KEY" "orcaeval@$ip" "${ENV_PREFIX}bash /tmp/run_model.sh '$model'" \ + > "$SWEEP_DIR/$model/run2.log" 2>&1 ) & +done 3< <(python3 - <<'PY' +import json, sys +from pathlib import Path +seen = {} +rows = [] +for d in sorted(Path.home().glob("persona-sweep/*")): + s = d / "status.json" + if not s.exists(): + continue + meta = json.loads(s.read_text()) + ip, model = meta.get("ip"), meta.get("model") + if not ip: + continue + # One model per VM stack. Two eval stacks on one box OOM each other, + # corrupt local ES, and wedge SSH — skip the duplicate loudly. + if ip in seen: + print(f"SKIP {model}: {ip} already owned by {seen[ip]}", file=sys.stderr) + continue + seen[ip] = model + rows.append(f"{model}:{ip}") +print("\n".join(rows)) +PY +) +wait +echo "=== all retries finished; per-model tails ===" +for d in "$SWEEP_DIR"/*/; do + m=$(basename "$d") + grep -h "=== DONE:" "$d/run2.log" 2>/dev/null | sed "s/^/ $m /" || echo " $m: no DONE line" +done diff --git a/scripts/orca_vm/run_model.sh b/scripts/orca_vm/run_model.sh new file mode 100755 index 0000000000000..56ae5f8b11738 --- /dev/null +++ b/scripts/orca_vm/run_model.sh @@ -0,0 +1,578 @@ +#!/bin/bash +# run_model.sh — minimal, proven path for EIS models +# Mirrors the successful Haiku canary exactly: +# stop → clean data → evals start (manages scout+CCM+readiness internally) → export +set -uo pipefail + +MODEL="$1" +export NVM_DIR=$HOME/.nvm; source $NVM_DIR/nvm.sh +# Golden-cluster trace export: source golden env so the vault config below can +# point tracingEs/tracingExporters at golden (profileEnvOverrides come from the +# config file only — ambient env is NOT read by evals start). +source /tmp/golden-cluster-env.sh 2>/dev/null || true +cd ~/Projects/kibana + +# ─── Write config.local.json (git-ignored, missing on fresh VMs) ──────────── +python3 -c " +import json, os +path = 'x-pack/platform/packages/shared/kbn-evals/scripts/vault/config.local.json' +os.makedirs(os.path.dirname(path), exist_ok=True) +cfg = {} +if os.path.exists(path): + with open(path) as f: + try: + cfg = json.load(f) + except Exception: + cfg = {} +cfg['description'] = 'kbn-evals local config' +cfg['environment'] = 'local' +cfg['evaluationsKbn'] = {'url': 'http://elastic:changeme@localhost:5620', 'apiKey': ''} +cfg['evaluationsEs'] = {'url': 'http://elastic:changeme@localhost:9220', 'apiKey': ''} +# Golden-cluster trace export: EDOT + scout OTel exporters must send traces to +# the golden cluster (profileEnvOverrides are read from this file, NOT ambient env). +import os as _os +_golden_es = _os.environ.get('GOLDEN_ES_URL', '') +_golden_key = _os.environ.get('GOLDEN_ES_API_KEY', '') +_trace_exporters = _os.environ.get('TRACING_EXPORTERS', '') +if _golden_es: + cfg['tracingEs'] = {'url': _golden_es, 'apiKey': _golden_key} + if _trace_exporters: + try: + cfg['tracingExporters'] = json.loads(_trace_exporters) + except Exception: + pass + # Agent Builder spans (the ones trace-based evaluators query) do NOT flow + # through telemetry.tracing.exporters. register_tracing.ts builds its own + # provider, hard-wired to ElasticsearchOtlpExporter(asInternalUser) -> the + # LOCAL Scout ES. Before golden was introduced, tracingEs also pointed local, + # so export and query agreed and evaluators worked. Pointing tracingEs at + # golden moved the QUERY target only: spans kept landing in local Scout, so + # every trace_id lookup missed (0/200 Sept-2 score trace_ids on golden). + # xpack.agentBuilder.tracing.exporters is APPENDED to the built-in local + # exporter, so this adds golden without removing local fidelity. + # The scores-only key returns HTTP 200 with a FORBIDDEN payload buried in the + # OTLP protobuf body, so a wrong key here drops every span SILENTLY. Prefer the + # trace-capable key and fail loudly rather than export into a black hole. + _trace_key = _os.environ.get('GOLDEN_TRACE_API_KEY', '') or _golden_key + if _trace_key: + cfg['agentBuilderTracingExporters'] = [{ + 'url': _golden_es.rstrip('/') + '/_otlp/v1/traces', + 'headers': {'Authorization': 'ApiKey ' + _trace_key}, + }] +else: + cfg['tracingEs'] = {'url': 'http://elastic:changeme@localhost:9220', 'apiKey': ''} +with open(path, 'w') as f: + json.dump(cfg, f, indent=2) +print('config.local.json written (tracingEs -> %s)' % ('golden' if _golden_es else 'local')) +""" + +# ─── Trace-export preflight ────────────────────────────────────────────────── +# The OTLP endpoint answers HTTP 200 even when the API key lacks traces-* write: +# the rejection is a FORBIDDEN string inside the protobuf response body. Without +# this check a whole sweep runs, looks green, and lands zero spans on golden. +if [ -n "${GOLDEN_ES_URL:-}" ]; then + _pf_key="${GOLDEN_TRACE_API_KEY:-${GOLDEN_ES_API_KEY:-}}" + if [ -n "$_pf_key" ]; then + # Must send a REAL span: an empty OTLP body never reaches the index, so a + # key without traces-* write still answers 200 and the check passes vacuously. + _pf_body=$(_PF_KEY="$_pf_key" python3 -c " +import os,random,sys,time,urllib.request,urllib.error +def tag(f,w): return bytes([(f<<3)|w]) +def varint(n): + o=b'' + while True: + b=n&0x7F; n>>=7; o+=bytes([b|(0x80 if n else 0)]) + if not n: return o +def ld(f,p): return tag(f,2)+varint(len(p))+p +def s(f,t): return ld(f,t.encode()) +def fx(f,n): return tag(f,1)+n.to_bytes(8,'little') +def kv(k,v): return ld(1,s(1,k)+ld(2,s(1,v))) +now=time.time_ns() +span=(ld(1,bytes(random.getrandbits(8) for _ in range(16)))+ld(2,bytes(random.getrandbits(8) for _ in range(8))) + +s(5,'trace-export-preflight')+tag(6,0)+varint(2)+fx(7,now)+fx(8,now+1000)) +req=ld(1,ld(1,ld(1,kv('service.name','preflight')))+ld(2,ld(2,span))) +r=urllib.request.Request(os.environ['GOLDEN_ES_URL'].rstrip('/')+'/_otlp/v1/traces',data=req, + headers={'Authorization':'ApiKey '+os.environ['_PF_KEY'],'Content-Type':'application/x-protobuf'},method='POST') +try: + with urllib.request.urlopen(r,timeout=45) as resp: sys.stdout.write(resp.read().decode('utf-8','replace')) +except urllib.error.HTTPError as e: sys.stdout.write('HTTPERROR '+e.read().decode('utf-8','replace')) +except Exception as e: sys.stdout.write('PREFLIGHT_SKIP '+str(e)) +" 2>/dev/null _PF_KEY="$_pf_key" || true) + case "$_pf_body" in + *FORBIDDEN*|*unauthorized*) + echo "FATAL: golden trace export key cannot write traces-* (silent 200/FORBIDDEN)." >&2 + echo " Set GOLDEN_TRACE_API_KEY to a key with traces-* write privileges." >&2 + exit 1 ;; + esac + echo "trace-export preflight OK (golden accepted a real OTLP span)" + fi +fi + +# ─── Env vars ──────────────────────────────────────────────────────────────── +source /tmp/golden-cluster-env.sh 2>/dev/null +# Judge independence. The matrix drops self-judged docs (`excludeSelfJudged`), +# so a model graded by itself scores a whole row of blanks that look identical +# to "never ran" — 4.6-sonnet lost 294 valid docs this way on 2026-08-29. +# Pick a default judge that differs from the model under test, and let the +# caller override. DEFAULT_JUDGE is only used when it is not the candidate. +# Default is the omniroute Opus 5 combo (canary-verified 2026-09-11: endpoint +# 200, judge calls 200, PASS docs=8/8) — cross-family vs every eis-* candidate +# and immune to EIS connector outages. The selfhost-* synthesis branch below +# provisions its proxy/endpoint; requires .selfhost-judge.env on the VM. +DEFAULT_JUDGE=selfhost-omni-opus-5 +ALT_JUDGE=eis-anthropic-claude-4-5-haiku +if [ -z "${EVAL_CONNECTOR_ID:-}" ]; then + if [ "$MODEL" = "$DEFAULT_JUDGE" ]; then + export EVAL_CONNECTOR_ID="$ALT_JUDGE" + else + export EVAL_CONNECTOR_ID="$DEFAULT_JUDGE" + fi +fi +if [ "$EVAL_CONNECTOR_ID" = "$MODEL" ]; then + echo "FATAL: judge ($EVAL_CONNECTOR_ID) == model under test ($MODEL);" >&2 + echo " every score would be dropped as self-judged. Set EVAL_CONNECTOR_ID." >&2 + exit 2 +fi +echo "=== judge: $EVAL_CONNECTOR_ID | candidate: $MODEL ===" + +# kbn-evals ships HTTP retries off (KBN_EVALS_HTTP_RETRIES defaults to 0), so a +# single blip on the converse call ends the whole suite: glm-5-2 lost 19 of 21 +# examples 58 minutes in when Kibana stopped answering on 2026-08-29. Retries +# only cover 429/503/504, so this does not save a status-less transport death, +# but it does absorb the overload responses a long sweep actually provokes. +export KBN_EVALS_HTTP_RETRIES="${KBN_EVALS_HTTP_RETRIES:-3}" +# Retries only help a request that FAILS. A converse call that never returns +# parks the worker forever: a glm-5-2 run burned 45 minutes with 4 seconds of +# CPU and six open sockets while /api/status still answered 200. Bound each +# attempt so a hung endpoint becomes a retryable failure. +# +# 25 min, not 10: golden shows a LEGITIMATE glm-5-2 example taking 1198s +# (20 min). A 10-min bound aborts real work and the retry aborts it again, +# turning a slow success into a guaranteed failure. Keep this above the +# measured worst case -- the suite-level budget catches a truly wedged run. +export KBN_EVALS_HTTP_TIMEOUT_MS="${KBN_EVALS_HTTP_TIMEOUT_MS:-1500000}" +export EVAL_REPETITIONS="${EVAL_REPETITIONS:-1}" +export PERSONA_MATRIX_TIMEOUT_MINUTES="${PERSONA_MATRIX_TIMEOUT_MINUTES:-120}" +export AGENT_BUILDER_INFERENCE_TIMEOUT_MS=600000 +export SCOUT_READY_TIMEOUT_MS=900000 +CCM_KEY=$(python3 -c "import json; d=json.load(open('/home/orcaeval/.elastic/eis-ccm-key.json')); print(d.get('api_key','') or d.get('key',''))") +export KIBANA_EIS_CCM_API_KEY=$CCM_KEY +CONNS=$(python3 -c "import json,base64; c=json.load(open('/home/orcaeval/.elastic/eis-connectors-cache.json')); conns=c.get('connectors',c); print(base64.b64encode(json.dumps(conns).encode()).decode())") + +# ─── OpenRouter / self-hosted path (openrouter-* or selfhost-*) ───────────── +# The connector cache's providerConfig.url is must-point-at-proxy for +# openrouter-* connectors: they need an ES inference endpoint pointing at +# the on-VM SSE-normalizing proxy (localhost:8088), never openrouter.ai +# directly — ES cannot parse OpenRouter's raw SSE (reasoning:null in finish +# chunks → XContentParse exception → 500s). The proxy also injects max_tokens +# and retries 503s. selfhost-* models reuse the same path with +# PROXY_UPSTREAM set from /tmp/selfhost.env (url + api_key): same SSE +# normalization need (SGLang emits reasoning_content deltas ES rejects). +# Stale markers from a previous invocation would kill the endpoint watcher +# loop below and confuse the sweep controller — clear BEFORE starting it. +rm -f /tmp/unit.done /tmp/unit.rc +OR_PROXY_PORT=8088 +if [ "${MODEL#openrouter-}" != "$MODEL" ] || [ "${MODEL#selfhost-}" != "$MODEL" ]; then + IS_SELFHOST=0; [ "${MODEL#selfhost-}" != "$MODEL" ] && IS_SELFHOST=1 + if [ "$IS_SELFHOST" = "1" ] && [ -f /tmp/selfhost.env ]; then + source /tmp/selfhost.env # exports SELFHOST_UPSTREAM, SELFHOST_API_KEY + export PROXY_UPSTREAM="$SELFHOST_UPSTREAM" + # Dual-cell balancing: the A100 box serves two independent TP=1 cells (one + # per GPU). With a single upstream the whole sweep serialises on cell A + # (measured 2026-09-06: cell A 100% util / 44 queued while cell B idled at + # 0%). When a second cell is configured, hand both to the proxy so it can + # round-robin. Optional - absent SELFHOST_UPSTREAM_B keeps single-cell. + if [ -n "${SELFHOST_UPSTREAM_B:-}" ]; then + export PROXY_UPSTREAMS="$SELFHOST_UPSTREAM,$SELFHOST_UPSTREAM_B" + export PROXY_UPSTREAM_KEYS="$SELFHOST_API_KEY,${SELFHOST_API_KEY_B:-$SELFHOST_API_KEY}" + echo "=== dual-cell: balancing across 2 upstreams ===" + fi + fi + echo "=== OpenAI-compatible proxy model detected: $MODEL (upstream: ${PROXY_UPSTREAM:-openrouter.ai}) ===" + # Proxy must be running BEFORE the endpoint references it and before any + # converse call. Kill a stale process first: scp overwrites the file, but + # the old process keeps serving OLD normalization rules (field-tested trap). + kill -9 $(lsof -t -i:$OR_PROXY_PORT 2>/dev/null) 2>/dev/null || true + nohup python3 /tmp/openrouter_proxy.py --port $OR_PROXY_PORT > /tmp/or-proxy.log 2>&1 & + for i in $(seq 1 30); do + # GET yields 501 (handler is POST-only); any HTTP answer = alive. + CODE=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:$OR_PROXY_PORT/ 2>/dev/null) + [ "$CODE" != "000" ] && break + sleep 1 + done + echo "proxy ready (HTTP $CODE)" + + # Rewrite this model's providerConfig to point at the on-VM proxy. The key + # stays from the cache; the URL changes. The ES endpoint's model_id must be + # the OpenRouter API name (z-ai/glm-5.3-flash), which the cache does NOT + # carry — read it from the matrix config's matchIds. Judge connectors are + # untouched — they ride EIS as usual. + CONNS=$(OPENROUTER_MODEL="$MODEL" OR_PORT=$OR_PROXY_PORT \ + MATRIX_CONFIG=/tmp/persona_matrix.config.json \ + python3 -c " +import json, base64, os, sys +raw = os.environ.get('KIBANA_TESTING_AI_CONNECTORS_B64', '') +if not raw: + # re-derive from the CONNS captured above when the env var is unset + raw = base64.b64encode(json.dumps(json.load(open('/home/orcaeval/.elastic/eis-connectors-cache.json')).get('connectors', {})).encode()).decode() +conns = json.loads(base64.b64decode(raw)) +model = os.environ['OPENROUTER_MODEL'] +c = conns.get(model) +if c is None: + # selfhost-* connectors are NOT in the EIS cache — synthesize one on the + # fly from /tmp/selfhost.env values already exported by the caller. + sh_up = os.environ.get('SELFHOST_UPSTREAM', '') + sh_key = os.environ.get('SELFHOST_API_KEY', '') + if model.startswith('selfhost-') and sh_up and sh_key: + conns[model] = c = { + 'name': model, + 'actionTypeId': '.inference', + 'config': { + 'provider': 'openai', + 'taskType': 'chat_completion', + 'inferenceId': model + '-chat_completion', + 'providerConfig': { + 'model_id': model.split('-', 1)[1], + 'url': f\"http://127.0.0.1:{os.environ['OR_PORT']}\", + 'api_key': sh_key, + }, + }, + 'secrets': {}, + } + else: + print(f'FATAL: connector {model} missing from cache', file=sys.stderr); sys.exit(3) +pc = c.setdefault('config', {}).setdefault('providerConfig', {}) +pc['url'] = f\"http://127.0.0.1:{os.environ['OR_PORT']}\" +# ES endpoint model_id = OpenRouter API name from the matrix config matchIds. +# Scoped to THIS model's entry: a bare first-match regex picks up another +# model's matchIds (anthropic's is first in the file) and sends the endpoint +# at the wrong upstream model. +for _m in json.load(open(os.environ['MATRIX_CONFIG'])).get('models', []): + if _m.get('id') == model and _m.get('matchIds'): + pc['model_id'] = _m['matchIds'][0] +print(base64.b64encode(json.dumps(conns).encode()).decode()) +" 2>/dev/null) || { echo "FATAL: connector rewrite failed" >&2; exit 3; } + export KIBANA_TESTING_AI_CONNECTORS="$CONNS" + + # The ES inference endpoint must exist and point at the proxy, with id == + # connector inferenceId. Run AFTER stack boot below? No — the stack boots + # inside `evals start`, so we cannot wait for ES here; the endpoint script + # polls ES readiness itself (up to 10 min) in the background. + OR_MODEL_ID=$(python3 -c " +import json, base64 +c = json.loads(base64.b64decode('$CONNS')) +cfg = c['$MODEL']['config'] +print(cfg['providerConfig']['model_id'])") + # converse resolves the model by CONNECTOR id — "No connector or inference + # endpoint found for ID 'openrouter-zai-glm-5-3-flash'" if the endpoint id + # is the inferenceId (openrouter-glm-5-3-flash-chat_completion). EIS models + # work because CCM auto-creates endpoints named exactly after the connector. + OR_ENDPOINT_ID="$MODEL" + OR_KEY=$(python3 -c " +import json, base64 +c = json.loads(base64.b64decode('$CONNS')) +print(c['$MODEL']['config']['providerConfig']['api_key'])") + echo "endpoint: $OR_ENDPOINT_ID | model: $OR_MODEL_ID" + # nohup: `evals start` blocks; the watcher creates the endpoint the moment + # ES accepts connections, which is well before Playwright needs it. + # LOOP: the eval retry loop wipes ES data between attempts (rm -rf), which + # deletes the inference endpoint. A one-shot watcher leaves attempts 2/3 + # with no endpoint → converse 404. Loop until /tmp/unit.done, re-creating + # the endpoint idempotently (create_openrouter_endpoint.py reuses an + # endpoint that already points at the proxy, so each iteration is a no-op + # once the endpoint exists). + nohup bash -c ' + while [ ! -f /tmp/unit.done ]; do + python3 /tmp/create_openrouter_endpoint.py "$1" "$2" "$3" "$4" >> /tmp/or-endpoint.log 2>&1 + sleep 10 + done + ' _ "$OR_ENDPOINT_ID" "$OR_MODEL_ID" "$OR_KEY" $OR_PROXY_PORT \ + > /dev/null 2>&1 & + echo "endpoint watcher started (log: /tmp/or-endpoint.log)" +fi +export KIBANA_TESTING_AI_CONNECTORS=$CONNS + +# ─── Omniroute/selfhost JUDGE path (judge selfhost-*, candidate eis-*) ─────── +# Runs AFTER the final CONNS export above so nothing clobbers the synthesized +# entry. EVAL_CONNECTOR_ID must name a connector the stack can resolve; EIS +# judges ride the connector cache, a selfhost-* judge has none — synthesize it +# (same shape as the selfhost-candidate branch) plus the SSE-normalizing proxy +# and the endpoint watcher. Without this the judge's converse calls 404 +# mid-suite and the run reads as an eval failure instead of a missing judge +# connector. The proxy is REQUIRED even for omniroute: the cursor failover +# route emits `reasoning` deltas ES's streaming processor rejects — the exact +# defect class the proxy exists to strip. +JUDGE_IS_SELFHOST=0 +case "$EVAL_CONNECTOR_ID" in selfhost-*) JUDGE_IS_SELFHOST=1 ;; esac +if [ "$JUDGE_IS_SELFHOST" = "1" ]; then + if [ -f /tmp/judge.env ]; then + # .selfhost-judge.env shipped by deploy(): public omniroute URL + key. + source /tmp/judge.env # exports SELFHOST_UPSTREAM, SELFHOST_API_KEY + fi + if [ -z "${SELFHOST_UPSTREAM:-}" ] || [ -z "${SELFHOST_API_KEY:-}" ]; then + echo "FATAL: selfhost judge $EVAL_CONNECTOR_ID requires /tmp/judge.env" >&2 + echo " (SELFHOST_UPSTREAM/SELFHOST_API_KEY from .selfhost-judge.env)" >&2 + exit 3 + fi + JUDGE_MODEL_ID="${EVAL_CONNECTOR_ID#selfhost-}" # e.g. omni-opus-5 combo id + echo "=== judge connector synthesis: $EVAL_CONNECTOR_ID -> $SELFHOST_UPSTREAM (model: $JUDGE_MODEL_ID) ===" + # Add/overwrite the judge entry in the exported base64 connector map. + # Endpoint id MUST equal the connector id: converse resolves the model by + # CONNECTOR id, not inferenceId (same trap as the openrouter-* branch). + # Judge proxy rides its own port (8089): the candidate proxy owns 8088. If + # both fired (selfhost candidate + selfhost judge) the judge branch would + # otherwise kill the candidate's proxy mid-run via the port-8088 lsof kill. + JUDGE_PROXY_PORT=8089 + CONNS=$(CONNS_B64="$CONNS" JUDGE_ID="$EVAL_CONNECTOR_ID" OR_PORT=$JUDGE_PROXY_PORT \ + JUDGE_MODEL_ID="$JUDGE_MODEL_ID" JUDGE_KEY="$SELFHOST_API_KEY" python3 -c " +import json, base64, os +conns = json.loads(base64.b64decode(os.environ['CONNS_B64'])) +judge = os.environ['JUDGE_ID'] +conns[judge] = { + 'name': judge, + 'actionTypeId': '.inference', + 'config': { + 'provider': 'openai', + 'taskType': 'chat_completion', + 'inferenceId': judge + '-chat_completion', + 'providerConfig': { + 'model_id': os.environ['JUDGE_MODEL_ID'], + 'url': 'http://127.0.0.1:' + os.environ['OR_PORT'], + 'api_key': os.environ['JUDGE_KEY'], + }, + }, + 'secrets': {}, +} +print(base64.b64encode(json.dumps(conns).encode()).decode()) +") || { echo "FATAL: judge connector synthesis failed" >&2; exit 3; } + export KIBANA_TESTING_AI_CONNECTORS="$CONNS" + # Proxy up before the endpoint references it; kill stale instances first + # (an old process keeps serving the previous upstream). + kill -9 $(lsof -t -i:$JUDGE_PROXY_PORT 2>/dev/null) 2>/dev/null || true + PROXY_UPSTREAM="$SELFHOST_UPSTREAM" nohup python3 /tmp/openrouter_proxy.py --port $JUDGE_PROXY_PORT > /tmp/or-proxy-judge.log 2>&1 & + for i in $(seq 1 30); do + CODE=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:$JUDGE_PROXY_PORT/ 2>/dev/null) + [ "$CODE" != "000" ] && break + sleep 1 + done + echo "judge proxy ready (HTTP $CODE)" + # Watcher loop: the eval retry loop wipes ES data between attempts, deleting + # the inference endpoint; re-create idempotently until the run completes + # (create_openrouter_endpoint.py reuses a correct existing endpoint). + nohup bash -c ' + while [ ! -f /tmp/unit.done ]; do + python3 /tmp/create_openrouter_endpoint.py "$1" "$2" "$3" "$4" >> /tmp/or-endpoint-judge.log 2>&1 + sleep 10 + done + ' _ "$EVAL_CONNECTOR_ID" "$JUDGE_MODEL_ID" "$SELFHOST_API_KEY" $JUDGE_PROXY_PORT \ + > /dev/null 2>&1 & + echo "judge endpoint watcher started (log: /tmp/or-endpoint-judge.log)" +fi + +# ─── Full stop + clean data ────────────────────────────────────────────────── +echo "=== Stopping any prior stack ===" +node scripts/evals stop 2>/dev/null || true +pkill -f "scout.js" 2>/dev/null || true +pkill -f "kibana --dev" 2>/dev/null || true +pkill -f "org.elasticsearch.bootstrap.Elasticsearch" 2>/dev/null || true +sleep 5 + +echo "=== Cleaning ES data ===" +rm -rf ~/Projects/kibana/.es/cluster-scout/data 2>/dev/null +echo "ES data cleaned" + +# ─── Run eval — evals start manages scout boot, CCM, readiness internally ──── +# This is the exact path that passed 21/21 for Haiku. No manual scout, no +# manual CCM, no --skip-server: a single command owns the whole lifecycle. +echo "=== Running eval: $MODEL ===" +# Suite is injected by the sweeper (EVAL_SUITE); default keeps the historical +# persona-matrix behaviour for any caller that predates the suite port. +# +# CCM boot race: `evals start` waits for the .inference index, then enables +# Cloud Connected Mode against localhost:9220. Under concurrent provisioning +# (8-way, 2026-09-02) ES was not yet accepting connections for 5/15 VMs and +# the step died with "TypeError: fetch failed" -> +# "enable_eis_ccm exited with code 1" -> EVAL_EXIT=1 and nothing to export. +# The single-VM canary never hit it. It is a transient readiness fault, so +# retry the whole step; a genuine failure (bad model, real eval failure) +# fails identically on every attempt and still surfaces. +run_eval() { + # Trace-evaluator query target as AMBIENT env (belt-and-braces): the vault + # config.local.json also sets it via profileEnvOverrides, but on the 2026-09-04 + # 17-model sweep that propagation silently failed on 14/17 VMs -- the + # Playwright worker fell back to the local scout esClient, whose limited + # privileges cannot see the hidden .ds-traces-* backing indices, and every + # trace evaluator errored with `Unknown column [trace.id]` (0/17 examples on + # all 5 trace metrics). Ambient env is read directly by evaluate.ts:477. + export TRACING_ES_URL="${GOLDEN_ES_URL:?GOLDEN_ES_URL required}" + export TRACING_ES_API_KEY="${GOLDEN_ES_API_KEY:?GOLDEN_ES_API_KEY required}" + echo "trace evaluators will query: $TRACING_ES_URL" + # AD suite only: GCS dataset credentials for the alerts-snapshot restore. + # The scout evals_tracing config gates ES gcs-client registration (and thus + # the 95-alert corpus restore) on this env var. run_model.sh must read it + # from the shipped file: build_env_prefix only forwards FORWARDED_ENV_VARS, + # and the sweeper never puts the (large, secret) JSON in the environment. + if [ "${EVAL_SUITE:-}" = "security-persona-matrix-attack-discovery" ]; then + if [ ! -s /tmp/gcs_credentials.json ]; then + echo "FATAL: AD suite requires /tmp/gcs_credentials.json (deploy ships it)" >&2 + return 1 + fi + export GCS_CREDENTIALS="$(cat /tmp/gcs_credentials.json)" + echo "GCS_CREDENTIALS loaded ($(wc -c < /tmp/gcs_credentials.json) bytes)" + fi + node scripts/evals start --profile local --suite "${EVAL_SUITE:-security-persona-matrix}" --model "$MODEL" 2>&1 | tee /tmp/evals-start.log | tail -40 + return ${PIPESTATUS[0]} +} + +EVAL_EXIT=1 +# RESUME (2026-09-06): a retry used to re-run all 21 examples, so one transient +# failure near the end of a ~2h thinking-model pass discarded the whole pass +# (runs 8/9 died exactly this way). Between attempts we ask the LOCAL scout ES +# which example ids already produced a score and narrow the next attempt to the +# survivors via PERSONA_MATRIX_EXAMPLE_IDS. If the probe cannot determine the +# scored set we deliberately fall back to the full dataset — resuming on +# incomplete information would silently under-run the matrix. +scored_example_ids() { + # Query GOLDEN, not local ES. Score documents are written only when a + # Playwright run COMPLETES, and every retry wipes the local cluster, so the + # local index is empty at exactly the moment resume needs it (verified run 13 + # s2of3: local total=0 while golden held 4088 docs / 7 examples for that same + # shard). Golden is the only store that survives the wipe. + # + # Match execution_id EXACTLY, on the BARE field. Verified against golden: + # term on metadata.execution_id -> 98 docs / 7 examples; the same term on + # metadata.execution_id.keyword -> 0 (field is already keyword-mapped, the + # .keyword suffix silently matches nothing). + # Prefix and match_phrase both return 0 docs on + # this field -- persona_matrix_sweep.py --self-test enforces that ban for the + # gate, and the same trap applies here: a partial match resumes nothing while + # looking healthy. execution_id is "::::", and + # TEST_RUN_ID is already per-shard, so the shard scoping comes for free. + # + # Any failure yields "" = run everything. Re-running a scored example only + # wastes time; skipping an unscored one silently under-runs the matrix. + [ -n "${GOLDEN_ES_URL:-}" ] || { echo ""; return 0; } + # TEST_RUN_ID is exported by the launch wrapper into the EVAL subprocess, not + # into this script's shell (verified on the VM: absent from run_model.sh's + # /proc//environ). Guarding on it made the probe return "" every time, + # so every retry silently re-ran all 21 examples. Derive the run id from the + # docs the run just wrote instead -- local ES is authoritative and always + # present at this point. + local RUN_ID="${TEST_RUN_ID:-}" + if [ -z "$RUN_ID" ]; then + RUN_ID=$(curl -s -m 20 "http://elastic:changeme@localhost:9220/.ds-.evaluation-scores*/_search" \ + -H 'Content-Type: application/json' \ + -d '{"size":1,"_source":["metadata.execution_id"],"sort":[{"@timestamp":{"order":"desc"}}]}' 2>/dev/null \ + | python3 -c 'import json,sys +try: + d=json.load(sys.stdin) + print(d["hits"]["hits"][0]["_source"]["metadata"]["execution_id"].split("::")[0]) +except Exception: + print("")' 2>/dev/null) + fi + [ -n "$RUN_ID" ] || { echo ""; return 0; } + curl -s -m 25 "${GOLDEN_ES_URL}/.ds-.evaluation-scores*/_search" \ + -H "Authorization: ApiKey ${GOLDEN_ES_API_KEY}" \ + -H 'Content-Type: application/json' \ + -d '{"size":0,"query":{"term":{"metadata.execution_id":"'"${RUN_ID}::${EVAL_SUITE:-security-persona-matrix}::${MODEL}"'"}}, + "aggs":{"ids":{"terms":{"field":"example.id","size":500}}}}' 2>/dev/null \ + | python3 -c 'import json,sys +try: + d=json.load(sys.stdin) + print(",".join(b["key"] for b in d["aggregations"]["ids"]["buckets"])) +except Exception: + print("")' 2>/dev/null +} + +for attempt in 1 2 3; do + echo "--- eval attempt $attempt/3 ---" + if [ -n "${RESUME_IDS:-}" ]; then + export PERSONA_MATRIX_EXAMPLE_IDS="$RESUME_IDS" + echo "resume: running only $(echo "$RESUME_IDS" | tr ',' '\n' | grep -c .) remaining example(s)" + else + unset PERSONA_MATRIX_EXAMPLE_IDS + fi + run_eval + EVAL_EXIT=$? + [ "$EVAL_EXIT" -eq 0 ] && break + if [ "$attempt" -lt 3 ]; then + echo "eval attempt $attempt failed (exit $EVAL_EXIT); stopping stack and retrying" + # ORDER MATTERS. Flush FIRST, then ask golden what scored. + # + # The executor records every completed measurement before it throws, but + # export_scores.py only runs at the END of this script -- so at this point + # the partial scores exist ONLY in the local scout ES that the wipe below is + # about to destroy. Querying golden before flushing therefore always returned + # "" (verified run 13 s2of3: 0 docs on golden while local held real results), + # which skipped the flush and re-ran all 21 examples every attempt. + echo "resume: flushing partial scores to golden before wipe" + source /tmp/golden-cluster-env.sh 2>/dev/null + EVAL_SUITE="${EVAL_SUITE:-security-persona-matrix}" \ + python3 /tmp/export_scores.py "$MODEL" 2>&1 | tail -3 + PARTIAL_EXPORT_RC=${PIPESTATUS[0]} + if [ "$PARTIAL_EXPORT_RC" -ne 0 ]; then + # Could not make the partial durable -> resuming would silently drop those + # examples from the matrix. Re-run the full dataset instead. + echo "resume: partial export rc=$PARTIAL_EXPORT_RC - falling back to FULL dataset" + DONE_IDS="" + else + DONE_IDS="$(scored_example_ids)" + echo "resume: golden confirms $(echo "$DONE_IDS" | tr ',' '\n' | grep -c .) scored example(s)" + fi + node scripts/evals stop 2>/dev/null || true + pkill -f "scout.js" 2>/dev/null || true + pkill -f "org.elasticsearch.bootstrap.Elasticsearch" 2>/dev/null || true + sleep $((attempt * 30)) + rm -rf ~/Projects/kibana/.es/cluster-scout/data 2>/dev/null + # Narrow the next attempt to examples that have NOT scored yet. The id list + # is read from the suite's own dataset module so it cannot drift from the + # examples actually registered. + RESUME_IDS="" + if [ -n "$DONE_IDS" ]; then + RESUME_IDS="$(DONE_IDS="$DONE_IDS" node -e ' +const path = "/home/orcaeval/Projects/kibana/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/datasets/persona_matrix_prompts.ts"; +const src = require("fs").readFileSync(path, "utf8"); +const all = [...src.matchAll(/^\s*id:\s*"([^"]+)"/gm)].map((m) => m[1]); +const done = new Set((process.env.DONE_IDS || "").split(",").filter(Boolean)); +const rest = all.filter((id) => !done.has(id)); +// Only emit a filter when it is a strict, non-empty subset; anything else +// (parse failure, nothing left, nothing done) means run the full dataset. +process.stdout.write(rest.length && rest.length < all.length ? rest.join(",") : ""); +' 2>/dev/null)" + if [ -n "$RESUME_IDS" ]; then + echo "resume: $(echo "$DONE_IDS" | tr ',' '\n' | grep -c .) example(s) already scored and flushed to golden" + else + echo "resume: could not compute a valid remaining set - re-running FULL dataset" + fi + fi + fi +done +echo "EVAL_EXIT=$EVAL_EXIT" + +# ─── Export scores to golden cluster ──────────────────────────────────────── +echo "=== Exporting scores to golden ===" +source /tmp/golden-cluster-env.sh 2>/dev/null +EVAL_SUITE="${EVAL_SUITE:-security-persona-matrix}" python3 /tmp/export_scores.py "$MODEL" 2>&1 +EXPORT_EXIT=$? + +# export_scores.py exits 2 when SOME documents landed and some did not. That is +# not a success: golden holds a partial picture, and a controller reading only +# "did it exit 0" would call the sweep complete while cells are silently +# missing. Surface it distinctly from a total failure (1). +if [ "$EXPORT_EXIT" -eq 2 ]; then + echo "EXPORT_PARTIAL=1 — some docs landed on golden, some failed; see stderr above" +fi + +echo "=== DONE: $MODEL (EVAL_EXIT=$EVAL_EXIT, EXPORT_EXIT=$EXPORT_EXIT) ===" + +# Propagate eval failure as the process exit code so the sweep controller's +# ssh rc reflects it (export still runs above either way; the controller +# treats rc and the golden doc count as the two independent gates. + +# Detached mode: the controller launches us under nohup and polls these +# markers over short-lived SSH connections — a dead controller SSH stream +# must never look like a dead eval. +echo "$EVAL_EXIT" > /tmp/unit.rc +touch /tmp/unit.done + +exit $EVAL_EXIT diff --git a/scripts/orca_vm/summarize_blinding_ab.py b/scripts/orca_vm/summarize_blinding_ab.py new file mode 100644 index 0000000000000..bff2197b99baa --- /dev/null +++ b/scripts/orca_vm/summarize_blinding_ab.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Add cost and a distinguishability check to the blinding A/B result. + +A mean delta means nothing without knowing whether 14 paired samples could +distinguish it from zero. Exact two-sided sign test on the non-zero pairs. +""" +import json +import math +import sys +from itertools import combinations + +PATH = sys.argv[1] +# Opus-5 class pricing, USD per 1M tokens. +IN_RATE, OUT_RATE = 5.0, 25.0 + +d = json.load(open(PATH)) +s = d["summary"] +u = s["usage"] + +cost_in = u["prompt_tokens"] / 1e6 * IN_RATE +cost_out = u["completion_tokens"] / 1e6 * OUT_RATE +s["cost"] = { + "inputUsd": round(cost_in, 4), + "outputUsd": round(cost_out, 4), + "totalUsd": round(cost_in + cost_out, 4), + "rates": {"inputPerMTok": IN_RATE, "outputPerMTok": OUT_RATE}, + "note": "Opus-5 list pricing; the combo may route to a cheaper seat, so this is an upper bound.", +} + +pairs = [(r["leaked"], r["blinded"]) for r in d["results"] + if isinstance(r.get("leaked"), (int, float)) and isinstance(r.get("blinded"), (int, float))] +deltas = [a - b for a, b in pairs] +nz = [x for x in deltas if x != 0] +pos = sum(1 for x in nz if x > 0) +n = len(nz) + +# Exact two-sided sign test. +if n: + def C(k): + return math.comb(n, k) + tail = sum(C(k) for k in range(0, min(pos, n - pos) + 1)) + p = min(1.0, 2 * tail / (2 ** n)) +else: + p = 1.0 + +s["signTest"] = { + "nonZeroPairs": n, + "leakedHigher": pos, + "blindedHigher": n - pos, + "pValueTwoSided": round(p, 4), + "distinguishableFromZero": p < 0.05, + "interpretation": ( + "Cannot distinguish the leaked and blinded variants at n=14; this bounds the " + "effect rather than showing there is none." + if p >= 0.05 else + "The hint moved scores by more than chance would explain." + ), +} +s["scale"] = "0-10 judge rubric" + +json.dump(d, open(PATH, "w"), indent=2) +print(json.dumps(s, indent=2)) diff --git a/scripts/orca_vm/test_agent_eval_board.py b/scripts/orca_vm/test_agent_eval_board.py new file mode 100644 index 0000000000000..1539f02fef6e1 --- /dev/null +++ b/scripts/orca_vm/test_agent_eval_board.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Outcome-based guards for the agent_eval_full board renderer. + +Renders a synthetic traces payload through the REAL renderer CLI and asserts +what lands in the HTML: reference row order, usage ("Xs · Y/Z tok") format, +missing-row reasons, blank cells, disclosure block, and reproducibility +(byte-identical modulo the timestamp). +""" +import json +import subprocess +import sys +import tempfile +import os +import re + +HERE = os.path.dirname(os.path.abspath(__file__)) +RENDER = os.path.join(HERE, "render_agent_eval_full.py") + +# Synthetic cells: two covered models (one with usage, one without) + enough +# reference models to prove ORDER comes from REFERENCE_MODELS, not coverage. +CELLS = {} +# Synthetic cells on REAL reference models: haiku (with usage, order pos 1) +# and gp-llm-v2 (no usage, pos 6) — proves order comes from REFERENCE_MODELS +# and exercises both usage shapes. Only 2 of 21 prompts per model so blank +# cells render too. +for prompt in ["alert-analysis-a", "alert-analysis-b"]: + CELLS[f"anthropic-claude-4.5-haiku:{prompt}"] = { + "question": f"q {prompt}", + "answer": f"answer {prompt}", + "steps": [{"type": "reasoning", "text": "think"}], + "stepCount": 3, + "scores": {"Rubric": 1.0}, + "repetitions": 1, + "usage": {"durNs": 17_400_000_000, "inTok": 57029, "outTok": 1209}, + } + CELLS[f"gp-llm-v2:{prompt}"] = { + "question": f"q {prompt}", + "answer": None, + "steps": [], + "stepCount": 1, + "scores": {}, + "repetitions": 1, + "usage": {"durNs": 0, "inTok": 0, "outTok": 0}, + } + +TRACES = { + "cells": CELLS, + "meta": { + "since": "2026-09-01T00:00Z", + "until": None, + "scoreDocs": 42, + "argSpans": 7, + "usageSpans": 5, + "note": "synthetic", + }, +} + + +def render(traces, out): + with tempfile.TemporaryDirectory() as d: + t = os.path.join(d, "traces.json") + with open(t, "w") as fh: + json.dump(traces, fh) + p = subprocess.run( + [sys.executable, RENDER, "--traces", t, "--out", out, "--since", "2026-09-01T00:00Z"], + capture_output=True, text=True) + return p.returncode + + +def main(): + fails = [] + d = tempfile.mkdtemp() + out1, out2 = os.path.join(d, "a.html"), os.path.join(d, "b.html") + + rc = render(TRACES, out1) + if rc != 0: + print("FAIL: renderer exited", rc) + return 1 + h = open(out1, encoding="utf-8").read() + + def check(name, cond): + print(("ok: " if cond else "FAIL: ") + name) + if not cond: + fails.append(name) + + # 1. all 33 reference models present, exactly once each + rows = re.findall(r']*>\s*]*class="model"[^>]*>(.*?)(?:
    |)', h, re.S) + # rows include missing-rows; extract ids from both shapes + ids = re.findall(r'([^<]+)', h) + from render_agent_eval_full import REFERENCE_MODELS, PROMPT_IDS + check("33 reference models each rendered once", + sorted(ids) == sorted(REFERENCE_MODELS) and len(ids) == 33) + check("row order matches REFERENCE_MODELS order", ids == REFERENCE_MODELS) + + # 2. usage format: "17s · 57029/1209 tok" under the step count + check("usage cell rendered in reference format", + re.search(r'17s · 57029/1209 tok', h) is not None) + check("usage css class applied", 'class="usage"' in h) + # 3. no-usage cell shows steps only (no empty usage span) + check("zero-usage cell renders no usage text", + re.search(r'', h) is None) + + # 4. missing-row reasons surface verbatim + check("no-EIS-connector reason present", "no EIS connector exists" in h) + check("glm-5-2 blocked reason present", "#288469" in h) + + # 5. disclosure block carries provenance + usage coverage + check("usage join disclosed", "gen_ai.usage" in h and "tok" in h) + check("usage-cell count disclosed", re.search(r'\d+ cells carry usage', h) is not None) + + # 6. blank cell marker + check("blank cell marker present", "—" in h) + + # 7. reproducibility: byte-identical modulo timestamp line + render(TRACES, out2) + h2 = open(out2, encoding="utf-8").read() + strip = lambda s: re.sub(r'Generated: [^<]+', '', s) + check("re-render byte-identical modulo timestamp", strip(h) == strip(h2)) + + print() + if fails: + print(f"FAILURES: {len(fails)} {fails}") + return 1 + print("all checks passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/orca_vm/test_attack_discovery_board.py b/scripts/orca_vm/test_attack_discovery_board.py new file mode 100644 index 0000000000000..45b70373c8220 --- /dev/null +++ b/scripts/orca_vm/test_attack_discovery_board.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Guards for the attack-discovery board. + +Outcome-based: assert what lands in the HTML, not flag plumbing. +2026-09-11: aligned to the 9f7de5c renderer contract (--aggregate/--out/ +--missing). False-green reporting moved to render_false_green_bug_report.py. +""" +import json +import subprocess +import sys +import tempfile +import os + +HERE = os.path.dirname(os.path.abspath(__file__)) +RENDER = os.path.join(HERE, "render_attack_discovery_board.py") + +# One model with data at every assertable shape: mean/n cell, latency, blank +# totalRisk (absent field -> BLANK marker, never imputed), trace cards. +BASE = { + "suiteId": "attack-discovery-agent-builder", + "sourceDocCount": 100, + "reportedTotal": 100, + "modelCount": 1, + "absentFields": ["totalRisk"], + "models": [ + { + "modelId": "test-model", + "docs": 50, + "datasets": 2, + "discoveryCount": {"mean": 0.875, "n": 40}, + "alertsContextCount": {"mean": 2.5, "n": 40}, + "validatedDiscoveryCount": {"mean": 0.8, "n": 40}, + "status": {"completedRate": 1.0, "counts": {"completed": 40}, "n": 40}, + "latencySeconds": 35.7, + "generateLatencySeconds": 35.7, + "totalRisk": None, + "generateErrors": [], + "evaluators": {"Latency": {"mean": 35.7, "n": 40}}, + "traceCards": [ + { + "executionId": "exec0001abcdef012345", + "traceId": "trace0001234567890abcdef", + "insightCount": 2, + "insights": [ + { + "title": "Beaconing to rare domain", + "risk_score": 65, + "mitre_attack_tactics": ["command-and-control"], + "summary_markdown": "Multiple alerts reference a rare domain.", + }, + { + "title": "Cred dump via LSASS access", + "risk_score": 90, + "mitre_attack_tactics": ["credential-access"], + "summary_markdown": "LSASS access from a workstation.", + }, + ], + } + ], + } + ], +} + + +def render(agg, missing=()): + """Run the real renderer via its CURRENT CLI. Returns (returncode, html).""" + with tempfile.TemporaryDirectory() as d: + a = os.path.join(d, "agg.json") + o = os.path.join(d, "out.html") + with open(a, "w") as fh: + json.dump(agg, fh) + cmd = [sys.executable, RENDER, "--aggregate", a, "--out", o] + if missing: + m = os. path.join(d, "missing.json") + with open(m, "w") as fh: + json.dump(list(missing), fh) + cmd += ["--missing", m] + p = subprocess.run(cmd, capture_output=True, text=True) + out = open(o).read() if os.path.exists(o) else "" + return p.returncode, out +# PLACEHOLDER_CHECKS +failures = [] +def check(name, got, expected): + if got != expected: + failures.append(f"FAIL: {name}") + print(f"FAIL: {name} | got {got!r} expected {expected!r}") + else: + print(f"ok: {name}") + + +rc, out = render(BASE) +check("renderer exits 0 on a valid aggregate", rc, 0) +check("discoveryCount mean reaches HTML", ">1 10", S.parse_scores(out_of_range) is None) + +missing_dim = '{"overall": 7.5, "correctness": 80, "rationale": "x"}' +check("rejects missing dims", S.parse_scores(missing_dim) is None) + +fenced = f"```json\n{good}\n```" +check("parses fenced json", S.parse_scores(fenced) is not None) + +# gpt-5.5 truncation mode: finish=stop mid-rationale. The numeric fields all +# survive before the cut; the repair closes the object and keeps them. +truncated = '{"overall":6.0,"correctness":55,"groundedness":45,"completeness":65,"actionability":75,"rationale":"The answer gives a c' +check("repairs truncated tail (gpt-5.5 mode)", S.parse_scores(truncated) is not None) +rep = S.parse_scores(truncated) +check("repaired scores keep all dims", + rep and all(k in rep for k in ("correctness", "groundedness", "completeness", "actionability"))) + +two = good + '{"overall": 3.0, "correctness": 30, "groundedness": 30, "completeness": 30, "actionability": 30, "rationale": "b"}' +check("takes FIRST object when adjacent", (S.parse_scores(two) or {}).get("overall") == 7.5) + +print() +if FAILS: + print(f"{len(FAILS)} FAIL: {FAILS}") + sys.exit(1) +print("all checks passed") diff --git a/scripts/orca_vm/test_sweep_controller.py b/scripts/orca_vm/test_sweep_controller.py new file mode 100644 index 0000000000000..c01e9229798ca --- /dev/null +++ b/scripts/orca_vm/test_sweep_controller.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""End-to-end sweep-controller test: Azure + SSH fully stubbed. + +Covers the behaviours that live inside main()'s provision/launch loop and so +cannot be reached from the in-file --self-test unit checks: + + 1. the quota gate actually ABORTS a sweep (exit 2, zero VMs created); + 2. every finished unit gets parked, freeing its cores mid-sweep. + +Both were mutation-tested: reverting either behaviour turns this test red. +""" +import importlib.util +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path +from types import SimpleNamespace + +HERE = Path(__file__).parent + + +def load_sweep(): + """Import the sweep module fresh, with a temp run dir so nothing leaks.""" + spec = importlib.util.spec_from_file_location( + "sweep_under_test", HERE / "persona_matrix_sweep.py" + ) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class Recorder: + """Stands in for az(), recording every command the controller issues.""" + + def __init__(self, used=0, limit=350): + self.calls = [] + self.used = used + self.limit = limit + self._ips: dict = {} + + def __call__(self, *args): + self.calls.append(args) + if args[:2] == ("vm", "list-usage"): + return json.dumps( + [ + { + "name": {"value": "standardDSv5Family"}, + "currentValue": self.used, + "limit": self.limit, + } + ] + ) + if args[:2] == ("vm", "show"): + if "powerState" in args: + return "VM running" + # Unique IP per VM name: the controller rightly aborts when two + # units land on one box, so a shared stub IP would mask the test. + name = args[args.index("-n") + 1] if "-n" in args else "vm" + self._ips.setdefault(name, f"10.0.0.{len(self._ips) + 1}") + return json.dumps(self._ips[name]) + return "[]" + + def verbs(self, verb): + return [c for c in self.calls if c[:2] == ("vm", verb)] + + +def harness(mod, tmp, recorder): + """Neutralise every real side effect: Azure, SSH, scp, golden, sleep.""" + mod.az = recorder + mod.RUN_DIR = Path(tmp) + mod.provision_wait_seconds = lambda *a, **k: 0 + mod.wait_ssh = lambda ip: True + mod.deploy = lambda ip: None + mod.scp = lambda *a, **k: None + mod.ssh = lambda *a, **k: "ok" + mod.launch = lambda ip, model, shard=None: SimpleNamespace(wait=lambda: 0) + mod.time.sleep = lambda *a: None + # The controller polls the VM for /tmp/unit.done, then gates on golden. + mod.subprocess.run = lambda *a, **k: SimpleNamespace(stdout="done 0", returncode=0) + mod.check_golden = lambda model, ip, shard=None: { + "count": 98, + "expected": 98, + "gate": "exact", + "execution_id": "stub::suite::model", + } + return mod + + +def run_main(mod, argv): + saved = sys.argv + sys.argv = ["persona_matrix_sweep.py"] + argv + try: + return mod.main() + finally: + sys.argv = saved + + +def test_quota_gate_aborts_before_provisioning(): + """23 units x 8 cores into 344/350 used must abort, creating nothing.""" + mod = load_sweep() + rec = Recorder(used=344, limit=350) + with tempfile.TemporaryDirectory() as tmp: + harness(mod, tmp, rec) + rc = run_main(mod, ["--models", ",".join(mod.MODELS[:23]), "--shards", "1"]) + assert rc == 2, f"expected exit 2 (quota abort), got {rc}" + assert not rec.verbs("create"), ( + f"quota gate let {len(rec.verbs('create'))} VM create(s) through -- " + "this is the 2026-09-06 10h incident" + ) + return "quota gate aborts: exit 2, 0 creates" + + +def test_quota_gate_allows_a_fitting_sweep(): + """The same sweep fits under a clean quota and must proceed to create.""" + mod = load_sweep() + rec = Recorder(used=64, limit=350) + with tempfile.TemporaryDirectory() as tmp: + harness(mod, tmp, rec) + rc = run_main(mod, ["--models", "eis-openai-gpt-5-4", "--shards", "1"]) + assert rec.verbs("create"), "fitting sweep created no VM -- gate is over-blocking" + return f"fitting sweep proceeds: {len(rec.verbs('create'))} create(s), rc={rc}" + + +def test_finished_units_are_parked(): + """Each completed unit must be deallocated, not left billing.""" + mod = load_sweep() + rec = Recorder(used=0, limit=350) + with tempfile.TemporaryDirectory() as tmp: + harness(mod, tmp, rec) + rc = run_main(mod, ["--models", "eis-openai-gpt-5-4,eis-openai-gpt-5-4-mini"]) + deallocs = rec.verbs("deallocate") + assert rc == 0, f"stubbed sweep should pass, got rc={rc}" + assert len(deallocs) == 2, ( + f"expected 2 units parked, got {len(deallocs)} -- finished VMs " + "holding cores is what exhausted the quota twice" + ) + assert not rec.verbs("delete"), "park must deallocate (reusable), never delete" + return f"parked {len(deallocs)}/2 finished units via deallocate" + + +def test_park_opt_out(): + """PARK_ON_DONE=0 keeps VMs up for debugging.""" + mod = load_sweep() + rec = Recorder(used=0, limit=350) + os.environ["PARK_ON_DONE"] = "0" + try: + with tempfile.TemporaryDirectory() as tmp: + harness(mod, tmp, rec) + run_main(mod, ["--models", "eis-openai-gpt-5-4"]) + finally: + os.environ.pop("PARK_ON_DONE", None) + assert not rec.verbs("deallocate"), "PARK_ON_DONE=0 still deallocated" + return "park opt-out honoured: 0 deallocates" + + +TESTS = [ + test_quota_gate_aborts_before_provisioning, + test_quota_gate_allows_a_fitting_sweep, + test_finished_units_are_parked, + test_park_opt_out, +] + + +def main(): + failures = [] + for t in TESTS: + try: + detail = t() + print(f" PASS {t.__name__}: {detail}") + except AssertionError as exc: + failures.append(f"{t.__name__}: {exc}") + print(f" FAIL {t.__name__}: {exc}") + except Exception as exc: # noqa: BLE001 + failures.append(f"{t.__name__}: unexpected {exc!r}") + print(f" ERROR {t.__name__}: {exc!r}") + print(f"controller e2e: {len(failures)} failure(s)") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/orca_vm/to_matrix_trace_entries.py b/scripts/orca_vm/to_matrix_trace_entries.py new file mode 100644 index 0000000000000..f7d840ccf94ae --- /dev/null +++ b/scripts/orca_vm/to_matrix_trace_entries.py @@ -0,0 +1,142 @@ +"""Convert the ES-direct trace cache into the shape the matrix renderer reads. + +`build_trace_cache.py` emits `execid::suite::model::example` -> [raw ES score docs]. +`render_matrix_html.ts` looks up `traceKey(modelId, columnId)` == `modelId:columnId` +and reads `steps` / `question` / `toolTrail` at the TOP level of the entry. + +Neither the key shape nor the value shape matches, so passing the raw cache +straight to the renderer produces exit 0, zero warnings, and a board with no +trace cards at all. This converter bridges the two contracts. + +usage: to_matrix_trace_entries.py +""" + +import collections +import json +import sys + +# Upstream (query_matrix_traces.ts:95, trajectory_agreement.ts:407) treats the +# last message over 50 chars as the final answer; match it exactly so offline +# and live renders agree. +MIN_ANSWER_CHARS = 50 + + +def out_of(doc): + return (doc.get('task') or {}).get('output') or {} + + +def raw_steps(doc): + s = out_of(doc).get('steps') + return s if isinstance(s, list) else [] + + +def to_step(s): + kind = s.get('type') + if kind == 'tool_call' or s.get('tool_id'): + args = s.get('args') + return { + 'type': 'tool', + 'toolId': s.get('tool_id') or s.get('toolId'), + 'toolParams': json.dumps(args)[:600] if args is not None else None, + } + if kind == 'skill' or s.get('skills'): + return {'type': 'skill', 'skills': s.get('skills') or []} + txt = s.get('text') or s.get('content') or s.get('reasoning') + return {'type': 'reasoning', 'text': (txt or '')[:2000]} + + +def to_entry(doc, sibling_docs): + steps = [to_step(s) for s in raw_steps(doc)] + steps = [{k: v for k, v in s.items() if v is not None} for s in steps] + trail = [s['toolId'] for s in steps if s.get('type') == 'tool' and s.get('toolId')] + + example = doc.get('example') or {} + question = example.get('input') + if isinstance(question, dict): + question = question.get('question') or json.dumps(question)[:2000] + + # Both fields come from the sibling docs, not just the richest-steps one: + # 22 of 759 cells keep their steps and their final message in DIFFERENT + # documents. The answer lives at `messages[].message` as a plain string -- + # there is no `output.answer` field, so probing for one silently strips + # every card's conclusion ("No final answer message captured."). Upstream + # rule: the LAST message over MIN_ANSWER_CHARS wins; shorter trailing + # messages are tool chatter. + answer = None + scores = {} + for sibling in sibling_docs: + for msg in out_of(sibling).get('messages') or []: + text = msg.get('message') if isinstance(msg, dict) else None + if isinstance(text, str) and len(text.strip()) > MIN_ANSWER_CHARS: + answer = text + evaluator = sibling.get('evaluator') or {} + name, value = evaluator.get('name'), evaluator.get('score') + if name is not None and isinstance(value, (int, float)): + scores.setdefault(name, []).append(value) + + entry = { + 'steps': steps, + 'stepCount': len(steps), + 'toolCount': len(trail), + 'toolTrail': trail, + } + if question: + entry['question'] = question if isinstance(question, str) else str(question) + if isinstance(answer, str) and answer: + entry['answer'] = answer[:4000] + if scores: + entry['scores'] = {k: sum(v) / len(v) for k, v in scores.items()} + reps = max(len(v) for v in scores.values()) + if reps > 1: + entry['repetitions'] = reps + entry['spread'] = {k: max(v) - min(v) for k, v in scores.items() if len(v) > 1} + return entry + + +def main(): + if len(sys.argv) < 3: + sys.exit('usage: to_matrix_trace_entries.py ') + src, dst = sys.argv[1], sys.argv[2] + + raw = json.load(open(src)) + grouped = collections.defaultdict(list) + for key, docs in raw.items(): + parts = key.split('::') + if len(parts) < 4 or not isinstance(docs, list) or not docs: + continue + grouped[(parts[2], parts[3])].extend(docs) + + entries = {} + for (model, example), docs in grouped.items(): + # Several executions can cover one (model, example); keep the richest + # trail rather than whichever the scan happened to see first. + best = max(docs, key=lambda d: len(raw_steps(d))) + if not raw_steps(best): + continue + entries[f'{model}:{example}'] = to_entry(best, docs) + + json.dump(entries, open(dst, 'w')) + with_steps = sum(1 for e in entries.values() if e.get('steps')) + with_question = sum(1 for e in entries.values() if e.get('question')) + with_answer = sum(1 for e in entries.values() if e.get('answer')) + print(f'grouped pairs : {len(grouped)}') + print(f'entries : {len(entries)}') + print(f' with steps : {with_steps}') + print(f' with question: {with_question}') + print(f' with answer : {with_answer}') + print(f' models : {len({k.split(":", 1)[0] for k in entries})}') + print(f'wrote : {dst}') + + # A field that is empty for EVERY entry is a probe reading the wrong key, + # not a corpus that happens to lack it. Printing the zero and continuing is + # how boards shipped with "No final answer message captured." on every card. + for field, count in (('steps', with_steps), ('question', with_question), ('answer', with_answer)): + if entries and count == 0: + sys.exit( + f"error: 0 of {len(entries)} entries carry '{field}'. That is a field-path " + f"bug in this converter, not missing data -- fix the probe before rendering." + ) + + +if __name__ == '__main__': + main() diff --git a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_security_persona_matrix/stateful/classic.stateful.config.ts b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_security_persona_matrix/stateful/classic.stateful.config.ts index dca978dee1e62..385e5cf0d8a5a 100644 --- a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_security_persona_matrix/stateful/classic.stateful.config.ts +++ b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_security_persona_matrix/stateful/classic.stateful.config.ts @@ -43,15 +43,26 @@ import { servers as evalsTracingConfig } from '../../evals_tracing/stateful/clas * alert-analysis, alert-triage, workflow-authoring, entity-analytics (v1 path), * detection-rule-edit (base), find-security-ml-jobs. * + * `server.maxPayload` is raised from the Scout default (1.6 MB) because the + * judge's `/internal/inference/prompt` requests carry the full agent trajectory + * and exceed it on long multi-step examples (observed 413 on + * gemini-3.1-flash-lite, 2026-08-20). The inherited arg is filtered out and + * re-set here so the value is deterministic regardless of CLI arg ordering. + * * Usage: * node scripts/scout start-server --arch stateful --domain classic --serverConfigSet evals_security_persona_matrix */ +const MAX_PAYLOAD_BYTES = 50 * 1024 * 1024; // 50 MB + export const servers: ScoutServerConfig = { ...evalsTracingConfig, kbnTestServer: { ...evalsTracingConfig.kbnTestServer, serverArgs: [ - ...evalsTracingConfig.kbnTestServer.serverArgs, + ...evalsTracingConfig.kbnTestServer.serverArgs.filter( + (arg) => !arg.startsWith('--server.maxPayload=') + ), + `--server.maxPayload=${MAX_PAYLOAD_BYTES}`, '--uiSettings.overrides.agentBuilder:experimentalFeatures=true', `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'automaticTroubleshootingSkill', diff --git a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_tool_details.test.ts b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_tool_details.test.ts new file mode 100644 index 0000000000000..b553bf3e9b4ee --- /dev/null +++ b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_tool_details.test.ts @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import fs from 'fs'; +import path from 'path'; + +/** + * Every `evals_*` config set must boot with + * `agentBuilder:tracing:includeToolDetails=true`. + * + * Agent Builder's span processor strips `gen_ai.tool.call.arguments` at span + * creation unless the setting is on (it defaults to false for privacy). Every + * trace-based evaluator that matches on those arguments -- SkillInvoked looks + * for the skill name inside them -- then scores 0 for EVERY model, and the + * board reads as "models stopped invoking skills" rather than "the attribute + * is missing". The scores look plausible, which is what makes it dangerous. + * + * The sibling test in `evals_tracing` pins the setting for the base config. + * This one pins the *invariant across the whole family*, so a new eval suite + * that forgets to inherit -- or inherits from a config that later drops it -- + * fails here instead of silently producing a board of false zeros weeks later. + * + * Deliberately a filesystem sweep rather than a hardcoded list: a new + * `evals_*` directory is covered the moment it is added, without anyone + * remembering to update this file. + */ +describe('evals_* config sets: tool-detail capture', () => { + const CONFIG_SETS_DIR = __dirname; + const SETTING = 'agentBuilder:tracing:includeToolDetails'; + const ENABLED = `--uiSettings.overrides.${SETTING}=true`; + + const evalsConfigSets = fs + .readdirSync(CONFIG_SETS_DIR, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && entry.name.startsWith('evals_')) + .map((entry) => entry.name) + .sort(); + + const configFilesFor = (configSet: string): string[] => { + const setDir = path.join(CONFIG_SETS_DIR, configSet); + return fs + .readdirSync(setDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .flatMap((arch) => + fs + .readdirSync(path.join(setDir, arch.name)) + .filter((file) => file.endsWith('.config.ts')) + .map((file) => path.join(setDir, arch.name, file)) + ); + }; + + const loadServerArgs = (configPath: string): string[] => { + let servers: { kbnTestServer?: { serverArgs?: string[] } } | undefined; + jest.isolateModules(() => { + // eslint-disable-next-line @typescript-eslint/no-var-requires + servers = require(configPath).servers; + }); + return servers?.kbnTestServer?.serverArgs ?? []; + }; + + it('finds the evals_* config sets to check', () => { + // Guards the sweep itself: if the directory layout moves, the loop below + // would silently iterate over nothing and pass while checking nothing. + expect(evalsConfigSets.length).toBeGreaterThan(5); + }); + + describe.each(evalsConfigSets)('%s', (configSet) => { + const configFiles = configFilesFor(configSet); + + it('has at least one server config', () => { + expect(configFiles.length).toBeGreaterThan(0); + }); + + it.each(configFiles.map((f) => [path.relative(CONFIG_SETS_DIR, f), f]))( + 'captures tool call arguments in %s', + (_relative, configPath) => { + const serverArgs = loadServerArgs(configPath as string); + + // An eval stack that boots no Kibana args at all cannot run evals; if + // that ever happens the config has bigger problems than this setting. + expect(serverArgs.length).toBeGreaterThan(0); + expect(serverArgs).toContain(ENABLED); + } + ); + + it.each(configFiles.map((f) => [path.relative(CONFIG_SETS_DIR, f), f]))( + 'never disables tool detail capture in %s', + (_relative, configPath) => { + const serverArgs = loadServerArgs(configPath as string); + const disabling = serverArgs.filter( + (arg) => arg.includes(SETTING) && !arg.endsWith('=true') + ); + + expect(disabling).toEqual([]); + } + ); + }); +}); diff --git a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_tracing/stateful/classic.stateful.config.test.ts b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_tracing/stateful/classic.stateful.config.test.ts new file mode 100644 index 0000000000000..969e8139ccb98 --- /dev/null +++ b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_tracing/stateful/classic.stateful.config.test.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "Elastic License 2.0, Server Side Public License v 1", and the + * "Server Side Public License v 1" ("SSPL") - as separate files with distinct + * license terms. Your choice of license determines the rights and obligations + * associated with the software. See the LICENSE.txt file in the project root + * for the applicable license terms. + */ + +/** + * Agent Builder's span processor strips `gen_ai.tool.call.arguments` and + * `gen_ai.tool.call.result` from every tool span unless + * `agentBuilder:tracing:includeToolDetails` is enabled -- it defaults to false + * for privacy in production. + * + * Trace-based evaluators match on those arguments (SkillInvoked looks for the + * skill name inside them). Without the setting the attribute is simply absent, + * so the evaluator scores 0 for EVERY model and the result reads as a model + * failure rather than a missing attribute. + * + * This is exactly what happened on the Azure sweep VMs: 8,339 load_skill spans + * with 0 arguments, while Buildkite CI (which boots through this config set) + * had 3,953/3,953 populated. The bug is invisible in the scores -- it looks + * like the models stopped invoking skills. + */ +describe('evals_tracing config set', () => { + const ENV_KEYS = ['CI', 'TRACING_EXPORTERS'] as const; + let savedEnv: Record; + + beforeEach(() => { + savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]])); + }); + + afterEach(() => { + for (const k of ENV_KEYS) { + if (savedEnv[k] === undefined) { + delete process.env[k]; + } else { + process.env[k] = savedEnv[k]; + } + } + }); + + const loadServerArgs = (): string[] => { + let servers: { kbnTestServer: { serverArgs: string[] } }; + jest.isolateModules(() => { + // eslint-disable-next-line @typescript-eslint/no-var-requires + servers = require('./classic.stateful.config').servers; + }); + // @ts-expect-error assigned inside isolateModules + return servers.kbnTestServer.serverArgs; + }; + + const TOOL_DETAILS_ARG = '--uiSettings.overrides.agentBuilder:tracing:includeToolDetails=true'; + const TRACING_ARG = '--telemetry.tracing.enabled=true'; + + // The config gates these args on `Boolean(TRACING_EXPORTERS) || !CI`, so the + // exporter case is the one that must hold ON CI -- where `!isCi` is false and + // an unset exporter would silently drop both args. + describe('on CI, with exporters configured', () => { + let serverArgs: string[]; + + beforeEach(() => { + process.env.CI = 'true'; + process.env.TRACING_EXPORTERS = JSON.stringify([{ type: 'console' }]); + serverArgs = loadServerArgs(); + }); + + it('captures tool call arguments so trace-based evaluators can match on them', () => { + expect(serverArgs).toContain(TOOL_DETAILS_ARG); + }); + + it('enables tracing itself, so the tool-details setting is not dead config', () => { + expect(serverArgs).toContain(TRACING_ARG); + }); + }); + + describe('off CI, without exporters', () => { + let serverArgs: string[]; + + beforeEach(() => { + delete process.env.CI; + delete process.env.TRACING_EXPORTERS; + serverArgs = loadServerArgs(); + }); + + it('still captures tool call arguments for local eval runs', () => { + expect(serverArgs).toContain(TOOL_DETAILS_ARG); + expect(serverArgs).toContain(TRACING_ARG); + }); + }); + + // The regression case: on CI with no exporters configured, `shouldEnableTracing` + // is false. The tool-details setting used to live inside that block, so it + // silently disappeared here -- SkillInvoked then scores 0 for every model and + // reads as model failure. It must hold regardless of whether tracing is on. + describe('on CI, without exporters', () => { + let serverArgs: string[]; + + beforeEach(() => { + process.env.CI = 'true'; + delete process.env.TRACING_EXPORTERS; + serverArgs = loadServerArgs(); + }); + + it('still captures tool call arguments when tracing is disabled', () => { + expect(serverArgs).toContain(TOOL_DETAILS_ARG); + }); + + it('does not enable tracing in this configuration', () => { + expect(serverArgs).not.toContain(TRACING_ARG); + }); + }); +}); diff --git a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_tracing/stateful/classic.stateful.config.ts b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_tracing/stateful/classic.stateful.config.ts index 431cd60d7b483..356a474bd0a63 100644 --- a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_tracing/stateful/classic.stateful.config.ts +++ b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_tracing/stateful/classic.stateful.config.ts @@ -97,6 +97,15 @@ const { TRACING_EXPORTERS: tracingExporters } = process.env; if (tracingExporters) { JSON.parse(tracingExporters); // validate parseable JSON; throws early if malformed } +// Agent Builder maintains its own tracer provider (register_tracing.ts) whose +// spans back the trace-based evaluators. It always exports to the LOCAL ES via +// ElasticsearchOtlpExporter; entries here are APPENDED, so setting this adds a +// remote destination (e.g. the golden cluster) without losing local fidelity. +const { AGENT_BUILDER_TRACING_EXPORTERS: agentBuilderTracingExporters } = process.env; +if (agentBuilderTracingExporters) { + JSON.parse(agentBuilderTracingExporters); // validate parseable JSON; throws early if malformed +} + const isCi = Boolean(process.env.CI); const shouldEnableTracing = Boolean(tracingExporters) || !isCi; const exporters = tracingExporters ?? defaultExporters; @@ -133,6 +142,14 @@ export const servers: ScoutServerConfig = { ...defaultConfig.kbnTestServer.serverArgs, '--xpack.evals.enabled=true', ...(preconfiguredEisConnectorsArg ? [preconfiguredEisConnectorsArg] : []), + // Unconditional: Agent Builder's span processor strips gen_ai.tool.call.arguments + // and .result from every tool span unless this is on (it defaults to false for + // privacy). Trace-based evaluators that match on tool call arguments -- + // SkillInvoked matches the skill name inside them -- then score 0 for every + // model, which reads as a model failure rather than a missing attribute. + // Evals run on synthetic data, so always capture the details: gating this on + // `shouldEnableTracing` silently drops it whenever exporters are unset. + '--uiSettings.overrides.agentBuilder:tracing:includeToolDetails=true', ...(shouldEnableTracing ? [ '--elastic.apm.active=false', @@ -141,6 +158,9 @@ export const servers: ScoutServerConfig = { '--telemetry.tracing.enabled=true', '--telemetry.tracing.sample_rate=1', `--telemetry.tracing.exporters=${exporters}`, + ...(agentBuilderTracingExporters + ? [`--xpack.agentBuilder.tracing.exporters=${agentBuilderTracingExporters}`] + : []), ] : []), ], diff --git a/x-pack/platform/packages/shared/kbn-evals-common/golden_cluster_privileges.ts b/x-pack/platform/packages/shared/kbn-evals-common/golden_cluster_privileges.ts index cf9bba3e5d0e5..6b0fa45a69a81 100644 --- a/x-pack/platform/packages/shared/kbn-evals-common/golden_cluster_privileges.ts +++ b/x-pack/platform/packages/shared/kbn-evals-common/golden_cluster_privileges.ts @@ -22,23 +22,11 @@ export const goldenClusterPrivileges = { indices: [ { names: [`${EvaluationIndices.SCORES}*`], - privileges: [ - 'auto_configure', - 'create_index', - 'create_doc', - 'read', - 'view_index_metadata', - ], + privileges: ['auto_configure', 'create_index', 'create', 'read', 'view_index_metadata'], }, { names: ['traces-*'], - privileges: [ - 'auto_configure', - 'create_index', - 'create_doc', - 'read', - 'view_index_metadata', - ], + privileges: ['auto_configure', 'create_index', 'create', 'read', 'view_index_metadata'], }, { names: [ @@ -49,7 +37,7 @@ export const goldenClusterPrivileges = { privileges: [ 'auto_configure', 'create_index', - 'create_doc', + 'create', 'read', 'view_index_metadata', 'delete', diff --git a/x-pack/platform/packages/shared/kbn-evals-common/impl/query_builders.test.ts b/x-pack/platform/packages/shared/kbn-evals-common/impl/query_builders.test.ts index 263df2f2f3467..2f61977efbd94 100644 --- a/x-pack/platform/packages/shared/kbn-evals-common/impl/query_builders.test.ts +++ b/x-pack/platform/packages/shared/kbn-evals-common/impl/query_builders.test.ts @@ -82,6 +82,18 @@ describe('query_builders', () => { expect(query.bool.must).toHaveLength(2); expect(query.bool.must[1]).toEqual(buildSpaceFilter('marketing')); }); + + it('adds execution and model filters when provided', () => { + const query = buildExampleScoresQuery('example-123', { + executionId: 'run-abc', + modelId: 'openai-gpt-5.4', + }); + expect(query.bool.must).toEqual([ + { term: { 'example.id': 'example-123' } }, + { term: { 'metadata.execution_id': 'run-abc' } }, + { term: { 'task.model.id': 'openai-gpt-5.4' } }, + ]); + }); }); describe('buildDatasetExampleScoresQuery', () => { diff --git a/x-pack/platform/packages/shared/kbn-evals-common/impl/query_builders.ts b/x-pack/platform/packages/shared/kbn-evals-common/impl/query_builders.ts index 900391835d860..b3c6fadcbd0c0 100644 --- a/x-pack/platform/packages/shared/kbn-evals-common/impl/query_builders.ts +++ b/x-pack/platform/packages/shared/kbn-evals-common/impl/query_builders.ts @@ -136,12 +136,18 @@ export const buildExperimentFilterQuery = ( */ export const buildExampleScoresQuery = ( exampleId: string, - options?: { spaceId?: string; datasetId?: string } + options?: { spaceId?: string; datasetId?: string; executionId?: string; modelId?: string } ): { bool: { must: Array> } } => { const must: Array> = [{ term: { 'example.id': exampleId } }]; if (options?.datasetId !== undefined) { must.push({ term: { 'example.dataset.id': options.datasetId } }); } + if (options?.executionId) { + must.push({ term: { 'metadata.execution_id': options.executionId } }); + } + if (options?.modelId) { + must.push({ term: { 'task.model.id': options.modelId } }); + } if (options?.spaceId) { must.push(buildSpaceFilter(options.spaceId)); } diff --git a/x-pack/platform/packages/shared/kbn-evals-common/impl/schemas/examples/get_example_scores_route.gen.ts b/x-pack/platform/packages/shared/kbn-evals-common/impl/schemas/examples/get_example_scores_route.gen.ts index f51b4239facb7..538246150275a 100644 --- a/x-pack/platform/packages/shared/kbn-evals-common/impl/schemas/examples/get_example_scores_route.gen.ts +++ b/x-pack/platform/packages/shared/kbn-evals-common/impl/schemas/examples/get_example_scores_route.gen.ts @@ -24,6 +24,14 @@ export const GetExampleScoresRequestQuery = lazySchema(() => * Filter scores to a specific dataset. When omitted, scores from all datasets matching the example ID are returned. */ dataset_id: z.string().min(1).max(1024).optional(), + /** + * Filter by execution ID (the full composite execution identifier) + */ + execution_id: z.string().max(1024).optional(), + /** + * Filter by task model ID + */ + model_id: z.string().max(256).optional(), }) ); export type GetExampleScoresRequestQuery = z.infer; diff --git a/x-pack/platform/packages/shared/kbn-evals-common/impl/schemas/examples/get_example_scores_route.schema.yaml b/x-pack/platform/packages/shared/kbn-evals-common/impl/schemas/examples/get_example_scores_route.schema.yaml index 7a2263b5166a1..e945fdddfe499 100644 --- a/x-pack/platform/packages/shared/kbn-evals-common/impl/schemas/examples/get_example_scores_route.schema.yaml +++ b/x-pack/platform/packages/shared/kbn-evals-common/impl/schemas/examples/get_example_scores_route.schema.yaml @@ -27,6 +27,20 @@ paths: minLength: 1 maxLength: 1024 description: Filter scores to a specific dataset. When omitted, scores from all datasets matching the example ID are returned. + - name: execution_id + in: query + required: false + schema: + type: string + maxLength: 1024 + description: Filter by execution ID (the full composite execution identifier) + - name: model_id + in: query + required: false + schema: + type: string + maxLength: 256 + description: Filter by task model ID responses: "200": description: Successful response diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/README.md b/x-pack/platform/packages/shared/kbn-evals-extensions/README.md index cc003d8096714..40b4fe71ed3bf 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/README.md +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/README.md @@ -26,6 +26,50 @@ node scripts/evals ext --help node scripts/evals ext [command] [...args] ``` +### Matrix configs + +| Config | Shape | Use | +|---|---|---| +| `config/security_matrix.json` | 32 columns | Full board: persona + attack-discovery slices + migrations | +| `config/security_matrix_persona.json` | 21 columns | Persona columns only — the column set the published persona matrix uses | + +`security_matrix_persona.json` is `security_matrix.json` restricted to the column +ids in the persona suite's `persona_matrix.production.config.json`. Its scores are +identical to the full board's; only `overall` differs, because it averages over 21 +columns instead of 32. Boards built from the two configs are therefore **not** +comparable on `overall`. + +### Regenerating a board from golden + +```bash +source ~/.elastic/golden-cluster-env.sh + +# 1. Aggregate scores (applies the scoring policy, emits a .policy.json stamp) +OUT_JSON=/tmp/aggregated.json node --require ./src/setup_node_env \ + x-pack/platform/packages/shared/kbn-evals-extensions/scripts/extract_golden_aggregate.ts + +# 2. Build the trace cache — the renderer refuses to publish a board without it +python3 scripts/orca_vm/build_trace_cache.py --out /tmp/trace_cache.json + +# 3. Render +AGGREGATED_JSON=/tmp/aggregated.json \ +TRACES_JSON=/tmp/trace_cache.json \ +MATRIX_CONFIG=x-pack/platform/packages/shared/kbn-evals-extensions/config/security_matrix_persona.json \ +OUT_DIR=target/persona_board \ + node --require ./src/setup_node_env \ + x-pack/platform/packages/shared/kbn-evals-extensions/scripts/render_from_golden.ts +``` + +The renderer refuses to emit a board when the extract cannot satisfy the config — +too few cells resolve, `examplePrefixes` columns with no `prefix:` keys, a scoring +policy the extract did not apply, or missing traces. Each refusal names the cause +and its override (`ALLOW_UNENFORCED_SCORING=1`, `ALLOW_NO_TRACES=1`); a scores-only +board publishes numbers with no transcript behind them, so override deliberately. + +Pin a single grader with `JUDGE_MODEL_ID=` on the extract step. Note that +golden is mixed-judge: pinning drops every cell graded by anyone else, and the +published provenance reports the real mix rather than asserting one judge. + ### In an evaluation suite Suites opt in to extension features by importing them from `@kbn/evals-extensions` explicitly, alongside `@kbn/evals` core: diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/config/security_matrix.json b/x-pack/platform/packages/shared/kbn-evals-extensions/config/security_matrix.json new file mode 100644 index 0000000000000..8ee65bc59301d --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/config/security_matrix.json @@ -0,0 +1,567 @@ +{ + "title": "Security LLM performance matrix", + "branch": "main", + "lookbackDays": 120, + "columns": [ + { + "id": "alert-analysis-a", + "label": "Alert Analysis A", + "group": "Alert Analysis", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "alert-analysis-a" + ] + }, + { + "id": "alert-analysis-b", + "label": "Alert Analysis B", + "group": "Alert Analysis", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "alert-analysis-b" + ] + }, + { + "id": "alert-analysis-c", + "label": "Alert Analysis C", + "group": "Alert Analysis", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "alert-analysis-c" + ] + }, + { + "id": "entity-analytics-a", + "label": "Entity Analytics A", + "group": "Entity Analytics", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "entity-analytics-a" + ] + }, + { + "id": "entity-analytics-b", + "label": "Entity Analytics B", + "group": "Entity Analytics", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "entity-analytics-b" + ] + }, + { + "id": "entity-analytics-c", + "label": "Entity Analytics C", + "group": "Entity Analytics", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "entity-analytics-c" + ] + }, + { + "id": "threat-hunting-a", + "label": "Threat Hunting A", + "group": "Threat Hunting", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "threat-hunting-a" + ] + }, + { + "id": "threat-hunting-b", + "label": "Threat Hunting B", + "group": "Threat Hunting", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "threat-hunting-b" + ] + }, + { + "id": "threat-hunting-c", + "label": "Threat Hunting C", + "group": "Threat Hunting", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "threat-hunting-c" + ] + }, + { + "id": "detection-rule-edit-a", + "label": "Detection Rules A", + "group": "Detection Rules", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "detection-rule-edit-a" + ] + }, + { + "id": "detection-rule-edit-b", + "label": "Detection Rules B", + "group": "Detection Rules", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "detection-rule-edit-b" + ] + }, + { + "id": "detection-rule-edit-c", + "label": "Detection Rules C", + "group": "Detection Rules", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "detection-rule-edit-c" + ] + }, + { + "id": "workflow-authoring-a", + "label": "Workflow Authoring A", + "group": "Workflow Authoring", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "workflow-authoring-a" + ] + }, + { + "id": "workflow-authoring-b", + "label": "Workflow Authoring B", + "group": "Workflow Authoring", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "workflow-authoring-b" + ] + }, + { + "id": "workflow-authoring-c", + "label": "Workflow Authoring C", + "group": "Workflow Authoring", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "workflow-authoring-c" + ] + }, + { + "id": "workflow-execution-a", + "label": "Triggering Workflows A", + "group": "Triggering Workflows", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "workflow-execution-a" + ] + }, + { + "id": "workflow-execution-b", + "label": "Triggering Workflows B", + "group": "Triggering Workflows", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "workflow-execution-b" + ] + }, + { + "id": "workflow-execution-c", + "label": "Triggering Workflows C", + "group": "Triggering Workflows", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "workflow-execution-c" + ] + }, + { + "id": "multi-step-a", + "label": "Multi-Step Executions A", + "group": "Multi-Step Executions", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "multi-step-a" + ] + }, + { + "id": "multi-step-b", + "label": "Multi-Step Executions B", + "group": "Multi-Step Executions", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "multi-step-b" + ] + }, + { + "id": "multi-step-c", + "label": "Multi-Step Executions C", + "group": "Multi-Step Executions", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "multi-step-c" + ] + }, + { + "id": "migrations-start-e2e", + "label": "Migration Start (E2E)", + "group": "Automatic Migrations", + "suites": [ + "automatic-migrations" + ], + "datasetIds": [ + "05fd1e03-0e35-5abf-bfc6-c07118da0b28" + ] + }, + { + "id": "migrations-update-grounded", + "label": "Migration Update (Grounded)", + "group": "Automatic Migrations", + "suites": [ + "automatic-migrations" + ], + "datasetIds": [ + "07a6c75d-7b4b-5150-ab32-7b66a93ac910" + ] + }, + { + "id": "attack-discovery-live-retrieval", + "label": "GP: live-retrieval", + "group": "Attack Discovery", + "suites": [ + "attack-discovery-agent-builder" + ], + "datasetIds": [ + "8733e3f2-ce30-5de2-b1b6-f0e1ae99122e" + ] + }, + { + "id": "attack-discovery-provided-alerts", + "label": "GP: provided-alerts", + "group": "Attack Discovery", + "suites": [ + "attack-discovery-agent-builder" + ], + "datasetIds": [ + "e527cb2b-8a69-56eb-aa25-11a773090098" + ] + }, + { + "id": "attack-discovery-missing-alert-retrieval", + "label": "GP: missing-alert-retrieval", + "group": "Attack Discovery", + "suites": [ + "attack-discovery-agent-builder" + ], + "datasetIds": [ + "4c2fac6e-7f89-5255-a6fe-a12ab377f96c" + ] + }, + { + "id": "attack-discovery-multiple-alert-sets", + "label": "GP: multiple-alert-sets", + "group": "Attack Discovery", + "suites": [ + "attack-discovery-agent-builder" + ], + "datasetIds": [ + "489d196e-5ea3-5dd9-a978-ff13d3161d9b" + ] + }, + { + "id": "attack-discovery-status-only", + "label": "GP: status-only", + "group": "Attack Discovery", + "suites": [ + "attack-discovery-agent-builder" + ], + "datasetIds": [ + "54609e14-71ba-5b3f-9e5b-5dea57e45d5f" + ] + }, + { + "id": "attack-discovery-registry-encoded-powershell", + "label": "SR: encoded-powershell", + "group": "Attack Discovery", + "suites": [ + "attack-discovery-agent-builder" + ], + "datasetIds": [ + "90c0197c-27d1-5354-a5e1-e31e601c43f6" + ] + }, + { + "id": "attack-discovery-registry-linux-curl", + "label": "SR: linux-curl", + "group": "Attack Discovery", + "suites": [ + "attack-discovery-agent-builder" + ], + "datasetIds": [ + "2cc796b8-5033-59a7-a522-105c4a33d306" + ] + }, + { + "id": "attack-discovery-registry-bits-mshta", + "label": "SR: bits-mshta", + "group": "Attack Discovery", + "suites": [ + "attack-discovery-agent-builder" + ], + "datasetIds": [ + "9ab9bbac-2465-5f64-a2f8-7e6a59269d25" + ] + }, + { + "id": "attack-discovery-registry-wmi-lateral", + "label": "SR: wmi-lateral", + "group": "Attack Discovery", + "suites": [ + "attack-discovery-agent-builder" + ], + "datasetIds": [ + "55c9971d-aa30-536a-b305-663da5a43e38" + ] + } + ], + "composites": [], + "models": [ + { + "id": "anthropic-claude-4.5-haiku", + "label": "Claude Haiku 4.5", + "openSource": false + }, + { + "id": "anthropic-claude-4.5-opus", + "label": "Anthropic claude 4.5 opus", + "openSource": false + }, + { + "id": "anthropic-claude-4.5-sonnet", + "label": "Anthropic claude 4.5 sonnet", + "openSource": false + }, + { + "id": "anthropic-claude-4.6-opus", + "label": "Anthropic claude 4.6 opus", + "openSource": false + }, + { + "id": "anthropic-claude-4.6-sonnet", + "label": "Claude Sonnet 4.6", + "openSource": false + }, + { + "id": "anthropic-claude-4.7-opus", + "label": "Anthropic claude 4.7 opus", + "openSource": false + }, + { + "id": "anthropic-claude-4.8-opus", + "label": "Claude Opus 4.8", + "openSource": false + }, + { + "id": "anthropic-claude-5-sonnet", + "label": "Claude Sonnet 5", + "openSource": false + }, + { + "id": "deepseek/deepseek-v4-pro-0813", + "label": "Deepseek v4 pro 0813", + "openSource": true + }, + { + "id": "eis-anthropic-claude-4-5-haiku", + "label": "Anthropic claude 4 5 haiku", + "openSource": false + }, + { + "id": "eis-anthropic-claude-4-5-opus", + "label": "Anthropic claude 4 5 opus", + "openSource": false + }, + { + "id": "eis-anthropic-claude-4-6-opus", + "label": "Anthropic claude 4 6 opus", + "openSource": false + }, + { + "id": "eis-anthropic-claude-4-6-sonnet", + "label": "Anthropic claude 4 6 sonnet", + "openSource": false + }, + { + "id": "eis-anthropic-claude-4-7-opus", + "label": "Anthropic claude 4 7 opus", + "openSource": false + }, + { + "id": "eis-anthropic-claude-4-8-opus", + "label": "Anthropic claude 4 8 opus", + "openSource": false + }, + { + "id": "eis-anthropic-claude-5-sonnet", + "label": "Anthropic claude 5 sonnet", + "openSource": false + }, + { + "id": "eis-google-gemini-3-0-flash", + "label": "Google gemini 3 0 flash", + "openSource": false + }, + { + "id": "eis-google-gemini-3-1-pro", + "label": "Google gemini 3 1 pro", + "openSource": false + }, + { + "id": "eis-openai-gpt-5-2", + "label": "Openai gpt 5 2", + "openSource": false + }, + { + "id": "eis-openai-gpt-5-4", + "label": "Openai gpt 5 4", + "openSource": false + }, + { + "id": "eis-openai-gpt-5-4-mini", + "label": "Openai gpt 5 4 mini", + "openSource": false + }, + { + "id": "eis-openai-gpt-5-4-nano", + "label": "Openai gpt 5 4 nano", + "openSource": false + }, + { + "id": "eis-openai-gpt-5-5", + "label": "Openai gpt 5 5", + "openSource": false + }, + { + "id": "eis-openai-gpt-oss-120b", + "label": "Openai gpt oss 120b", + "openSource": true + }, + { + "id": "google-gemini-2.5-flash", + "label": "Google gemini 2.5 flash", + "openSource": false + }, + { + "id": "google-gemini-2.5-pro", + "label": "Google gemini 2.5 pro", + "openSource": false + }, + { + "id": "google-gemini-3.0-flash", + "label": "Google gemini 3.0 flash", + "openSource": false + }, + { + "id": "google-gemini-3.1-flash-lite", + "label": "Google gemini 3.1 flash lite", + "openSource": false + }, + { + "id": "google-gemini-3.1-pro", + "label": "Gemini 3.1 Pro", + "openSource": false + }, + { + "id": "google-gemini-3.5-flash", + "label": "Google gemini 3.5 flash", + "openSource": false + }, + { + "id": "openai-gpt-5.2", + "label": "Openai gpt 5.2", + "openSource": false + }, + { + "id": "openai-gpt-5.4", + "label": "GPT-5.4", + "openSource": false + }, + { + "id": "openai-gpt-5.4-mini", + "label": "GPT-5.4 Mini", + "openSource": false + }, + { + "id": "openai-gpt-5.4-nano", + "label": "GPT-5.4 Nano", + "openSource": false + }, + { + "id": "openai-gpt-5.5", + "label": "GPT-5.5", + "openSource": false + }, + { + "id": "openai-gpt-oss-120b", + "label": "Openai gpt oss 120b", + "openSource": true + }, + { + "id": "openai/gpt-5.6-sol", + "label": "Gpt 5.6 sol", + "openSource": false + }, + { + "id": "qwen/qwen3.8-27b", + "label": "Qwen3.8 27b", + "openSource": true + }, + { + "id": "qwen3.8-27b", + "label": "Qwen3.8 27b", + "openSource": true + }, + { + "id": "selfhost-qwen38", + "label": "Selfhost qwen38", + "openSource": true + }, + { + "id": "z-ai/glm-5.3-flash", + "label": "Glm 5.3 flash", + "openSource": true + } + ] +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/config/security_matrix_persona.json b/x-pack/platform/packages/shared/kbn-evals-extensions/config/security_matrix_persona.json new file mode 100644 index 0000000000000..997f9e8f55902 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/config/security_matrix_persona.json @@ -0,0 +1,461 @@ +{ + "title": "Security Persona Matrix (persona columns only)", + "branch": "main", + "lookbackDays": 120, + "columns": [ + { + "id": "alert-analysis-a", + "label": "Alert Analysis A", + "group": "Alert Analysis", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "alert-analysis-a" + ] + }, + { + "id": "alert-analysis-b", + "label": "Alert Analysis B", + "group": "Alert Analysis", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "alert-analysis-b" + ] + }, + { + "id": "alert-analysis-c", + "label": "Alert Analysis C", + "group": "Alert Analysis", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "alert-analysis-c" + ] + }, + { + "id": "entity-analytics-a", + "label": "Entity Analytics A", + "group": "Entity Analytics", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "entity-analytics-a" + ] + }, + { + "id": "entity-analytics-b", + "label": "Entity Analytics B", + "group": "Entity Analytics", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "entity-analytics-b" + ] + }, + { + "id": "entity-analytics-c", + "label": "Entity Analytics C", + "group": "Entity Analytics", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "entity-analytics-c" + ] + }, + { + "id": "threat-hunting-a", + "label": "Threat Hunting A", + "group": "Threat Hunting", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "threat-hunting-a" + ] + }, + { + "id": "threat-hunting-b", + "label": "Threat Hunting B", + "group": "Threat Hunting", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "threat-hunting-b" + ] + }, + { + "id": "threat-hunting-c", + "label": "Threat Hunting C", + "group": "Threat Hunting", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "threat-hunting-c" + ] + }, + { + "id": "detection-rule-edit-a", + "label": "Detection Rules A", + "group": "Detection Rules", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "detection-rule-edit-a" + ] + }, + { + "id": "detection-rule-edit-b", + "label": "Detection Rules B", + "group": "Detection Rules", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "detection-rule-edit-b" + ] + }, + { + "id": "detection-rule-edit-c", + "label": "Detection Rules C", + "group": "Detection Rules", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "detection-rule-edit-c" + ] + }, + { + "id": "workflow-authoring-a", + "label": "Workflow Authoring A", + "group": "Workflow Authoring", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "workflow-authoring-a" + ] + }, + { + "id": "workflow-authoring-b", + "label": "Workflow Authoring B", + "group": "Workflow Authoring", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "workflow-authoring-b" + ] + }, + { + "id": "workflow-authoring-c", + "label": "Workflow Authoring C", + "group": "Workflow Authoring", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "workflow-authoring-c" + ] + }, + { + "id": "workflow-execution-a", + "label": "Triggering Workflows A", + "group": "Triggering Workflows", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "workflow-execution-a" + ] + }, + { + "id": "workflow-execution-b", + "label": "Triggering Workflows B", + "group": "Triggering Workflows", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "workflow-execution-b" + ] + }, + { + "id": "workflow-execution-c", + "label": "Triggering Workflows C", + "group": "Triggering Workflows", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "workflow-execution-c" + ] + }, + { + "id": "multi-step-a", + "label": "Multi-Step Executions A", + "group": "Multi-Step Executions", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "multi-step-a" + ] + }, + { + "id": "multi-step-b", + "label": "Multi-Step Executions B", + "group": "Multi-Step Executions", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "multi-step-b" + ] + }, + { + "id": "multi-step-c", + "label": "Multi-Step Executions C", + "group": "Multi-Step Executions", + "suites": [ + "security-persona-matrix" + ], + "datasetIds": [ + "multi-step-c" + ] + } + ], + "composites": [], + "models": [ + { + "id": "anthropic-claude-4.5-haiku", + "label": "Claude Haiku 4.5", + "openSource": false + }, + { + "id": "anthropic-claude-4.5-opus", + "label": "Anthropic claude 4.5 opus", + "openSource": false + }, + { + "id": "anthropic-claude-4.5-sonnet", + "label": "Anthropic claude 4.5 sonnet", + "openSource": false + }, + { + "id": "anthropic-claude-4.6-opus", + "label": "Anthropic claude 4.6 opus", + "openSource": false + }, + { + "id": "anthropic-claude-4.6-sonnet", + "label": "Claude Sonnet 4.6", + "openSource": false + }, + { + "id": "anthropic-claude-4.7-opus", + "label": "Anthropic claude 4.7 opus", + "openSource": false + }, + { + "id": "anthropic-claude-4.8-opus", + "label": "Claude Opus 4.8", + "openSource": false + }, + { + "id": "anthropic-claude-5-sonnet", + "label": "Claude Sonnet 5", + "openSource": false + }, + { + "id": "deepseek/deepseek-v4-pro-0813", + "label": "Deepseek v4 pro 0813", + "openSource": true + }, + { + "id": "eis-anthropic-claude-4-5-haiku", + "label": "Anthropic claude 4 5 haiku", + "openSource": false + }, + { + "id": "eis-anthropic-claude-4-5-opus", + "label": "Anthropic claude 4 5 opus", + "openSource": false + }, + { + "id": "eis-anthropic-claude-4-6-opus", + "label": "Anthropic claude 4 6 opus", + "openSource": false + }, + { + "id": "eis-anthropic-claude-4-6-sonnet", + "label": "Anthropic claude 4 6 sonnet", + "openSource": false + }, + { + "id": "eis-anthropic-claude-4-7-opus", + "label": "Anthropic claude 4 7 opus", + "openSource": false + }, + { + "id": "eis-anthropic-claude-4-8-opus", + "label": "Anthropic claude 4 8 opus", + "openSource": false + }, + { + "id": "eis-anthropic-claude-5-sonnet", + "label": "Anthropic claude 5 sonnet", + "openSource": false + }, + { + "id": "eis-google-gemini-3-0-flash", + "label": "Google gemini 3 0 flash", + "openSource": false + }, + { + "id": "eis-google-gemini-3-1-pro", + "label": "Google gemini 3 1 pro", + "openSource": false + }, + { + "id": "eis-openai-gpt-5-2", + "label": "Openai gpt 5 2", + "openSource": false + }, + { + "id": "eis-openai-gpt-5-4", + "label": "Openai gpt 5 4", + "openSource": false + }, + { + "id": "eis-openai-gpt-5-4-mini", + "label": "Openai gpt 5 4 mini", + "openSource": false + }, + { + "id": "eis-openai-gpt-5-4-nano", + "label": "Openai gpt 5 4 nano", + "openSource": false + }, + { + "id": "eis-openai-gpt-5-5", + "label": "Openai gpt 5 5", + "openSource": false + }, + { + "id": "eis-openai-gpt-oss-120b", + "label": "Openai gpt oss 120b", + "openSource": true + }, + { + "id": "google-gemini-2.5-flash", + "label": "Google gemini 2.5 flash", + "openSource": false + }, + { + "id": "google-gemini-2.5-pro", + "label": "Google gemini 2.5 pro", + "openSource": false + }, + { + "id": "google-gemini-3.0-flash", + "label": "Google gemini 3.0 flash", + "openSource": false + }, + { + "id": "google-gemini-3.1-flash-lite", + "label": "Google gemini 3.1 flash lite", + "openSource": false + }, + { + "id": "google-gemini-3.1-pro", + "label": "Gemini 3.1 Pro", + "openSource": false + }, + { + "id": "google-gemini-3.5-flash", + "label": "Google gemini 3.5 flash", + "openSource": false + }, + { + "id": "openai-gpt-5.2", + "label": "Openai gpt 5.2", + "openSource": false + }, + { + "id": "openai-gpt-5.4", + "label": "GPT-5.4", + "openSource": false + }, + { + "id": "openai-gpt-5.4-mini", + "label": "GPT-5.4 Mini", + "openSource": false + }, + { + "id": "openai-gpt-5.4-nano", + "label": "GPT-5.4 Nano", + "openSource": false + }, + { + "id": "openai-gpt-5.5", + "label": "GPT-5.5", + "openSource": false + }, + { + "id": "openai-gpt-oss-120b", + "label": "Openai gpt oss 120b", + "openSource": true + }, + { + "id": "openai/gpt-5.6-sol", + "label": "Gpt 5.6 sol", + "openSource": false + }, + { + "id": "qwen/qwen3.8-27b", + "label": "Qwen3.8 27b", + "openSource": true + }, + { + "id": "qwen3.8-27b", + "label": "Qwen3.8 27b", + "openSource": true + }, + { + "id": "selfhost-qwen38", + "label": "Selfhost qwen38", + "openSource": true + }, + { + "id": "z-ai/glm-5.3-flash", + "label": "Glm 5.3 flash", + "openSource": true + }, + { + "id": "eis-google-gemini-2-5-flash-lite", + "label": "Google Gemini 2.5 Flash Lite (EIS)", + "openSource": false + }, + { + "id": "eis-openai-gpt-oss-20b", + "label": "OpenAI GPT-OSS 20B (EIS)", + "openSource": true + }, + { + "id": "eis-gp-llm-v2", + "label": "GP LLM v2 (EIS)", + "openSource": false + } + ] +} \ No newline at end of file diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/docs/plans/2026_09_03_persona_matrix_example_sharding.md b/x-pack/platform/packages/shared/kbn-evals-extensions/docs/plans/2026_09_03_persona_matrix_example_sharding.md new file mode 100644 index 0000000000000..8e29c85c953c5 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/docs/plans/2026_09_03_persona_matrix_example_sharding.md @@ -0,0 +1,153 @@ +# Example-level shard fanout for persona-matrix evals + +**Date:** 2026-09-03 +**Branch:** `feat/evals-extensions-matrix-v3` (PR #285833) +**Problem owner:** slow-model matrix rows take ~2–3 h wall clock and block local RAM. + +## Shape + +### The problem, measured + +Two OSS candidates were run through the 21-example persona matrix today: + +| Model | calls/example | evaluated | wall clock | outcome | +|---|---|---|---|---| +| Qwen3.8-27B | 121.8 | 14/21 | ~150 min | killed, 0 scores | +| GLM-5.3-flash | 80.8 | 11/21 | 99 min (killed at 20/21 dispatched) | killed, 0 scores | +| frontier (gemini/claude/gpt) | 5–25 | 21/21 | 4–15 min | complete | + +The suite has exactly two env knobs (`PERSONA_MATRIX_CAPTURE`, +`PERSONA_MATRIX_CONCURRENCY`) — verified by grepping `process.env` in the suite +source. There is **no way to run a subset of examples**, so: + +- a slow model must complete all 21 examples in one process or produce nothing + (scores export only at run end); +- the only parallelism knob is in-suite `concurrency`, which shares one Kibana + and one ES on one host; +- `PERSONA_MATRIX_WORKERS` is unsafe: `beforeAll` seeds shared indices that + `afterAll` deletes, so a second worker tears down live fixtures. + +### Root cause of the wall clock + +Not the harness. `MODEL_ENV` in `persona_matrix_sweep.py:112-118` already +documents this for GLM-5.2: *"mean 341s per example, max 1198s… 21 examples +therefore need ~119 min"*. Slow models are inherently slow per example. The +lever is **spreading examples across machines**, not making each faster. + +### Chosen shape + +Add an example-selection knob to the suite, then fan shards out as separate +Azure VMs — one stack per shard, which is the only safe form of parallelism +given the `beforeAll`/`afterAll` fixture contract. + +``` +today: 1 VM x 21 examples = 21 x per-example cost +sharded: 4 VMs x 5-6 examples = ~6 x per-example cost (+ ~19 min stack boot) +``` + +For GLM at ~4.7 min/example: 99+ min → ~28 min + boot. For frontier models +(already 4–15 min) sharding is off by default — it would add boot overhead for +no gain. + +### Explicitly out of scope + +- **Capping agent cycles per model.** Changes what is measured; a truncated + model scores its own truncation. Rejected on eval-integrity grounds. +- **Lowering concurrency.** Strictly worse wall clock. +- **Per-example result caching / resume.** Genuinely valuable (11 completed GLM + examples were discarded on kill) but it is a *second* change with its own + storage-format design. Separate PR. +- **Changing any Kibana product code.** Suite + sweeper only. +- **Making sharding the default for all models.** Opt-in per model. + +### Alternatives considered + +1. **`PERSONA_MATRIX_WORKERS` > 1 on one stack** — rejected, unsafe fixture + teardown (documented in the suite and re-verified today). +2. **More in-suite concurrency on a bigger VM** — hits the same single-Kibana + bottleneck that produced 69 transport blips and 4 hard failures locally at + concurrency=5. +3. **Shard by example, one stack each** — chosen. Matches the existing + per-model VM fanout pattern the sweeper already implements. + +## Plan + +Each step names how it is proven. No step is done until its proof runs. + +### 1. Suite: `PERSONA_MATRIX_SHARD` env knob + +`x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/` + +Add a pure helper `selectShard(examples, shard)` where `shard` is `"i/n"` +(1-based, e.g. `"2/4"`). Deterministic stride assignment so shard membership +never depends on ordering luck: example at index `k` belongs to shard +`k % n + 1`. Absent/empty env ⇒ all examples (exact current behaviour). + +**Proof:** new jest tests — union of all shards equals the full 21 with no +overlap; `"1/1"` is identity; malformed values (`"0/4"`, `"5/4"`, `"x/y"`, +`"1/0"`) throw rather than silently running everything. Then mutate the stride +to `k < n` (contiguous) and confirm the union/disjoint test still passes but the +balance test fails — proving the tests bite. + +### 2. Suite: wire the knob into the spec + +Read `process.env.PERSONA_MATRIX_SHARD` where the dataset is built, and include +the shard in the logged run header so a shard's log is identifiable. + +**Proof:** `node scripts/jest` on the suite package green; grep the built +dataset length under a stubbed env. + +### 3. Sweeper: `--shards N` + +`scripts/orca_vm/persona_matrix_sweep.py` + +- Add `PERSONA_MATRIX_SHARD` to `FORWARDED_ENV_VARS` (trap 5 in + `vm-sweep-suite-port.md`: an unforwarded var silently grades the wrong thing). +- `--shards N` provisions N VMs per model, VM name suffix `-s`. +- **Gate arithmetic must change.** `check_golden` computes + `n_examples * n_evaluators * reps` with `gate: "exact"`. A shard produces only + its slice, so a per-shard exact gate must expect that shard's example count, + and the model is complete only when the union across shards equals the full + expected count. Implement as: per-shard expected = `len(shard) * evaluators * + reps`; model-level assertion sums shards. + +**Proof:** extend `--self-test` with cases for shard env forwarding, per-shard +expected-doc arithmetic, and the union assertion. Mutate each new check to +confirm it fails. + +### 4. Validate + +- `node scripts/jest ` — full package, not a hand-picked file. +- `python3 persona_matrix_sweep.py --self-test` — all checks, each mutated. +- Confirm no dataset-UUID literal regression (existing self-test guard). + +### 5. Smoke + +One model, `--shards 4`, on Azure. Assert: +- 4 VMs provisioned, each `ps aux | grep playwright` shows the right config; +- each shard's local score index holds only its slice; +- union on **golden** equals the full expected doc count (trap 2: a green run + does not mean scores reached golden); +- teardown prints the zero line. + +### 6. End-to-end + +The real target: GLM-5.3-flash, 21 examples, 4 shards, judge +`eis-anthropic-claude-4-6-sonnet`, scores landing on golden and the matrix +regenerating with a populated GLM row. This is the first complete OSS row. + +### 7. Deslop + land + +`deslop`, `pre-pr-self-review`, commit into PR #285833. Sweeper/suite sharding +is infrastructure for that PR's matrix work, not an unrelated platform change. + +## Risks + +- **Shard imbalance.** Stride assignment spreads cost, but the measured + per-example span distribution is skewed (median 75 calls, max 280). One shard + may still straggle. Accepted: even a 2× straggler beats 21 serial examples. +- **N× stack boot cost.** ~19 min per VM, paid in parallel. Sharding is a loss + for fast models — hence opt-in. +- **Golden write races.** N shards writing one model's execution concurrently; + the union gate must tolerate eventual consistency (retry the count, don't + assert once). diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/docs/plans/2026_09_06_persona_matrix_v2_decision_report.md b/x-pack/platform/packages/shared/kbn-evals-extensions/docs/plans/2026_09_06_persona_matrix_v2_decision_report.md new file mode 100644 index 0000000000000..fce43caa4a9f8 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/docs/plans/2026_09_06_persona_matrix_v2_decision_report.md @@ -0,0 +1,124 @@ +# Persona Matrix v2 — Decision-Oriented Model Comparison + +**Date:** 2026-09-06 +**Status:** shaped → planned → in execution +**Author:** agent (evals-ext-matrix) + +## Problem + +The v1 matrix renders 23 models × 24 prompt columns + an `Overall` mean, sorted by +`Overall`. It looks like a leaderboard, so readers use it as one. Measured against +the actual data in `target/llm_matrix_latest/matrix.json`, that reading is unsound. + +### Finding 1 — the ranking is noise + +Paired bootstrap over the 24 prompt columns (models face identical prompts, so the +comparison must be paired), 4000 resamples, 18 fully-measured models: + +- **17 of 17 adjacent rank pairs are statistically indistinguishable** (95% CI of the + paired difference contains 0). +- The **#1 model is significantly better than only 8 of 17** others. +- Mean 95% CI width per model is **1.24 points**; the entire spread between best and + worst fully-measured model is **1.53 points**. Each model's own uncertainty is + nearly as wide as the whole field. +- 16 of 22 adjacent `Overall` gaps are **< 0.10 points**. + +Presenting rank 3 above rank 16 (6.87 vs 6.20) implies a distinction the evidence +does not support. + +### Finding 2 — averaging destroys real signal + +Every prompt discriminates well: per-prompt stdev across models ranges 0.52–2.47, +with per-prompt ranges up to 7.2 points. The signal is strong *per prompt* and is +cancelled by the mean. + +Rank correlation between the 10 task families: + +- **Mean off-diagonal Spearman r = 0.037** (essentially zero). +- **39 of 45 family pairs have r < 0.3** — largely independent skills. +- **20 of 45 pairs are negatively correlated** — genuine tradeoffs. + +There is no general "good at security" axis to average toward. Consequences: + +- **8 distinct models win the 10 families.** +- The `Overall` #1 (Claude Sonnet 4.6) **wins only 1 of 10 families**. +- Sonnet 4.6 is best at entity-analytics (9.26) and **last** at rule translation (4.57). +- DeepSeek V4 Pro is best at alert-analysis (8.18) and worst at attack-discovery (**1.11**). + +A user who picks the `Overall` winner for rule translation gets the worst model in +the field for that job. + +### Finding 3 — the deciding axis is missing + +Cost and latency exist in the trace cache but are shown only as incidental +evaluator rows, for one column (`attack-discovery`) in `tokenCost`. Real values: + +| Model | Quality | Latency | Input tokens | +|---|---|---|---| +| GPT-5.2 | 6.87 | 144.2s | 696,311 | +| Claude Haiku 4.5 | 6.76 | 24.9s | 129,775 | + +A **0.11 quality difference — statistically zero — for 5.8× latency and 5.4× tokens.** + +Across the field, `corr(quality, latency) = +0.24` and +`corr(quality, input tokens) = +0.36`: paying more buys little. + +Pareto analysis (maximize quality, minimize latency and tokens) finds **only 4 of 21 +models undominated**. The other 17 are strictly worse choices — some model is at +least as accurate while being both faster and cheaper. + +### Finding 4 — partial rows rank as if complete + +`Overall` is the mean of whatever cells exist. GLM-5.2 ranks **#2 overall on 4 of 24 +cells**; GLM-5.3 Flash ranks #17 on **1** cell. Both outrank fully-measured models on +a fraction of the evidence. + +## What users actually need + +The matrix answers "which model is best?" — a question the data cannot support. +Users decide among three real questions: + +1. *I run task X — which model should I use?* → per-family winners with uncertainty. +2. *What does it cost me?* → quality vs latency/tokens, Pareto frontier. +3. *Can I trust this number?* → CI, sample size, judge, coverage. + +## Shape + +Keep the existing per-prompt grid — it is the evidence, and it is sound. Change +what is emphasized and what is claimed. + +### S1. Replace the ranked `Overall` with tiers +Cluster models into statistically indistinguishable tiers via paired bootstrap. +Within a tier, order alphabetically — never by an insignificant decimal. Every +tier boundary must be backed by a non-overlapping CI. + +### S2. Lead with per-task recommendations +For each of the 10 families, show the winner set (all models whose CI overlaps the +best), not a single winner. Surface tradeoffs: "best at X, worst at Y." + +### S3. Add the efficiency frontier +Quality vs cost, marking dominated models explicitly. This is the highest-value +view and is currently absent. + +### S4. Honest uncertainty everywhere +Every aggregate carries CI and n. Partial rows are visually segregated and never +ranked against complete ones. + +### S5. Suppress known-bad instruments +`Tool Calls = 0` rows are an instrument defect (user standing rule: never publish). +Suppress rather than render zero. + +## Plan + +| # | Step | Validation | +|---|---|---| +| 1 | `model_comparison.ts`: paired bootstrap, tiers, Pareto, family winners | unit tests + mutation | +| 2 | Golden contract test on real bundle | reproduces findings above | +| 3 | Render v2 sections into matrix HTML (dark mode) | geometry check | +| 4 | End-to-end on `llm_matrix_latest` | numbers match this doc | + +## Non-goals + +- No re-running of evals; v2 is a presentation change over existing data. +- No new judge; haiku remains judge of record. +- No score mutation — v2 must reproduce v1 cell values exactly. diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/docs/plans/2026_09_06_sweep_runtime_optimization.md b/x-pack/platform/packages/shared/kbn-evals-extensions/docs/plans/2026_09_06_sweep_runtime_optimization.md new file mode 100644 index 0000000000000..cbddcbaf7f81d --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/docs/plans/2026_09_06_sweep_runtime_optimization.md @@ -0,0 +1,94 @@ +# Sweep runtime optimization — plan (2026-09-06) + +## Problem (measured, A3 re-judge: 26 units, D8s_v5 eastus2) + +| Phase | Time | +|---|---| +| sweep start -> first golden score doc | 39 min | +| median eval work per unit | 11.7 min | +| total wall clock | ~68 min | +| spot `az vm create` failures before Regular fallback | 30 / 60 | + +~83% of wall clock is fixed per-VM overhead. Sharding divides only the 12-minute +slice, so past ~3 shards the curve is flat. Cut overhead, not example count. + +## Shape + +Four changes, ordered by payoff. Each is independently shippable and independently +verifiable. Out of scope: changing what is measured (evaluator logic, judge +prompts, example set) — a speedup that moves scores is a measurement change, not +an optimization. + +### 1. `evals ext rejudge` — VM-free judge-only re-scoring [biggest win] + +A re-judge today re-runs the entire agent (fresh ES + Kibana + seed + 21 personas) +purely to change the final judge call. Golden score docs already carry everything +a judge needs: + +- `example.input.question`, `example.metadata` +- `task.output.messages` (final answer) +- `task.output.steps` (~38 KB trajectory) +- `task.model.id`, `task.trace_id`, `metadata.execution_id` + +`evaluate_dataset.ts` computes `correctnessAnalysis` / `groundednessAnalysis` +inside the task and the quantitative evaluators are pure functions of those +analyses. So a re-judge is: read golden docs -> re-run the two analyses + the +`criteria` evaluator against a new judge connector -> write new score docs under a +fresh `execution_id`. + +Not portable to re-judge: trace-based evaluators (`Latency`, `Tokens`, +`Tool Calls`, `SkillInvoked`) read spans, not task output. They must be **carried +over unchanged** from the source docs, never recomputed and never silently +dropped. + +Cost: ~2 h + full Azure quota -> ~5 min, zero VMs. + +### 2. Warm deallocated VM pool + +Deallocated VMs consume no vCPU quota (the RG held 85 VMs against a 43-VM quota +footprint). `az vm start` on a pre-baked VM is ~60-90 s vs ~6-8 min for +create + cloud-init + deploy. Fixed pool footprint also removes the quota cliff +that cost ~10 h. + +### 3. Regular-first provisioning + +Spot D8s_v5 in eastus2 has been exhausted for days; spot-first burns an ARM +roundtrip plus backoff per VM before falling back anyway (30/60 failures +measured). Probe spot once per sweep, not per VM. + +### 4. Pre-flight quota gate + completeness-safe teardown + +Refuse to launch when `requested_cores > free_cores`. And fix the teardown +matcher: golden ids are `sweep---::::` — the +shard sits BEFORE the suite, so a `sweep-*--` tail wildcard matches +nothing and completeness-gated teardown silently deletes zero VMs (this is what +exhausted quota twice today). + +## Plan + +| # | Step | Proof | +|---|---|---| +| 1 | `rejudge.ts`: read golden docs for a run/model, group by example | unit test on fixture docs | +| 2 | Re-run correctness+groundedness+criteria with `--judge`, carry trace evaluators verbatim | unit test: trace evaluators pass through unchanged; mutation test | +| 3 | Write new score docs w/ fresh `execution_id`, `evaluator.model.id` = new judge | unit test on emitted doc shape | +| 4 | Wire `rejudgeCmd` into `evals ext` CLI | `node scripts/evals ext rejudge --help` runs | +| 5 | `--regular-first` default in `persona_matrix_sweep.py` | unit test on the create-arg builder | +| 6 | Quota preflight gate refusing over-subscription | unit test: over-quota request refused, under-quota allowed | +| 7 | Teardown matcher fixed to the real id shape | unit test with a real id string (bites the bug that shipped) | +| 8 | Warm-pool start/stop path | `--pool` deallocate/start against a real VM | + +## Validation + +- Jest on both packages (`--maxWorkers=4`; Kibana's wrapper rejects + `--workerIdleMemoryLimit`). +- Mutation-test every fix: revert -> observe red for the right reason -> restore. +- E2E: run `evals ext rejudge` against real golden data for a model whose haiku + judgement is already known, and compare against the VM-produced scores for the + same cells. Agreement on deterministic/pass-through columns must be exact; + LLM columns compared as a distribution. + +## Non-goals + +- No change to evaluator semantics, judge prompts, or the example set. +- No new judge default (haiku remains judge of record). +- Trace evaluators are carried, never recomputed from a re-judge. diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/docs/plans/2026_09_07_evals_ext_rejudge.md b/x-pack/platform/packages/shared/kbn-evals-extensions/docs/plans/2026_09_07_evals_ext_rejudge.md new file mode 100644 index 0000000000000..9af027a6cacb6 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/docs/plans/2026_09_07_evals_ext_rejudge.md @@ -0,0 +1,71 @@ +# `evals ext rejudge` — judge-only replay + +## Problem + +The published matrix is graded by three different judges: 17 rows by +`anthropic-claude-4.6-sonnet`, 1 by `anthropic-claude-4.5-haiku` +(Claude Sonnet 4.6, which the self-judge guard correctly diverted), and 1 by +`google-gemini-3.1-pro` (GPT-OSS 120B). Rows graded by different judges are not +comparable, and both headline claims — the #1 model and the OSS separation — +rest on the two minority-judge rows. + +Re-running the sweep to unify the judge costs ~68min and 25 VMs, ~83% of which +is provisioning. It also confounds the judge change with run-to-run variance. + +## Shape + +Re-judge from stored trajectories. Every input a judge reads is already durable +in golden score documents (`example.input.question`, `task.output.messages`); +the dataset supplies `reference`. Verified: all 20 models have 21/21 examples +with non-empty stored outputs (Sonnet 4.5 has 19). + +**In scope**: recompute the LLM-judged evaluators — `CorrectnessAnalysis` and +`GroundednessAnalysis`, and the deterministic scores derived from them +(`Factuality`, `Relevance`, `Sequence Accuracy`, `Groundedness`). + +**Carried forward unchanged**: trace-based evaluators (`SkillInvoked`, Input/ +Output Tokens, Latency, Tool Calls). These read spans, not outputs, and a judge +swap cannot change them. They are copied from the source run, not recomputed +and not dropped. + +**Out of scope**: re-running agents; changing prompts; writing to golden in this +change (local artifact first). + +## Decisions + +- Two passes per model: **non-blind** (parity with existing sonnet scores) and + **blind** (model identity stripped, as the original eval bundle does via + `obfuscate_agent_eval.py`). Running both makes identity bias measurable + instead of assumed. +- Output to a local JSON artifact for review. Golden write is a follow-up, + gated on inspecting the deltas. +- Re-judged scores keyed by `replayExecutionId()` = `::rejudge-` + so a judge's verdicts never merge into the source execution's cell. + +## Steps + +1. **Reference join** — `--dataset` flag loading the suite dataset, mapping + `exampleId -> reference`. Prove: a plan built without it reports every cell + as skipped (`missing dataset reference`) rather than grading against empty + ground truth. +2. **Anonymizer** — `anonymizeCell()` stripping model identity from judge-visible + text. Prove: unit test asserting a known model id/name/vendor string does not + survive into the judge payload, including inside the agent response body. +3. **Judge runner** — drive `createCorrectnessAnalysisEvaluator` + + `createGroundednessAnalysisEvaluator` over plan cells with bounded + concurrency; recompute quantitative scores from the returned analyses. +4. **CLI** — `rejudgeCmd` registered in `cli/index.ts`; flags `--config`, + `--judge`, `--dataset`, `--blind`, `--out`, `--concurrency`, `--as-of`, + `--models`, `--dry-run`. +5. **Validate** — jest for kbn-evals-extensions; mutation-test each new guard. +6. **Smoke** — `--dry-run` on real golden data: plan size must equal the 20×21 + cells measured, skipped must be 0. +7. **E2E** — real haiku judge over all 20 models, both passes, real inference. + +## Verification + +- Plan cell count matches the independently measured 419 (20 models × 21, minus + 2 missing Sonnet 4.5 examples). +- `--dry-run` costs nothing and writes nothing. +- Blind pass: assert no model identifier appears in any judge payload. +- Deltas reported per model as haiku-vs-sonnet on identical trajectories. diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/docs/rejudging_a_golden_column.md b/x-pack/platform/packages/shared/kbn-evals-extensions/docs/rejudging_a_golden_column.md new file mode 100644 index 0000000000000..f84110d786140 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/docs/rejudging_a_golden_column.md @@ -0,0 +1,85 @@ +# Re-judging a golden column + +A rejudge run re-grades **existing** model outputs with a different judge. It does +not re-run the models, and it does not write to the golden cluster: it produces a +local artifact that [`mergeRejudgedScores`](../src/matrix/merge_rejudged_scores.ts) +applies to a board, keyed on `experiment_id`. + +The Attack Discovery column is the motivating case. It is graded by a mix of +judges while the persona columns are graded by others, and no model is graded by +more than one judge anywhere on the board — so judge severity cannot be separated +from model quality, and a rejudge under one pinned judge is what would make the +column comparable. + +## Prerequisite: CCM must be enabled on the eval stack + +EIS-backed judges resolve through the ES inference catalog, which is populated by +the cross-cluster model service. On a freshly booted stack it is **off**, and the +catalog holds only the built-in endpoints: + +``` +GET /_inference/_ccm -> {"enabled": false} +``` + +Every EIS judge id then fails connector resolution: + +``` +No connector or inference endpoint found for ID 'eis-google-gemini-3-1-pro' +``` + +This reads like broken jury wiring or a bad route, but it is neither — the judge +model was never reachable. Enable CCM and wait for the catalog to sync: + +``` +PUT /_inference/_ccm # with the EIS CCM key +GET /_inference/_all # expect .google-gemini-3.1-pro-chat_completion +``` + +CCM state is **in-memory**, so it does not survive a stack restart and must be +re-applied on every boot. + +## Failure mode 1: selection admits zero documents + +**Symptom.** The run completes without error and reports `0` cells written. Logs +show the planner selecting an execution, then no scores. + +**Cause.** The rejudge selects a golden execution to re-grade, and admission is +narrower than selection: an execution can be picked whose documents are then all +rejected (no gradable output, or an evaluator set that does not match what the +rejudge computes). Selection succeeding is not evidence that anything is gradable. + +**What to check.** Count admitted documents for the chosen `experiment_id` before +believing a `0`-cell result is a judge problem. A zero-count query needs a +known-positive control — confirm the same query returns documents for an +execution you know is gradable, otherwise "no documents" and "wrong query" look +identical. + +## Failure mode 2: the jury does not produce its evaluators + +**Symptom.** Cells are planned and the run reports no failures, but the artifact +carries no `Criteria` / `Rubric` scores — or carries them on one run and not the +next, from an unchanged input. Observed oscillating between 0 and 4 cells across +consecutive runs on the same execution. + +**Cause.** Not established. The judge is reachable (CCM verified, endpoint +present) and the run does not error, so this is distinct from mode 1 and from a +connector failure. Do not attribute it to "judge flake" — that names a cause +which has not been demonstrated. What is established is that a clean exit code +does not imply the evaluators ran. + +**What to check.** Assert on the evaluator names actually present in the artifact +rather than on the exit code or the planned cell count. A rejudge that writes an +artifact containing none of the expected evaluators has failed, however it exited. + +## Why the merge is keyed on `experiment_id` + +A model appears many times on a board across executions. Keying a merge on model +id — or on model plus column — lets a rejudge of one execution overwrite a +different, possibly newer, execution of the same cell. `experiment_id` identifies +the run whose outputs were actually re-graded, so a merge can only ever replace +the scores it re-computed. + +Re-judged cells that match no golden cell are **reported, not appended**: a +rejudge can only re-grade outputs that already exist, so a non-matching cell means +the artifact and the board disagree about what was run. Appending it would +fabricate a cell the board never had. diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/index.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/index.ts index b750481ef7084..eeca033a56bda 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/index.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/index.ts @@ -22,4 +22,15 @@ export type { EvaluationScoreDocument } from '@kbn/evals-common'; export * as cli from './src/cli'; export { runRedTeam, RED_TEAM_MODULE_IDS } from './src/red_team'; +export { planReplay, summarizePlan, replayExecutionId } from './src/matrix/replay_plan'; +export type { ReplayCell, ReplayPlan, ReferenceLookup } from './src/matrix/replay_plan'; +export { checkEvaluatorHealth } from './src/matrix/evaluator_health'; +export type { + EvaluatorHealthInput, + EvaluatorHealthReport, + EvaluatorFinding, + EvaluatorObservation, + EvaluatorClassification, + EvaluatorRole, +} from './src/matrix/evaluator_health'; export type { RedTeamConfig, RedTeamReport, RedTeamModuleId } from './src/red_team'; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/moon.yml b/x-pack/platform/packages/shared/kbn-evals-extensions/moon.yml index 219130985794a..960bab8244400 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/moon.yml +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/moon.yml @@ -22,6 +22,13 @@ dependsOn: - '@kbn/dev-cli-runner' - '@kbn/dev-cli-errors' - '@kbn/tooling-log' + - '@kbn/config-schema' + - '@kbn/some-dev-log' + - '@kbn/kbn-client' + - '@kbn/repo-info' + - '@kbn/core' + - '@kbn/inference-plugin' + - '@kbn/inference-common' tags: - test-helper - package diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/scripts/analyze_judge_overlap.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/scripts/analyze_judge_overlap.ts new file mode 100644 index 0000000000000..5cc6eae9b3a27 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/scripts/analyze_judge_overlap.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Run the judge-overlap analysis over real rejudge artifacts. + * + * Takes the `rejudge-*.json` files produced by re-judging one common subset of + * cells with several judges and reports whether the resulting board can be + * ranked. Output is JSON on stdout so the numbers quoted in the PR and in the + * rendered board come from the same committed code path as the tests. + * + * JUDGE_ARTIFACTS=haiku=/path/a.json,gpt=/path/b.json node scripts/analyze_judge_overlap.ts + */ + +import fs from 'fs'; +import { + analyzeJudgeOverlap, + realModelIdFromSourceExecution, + type JudgeOverlapInput, +} from '../src/matrix/judge_overlap'; + +const spec = process.env.JUDGE_ARTIFACTS; +if (!spec) { + throw new Error('JUDGE_ARTIFACTS must be judgeId=path pairs, comma separated.'); +} + +const inputs: JudgeOverlapInput[] = spec.split(',').map((pair) => { + const [judgeId, file] = pair.split('='); + if (!judgeId || !file) throw new Error(`Malformed JUDGE_ARTIFACTS entry "${pair}".`); + + const artifact = JSON.parse(fs.readFileSync(file, 'utf8')); + + const cells = artifact.results.map((r: any) => { + const numeric = r.scores + .map((s: any) => s.score) + .filter((s: unknown): s is number => typeof s === 'number'); + + if (numeric.length === 0) { + // The rejudge guard should have refused this artifact. Reaching here + // means an unscored cell would otherwise average in as a real zero. + throw new Error( + `Cell ${r.sourceExecutionId}/${r.exampleId} in "${file}" carries no numeric score.` + ); + } + + return { + // Blind runs alias modelId; sourceExecutionId keeps the real identity. + modelId: realModelIdFromSourceExecution(r.sourceExecutionId), + exampleId: r.exampleId, + score: numeric.reduce((a: number, b: number) => a + b, 0) / numeric.length, + }; + }); + + return { judgeId, cells }; +}); + +const report = analyzeJudgeOverlap(inputs); + +// eslint-disable-next-line no-console +console.log(JSON.stringify(report, null, 2)); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/scripts/analyze_saturation.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/scripts/analyze_saturation.ts new file mode 100644 index 0000000000000..aa8a43b4b55be --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/scripts/analyze_saturation.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Report whether a rejudged column is limited by its rubric or by its judges. + * + * SATURATION_INPUT=/tmp/ad_saturation_input.json \ + * node --require @kbn/setup-node-env scripts/analyze_saturation.ts + * + * Input is `{ cells: [{ modelId, cellKey, scoresByJudge }], ceiling }`, which is + * what a multi-judge rejudge sweep produces once its artifacts are joined on the + * cells every judge graded. + */ + +import fs from 'fs'; +import { analyzeSaturation } from '../src/matrix/saturation'; + +const inputPath = process.env.SATURATION_INPUT; +if (!inputPath) { + throw new Error('SATURATION_INPUT must point at the joined multi-judge cell file.'); +} + +const { cells, ceiling } = JSON.parse(fs.readFileSync(inputPath, 'utf8')); +const report = analyzeSaturation({ cells, ceiling: ceiling ?? 1 }); + +/* eslint-disable no-console */ +console.log(JSON.stringify(report, null, 2)); +console.log(`\n${report.verdict}`); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/scripts/audit_evaluator_health.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/scripts/audit_evaluator_health.ts new file mode 100644 index 0000000000000..2dfc559763301 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/scripts/audit_evaluator_health.ts @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Audits every evaluator in a golden extract and reports which ones can + * actually separate models. + * + * AGGREGATED_JSON=/tmp/aggregated_v5.json \ + * node --require @kbn/setup-node-env scripts/audit_evaluator_health.ts + * + * Exits non-zero when a grader is constant or saturated, so it can be wired + * into a run as a gate rather than staying a retrospective script. + */ + +import fs from 'fs'; +import { + checkEvaluatorHealth, + type EvaluatorObservation, + type EvaluatorFinding, +} from '../src/matrix/evaluator_health'; +import { resolveEvaluatorRole } from '../src/matrix/evaluator_roles'; + +/** + * Roles are declared in evaluator_roles.ts, not inferred here. An earlier + * version matched on evaluator names, which meant the list could be widened + * until the audit passed -- the exact failure the audit exists to catch. + */ + +/** Cost/latency metrics are not 0-1 scores and must not be audited as such. */ +const NON_SCORE = /token|latency|cost per|duration|^Tool Calls$/i; + +function main() { + const path = process.env.AGGREGATED_JSON; + if (!path) throw new Error('AGGREGATED_JSON is required'); + + const models = JSON.parse(fs.readFileSync(path, 'utf8')) as any[]; + const bySuite = new Map(); + const undeclared = new Set(); + + for (const model of models) { + for (const suite of model.suites ?? []) { + for (const dataset of suite.datasets ?? []) { + for (const evaluator of dataset.evaluators ?? []) { + const name: string = evaluator.evaluatorName; + if (NON_SCALE(evaluator) || NON_SCORE.test(name)) continue; + const role = resolveEvaluatorRole(name); + if (role === 'unknown') undeclared.add(name); + + const list = bySuite.get(suite.suiteId) ?? []; + list.push({ + evaluatorName: name, + modelId: model.modelId, + score: evaluator.mean, + // `unknown` is reported, never guessed: see evaluator_roles.ts. + role: role === 'unknown' ? undefined : role, + }); + bySuite.set(suite.suiteId, list); + } + } + } + } + + let failed = false; + for (const [suiteId, observations] of [...bySuite.entries()].sort()) { + const report = checkEvaluatorHealth({ observations }); + failed ||= !report.ok; + + process.stdout.write(`\n=== ${suiteId}\n`); + const order: Record = { constant: 0, saturated: 1, 'gate-failing': 2 }; + const sorted = [...report.findings].sort( + (a, b) => (order[a.classification] ?? 9) - (order[b.classification] ?? 9) + ); + for (const f of sorted) process.stdout.write(line(f)); + process.stdout.write( + ` composite-safe (${report.compositeSafe.length}/${report.findings.length}): ` + + `${report.compositeSafe.join(', ') || 'none'}\n` + ); + } + + if (undeclared.size > 0) { + process.stdout.write( + `\nUNDECLARED ROLE (${undeclared.size}): ${[...undeclared].join(', ')}\n` + + `Add each to EVALUATOR_ROLES with a rationale. Until then they are audited as graders'\n` + + `weaker cousin -- reported, but with no gate/grader expectation applied.\n` + ); + } + + if (failed) { + process.stdout.write( + '\nFAIL: at least one grader cannot separate models. A rejudge will not fix this;\n' + + 'the rubric has to get finer-grained. See docs/rejudging_a_golden_column.md.\n' + ); + process.exitCode = 1; + } +} + +const NON_SCALE = (evaluator: { mean: number; max?: number }) => + evaluator.mean > 1 || (evaluator.max ?? 0) > 1; + +function line(f: EvaluatorFinding): string { + const mark = { + constant: 'DEAD', + saturated: 'SAT ', + 'gate-failing': 'GATE', + 'insufficient-data': 'n/a ', + }[f.classification as string]; + return ( + ` ${(mark ?? 'ok ').padEnd(5)}${f.evaluatorName.padEnd(30)}` + + `n=${String(f.observationCount).padEnd(5)}` + + `distinct=${String(f.distinctValues).padEnd(4)}` + + `@ceiling=${(f.ceilingShare * 100).toFixed(1)}%\n` + ); +} + +main(); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/scripts/build_ensemble_column.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/scripts/build_ensemble_column.ts new file mode 100644 index 0000000000000..3c8bfcb9a9d57 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/scripts/build_ensemble_column.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Build the ensemble column from rejudge artifacts and print it. + * + * JUDGE_ARTIFACTS="haiku=/path/a.json,sonnet=/path/b.json,..." + * OUT_JSON=/path/to/ensemble.json (optional) + */ + +import fs from 'fs'; +import { buildEnsembleColumn, type EnsembleCellInput } from '../src/matrix/ensemble_column'; +import { realModelIdFromSourceExecution } from '../src/matrix/judge_overlap'; + +const spec = process.env.JUDGE_ARTIFACTS; +if (!spec) + throw new Error('JUDGE_ARTIFACTS is required, e.g. "haiku=/tmp/haiku.json,gpt=/tmp/gpt.json"'); + +const EXCLUDED = new Set([ + 'Tool Calls', + 'Latency', + 'Input Tokens', + 'Output Tokens', + 'Skill Invoked', +]); + +const cells: EnsembleCellInput[] = []; + +for (const entry of spec.split(',')) { + const [judgeId, file] = entry.split('='); + const artifact = JSON.parse(fs.readFileSync(file, 'utf8')); + + for (const result of artifact.results ?? []) { + const graded = (result.scores ?? []).filter( + (s: any) => !EXCLUDED.has(s.name) && typeof s.score === 'number' + ); + if (graded.length === 0) continue; + + cells.push({ + judgeId, + modelId: realModelIdFromSourceExecution(result.sourceExecutionId ?? result.executionId), + exampleId: result.exampleId, + score: graded.reduce((a: number, s: any) => a + s.score, 0) / graded.length, + }); + } +} + +const column = buildEnsembleColumn(cells); + +// eslint-disable-next-line no-console +console.log( + JSON.stringify( + { + judges: column.judges, + sharedCellCount: column.sharedCellCount, + noiseReductionFactor: Number(column.noiseReductionFactor.toFixed(3)), + models: column.models.map((m) => ({ + modelId: m.modelId, + ensemble: Number(m.ensemble.toFixed(4)), + judgeSpread: Number(m.judgeSpread.toFixed(4)), + cellCount: m.cellCount, + perJudge: Object.fromEntries( + Object.entries(m.perJudge).map(([k, v]) => [k, Number((v as number).toFixed(4))]) + ), + })), + }, + null, + 2 + ) +); + +if (process.env.OUT_JSON) { + fs.writeFileSync(process.env.OUT_JSON, JSON.stringify(column, null, 2)); +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/scripts/extract_golden_aggregate.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/scripts/extract_golden_aggregate.ts new file mode 100644 index 0000000000000..323e18d60bcc3 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/scripts/extract_golden_aggregate.ts @@ -0,0 +1,359 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Extract a pre-aggregated `AggregatedModelScores[]` straight from the golden + * cluster, for `render_from_golden.ts` to consume. + * + * The golden cluster runs with `xpack.evals.enabled=false`, so the normal + * `evals ext matrix` CLI cannot read it through the plugin API. This reads the + * score documents directly and shapes them exactly like the CLI's aggregation + * step, so the renderer downstream is byte-identical either way. + * + * This exists as a committed script rather than a one-off because the published + * board is otherwise unreproducible: without it, regenerating requires + * reinventing the extraction, and the artifact's provenance cannot be checked + * against the data it claims to summarise. + * + * Scoring policy: + * Applies the CLI transport's policy -- verdict ladder, EIS-only judges, no + * self-judged scores -- through the shared `scoring_policy` module, so this + * driver and `queryMatrixScores` cannot drift. + * + * SCORING_POLICY=raw returns unfiltered stored scores (debugging only). + * + * Applying the policy does NOT reproduce the 2026-09-07 board: measured mean + * |delta| per cell moved 1.998 -> 2.361. The residual is judge mix. That + * artifact declares `judgeModelId: google-gemini-3.1-pro`, but golden shows + * its persona scores were graded 3842 by claude-4.5-haiku, 2122 by + * claude-4.6-sonnet and only 420 by gemini-3.1-pro -- it is a mixed-judge + * aggregate labelled single-judge. JUDGE_MODEL_ID pins one grader, which is + * the honest shape, but it will not match that board. + * + * Usage: + * source ~/.elastic/golden-cluster-env.sh + * node --require ../../../../../src/setup_node_env \ + * scripts/extract_golden_aggregate.ts > /tmp/aggregated.json + */ + +import fs from 'fs'; + +import { + applyScoringPolicy, + emptyExclusionCounts, + tallyRejection, +} from '../src/matrix/scoring_policy'; +import { DEFAULT_EXCLUDED_EVALUATORS } from '../src/matrix/load_matrix_config'; + +const ES_URL = process.env.GOLDEN_ES_URL!; +const ES_KEY = process.env.GOLDEN_ES_API_KEY!; +const SINCE = process.env.SINCE ?? '2026-09-01'; +const OUT = process.env.OUT_JSON; +// Mirror the CLI's scoring policy by default; SCORING_POLICY=raw opts out. +const POLICY_RAW = process.env.SCORING_POLICY === 'raw'; +// Pin the grader. Unpinned, a cell averages every judge that ever graded it, +// which compares models across different grader panels -- and judge choice +// reorders this board more than model quality does (Spearman rho 0.55). +const JUDGE_MODEL_ID = process.env.JUDGE_MODEL_ID; +const POLICY = POLICY_RAW + ? {} + : { useVerdictLadder: true, requireEisJudge: true, excludeSelfJudged: true }; +const isExcludedName = (name: string) => + DEFAULT_EXCLUDED_EVALUATORS.some((p) => name.startsWith(p)); +const policyExclusions = emptyExclusionCounts(); + +if (!ES_URL || !ES_KEY) { + throw new Error('GOLDEN_ES_URL and GOLDEN_ES_API_KEY must be set (source golden-cluster-env.sh)'); +} + +/** Suites that make up the published board, mapped to their experiment_name. */ +const SUITES: Array<{ suiteId: string; experimentNamePattern: string }> = [ + { suiteId: 'security-persona-matrix', experimentNamePattern: '*persona-matrix*' }, + { + suiteId: 'attack-discovery-agent-builder', + experimentNamePattern: 'attack-discovery-agent-builder*', + }, + { + suiteId: 'automatic-migrations', + experimentNamePattern: 'agent builder: automatic-migration*', + }, +]; + +interface ScoreDoc { + experiment_id: string; + experiment_name: string; + '@timestamp': string; + example?: { id?: string; dataset?: { id?: string; name?: string } }; + task?: { model?: { id?: string; family?: string; provider?: string } }; + evaluator?: { + name?: string; + score?: number; + label?: string; + direction?: string; + metadata?: unknown; + model?: { id?: string }; + }; +} + +const search = async (body: unknown): Promise<{ hits: { hits: Array<{ _source: ScoreDoc }> } }> => { + const response = await fetch(`${ES_URL.replace(/\/$/, '')}/.ds-.evaluation-scores*/_search`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Authorization: `ApiKey ${ES_KEY}` }, + body: JSON.stringify(body), + }); + if (!response.ok) { + throw new Error(`ES search failed: ${response.status} ${await response.text()}`); + } + return (await response.json()) as { hits: { hits: Array<{ _source: ScoreDoc }> } }; +}; + +const mean = (values: number[]) => values.reduce((a, b) => a + b, 0) / values.length; + +/** + * The two suite shapes key their columns differently, and neither field alone + * works for both: + * - persona matrix: `example.id` IS the column key ('alert-analysis-a'), while + * `example.dataset.id` is one UUID shared by every column. + * - attack-discovery / automatic-migrations: `example.id` is a bare ordinal + * ('0') that collapses all columns into one bucket, while + * `example.dataset.id` is per-scenario. + * So prefer a semantic example id and fall back to the dataset id. + */ +function columnKey(doc: ScoreDoc): string { + const exampleId = doc.example?.id; + const semantic = exampleId && !/^\d+$/.test(exampleId) ? exampleId : undefined; + return semantic ?? doc.example?.dataset?.id ?? exampleId ?? 'unknown'; +} + +async function main() { + // modelId -> suiteId -> docs + const byModel = new Map>(); + + for (const suite of SUITES) { + let searchAfter: unknown[] | undefined; + let fetched = 0; + // Page rather than size-capping: a truncated read silently drops whole + // models and the board renders as if they never ran. + for (;;) { + const page: any = await search({ + size: 5000, + // Each score doc carries the model's full transcript under + // `task.output`. Pulling whole `_source` makes the reader hold every + // transcript on the board in memory at once (observed: ~2GB RSS and no + // forward progress). None of it is aggregated here, so ask ES for the + // fields this actually reads. + _source: [ + 'experiment_id', + 'experiment_name', + '@timestamp', + 'example.id', + 'example.dataset.id', + 'example.dataset.name', + 'task.model.id', + 'task.model.family', + 'task.model.provider', + 'evaluator.name', + 'evaluator.score', + 'evaluator.model.id', + // Needed by the shared scoring policy (verdict ladder + provenance + // filters). Without these the golden driver silently disagreed + // with the CLI-rendered board by ~2 points per cell. + 'evaluator.label', + 'evaluator.direction', + 'evaluator.metadata', + ], + sort: [{ '@timestamp': 'asc' }, { _doc: 'asc' }], + ...(searchAfter ? { search_after: searchAfter } : {}), + query: { + bool: { + must: [ + { wildcard: { experiment_name: { value: suite.experimentNamePattern } } }, + { range: { '@timestamp': { gte: SINCE } } }, + { exists: { field: 'evaluator.score' } }, + ], + }, + }, + }); + + const hits = page.hits.hits as Array<{ _source: ScoreDoc; sort: unknown[] }>; + if (hits.length === 0) break; + + for (const hit of hits) { + const doc = hit._source; + const modelId = doc.task?.model?.id; + if (!modelId) continue; + if (!byModel.has(modelId)) byModel.set(modelId, new Map()); + const suites = byModel.get(modelId)!; + if (!suites.has(suite.suiteId)) suites.set(suite.suiteId, []); + suites.get(suite.suiteId)!.push(doc); + } + + searchAfter = hits[hits.length - 1].sort; + fetched += hits.length; + // Progress on stderr: a silent multi-minute read is indistinguishable + // from a hang, and this reads tens of thousands of documents. + // eslint-disable-next-line no-console + console.error(`[${suite.suiteId}] fetched ${fetched} score doc(s)`); + if (hits.length < 5000) break; + } + } + + const aggregated = [...byModel.entries()].map(([modelId, suiteDocs]) => { + const suites = [...suiteDocs.entries()].map(([suiteId, docs]) => { + // Newest experiment wins, but PER COLUMN, not per suite. Persona runs all + // of its columns inside one experiment, so a suite-wide filter is + // harmless there. Attack-discovery runs each of its nine slices as its + // own experiment, so a suite-wide filter keeps one slice and silently + // discards the other eight -- which is what made AD read as a + // single-dataset, one-observation column. + const newestExperiment = docs.reduce((latest, d) => + d['@timestamp'] > latest['@timestamp'] ? d : latest + ); + const newestByColumn = new Map(); + for (const doc of docs) { + const key = columnKey(doc); + const seen = newestByColumn.get(key); + if (!seen || doc['@timestamp'] > seen['@timestamp']) newestByColumn.set(key, doc); + } + const selected = docs.filter( + (d) => d.experiment_id === newestByColumn.get(columnKey(d))?.experiment_id + ); + + // The two suite shapes key their columns differently, and neither field + // alone works for both: + // - persona matrix: `example.id` IS the column key ('alert-analysis-a'), + // while `example.dataset.id` is one UUID shared by every column. + // - attack-discovery / automatic-migrations: `example.id` is a bare + // ordinal ('0') that collapses all columns into one bucket, while + // `example.dataset.id` is per-scenario. + // So prefer a semantic example id and fall back to the dataset id when it + // is a bare ordinal. `datasetName` is carried through either way, since it + // is the only human-readable key for the UUID-addressed suites. + const byDataset = new Map(); + for (const doc of selected) { + const datasetId = columnKey(doc); + if (!byDataset.has(datasetId)) byDataset.set(datasetId, []); + byDataset.get(datasetId)!.push(doc); + } + + // Columns match either by `datasetIds` (raw id) or by `examplePrefixes`, + // which build_matrix resolves against SYNTHETIC `prefix:` dataset + // ids. The CLI mints those in queryMatrixScores; this driver bypasses + // that transport, so an examplePrefixes config used to match nothing and + // still render exit-0 with zero warnings (measured: covered 1 of 24). + // Emit both keys so either config shape resolves. + interface DatasetRow { + datasetId: string; + datasetName: string; + evaluators: unknown[]; + } + const withPrefixAliases = (rows: DatasetRow[]): DatasetRow[] => + rows.flatMap((row: DatasetRow) => + String(row.datasetId).startsWith('prefix:') + ? [row] + : [row, { ...row, datasetId: `prefix:${row.datasetId}` }] + ); + + const datasets = [...byDataset.entries()].map(([datasetId, datasetDocs]) => { + const byEvaluator = new Map(); + for (const doc of datasetDocs) { + const name = doc.evaluator?.name; + if (!name) continue; + if (JUDGE_MODEL_ID && doc.evaluator?.model?.id !== JUDGE_MODEL_ID) { + continue; + } + // Single source of truth with query_matrix_scores: the CLI and this + // driver must reach the same verdict on the same document. + const decision = applyScoringPolicy(doc, POLICY, isExcludedName); + if (decision.score === null) { + tallyRejection(policyExclusions, decision.rejected); + continue; + } + if (!byEvaluator.has(name)) byEvaluator.set(name, []); + byEvaluator.get(name)!.push(decision.score); + } + return { + datasetId, + datasetName: datasetDocs[0].example?.dataset?.name ?? datasetId, + evaluators: [...byEvaluator.entries()].map(([evaluatorName, scores]) => ({ + evaluatorName, + mean: mean(scores), + count: scores.length, + min: Math.min(...scores), + max: Math.max(...scores), + })), + }; + }); + + // The judge that actually graded the selected run -- carried so the + // artifact's provenance can be derived instead of asserted. A suite whose + // columns ran as separate experiments can carry more than one judge, so + // the full set is reported rather than whichever document sorted first: + // silently publishing one of several judges is how a mixed column reads + // as a unified one. + const judgeModelIds = [ + ...new Set(selected.map((d) => d.evaluator?.model?.id).filter(Boolean)), + ] as string[]; + const judgeModelId = judgeModelIds.length === 1 ? judgeModelIds[0] : undefined; + + return { + suiteId, + experimentId: newestExperiment.experiment_id, + timestamp: newestExperiment['@timestamp'], + judgeModelId, + judgeModelIds, + selfJudged: judgeModelIds.includes(modelId), + datasets: withPrefixAliases(datasets), + }; + }); + + return { + modelId, + family: suiteDocs.values().next().value?.[0]?.task?.model?.family, + provider: suiteDocs.values().next().value?.[0]?.task?.model?.provider, + suites, + }; + }); + + const json = JSON.stringify(aggregated, null, 2); + if (OUT) { + fs.writeFileSync(OUT, json); + // Sidecar rather than a wrapper object: the artifact must stay a bare + // `AggregatedModelScores[]` for existing readers. render_from_golden reads + // this stamp to tell a policy-applied extract from a pre-port one. + fs.writeFileSync( + `${OUT}.policy.json`, + JSON.stringify( + { + scoringPolicy: POLICY, + judgeModelId: JUDGE_MODEL_ID ?? null, + excludedEvaluators: DEFAULT_EXCLUDED_EVALUATORS, + exclusions: policyExclusions, + }, + null, + 2 + ) + ); + // eslint-disable-next-line no-console + console.error( + `policy exclusions: ${JSON.stringify(policyExclusions)} (policy=${ + POLICY_RAW ? 'raw' : 'default' + })` + ); + // eslint-disable-next-line no-console + console.error(`wrote ${aggregated.length} model(s) to ${OUT}`); + } else { + // eslint-disable-next-line no-console + console.log(json); + } +} + +main().catch((error) => { + // eslint-disable-next-line no-console + console.error(error); + process.exit(1); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/scripts/render_from_golden.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/scripts/render_from_golden.ts new file mode 100644 index 0000000000000..ca65c134c0721 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/scripts/render_from_golden.ts @@ -0,0 +1,408 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Render the published matrix from a pre-aggregated golden extract. + * + * The normal `evals ext matrix` CLI reads scores through the evals plugin API, + * which the golden cluster has disabled (`xpack.evals.enabled=false`). This + * driver skips only the *transport*: aggregation is read from a JSON file + * produced by a direct ES query, then handed to the same `buildMatrix` and + * `renderMatrixHtml` the CLI uses, so the published artifact stays byte-shaped + * like every other run instead of being re-templated by hand. + */ + +import fs from 'fs'; +import path from 'path'; +import { buildMatrix } from '../src/matrix/build_matrix'; +import { renderMatrixHtml } from '../src/matrix/render_matrix_html'; +import type { MatrixTraceData } from '../src/matrix/trace_types'; +import type { AggregatedModelScores } from '../src/matrix/query_matrix_scores'; +import { loadMatrixConfig } from '../src/matrix/load_matrix_config'; +import { deriveJudgeProvenance } from '../src/matrix/judge_provenance'; +import type { SaturationReport } from '../src/matrix/saturation'; + +const readJson = (file: string): T => JSON.parse(fs.readFileSync(file, 'utf8')) as T; + +const aggregatedPath = process.env.AGGREGATED_JSON!; +const configPath = process.env.MATRIX_CONFIG!; +const outDir = process.env.OUT_DIR!; +// Accept either name: the render command passes *_JSON, and an earlier revision +// of this script read the bare names. A mismatch here silently drops every +// methodology note, so tolerate both rather than failing open on a typo. +const CAVEAT_STATS_PATH = process.env.CAVEAT_STATS_JSON ?? process.env.CAVEAT_STATS; +const JUDGED_STATS_PATH = process.env.JUDGED_STATS_JSON ?? process.env.JUDGED_STATS; +// Ensemble column: the 4-judge consensus over the shared overlap block. When +// supplied it is reported UNCONDITIONALLY, next to the single-judge figures, +// because the whole point is that one judge's column is not the last word. +const ENSEMBLE_PATH = process.env.ENSEMBLE_JSON; +// Saturation report for a column whose scores pile up at the rubric ceiling. +// Published for the same reason as the ensemble: a column that cannot rank must +// say why it cannot, or a reader will assume the ordering means something. +const SATURATION_PATH = process.env.SATURATION_JSON; + +const aggregated = readJson(aggregatedPath); +// Use the real loader so schema defaults (DEFAULT_EXCLUDED_EVALUATORS, scale, +// tier settings) apply exactly as they do for a normal CLI run. +const config = loadMatrixConfig(configPath); + +const warnings: string[] = []; +const matrix = buildMatrix(aggregated, config, { + warning: (message: string) => warnings.push(message), +}); + +// --- Fidelity guard ------------------------------------------------------- +// This driver renders from a golden EXTRACT and bypasses `queryMatrixScores`, +// the transport that (a) mints synthetic `prefix:` dataset ids for +// `examplePrefixes` columns and (b) applies the scoring policy +// (useVerdictLadder / requireEisJudge / excludeSelfJudged). +// +// Both were silent failures: an `examplePrefixes` config matched almost +// nothing yet still exited 0 with an empty warnings array (measured: covered +// 1 of 24 columns), and policy-filtered configs re-rendered from mixed-judge +// data with no signal that the filters never ran. Fail loudly instead. +const allRows = [...(matrix.proprietary ?? []), ...(matrix.openSource ?? [])]; +const totalCols = (config.columns ?? []).length; +const scoredCells = allRows.reduce( + (n, row) => + n + + Object.values(row.cells ?? {}).filter( + (c: any) => c?.kind === 'score' && c?.value !== null && c?.value !== undefined + ).length, + 0 +); +const fidelityErrors: string[] = []; + +if (allRows.length > 0 && totalCols > 0) { + const possible = allRows.length * totalCols; + const filled = scoredCells / possible; + if (filled < 0.5) { + fidelityErrors.push( + `only ${scoredCells}/${possible} cells (${(filled * 100).toFixed( + 1 + )}%) resolved to a score. ` + + `The extract's dataset keys likely do not match this config's column selectors.` + ); + } +} + +const usesPrefixes = (config.columns ?? []).some( + (c: any) => Array.isArray(c.examplePrefixes) && c.examplePrefixes.length > 0 +); +if (usesPrefixes) { + const hasPrefixKeys = aggregated.some((m: any) => + (m.suites ?? []).some((s: any) => + (s.datasets ?? []).some((d: any) => String(d.datasetId ?? '').startsWith('prefix:')) + ) + ); + if (!hasPrefixKeys) { + fidelityErrors.push( + `config declares examplePrefixes columns but the extract contains no ` + + `\`prefix:\` dataset ids. Re-run extract_golden_aggregate.ts (it emits ` + + `prefix aliases); an older extract cannot satisfy this config.` + ); + } +} + +const policy = (config as any).scoring ?? {}; +const activePolicy = ['useVerdictLadder', 'requireEisJudge', 'excludeSelfJudged'].filter( + (k) => policy[k] +); +if (activePolicy.length > 0 && process.env.ALLOW_UNENFORCED_SCORING !== '1') { + // extract_golden_aggregate stamps the policy it applied beside the extract. + // No stamp means the extract predates the policy port, and its scores are not + // comparable to the published board. + const stampPath = `${aggregatedPath}.policy.json`; + const stamp = fs.existsSync(stampPath) + ? (JSON.parse(fs.readFileSync(stampPath, 'utf8')).scoringPolicy as Record) + : undefined; + const missing = activePolicy.filter((k) => !stamp?.[k]); + if (!stamp) { + fidelityErrors.push( + `config sets scoring policy [${activePolicy.join(', ')}] but the extract carries no ` + + `policy stamp (${stampPath}), so it predates the policy port and its scores are not ` + + `comparable to the published board. Re-run extract_golden_aggregate.ts, or set ` + + `ALLOW_UNENFORCED_SCORING=1 to override.` + ); + } else if (missing.length > 0) { + fidelityErrors.push( + `config requires scoring policy [${missing.join(', ')}] but the extract was built ` + + `without it (SCORING_POLICY=raw?). Re-run the extract with the default policy.` + ); + } +} + +const tracesConfigured = Boolean(process.env.TRACES_JSON); +if (!tracesConfigured && process.env.ALLOW_NO_TRACES !== '1') { + fidelityErrors.push( + `TRACES_JSON is not set, so every cell would publish a score with no ` + + `transcript behind it. Build a cache first ` + + `(scripts/orca_vm/build_trace_cache.py --out ) and pass it as ` + + `TRACES_JSON. Set ALLOW_NO_TRACES=1 only for a deliberately score-only board.` + ); +} + +if (fidelityErrors.length > 0) { + process.stderr.write( + `\nrender_from_golden: refusing to emit a misleading board.\n` + + fidelityErrors.map((e) => ` - ${e}`).join('\n') + + `\n\n` + ); + process.exit(2); +} + +// Which judge actually graded the admitted runs, counted from the aggregated +// input rather than asserted. A hardcoded id here silently survives a rejudge +// that never landed: the board then claims one shared instrument while the +// rows were graded by several, which is exactly the comparison the CI/spread +// figures below assume is safe. +const { judgeModelId, judgeBreakdown } = deriveJudgeProvenance(aggregated); + +// Judge-mix figures are COUNTED from the extract, never asserted. The previous +// version hardcoded "ZERO models are graded by more than one judge" alongside a +// fixed per-judge model census; both silently went stale and the board then +// published a disclosure its own data contradicted (measured: 24 of 42 models +// carry more than one judge). A stale caveat is worse than none -- it reads as +// verified. +const judgeCensus = (() => { + const judgesPerModel = new Map>(); + for (const model of aggregated as any[]) { + const set = new Set(); + for (const suite of model.suites ?? []) { + for (const judge of suite.judgeModelIds ?? (suite.judgeModelId ? [suite.judgeModelId] : [])) { + set.add(judge); + } + } + if (set.size > 0) judgesPerModel.set(model.modelId, set); + } + const multi = [...judgesPerModel.values()].filter((s) => s.size > 1).length; + const single = [...judgesPerModel.values()].filter((s) => s.size === 1).length; + const singleBlocks = new Map(); + for (const set of judgesPerModel.values()) { + if (set.size === 1) { + const j = [...set][0]; + singleBlocks.set(j, (singleBlocks.get(j) ?? 0) + 1); + } + } + return { multi, single, graded: judgesPerModel.size, singleBlocks }; +})(); + +const JUDGE_NOTES: string[] = [ + `${ + judgeBreakdown.length === 1 + ? `SINGLE JUDGE OF RECORD: every admitted suite was graded by ${judgeBreakdown[0].judgeModelId}.` + : 'THERE IS NO SINGLE JUDGE OF RECORD.' + } Counted over this extract (${judgeCensus.graded} graded models): ${judgeBreakdown + .map((j) => `${j.judgeModelId} ${j.share.toFixed(1)}% of suites`) + .join(', ')}. ${judgeCensus.multi} model(s) were graded by more than one judge and ${ + judgeCensus.single + } by exactly one${ + judgeCensus.singleBlocks.size > 0 + ? ` (${[...judgeCensus.singleBlocks.entries()] + .sort((a, b) => b[1] - a[1]) + .map(([j, n]) => `${n} under ${j}`) + .join(', ')})` + : '' + }. Where a model sits in a single-judge block, judge severity cannot be separated from model quality, so comparisons ACROSS blocks are not safe. Every figure here is derived from the data, not asserted.`, + `Judge identity is not cosmetic: re-scoring identical trajectories with a different judge reorders this board (Spearman rho 0.55), which is more movement than the model-quality differences being read off it. A board aggregated over several judges is measuring the grader mix as well as the models.`, +]; + +const provenance = { + generatedAt: new Date().toISOString(), + commitSha: process.env.COMMIT_SHA, + source: 'golden cluster .ds-.evaluation-scores* (wave-2)', + judgeModelId, + judgeBreakdown, + traceCache: process.env.TRACES_JSON ? 'golden .ds-.evaluation-scores* (task.output)' : 'none', + // Statistical honesty notes ride in the template's own methodologyNotes slot, + // so nothing in the original layout is removed to make room for them. + // + // The judge-provenance notes are UNCONDITIONAL. They used to live inside the + // optional caveat-stats branch, which meant a plain render silently published + // a board with no judge disclosure at all -- the same failure that let a + // hardcoded "single unified judge" claim survive. A caveat that disappears + // when an env var is unset is not a disclosure. + methodologyNotes: [ + ...JUDGE_NOTES, + ...(ENSEMBLE_PATH + ? (() => { + const e = JSON.parse(fs.readFileSync(ENSEMBLE_PATH, 'utf8')); + const top = e.models[0]; + const widest = [...e.models].sort((a: any, b: any) => b.judgeSpread - a.judgeSpread)[0]; + return [ + `ENSEMBLE COLUMN: ${e.models.length} models were re-graded by all ${e.judges.length} judges ` + + `(${e.judges.join(', ')}) over ${ + e.sharedCellCount + } cells each judge scored, so judge severity ` + + `is separable from model quality on this block. Averaging ${e.judges.length} judges cuts the ` + + `independent component of judge noise by up to ${e.noiseReductionFactor.toFixed( + 2 + )}x -- an upper ` + + `bound, since judges are correlated rather than independent.`, + // Derived from the artifact, never asserted: a hardcoded pair count + // silently survives a re-run that produced different separability, + // which is the same failure mode as a hardcoded judge id. + `Under the ensemble, ${top.modelId} leads at ${top.ensemble.toFixed(3)}, and ${ + e.separablePairs + } of ${e.totalPairs} model pairs ` + + `separate by a paired bootstrap over shared examples (95% CI excluding zero) -- against exactly ` + + `1 judge-independent rank under any single judge. ${ + e.totalPairs - e.separablePairs + } of ${e.totalPairs} pairs are tied even by that measure.`, + `Do not read any of this as a ranking. The bootstrap resamples cells within a SINGLE run, so it ` + + `sees judge and example variance but is structurally blind to run-to-run variance. Seven models ` + + `in this golden were run twice (Sep 6 and Sep 7); across those re-run pairs the mean absolute ` + + `difference on 0-1 quality evaluators is 0.099 and only 71% of cells reproduce exactly. Every ` + + `pairwise gap on this ensemble (median 0.033, max 0.075) is SMALLER than that re-run noise ` + + `floor, including all ${e.separablePairs} "separable" and the ${ + e.separablePairs - e.borderlinePairs + } once called robust. The persona column orders nothing yet.`, + `That noise floor has now been MEASURED directly rather than inferred. A seed-only re-run of three ` + + `models (identical suite, examples and judge; 9/9 shards, 882 score docs, 2026-09-09) gives a ` + + `mean absolute difference of 0.0963 over 582 cells, with only ~64% of cells reproducing exactly ` + + `-- within 0.003 of the 0.099 upper bound, so the naming-convention confounds contributed ` + + `almost nothing and this is genuine sampling variance. Zero of 15 ensemble gaps clear it. ` + + `Averaging more judges cannot recover the loss, because the variance originates in the model's ` + + `own sampling rather than in judge disagreement.`, + `Per-model judge spread is published next to every ensemble score. ${widest.modelId} has the widest ` + + `at ${widest.judgeSpread.toFixed( + 3 + )}, meaning its consensus score averages over substantial judge ` + + `disagreement; a mean that hides that spread would overstate what the judges actually agreed on.`, + ]; + })() + : []), + ...(SATURATION_PATH + ? (() => { + const s: SaturationReport = JSON.parse(fs.readFileSync(SATURATION_PATH!, 'utf8')); + return [ + `The attack-discovery column cannot be ranked, and the limiting factor is ${s.limitingFactor}: ` + + `${(s.ceilingShare * 100).toFixed(1)}% of its ${ + s.cellCount + } scored cells sit at the rubric ` + + `ceiling and ${s.saturatedModels} of ${s.modelCount} models are indistinguishable from it.`, + `${ + s.judgeCount + } judges regraded those cells and disagreed by only ${s.judgeSpread.toFixed( + 3 + )} per cell, against a model-to-model spread of ${s.modelSpread.toFixed(3)}. ` + + `Judge noise is not what hides the ordering, so adding judges cannot recover it -- ` + + `only fixtures the models actually fail can. ${s.verdict}`, + ]; + })() + : []), + ...(CAVEAT_STATS_PATH + ? (() => { + const s = JSON.parse(fs.readFileSync(CAVEAT_STATS_PATH!, 'utf8')); + // Judged-evaluator subset, computed independently by the rejudge + // analysis (/tmp/matrix_final.json). Published alongside the + // all-evaluator figures so neither framing can be cherry-picked. + const judged = JSON.parse(fs.readFileSync(JUDGED_STATS_PATH!, 'utf8')); + const [j1, j2] = judged.rows; + // Derived, not asserted: top-two difference over pooled 95% CI. + judged.top2_t = ( + Math.abs(j1.score - j2.score) / Math.sqrt(j1.ci95 ** 2 + j2.ci95 ** 2) + ).toFixed(2); + return [ + `THIS BOARD DOES NOT RANK. Total spread across all ${s.ranking.length} models is ${s.spread} points on a 0-10 scale, while the 95% CI on a single model's mean reaches +/-${s.max_ci}. Only ${s.separable} of ${s.pairs} model pairs are distinguishable by a paired t-test over shared examples (two-sided .05); the #1-vs-#2 gap has t=${s.top2_t}. Every row is tie-tier T1. Read the ordering as arbitrary within the error bars.`, + `On the four LLM-judged evaluators alone -- the judged-quality axis, and the subset the ` + + `unified rejudge actually recomputed -- the spread is ${judged.spread} points against a median ` + + `95% CI of +/-${judged.median_ci95}, and only ${judged.n_distinguishable} of ${judged.n_pairs} model pairs ` + + `separate (paired t-test over shared examples, |t|>2). Five of those six are "beats Claude Opus 4.8"; ` + + `the #1-vs-#2 gap is t=${judged.top2_t}. Both framings agree: this board does not rank.`, + `Restricted to the four LLM-judged evaluators (Factuality, Groundedness, Relevance, Sequence Accuracy), the same computation gives a 0.70 spread against a +/-0.45 CI with the same ${s.separable}/${s.pairs} separable pairs -- the conclusion does not depend on which evaluator subset is used.`, + `The Attack Discovery and Automatic Migrations columns come from SEPARATE golden runs that were NOT part of the wave-2 unified rejudge. Their LLM-judged evaluators were scored by a mix of judges (mostly anthropic-claude-4.6-sonnet, some google-gemini-3.1-pro), and Attack Discovery is a SINGLE example -- which is why several rows read a saturated 10.0. Do not compare those three columns across models with the same confidence as the 21 persona columns, and note the headline spread/CI figures above are computed on the persona columns only, since these suites carry raw-count evaluators on a different scale.`, + `Re-scoring identical trajectories with a different judge (claude-4.5-haiku) reorders the board (Spearman rho=0.55): judge choice moves rank more than model quality does. That is the strongest argument against reading this as a leaderboard.`, + `Scores cluster because the suite saturates, not because the models are equal. Real discrimination needs repetitions (to measure the noise floor directly) and harder examples that frontier models actually fail.`, + `GLM models are excluded from this board entirely. gpt-5.4-nano contributes 19 of 21 examples: two produced no agent response at all and are omitted rather than scored as zero.`, + ]; + })() + : []), + ], + note: 'Judged evaluators carry the judge that actually graded them (derived per suite from the golden extract, not asserted); trace-based evaluators carried forward.', +}; + +const tracesPath = process.env.TRACES_JSON; +const traces = tracesPath + ? (JSON.parse(fs.readFileSync(tracesPath, 'utf8')) as MatrixTraceData) + : undefined; + +// A trace cache that LOADS is not a trace cache that LANDS. The ES-direct +// builder keys entries `execid::suite::model::example` and nests steps under +// `task.output.steps`, while the renderer looks up `traceKey(modelId, columnId)` +// and reads `entry.steps` at the top level. Passing the raw cache through +// yields exit 0, zero warnings, and a board with no trace cards at all -- +// exactly the defect TRACES_JSON was added to prevent. So assert on resolved +// lookups against the config, not on the env var being set. +if (traces) { + const traceRows = [...matrix.proprietary, ...matrix.openSource]; + const cols = config.columns.map((c) => c.id); + let hits = 0; + for (const row of traceRows) { + for (const col of cols) { + if (traces[`${row.modelId}:${col}`]?.steps?.length) hits++; + } + } + const withSteps = Object.values(traces).filter((t) => t?.steps?.length).length; + if (withSteps === 0) { + throw new Error( + `render_from_golden: ${tracesPath} has ${Object.keys(traces).length} entries but NONE ` + + `carry a top-level 'steps' array. This is the raw ES cache shape ` + + `(steps nested under task.output.steps); convert it to MatrixTraceEntry first ` + + `(scripts/orca_vm/to_matrix_trace_entries.py).` + ); + } + if (hits === 0) { + throw new Error( + `render_from_golden: ${tracesPath} resolved 0 of ${traceRows.length * cols.length} ` + + `(model, column) lookups. Entries are keyed ` + + `'${Object.keys(traces)[0]}' but the renderer looks up ` + + `'${traceRows[0]?.modelId}:${cols[0]}'. Re-key to traceKey(modelId, columnId).` + ); + } + // eslint-disable-next-line no-console + console.log( + `traces: ${hits}/${traceRows.length * cols.length} cell lookups resolved ` + + `(${withSteps} entries with steps)` + ); +} + +const html = renderMatrixHtml(matrix, config, provenance, traces); + +fs.mkdirSync(outDir, { recursive: true }); +// The upstream CLI (cli/commands/matrix.ts:437) writes the renderMatrixHtml +// output as matrix.html -- it never writes index.html. Match that name so the +// artifact is comparable to target/llm_matrix_final/matrix.html, the actual +// product of this renderer. (llm_matrix_final/index.html is a separate +// HAND-AUTHORED summary page with hardcoded hex colors and no generator in the +// repo; reproducing it would mean copying, which the anti-cheat rule forbids.) +fs.writeFileSync(path.join(outDir, 'matrix.html'), html); +// Keep index.html as a byte-identical alias so existing links/gates resolve. +fs.writeFileSync(path.join(outDir, 'index.html'), html); +fs.writeFileSync( + path.join(outDir, 'matrix.json'), + JSON.stringify({ ...matrix, provenance }, null, 2) +); + +const rows = [...matrix.proprietary, ...matrix.openSource]; +// eslint-disable-next-line no-console +console.log( + JSON.stringify( + { + rows: rows.length, + htmlChars: html.length, + tiers: rows.map((r) => ({ + model: r.modelId, + overall: r.overall?.kind === 'score' ? r.overall.value : r.overall?.kind, + tier: r.tier, + })), + warnings, + }, + null, + 1 + ) +); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.test.ts new file mode 100644 index 0000000000000..0164bda94bf3e --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.test.ts @@ -0,0 +1,240 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { buildMatrix } from '../../matrix/build_matrix'; +import { renderMatrix } from '../../matrix/render_matrix'; +import { branchBySuiteFromColumns } from './matrix'; +import { parseMatrixConfig } from '../../matrix/load_matrix_config'; +import type { AggregatedModelScores } from '../../matrix/query_matrix_scores'; +import { matrixScoreQuery } from './matrix'; + +describe('matrixScoreQuery', () => { + const query = (overrides = {}) => + matrixScoreQuery( + parseMatrixConfig({ + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'], examplePrefixes: ['a'] }], + models: [{ id: 'model-a', label: 'Model A' }], + ...overrides, + }), + { suiteIds: ['suite-a'], modelIds: ['model-a'], asOf: undefined } + ); + + it('forwards asOf to the score query', () => { + // The CLI parses --as-of, but selection happens in queryMatrixScores. If + // this object drops the value the flag is accepted and silently ignored, + // and the matrix publishes the runs it was told to exclude. + const asOf = Date.parse('2026-09-01T00:00:00.000Z'); + const q = matrixScoreQuery( + parseMatrixConfig({ + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'] }], + models: [{ id: 'model-a', label: 'Model A' }], + }), + { suiteIds: ['suite-a'], modelIds: ['model-a'], asOf } + ); + + expect(q.asOf).toBe(asOf); + }); + + it('lets a column opt out of the global self-judge exclusion', () => { + // gemini-3.1-pro judges the attack-discovery suite and is also ranked in + // it. The global policy blanks its cell; measured self-preference on that + // suite is absent (it ranks itself 4th of 6), so the column opts out + // without relaxing the policy for suites that were never audited. + const q = query({ + scoring: { excludeSelfJudged: true }, + columns: [ + { id: 'triage', label: 'Triage', suites: ['suite-a'] }, + { + id: 'kill-chain', + label: 'Kill-Chain Discovery', + suites: ['attack-discovery-agent-builder'], + allowSelfJudged: true, + }, + ], + }); + + // The audited suite opts out... + expect(q.scoringBySuite?.['attack-discovery-agent-builder']?.excludeSelfJudged).toBe(false); + // ...while every other suite is left out of the map entirely and falls + // back to the strict global policy at the suite loop. + expect(q.scoringBySuite?.['suite-a']).toBeUndefined(); + expect(q.scoring?.excludeSelfJudged).toBe(true); + }); + + it('forwards the opted-in scoring policy to the aggregator', () => { + expect(query({ scoring: { useVerdictLadder: true, requireEisJudge: true } }).scoring).toEqual({ + useVerdictLadder: true, + requireEisJudge: true, + excludeSelfJudged: false, + }); + }); + + it('forwards no policy when the config does not opt in', () => { + expect(query().scoring).toBeUndefined(); + }); + + it('de-duplicates example prefixes across columns', () => { + const options = query({ + columns: [ + { id: 'a', label: 'A', suites: ['s'], examplePrefixes: ['dup'] }, + { id: 'b', label: 'B', suites: ['s'], examplePrefixes: ['dup', 'other'] }, + ], + }); + + expect(options.prefixesBySuite).toEqual({ s: ['dup', 'other'] }); + }); + + // Suite histories are not co-located: the migrations suite publishes on a + // feature branch while every persona column lives on `main`. A single global + // branch can only satisfy one of them, so the other renders blank. + it('maps per-column branch overrides onto their suites', () => { + const options = query({ + columns: [ + { id: 'persona', label: 'Persona', suites: ['persona-suite'] }, + { + id: 'migrations', + label: 'Migrations', + suites: ['migrations-suite'], + branch: 'feat/matrix-v3', + }, + ], + }); + + expect(options.branchBySuite).toEqual({ 'migrations-suite': 'feat/matrix-v3' }); + }); + + it('omits suites that do not override the branch', () => { + expect(query().branchBySuite).toEqual({}); + }); + + it('rejects conflicting branch overrides for a shared suite', () => { + expect(() => + query({ + columns: [ + { id: 'a', label: 'A', suites: ['shared'], branch: 'branch-one' }, + { id: 'b', label: 'B', suites: ['shared'], branch: 'branch-two' }, + ], + }) + ).toThrow(/Conflicting branch overrides for suite "shared"/); + }); + + it('accepts agreeing branch overrides for a shared suite', () => { + const options = query({ + columns: [ + { id: 'a', label: 'A', suites: ['shared'], branch: 'same-branch' }, + { id: 'b', label: 'B', suites: ['shared'], branch: 'same-branch' }, + ], + }); + + expect(options.branchBySuite).toEqual({ shared: 'same-branch' }); + }); +}); + +describe('matrix command empty-result guard', () => { + const config = parseMatrixConfig({ + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'], weight: 1 }], + models: [{ id: 'model-a', label: 'Model A' }], + }); + + it('renders header-only CSVs when no experiments match', () => { + const rendered = renderMatrix(buildMatrix([], config), config); + + expect(rendered.proprietaryCsv.trim().split('\n')).toHaveLength(1); + expect(rendered.openSourceCsv.trim().split('\n')).toHaveLength(1); + }); + + it('renders populated CSVs when experiments do match', () => { + const aggregated: AggregatedModelScores[] = [ + { + modelId: 'model-a', + provider: 'anthropic', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'experiment-a', + datasets: [ + { + datasetId: 'dataset-a-id', + datasetName: 'dataset-a', + evaluators: [{ evaluatorName: 'correctness', mean: 0.9, count: 10 }], + }, + ], + }, + ], + }, + ]; + + const rendered = renderMatrix(buildMatrix(aggregated, config), config); + + expect(rendered.proprietaryCsv.trim().split('\n').length).toBeGreaterThan(1); + }); + + it('projects a branch LIST onto every suite the column reads', () => { + // Golden migrations data is split across branches by model, so a column + // may need several branches unioned rather than one pinned branch. + const withList = { + ...config, + columns: [ + { + ...config.columns[0], + suites: ['migrations-suite'], + branch: ['elastic:fix/weekly-evals-matrix', 'feat/evals-extensions-matrix-v3'], + }, + ], + } as unknown as typeof config; + + expect(branchBySuiteFromColumns(withList)).toEqual({ + 'migrations-suite': ['elastic:fix/weekly-evals-matrix', 'feat/evals-extensions-matrix-v3'], + }); + }); + + it('treats two columns declaring the same branch list as agreeing', () => { + // The conflict guard compares by value: two columns sharing a suite and + // declaring an equal list must not throw just because the arrays are + // distinct objects. + const shared = { + ...config, + columns: [ + { ...config.columns[0], suites: ['migrations-suite'], branch: ['a', 'b'] }, + { ...config.columns[0], suites: ['migrations-suite'], branch: ['a', 'b'] }, + ], + } as unknown as typeof config; + + expect(() => branchBySuiteFromColumns(shared)).not.toThrow(); + }); + + it('still rejects columns that disagree on a suite branch list', () => { + const conflicting = { + ...config, + columns: [ + { ...config.columns[0], suites: ['migrations-suite'], branch: ['a', 'b'] }, + { ...config.columns[0], suites: ['migrations-suite'], branch: ['a', 'c'] }, + ], + } as unknown as typeof config; + + expect(() => branchBySuiteFromColumns(conflicting)).toThrow(/Conflicting branch overrides/); + }); + + it('produces no model rows when no experiments match', () => { + const matrix = buildMatrix([], config); + + expect(matrix.proprietary).toHaveLength(0); + expect(matrix.openSource).toHaveLength(0); + }); + + // Deleting either preflight call from the command is invisible to every + // other test -- the warnings simply stop appearing, and a silently unhooked + // warning is worse than no warning because the artifact looks checked. + it('wires both data preflights into the matrix command', () => { + const source = readFileSync(join(__dirname, 'matrix.ts'), 'utf8'); + + expect(source).toContain('warnOnConfiguredNamesMissingFromData(config, aggregated, log)'); + expect(source).toContain('warnOnDataAboutToLeaveLookback(config, aggregated, log)'); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.ts new file mode 100644 index 0000000000000..8e1e3ab00c00e --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.ts @@ -0,0 +1,455 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Fs from 'fs'; +import Path from 'path'; +import { createFailError, createFlagError } from '@kbn/dev-cli-errors'; +import type { Command } from '@kbn/dev-cli-runner'; +import { + EvalsClient, + getEvaluationsKbnClient, + envFromDatasetsProfile, + DEFAULT_EVALUATIONS_KBN_URL, +} from '@kbn/evals'; +import { KbnClient } from '@kbn/kbn-client'; +import type { EvaluationScoreDocument } from '@kbn/evals-common'; +import { loadMatrixConfig, applyModelOverrides } from '../../matrix/load_matrix_config'; +import type { MatrixConfig } from '../../matrix/load_matrix_config'; +import { queryMatrixScores } from '../../matrix/query_matrix_scores'; +import type { + QueryMatrixScoresOptions, + ScoreAggregationOptions, +} from '../../matrix/query_matrix_scores'; +import { buildMatrix } from '../../matrix/build_matrix'; +import { renderMatrix } from '../../matrix/render_matrix'; +import { renderMatrixHtml } from '../../matrix/render_matrix_html'; +import { renderReliabilityHtml } from '../../matrix/render_reliability_html'; +import { queryMatrixTraces } from '../../matrix/query_matrix_traces'; +import type { MatrixTraceData } from '../../matrix/trace_types'; +import { readLocalGitState } from '../../matrix/local_git_state'; +import { + warnOnConfiguredNamesMissingFromData, + warnOnDataAboutToLeaveLookback, +} from '../../matrix/config_data_preflight'; + +const DEFAULT_OUT_DIR = 'target/llm_matrix'; + +/** + * Build the aggregation query for a matrix run. + * + * Extracted from the command body so the config-to-aggregation link is + * reachable from a unit test: the scoring policy is only worth anything if a + * config value actually survives the trip, and a test that cannot see this + * object cannot notice when it stops being passed. + */ +export const matrixScoreQuery = ( + config: MatrixConfig, + { + suiteIds, + modelIds, + branch, + lookbackDays, + asOf, + }: Omit< + QueryMatrixScoresOptions, + 'prefixesBySuite' | 'scoring' | 'branchBySuite' | 'scoringBySuite' | 'asOf' + > & { + /** + * Required (though nullable) so a caller that forgets it fails to compile. + * An optional `asOf` was silently dropped at the command call site once + * already: the flag parsed, selection ignored it, and the matrix published + * exactly the runs the cutoff existed to exclude. + */ + asOf: number | undefined; + } +): QueryMatrixScoresOptions => ({ + suiteIds, + modelIds, + branch, + branchBySuite: branchBySuiteFromColumns(config), + lookbackDays, + asOf, + prefixesBySuite: prefixesBySuiteFromColumns(config), + scoring: config.scoring, + scoringBySuite: scoringBySuiteFromColumns(config), +}); + +/** + * Collapses per-column `allowSelfJudged` into a suite-keyed scoring policy. + * + * A judge that is also a ranked model has its own row dropped by the global + * `excludeSelfJudged`, blanking a cell it genuinely earned. Opting out is + * scoped to the suite whose judge was actually audited: gemini-3.1-pro also + * self-judges 100% of security-automatic-migrations, so a global flip would + * admit self-judged scores for suites nobody measured. + */ +export const scoringBySuiteFromColumns = ( + config: MatrixConfig +): Record => { + const bySuite: Record = {}; + for (const column of config.columns) { + if (column.allowSelfJudged === undefined) { + continue; + } + for (const suiteId of column.suites ?? []) { + bySuite[suiteId] = { ...config.scoring, excludeSelfJudged: !column.allowSelfJudged }; + } + } + return bySuite; +}; + +/** + * Collapses per-column `examplePrefixes` into a suite-keyed map. + * + * A single global union made the per-prefix fetch run for every suite, so a + * suite whose column declares no prefixes still paid the extra query and + * then reported every score as an unmapped verdict. attack-discovery writes + * a constant example id and produced 63 such rejections per model while its + * column scored correctly from aggregate stats. + */ +export const prefixesBySuiteFromColumns = (config: MatrixConfig): Record => { + const bySuite: Record = {}; + + for (const column of config.columns) { + if (!column.examplePrefixes?.length) { + continue; + } + for (const suiteId of column.suites) { + bySuite[suiteId] = [...new Set([...(bySuite[suiteId] ?? []), ...column.examplePrefixes])]; + } + } + + return bySuite; +}; +/** + * Collapses per-column `branch` overrides into the suite-keyed map the query + * layer consumes. + * + * Columns address suites, but the score query iterates suites, so an override + * declared on a column has to be projected onto every suite that column reads. + * Two columns sharing a suite must agree: silently honouring the first would + * make the resulting cells depend on config ordering, so a genuine conflict + * throws rather than resolving arbitrarily. + */ +export const branchBySuiteFromColumns = ( + config: MatrixConfig +): Record => { + const bySuite: Record = {}; + // Compare by value: a branch override may be a list, and two columns + // declaring equal lists agree even though the arrays are distinct objects. + const describe = (branch: string | string[]): string => + Array.isArray(branch) ? branch.join(', ') : branch; + + for (const column of config.columns) { + if (!column.branch) { + continue; + } + for (const suiteId of column.suites) { + const existing = bySuite[suiteId]; + if (existing !== undefined && describe(existing) !== describe(column.branch)) { + throw new Error( + `Conflicting branch overrides for suite "${suiteId}": ` + + `"${describe(existing)}" and "${describe(column.branch)}". A suite is queried ` + + `once, so its columns must agree on which branch to read.` + ); + } + bySuite[suiteId] = column.branch; + } + } + + return bySuite; +}; + +export const matrixCmd: Command = { + name: 'matrix', + description: ` + Generate an LLM performance matrix artifact from exported evaluation results. + + Reads the latest experiment per (model, suite) from the evals plugin on the + target Kibana, maps suites/datasets/evaluators onto matrix columns via a config + file, normalizes scores onto a 0-10 scale, and writes markdown + CSV + JSON. + + Configure target/auth with EVAL_KBN_URL and EVAL_KBN_API_KEY, + with --kbn-url/--kbn-api-key, or with --profile (e.g. dev-vault for the golden + cluster, or a config..json file). + + Example: + node scripts/evals ext matrix \\ + --config .buildkite/pipelines/evals/security_matrix.config.json \\ + --profile dev-vault --branch main --out target/llm_matrix + `, + flags: { + string: [ + 'config', + 'out', + 'branch', + 'lookback-days', + 'as-of', + 'profile', + 'kbn-url', + 'kbn-api-key', + 'model', + 'trace-cache', + ], + boolean: ['html'], + allowUnexpected: false, + help: ` + --config Path to the matrix config JSON (required). + --out Output directory for artifacts (default: ${DEFAULT_OUT_DIR}). + --branch Git branch filter override (default: config.branch). + --lookback-days Only consider experiments newer than now-d (default: config.lookbackDays). + --as-of Render the matrix as of an ISO date/instant, ignoring runs + after it for every model alike (e.g. 2026-09-01). Use to + reproduce an earlier matrix or exclude a known-bad window. + --model Replace the config's model set for an on-demand run. + Format: id[:label][:open-source]. Repeatable. + e.g. --model gpt-5-preview:GPT-5 --model qwen3:Qwen3:open-source + --profile Golden-cluster config profile providing EVAL_KBN_URL/API_KEY + (e.g. 'dev-vault' for runtime Vault, or a config..json file). + --trace-cache Path to a trace-cache JSON (executionId::exampleId -> score docs). + --kbn-url Kibana URL override. + --kbn-api-key Kibana API key override. + --html Also generate a self-contained HTML report (matrix.html). + `, + }, + run: async ({ log, flagsReader }) => { + const configPath = flagsReader.string('config'); + if (!configPath) { + throw createFlagError('--config is required. Provide the path to a matrix config JSON.'); + } + + const repoRoot = process.cwd(); + const baseConfig = loadMatrixConfig(Path.resolve(repoRoot, configPath)); + + const modelOverrides = flagsReader.arrayOfStrings('model') ?? []; + let config: MatrixConfig; + try { + config = applyModelOverrides(baseConfig, modelOverrides); + } catch (error) { + throw createFlagError(error instanceof Error ? error.message : String(error)); + } + if (modelOverrides.length > 0) { + log.info( + `Overriding config model set with ${ + config.models.length + } on-demand model(s): ${config.models.map((model) => model.id).join(', ')}` + ); + } + + const profile = flagsReader.string('profile') ?? undefined; + const profileEnv = envFromDatasetsProfile(repoRoot, profile); + + const evaluationsKbnUrl = + flagsReader.string('kbn-url') ?? profileEnv.EVAL_KBN_URL ?? process.env.EVAL_KBN_URL; + if (!evaluationsKbnUrl) { + log.warning(`EVAL_KBN_URL not set; defaulting to ${DEFAULT_EVALUATIONS_KBN_URL}.`); + } + + const evaluationsKbnApiKey = + flagsReader.string('kbn-api-key') ?? + profileEnv.EVAL_KBN_API_KEY ?? + process.env.EVAL_KBN_API_KEY; + + const branch = flagsReader.string('branch') ?? config.branch; + const lookbackDaysFlag = flagsReader.string('lookback-days'); + const lookbackDays = lookbackDaysFlag ? Number(lookbackDaysFlag) : config.lookbackDays; + if (Number.isNaN(lookbackDays) || lookbackDays < 1) { + throw createFlagError('--lookback-days must be a positive number.'); + } + + const asOfFlag = flagsReader.string('as-of'); + const asOf = asOfFlag === undefined ? undefined : Date.parse(asOfFlag); + if (asOf !== undefined && !Number.isFinite(asOf)) { + throw createFlagError('--as-of must be an ISO date or instant, e.g. 2026-09-01.'); + } + + const outDir = Path.resolve(repoRoot, flagsReader.string('out') ?? DEFAULT_OUT_DIR); + const suiteIds = [...new Set(config.columns.flatMap((column) => column.suites))]; + // Query per (suite, model) pair: the experiments route answers from a + // terms aggregation that grows with the page number, so the listing must + // stay bounded per pair. Include matchIds so aliased model rows are found. + const modelIds = [ + ...new Set(config.models.flatMap((model) => [model.id, ...(model.matchIds ?? [])])), + ]; + + const defaultKbnClient = new KbnClient({ log, url: DEFAULT_EVALUATIONS_KBN_URL }); + const kbnClient = getEvaluationsKbnClient({ + kbnClient: defaultKbnClient, + log, + evaluationsKbnUrl, + evaluationsKbnApiKey, + }); + const evalsClient = new EvalsClient(kbnClient, log); + + try { + await evalsClient.assertPluginEnabled(); + } catch (error) { + throw createFlagError( + [ + error instanceof Error ? error.message : String(error), + 'Set EVAL_KBN_URL to a Kibana instance with xpack.evals.enabled=true.', + 'Set EVAL_KBN_API_KEY when authenticating to a non-local target.', + ].join('\n') + ); + } + + log.info( + `Querying matrix scores from ${evaluationsKbnUrl ?? DEFAULT_EVALUATIONS_KBN_URL} (branch: ${ + branch ?? 'any' + })` + ); + + const aggregated = await queryMatrixScores( + evalsClient, + log, + matrixScoreQuery(config, { suiteIds, modelIds, branch, lookbackDays, asOf }) + ); + + if (aggregated.length === 0) { + // Empty CSVs would publish as a blank matrix in customer-facing docs. + throw createFailError( + [ + 'No experiments matched the configured filters, refusing to write an empty matrix.', + `Filters: suites=[${suiteIds.join(', ')}] models=[${modelIds.join(', ')}] branch=${ + branch ?? 'any' + } lookbackDays=${lookbackDays}`, + 'Check that the weekly eval run published results for these suites in the lookback window.', + ].join('\n') + ); + } + + warnOnConfiguredNamesMissingFromData(config, aggregated, log); + warnOnDataAboutToLeaveLookback(config, aggregated, log); + + const matrix = buildMatrix(aggregated, config, log); + const generateHtml = flagsReader.boolean('html'); + + // Query traces before rendering so they can be embedded in matrix.json — + // the artifact then carries everything needed to audit a cell's full + // conversation without re-querying the evals cluster. + let traces: MatrixTraceData | undefined; + // Recorded in the artifact's provenance. A matrix built from a dirty tree or + // from a different cache than a later regen will not reproduce, and every + // stale-artifact mix-up this pipeline has had looked exactly like a real + // result until someone diffed it. + const traceCacheForProvenance = flagsReader.string('trace-cache') ?? 'none'; + const localGit = readLocalGitState(repoRoot, log); + if (generateHtml) { + log.info('Querying trace data for HTML report...'); + // --trace-cache : pre-pulled score documents keyed + // `${executionId}::${exampleId}`, e.g. fetched directly from the evals + // cluster's ES when the Kibana route can't serve them (older plugin + // builds ignore execution filters and trip the response-size cap on + // heavy examples). Cached cells skip the server fetch entirely. + const traceCachePath = flagsReader.string('trace-cache'); + let traceCache: Record | undefined; + if (traceCachePath) { + traceCache = JSON.parse(Fs.readFileSync(traceCachePath, 'utf8')) as Record< + string, + EvaluationScoreDocument[] + >; + const cellCount = traceCache ? Object.keys(traceCache).length : 0; + log.info(`Loaded trace cache: ${cellCount} cells from ${traceCachePath}`); + } + traces = await queryMatrixTraces( + evalsClient, + log, + aggregated, + traceCache, + config.toolCallWarnAbove, + new Map( + config.models + .filter((model) => (model.matchIds ?? []).length > 0) + .map((model) => [model.id, model.matchIds ?? []]) + ) + ); + + // Traces fetched from the server come back hollow (stepCount 0) when the + // evals plugin cannot serve step payloads, so the report renders panels + // with nothing in them and still exits 0. Counting panels is not enough + // -- count the ones that actually carry steps. + const traceCells = Object.values(traces ?? {}); + const withSteps = traceCells.filter((t) => (t?.stepCount ?? 0) > 0).length; + if (traceCells.length === 0) { + log.warning('no traces resolved -- the report will have no trace panels'); + } else if (withSteps === 0) { + log.warning( + `all ${traceCells.length} traces came back without steps` + + (traceCachePath + ? '' + : ' -- pass --trace-cache to load step payloads from a pre-pulled cache') + ); + } + } + + const rendered = renderMatrix( + matrix, + config, + { + branch, + lookbackDays, + asOf, + suiteIds, + commitSha: process.env.BUILDKITE_COMMIT ?? localGit.sha, + dirtyWorkingTree: localGit.dirty, + traceCache: traceCacheForProvenance, + buildUrl: process.env.BUILDKITE_BUILD_URL, + }, + traces + ); + + Fs.mkdirSync(outDir, { recursive: true }); + const writes: Array<[string, string]> = [ + ['proprietary-models.csv', rendered.proprietaryCsv], + ['open-source-models.csv', rendered.openSourceCsv], + ['matrix.md', rendered.markdown], + ['matrix.json', rendered.json], + // Raw, pre-scaling per-evaluator means/counts, so reviewers can audit which + // evaluators feed a cell without re-querying. + ['scores.debug.json', `${JSON.stringify(aggregated, null, 2)}\n`], + ]; + for (const [fileName, contents] of writes) { + Fs.writeFileSync(Path.join(outDir, fileName), contents); + } + + if (generateHtml && traces) { + const htmlContent = renderMatrixHtml( + matrix, + config, + { + branch, + lookbackDays, + asOf, + suiteIds, + commitSha: process.env.BUILDKITE_COMMIT ?? localGit.sha, + dirtyWorkingTree: localGit.dirty, + traceCache: traceCacheForProvenance, + buildUrl: process.env.BUILDKITE_BUILD_URL, + fixtureFingerprint: config.provenance?.fixtureFingerprint, + methodologyNotes: config.provenance?.methodologyNotes, + }, + traces + ); + Fs.writeFileSync(Path.join(outDir, 'matrix.html'), htmlContent); + const reliabilityHtml = renderReliabilityHtml(matrix, traces, { + branch, + lookbackDays, + asOf, + commitSha: process.env.BUILDKITE_COMMIT ?? localGit.sha, + dirtyWorkingTree: localGit.dirty, + }); + Fs.writeFileSync(Path.join(outDir, 'matrix.reliability.html'), reliabilityHtml); + log.info(`Wrote matrix.html and matrix.reliability.html to ${outDir}`); + } + + log.info( + `Wrote matrix artifacts to ${outDir} ` + + `(${matrix.proprietary.length} proprietary, ${matrix.openSource.length} open-source models)` + ); + log.info(`\n${rendered.markdown}`); + }, +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/rejudge.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/rejudge.test.ts new file mode 100644 index 0000000000000..8e3af6777544f --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/rejudge.test.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Fs from 'fs'; +import Os from 'os'; +import Path from 'path'; +import { buildJudgeAuthHeader, loadSuiteRubricItems } from './rejudge'; + +describe('buildJudgeAuthHeader', () => { + it('base64-encodes raw user:pass credentials', () => { + expect(buildJudgeAuthHeader({ basicAuth: 'elastic:changeme' })).toBe( + `Basic ${Buffer.from('elastic:changeme').toString('base64')}` + ); + }); + + // Regression: a pre-encoded JUDGE_KBN_AUTH double-encodes, and the resulting + // 401 only surfaces after every planned cell has been judged and discarded -- + // an entire AD rejudge (99 cells, two judges) was lost to it. + it('rejects an already base64-encoded value instead of double-encoding', () => { + const encoded = Buffer.from('elastic:changeme').toString('base64'); + expect(() => buildJudgeAuthHeader({ basicAuth: encoded })).toThrow(/raw "user:pass"/); + }); + + it('rejects a value that already carries the Basic scheme', () => { + expect(() => buildJudgeAuthHeader({ basicAuth: 'Basic abc123' })).toThrow(/raw "user:pass"/); + }); + + it('falls back to the API key when no basic auth is given', () => { + expect(buildJudgeAuthHeader({ apiKey: 'abc' })).toBe('ApiKey abc'); + }); + + it('prefers basic auth over an API key, matching the stack that needs it', () => { + expect(buildJudgeAuthHeader({ basicAuth: 'a:b', apiKey: 'abc' })).toBe( + `Basic ${Buffer.from('a:b').toString('base64')}` + ); + }); + + it('returns undefined when neither credential is present', () => { + expect(buildJudgeAuthHeader({})).toBeUndefined(); + }); +}); + +describe('loadSuiteRubricItems', () => { + // The suite owns the rubric; the jury keeps a fallback copy. These tests pin + // the resolution so a suite-side rubric change cannot be silently shadowed by + // the stale copy -- the exact drift that let a rejudge grade a collapsed + // "5 of 7 -> Y or N" rubric while the suite scored seven items separately. + let root: string; + + beforeEach(() => { + root = Fs.mkdtempSync(Path.join(Os.tmpdir(), 'rubric-load-')); + }); + + afterEach(() => { + Fs.rmSync(root, { recursive: true, force: true }); + }); + + const writeEvaluator = (body: string) => { + const evaluatorsDir = Path.join(root, 'src', 'evaluators'); + Fs.mkdirSync(evaluatorsDir, { recursive: true }); + Fs.writeFileSync( + Path.join(evaluatorsDir, 'attack_discovery_rubric_evaluator.ts'), + body, + 'utf8' + ); + const datasetsDir = Path.join(root, 'src', 'datasets'); + Fs.mkdirSync(datasetsDir, { recursive: true }); + const datasetPath = Path.join(datasetsDir, 'some_dataset.ts'); + Fs.writeFileSync(datasetPath, 'export const examples = [];\n', 'utf8'); + return datasetPath; + }; + + it('loads the rubric items the suite exports', async () => { + const datasetPath = writeEvaluator( + "export const ATTACK_DISCOVERY_RUBRIC_ITEMS = ['item one', 'item two', 'item three'];\n" + ); + + await expect(loadSuiteRubricItems(datasetPath)).resolves.toEqual([ + 'item one', + 'item two', + 'item three', + ]); + }); + + it('returns undefined when the suite exports no rubric items', async () => { + const datasetPath = writeEvaluator('export const SOMETHING_ELSE = 1;\n'); + + await expect(loadSuiteRubricItems(datasetPath)).resolves.toBeUndefined(); + }); + + it('returns undefined rather than throwing when the evaluator is absent', async () => { + const datasetsDir = Path.join(root, 'src', 'datasets'); + Fs.mkdirSync(datasetsDir, { recursive: true }); + const datasetPath = Path.join(datasetsDir, 'orphan_dataset.ts'); + Fs.writeFileSync(datasetPath, 'export const examples = [];\n', 'utf8'); + + await expect(loadSuiteRubricItems(datasetPath)).resolves.toBeUndefined(); + }); + + it('ignores a non-string rubric export instead of passing it to the judge', async () => { + // A malformed export must not reach the judge as criteria: it would be + // stringified into nonsense and graded as if it were a requirement. + const datasetPath = writeEvaluator('export const ATTACK_DISCOVERY_RUBRIC_ITEMS = [1, 2, 3];\n'); + + await expect(loadSuiteRubricItems(datasetPath)).resolves.toBeUndefined(); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/rejudge.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/rejudge.ts new file mode 100644 index 0000000000000..963e04e939de0 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/rejudge.ts @@ -0,0 +1,668 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Fs from 'fs'; +import Path from 'path'; +import { REPO_ROOT } from '@kbn/repo-info'; +import { createFailError, createFlagError } from '@kbn/dev-cli-errors'; +import type { Command } from '@kbn/dev-cli-runner'; +import { + envFromDatasetsProfile, + createCorrectnessAnalysisEvaluator, + createQuantitativeCorrectnessEvaluators, + createGroundednessAnalysisEvaluator, + createQuantitativeGroundednessEvaluator, + createCriteriaEvaluator, + type EvaluationCriterion, +} from '@kbn/evals'; +import type { BoundInferenceClient } from '@kbn/inference-common'; +import type { ToolingLog } from '@kbn/tooling-log'; +import type { HttpHandler } from '@kbn/core/public'; +import { createRestClient } from '@kbn/inference-plugin/common'; +import { loadMatrixConfig } from '../../matrix/load_matrix_config'; +import { planReplay, summarizePlan, type ReplayCell } from '../../matrix/replay_plan'; +import { fetchScoreDocs } from '../../matrix/fetch_score_docs'; +import { anonymizeCell, buildAliasMap } from '../../matrix/anonymize_cell'; +import { runRejudge, type CellJudge, type RejudgeScore } from '../../matrix/run_rejudge'; +import { + collectExamples, + selectAdapter, + buildStructuredReferences, + REFERENCE_ADAPTERS, + DEFAULT_JOIN_FIELD, +} from '../../matrix/reference_adapters'; +import { selectJury, checkJuryCoverage, JURY_ADAPTERS } from '../../matrix/jury_adapters'; +import type { JuryAdapter, JuryArgs } from '../../matrix/jury_adapters'; + +const DEFAULT_OUT_DIR = 'target/llm_matrix_rejudge'; + +/** + * Load `exampleId -> reference` from a suite dataset module. + * + * Golden score documents carry an empty `example.output`, so the ground truth a + * correctness judge compares against exists only in the suite's dataset. A + * replay without it grades every answer against an empty reference and + * manufactures uniform inaccuracy verdicts. + */ +/** + * Load the AD rubric items from the suite package at runtime. + * + * A platform package must not statically import the private solutions-side + * suite, so the jury keeps a fallback copy of the rubric. Two copies drift: + * the suite moved to per-item scoring while the jury still collapsed all seven + * items into one "5 of 7 -> Y or N" question, so a rejudge silently graded a + * different thing under the same column name. + * + * Resolving the suite's own export keeps one definition authoritative. Returns + * undefined when the module cannot be resolved -- the jury's fallback then + * applies, and it is the same per-item form, so a miss degrades to a stale + * wording rather than to the collapsed rubric. + */ +export async function loadSuiteRubricItems( + datasetPath: string, + log?: { info: (msg: string) => void } +): Promise { + const datasetDir = Path.dirname(Path.resolve(process.cwd(), datasetPath)); + // The evaluator sits under `src/evaluators` in the suite package; walk up + // from the dataset module rather than hardcoding a repo-relative path. + const candidates = [ + Path.resolve(datasetDir, '../evaluators/attack_discovery_rubric_evaluator.ts'), + Path.resolve(datasetDir, '../../evaluators/attack_discovery_rubric_evaluator.ts'), + Path.resolve(datasetDir, 'evaluators/attack_discovery_rubric_evaluator.ts'), + ]; + + for (const candidate of candidates) { + if (!Fs.existsSync(candidate)) { + continue; + } + try { + const mod = (await import(candidate)) as Record; + const items = mod.ATTACK_DISCOVERY_RUBRIC_ITEMS; + if (Array.isArray(items) && items.length > 0 && items.every((i) => typeof i === 'string')) { + log?.info(`Loaded ${items.length} rubric item(s) from ${candidate}`); + return items as string[]; + } + } catch (error) { + // Fall through to the jury's copy rather than failing the whole replay. + log?.info( + `Could not load rubric items from ${candidate}: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } + + return undefined; +} + +async function loadReferences( + datasetPath: string, + log?: { info: (msg: string) => void } +): Promise<{ + references: Map; + structured: Map>; + adapterName: string; + joinField: string; +}> { + const resolved = Path.resolve(process.cwd(), datasetPath); + if (!Fs.existsSync(resolved)) { + throw createFlagError(`--dataset path does not exist: ${resolved}`); + } + + const mod = await import(resolved); + const examples = collectExamples(mod as Record); + + if (examples.length === 0) { + throw createFlagError( + `--dataset module ${resolved} does not export an examples array ` + + `(looked for ${[...new Set(REFERENCE_ADAPTERS.flatMap((a) => a.exportNames))].join(', ')})` + ); + } + + // Each suite states ground truth in its own shape; pick the adapter whose + // contract these examples actually satisfy rather than assuming a reference + // string exists. + const adapter = selectAdapter(examples); + if (!adapter) { + throw createFlagError( + `--dataset module ${resolved} matches no reference adapter ` + + `(tried: ${REFERENCE_ADAPTERS.map((a) => a.name).join(', ')}). ` + + `Add an adapter in matrix/reference_adapters.ts for this suite.` + ); + } + + const references = adapter.build(examples); + log?.info(`Reference adapter "${adapter.name}" resolved ${references.size} example(s)`); + + if (references.size === 0) { + throw createFlagError( + `--dataset module ${resolved} yielded no example references via the ` + + `"${adapter.name}" adapter` + ); + } + + return { + references, + structured: buildStructuredReferences(examples), + adapterName: adapter.name, + joinField: adapter.joinField ?? DEFAULT_JOIN_FIELD, + }; +} + +/** + * JUDGE_KBN_AUTH is raw `user:pass` and is base64-encoded here. A value that is + * already encoded (or already carries the `Basic ` scheme) double-encodes into a + * 401 that only surfaces after every cell has been judged and thrown away, so it + * is rejected up front rather than after the run. + */ +export function buildJudgeAuthHeader({ + basicAuth, + apiKey, +}: { + basicAuth?: string; + apiKey?: string; +}): string | undefined { + if (basicAuth) { + if (!basicAuth.includes(':')) { + throw createFlagError( + 'JUDGE_KBN_AUTH must be raw "user:pass" -- it is base64-encoded for you. ' + + 'A pre-encoded or "Basic ..."-prefixed value double-encodes and fails every ' + + 'judge call with 401 invalid basic authentication header value.' + ); + } + return `Basic ${Buffer.from(basicAuth).toString('base64')}`; + } + return apiKey ? `ApiKey ${apiKey}` : undefined; +} + +export const rejudgeCmd: Command = { + name: 'rejudge', + description: ` + Re-grade already-recorded trajectories with a different judge. + + Reads stored agent outputs from the evals cluster and re-runs only the + LLM-judged evaluators. No agent, Kibana stack, or seeded data is needed: + every input a judge reads is durable in the score documents. Trace-based + evaluators (SkillInvoked, tokens, latency) are unaffected by a judge swap + and are left on their source run rather than recomputed. + + Results are written to a local JSON artifact for inspection. + `, + flags: { + string: [ + 'config', + 'judge', + 'dataset', + 'out', + 'models', + 'concurrency', + 'as-of', + 'execution-id', + 'from-matrix', + 'profile', + 'suite', + ], + boolean: ['blind', 'dry-run'], + help: ` + --config Matrix config JSON (selects models/suites). + --judge Judge tag recorded on results, e.g. "haiku". Required. + --dataset Path to the suite dataset module supplying references. Required. + --suite Suite id selecting the jury (evaluator set) to recompute. + Defaults to the jury matching the dataset's reference adapter. + --from-matrix scores.debug.json from a rendered matrix; re-judges + exactly the executions that matrix published + --execution-id Re-judge specific execution ids (comma separated). + --models Restrict to these model ids (comma separated). + --blind Strip model identity before judging. + --dry-run Plan only: report cell counts and cost, call no judge. + --concurrency Parallel judge calls (default 5). + --out Output directory (default ${DEFAULT_OUT_DIR}). + --profile Golden-cluster config profile providing EVAL_KBN_URL/API_KEY. + --as-of Only replay runs recorded before this ISO timestamp. + `, + }, + run: async ({ log, flagsReader }) => { + const judgeTag = flagsReader.string('judge'); + if (!judgeTag) { + throw createFlagError('--judge is required so re-judged scores stay separable'); + } + + const datasetPath = flagsReader.string('dataset'); + if (!datasetPath) { + throw createFlagError('--dataset is required: golden documents carry no ground truth'); + } + + const configPath = flagsReader.string('config'); + const blind = flagsReader.boolean('blind'); + const dryRun = flagsReader.boolean('dry-run'); + const outDir = flagsReader.string('out') ?? DEFAULT_OUT_DIR; + const concurrency = Number(flagsReader.string('concurrency') ?? '5'); + const asOfFlag = flagsReader.string('as-of'); + if (asOfFlag && Number.isNaN(Date.parse(asOfFlag))) { + throw createFlagError(`--as-of must be an ISO date, got "${asOfFlag}"`); + } + + const explicitExecutionIds: string[] = (flagsReader.string('execution-id') ?? '') + .split(',') + .map((s: string) => s.trim()) + .filter(Boolean); + const modelFilter = new Set( + (flagsReader.string('models') ?? '') + .split(',') + .map((s: string) => s.trim()) + .filter(Boolean) + ); + + const config = configPath ? loadMatrixConfig(configPath) : undefined; + const { references, structured, adapterName, joinField } = await loadReferences( + datasetPath, + log + ); + log.info(`Loaded ${references.size} dataset reference(s) from ${datasetPath}`); + + // The jury is resolved from the suite the scores belong to, not from the + // dataset path. An unregistered suite is a hard error: falling back to the + // persona jury is what silently produced Factuality/Relevance verdicts for + // an Attack Discovery replay, which look like scores but grade the wrong + // artefact with the wrong rubric. + const suiteFlag = flagsReader.string('suite'); + const jury = selectJury(suiteFlag ?? adapterName); + if (!jury) { + throw createFlagError( + `No jury adapter for suite "${suiteFlag ?? adapterName}". ` + + `Known juries: ${JURY_ADAPTERS.map((j) => `${j.name} (${j.suiteIds.join(', ')})`).join( + '; ' + )}. Add one in matrix/jury_adapters.ts rather than replaying with a ` + + `jury built for a different suite.` + ); + } + log.info(`Jury "${jury.name}" recomputes: ${jury.evaluatorNames.join(', ')}`); + + // Only consult the profile when one was asked for: envFromDatasetsProfile + // shells out to Vault, which blocks for minutes when no Vault is reachable. + const profileFlag = flagsReader.string('profile') ?? undefined; + const profileEnv = profileFlag ? envFromDatasetsProfile(REPO_ROOT, profileFlag) : {}; + const evaluationsKbnUrl = profileEnv.EVAL_KBN_URL ?? process.env.EVAL_KBN_URL; + const evaluationsKbnApiKey = profileEnv.EVAL_KBN_API_KEY ?? process.env.EVAL_KBN_API_KEY; + + // Read score documents straight from Elasticsearch rather than through the + // evals plugin API: the golden cluster stores the data but does not run the + // plugin (/internal/evals answers 400 "not available with the current + // configuration"), so the API path cannot reach the archive being replayed. + const esUrl = process.env.GOLDEN_ES_URL; + const esApiKey = process.env.GOLDEN_ES_API_KEY; + if (!esUrl || !esApiKey) { + throw createFlagError( + 'GOLDEN_ES_URL and GOLDEN_ES_API_KEY must point at the cluster holding the scores.' + ); + } + + // Which runs to re-judge is not a model filter. A model can have a dozen + // archived reruns (25 config model ids match 136 executions), so filtering + // by model would grade runs the matrix never published. + // + // queryMatrixScores cannot make this selection here: it needs an + // EvalsClient, and the golden cluster does not run the evals plugin. The + // rendered matrix already records the executions it published, so the + // replay reuses that artifact and grades exactly those trajectories. + let executionIds: string[] | undefined = explicitExecutionIds.length + ? explicitExecutionIds + : undefined; + + const matrixDebugPath = flagsReader.string('from-matrix'); + if (!executionIds && matrixDebugPath) { + const resolved = Path.resolve(process.cwd(), matrixDebugPath); + if (!Fs.existsSync(resolved)) { + throw createFlagError(`--from-matrix path does not exist: ${resolved}`); + } + + const debug = JSON.parse(Fs.readFileSync(resolved, 'utf8')) as Array<{ + modelId: string; + suites: Array<{ experimentId: string }>; + }>; + + executionIds = [ + ...new Set( + debug + .filter((row) => modelFilter.size === 0 || modelFilter.has(row.modelId)) + .flatMap((row) => row.suites.map((suite) => suite.experimentId)) + ), + ]; + + log.info(`Selected ${executionIds.length} published execution(s) from ${matrixDebugPath}`); + } + + const docs = await fetchScoreDocs({ + esUrl, + apiKey: esApiKey, + exampleIds: [...references.keys()], + joinField, + executionIds, + modelIds: modelFilter.size > 0 ? [...modelFilter] : undefined, + configModelIds: config?.models.flatMap((m) => [m.id, ...(m.matchIds ?? [])]), + suiteIds: config ? [...new Set(config.columns.flatMap((c) => c.suites))] : [...jury.suiteIds], + asOf: asOfFlag ? Date.parse(asOfFlag) : undefined, + }); + log.info(`Fetched ${docs.length} score document(s) from ${esUrl}`); + + // planReplay both builds the judgeable cells and reports why the rest are + // unusable. A second extractor over the same documents could silently + // disagree with the plan it is reported alongside. + const plan = planReplay(docs as never[], (id) => references.get(id), { + jury, + joinField, + structuredReferenceFor: (id) => structured.get(id), + }); + const cellsToJudge = plan.cells; + log.info(`Replay plan: ${summarizePlan(plan)}`); + + if (plan.skipped.length > 0) { + log.warning(`${plan.skipped.length} cell(s) cannot be replayed:`); + for (const issue of plan.skipped.slice(0, 10)) { + log.warning(` ${issue.executionId} / ${issue.exampleId}: ${issue.reason}`); + } + } + + if (dryRun) { + log.info('--dry-run: no judge was called and nothing was written.'); + return; + } + + if (cellsToJudge.length === 0) { + throw createFailError('Replay plan is empty; refusing to write an empty rejudge artifact.'); + } + + const aliases = buildAliasMap(cellsToJudge); + const cells: ReplayCell[] = blind + ? cellsToJudge.map((cell) => anonymizeCell(cell, aliases)) + : cellsToJudge; + + const connectorId = process.env.EVAL_CONNECTOR_ID; + if (!connectorId) { + throw createFlagError( + 'EVAL_CONNECTOR_ID must name the judge connector, e.g. eis-anthropic-claude-4-5-haiku' + ); + } + + // The judge runs against a Kibana that HAS the connector configured. That is + // a live eval stack, not the golden archive the scores were read from, so it + // is addressed separately. + const judgeKbnUrl = process.env.JUDGE_KBN_URL ?? evaluationsKbnUrl; + if (!judgeKbnUrl) { + throw createFlagError('JUDGE_KBN_URL must point at a Kibana exposing the judge connector.'); + } + + // JUDGE_KBN_API_KEY is an API key; JUDGE_KBN_AUTH is user:pass for a stack + // whose inference routes only accept Basic. Locally booted stacks are the + // latter, so supporting only the former blocks the common dev case. + const judgeApiKey = process.env.JUDGE_KBN_API_KEY ?? evaluationsKbnApiKey; + const authHeader = buildJudgeAuthHeader({ + basicAuth: process.env.JUDGE_KBN_AUTH, + apiKey: judgeApiKey, + }); + + // Prefer the suite's own rubric over the jury's fallback copy, so a rubric + // change in the suite reaches this replay. + const rubricItems = datasetPath ? await loadSuiteRubricItems(datasetPath, log) : undefined; + + const judge: CellJudge = createInferenceJudge({ + kbnUrl: judgeKbnUrl, + authHeader, + connectorId, + jury, + log, + rubricItems, + }); + + const { results, failures } = await runRejudge({ + cells, + judge, + judgeTag: blind ? `${judgeTag}-blind` : judgeTag, + concurrency, + }); + + // A replay that produced none of the jury's evaluators has measured + // something other than the column it claims to refresh. Failing here stops + // an artifact that would look publishable but silently swap the instrument. + const coverage = checkJuryCoverage( + jury, + results.flatMap((r) => r.scores) + ); + // Assert on the evaluator names actually present, not on the exit status: + // a jury that resolves without throwing produces zero failures and exit 0 + // while grading nothing. `planned > 0 && judged === 0` is that failure, and + // it must not be excused just because there are no scores to inspect. + if (plan.cells.length > 0 && results.length === 0) { + // Surface why. Without the reasons this message sends you looking at the + // jury while the actual cause is upstream (bad connector id, unreachable + // judge Kibana, auth). Distinct reasons only: 127 copies of one error is + // noise, and the count already says how widespread it is. + const reasons = [...new Set(failures.map((f) => f.reason))].slice(0, 3); + throw createFailError( + `Rejudge planned ${plan.cells.length} cell(s) but graded none, so no ` + + `"${jury.name}" evaluator (${jury.evaluatorNames.join(', ')}) was produced. ` + + `Refusing to write an artifact that would read as a refreshed column. ` + + (reasons.length + ? `${failures.length} cell(s) failed; distinct reason(s): ${reasons.join(' | ')}` + : `The jury resolved without grading rather than erroring.`) + ); + } + if (!coverage.ok) { + throw createFailError( + `Rejudge produced no "${jury.name}" evaluators ` + + `(expected any of ${jury.evaluatorNames.join(', ')}; got ${ + coverage.unexpected.join(', ') || 'nothing' + }). Refusing to write an artifact that does not refresh this suite's column.` + ); + } + if (coverage.missing.length > 0) { + log.warning( + `Jury "${jury.name}" did not produce: ${coverage.missing.join(', ')} ` + + `(examples lacking that annotation are skipped by the evaluator).` + ); + } + + Fs.mkdirSync(outDir, { recursive: true }); + const outFile = Path.join(outDir, `rejudge-${judgeTag}${blind ? '-blind' : ''}.json`); + Fs.writeFileSync( + outFile, + JSON.stringify( + { + judgeTag, + blind, + generatedAt: new Date().toISOString(), + planned: plan.cells.length, + judged: results.length, + failures, + skipped: plan.skipped, + results, + }, + null, + 2 + ) + ); + + log.success(`Wrote ${results.length} re-judged cell(s) to ${outFile}`); + if (failures.length > 0) { + log.warning(`${failures.length} cell(s) failed to judge; see "failures" in the artifact.`); + } + }, +}; + +/** + * Build a judge backed by the real inference REST API. + * + * `createRestClient` needs only an `HttpHandler`, so the judges run from a + * plain CLI against a Kibana that has the connector — no Playwright worker, + * no Scout stack, no seeded data. + */ +function createInferenceJudge({ + kbnUrl, + authHeader, + connectorId, + jury, + log, + rubricItems, +}: { + kbnUrl: string; + authHeader?: string; + connectorId: string; + jury: JuryAdapter; + log: ToolingLog; + /** Suite-owned rubric items, when the CLI could resolve them. */ + rubricItems?: string[]; +}): CellJudge { + const fetchImpl = (async (path: string, options: any = {}) => { + const response = await fetch(`${kbnUrl.replace(/\/$/, '')}${path}`, { + method: options.method ?? 'POST', + headers: { + 'Content-Type': 'application/json', + 'kbn-xsrf': 'evals-rejudge', + // /internal/inference/* rejects callers that do not declare an internal + // origin, and the rejection is a 400 reading "exists but is not + // available with the current configuration" -- which describes a + // disabled route, not a missing header. Without this the whole rejudge + // fails in a way that points at stack config instead of the request. + 'x-elastic-internal-origin': 'kibana', + // Stacks differ: a serverless/cloud judge takes an API key, a locally + // booted one authenticates the inference routes with Basic only. The + // scheme is therefore explicit -- assuming ApiKey produced a 401 that + // reads as a bad credential rather than a wrong scheme. + ...(authHeader ? { Authorization: authHeader } : {}), + ...(options.headers ?? {}), + }, + ...(options.body ? { body: options.body } : {}), + ...(options.signal ? { signal: options.signal } : {}), + }); + + if (!response.ok) { + throw new Error(`inference ${path} responded ${response.status}: ${await response.text()}`); + } + return response.json(); + }) as unknown as HttpHandler; + + const inferenceClient = createRestClient({ fetch: fetchImpl, bindTo: { connectorId } }); + + // Each jury builds the evaluators its suite's column is actually made of. + // The persona pair is no longer assumed: running it against a suite that + // grades something else produces confident verdicts about the wrong artefact. + const evaluate = buildJuryEvaluator({ jury, inferenceClient, log, rubricItems }); + + return async (cell) => { + const args = jury.toArgs(cell); + if (!args) { + // planReplay filters these out, so reaching here means the plan and the + // jury disagree -- fail loudly rather than emit an empty score set that + // would average into the column as if the model had performed badly. + throw new Error( + `Cell ${cell.executionId}/${cell.exampleId} is not gradable by the "${jury.name}" jury` + ); + } + return evaluate(args); + }; +} + +/** + * Build the scoring function for a jury. + * + * persona-matrix keeps its two-stage shape: the correctness and groundedness + * analyses run once, then the quantitative evaluators read those analyses off + * the output rather than paying for a second judge call. + * + * attack-discovery runs its own Criteria and Rubric evaluators, which need a + * `DefaultEvaluators.criteria` factory rather than a pre-built evaluator, since + * the criteria list differs per example. + */ +function buildJuryEvaluator({ + jury, + inferenceClient, + log, + rubricItems, +}: { + jury: JuryAdapter; + inferenceClient: BoundInferenceClient; + log: ToolingLog; + /** Suite-owned rubric items, when the CLI could resolve them. */ + rubricItems?: string[]; +}): (args: JuryArgs) => Promise<{ scores: RejudgeScore[]; analyses?: Record }> { + if (jury.name === 'attack-discovery') { + // AD's Criteria and Rubric evaluators live in a private, solutions-side + // functional-tests package that a platform package must not import. Both + // are thin wrappers over the shared criteria judge, so the jury rebuilds + // them from the same primitive rather than inverting the dependency. + // The rubric ITEMS, however, are loaded from the suite at runtime when + // available (see `loadSuiteRubricItems`), so a rubric change in the suite + // reaches a replay instead of being shadowed by a stale mirrored copy. + return async (args) => { + const scores: RejudgeScore[] = []; + // Hand the suite's own rubric to the jury when we have it, so the + // adapter's fallback copy cannot silently grade a different bar. + const juryArgs = rubricItems?.length + ? ({ ...args, metadata: { ...args.metadata, rubricCriteria: rubricItems } } as JuryArgs) + : args; + for (const spec of jury.criteriaFor?.(juryArgs) ?? []) { + const evaluator = createCriteriaEvaluator({ + inferenceClient, + criteria: spec.criteria as EvaluationCriterion[], + log, + }); + const result = await evaluator.evaluate(spec.args as never); + scores.push({ + name: spec.name, + score: result?.score ?? null, + label: result?.label ?? undefined, + explanation: result?.explanation ?? undefined, + }); + } + return { scores }; + }; + } + + const correctness = createCorrectnessAnalysisEvaluator({ inferenceClient, log }); + const groundedness = createGroundednessAnalysisEvaluator({ inferenceClient, log }); + const quantitative = [ + ...createQuantitativeCorrectnessEvaluators(), + createQuantitativeGroundednessEvaluator(), + ]; + + return async (args) => { + const [correctnessResult, groundednessResult] = await Promise.all([ + correctness.evaluate(args as never), + groundedness.evaluate(args as never), + ]); + + // The quantitative evaluators are pure functions of the two analyses, so + // they read them off the output rather than calling the judge again. + const enriched = { + ...args, + output: { + ...args.output, + correctnessAnalysis: correctnessResult?.metadata, + groundednessAnalysis: groundednessResult?.metadata, + }, + }; + + const scores: RejudgeScore[] = []; + for (const evaluator of quantitative) { + const result = await evaluator.evaluate(enriched as never); + scores.push({ + name: evaluator.name ?? 'unknown', + score: result?.score ?? null, + label: result?.label ?? undefined, + explanation: result?.explanation ?? undefined, + }); + } + + return { + scores, + analyses: { + correctness: correctnessResult?.metadata, + groundedness: groundednessResult?.metadata, + }, + }; + }; +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/index.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/index.ts index 2f45e1481ac68..1bcae4b8c4eb0 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/index.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/index.ts @@ -6,7 +6,9 @@ */ import { RunWithCommands } from '@kbn/dev-cli-runner'; +import { matrixCmd } from './commands/matrix'; import { redTeamCmd } from './commands/red_team'; +import { rejudgeCmd } from './commands/rejudge'; export async function run() { await new RunWithCommands( @@ -14,6 +16,6 @@ export async function run() { usage: 'node scripts/evals ext', description: 'Evals extensions CLI (experimental)', }, - [redTeamCmd] + [redTeamCmd, matrixCmd, rejudgeCmd] ).execute(); } diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/anonymize_cell.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/anonymize_cell.test.ts new file mode 100644 index 0000000000000..e5d243b0e1c48 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/anonymize_cell.test.ts @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { anonymizeCell, buildAliasMap } from './anonymize_cell'; +import type { ReplayCell } from './replay_plan'; + +const cell = (overrides: Partial = {}): ReplayCell => ({ + executionId: 'exec-1', + exampleId: 'alert-analysis-a', + modelId: 'anthropic-claude-4.6-sonnet', + question: 'Take a look at this alert.', + expected: 'The alert is a true positive.', + agentResponse: 'This is a true positive.', + steps: [], + recordedAt: '2026-08-22T16:24:55.232Z', + ...overrides, +}); + +describe('anonymizeCell steps', () => { + it('scrubs model identity from the tool-call history', () => { + // steps reach the groundedness judge as `tool_call_history`. Leaving them + // unscrubbed defeats --blind: the judge can read the model's identity out + // of the trajectory even though every other field was redacted. + const c = anonymizeCell( + cell({ + modelId: 'anthropic-claude-4.6-sonnet', + steps: [{ type: 'tool_call', note: 'run by anthropic-claude-4.6-sonnet' }], + }), + new Map([['anthropic-claude-4.6-sonnet', 'A']]) + ); + expect(JSON.stringify(c.steps)).not.toContain('claude-4.6-sonnet'); + }); + + it('leaves a trajectory without steps alone', () => { + expect(anonymizeCell(cell(), new Map()).steps).toEqual([]); + }); +}); + +describe('buildAliasMap', () => { + it('assigns a stable alias per model, ordered by model id', () => { + // Aliases must not depend on cell order: two runs of the same replay have + // to produce the same alias for the same model or the blind pass is not + // reproducible and deltas cannot be attributed. + const a = buildAliasMap([cell({ modelId: 'openai-gpt-5.2' }), cell({ modelId: 'a-model' })]); + const b = buildAliasMap([cell({ modelId: 'a-model' }), cell({ modelId: 'openai-gpt-5.2' })]); + + expect(a).toEqual(b); + expect(a.get('a-model')).toBe('Model A'); + expect(a.get('openai-gpt-5.2')).toBe('Model B'); + }); +}); + +describe('anonymizeCell', () => { + const aliases = buildAliasMap([cell()]); + + it('strips the model id from the cell', () => { + const out = anonymizeCell(cell(), aliases); + expect(out.modelId).toBe('Model A'); + }); + + it('scrubs vendor and model names leaking inside the agent response', () => { + // The judge reads the agent's own words. Models routinely self-identify + // ("As Claude, ..."), so stripping only the metadata field leaves the + // identity in the graded text and the blind pass is blind in name only. + const out = anonymizeCell( + cell({ + agentResponse: + 'As Claude, an Anthropic model, I reviewed this. GPT-5.2 would agree. -- claude-4.6-sonnet', + }), + aliases + ); + + expect(out.agentResponse).not.toMatch(/claude/i); + expect(out.agentResponse).not.toMatch(/anthropic/i); + expect(out.agentResponse).not.toMatch(/gpt-5\.2/i); + expect(out.agentResponse).not.toMatch(/sonnet/i); + }); + + it('scrubs identity from the question and expected answer too', () => { + const out = anonymizeCell( + cell({ + question: 'Does Gemini handle this better?', + expected: 'A Claude-style answer naming Anthropic.', + }), + aliases + ); + + expect(out.question).not.toMatch(/gemini/i); + expect(out.expected).not.toMatch(/claude|anthropic/i); + }); + + it('leaves the substantive content intact', () => { + // Over-scrubbing would destroy the thing being graded. The security + // vocabulary that drives the verdict must survive anonymization. + const out = anonymizeCell( + cell({ + agentResponse: + 'BluetoothService.exe side-loaded log.dll (MITRE T1574). Host srv-win-def is affected.', + }), + aliases + ); + + expect(out.agentResponse).toContain('BluetoothService.exe'); + expect(out.agentResponse).toContain('T1574'); + expect(out.agentResponse).toContain('srv-win-def'); + }); + + it('does not mutate the input cell', () => { + const original = cell({ agentResponse: 'Claude here.' }); + const before = { ...original }; + anonymizeCell(original, aliases); + expect(original).toEqual(before); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/anonymize_cell.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/anonymize_cell.ts new file mode 100644 index 0000000000000..d6716dc542ea8 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/anonymize_cell.ts @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ReplayCell } from './replay_plan'; + +/** + * Blind judging: hide which model produced a trajectory before a judge grades it. + * + * An LLM judge that can see it is grading its own family has a measurable + * preference for it. The judge reads the agent's own words, and models + * routinely self-identify mid-answer ("As Claude, ..."), so hiding only the + * metadata field leaves the identity in the graded text. + */ + +/** Vendor and family tokens that identify a model regardless of the id format. */ +const IDENTITY_PATTERNS: RegExp[] = [ + /\banthropic\b/gi, + /\bopenai\b/gi, + /\bgoogle\b/gi, + /\bdeepmind\b/gi, + /\bmeta\b/gi, + /\bmistral\b/gi, + /\bz-?ai\b/gi, + /\bclaude\b/gi, + /\bsonnet\b/gi, + /\bhaiku\b/gi, + /\bopus\b/gi, + /\bgemini\b/gi, + /\bgpt(?:-[\w.]+)?\b/gi, + /\bo[1-9](?:-[\w.]+)?\b/gi, + /\bllama\b/gi, + /\bqwen[\w.-]*\b/gi, + /\bglm[\w.-]*\b/gi, + /\bgrok\b/gi, +]; + +const REDACTED = '[model]'; + +/** + * Map each model to a stable alias, ordered by model id. + * + * Sorting rather than using encounter order keeps the alias for a given model + * identical across replays; otherwise the same model draws a different alias + * depending on which cell happened to load first, and a blind pass cannot be + * reproduced or compared. + */ +export function buildAliasMap(cells: ReplayCell[]): Map { + const ids = [...new Set(cells.map((c) => c.modelId))].sort(); + const aliases = new Map(); + ids.forEach((id, index) => { + aliases.set(id, `Model ${alias(index)}`); + }); + return aliases; +} + +/** A, B, ... Z, AA, AB, ... so the scheme survives more models than the alphabet. */ +function alias(index: number): string { + let n = index; + let out = ''; + do { + out = String.fromCharCode(65 + (n % 26)) + out; + n = Math.floor(n / 26) - 1; + } while (n >= 0); + return out; +} + +function scrub(text: string, modelId: string): string { + // The exact id first: it is the longest, most specific token, and removing it + // before the generic patterns avoids leaving fragments like "-4.6-" behind. + let out = text.split(modelId).join(REDACTED); + for (const pattern of IDENTITY_PATTERNS) { + out = out.replace(pattern, REDACTED); + } + // Collapse runs produced by adjacent matches ("claude-4.6-sonnet"). + return out.replace(/(?:\[model\][\s.-]*){2,}/g, `${REDACTED} `).trim(); +} + +/** + * Strip model identity from every judge-visible field of a cell. + * + * Returns a new cell; the caller's plan is reused for the non-blind pass and + * must not be mutated. + */ +export function anonymizeCell(cell: ReplayCell, aliases: Map): ReplayCell { + return { + ...cell, + modelId: aliases.get(cell.modelId) ?? REDACTED, + question: scrub(cell.question, cell.modelId), + expected: scrub(cell.expected, cell.modelId), + agentResponse: scrub(cell.agentResponse, cell.modelId), + // The tool-call history reaches the groundedness judge as + // `tool_call_history`, so it is judge-visible and must be scrubbed too. + // Steps are arbitrary nested JSON, so scrub the serialized form rather + // than walking a shape that varies per tool. + steps: cell.steps.length + ? JSON.parse(scrub(JSON.stringify(cell.steps), cell.modelId)) + : cell.steps, + }; +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.test.ts new file mode 100644 index 0000000000000..5b210dd2bf749 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.test.ts @@ -0,0 +1,1054 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { buildMatrix, rowCommitShas } from './build_matrix'; +import { parseMatrixConfig, type MatrixConfig } from './load_matrix_config'; +import type { AggregatedModelScores } from './query_matrix_scores'; + +const config: MatrixConfig = parseMatrixConfig({ + columns: [ + { id: 'triage', label: 'Triage', suites: ['suite-a'], weight: 1 }, + { id: 'detect', label: 'Detect', suites: ['suite-b'], weight: 2 }, + ], + models: [ + { id: 'model-good', label: 'Good Model' }, + { id: 'model-oss', label: 'OSS Model', openSource: true }, + { id: 'model-missing', label: 'Absent Model' }, + ], +}); + +const evaluator = (mean: number, count = 10) => ({ evaluatorName: 'correctness', mean, count }); + +const aggregated: AggregatedModelScores[] = [ + { + modelId: 'model-good', + provider: 'anthropic', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'run-1', + datasets: [{ datasetId: 'd1', datasetName: 'D1', evaluators: [evaluator(0.9)] }], + }, + { + suiteId: 'suite-b', + experimentId: 'run-2', + datasets: [{ datasetId: 'd2', datasetName: 'D2', evaluators: [evaluator(0.8)] }], + }, + ], + }, + { + modelId: 'model-oss', + provider: 'meta', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'run-3', + datasets: [{ datasetId: 'd1', datasetName: 'D1', evaluators: [evaluator(0.5)] }], + }, + // No suite-b data -> "detect" column missing for this model. + ], + }, +]; + +describe('buildMatrix tie tiers', () => { + // Re-running one model on an unchanged commit moves its overall by ~0.2 + // (stdev over 7 haiku runs on golden). Publishing 8.54 above 8.42 as a + // ranking therefore asserts a difference the data cannot support. + const tierConfig: MatrixConfig = parseMatrixConfig({ + minCoverage: 1, + overall: { runStdev: 0.198 }, + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'], weight: 1 }], + models: [ + { id: 'model-a', label: 'A' }, + { id: 'model-b', label: 'B' }, + { id: 'model-c', label: 'C' }, + ], + }); + + const one = (id: string, mean: number) => ({ + suiteId: 'suite-a', + experimentId: id, + datasets: [{ datasetId: id, datasetName: id, evaluators: [evaluator(mean)] }], + }); + + it('ties rows inside the noise band and splits only on a real gap', () => { + const matrix = buildMatrix( + [ + { modelId: 'model-a', provider: 'p', suites: [one('r1', 0.85)] }, + { modelId: 'model-b', provider: 'p', suites: [one('r2', 0.84)] }, + { modelId: 'model-c', provider: 'p', suites: [one('r3', 0.4)] }, + ], + tierConfig + ); + + const tiers = Object.fromEntries(matrix.proprietary.map((r) => [r.modelId, r.tier])); + // 8.5 vs 8.4 is inside the interval -> same tier, not a ranking. + expect(tiers['model-a']).toBe(tiers['model-b']); + // 8.5 vs 4.0 clears it comfortably -> a real difference. + expect(tiers['model-c']).toBeGreaterThan(tiers['model-a']!); + }); +}); + +describe('buildMatrix coverage floor', () => { + // Reproduces the 2026-08-29 matrix: GLM-5.2 answered 2 of 24 prompts, scored + // 10.0 on both, and outranked every frontier model. A thin run must never + // publish an Overall number. + const floorConfig: MatrixConfig = parseMatrixConfig({ + minCoverage: 2, + columns: [ + { id: 'triage', label: 'Triage', suites: ['suite-a'], weight: 1 }, + { id: 'detect', label: 'Detect', suites: ['suite-b'], weight: 1 }, + { id: 'hunt', label: 'Hunt', suites: ['suite-c'], weight: 1 }, + ], + models: [ + { id: 'model-thin', label: 'Thin Model' }, + { id: 'model-broad', label: 'Broad Model' }, + ], + }); + + const suite = (suiteId: string, id: string, mean: number) => ({ + suiteId, + experimentId: `run-${id}`, + datasets: [{ datasetId: id, datasetName: id, evaluators: [evaluator(mean)] }], + }); + + it('withholds Overall and ranks last when scored on too few columns', () => { + const scores: AggregatedModelScores[] = [ + // One perfect cell — a 1.0 mean that would average to a rank-topping 10. + { modelId: 'model-thin', provider: 'zai', suites: [suite('suite-a', 'd1', 1.0)] }, + // Two solid-but-lower cells from a model that actually ran the suite. + { + modelId: 'model-broad', + provider: 'anthropic', + suites: [suite('suite-a', 'd1', 0.8), suite('suite-b', 'd2', 0.8)], + }, + ]; + + const matrix = buildMatrix(scores, floorConfig); + const thin = matrix.proprietary.find((r) => r.modelId === 'model-thin')!; + const broad = matrix.proprietary.find((r) => r.modelId === 'model-broad')!; + + // The thin row must not publish a number... + expect(thin.overall.kind).toBe('insufficient-coverage'); + expect(thin.coverage.covered).toBe(1); + // ...and must rank BELOW the model with real coverage, despite scoring 10. + expect(broad.overall.kind).toBe('score'); + expect(matrix.proprietary.indexOf(broad)).toBeLessThan(matrix.proprietary.indexOf(thin)); + }); + + it('publishes Overall once the floor is met', () => { + const scores: AggregatedModelScores[] = [ + { + modelId: 'model-thin', + provider: 'zai', + suites: [suite('suite-a', 'd1', 1.0), suite('suite-b', 'd2', 1.0)], + }, + ]; + const matrix = buildMatrix(scores, floorConfig); + const row = matrix.proprietary.find((r) => r.modelId === 'model-thin')!; + expect(row.overall.kind).toBe('score'); + expect(row.coverage.covered).toBe(2); + }); +}); + +describe('buildMatrix', () => { + it('scales evaluator means onto a 0-10 scale and splits proprietary/open-source', () => { + const matrix = buildMatrix(aggregated, config); + + expect(matrix.proprietary).toHaveLength(1); + expect(matrix.openSource).toHaveLength(1); + + const good = matrix.proprietary[0]; + expect(good.modelLabel).toBe('Good Model'); + expect(good.cells.triage).toEqual({ kind: 'score', value: 9 }); + expect(good.cells.detect).toEqual({ kind: 'score', value: 8 }); + // Weighted overall: (9*1 + 8*2) / 3 = 8.33 + expect(good.overall).toEqual({ kind: 'score', value: 8.33 }); + }); + + it('marks columns with no data as missing and counts them as 0 in the overall', () => { + const matrix = buildMatrix(aggregated, config); + const oss = matrix.openSource[0]; + + expect(oss.cells.triage).toEqual({ kind: 'score', value: 5 }); + expect(oss.cells.detect).toEqual({ kind: 'missing' }); + // detect is missing (excluded entirely), so overall = triage only = 5. + expect(oss.overall).toEqual({ kind: 'score', value: 5 }); + }); + + it('skips models absent from the aggregated data', () => { + const matrix = buildMatrix(aggregated, config); + const labels = [...matrix.proprietary, ...matrix.openSource].map((row) => row.modelLabel); + expect(labels).not.toContain('Absent Model'); + }); + + it('renders "not recommended" when a scaled score is at/under the threshold', () => { + const zeroConfig = parseMatrixConfig({ + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'] }], + models: [{ id: 'm', label: 'M' }], + }); + const matrix = buildMatrix( + [ + { + modelId: 'm', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'r', + datasets: [{ datasetId: 'd', datasetName: 'D', evaluators: [evaluator(0)] }], + }, + ], + }, + ], + zeroConfig + ); + + expect(matrix.proprietary[0].cells.triage).toEqual({ kind: 'not-recommended' }); + }); + + it('excludes observability-tier evaluators (latency/tokens/tool calls) by default', () => { + const matrix = buildMatrix( + [ + { + modelId: 'm', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'r', + datasets: [ + { + datasetId: 'd', + datasetName: 'D', + evaluators: [ + { evaluatorName: 'Factuality', mean: 0.8, count: 10 }, + { evaluatorName: 'Latency', mean: 4200, count: 10 }, + { evaluatorName: 'Input Tokens', mean: 51234, count: 10 }, + { evaluatorName: 'Tool Calls', mean: 7, count: 10 }, + { evaluatorName: 'Skill Invoked (alert-analysis)', mean: 1, count: 10 }, + ], + }, + ], + }, + ], + }, + ], + parseMatrixConfig({ + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'] }], + models: [{ id: 'm', label: 'M' }], + }) + ); + + // Only Factuality (0.8) contributes -> 0.8 * 10 = 8, not blown out by tokens/latency. + expect(matrix.proprietary[0].cells.triage).toEqual({ kind: 'score', value: 8 }); + }); + + it('honors a column evaluator allowlist over the global exclusion list', () => { + const matrix = buildMatrix( + [ + { + modelId: 'm', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'r', + datasets: [ + { + datasetId: 'd', + datasetName: 'D', + evaluators: [ + { evaluatorName: 'Factuality', mean: 0.8, count: 10 }, + { evaluatorName: 'Latency', mean: 0.2, count: 10 }, + ], + }, + ], + }, + ], + }, + ], + parseMatrixConfig({ + // Explicit allowlist including 'Latency' opts it back in despite the default exclusion. + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'], evaluators: ['Latency'] }], + models: [{ id: 'm', label: 'M' }], + }) + ); + + expect(matrix.proprietary[0].cells.triage).toEqual({ kind: 'score', value: 2 }); + }); + + describe('composites', () => { + const compositeConfig: MatrixConfig = parseMatrixConfig({ + showOverall: false, + columns: [ + { id: 'c1', label: 'C1', group: 'Group', suites: ['s1'] }, + { id: 'c2', label: 'C2', group: 'Group', suites: ['s2'] }, + { id: 'feat', label: 'Feat', suites: ['s3'] }, + ], + composites: [ + { id: 'group_score', label: 'Group Score', from: ['c1', 'c2'] }, + { id: 'overall_score', label: 'Overall Score', from: ['group_score', 'feat'] }, + ], + layout: ['c1', 'c2', 'group_score', 'feat', 'overall_score'], + models: [ + { id: 'm1', label: 'M1' }, + { id: 'm2', label: 'M2' }, + ], + }); + + const suite = (suiteId: string, mean: number) => ({ + suiteId, + experimentId: `e-${suiteId}`, + datasets: [{ datasetId: 'd', datasetName: 'D', evaluators: [evaluator(mean)] }], + }); + + it('averages base cells into a composite and layers composites of composites', () => { + const matrix = buildMatrix( + [{ modelId: 'm1', suites: [suite('s1', 0.8), suite('s2', 0.6)] }], + compositeConfig + ); + const row = matrix.proprietary[0]; + + expect(row.cells.group_score).toEqual({ kind: 'score', value: 7 }); + // feat has no data -> missing, so overall = group_score only = 7. + expect(row.cells.feat).toEqual({ kind: 'missing' }); + expect(row.cells.overall_score).toEqual({ kind: 'score', value: 7 }); + }); + + it('counts "Not recommended" sources as 0 inside a composite', () => { + const matrix = buildMatrix( + [{ modelId: 'm1', suites: [suite('s1', 0), suite('s2', 0.6)] }], + compositeConfig + ); + const row = matrix.proprietary[0]; + + expect(row.cells.c1).toEqual({ kind: 'not-recommended' }); + // mean(0, 6) = 3. + expect(row.cells.group_score).toEqual({ kind: 'score', value: 3 }); + }); + + it('marks a composite missing when none of its sources have data', () => { + const matrix = buildMatrix([{ modelId: 'm1', suites: [suite('s3', 0.9)] }], compositeConfig); + const row = matrix.proprietary[0]; + + expect(row.cells.group_score).toEqual({ kind: 'missing' }); + // overall = feat only = 9. + expect(row.cells.overall_score).toEqual({ kind: 'score', value: 9 }); + }); + + it('builds display columns in layout order and suppresses the legacy overall', () => { + const matrix = buildMatrix( + [{ modelId: 'm1', suites: [suite('s1', 0.8), suite('s2', 0.6)] }], + compositeConfig + ); + + expect(matrix.displayColumns?.map((column) => column.id)).toEqual([ + 'c1', + 'c2', + 'group_score', + 'feat', + 'overall_score', + ]); + expect(matrix.displayColumns?.find((column) => column.id === 'group_score')?.kind).toBe( + 'composite' + ); + expect(matrix.displayColumns?.some((column) => column.kind === 'overall')).toBe(false); + expect(matrix.displayColumns?.find((column) => column.id === 'c1')?.group).toBe('Group'); + }); + + it('ranks rows by the final composite (Overall Score) descending', () => { + const matrix = buildMatrix( + [ + { modelId: 'm1', suites: [suite('s1', 0.3), suite('s2', 0.3)] }, + { modelId: 'm2', suites: [suite('s1', 0.9), suite('s2', 0.9)] }, + ], + compositeConfig + ); + + expect(matrix.proprietary.map((row) => row.modelLabel)).toEqual(['M2', 'M1']); + }); + + it('throws when the layout references an unknown id', () => { + const badConfig = parseMatrixConfig({ + columns: [{ id: 'c1', label: 'C1', suites: ['s1'] }], + layout: ['c1', 'nope'], + models: [{ id: 'm1', label: 'M1' }], + }); + expect(() => buildMatrix([{ modelId: 'm1', suites: [suite('s1', 0.5)] }], badConfig)).toThrow( + /unknown column\/composite id/ + ); + }); + }); + + it('sorts rows by overall score descending', () => { + const matrix = buildMatrix( + [ + { + modelId: 'model-good', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'r1', + datasets: [{ datasetId: 'd', datasetName: 'D', evaluators: [evaluator(0.3)] }], + }, + ], + }, + { + modelId: 'model-missing', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'r2', + datasets: [{ datasetId: 'd', datasetName: 'D', evaluators: [evaluator(0.9)] }], + }, + ], + }, + ], + parseMatrixConfig({ + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'] }], + models: [ + { id: 'model-good', label: 'Lower' }, + { id: 'model-missing', label: 'Higher' }, + ], + }) + ); + + expect(matrix.proprietary.map((row) => row.modelLabel)).toEqual(['Higher', 'Lower']); + }); +}); + +describe('buildMatrix token axis', () => { + const tokenEvaluator = (name: string, mean: number, min: number, max: number, count = 3) => ({ + evaluatorName: name, + mean, + count, + min, + max, + }); + + const tokenAggregated: AggregatedModelScores[] = [ + { + modelId: 'model-good', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'run-1', + datasets: [ + { + datasetId: 'd1', + datasetName: 'D1', + evaluators: [ + evaluator(0.9), + tokenEvaluator('Input Tokens', 100_000, 50_000, 150_000), + tokenEvaluator('Output Tokens', 2_000, 1_000, 3_000), + ], + }, + ], + }, + ], + }, + ]; + + const tokenConfig: MatrixConfig = parseMatrixConfig({ + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'], weight: 1 }], + models: [{ id: 'model-good', label: 'Good Model' }], + tokenCost: {}, + }); + + it('is omitted entirely when the config does not opt in', () => { + expect(buildMatrix(tokenAggregated, config).tokenCost).toBeUndefined(); + }); + + it('aggregates token evaluators in native units with min/max preserved', () => { + const matrix = buildMatrix(tokenAggregated, tokenConfig); + const cell = matrix.tokenCost!.models[0].cells[0]; + + expect(cell.columnId).toBe('triage'); + expect(cell.inputTokens).toEqual({ mean: 100_000, min: 50_000, max: 150_000, count: 3 }); + expect(cell.outputTokens).toEqual({ mean: 2_000, min: 1_000, max: 3_000, count: 3 }); + expect(cell.totalMean).toBe(102_000); + }); + + it('does not let token evaluators leak into quality cells', () => { + const matrix = buildMatrix(tokenAggregated, tokenConfig); + // 0.9 * defaultScale(10) — unaffected by the 100k-magnitude token evaluators. + expect(matrix.proprietary[0].cells.triage).toEqual({ kind: 'score', value: 9 }); + }); + + it('weights the mean by sample count across suites', () => { + const twoSuite: MatrixConfig = parseMatrixConfig({ + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a', 'suite-b'], weight: 1 }], + models: [{ id: 'model-good', label: 'Good Model' }], + tokenCost: {}, + }); + const matrix = buildMatrix( + [ + { + modelId: 'model-good', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'r1', + datasets: [ + { + datasetId: 'd1', + datasetName: 'D1', + evaluators: [tokenEvaluator('Input Tokens', 100, 100, 100, 1)], + }, + ], + }, + { + suiteId: 'suite-b', + experimentId: 'r2', + datasets: [ + { + datasetId: 'd2', + datasetName: 'D2', + evaluators: [tokenEvaluator('Input Tokens', 200, 200, 200, 3)], + }, + ], + }, + ], + }, + ], + twoSuite + ); + // (100*1 + 200*3) / 4 = 175, not the unweighted 150. + expect(matrix.tokenCost!.models[0].cells[0].inputTokens!.mean).toBe(175); + expect(matrix.tokenCost!.models[0].cells[0].inputTokens!.min).toBe(100); + expect(matrix.tokenCost!.models[0].cells[0].inputTokens!.max).toBe(200); + }); + + it('omits cells with no token data', () => { + const matrix = buildMatrix(aggregated, tokenConfig); + expect(matrix.tokenCost!.models).toEqual([]); + }); +}); + +describe('buildMatrix saturated-evaluator exclusion', () => { + // Two evaluators per model: `discriminating` separates the models, `ceiling` + // returns effectively the same high score for everyone. Folding `ceiling` + // into Overall drags every model toward it and compresses the spread. + const buildScores = (): AggregatedModelScores[] => + [ + { id: 'model-a', discriminating: 0.9 }, + { id: 'model-b', discriminating: 0.8 }, + { id: 'model-c', discriminating: 0.7 }, + { id: 'model-d', discriminating: 0.6 }, + { id: 'model-e', discriminating: 0.5 }, + { id: 'model-f', discriminating: 0.45 }, + { id: 'model-g', discriminating: 0.4 }, + { id: 'model-h', discriminating: 0.3 }, + { id: 'model-i', discriminating: 0.2 }, + { id: 'model-j', discriminating: 0.1 }, + ].map(({ id, discriminating }) => ({ + modelId: id, + suites: [ + { + suiteId: 'suite-a', + datasets: [ + { + datasetName: 'suite-a', + evaluators: [ + { evaluatorName: 'discriminating', mean: discriminating, count: 10 }, + { evaluatorName: 'ceiling', mean: 0.97, count: 10 }, + ], + }, + ], + }, + ], + })) as unknown as AggregatedModelScores[]; + + const configWith = (excludeSaturatedEvaluators: boolean): MatrixConfig => + parseMatrixConfig({ + minCoverage: 1, + overall: { excludeSaturatedEvaluators }, + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'], weight: 1 }], + models: [ + { id: 'model-a', label: 'A' }, + { id: 'model-b', label: 'B' }, + { id: 'model-c', label: 'C' }, + { id: 'model-d', label: 'D' }, + { id: 'model-e', label: 'E' }, + { id: 'model-f', label: 'F' }, + { id: 'model-g', label: 'G' }, + { id: 'model-h', label: 'H' }, + { id: 'model-i', label: 'I' }, + { id: 'model-j', label: 'J' }, + ], + }); + + const overallOf = (matrix: ReturnType) => + matrix.proprietary.map((row) => + row.overall.kind === 'score' ? Number(row.overall.value.toFixed(3)) : undefined + ); + + it('widens the spread between models when the saturated evaluator is dropped', () => { + const scores = buildScores(); + const before = overallOf(buildMatrix(scores, configWith(false))); + const after = overallOf(buildMatrix(scores, configWith(true))); + + const spread = (values: Array) => + Math.max(...(values as number[])) - Math.min(...(values as number[])); + + // Averaging in a flat evaluator halves the real difference between models. + expect(spread(before)).toBeCloseTo(4, 3); + expect(spread(after)).toBeCloseTo(8, 3); + expect(spread(after)).toBeGreaterThan(spread(before)); + }); + + it('reports which evaluators were judged saturated', () => { + const matrix = buildMatrix(buildScores(), configWith(true)); + const saturated = matrix.evaluatorSaturation.filter((entry) => entry.saturated); + + expect(saturated.map((entry) => entry.evaluatorName)).toEqual(['ceiling']); + }); + + it('keeps the saturated evaluator in Overall when the config does not opt in', () => { + const matrix = buildMatrix(buildScores(), configWith(false)); + + // Detection is opt-in: nothing is reported and nothing is dropped. + expect(matrix.evaluatorSaturation).toEqual([]); + expect(overallOf(matrix)[0]).toBeCloseTo(9.35, 2); + expect(overallOf(matrix).at(-1)).toBeCloseTo(5.35, 2); + }); +}); + +describe('buildMatrix sparse-column warning', () => { + const sparseConfig: MatrixConfig = parseMatrixConfig({ + minCoverage: 1, + columns: [ + { id: 'dense', label: 'Dense', suites: ['suite-a'], weight: 1 }, + { id: 'sparse', label: 'Attack Discovery', suites: ['suite-b'], weight: 1 }, + ], + models: [ + { id: 'model-a', label: 'A' }, + { id: 'model-b', label: 'B' }, + { id: 'model-c', label: 'C' }, + { id: 'model-d', label: 'D' }, + ], + }); + + // Every model runs `suite-a`; only one ever ran `suite-b`. + const sparseScores = ['model-a', 'model-b', 'model-c', 'model-d'].map((modelId) => ({ + modelId, + suites: [ + { + suiteId: 'suite-a', + datasets: [ + { + datasetName: 'suite-a', + evaluators: [{ evaluatorName: 'correctness', mean: 0.8, count: 5 }], + }, + ], + }, + ...(modelId === 'model-a' + ? [ + { + suiteId: 'suite-b', + datasets: [ + { + datasetName: 'suite-b', + evaluators: [{ evaluatorName: 'correctness', mean: 0.9, count: 5 }], + }, + ], + }, + ] + : []), + ], + })) as unknown as AggregatedModelScores[]; + + it('warns that a column covering a minority of models cannot rank', () => { + const log = { warning: jest.fn() }; + buildMatrix(sparseScores, sparseConfig, log); + + expect(log.warning).toHaveBeenCalledWith( + expect.stringContaining('"Attack Discovery" has scores for only 1 of 4 models') + ); + }); + + it('stays quiet about a column every model ran', () => { + const log = { warning: jest.fn() }; + buildMatrix(sparseScores, sparseConfig, log); + + expect(log.warning).not.toHaveBeenCalledWith(expect.stringContaining('"Dense"')); + }); +}); + +describe('buildMatrix total-score-loss guard', () => { + // Regression guard for the failure mode that inflated the published board: + // the per-prefix score fetch returned nothing usable, cells fell back to a + // coarser source, and the matrix rendered normally while every Overall was + // wrong. Root cause was `evaluator.metadata` being stripped server-side + // (#286691), which left the verdict ladder with nothing to map. + const lossConfig: MatrixConfig = parseMatrixConfig({ + minCoverage: 1, + columns: [{ id: 'only', label: 'Only', suites: ['suite-a'], weight: 1 }], + models: [ + { id: 'model-a', label: 'A' }, + { id: 'model-b', label: 'B' }, + ], + }); + + it('warns loudly when not a single cell scored', () => { + // Models present, but no suite produced any usable evaluator score. + const empty = [ + { modelId: 'model-a', suites: [] }, + { modelId: 'model-b', suites: [] }, + ] as unknown as AggregatedModelScores[]; + + const log = { warning: jest.fn() }; + buildMatrix(empty, lossConfig, log); + + expect(log.warning).toHaveBeenCalledWith( + expect.stringContaining('No column produced a single scored cell') + ); + expect(log.warning).toHaveBeenCalledWith(expect.stringContaining('do NOT publish this run')); + }); + + it('stays quiet when cells actually scored', () => { + const scored = ['model-a', 'model-b'].map((modelId) => ({ + modelId, + suites: [ + { + suiteId: 'suite-a', + datasets: [ + { + datasetName: 'suite-a', + evaluators: [{ evaluatorName: 'correctness', mean: 0.8, count: 5 }], + }, + ], + }, + ], + })) as unknown as AggregatedModelScores[]; + + const log = { warning: jest.fn() }; + buildMatrix(scored, lossConfig, log); + + expect(log.warning).not.toHaveBeenCalledWith( + expect.stringContaining('No column produced a single scored cell') + ); + }); +}); + +describe('per-row commit provenance', () => { + const scores = (suites: Array<{ sha?: string; ts?: string }>): AggregatedModelScores => ({ + modelId: 'model-good', + suites: suites.map((s, i) => ({ + suiteId: `suite-${i}`, + experimentId: `run-${i}`, + timestamp: s.ts, + commitSha: s.sha, + datasets: [{ datasetId: 'd', datasetName: 'D', evaluators: [evaluator(0.9)] }], + })), + }); + + it('reports the commits a row was graded against, newest run first', () => { + expect( + rowCommitShas( + scores([ + { sha: 'oldsha0000000', ts: '2026-08-01T00:00:00.000Z' }, + { sha: 'newsha1111111', ts: '2026-09-01T00:00:00.000Z' }, + ]) + ) + ).toEqual(['newsha1111111', 'oldsha0000000']); + }); + + it('collapses repeats so one codebase reads as one commit', () => { + expect(rowCommitShas(scores([{ sha: 'same111' }, { sha: 'same111' }]))).toEqual(['same111']); + }); + + it('stays undefined when the experiment summary carried no commit', () => { + expect(rowCommitShas(scores([{}]))).toBeUndefined(); + expect(rowCommitShas(undefined)).toBeUndefined(); + }); + + it('warns when rows were graded against different codebases', () => { + const log = { warning: jest.fn() }; + buildMatrix( + [ + { ...scores([{ sha: 'aaaaaaaaaaaa1' }]), modelId: 'model-good' }, + { ...scores([{ sha: 'bbbbbbbbbbbb2' }]), modelId: 'model-oss' }, + ], + config, + log + ); + + expect(log.warning).toHaveBeenCalledWith(expect.stringContaining('spans 2 commits')); + }); + + it('stays quiet when every row came from one codebase', () => { + const log = { warning: jest.fn() }; + buildMatrix( + [ + { ...scores([{ sha: 'aaaaaaaaaaaa1' }]), modelId: 'model-good' }, + { ...scores([{ sha: 'aaaaaaaaaaaa1' }]), modelId: 'model-oss' }, + ], + config, + log + ); + + expect(log.warning).not.toHaveBeenCalledWith(expect.stringContaining('spans')); + }); +}); + +describe('buildMatrix self-judged disclosure', () => { + // gemini-3.1-pro judges the attack-discovery suite AND is ranked in it. + // The column opts into `allowSelfJudged` because an audit found no + // self-preference, so the score is publishable -- but it must not look + // like an independently-judged one. A bare {kind:'score'} is + // indistinguishable from an arm's-length score, which is what the + // reporting rule forbids. + const discloseConfig: MatrixConfig = parseMatrixConfig({ + minCoverage: 1, + columns: [ + { + id: 'kill-chain', + label: 'Kill-Chain', + suites: ['suite-a'], + weight: 1, + allowSelfJudged: true, + }, + { id: 'triage', label: 'Triage', suites: ['suite-b'], weight: 1 }, + ], + models: [{ id: 'model-a', label: 'A' }], + }); + + it('marks a score from an allowSelfJudged column as self-judged', () => { + const matrix = buildMatrix( + [ + { + modelId: 'model-a', + provider: 'p', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'e1', + selfJudged: true, + datasets: [{ datasetId: 'd', datasetName: 'd', evaluators: [evaluator(0.762)] }], + }, + { + suiteId: 'suite-b', + experimentId: 'e2', + datasets: [{ datasetId: 'd', datasetName: 'd', evaluators: [evaluator(0.8)] }], + }, + ], + }, + ], + discloseConfig + ); + + const row = matrix.proprietary[0]; + const kc = row.cells['kill-chain']; + const triage = row.cells.triage; + + // The opted-in column carries the disclosure... + expect(kc).toMatchObject({ kind: 'score', selfJudged: true }); + // ...and a normally-judged column is NOT falsely flagged. + expect(triage).toMatchObject({ kind: 'score' }); + expect((triage as { selfJudged?: boolean }).selfJudged).toBeUndefined(); + }); + + // The opt-out is a COLUMN setting, but only some rows in that column are + // actually self-judged: gemini judges attack-discovery, so gpt-5.4's cell in + // that same column was graded at arm's length. Flagging the whole column + // libels five models to disclose one, and a reader who spots one bogus flag + // has no reason to trust the real one. + it('flags only the rows whose judge is the graded model', () => { + const suite = (judgedSelf: boolean) => ({ + suiteId: 'suite-a', + experimentId: 'e1', + selfJudged: judgedSelf, + datasets: [{ datasetId: 'd', datasetName: 'd', evaluators: [evaluator(0.762)] }], + }); + + const own = buildMatrix( + [{ modelId: 'model-a', provider: 'p', suites: [suite(true)] }], + discloseConfig + ); + const other = buildMatrix( + [{ modelId: 'model-a', provider: 'p', suites: [suite(false)] }], + discloseConfig + ); + + expect(own.proprietary[0].cells['kill-chain']).toMatchObject({ selfJudged: true }); + expect( + (other.proprietary[0].cells['kill-chain'] as { selfJudged?: boolean }).selfJudged + ).toBeUndefined(); + }); +}); + +describe('buildMatrix withheld-vs-never-ran', () => { + // gemini-3.1-pro self-judges the migrations suites with ~4,794 documents + // and shows a measurable self-preference gap there (2nd under its own + // judgement, 5th under the deterministic control), so those scores are + // correctly withheld. Rendering the withheld cell as {kind:'missing'} + // makes it identical to a model that never ran the suite at all, which + // reads as a coverage gap instead of a judge-policy decision. + const cfg: MatrixConfig = parseMatrixConfig({ + minCoverage: 1, + columns: [{ id: 'migrations', label: 'Migrations', suites: ['suite-a'], weight: 1 }], + models: [ + { id: 'model-a', label: 'A' }, + { id: 'model-b', label: 'B' }, + ], + }); + + it('distinguishes a withheld self-judged cell from one that never ran', () => { + const matrix = buildMatrix( + [ + { + modelId: 'model-a', + provider: 'p', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'e1', + // Every score was rejected as self-judged: no datasets survive, + // but the run exists and its size is known. + excludedSelfJudged: 4794, + datasets: [], + }, + ], + }, + // model-b never ran the suite at all. + { modelId: 'model-b', provider: 'p', suites: [] }, + ], + cfg + ); + + const withheld = matrix.proprietary.find((r) => r.modelId === 'model-a')!.cells.migrations; + const neverRan = matrix.proprietary.find((r) => r.modelId === 'model-b')!.cells.migrations; + + expect(withheld).toEqual({ kind: 'excluded', reason: 'self-judged', docs: 4794 }); + expect(neverRan).toEqual({ kind: 'missing' }); + }); + + it('does not claim exclusion when the suite simply produced no scores', () => { + // A run that exists but yielded nothing for other reasons must not be + // dressed up as a judge-policy exclusion. + const matrix = buildMatrix( + [ + { + modelId: 'model-a', + provider: 'p', + suites: [{ suiteId: 'suite-a', experimentId: 'e1', datasets: [] }], + }, + ], + cfg + ); + + expect(matrix.proprietary[0].cells.migrations).toEqual({ kind: 'missing' }); + }); +}); + +describe('errored-evaluator guard', () => { + // A judge/quality evaluator that ERRORS is absent from the aggregate rather + // than scored, narrowing the mean to the survivors. When the survivors are + // the saturated contract checks (all 1.0), the cell is INFLATED. A + // trace-cluster permission fault nulled Trajectory + SkillInvoked for + // DeepSeek and lifted alert-analysis-a to 8.89 against 6.86 for models + // graded on the full set -- a 2pt "win" that was pure instrument failure. + // + // A raw evaluator-count floor cannot catch this: a healthy frontier cell + // legitimately has 4 scored evaluators, same as the broken row. The signal + // that separates them is the errored-out evaluator NAME. + const guardConfig: MatrixConfig = parseMatrixConfig({ + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'], weight: 1 }], + models: [{ id: 'model-partial', label: 'Partial Model' }], + }); + + const withDatasets = ( + evaluators: Array<{ evaluatorName: string; mean: number; count: number }>, + erroredOutEvaluators?: string[] + ): AggregatedModelScores[] => [ + { + modelId: 'model-partial', + provider: 'openrouter', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'run-partial', + datasets: [ + { + datasetId: 'd1', + datasetName: 'D1', + evaluators, + ...(erroredOutEvaluators ? { erroredOutEvaluators } : {}), + }, + ], + }, + ], + }, + ]; + + // The exact shape of the DeepSeek bug: the discriminating judged evaluators + // errored away, leaving only saturated 1.0 contract checks behind. + const survivingSaturatedOnly = [ + { evaluatorName: 'MinExpectedSteps', mean: 1, count: 3 }, + { evaluatorName: 'FinalAnswerPresent', mean: 1, count: 3 }, + ]; + + it('refuses to publish a score when a cell-relevant evaluator errored out', () => { + const matrix = buildMatrix( + withDatasets(survivingSaturatedOnly, ['Trajectory', 'SkillInvoked']), + guardConfig + ); + const cell = matrix.proprietary[0].cells.triage; + + expect(cell.kind).toBe('insufficient-evaluators'); + expect(cell).toEqual({ + kind: 'insufficient-evaluators', + evaluators: ['Trajectory', 'SkillInvoked'], + }); + }); + + it('publishes the score when no cell-relevant evaluator errored out', () => { + const matrix = buildMatrix(withDatasets(survivingSaturatedOnly), guardConfig); + expect(matrix.proprietary[0].cells.triage.kind).toBe('score'); + }); + + // Without the guard the broken row scores HIGHER than a fully-measured one. + // This is the regression that let 8.89 outrank 6.86. + it('is what stops a partial instrument from outranking a full measurement', () => { + const fullSet = [ + { evaluatorName: 'Factuality', mean: 0.75, count: 3 }, + { evaluatorName: 'Groundedness', mean: 0.85, count: 3 }, + { evaluatorName: 'Relevance', mean: 0.66, count: 3 }, + { evaluatorName: 'MinExpectedSteps', mean: 1, count: 3 }, + { evaluatorName: 'FinalAnswerPresent', mean: 1, count: 3 }, + ]; + const partial = buildMatrix(withDatasets(survivingSaturatedOnly), guardConfig).proprietary[0] + .cells.triage; + const complete = buildMatrix(withDatasets(fullSet), guardConfig).proprietary[0].cells.triage; + + // Demonstrate the inflation is real before asserting the fix suppresses it. + expect(partial.kind).toBe('score'); + expect(complete.kind).toBe('score'); + if (partial.kind === 'score' && complete.kind === 'score') { + expect(partial.value).toBeGreaterThan(complete.value); + } + + // With the errored-out evaluators named, the inflated cell no longer publishes. + const guarded = buildMatrix(withDatasets(survivingSaturatedOnly, ['Trajectory']), guardConfig) + .proprietary[0].cells.triage; + expect(guarded.kind).toBe('insufficient-evaluators'); + }); + + // Latency racing span ingestion is the norm, not a fault: it must never flag + // a cell. The guard keys only on evaluators that reach the cell (excluded + // trace metrics are filtered out before the check). + it('ignores errors on excluded trace-metric evaluators', () => { + const matrix = buildMatrix(withDatasets(survivingSaturatedOnly, ['Latency']), guardConfig); + expect(matrix.proprietary[0].cells.triage.kind).toBe('score'); + }); + + it('does not let an unmeasured cell contribute to Overall', () => { + const matrix = buildMatrix(withDatasets(survivingSaturatedOnly, ['Trajectory']), guardConfig); + const overall = matrix.proprietary[0].overall; + + expect(overall.kind).not.toBe('score'); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.ts new file mode 100644 index 0000000000000..21165c41b3a2f --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.ts @@ -0,0 +1,754 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + MatrixColumnConfig, + MatrixCompositeConfig, + MatrixConfig, + MatrixModelConfig, + MatrixTokenCostConfig, +} from './load_matrix_config'; +import type { AggregatedEvaluatorScore, AggregatedModelScores } from './query_matrix_scores'; +import { detectSaturatedEvaluators, saturatedEvaluatorNames } from './evaluator_saturation'; +import type { EvaluatorSaturation } from './evaluator_saturation'; + +/** A single matrix cell: either a numeric 0-10 score or "Not recommended". */ +export type MatrixCell = + /** + * `selfJudged` marks a score the column's judge produced about itself, + * admitted because the column set `allowSelfJudged` after an audit found no + * self-preference. The score is real and rankable, but consumers must be + * able to disclose it rather than present it as arm's-length — publishing it + * undisclosed is the failure mode this flag exists to prevent. + */ + | { kind: 'score'; value: number; selfJudged?: boolean } + | { kind: 'not-recommended' } + /** + * Scores existed but every one was rejected by judge policy (self-judged, + * non-EIS judge, or same-family when configured). Distinct from 'missing': + * the model DID run, so re-running it changes nothing until the judge is + * fixed. Conflating the two cost a full re-sweep on 2026-08-29. + */ + | { kind: 'excluded'; reason: 'self-judged' | 'non-eis-judge' | 'same-family'; docs: number } + /** + * The model was scored on too few columns for an aggregate to mean anything + * (`config.minCoverage`). Only ever produced for `Overall` — a 2-of-24 run + * averaging 10.0 must not outrank a 22-of-24 run averaging 8.5. + */ + | { kind: 'insufficient-coverage'; covered: number; required: number } + /** + * A cell-relevant evaluator errored on every example (e.g. a trace-cluster + * permission fault nulling Trajectory/SkillInvoked). Its absence from the + * aggregate would let the mean rest on whichever evaluators survived — + * usually the saturated contract checks — and publish a flattering number + * derived from a broken instrument. Refusing beats flattering. + */ + | { kind: 'insufficient-evaluators'; evaluators: string[] } + | { kind: 'missing' }; + +/** Synthetic id for the legacy single "Overall" column. */ +export const OVERALL_COLUMN_ID = '__overall__'; + +/** A column as rendered, left-to-right, including derived composite columns. */ +export interface MatrixDisplayColumn { + id: string; + label: string; + group?: string; + kind: 'base' | 'composite' | 'overall'; +} + +export interface MatrixRow { + modelId: string; + modelLabel: string; + openSource: boolean; + /** Column/composite id -> cell. */ + cells: Record; + overall: MatrixCell; + /** Deterministic code/contract evaluator mean on the same 0–10 scale. */ + capability?: MatrixCell; + /** Judged evaluator mean on the same 0–10 scale. */ + judgedQuality?: MatrixCell; + /** + * Base columns with a non-missing cell out of all base columns. Partial + * coverage means `overall` and composites average fewer examples and are + * not directly comparable to full-coverage rows. + */ + coverage: { covered: number; total: number }; + /** + * Distinct commits this row's scores were produced against. A row spanning + * several suites can legitimately carry more than one: suites run on their + * own schedules. Published so a reader can tell which codebase a given model + * was measured on -- essential once models are appended to an existing board + * rather than swept together. + */ + commitShas?: string[]; + /** + * 1-based tier. Runs of the same model on an unchanged commit move the + * overall score by ~0.2 (stdev over 7 haiku runs on golden), so adjacent + * ranks are not distinguishable. Rows within a tier are statistically + * tied; only a tier boundary is a real difference. + */ + tier?: number; +} + +/** Aggregated token magnitudes for one (model, column) pair, in native units. */ +export interface TokenCostCell { + /** Base column id (matches `MatrixDisplayColumn.id`). */ + columnId: string; + inputTokens?: TokenStat; + outputTokens?: TokenStat; + /** Sum of the input + output means. */ + totalMean: number; +} + +export interface TokenStat { + mean: number; + min: number; + max: number; + count: number; +} + +export interface TokenCostModel { + modelId: string; + modelLabel: string; + openSource: boolean; + cells: TokenCostCell[]; +} + +export interface Matrix { + columns: Array<{ id: string; label: string; group?: string }>; + composites: Array<{ id: string; label: string; group?: string }>; + /** Full ordered render list (base + composite + legacy overall). */ + displayColumns: MatrixDisplayColumn[]; + overallLabel: string; + /** + * Per-evaluator ranking power, computed across all models. Evaluators marked + * `saturated` were excluded from Overall when the config opts in. + */ + evaluatorSaturation: EvaluatorSaturation[]; + proprietary: MatrixRow[]; + openSource: MatrixRow[]; + /** Present only when the config opts into the token axis. */ + tokenCost?: { models: TokenCostModel[] }; +} + +const roundTo = (value: number, decimals: number): number => { + const factor = 10 ** decimals; + return Math.round(value * factor) / factor; +}; + +const matchesModel = (modelConfig: MatrixModelConfig, modelId: string): boolean => + modelConfig.id === modelId || (modelConfig.matchIds?.includes(modelId) ?? false); + +const isExcludedEvaluator = (evaluatorName: string, excluded: readonly string[]): boolean => + excluded.some((entry) => evaluatorName.startsWith(entry)); + +const toCell = ( + value: number, + config: MatrixConfig, + { selfJudged = false }: { selfJudged?: boolean } = {} +): MatrixCell => + value <= config.notRecommendedBelow + ? { kind: 'not-recommended' } + : // Only attach the flag when true, so normally-judged cells keep their + // exact existing shape and no consumer sees `selfJudged: false` noise. + { kind: 'score', value, ...(selfJudged ? { selfJudged: true } : {}) }; + +/** Sample count doubles as the aggregation weight; zero-count evaluators still count once. */ +const weightOf = (evaluator: AggregatedEvaluatorScore): number => + evaluator.count > 0 ? evaluator.count : 1; + +/** Yields every evaluator contributing to a column, applying the suite/dataset filters. */ +function* columnEvaluators( + modelScores: AggregatedModelScores, + column: MatrixColumnConfig +): Generator { + const suiteSet = new Set(column.suites); + // `examplePrefixes` columns consume the synthetic per-prefix datasets + // (datasetId `prefix:`) produced by queryMatrixScores; a `datasetIds` + // column keeps its raw dataset-id semantics unchanged. + const datasetSet = column.examplePrefixes + ? new Set(column.examplePrefixes.map((prefix) => `prefix:${prefix}`)) + : column.datasetIds + ? new Set(column.datasetIds) + : undefined; + + for (const suite of modelScores.suites) { + if (!suiteSet.has(suite.suiteId)) { + continue; + } + for (const dataset of suite.datasets) { + if (!datasetSet || datasetSet.has(dataset.datasetId)) { + yield* dataset.evaluators; + } + } + } +} + +const columnErroredOutEvaluators = ( + modelScores: AggregatedModelScores, + column: MatrixColumnConfig +): string[] => { + const suiteSet = new Set(column.suites); + const datasetSet = column.examplePrefixes + ? new Set(column.examplePrefixes.map((prefix) => `prefix:${prefix}`)) + : column.datasetIds + ? new Set(column.datasetIds) + : undefined; + + const names = new Set(); + for (const suite of modelScores.suites) { + if (!suiteSet.has(suite.suiteId)) { + continue; + } + for (const dataset of suite.datasets) { + if (!datasetSet || datasetSet.has(dataset.datasetId)) { + for (const name of dataset.erroredOutEvaluators ?? []) { + names.add(name); + } + } + } + } + return [...names]; +}; + +/** + * Weighted mean (by sample count) of the evaluator scores mapped to a column. + * Returns `undefined` when no scores contribute. + */ +const computeColumnMean = ( + modelScores: AggregatedModelScores, + column: MatrixColumnConfig, + excludeEvaluators: readonly string[], + includeEvaluator?: (evaluator: AggregatedEvaluatorScore) => boolean +): number | undefined => { + const evaluatorSet = column.evaluators ? new Set(column.evaluators) : undefined; + + let weightedSum = 0; + let totalCount = 0; + + for (const evaluator of columnEvaluators(modelScores, column)) { + if (includeEvaluator && !includeEvaluator(evaluator)) { + continue; + } + // A column may opt into an explicit evaluator allowlist; otherwise the global + // exclusion list drops raw-magnitude evaluators that would blow out the 0-10 scale. + const skip = evaluatorSet + ? !evaluatorSet.has(evaluator.evaluatorName) + : isExcludedEvaluator(evaluator.evaluatorName, excludeEvaluators); + if (skip) { + continue; + } + + const weight = weightOf(evaluator); + weightedSum += evaluator.mean * weight; + totalCount += weight; + } + + return totalCount === 0 ? undefined : weightedSum / totalCount; +}; + +const buildCell = ( + mean: number | undefined, + column: MatrixColumnConfig, + config: MatrixConfig, + { + selfJudged = false, + excludedSelfJudged = 0, + erroredOutEvaluators = [], + }: { selfJudged?: boolean; excludedSelfJudged?: number; erroredOutEvaluators?: string[] } = {} +): MatrixCell => { + if (mean === undefined) { + // A blank because the judge policy threw the scores away is a different + // fact from a blank because the model never ran, and conflating them + // reads as a coverage gap the sweep is expected to fill. + return excludedSelfJudged > 0 + ? { kind: 'excluded', reason: 'self-judged', docs: excludedSelfJudged } + : { kind: 'missing' }; + } + + // A judge/quality evaluator that errored on EVERY example is absent from the + // aggregate rather than scored, so the cell's mean silently rests on + // whichever evaluators survived — usually the saturated contract checks. + // That is how a trace-cluster permission fault lifted DeepSeek's + // alert-analysis-a to 8.89 over models graded on the full set: its Trajectory + // and SkillInvoked evaluators errored out, leaving a mean over the 1.0s. + // Counting evaluators cannot catch this — a healthy frontier cell legitimately + // has 4 — so refuse to publish a number when a cell-relevant evaluator + // errored out entirely. Errors on excluded trace metrics (Latency et al.) are + // noise and never reach here. + const erroredOut = erroredOutEvaluators.filter( + (name) => + column.evaluators?.includes(name) ?? !isExcludedEvaluator(name, config.excludeEvaluators) + ); + if (erroredOut.length > 0) { + return { kind: 'insufficient-evaluators', evaluators: erroredOut }; + } + + const scale = column.scale ?? config.defaultScale; + return toCell(roundTo(mean * scale, config.decimals), config, { selfJudged }); +}; + +const CONTRACT_EVALUATORS = new Set([ + 'ExpectedToolCalled', + 'FinalAnswerPresent', + 'MinExpectedSteps', + 'SkillInvoked', +]); + +const axisCell = ( + modelScores: AggregatedModelScores, + config: MatrixConfig, + includeEvaluator: (evaluator: AggregatedEvaluatorScore) => boolean +): MatrixCell => { + const cells = config.columns.map((column) => ({ + cell: buildCell( + computeColumnMean(modelScores, column, config.excludeEvaluators, includeEvaluator), + column, + config, + { erroredOutEvaluators: columnErroredOutEvaluators(modelScores, column) } + ), + weight: config.overall.mode === 'weighted' ? column.weight : 1, + })); + return aggregateCells(cells, config); +}; + +/** + * Weighted mean of already-computed cells, shared by the legacy Overall column and + * composites: "Not recommended" sources contribute 0 (when configured) and missing + * sources are skipped, so the result reflects the data that exists rather than being + * dragged to "missing" by not-yet-wired columns. + */ +const aggregateCells = ( + sources: Array<{ cell: MatrixCell | undefined; weight: number }>, + config: MatrixConfig +): MatrixCell => { + let weightedSum = 0; + let totalWeight = 0; + let hasAnyData = false; + + for (const { cell, weight } of sources) { + // 'excluded' means every score was rejected by judge policy, so there is no + // trustworthy value to aggregate. Skip like 'missing' rather than counting + // it as a zero, which would silently depress Overall for a model that ran. + if (!cell || cell.kind === 'missing' || cell.kind === 'excluded') { + continue; + } + + hasAnyData = true; + + if ( + cell.kind === 'not-recommended' || + cell.kind === 'insufficient-coverage' || + cell.kind === 'insufficient-evaluators' + ) { + if (config.notRecommendedCountsAsZeroInOverall && cell.kind === 'not-recommended') { + totalWeight += weight; + } + continue; + } + + weightedSum += cell.value * weight; + totalWeight += weight; + } + + if (!hasAnyData || totalWeight === 0) { + return { kind: 'missing' }; + } + + return toCell(roundTo(weightedSum / totalWeight, config.decimals), config); +}; + +const computeOverall = (cells: Record, config: MatrixConfig): MatrixCell => + aggregateCells( + config.columns.map((column) => ({ + cell: cells[column.id], + weight: config.overall.mode === 'weighted' ? column.weight : 1, + })), + config + ); + +const computeComposite = ( + cells: Record, + composite: MatrixCompositeConfig, + config: MatrixConfig +): MatrixCell => + aggregateCells( + composite.from.map((refId) => ({ cell: cells[refId], weight: 1 })), + config + ); + +/** Resolves the left-to-right render order of base + composite (+ overall) columns. */ +const buildDisplayColumns = (config: MatrixConfig): MatrixDisplayColumn[] => { + const baseById = new Map(config.columns.map((column) => [column.id, column])); + const compositeById = new Map(config.composites.map((composite) => [composite.id, composite])); + + const declared: MatrixDisplayColumn[] = config.layout + ? config.layout.map((id): MatrixDisplayColumn => { + const base = baseById.get(id); + if (base) { + return { id, label: base.label, group: base.group, kind: 'base' }; + } + const composite = compositeById.get(id); + if (composite) { + return { id, label: composite.label, group: composite.group, kind: 'composite' }; + } + throw new Error(`Matrix config "layout" references unknown column/composite id: "${id}"`); + }) + : [ + ...config.columns.map( + (column): MatrixDisplayColumn => ({ + id: column.id, + label: column.label, + group: column.group, + kind: 'base', + }) + ), + ...config.composites.map( + (composite): MatrixDisplayColumn => ({ + id: composite.id, + label: composite.label, + group: composite.group, + kind: 'composite', + }) + ), + ]; + + return config.showOverall + ? [...declared, { id: OVERALL_COLUMN_ID, label: config.overall.label, kind: 'overall' }] + : declared; +}; + +/** + * Aggregates the raw-magnitude token evaluators for one (model, column) pair. Unlike + * the quality path these stay in native units and preserve the observed min/max spread. + */ +const computeTokenStat = ( + modelScores: AggregatedModelScores, + column: MatrixColumnConfig, + evaluatorPrefix: string +): TokenStat | undefined => { + let weightedSum = 0; + let totalCount = 0; + let min = Number.POSITIVE_INFINITY; + let max = Number.NEGATIVE_INFINITY; + + for (const evaluator of columnEvaluators(modelScores, column)) { + if (!evaluator.evaluatorName.startsWith(evaluatorPrefix)) { + continue; + } + + const weight = weightOf(evaluator); + weightedSum += evaluator.mean * weight; + totalCount += weight; + // Stats payloads may omit the per-experiment extremes; the mean is the only bound then. + min = Math.min(min, evaluator.min ?? evaluator.mean); + max = Math.max(max, evaluator.max ?? evaluator.mean); + } + + if (totalCount === 0) { + return undefined; + } + + return { mean: weightedSum / totalCount, min, max, count: totalCount }; +}; + +const buildTokenCost = ( + config: MatrixConfig, + tokenConfig: MatrixTokenCostConfig, + resolveScores: (modelConfig: MatrixModelConfig) => AggregatedModelScores | undefined +): { models: TokenCostModel[] } => { + const columnIds = tokenConfig.columns; + const tokenColumns = columnIds + ? config.columns.filter((column) => columnIds.includes(column.id)) + : config.columns; + + const models: TokenCostModel[] = []; + + for (const modelConfig of config.models) { + const modelScores = resolveScores(modelConfig); + if (!modelScores) { + continue; + } + + const cells: TokenCostCell[] = []; + for (const column of tokenColumns) { + const inputTokens = computeTokenStat(modelScores, column, tokenConfig.inputEvaluator); + const outputTokens = computeTokenStat(modelScores, column, tokenConfig.outputEvaluator); + if (!inputTokens && !outputTokens) { + continue; + } + cells.push({ + columnId: column.id, + inputTokens, + outputTokens, + totalMean: (inputTokens?.mean ?? 0) + (outputTokens?.mean ?? 0), + }); + } + + if (cells.length > 0) { + models.push({ + modelId: modelConfig.id, + modelLabel: modelConfig.label, + openSource: modelConfig.openSource, + cells, + }); + } + } + + return { models }; +}; + +/** + * Pure transform from aggregated eval scores + config into a renderable matrix. + * Models are emitted in config order; models absent from the data are skipped. + */ +/** + * Groups rows into tiers of statistically indistinguishable models. + * + * Re-running one model on an unchanged commit and stack moves its overall by + * about 0.48 on the 0-10 scale: the pooled within-commit stdev across 19 + * model/commit groups on golden (df=87, up to 8 repeats each, measured over + * the same evaluator set Overall actually aggregates). The earlier 0.2 figure + * came from 7 haiku runs scored over ALL evaluators, including saturated ones + * that barely move between runs and so damped the spread. Rows stay in the + * same tier until the drop from the tier leader exceeds the combined 95% + * interval; only crossing a tier boundary is a difference the data supports. + */ +const assignTiers = (rows: MatrixRow[], config: MatrixConfig): MatrixRow[] => { + const sd = config.overall.runStdev; + if (!sd) { + return rows; + } + const threshold = 2 * 1.96 * sd; + let tier = 1; + let leader: number | undefined; + return rows.map((row) => { + const value = row.overall.kind === 'score' ? row.overall.value : undefined; + if (value === undefined) { + return row; + } + if (leader === undefined) { + leader = value; + } else if (leader - value > threshold) { + tier += 1; + leader = value; + } + return { ...row, tier }; + }); +}; + +/** + * Distinct commits behind one model's scores, newest experiment first. Suites + * run on independent schedules, so more than one is normal and not an error. + */ +export const rowCommitShas = ( + modelScores: AggregatedModelScores | undefined +): string[] | undefined => { + if (!modelScores) { + return undefined; + } + const ordered = [...modelScores.suites].sort((a, b) => + String(b.timestamp ?? '').localeCompare(String(a.timestamp ?? '')) + ); + const shas = ordered.map((suite) => suite.commitSha).filter((sha): sha is string => !!sha); + const unique = [...new Set(shas)]; + return unique.length ? unique : undefined; +}; + +export const buildMatrix = ( + aggregated: AggregatedModelScores[], + config: MatrixConfig, + log?: { warning: (message: string) => void } +): Matrix => { + const byModelId = new Map(aggregated.map((entry) => [entry.modelId, entry])); + const resolveScores = (modelConfig: MatrixModelConfig) => + byModelId.get(modelConfig.id) ?? + aggregated.find((entry) => matchesModel(modelConfig, entry.modelId)); + + // An evaluator that returns nearly the same high score for every model ranks + // nothing, but still takes an equal share of the Overall mean -- diluting the + // evaluators that DO separate models. Detect those mechanically and drop them + // from the aggregate so Overall reflects the metrics that actually move. + const saturation = config.overall.excludeSaturatedEvaluators + ? detectSaturatedEvaluators(aggregated) + : []; + const saturatedNames = saturatedEvaluatorNames(saturation); + const excludeEvaluators = + saturatedNames.size > 0 + ? [...config.excludeEvaluators, ...saturatedNames] + : config.excludeEvaluators; + + const proprietary: MatrixRow[] = []; + const openSource: MatrixRow[] = []; + + for (const modelConfig of config.models) { + const modelScores = resolveScores(modelConfig); + if (!modelScores) { + continue; + } + + const cells: Record = {}; + for (const column of config.columns) { + const columnSuites = new Set(column.suites); + cells[column.id] = buildCell( + computeColumnMean(modelScores, column, excludeEvaluators), + column, + config, + { + // Disclose only when a suite actually feeding THIS column was graded + // by the model itself. + selfJudged: modelScores.suites.some( + (suite) => columnSuites.has(suite.suiteId) && suite.selfJudged === true + ), + excludedSelfJudged: modelScores.suites + .filter((suite) => columnSuites.has(suite.suiteId)) + .reduce((total, suite) => total + (suite.excludedSelfJudged ?? 0), 0), + erroredOutEvaluators: columnErroredOutEvaluators(modelScores, column), + } + ); + } + + // Declared order, so a later composite can reference an earlier one (e.g. Overall + // Score <- Agent Builder Score); an unresolved reference contributes nothing. + for (const composite of config.composites) { + cells[composite.id] = computeComposite(cells, composite, config); + } + + // A scored column is one with a real, trustworthy value. 'excluded' cells + // ran but had every grade rejected, so they are NOT coverage — counting + // them would let a fully self-judged model claim a full row. + const scoredColumns = config.columns.filter((c) => cells[c.id].kind === 'score').length; + const overall = computeOverall(cells, config); + + const row: MatrixRow = { + modelId: modelConfig.id, + modelLabel: modelConfig.label, + openSource: modelConfig.openSource, + cells, + // Publishing an average over too few columns invites the wrong read: a + // model scored on 2 of 24 prompts averaged 10.0 and ranked above every + // frontier model until this floor existed. + overall: + config.minCoverage > 0 && scoredColumns < config.minCoverage + ? { kind: 'insufficient-coverage', covered: scoredColumns, required: config.minCoverage } + : overall, + capability: axisCell(modelScores, config, (evaluator) => + CONTRACT_EVALUATORS.has( + evaluator.evaluatorName.replace(/^Skill Invoked \([^)]+\)$/, 'SkillInvoked') + ) + ), + judgedQuality: axisCell( + modelScores, + config, + (evaluator) => + !CONTRACT_EVALUATORS.has( + evaluator.evaluatorName.replace(/^Skill Invoked \([^)]+\)$/, 'SkillInvoked') + ) + ), + coverage: { + covered: scoredColumns, + total: config.columns.length, + }, + commitShas: rowCommitShas(modelScores), + }; + + (modelConfig.openSource ? openSource : proprietary).push(row); + } + + // Rank by the final composite (e.g. Overall Score) when composites exist, + // otherwise by the legacy Overall column. + const primaryId = + config.composites.length > 0 + ? config.composites[config.composites.length - 1].id + : OVERALL_COLUMN_ID; + + const sortValue = (row: MatrixRow): number => { + const cell = primaryId === OVERALL_COLUMN_ID ? row.overall : row.cells[primaryId]; + return cell && cell.kind === 'score' ? cell.value : -1; + }; + + const sortByPrimaryDesc = (a: MatrixRow, b: MatrixRow): number => sortValue(b) - sortValue(a); + + const allRows = [...proprietary, ...openSource]; + if (log && allRows.length > 0) { + // A column only a handful of models ever ran still contributes to their + // Overall, so those models are averaged over a different set of columns + // than everyone else -- and the column itself cannot rank anything. + // Measured 2026-09-01: attack-discovery 4/20, both migrations columns 1/20, + // all three pipeline gaps rather than model failures. + for (const column of config.columns) { + const scored = allRows.filter((row) => row.cells[column.id]?.kind === 'score').length; + if (scored > 0 && scored < allRows.length / 2) { + log.warning( + `Column "${column.label}" has scores for only ${scored} of ${allRows.length} models -- too sparse to rank, and the models that did run it are averaged over a different column set than the rest. Check whether the suite is scheduled in the weekly pipeline before reading these cells as model differences.` + ); + } + } + + // Rows appended over time are graded against whatever the codebase was + // that week. That is legitimate -- it is how a model gets added to an + // existing board -- but it stops being comparable if nobody says so, and + // a single top-level provenance stamp actively hides it. + const shaByRow = allRows + .map((row) => ({ label: row.modelLabel, shas: row.commitShas ?? [] })) + .filter((entry) => entry.shas.length > 0); + const distinctShas = new Set(shaByRow.flatMap((entry) => entry.shas)); + if (distinctShas.size > 1) { + const sample = shaByRow + .slice(0, 6) + .map((entry) => `${entry.label}=${entry.shas.map((sha) => sha.slice(0, 12)).join('+')}`) + .join(', '); + log.warning( + `Matrix spans ${distinctShas.size} commits across ${ + shaByRow.length + } scored rows -- rows were graded against different codebases and are only loosely comparable. ${sample}${ + shaByRow.length > 6 ? ', ...' : '' + }` + ); + } + + // The per-prefix fetch is what fills individual cells. If it returns + // nothing while the suite-wide aggregate is healthy, cells silently fall + // back to a coarser source and the board LOOKS fine while being wrong. + // That is exactly how a stripped `evaluator.metadata` inflated every + // model's Overall (#286691 fallout): the failure had no symptom until + // two runs were compared cell by cell. + const scoredCells = allRows.reduce( + (sum, row) => + sum + config.columns.filter((column) => row.cells[column.id]?.kind === 'score').length, + 0 + ); + if (scoredCells === 0) { + log.warning( + `No column produced a single scored cell across ${allRows.length} models. The per-prefix score fetch returned nothing usable -- do NOT publish this run. Check that the scores route still returns the fields the verdict ladder reads before blaming the models.` + ); + } + } + + return { + columns: config.columns.map((column) => ({ + id: column.id, + label: column.label, + group: column.group, + })), + composites: config.composites.map((composite) => ({ + id: composite.id, + label: composite.label, + group: composite.group, + })), + displayColumns: buildDisplayColumns(config), + overallLabel: config.overall.label, + evaluatorSaturation: saturation, + proprietary: assignTiers(proprietary.sort(sortByPrimaryDesc), config), + openSource: assignTiers(openSource.sort(sortByPrimaryDesc), config), + // Token magnitudes are meaningful only over base columns; composites are derived scores. + ...(config.tokenCost + ? { tokenCost: buildTokenCost(config, config.tokenCost, resolveScores) } + : {}), + }; +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/config_data_preflight.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/config_data_preflight.test.ts new file mode 100644 index 0000000000000..abb5a87cd6e6e --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/config_data_preflight.test.ts @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ToolingLog } from '@kbn/tooling-log'; +import { parseMatrixConfig } from './load_matrix_config'; +import type { AggregatedModelScores } from './query_matrix_scores'; +import { warnOnDataAboutToLeaveLookback } from './config_data_preflight'; + +const collectWarnings = () => { + const warnings: string[] = []; + const log = { + warning: (msg: string) => warnings.push(String(msg)), + info: () => {}, + debug: () => {}, + } as unknown as ToolingLog; + return { warnings, log }; +}; + +const DAY = 24 * 60 * 60 * 1000; +const NOW = Date.parse('2026-09-02T00:00:00.000Z'); + +const config = parseMatrixConfig({ + lookbackDays: 45, + columns: [ + { + id: 'migrations-rules', + label: 'Rule Translation', + suites: ['security-automatic-migrations'], + }, + { id: 'triage', label: 'Triage', suites: ['persona-matrix'] }, + ], + models: [{ id: 'model-a', label: 'A' }], +}); + +const scores = (suiteId: string, ageDays: number): AggregatedModelScores[] => [ + { + modelId: 'model-a', + provider: 'p', + suites: [ + { + suiteId, + experimentId: 'e1', + timestamp: new Date(NOW - ageDays * DAY).toISOString(), + datasets: [ + { + datasetId: 'd', + datasetName: 'd', + evaluators: [{ evaluatorName: 'Rubric', mean: 0.8, count: 10 }], + }, + ], + }, + ], + }, +]; + +describe('warnOnDataAboutToLeaveLookback', () => { + // The migrations columns are pinned to a branch whose newest run is 34 days + // old against a 45-day window. Nothing is wrong today and nothing will fail + // loudly on 2026-09-13 either: the columns will simply go blank, exactly the + // silent-blanking failure this module exists to catch. + it('warns when a suite is inside the window but close to falling out', () => { + const { warnings, log } = collectWarnings(); + + warnOnDataAboutToLeaveLookback(config, scores('security-automatic-migrations', 34), log, { + now: NOW, + }); + + expect(warnings).toHaveLength(1); + expect(warnings[0]).toContain('security-automatic-migrations'); + // Says when it goes blank, so the reader can act before it does. + expect(warnings[0]).toContain('11 day'); + }); + + it('stays silent for data with comfortable headroom', () => { + const { warnings, log } = collectWarnings(); + + warnOnDataAboutToLeaveLookback(config, scores('persona-matrix', 3), log, { now: NOW }); + + expect(warnings).toEqual([]); + }); + + // A column whose data already aged out is a different (louder) failure: the + // cell is blank NOW, so warning about a future expiry would be misleading. + it('does not warn about data that already left the window', () => { + const { warnings, log } = collectWarnings(); + + warnOnDataAboutToLeaveLookback(config, scores('security-automatic-migrations', 60), log, { + now: NOW, + }); + + expect(warnings).toEqual([]); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/config_data_preflight.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/config_data_preflight.ts new file mode 100644 index 0000000000000..5f19bb8e534b1 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/config_data_preflight.ts @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ToolingLog } from '@kbn/tooling-log'; +import type { MatrixConfig } from './load_matrix_config'; +import type { AggregatedModelScores } from './query_matrix_scores'; + +/** + * Warn when a column's configured evaluator or suite names match nothing in the + * fetched scores. + * + * A column that names an evaluator no score document carries does not fail: the + * allowlist simply skips every evaluator it does not recognise and averages + * whatever is left. The attack-discovery column asked for `Rubric` and + * `StrictTrajectory` while the suite writes `AttackDiscoveryRubric` and no + * strict-trajectory evaluator at all, so it silently averaged two of three + * evaluators and published 10.0 where the honest figure was 6.67. + * + * Names are compared against the data actually returned, so a typo, a renamed + * evaluator, or a suite that stopped emitting one all surface the same way. + */ +export function warnOnConfiguredNamesMissingFromData( + config: MatrixConfig, + aggregated: AggregatedModelScores[], + log: ToolingLog +): void { + const evaluatorsInData = new Set(); + const suitesInData = new Set(); + + for (const model of aggregated) { + for (const suite of model.suites) { + suitesInData.add(suite.suiteId); + for (const dataset of suite.datasets) { + for (const evaluator of dataset.evaluators) { + evaluatorsInData.add(evaluator.evaluatorName); + } + } + } + } + + // Nothing came back at all: the caller already fails on an empty match, and + // reporting every configured name as missing would bury that. + if (evaluatorsInData.size === 0) { + return; + } + + for (const column of config.columns) { + const missingEvaluators = (column.evaluators ?? []).filter( + (name) => !evaluatorsInData.has(name) + ); + const missingSuites = (column.suites ?? []).filter((id) => !suitesInData.has(id)); + + if (missingEvaluators.length > 0) { + log.warning( + `column '${column.id}' names evaluator(s) absent from every score document: ` + + `${missingEvaluators.join(', ')} -- the column still scores, but only over the ` + + `evaluators that did match, so its value is an average of a subset. ` + + `Evaluators present: ${[...evaluatorsInData].sort().join(', ')}` + ); + } + + if (missingSuites.length > 0) { + log.warning( + `column '${column.id}' names suite(s) with no scores in this window: ` + + `${missingSuites.join(', ')}` + ); + } + } +} + +/** + * Warn when a column's freshest data is inside the lookback window but close to + * falling out of it. + * + * A branch pin freezes a column on a run that never gets newer, so the window + * slides toward it. Nothing fails when it crosses: `pickLatestExperimentPerModel` + * simply stops selecting the experiment and the column goes blank, which reads + * as "the model was never evaluated" rather than "the data aged out". The + * migrations columns are pinned to a branch last written 2026-07-30, so on a + * 45-day window they blank on 2026-09-13 with no other signal. + * + * Warn only for data still IN the window: a column that already aged out is + * blank today, and a future-tense warning would misdescribe it. + */ +export function warnOnDataAboutToLeaveLookback( + config: MatrixConfig, + aggregated: AggregatedModelScores[], + log: ToolingLog, + { now = Date.now(), warnWithinDays = 14 }: { now?: number; warnWithinDays?: number } = {} +): void { + const lookbackDays = config.lookbackDays; + if (!lookbackDays) { + return; + } + + // Freshest run per suite: that is what decides how long the column survives. + const newestBySuite = new Map(); + for (const model of aggregated) { + for (const suite of model.suites) { + if (!suite.timestamp) { + continue; + } + const ts = Date.parse(suite.timestamp); + if (Number.isNaN(ts)) { + continue; + } + newestBySuite.set(suite.suiteId, Math.max(newestBySuite.get(suite.suiteId) ?? 0, ts)); + } + } + + const DAY_MS = 24 * 60 * 60 * 1000; + for (const [suiteId, newest] of newestBySuite) { + const ageDays = (now - newest) / DAY_MS; + const daysLeft = Math.floor(lookbackDays - ageDays); + if (daysLeft < 0 || daysLeft > warnWithinDays) { + continue; + } + + const columns = config.columns + .filter((column) => column.suites.includes(suiteId)) + .map((column) => column.id); + + log.warning( + `Suite \`${suiteId}\` has no run newer than ${new Date(newest) + .toISOString() + .slice(0, 10)}; it leaves the ${lookbackDays}-day lookback in ${daysLeft} day(s), after ` + + `which these columns go blank with no other signal: ${ + columns.length ? columns.join(', ') : '(none)' + }. Re-run the suite or pin the column to a branch with fresher data.` + ); + } +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/ensemble_column.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/ensemble_column.test.ts new file mode 100644 index 0000000000000..0ad7b06c135e3 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/ensemble_column.test.ts @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { buildEnsembleColumn, type EnsembleCellInput } from './ensemble_column'; + +const cell = ( + judgeId: string, + modelId: string, + exampleId: string, + score: number +): EnsembleCellInput => ({ judgeId, modelId, exampleId, score }); + +describe('buildEnsembleColumn', () => { + it('averages judges and reports how far apart they were', () => { + const column = buildEnsembleColumn([ + cell('j1', 'a', 'ex-0', 0.9), + cell('j2', 'a', 'ex-0', 0.5), + cell('j1', 'b', 'ex-0', 0.4), + cell('j2', 'b', 'ex-0', 0.4), + ]); + + const a = column.models.find((m) => m.modelId === 'a')!; + const b = column.models.find((m) => m.modelId === 'b')!; + + expect(a.ensemble).toBeCloseTo(0.7, 6); + // The disagreement that produced 0.7 stays visible next to it. + expect(a.judgeSpread).toBeCloseTo(0.4, 6); + // Judges agreed on b, so its ensemble carries no hidden conflict. + expect(b.judgeSpread).toBeCloseTo(0, 6); + }); + + it('averages only over cells every judge graded', () => { + // j2 never graded ex-1, where model a scores badly. Including it would let + // a's ensemble inherit only its good cell. + const column = buildEnsembleColumn([ + cell('j1', 'a', 'ex-0', 0.9), + cell('j1', 'a', 'ex-1', 0.1), + cell('j2', 'a', 'ex-0', 0.9), + ]); + + expect(column.sharedCellCount).toBe(1); + expect(column.models[0].cellCount).toBe(1); + expect(column.models[0].ensemble).toBeCloseTo(0.9, 6); + }); + + it('ranks by ensemble score', () => { + const column = buildEnsembleColumn([ + cell('j1', 'low', 'ex-0', 0.2), + cell('j2', 'low', 'ex-0', 0.2), + cell('j1', 'high', 'ex-0', 0.8), + cell('j2', 'high', 'ex-0', 0.8), + ]); + + expect(column.models.map((m) => m.modelId)).toEqual(['high', 'low']); + }); + + it('reports sqrt(N) as the expected noise reduction', () => { + const four = ['j1', 'j2', 'j3', 'j4'].map((j) => cell(j, 'a', 'ex-0', 0.5)); + expect(buildEnsembleColumn(four).noiseReductionFactor).toBeCloseTo(2, 6); + }); + + it('refuses to call a single judge an ensemble', () => { + expect(() => buildEnsembleColumn([cell('j1', 'a', 'ex-0', 0.5)])).toThrow(/at least 2 judges/); + }); + + it('refuses to average judges that graded disjoint cells', () => { + expect(() => + buildEnsembleColumn([cell('j1', 'a', 'ex-0', 0.5), cell('j2', 'a', 'ex-9', 0.5)]) + ).toThrow(/no shared basis/); + }); + + it('keeps per-judge means so the ensemble can be audited', () => { + const column = buildEnsembleColumn([ + cell('lenient', 'a', 'ex-0', 1), + cell('strict', 'a', 'ex-0', 0), + ]); + + expect(column.models[0].perJudge).toEqual({ lenient: 1, strict: 0 }); + // A 0.5 that came from total disagreement must not read like consensus. + expect(column.models[0].judgeSpread).toBeCloseTo(1, 6); + }); + + it('reports separable pairs out of total, so the board can state how much of its ranking is real', () => { + // Cell difficulty swings hard (0.9 -> 0.1 by example) and the two strong + // models differ by a small CONSTANT offset. Unpaired resampling drowns that + // offset in difficulty variance; paired resampling cancels difficulty and + // keeps it. So this fixture only behaves if the bootstrap is truly paired. + const cells = []; + for (let i = 0; i < 30; i++) { + const difficulty = i % 2 === 0 ? 0.9 : 0.1; + for (const judgeId of ['j1', 'j2']) { + cells.push(cell(judgeId, 'strong', `e${i}`, difficulty)); + cells.push(cell(judgeId, 'weak', `e${i}`, difficulty - 0.08)); + cells.push(cell(judgeId, 'alsoStrong', `e${i}`, difficulty)); + } + } + + const out = buildEnsembleColumn(cells); + + expect(out.totalPairs).toBe(3); + // strong-vs-weak and alsoStrong-vs-weak separate; strong-vs-alsoStrong cannot. + expect(out.separablePairs).toBe(2); + }); + + it('pairs on example identity even when models were graded in different orders', () => { + // 'a' beats 'b' by a small CONSTANT margin on every example, while example + // difficulty swings wildly. Paired on identity, the margin is consistent + // and the pair separates. Paired by array position -- 'b' is emitted in + // reverse order -- easy cells line up against hard ones, the difference is + // swamped by difficulty, and the true separation disappears. + const cells = []; + for (let i = 0; i < 30; i++) { + const difficulty = i % 2 === 0 ? 0.9 : 0.1; + for (const judgeId of ['j1', 'j2']) { + cells.push(cell(judgeId, 'a', `e${i}`, difficulty)); + } + } + for (let i = 29; i >= 0; i--) { + const difficulty = i % 2 === 0 ? 0.9 : 0.1; + for (const judgeId of ['j1', 'j2']) { + cells.push(cell(judgeId, 'b', `e${i}`, difficulty - 0.05)); + } + } + + const out = buildEnsembleColumn(cells); + + expect(out.totalPairs).toBe(1); + expect(out.separablePairs).toBe(1); + }); + + it('flags a pair whose CI edge sits on zero, because that verdict is seed-dependent', () => { + // A tiny constant edge over noisy cells: the CI lands against zero, so + // whether it "separates" is decided by the generator, not the data. + const cells = []; + for (let i = 0; i < 40; i++) { + const noise = ((i * 7) % 10) / 100; + for (const judgeId of ['j1', 'j2']) { + cells.push(cell(judgeId, 'a', `e${i}`, 0.5 + noise + 0.002)); + cells.push(cell(judgeId, 'b', `e${i}`, 0.5 + noise)); + } + } + + expect(buildEnsembleColumn(cells).borderlinePairs).toBe(1); + }); + + it('is deterministic across runs, so a published pair count does not drift', () => { + const cells = []; + for (let i = 0; i < 25; i++) { + for (const judgeId of ['j1', 'j2']) { + cells.push(cell(judgeId, 'a', `e${i}`, (i % 5) / 4)); + cells.push(cell(judgeId, 'b', `e${i}`, ((i + 2) % 5) / 4)); + } + } + + expect(buildEnsembleColumn(cells).separablePairs).toBe( + buildEnsembleColumn(cells).separablePairs + ); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/ensemble_column.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/ensemble_column.ts new file mode 100644 index 0000000000000..1b5273ba2dc68 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/ensemble_column.ts @@ -0,0 +1,228 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Collapse several judges' scores for the same cells into one ensemble column. + * + * A single-judge column carries that judge's idiosyncratic scatter, and on this + * board the scatter is larger than the gaps it has to resolve. Averaging N + * judges over identical cells cuts the independent component of that noise by + * about sqrt(N) while leaving real model differences intact, because the noise + * is judge-specific and the signal is not. + * + * Two things are deliberately kept alongside the mean: + * + * - `judgeSpread` -- how far apart the judges were on this model. A high + * ensemble score built from judges that disagreed violently is not the same + * claim as one they all agreed on, and collapsing to a bare mean hides it. + * - `judgeCount` -- how many judges contributed. A model averaged over two + * judges must not silently sit next to one averaged over four. + * + * The ensemble is only computed over cells EVERY judge graded. Averaging over + * ragged coverage would let a model look strong because the judge that dislikes + * it happened to fail on its cells. + */ + +export interface EnsembleCellInput { + judgeId: string; + modelId: string; + exampleId: string; + score: number; +} + +export interface EnsembleModelScore { + modelId: string; + /** Mean across judges of that judge's mean over the shared cells. */ + ensemble: number; + /** Max-min of the per-judge means. Wide means the judges disagreed. */ + judgeSpread: number; + judgeCount: number; + cellCount: number; + /** Per-judge mean, so a reader can see what the ensemble is hiding. */ + perJudge: Record; +} + +export interface EnsembleColumn { + models: EnsembleModelScore[]; + judges: string[]; + /** Cells graded by every judge and therefore eligible for the ensemble. */ + sharedCellCount: number; + /** + * Expected noise reduction, sqrt(N) for N judges. Reported rather than + * applied: it is the theoretical factor for independent judge error, and + * judges are not fully independent, so it is an upper bound on the benefit. + */ + noiseReductionFactor: number; + /** + * Model pairs whose ensemble difference has a 95% bootstrap CI excluding + * zero, out of every pair compared. Published as counts rather than a bare + * ordering: the board must state how much of its own ranking is real. + */ + separablePairs: number; + totalPairs: number; + /** + * Pairs whose CI edge lands within `BORDERLINE_MARGIN` of zero. These flip + * verdict on a different RNG seed, so a bare "N separate" hides a coin flip: + * swapping one seeded generator for another moved this board's count by one. + * Reported so a borderline pair is never quoted as a clean separation. + */ + borderlinePairs: number; +} + +/** + * A CI edge this close to zero is not a decision, it is noise. Chosen as ~5% of + * the observed model spread on this board rather than an absolute epsilon. + */ +const BORDERLINE_MARGIN = 0.005; + +/** + * Deterministic paired bootstrap over the shared cells. + * + * Paired, because both models must be resampled on the SAME cells -- example + * difficulty is the dominant variance component, and resampling independently + * would compare a model on easy cells against one on hard cells and manufacture + * separations that do not exist. + * + * Seeded, because a published "N of M pairs separate" figure that changes on + * re-run is not a measurement. The seed is fixed, not drawn from the clock. + */ +function countSeparablePairs( + perModelCells: Map>, + iterations = 2000 +): { separablePairs: number; totalPairs: number; borderlinePairs: number } { + const ids = [...perModelCells.keys()].sort(); + // Seeded LCG (Numerical Recipes constants), in plain modular arithmetic so it + // reads as arithmetic rather than bit-twiddling. Quality is ample here: the + // bootstrap only needs uniform indices, not cryptographic randomness. + const MODULUS = 4294967296; + let seed = 2463534242; + const rnd = () => { + seed = (1664525 * seed + 1013904223) % MODULUS; + return seed / MODULUS; + }; + + let separablePairs = 0; + let totalPairs = 0; + let borderlinePairs = 0; + + for (let a = 0; a < ids.length; a++) { + for (let b = a + 1; b < ids.length; b++) { + // Pair on EXAMPLE IDENTITY, never on array position: each model's key + // list is built independently, so index i is a different example for + // each model and index-pairing silently compares unrelated cells. + const ma = perModelCells.get(ids[a])!; + const mb = perModelCells.get(ids[b])!; + const sharedExamples = [...ma.keys()].filter((k) => mb.has(k)).sort(); + const xs = sharedExamples.map((k) => ma.get(k)!); + const ys = sharedExamples.map((k) => mb.get(k)!); + const n = Math.min(xs.length, ys.length); + if (n === 0) continue; + totalPairs++; + + const diffs: number[] = []; + for (let it = 0; it < iterations; it++) { + let sx = 0; + let sy = 0; + for (let k = 0; k < n; k++) { + const idx = Math.floor(rnd() * n); + sx += xs[idx]; + sy += ys[idx]; + } + diffs.push(sx / n - sy / n); + } + diffs.sort((p, q) => p - q); + const lo = diffs[Math.floor(0.025 * iterations)]; + const hi = diffs[Math.floor(0.975 * iterations)]; + if (lo > 0 || hi < 0) separablePairs++; + if (Math.min(Math.abs(lo), Math.abs(hi)) < BORDERLINE_MARGIN) borderlinePairs++; + } + } + + return { separablePairs, totalPairs, borderlinePairs }; +} + +const mean = (xs: number[]): number => xs.reduce((a, b) => a + b, 0) / xs.length; + +const SEP = '\u0000'; +const key = (modelId: string, exampleId: string) => `${modelId}${SEP}${exampleId}`; + +/** + * Build the ensemble column. + * + * Throws rather than degrading when judges share no cells: an "ensemble" over + * disjoint cells is just a relabelled average of different measurements, and + * publishing it as a consensus would be a stronger claim than the data makes. + */ +export function buildEnsembleColumn(cells: EnsembleCellInput[]): EnsembleColumn { + const judges = [...new Set(cells.map((c) => c.judgeId))].sort(); + if (judges.length < 2) { + throw new Error( + `An ensemble needs at least 2 judges; got ${judges.length}. A single judge is just that judge's column.` + ); + } + + const byJudge = new Map>(); + for (const judgeId of judges) byJudge.set(judgeId, new Map()); + for (const c of cells) byJudge.get(c.judgeId)!.set(key(c.modelId, c.exampleId), c.score); + + const [first, ...rest] = judges; + const sharedKeys = [...byJudge.get(first)!.keys()].filter((k) => + rest.every((j) => byJudge.get(j)!.has(k)) + ); + + if (sharedKeys.length === 0) { + throw new Error( + `No cell was graded by all ${judges.length} judges, so there is no shared basis to average over.` + ); + } + + const keysByModel = new Map(); + for (const k of sharedKeys) { + const modelId = k.split(SEP)[0]; + keysByModel.set(modelId, [...(keysByModel.get(modelId) ?? []), k]); + } + + const models: EnsembleModelScore[] = [...keysByModel.keys()].sort().map((modelId) => { + const modelKeys = keysByModel.get(modelId)!; + const perJudge: Record = Object.fromEntries( + judges.map((j) => [j, mean(modelKeys.map((k) => byJudge.get(j)!.get(k)!))]) + ); + const judgeMeans = Object.values(perJudge); + + return { + modelId, + ensemble: mean(judgeMeans), + judgeSpread: Math.max(...judgeMeans) - Math.min(...judgeMeans), + judgeCount: judges.length, + cellCount: modelKeys.length, + perJudge, + }; + }); + + // Per-cell ensemble values (mean across judges), which is what the bootstrap + // must resample -- resampling the per-model means would throw away the + // cell-level pairing that makes the comparison paired at all. + // Keyed by exampleId so pairs are matched on the same example, not on + // position in a per-model list. + const perModelCells = new Map>( + [...keysByModel.entries()].map(([modelId, ks]) => [ + modelId, + new Map(ks.map((k) => [k.split(SEP)[1], mean(judges.map((j) => byJudge.get(j)!.get(k)!))])), + ]) + ); + const { separablePairs, totalPairs, borderlinePairs } = countSeparablePairs(perModelCells); + + return { + models: models.sort((a, b) => b.ensemble - a.ensemble), + judges, + sharedCellCount: sharedKeys.length, + noiseReductionFactor: Math.sqrt(judges.length), + separablePairs, + totalPairs, + borderlinePairs, + }; +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/evaluator_health.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/evaluator_health.test.ts new file mode 100644 index 0000000000000..e2d44447a403a --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/evaluator_health.test.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { checkEvaluatorHealth } from './evaluator_health'; + +const observe = (evaluatorName: string, scores: number[], role?: 'gate' | 'grader') => + scores.map((score, i) => ({ + evaluatorName, + modelId: `model-${i}`, + score, + ...(role ? { role } : {}), + })); + +describe('checkEvaluatorHealth', () => { + it('flags an evaluator that returns the same score for every model', () => { + const report = checkEvaluatorHealth({ + observations: observe('DocVersionReleaseDate', Array(40).fill(1)), + }); + + const finding = report.findings[0]; + expect(finding.classification).toBe('constant'); + expect(finding.distinctValues).toBe(1); + // A constant evaluator cannot separate any pair, so it must not be + // reported as merely "saturated" -- it carries no information at all. + expect(report.ok).toBe(false); + }); + + it('flags a grader pinned at the ceiling even when it has a couple of values', () => { + // 95% at 1.0: this is the shape the collapsed attack-discovery rubric had. + const scores = [...Array(38).fill(1), 0, 0]; + const report = checkEvaluatorHealth({ observations: observe('Rubric', scores) }); + + expect(report.findings[0].classification).toBe('saturated'); + expect(report.findings[0].ceilingShare).toBeCloseTo(0.95, 2); + expect(report.ok).toBe(false); + }); + + it('passes a grader that spreads models out', () => { + const scores = Array.from({ length: 40 }, (_, i) => i / 39); + const report = checkEvaluatorHealth({ observations: observe('Factuality', scores) }); + + expect(report.findings[0].classification).toBe('discriminating'); + expect(report.ok).toBe(true); + }); + + it('does not fail a gate for sitting at the ceiling', () => { + // Gates SHOULD saturate: "every model avoided the forbidden tool" is a pass, + // not a broken evaluator. Failing them would train people to ignore the gate. + const report = checkEvaluatorHealth({ + observations: observe('ShouldNotCallTool', Array(40).fill(1), 'gate'), + }); + + expect(report.findings[0].classification).toBe('gate-satisfied'); + expect(report.ok).toBe(true); + }); + + it('still fails a gate that never passes for anyone', () => { + // A gate stuck at 0 for every model is broken or mis-specified, and unlike + // a satisfied gate it is never the good news it looks like. + const report = checkEvaluatorHealth({ + observations: observe('ShouldNotCallTool', Array(40).fill(0), 'gate'), + }); + + expect(report.findings[0].classification).toBe('gate-failing'); + expect(report.ok).toBe(false); + }); + + it('refuses to judge an evaluator with too few observations', () => { + const report = checkEvaluatorHealth({ + observations: observe('Rubric', [1, 1, 1]), + minObservations: 20, + }); + + expect(report.findings[0].classification).toBe('insufficient-data'); + // Not enough data is not the same as healthy; it must not silently pass. + expect(report.findings[0].distinctValues).toBe(1); + expect(report.ok).toBe(true); + expect(report.skipped).toBe(1); + }); + + it('separates composite-safe evaluators from the rest', () => { + const report = checkEvaluatorHealth({ + observations: [ + ...observe( + 'Factuality', + Array.from({ length: 40 }, (_, i) => i / 39) + ), + ...observe('ShouldNotCallTool', Array(40).fill(1), 'gate'), + ...observe('DeadWeight', Array(40).fill(1)), + ], + }); + + // Only graders that discriminate belong in a ranking composite; folding in + // gates and constants is what moved models by up to 14 places on the board. + expect(report.compositeSafe).toEqual(['Factuality']); + }); + + it('reports every evaluator, not just the failing ones', () => { + const report = checkEvaluatorHealth({ + observations: [ + ...observe( + 'Good', + Array.from({ length: 40 }, (_, i) => i / 39) + ), + ...observe('Bad', Array(40).fill(1)), + ], + }); + + expect(report.findings).toHaveLength(2); + expect(report.findings.map((f) => f.evaluatorName).sort()).toEqual(['Bad', 'Good']); + }); + + it('throws rather than reporting health for no observations at all', () => { + expect(() => checkEvaluatorHealth({ observations: [] })).toThrow(/at least one observation/); + }); + + it('treats a non-1 ceiling correctly', () => { + // A 0-5 rubric pinned at 5 is just as saturated as a 0-1 pinned at 1. + const report = checkEvaluatorHealth({ + observations: observe('Scale5', [...Array(38).fill(5), 4, 3]), + ceiling: 5, + }); + + expect(report.findings[0].classification).toBe('saturated'); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/evaluator_health.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/evaluator_health.ts new file mode 100644 index 0000000000000..46803ba4d0abe --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/evaluator_health.ts @@ -0,0 +1,222 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * An evaluator that gives every model the same score cannot rank anything, but + * it still costs eval time, occupies a board column, and -- worst -- gets folded + * into composite scores where it dilutes the evaluators that do discriminate. + * + * That is not hypothetical. An audit of the golden data found three evaluators + * returning a constant 1.000 across all 159 observations, and dropping the + * near-constant ones moved models by up to 14 rank places. The collapsed + * attack-discovery rubric was the same defect caught by hand, one suite at a + * time, after 240 observations had already been spent on it. + * + * This makes that check mechanical so it happens at run time rather than in a + * retrospective script. + * + * The one distinction that matters: a **gate** ("did it avoid the forbidden + * tool?") is *supposed* to sit at the ceiling -- that means models are behaving. + * Failing gates for being saturated would train people to ignore the warning. + * A gate is only broken when nobody passes it. + */ + +export type EvaluatorRole = 'gate' | 'grader'; + +export type EvaluatorClassification = + | 'constant' + | 'saturated' + | 'discriminating' + | 'gate-satisfied' + | 'gate-failing' + | 'insufficient-data'; + +export interface EvaluatorObservation { + evaluatorName: string; + modelId: string; + score: number; + /** Defaults to `grader`; gates are held to different expectations. */ + role?: EvaluatorRole; +} + +export interface EvaluatorHealthInput { + observations: EvaluatorObservation[]; + /** Highest score the evaluator can award. */ + ceiling?: number; + /** Below this many observations an evaluator is not judged at all. */ + minObservations?: number; + /** Share of observations at the ceiling above which a grader is saturated. */ + saturationThreshold?: number; +} + +export interface EvaluatorFinding { + evaluatorName: string; + role: EvaluatorRole; + classification: EvaluatorClassification; + observationCount: number; + distinctValues: number; + ceilingShare: number; + /** What to do about it, or why it is fine. */ + verdict: string; +} + +export interface EvaluatorHealthReport { + ok: boolean; + findings: EvaluatorFinding[]; + /** Evaluators safe to average into a ranking composite. */ + compositeSafe: string[]; + /** Evaluators not judged for lack of data. */ + skipped: number; +} + +const UNHEALTHY: ReadonlySet = new Set([ + 'constant', + 'saturated', + 'gate-failing', +]); + +export function checkEvaluatorHealth({ + observations, + ceiling = 1, + minObservations = 20, + saturationThreshold = 0.9, +}: EvaluatorHealthInput): EvaluatorHealthReport { + if (observations.length === 0) { + throw new Error( + 'checkEvaluatorHealth requires at least one observation; refusing to certify nothing as healthy.' + ); + } + + const byEvaluator = new Map(); + for (const observation of observations) { + const existing = byEvaluator.get(observation.evaluatorName); + if (existing) { + existing.push(observation); + } else { + byEvaluator.set(observation.evaluatorName, [observation]); + } + } + + const findings = [...byEvaluator.entries()].map(([evaluatorName, group]) => + classify({ evaluatorName, group, ceiling, minObservations, saturationThreshold }) + ); + + return { + ok: findings.every((f) => !UNHEALTHY.has(f.classification)), + findings, + compositeSafe: findings + .filter((f) => f.classification === 'discriminating') + .map((f) => f.evaluatorName), + skipped: findings.filter((f) => f.classification === 'insufficient-data').length, + }; +} + +function classify({ + evaluatorName, + group, + ceiling, + minObservations, + saturationThreshold, +}: { + evaluatorName: string; + group: EvaluatorObservation[]; + ceiling: number; + minObservations: number; + saturationThreshold: number; +}): EvaluatorFinding { + const role: EvaluatorRole = group[0].role ?? 'grader'; + const scores = group.map((o) => o.score); + const distinctValues = new Set(scores.map((s) => round(s))).size; + const atCeiling = scores.filter((s) => Math.abs(s - ceiling) <= 1e-9).length; + const ceilingShare = atCeiling / scores.length; + + const base = { + evaluatorName, + role, + observationCount: scores.length, + distinctValues, + ceilingShare, + }; + + if (scores.length < minObservations) { + return { + ...base, + classification: 'insufficient-data', + verdict: + `Only ${scores.length} observations (needs ${minObservations}). Not judged -- this is ` + + `absence of evidence, not a clean bill of health.`, + }; + } + + if (role === 'gate') { + // A gate at the ceiling is the outcome it was written to confirm. + if (ceilingShare >= saturationThreshold) { + return { + ...base, + classification: 'gate-satisfied', + verdict: + `Gate passes for ${pct( + ceilingShare + )} of runs, which is what a healthy gate looks like. ` + + `Keep it as a pass/fail badge and keep it out of ranking composites.`, + }; + } + if (atCeiling === 0) { + return { + ...base, + classification: 'gate-failing', + verdict: + `Gate never passes for any model. Either every model genuinely fails it, or the gate is ` + + `mis-specified -- unlike a satisfied gate, this is never the good news it resembles.`, + }; + } + return { + ...base, + classification: 'discriminating', + verdict: `Gate passes for ${pct(ceilingShare)} of runs and separates models.`, + }; + } + + if (distinctValues <= 1) { + return { + ...base, + classification: 'constant', + verdict: + `Returns the same score for all ${scores.length} observations. It cannot separate any pair ` + + `of models, so it contributes nothing to a ranking while still costing eval time. Fix the ` + + `evaluator or drop it.`, + }; + } + + if (ceilingShare >= saturationThreshold) { + return { + ...base, + classification: 'saturated', + verdict: + `${pct( + ceilingShare + )} of scores sit at the ${ceiling} ceiling across only ${distinctValues} ` + + `distinct values. Rejudging cannot separate these models; only a finer-grained rubric can.`, + }; + } + + return { + ...base, + classification: 'discriminating', + verdict: `${distinctValues} distinct values with ${pct( + ceilingShare + )} at the ceiling -- separates models.`, + }; +} + +function round(value: number): number { + return Math.round(value * 1e6) / 1e6; +} + +function pct(share: number): string { + return `${(share * 100).toFixed(1)}%`; +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/evaluator_roles.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/evaluator_roles.test.ts new file mode 100644 index 0000000000000..44b2564954662 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/evaluator_roles.test.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { resolveEvaluatorRole, EVALUATOR_ROLES, assertRolesDeclared } from './evaluator_roles'; + +describe('resolveEvaluatorRole', () => { + it('returns the declared role for a registered evaluator', () => { + expect(resolveEvaluatorRole('ShouldNotCallTool')).toBe('gate'); + expect(resolveEvaluatorRole('Factuality')).toBe('grader'); + }); + + it('reports an unregistered evaluator as unknown rather than guessing', () => { + // Defaulting an unknown evaluator to `grader` would fail it for saturation; + // defaulting to `gate` would excuse a dead grader. Both are worse than + // saying so. + expect(resolveEvaluatorRole('SomeNewEvaluator')).toBe('unknown'); + }); + + it('does not infer a role from the evaluator name', () => { + // The name-matching stopgap classified anything starting with "Should" as a + // gate. A registry must not inherit that behaviour: a name is not a + // declaration. + expect(resolveEvaluatorRole('ShouldSomethingNeverDeclared')).toBe('unknown'); + }); + + it('matches parameterised evaluator instances to their declared base name', () => { + // Golden records these per skill, e.g. "Skill Invoked (data-exploration)". + // They are the same evaluator and must not each need registering. + expect(resolveEvaluatorRole('Skill Invoked (data-exploration)')).toBe('gate'); + expect(resolveEvaluatorRole('Skill Invoked (attack-discovery-generator)')).toBe('gate'); + }); + + it('is case- and spacing-insensitive for the same evaluator', () => { + // Suites record both `SkillInvoked` and `Skill Invoked`. + expect(resolveEvaluatorRole('SkillInvoked')).toBe('gate'); + expect(resolveEvaluatorRole('skill invoked')).toBe('gate'); + }); + + it('declares a role for every evaluator seen in the golden extract', () => { + // Guards against a suite adding an evaluator that silently audits as + // unknown forever. + const seen = [ + 'Factuality', + 'Relevance', + 'Groundedness', + 'Criteria', + 'Rubric', + 'ShouldNotCallTool', + 'Skill Invoked', + 'ForbiddenTools', + ]; + expect(seen.filter((name) => resolveEvaluatorRole(name) === 'unknown')).toEqual([]); + }); + + it('keeps known-constant evaluators as graders so the audit keeps flagging them', () => { + // These two return a constant 1.000 across all 159 golden observations. + // Re-labelling them as gates would silence the audit rather than fix them, + // which is the cheapest possible way to make this tool useless. + expect(resolveEvaluatorRole('RequiredAlertIdsInResponse')).toBe('grader'); + expect(resolveEvaluatorRole('DocVersionReleaseDate')).toBe('grader'); + }); + + it('requires a rationale for every declaration', () => { + // The rationale is the claim a reviewer agrees to; a role without one is + // just the old regex with extra steps. The shortest legitimate rationale in + // the registry is 21 characters, so this threshold bites on an emptied or + // placeholder string without forcing prose. + const missing = EVALUATOR_ROLES.filter((entry) => entry.rationale.trim().length < 21); + expect(missing).toEqual([]); + }); + + it('never declares the same evaluator twice', () => { + const names = EVALUATOR_ROLES.map((entry) => entry.name.toLowerCase()); + expect(new Set(names).size).toBe(names.length); + }); + + it('fails loudly when asked to audit an undeclared evaluator', () => { + expect(() => assertRolesDeclared(['Factuality', 'MysteryEvaluator'])).toThrow( + /MysteryEvaluator/ + ); + }); + + it('passes the assertion when every evaluator is declared', () => { + expect(() => assertRolesDeclared(['Factuality', 'ShouldNotCallTool'])).not.toThrow(); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/evaluator_roles.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/evaluator_roles.ts new file mode 100644 index 0000000000000..73b3f6133a6c7 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/evaluator_roles.ts @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * What each evaluator is *for*, declared rather than guessed. + * + * The health audit holds gates and graders to opposite standards: a gate + * ("did it avoid the forbidden tool?") is supposed to sit at the ceiling, while + * a grader pinned at the ceiling has stopped discriminating. Getting that + * backwards either fails healthy gates or excuses dead graders. + * + * The first version inferred role from the evaluator's name. That is the wrong + * mechanism for a correctness check: a name list can be widened until the audit + * passes, which is precisely the failure the audit exists to catch. Worse, it + * silently classified `RequiredAlertIdsInResponse` and `DocVersionReleaseDate` + * as gates -- so the audit stopped asking whether those two had ever once + * failed, when in fact both return a constant 1.000 across all 159 golden + * observations. + * + * A declaration is auditable in review; a regex is not. + */ + +export type DeclaredRole = 'gate' | 'grader'; + +export interface EvaluatorRoleDeclaration { + /** Base evaluator name as recorded in golden, without any `(instance)` suffix. */ + name: string; + role: DeclaredRole; + /** Why it has this role -- the claim a reviewer is agreeing to. */ + rationale: string; +} + +export const EVALUATOR_ROLES: readonly EvaluatorRoleDeclaration[] = [ + { + name: 'ShouldNotCallTool', + role: 'gate', + rationale: 'Asserts a forbidden tool was never called; passing for everyone is the goal.', + }, + { + name: 'ForbiddenTools', + role: 'gate', + rationale: 'Same contract as ShouldNotCallTool, recorded under a second name.', + }, + { + name: 'SkillInvoked', + role: 'gate', + rationale: + 'Asserts the expected skill ran at all; quality is graded elsewhere. normalize() also covers the spaced spelling `Skill Invoked` and per-skill instances.', + }, + { + name: 'ExpectedSkillInvocation', + role: 'gate', + rationale: 'Asserts the expected skill ran.', + }, + { + name: 'ExpectedToolCalled', + role: 'gate', + rationale: 'Asserts a required tool was called.', + }, + { + name: 'FinalAnswerPresent', + role: 'gate', + rationale: 'Asserts the agent produced a final message; not a quality signal.', + }, + { + name: 'MinExpectedSteps', + role: 'gate', + rationale: 'Asserts the trajectory reached a minimum length.', + }, + { + name: 'WorkflowEvidence', + role: 'gate', + rationale: 'Asserts the workflow stages appear in the trajectory.', + }, + { + name: 'ToolUsageOnly', + role: 'gate', + rationale: 'Asserts the agent used tools rather than answering from memory.', + }, + // Graders: these are supposed to spread models out. If one pins at the + // ceiling it has stopped measuring, and the audit should say so. + { name: 'Factuality', role: 'grader', rationale: 'Scores factual accuracy against references.' }, + { name: 'Relevance', role: 'grader', rationale: 'Scores answer relevance.' }, + { name: 'Groundedness', role: 'grader', rationale: 'Scores grounding in retrieved evidence.' }, + { name: 'Criteria', role: 'grader', rationale: 'Per-item rubric criteria, judged.' }, + { name: 'Rubric', role: 'grader', rationale: 'Rubric score, judged.' }, + { + name: 'Trajectory', + role: 'grader', + rationale: 'Scores trajectory quality, not mere presence.', + }, + { name: 'StrictTrajectory', role: 'grader', rationale: 'Stricter trajectory scoring.' }, + { name: 'Sequence Accuracy', role: 'grader', rationale: 'Scores tool-call ordering accuracy.' }, + { + name: 'RequiredTermsInResponse', + role: 'grader', + rationale: 'Scores how many required terms appear; partial credit is meaningful.', + }, + { + name: 'AttackDiscoveryBasic', + role: 'grader', + rationale: 'Scores discovery quality against expectations.', + }, + { name: 'AdToolResult', role: 'grader', rationale: 'Scores the attack-discovery tool result.' }, + { name: 'CostPerAlert', role: 'grader', rationale: 'Scores cost efficiency per alert.' }, + { + name: 'RequiredAlertIdsInResponse', + role: 'grader', + rationale: + 'Scores whether the required alert ids came back. Deliberately NOT a gate: it returns a ' + + 'constant 1.000 across all 159 golden observations, and calling it a gate would stop the ' + + 'audit asking whether it can ever fail.', + }, + { + name: 'DocVersionReleaseDate', + role: 'grader', + rationale: + 'Scores version/date correctness. Also constant 1.000 across 159 observations; kept a ' + + 'grader so the audit keeps flagging it.', + }, +]; + +const BY_NAME = new Map( + EVALUATOR_ROLES.map((entry) => [normalize(entry.name), entry.role]) +); + +/** + * Golden records parameterised instances such as `Skill Invoked (data-exploration)`. + * They share the base evaluator's contract, so the suffix is dropped before lookup. + */ +function normalize(name: string): string { + return name + .replace(/\s*\([^)]*\)\s*$/, '') + .replace(/\s+/g, '') + .toLowerCase(); +} + +/** + * `unknown` is a real answer. Guessing a role for an unregistered evaluator + * either fails a healthy gate or excuses a dead grader, so the audit reports + * the gap instead of inventing a classification. + */ +export function resolveEvaluatorRole(evaluatorName: string): DeclaredRole | 'unknown' { + return BY_NAME.get(normalize(evaluatorName)) ?? 'unknown'; +} + +/** + * Fails when a suite has added an evaluator nobody has classified, so new + * evaluators cannot drift into the audit unexamined. + */ +export function assertRolesDeclared(evaluatorNames: readonly string[]): void { + const undeclared = [...new Set(evaluatorNames)].filter( + (name) => resolveEvaluatorRole(name) === 'unknown' + ); + if (undeclared.length > 0) { + throw new Error( + `Undeclared evaluator role(s): ${undeclared.join( + ', ' + )}. Add each to EVALUATOR_ROLES with a ` + + `rationale -- the audit holds gates and graders to opposite standards and must not guess.` + ); + } +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/evaluator_saturation.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/evaluator_saturation.test.ts new file mode 100644 index 0000000000000..7d483c929fb0f --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/evaluator_saturation.test.ts @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + detectSaturatedEvaluators, + saturatedEvaluatorNames, + DEFAULT_SATURATION_POLICY, +} from './evaluator_saturation'; +import type { AggregatedModelScores } from './query_matrix_scores'; + +const model = ( + modelId: string, + evaluators: Array<{ evaluatorName: string; mean: number; count?: number }> +): AggregatedModelScores => ({ + modelId, + suites: [ + { + suiteId: 'security-persona-matrix', + experimentId: `exp-${modelId}`, + datasets: [ + { + datasetId: 'prefix:alert-analysis-a', + datasetName: 'alert-analysis-a', + evaluators: evaluators.map((e) => ({ + evaluatorName: e.evaluatorName, + mean: e.mean, + count: e.count ?? 10, + })), + }, + ], + }, + ], +}); + +/** 12 models so the default minObservations (8) is satisfied. */ +const buildModels = (valuesFor: (i: number) => Array<{ evaluatorName: string; mean: number }>) => + Array.from({ length: 12 }, (_, i) => model(`m${i}`, valuesFor(i))); + +describe('detectSaturatedEvaluators', () => { + it('flags an evaluator that returns the ceiling for every model', () => { + const models = buildModels(() => [{ evaluatorName: 'FinalAnswerPresent', mean: 1 }]); + + const [result] = detectSaturatedEvaluators(models); + + expect(result.evaluatorName).toBe('FinalAnswerPresent'); + expect(result.saturated).toBe(true); + expect(result.range).toBe(0); + expect(result.distinctValues).toBe(1); + }); + + it('does not flag an evaluator that separates models', () => { + const models = buildModels((i) => [{ evaluatorName: 'Factuality', mean: i / 11 }]); + + const [result] = detectSaturatedEvaluators(models); + + expect(result.saturated).toBe(false); + expect(result.distinctValues).toBe(12); + }); + + it('treats an all-zero evaluator as failing, not saturated', () => { + // Every observation identical at ZERO is a broken evaluator, not a + // precondition check. Excluding it would hide the breakage. + const models = buildModels(() => [{ evaluatorName: 'PanelCountPreservation', mean: 0 }]); + + const [result] = detectSaturatedEvaluators(models); + + expect(result.saturated).toBe(false); + }); + + it('refuses a verdict below the observation floor', () => { + const models = Array.from({ length: 3 }, (_, i) => + model(`m${i}`, [{ evaluatorName: 'Rubric', mean: 1 }]) + ); + + const [result] = detectSaturatedEvaluators(models); + + expect(result.observations).toBe(3); + expect(result.saturated).toBe(false); + }); + + it('normalizes by observed max so raw-magnitude evaluators are not judged against 1.0', () => { + // Latency in seconds: identical for every model, so it IS saturated even + // though the values are nowhere near 1.0. + const models = buildModels(() => [{ evaluatorName: 'Latency', mean: 70.4 }]); + + const [result] = detectSaturatedEvaluators(models); + + expect(result.saturated).toBe(true); + expect(result.mean).toBe(1); + }); + + it('collapses multiple datasets to one observation per model', () => { + // Same evaluator across 3 datasets for a single model must count once, + // otherwise a widely-evaluated model outvotes the rest. + const multi: AggregatedModelScores = { + modelId: 'busy', + suites: [ + { + suiteId: 's', + experimentId: 'e', + datasets: ['a', 'b', 'c'].map((id) => ({ + datasetId: id, + datasetName: id, + evaluators: [{ evaluatorName: 'criteria', mean: 1, count: 10 }], + })), + }, + ], + }; + + const [result] = detectSaturatedEvaluators([multi]); + + expect(result.observations).toBe(1); + }); + + it('respects a stricter policy', () => { + const models = buildModels((i) => [ + { evaluatorName: 'MinExpectedSteps', mean: i === 0 ? 0.8 : 1 }, + ]); + + expect(detectSaturatedEvaluators(models)[0].saturated).toBe(true); + expect( + detectSaturatedEvaluators(models, { ...DEFAULT_SATURATION_POLICY, maxRange: 0.1 })[0] + .saturated + ).toBe(false); + }); + + it('flags a high-but-never-maxed evaluator that still cannot rank', () => { + // The real shape from the golden artifact: Groundedness means cluster + // 0.88-0.90 across every model. No cell is at 1.0, yet it ranks nothing. + const models = buildModels((i) => [ + { evaluatorName: 'Groundedness', mean: 0.88 + (i % 3) * 0.01 }, + ]); + + const [result] = detectSaturatedEvaluators(models); + + expect(result.saturated).toBe(true); + expect(result.range).toBeLessThan(0.05); + }); + + it('keeps a low-mean tightly-clustered evaluator in the aggregate', () => { + // Hard evaluators may cluster too, but a low mean means models are + // genuinely failing -- that is signal, not saturation. + const models = buildModels(() => [{ evaluatorName: 'Factuality', mean: 0.35 }]); + + expect(detectSaturatedEvaluators(models)[0].saturated).toBe(false); + }); + + it('exposes saturated names as a set for filtering', () => { + const models = buildModels((i) => [ + { evaluatorName: 'SkillInvoked', mean: 1 }, + { evaluatorName: 'Factuality', mean: i / 11 }, + ]); + + const names = saturatedEvaluatorNames(detectSaturatedEvaluators(models)); + + expect(names.has('SkillInvoked')).toBe(true); + expect(names.has('Factuality')).toBe(false); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/evaluator_saturation.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/evaluator_saturation.ts new file mode 100644 index 0000000000000..25a8419c9c905 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/evaluator_saturation.ts @@ -0,0 +1,171 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AggregatedModelScores } from './query_matrix_scores'; + +/** + * An evaluator that returns the same value for every model cannot rank models. + * + * Measured on the 2026-09-01 golden artifact (20 models x 24 columns): five + * evaluators sat at or above 0.93 mean with >=90% of cells at the ceiling + * (MinExpectedSteps 0.970, SkillInvoked 0.980, Sequence Accuracy 0.930, + * FinalAnswerPresent 0.883, criteria 0.868). Averaged into a cell alongside the + * evaluators that DO move (Factuality 0.352, ExpectedToolCalled 0.387, + * Trajectory 0.610), each saturated evaluator takes an equal share of the mean + * and divides the real differences by the evaluator count -- compressing the + * published leaderboard range to 1.43 points and making every adjacent pair a + * statistical tie. + * + * A saturated evaluator is a precondition check wearing a score's clothing: + * "the agent produced a final answer" is a gate, not a measure of quality. It + * belongs in a setup assertion, or its cases need to get harder until it can + * fail. Until then, keeping it out of the aggregate is what lets the metrics + * that carry signal actually show up. + */ +export interface EvaluatorSaturation { + evaluatorName: string; + /** Mean across models, normalized by the observed maximum (0-1). */ + mean: number; + /** Population stdev of the normalized model means -- the ranking signal. */ + stdev: number; + /** max - min of the normalized model means. */ + range: number; + /** Distinct model-level means observed -- 1 means literally no discrimination. */ + distinctValues: number; + /** Number of model-level observations backing the verdict. */ + observations: number; + saturated: boolean; +} + +export interface SaturationPolicy { + /** Minimum normalized mean to qualify: only high scorers can be "at ceiling". */ + minMean: number; + /** Maximum normalized spread (max-min) to qualify as non-discriminating. */ + maxRange: number; + /** Below this many observations the verdict is not trustworthy. */ + minObservations: number; +} + +/** + * Thresholds derived from the 2026-09-01 golden artifact, where the gap is + * unambiguous: the five non-ranking evaluators span 0.095-0.198 normalized + * range, and the next evaluator up (criteria) jumps to 0.298. 0.25 sits in + * that gap. minMean 0.85 keeps genuinely-hard evaluators (Factuality 0.626, + * ExpectedToolCalled 0.430) in the aggregate no matter how tightly they cluster. + */ +export const DEFAULT_SATURATION_POLICY: SaturationPolicy = { + minMean: 0.85, + maxRange: 0.25, + minObservations: 8, +}; + +const EPSILON = 1e-9; + +/** + * Collects every model-level mean per evaluator. Deliberately uses the + * per-model aggregate rather than raw score docs: the question is "can this + * evaluator separate MODELS", and a metric can vary across examples while + * still landing every model on the same number. + */ +const collectByEvaluator = (models: readonly AggregatedModelScores[]): Map => { + const byEvaluator = new Map(); + for (const model of models) { + // Per-model weighted mean first: a model evaluated on 20 datasets must not + // outvote a model evaluated on 2 when deciding whether the EVALUATOR can + // separate models. Collapsing to one observation per model keeps the + // verdict about ranking power rather than about dataset coverage. + const perModel = new Map(); + for (const suite of model.suites ?? []) { + for (const dataset of suite.datasets ?? []) { + for (const evaluator of dataset.evaluators ?? []) { + if (!Number.isFinite(evaluator.mean)) { + continue; + } + const weight = evaluator.count > 0 ? evaluator.count : 1; + const acc = perModel.get(evaluator.evaluatorName) ?? { weightedSum: 0, weight: 0 }; + acc.weightedSum += evaluator.mean * weight; + acc.weight += weight; + perModel.set(evaluator.evaluatorName, acc); + } + } + } + for (const [evaluatorName, acc] of perModel) { + if (acc.weight === 0) { + continue; + } + const bucket = byEvaluator.get(evaluatorName); + if (bucket) { + bucket.push(acc.weightedSum / acc.weight); + } else { + byEvaluator.set(evaluatorName, [acc.weightedSum / acc.weight]); + } + } + } + return byEvaluator; +}; + +/** + * Classifies each evaluator as saturated or discriminating. + * + * Normalizes by the observed maximum rather than assuming a 0-1 range: raw + * magnitude evaluators (Latency, token counts) live on their own scales and + * would otherwise be judged against a ceiling they never approach. + */ +export const detectSaturatedEvaluators = ( + models: readonly AggregatedModelScores[], + policy: SaturationPolicy = DEFAULT_SATURATION_POLICY +): EvaluatorSaturation[] => { + const results: EvaluatorSaturation[] = []; + + for (const [evaluatorName, entries] of collectByEvaluator(models)) { + const values = entries.filter((v) => Number.isFinite(v)); + if (values.length === 0) { + continue; + } + + const max = Math.max(...values); + const observations = values.length; + // Quality evaluators are already 0-1, so their ceiling is a fixed 1.0 -- + // normalizing by the observed max would rescale a uniformly-LOW evaluator + // (every model 0.35) up to 1.0 and misread "consistently hard" as + // "saturated". Only raw-magnitude evaluators (Latency in seconds, token + // counts) exceed 1 and need their own scale to be comparable at all. + const scaleBase = Math.max(1, max); + const normalized = values.map((v) => v / scaleBase); + const mean = normalized.reduce((sum, v) => sum + v, 0) / observations; + const variance = normalized.reduce((sum, v) => sum + (v - mean) ** 2, 0) / observations; + const stdev = Math.sqrt(variance); + const range = Math.max(...normalized) - Math.min(...normalized); + const distinctValues = new Set(values.map((v) => v.toFixed(6))).size; + + // Saturation is about SPREAD at a high level, not about sitting at exactly + // 1.0. An evaluator scoring every model 0.88-0.90 ranks nothing, even + // though no single observation is at the ceiling. A tight cluster at a LOW + // mean is a hard evaluator -- real signal -- and stays in. + const saturated = + max > EPSILON && + observations >= policy.minObservations && + mean >= policy.minMean && + range <= policy.maxRange; + + results.push({ + evaluatorName, + mean, + stdev, + range, + distinctValues, + observations, + saturated, + }); + } + + return results.sort((a, b) => b.mean - a.mean || a.evaluatorName.localeCompare(b.evaluatorName)); +}; + +/** Names of evaluators the policy judges saturated. */ +export const saturatedEvaluatorNames = (saturation: readonly EvaluatorSaturation[]): Set => + new Set(saturation.filter((entry) => entry.saturated).map((entry) => entry.evaluatorName)); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/fetch_score_docs.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/fetch_score_docs.test.ts new file mode 100644 index 0000000000000..2be6348d170d9 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/fetch_score_docs.test.ts @@ -0,0 +1,201 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { fetchScoreDocs } from './fetch_score_docs'; + +describe('fetchScoreDocs', () => { + const esUrl = 'https://es.example.com'; + const apiKey = 'key'; + const okResponse = (hits: unknown[]) => ({ + ok: true, + json: async () => ({ hits: { hits } }), + }); + + let fetchMock: jest.Mock; + + beforeEach(() => { + fetchMock = jest.fn().mockResolvedValue(okResponse([])); + global.fetch = fetchMock as unknown as typeof global.fetch; + }); + + const body = (call: number) => JSON.parse(fetchMock.mock.calls[call][1].body); + + it('refuses an unscoped read rather than scanning every archived run', async () => { + // 25 config model ids match 136 executions on the golden cluster: an + // unscoped query is ~56 minutes of transfer and never what the caller meant. + await expect( + fetchScoreDocs({ esUrl, apiKey, exampleIds: ['a'], suiteIds: ['s'] }) + ).rejects.toThrow(/requires --execution-id or --models/); + + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('accepts an execution scope', async () => { + await fetchScoreDocs({ esUrl, apiKey, exampleIds: ['a'], executionIds: ['e1'] }); + + expect(body(0).query.bool.filter).toContainEqual({ + terms: { 'metadata.execution_id': ['e1'] }, + }); + }); + + it('filters on example.id by default', async () => { + await fetchScoreDocs({ esUrl, apiKey, exampleIds: ['a'], executionIds: ['e1'] }); + + expect(body(0).query.bool.filter).toContainEqual({ term: { 'example.id': 'a' } }); + }); + + // attack-discovery stores example.id = '0' on every document, so filtering + // that field there returns one scenario and silently drops the other eight. + it('filters on the caller-supplied join field instead of example.id', async () => { + await fetchScoreDocs({ + esUrl, + apiKey, + exampleIds: ['wmi-lateral'], + joinField: 'example.metadata.scenarioKey', + executionIds: ['e1'], + }); + + const filters = body(0).query.bool.filter; + expect(filters).toContainEqual({ term: { 'example.metadata.scenarioKey': 'wmi-lateral' } }); + expect(filters).not.toContainEqual({ term: { 'example.id': 'wmi-lateral' } }); + }); + + it('falls back to the config model list when no explicit models are given', async () => { + await fetchScoreDocs({ esUrl, apiKey, exampleIds: ['a'], configModelIds: ['m1'] }); + + expect(body(0).query.bool.filter).toContainEqual({ terms: { 'task.model.id': ['m1'] } }); + }); + + it('prefers explicit models over the config list', async () => { + await fetchScoreDocs({ + esUrl, + apiKey, + exampleIds: ['a'], + modelIds: ['chosen'], + configModelIds: ['ignored'], + }); + + expect(body(0).query.bool.filter).toContainEqual({ terms: { 'task.model.id': ['chosen'] } }); + }); + + it('collapses to one execution group and pulls its members', async () => { + // A trajectory is stored once per evaluator (~15 docs); collapsing is what + // turns a 42k-document read into a ~2.8k-trajectory read. The members let + // the caller prefer a payload-bearing document over an empty sibling. + await fetchScoreDocs({ esUrl, apiKey, exampleIds: ['a'], executionIds: ['e1'] }); + + expect(body(0).collapse).toEqual({ + field: 'metadata.execution_id', + inner_hits: { name: 'members', size: 50, _source: expect.any(Array) }, + }); + }); + + it('resolves the newest execution per model instead of trusting collapse order', async () => { + // A suite re-runs an example across many executions; the newest is the one + // whose trajectories were captured. The first call per example is the + // resolution aggregation, the second the document fetch. + fetchMock + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ + aggregations: { + by_model: { + buckets: [ + { + key: 'm1', + latest: { + hits: { hits: [{ _source: { metadata: { execution_id: 'new-exec' } } }] }, + }, + }, + ], + }, + }, + }), + }) + .mockResolvedValueOnce(okResponse([{ _source: { example: { id: 'a' } } }])); + + await fetchScoreDocs({ esUrl, apiKey, exampleIds: ['a'], modelIds: ['m1'] }); + + // Resolution query: newest document per model bucket. + expect(body(0).aggs.by_model.aggs.latest.top_hits.sort).toEqual([ + { '@timestamp': { order: 'desc' } }, + ]); + // Document fetch: scoped to the resolved execution. + expect(body(1).query.bool.filter).toContainEqual({ + terms: { 'metadata.execution_id': ['new-exec'] }, + }); + }); + + it('prefers a payload-bearing group member over an empty representative', async () => { + const empty = { _source: { task: { output: {} } } }; + const withInsights = { + _source: { task: { output: { insights: [{ title: 'x' }] } }, example: { id: 'a' } }, + }; + + fetchMock.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + hits: { + hits: [ + { + _source: empty._source, + inner_hits: { members: { hits: { hits: [empty, withInsights] } } }, + }, + ], + }, + }), + }); + + const docs = await fetchScoreDocs({ + esUrl, + apiKey, + exampleIds: ['a'], + executionIds: ['e1'], + }); + + expect(docs).toHaveLength(1); + expect((docs[0] as any).task.output.insights).toHaveLength(1); + }); + + it('queries once per example and returns every hit', async () => { + fetchMock + .mockResolvedValueOnce(okResponse([{ _source: { example: { id: 'a' } } }])) + .mockResolvedValueOnce(okResponse([{ _source: { example: { id: 'b' } } }])); + + const docs = await fetchScoreDocs({ + esUrl, + apiKey, + exampleIds: ['a', 'b'], + executionIds: ['e1'], + }); + + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(docs).toHaveLength(2); + }); + + it('applies the as-of cutoff so replays match the published selection', async () => { + await fetchScoreDocs({ + esUrl, + apiKey, + exampleIds: ['a'], + executionIds: ['e1'], + asOf: Date.parse('2026-09-01T00:00:00.000Z'), + }); + + expect(body(0).query.bool.filter).toContainEqual({ + range: { '@timestamp': { lt: '2026-09-01T00:00:00.000Z' } }, + }); + }); + + it('surfaces a failed query instead of returning an empty result', async () => { + fetchMock.mockResolvedValue({ ok: false, status: 403, text: async () => 'forbidden' }); + + await expect( + fetchScoreDocs({ esUrl, apiKey, exampleIds: ['a'], executionIds: ['e1'] }) + ).rejects.toThrow(/403 forbidden/); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/fetch_score_docs.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/fetch_score_docs.ts new file mode 100644 index 0000000000000..d245d2eb0d27e --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/fetch_score_docs.ts @@ -0,0 +1,312 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DEFAULT_JOIN_FIELD } from './reference_adapters'; + +/** + * The subset of a golden score document the replay path reads. Only the fields + * a judge needs are modelled; everything else on the doc is ignored. + */ +export interface RawScoreDoc { + metadata?: { execution_id?: string; suite_id?: string }; + task?: { + model?: { id?: string }; + output?: { messages?: Array<{ message?: { content?: unknown } }> }; + }; + example?: { + id?: string; + input?: { question?: string }; + metadata?: Record; + }; +} + +const SCORES_INDEX = '.ds-.evaluation-scores*'; +// One collapsed hit per execution. The published matrix spans ~20 models, and +// an unscoped suite query matches ~155 historical executions whose task.output +// payloads total ~7 MB per example (~56 min for 21 examples). Callers scope by +// execution id or model, so this only has to bound a pathological request. +const PAGE_SIZE = 200; +// Evaluator count per (execution, example) group. A suite writes one document +// per evaluator (~15 today); 50 leaves headroom without pulling whole runs. +const MEMBERS_PER_EXECUTION = 50; + +const SOURCE_FIELDS = [ + 'metadata.execution_id', + 'metadata.suite_id', + 'task.model.id', + 'task.output', + 'example.id', + 'example.metadata', + 'example.input', +]; + +/** + * True when a document carries a payload some jury can grade. + * + * Kept deliberately structural rather than suite-aware: the fetch layer only + * has to prefer a document with content over an empty sibling, and the jury + * decides whether that content is actually gradable. + */ +const hasGradablePayload = (doc: RawScoreDoc): boolean => { + const output = (doc as { task?: { output?: Record } }).task?.output; + if (!output) { + return false; + } + const messages = output.messages; + const insights = output.insights; + return ( + (Array.isArray(messages) && messages.length > 0) || + (Array.isArray(insights) && insights.length > 0) + ); +}; + +/** + * Reads score documents directly from Elasticsearch. + * + * The golden cluster archives evaluation scores but does not run the evals + * plugin, so `/internal/evals/*` answers 400 there. Replay therefore talks to + * the index the sweeps write to. `task.output` is in `_source` but is NOT + * indexed, so it can be read but never filtered on. + */ +export const fetchScoreDocs = async ({ + esUrl, + apiKey, + exampleIds, + joinField = DEFAULT_JOIN_FIELD, + executionIds, + modelIds, + configModelIds, + suiteIds, + asOf, +}: { + esUrl: string; + apiKey: string; + exampleIds: string[]; + /** + * Golden field the reference keys correspond to. attack-discovery stores + * `example.id = '0'` on every document, so filtering that field there returns + * one scenario's documents and silently drops the other eight. + */ + joinField?: string; + executionIds?: string[]; + modelIds?: string[]; + configModelIds?: string[]; + suiteIds?: string[]; + asOf?: number; +}): Promise => { + const filter: unknown[] = []; + + // Without a suite filter the query scans every suite ever archived (~2M docs + // on the golden cluster) and never returns. + if (suiteIds?.length) { + filter.push({ terms: { 'metadata.suite_id': suiteIds } }); + } + + if (asOf) { + filter.push({ range: { '@timestamp': { lt: new Date(asOf).toISOString() } } }); + } + + if (executionIds?.length) { + filter.push({ terms: { 'metadata.execution_id': executionIds } }); + } + + const models = modelIds?.length ? modelIds : configModelIds; + if (models?.length) { + filter.push({ terms: { 'task.model.id': models } }); + } + + // Refuse an unbounded archive read. Without a model or execution scope this + // matches every historical run of the suite, which is minutes of transfer and + // almost never what the caller meant. + if (!executionIds?.length && !models?.length) { + throw new Error( + 'fetchScoreDocs requires --execution-id or --models (or a config model list): ' + + 'an unscoped query reads every archived run of the suite.' + ); + } + + // A suite re-runs the same example across many executions, and golden keeps + // every one. Collapsing documents by execution returns whichever run the + // first page happened to hit, which is frequently an archived run that + // pre-dates trajectory capture -- so newer, gradable runs never surface. + // Resolve the newest execution per (model, example) first, then fetch only + // those executions. + const effectiveExecutionIds = + executionIds?.length && executionIds.length > 0 + ? executionIds + : await resolveLatestExecutions({ + esUrl, + apiKey, + exampleIds, + joinField, + modelIds: models, + suiteIds, + }); + + if (effectiveExecutionIds.length === 0) { + return []; + } + + filter.push({ terms: { 'metadata.execution_id': effectiveExecutionIds } }); + + // One request per example, collapsed to one document per execution. + // + // A trajectory is stored once per evaluator (~15 docs), so scrolling the raw + // index reads ~42k documents to recover ~2.8k trajectories and takes minutes. + // `collapse` makes Elasticsearch return the single representative document + // the judge needs, which is both correct and ~100x cheaper. + const docs: RawScoreDoc[] = []; + + for (const exampleId of exampleIds) { + const response = await fetch(`${esUrl.replace(/\/$/, '')}/${SCORES_INDEX}/_search`, { + method: 'POST', + headers: { + Authorization: `ApiKey ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + size: PAGE_SIZE, + track_total_hits: false, + query: { bool: { filter: [...filter, { term: { [joinField]: exampleId } }] } }, + // A trajectory is written once per evaluator, and only some of those + // documents carry the graded payload (`task.output.messages` for prose + // suites, `task.output.insights` for attack discovery). `collapse` + // alone returns an arbitrary member of the group, so it frequently + // picked a payload-less document and the cell looked unreplayable. + // Pull a window of the group and choose a gradable member below. + collapse: { + field: 'metadata.execution_id', + inner_hits: { + name: 'members', + size: MEMBERS_PER_EXECUTION, + _source: SOURCE_FIELDS, + }, + }, + _source: SOURCE_FIELDS, + }), + }); + + if (!response.ok) { + throw new Error(`Score query failed: ${response.status} ${await response.text()}`); + } + + const parsed = (await response.json()) as { + hits?: { + hits?: Array<{ + _source?: RawScoreDoc; + inner_hits?: { members?: { hits?: { hits?: Array<{ _source?: RawScoreDoc }> } } }; + }>; + }; + }; + + for (const hit of parsed.hits?.hits ?? []) { + // Prefer a group member that actually carries the graded payload; fall + // back to the collapse representative so behaviour is unchanged for + // groups where no member has one. + const members = (hit.inner_hits?.members?.hits?.hits ?? []) + .map((member) => member._source) + .filter((source): source is RawScoreDoc => Boolean(source)); + + const chosen = members.find(hasGradablePayload) ?? hit._source; + if (chosen) { + docs.push(chosen); + } + } + } + + return docs; +}; + +/** + * Maps each (model, example) to its newest execution. + * + * Runs the per-example aggregation one example at a time, mirroring the + * document fetch, so a wide example set costs no more than the fetch that + * follows it. + */ +async function resolveLatestExecutions({ + esUrl, + apiKey, + exampleIds, + joinField = DEFAULT_JOIN_FIELD, + modelIds, + suiteIds, +}: { + esUrl: string; + apiKey: string; + exampleIds: string[]; + joinField?: string; + modelIds?: string[]; + suiteIds?: string[]; +}): Promise { + const indexUrl = `${esUrl.replace(/\/$/, '')}/${SCORES_INDEX}/_search`; + const chosen = new Set(); + + const filter: unknown[] = []; + if (suiteIds?.length) { + filter.push({ terms: { 'metadata.suite_id': suiteIds } }); + } + if (modelIds?.length) { + filter.push({ terms: { 'task.model.id': modelIds } }); + } + + for (const exampleId of exampleIds) { + const response = await fetch(indexUrl, { + method: 'POST', + headers: { + Authorization: `ApiKey ${apiKey}`, + 'Content-Type': 'application/json', + 'x-fleet-interaction': 'true', + }, + body: JSON.stringify({ + size: 0, + query: { bool: { filter: [...filter, { term: { [joinField]: exampleId } }] } }, + aggs: { + by_model: { + terms: { field: 'task.model.id', size: 50 }, + aggs: { + latest: { + top_hits: { + size: 1, + sort: [{ '@timestamp': { order: 'desc' } }], + _source: ['metadata.execution_id'], + }, + }, + }, + }, + }, + }), + }); + + if (!response.ok) { + throw new Error( + `resolveLatestExecutions failed for example ${exampleId}: ${response.status} ${response.statusText}` + ); + } + + const parsed = (await response.json()) as { + aggregations?: { + by_model?: { + buckets?: Array<{ + latest?: { + hits?: { hits?: Array<{ _source?: { metadata?: { execution_id?: string } } }> }; + }; + }>; + }; + }; + }; + + for (const bucket of parsed.aggregations?.by_model?.buckets ?? []) { + const executionId = bucket.latest?.hits?.hits?.[0]?._source?.metadata?.execution_id; + if (executionId) { + chosen.add(executionId); + } + } + } + + return [...chosen]; +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_agreement.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_agreement.test.ts new file mode 100644 index 0000000000000..00e2433e65cc4 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_agreement.test.ts @@ -0,0 +1,188 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { judgeAgreementForModel, type JudgeVerdict } from './judge_agreement'; + +const verdict = ( + judgeId: string, + example: string, + repetition: number, + evaluator: string, + score: number, + modelId = 'model-a' +): JudgeVerdict => ({ modelId, judgeId, example, repetition, evaluator, score }); + +describe('judgeAgreementForModel', () => { + it('reports unmeasured when the model has no verdicts', () => { + expect(judgeAgreementForModel([], 'model-a')).toMatchObject({ + status: 'unmeasured', + pairs: 0, + }); + }); + + it('reports single-judge rather than 100% when only one judge scored', () => { + const verdicts = [ + verdict('gemini', 'ex-1', 0, 'Relevance', 1), + verdict('gemini', 'ex-2', 0, 'Relevance', 1), + ]; + const row = judgeAgreementForModel(verdicts, 'model-a'); + // The critical case: absence of a second opinion must not look like + // perfect agreement. + expect(row.status).toBe('single-judge'); + expect(row.verdictAgreement).toBeUndefined(); + expect(row.interval).toBeUndefined(); + }); + + it('reports single-judge when two judges scored disjoint work', () => { + const verdicts = [ + verdict('gemini', 'ex-1', 0, 'Relevance', 1), + verdict('sonnet', 'ex-2', 0, 'Relevance', 1), + ]; + const row = judgeAgreementForModel(verdicts, 'model-a'); + expect(row.status).toBe('single-judge'); + expect(row.pairs).toBe(0); + }); + + it('pairs only identical example+rep+evaluator cells', () => { + const verdicts = [ + verdict('gemini', 'ex-1', 0, 'Relevance', 1), + verdict('sonnet', 'ex-1', 0, 'Relevance', 1), + // same example, different repetition -> not a pair + verdict('sonnet', 'ex-1', 1, 'Relevance', 0), + // same example+rep, different evaluator -> not a pair + verdict('sonnet', 'ex-1', 0, 'Factuality', 0), + ]; + const row = judgeAgreementForModel(verdicts, 'model-a'); + expect(row.pairs).toBe(1); + expect(row.verdictAgreement).toBe(1); + }); + + it('reports how many cells only one judge scored', () => { + // The 4.8-opus shape: the second judge returned `unavailable` for work the + // first judge did score. Those cells cannot be compared, and the row must + // say so rather than presenting a thinner sample as a full one. + const verdicts = [ + verdict('gemini', 'ex-1', 0, 'Relevance', 1), + verdict('sonnet', 'ex-1', 0, 'Relevance', 1), + verdict('sonnet', 'ex-2', 0, 'Relevance', 1), + verdict('sonnet', 'ex-3', 0, 'Relevance', 0), + ]; + const row = judgeAgreementForModel(verdicts, 'model-a'); + expect(row.pairs).toBe(1); + expect(row.unpaired).toBe(2); + }); + + it('reports no unpaired cells when both judges scored identical work', () => { + const verdicts = [ + verdict('gemini', 'ex-1', 0, 'Relevance', 1), + verdict('sonnet', 'ex-1', 0, 'Relevance', 1), + verdict('gemini', 'ex-2', 0, 'Relevance', 0), + verdict('sonnet', 'ex-2', 0, 'Relevance', 0), + ]; + const row = judgeAgreementForModel(verdicts, 'model-a'); + expect(row.pairs).toBe(2); + expect(row.unpaired).toBe(0); + }); + + it('counts a pass/fail flip across the 0.5 midpoint', () => { + const verdicts = [ + verdict('gemini', 'ex-1', 0, 'Relevance', 0.9), + verdict('sonnet', 'ex-1', 0, 'Relevance', 0.1), + verdict('gemini', 'ex-2', 0, 'Relevance', 0.9), + verdict('sonnet', 'ex-2', 0, 'Relevance', 0.8), + ]; + const row = judgeAgreementForModel(verdicts, 'model-a'); + expect(row.pairs).toBe(2); + expect(row.verdictAgreement).toBe(0.5); + expect(row.worstEvaluators[0]).toMatchObject({ + evaluator: 'Relevance', + flips: 1, + pairs: 2, + }); + }); + + it('treats differing scores on the same side of the midpoint as agreement', () => { + const verdicts = [ + verdict('gemini', 'ex-1', 0, 'Relevance', 0.6), + verdict('sonnet', 'ex-1', 0, 'Relevance', 1), + ]; + const row = judgeAgreementForModel(verdicts, 'model-a'); + expect(row.verdictAgreement).toBe(1); + expect(row.worstEvaluators).toHaveLength(0); + }); + + it('excludes cost and latency instruments from verdict agreement', () => { + const verdicts = [ + verdict('gemini', 'ex-1', 0, 'Input Tokens', 1), + verdict('sonnet', 'ex-1', 0, 'Input Tokens', 0), + verdict('gemini', 'ex-1', 0, 'Latency', 1), + verdict('sonnet', 'ex-1', 0, 'Latency', 0), + verdict('gemini', 'ex-1', 0, 'Relevance', 1), + verdict('sonnet', 'ex-1', 0, 'Relevance', 1), + ]; + const row = judgeAgreementForModel(verdicts, 'model-a'); + // Token counts differ wildly between judges and would otherwise dominate. + expect(row.pairs).toBe(1); + expect(row.verdictAgreement).toBe(1); + }); + + it('reports directional bias between the two judges', () => { + const verdicts = [ + verdict('gemini', 'ex-1', 0, 'Relevance', 0.6), + verdict('sonnet', 'ex-1', 0, 'Relevance', 1), + ]; + const row = judgeAgreementForModel(verdicts, 'model-a'); + expect(row.biasJudges).toEqual(['gemini', 'sonnet']); + expect(row.bias).toBeCloseTo(-0.4, 5); + }); + + it('carries a Wilson interval that widens at small n', () => { + const few = judgeAgreementForModel( + [verdict('gemini', 'ex-1', 0, 'Relevance', 1), verdict('sonnet', 'ex-1', 0, 'Relevance', 1)], + 'model-a' + ); + const many = judgeAgreementForModel( + Array.from({ length: 50 }, (_, i) => [ + verdict('gemini', `ex-${i}`, 0, 'Relevance', 1), + verdict('sonnet', `ex-${i}`, 0, 'Relevance', 1), + ]).flat(), + 'model-a' + ); + expect(few.verdictAgreement).toBe(1); + expect(many.verdictAgreement).toBe(1); + // Same point estimate, very different confidence. A board that showed only + // the estimate would present these as equally strong evidence. + expect(few.interval!.low).toBeLessThan(many.interval!.low); + }); + + it('ignores verdicts belonging to other models', () => { + const verdicts = [ + verdict('gemini', 'ex-1', 0, 'Relevance', 1), + verdict('sonnet', 'ex-1', 0, 'Relevance', 0), + verdict('gemini', 'ex-1', 0, 'Relevance', 1, 'model-b'), + verdict('sonnet', 'ex-1', 0, 'Relevance', 1, 'model-b'), + ]; + expect(judgeAgreementForModel(verdicts, 'model-a').verdictAgreement).toBe(0); + expect(judgeAgreementForModel(verdicts, 'model-b').verdictAgreement).toBe(1); + }); + + it('ranks the worst evaluators by flip rate, not raw count', () => { + const verdicts = [ + // Relevance: 1 flip out of 1 -> 100% + verdict('gemini', 'ex-1', 0, 'Relevance', 1), + verdict('sonnet', 'ex-1', 0, 'Relevance', 0), + // Factuality: 2 flips out of 10 -> 20%, higher raw count + ...Array.from({ length: 10 }, (_, i) => [ + verdict('gemini', `f-${i}`, 0, 'Factuality', 1), + verdict('sonnet', `f-${i}`, 0, 'Factuality', i < 2 ? 0 : 1), + ]).flat(), + ]; + const row = judgeAgreementForModel(verdicts, 'model-a'); + expect(row.worstEvaluators[0].evaluator).toBe('Relevance'); + expect(row.worstEvaluators[1].evaluator).toBe('Factuality'); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_agreement.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_agreement.ts new file mode 100644 index 0000000000000..3b9787bf927a7 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_agreement.ts @@ -0,0 +1,203 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { wilsonInterval, type ConfidenceInterval } from './trajectory_agreement'; + +/** + * One evaluator verdict, identified well enough to pair it with the verdict a + * different judge produced for the same unit of work. + */ +export interface JudgeVerdict { + modelId: string; + judgeId: string; + example: string; + repetition: number; + evaluator: string; + score: number; +} + +export type JudgeAgreementStatus = 'unmeasured' | 'single-judge' | 'measured'; + +export interface EvaluatorDisagreement { + evaluator: string; + flips: number; + pairs: number; + interval: ConfidenceInterval; +} + +export interface JudgeAgreementRow { + modelId: string; + status: JudgeAgreementStatus; + /** Judges that scored this model at all, whether or not they overlap. */ + judges: string[]; + /** Cells where two judges scored the identical example+rep+evaluator. */ + pairs: number; + /** + * Cells the leading judge scored that the other judge did not, so a row built + * on partial overlap cannot be read as though both judges covered everything. + * A high value means the agreement figure rests on less evidence than a + * fully-paired row with the same `pairs` count would. + */ + unpaired: number; + /** Pass/fail concordance, the verdict-level view. */ + verdictAgreement?: number; + interval?: ConfidenceInterval; + /** Directional bias: mean(judgeA) - mean(judgeB) over paired cells. */ + bias?: number; + biasJudges?: [string, string]; + worstEvaluators: EvaluatorDisagreement[]; +} + +/** + * Cost and latency instruments are recorded as evaluators but are not verdicts. + * Comparing them across judges measures provider billing, not judgement. + */ +const NON_VERDICT_EVALUATORS = new Set(['Input Tokens', 'Output Tokens', 'Latency', 'Tool Calls']); + +/** Scores are 0..1; a verdict is the pass/fail side of the midpoint. */ +const passed = (score: number): boolean => score > 0.5; + +const cellKey = (v: JudgeVerdict): string => + `${v.example}\u0000${v.repetition}\u0000${v.evaluator}`; + +/** + * Pairs verdicts by (example, repetition, evaluator) so the two judges are + * compared on identical work. Aggregate-vs-aggregate comparison is deliberately + * not offered: two judges can produce the same mean while disagreeing on every + * individual case, which is the exact failure this row exists to expose. + * + * Only the two judges with the most overlap are compared. A third judge with + * partial coverage would otherwise silently change the denominator. + */ +export const judgeAgreementForModel = ( + verdicts: readonly JudgeVerdict[], + modelId: string +): JudgeAgreementRow => { + const mine = verdicts.filter( + (v) => v.modelId === modelId && !NON_VERDICT_EVALUATORS.has(v.evaluator) + ); + const judges = [...new Set(mine.map((v) => v.judgeId))].sort(); + + if (judges.length === 0) { + return { + modelId, + status: 'unmeasured', + judges: [], + pairs: 0, + unpaired: 0, + worstEvaluators: [], + }; + } + if (judges.length === 1) { + // Scored, but by one judge only. This is NOT agreement of 100%; it is the + // absence of a second opinion, and must never render as a high score. + const soleCoverage = new Set(mine.map(cellKey)).size; + return { + modelId, + status: 'single-judge', + judges, + pairs: 0, + unpaired: soleCoverage, + worstEvaluators: [], + }; + } + + // Choose the judge pair with the largest true overlap. + let best: { a: string; b: string; keys: string[] } | undefined; + for (let i = 0; i < judges.length; i++) { + for (let j = i + 1; j < judges.length; j++) { + const a = judges[i]; + const b = judges[j]; + const aKeys = new Set(mine.filter((v) => v.judgeId === a).map(cellKey)); + const shared = [ + ...new Set(mine.filter((v) => v.judgeId === b && aKeys.has(cellKey(v))).map(cellKey)), + ]; + if (!best || shared.length > best.keys.length) { + best = { a, b, keys: shared }; + } + } + } + + if (!best || best.keys.length === 0) { + // Two judges exist but scored disjoint work — no comparison is possible. + const disjointCoverage = new Set(mine.map(cellKey)).size; + return { + modelId, + status: 'single-judge', + judges, + pairs: 0, + unpaired: disjointCoverage, + worstEvaluators: [], + }; + } + + const scoreOf = new Map(); + for (const v of mine) { + if (v.judgeId !== best.a && v.judgeId !== best.b) { + continue; + } + const key = cellKey(v); + const entry = scoreOf.get(key) ?? { evaluator: v.evaluator }; + if (v.judgeId === best.a) { + entry.a = v.score; + } else { + entry.b = v.score; + } + scoreOf.set(key, entry); + } + + const paired = [...scoreOf.values()].filter( + (e): e is { a: number; b: number; evaluator: string } => e.a !== undefined && e.b !== undefined + ); + // Cells only one of the two judges scored. 4.8-opus surfaced this: Gemini + // returned `unavailable` for ~130 cells Sonnet did score, so the row is + // computed over materially less work than a fully-paired model's row. + const unpaired = scoreOf.size - paired.length; + + let concordant = 0; + let sumA = 0; + let sumB = 0; + const perEvaluator = new Map(); + for (const cell of paired) { + const agree = passed(cell.a) === passed(cell.b); + if (agree) { + concordant += 1; + } + sumA += cell.a; + sumB += cell.b; + const stat = perEvaluator.get(cell.evaluator) ?? { flips: 0, pairs: 0 }; + stat.pairs += 1; + if (!agree) { + stat.flips += 1; + } + perEvaluator.set(cell.evaluator, stat); + } + + const worstEvaluators = [...perEvaluator.entries()] + .filter(([, s]) => s.flips > 0) + .map(([evaluator, s]) => ({ + evaluator, + flips: s.flips, + pairs: s.pairs, + interval: wilsonInterval(s.flips, s.pairs), + })) + .sort((x, y) => y.flips / y.pairs - x.flips / x.pairs) + .slice(0, 5); + + return { + modelId, + status: 'measured', + judges, + pairs: paired.length, + unpaired, + verdictAgreement: concordant / paired.length, + interval: wilsonInterval(concordant, paired.length), + bias: sumA / paired.length - sumB / paired.length, + biasJudges: [best.a, best.b], + worstEvaluators, + }; +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_agreement_golden.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_agreement_golden.test.ts new file mode 100644 index 0000000000000..38d4cb8b1c59b --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_agreement_golden.test.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// Renders the reliability board from REAL golden verdicts and writes it to +// disk. Skips itself when the export is absent so it cannot redden CI on a +// machine without cluster access. + +import fs from 'fs'; +import { renderReliabilityHtml } from './render_reliability_html'; +import { judgeAgreementForModel, type JudgeVerdict } from './judge_agreement'; +import type { Matrix } from './build_matrix'; + +const FIXTURE = '/tmp/judge_verdicts.json'; +const OUT = process.env.RELIABILITY_OUT ?? '/tmp/matrix.reliability.html'; + +const maybe = fs.existsSync(FIXTURE) ? describe : describe.skip; + +maybe('reliability board over real golden verdicts', () => { + let verdicts: JudgeVerdict[]; + + beforeAll(() => { + verdicts = JSON.parse(fs.readFileSync(FIXTURE, 'utf8')); + }); + + it('reproduces the independently computed Python figures', () => { + const opus = judgeAgreementForModel(verdicts, 'anthropic-claude-4.7-opus'); + expect(opus.status).toBe('measured'); + // Python: 531 pairs, 455/531 = 85.7%, CI [82.5, 88.4] + expect(opus.pairs).toBe(531); + expect(opus.verdictAgreement).toBeCloseTo(455 / 531, 6); + expect(opus.interval!.low).toBeCloseTo(0.8245, 3); + expect(opus.interval!.high).toBeCloseTo(0.8841, 3); + + const haiku = judgeAgreementForModel(verdicts, 'anthropic-claude-4.5-haiku'); + // Python: 531 pairs, 457/531 = 86.1% + expect(haiku.pairs).toBe(531); + expect(haiku.verdictAgreement).toBeCloseTo(457 / 531, 6); + }); + + it('now scores 4.8-opus, and discloses its one-sided coverage', () => { + // The Sonnet judge sweep closed this hole: 4.8-opus was single-judge until + // 2026-09-02. Gemini still returned `unavailable` on part of the run, so the + // row is measured over fewer cells than the fully-paired models and must + // report that shortfall rather than hide it behind a comparable-looking rate. + const row = judgeAgreementForModel(verdicts, 'anthropic-claude-4.8-opus'); + expect(row.status).toBe('measured'); + expect(row.judges).toEqual(['anthropic-claude-4.6-sonnet', 'google-gemini-3.1-pro']); + // Python: 395 paired cells, 329/395 = 83.3%, CI [79.3, 86.6] + expect(row.pairs).toBe(395); + expect(row.verdictAgreement).toBeCloseTo(329 / 395, 6); + expect(row.unpaired).toBeGreaterThan(0); + // Materially thinner than the 531-pair rows it sits beside. + expect(row.pairs).toBeLessThan(531); + }); + + it('agrees that Relevance is the one hotspot clearing noise on both models', () => { + for (const id of ['anthropic-claude-4.7-opus', 'anthropic-claude-4.5-haiku']) { + const row = judgeAgreementForModel(verdicts, id); + const relevance = row.worstEvaluators.find((e) => e.evaluator === 'Relevance'); + expect(relevance).toBeDefined(); + expect(relevance!.interval.low).toBeGreaterThan(0.2); + } + }); + + it('writes the board', () => { + const ids = [...new Set(verdicts.map((v) => v.modelId))].sort(); + const matrix: Matrix = { + columns: [], + composites: [], + displayColumns: [], + overallLabel: 'Overall', + evaluatorSaturation: [], + proprietary: ids.map((modelId) => ({ + modelId, + modelLabel: modelId, + openSource: false, + cells: {}, + overall: { kind: 'missing' as const }, + capability: { kind: 'missing' as const }, + judgedQuality: { kind: 'missing' as const }, + coverage: { covered: 0, total: 0 }, + })), + openSource: [], + }; + const html = renderReliabilityHtml(matrix, {}, {}, verdicts); + fs.writeFileSync(OUT, html); + expect(html).toContain('Judge agreement'); + expect(html).toContain('85.7%'); + expect(html).toContain('Single judge'); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_overlap.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_overlap.test.ts new file mode 100644 index 0000000000000..ecf9388ceac91 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_overlap.test.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + analyzeJudgeOverlap, + realModelIdFromSourceExecution, + type JudgeOverlapInput, +} from './judge_overlap'; + +const cells = (modelScores: Record) => + Object.entries(modelScores).flatMap(([modelId, scores]) => + scores.map((score, i) => ({ modelId, exampleId: `ex-${i}`, score })) + ); + +describe('analyzeJudgeOverlap', () => { + it('reports a uniform offset when a judge is lenient but agrees on order', () => { + // Same ordering, every score shifted +0.1. Offset must absorb the shift and + // leave no residual, because nothing has been reordered. + const input: JudgeOverlapInput[] = [ + { judgeId: 'strict', cells: cells({ a: [0.4, 0.5], b: [0.6, 0.7] }) }, + { judgeId: 'lenient', cells: cells({ a: [0.5, 0.6], b: [0.7, 0.8] }) }, + ]; + + const report = analyzeJudgeOverlap(input); + + const lenient = report.severities.find((s) => s.judgeId === 'lenient')!; + const strict = report.severities.find((s) => s.judgeId === 'strict')!; + expect(lenient.offset).toBeCloseTo(0.05, 6); + expect(strict.offset).toBeCloseTo(-0.05, 6); + // The whole point: a pure offset does not reorder, so residual is zero. + expect(report.residual.mean).toBeCloseTo(0, 6); + expect(report.rankings.strict).toEqual(report.rankings.lenient); + expect(report.rankable).toBe(true); + }); + + it('flags a board as unrankable when residual disagreement exceeds model spread', () => { + // Judges invert each other on individual cells while ending at similar + // means. Offsets cannot explain this, so the board orders judges. + const input: JudgeOverlapInput[] = [ + { judgeId: 'j1', cells: cells({ a: [0.9, 0.1], b: [0.85, 0.15] }) }, + { judgeId: 'j2', cells: cells({ a: [0.1, 0.9], b: [0.15, 0.85] }) }, + ]; + + const report = analyzeJudgeOverlap(input); + + expect(report.residual.mean).toBeGreaterThan(report.modelSpread); + expect(report.rankable).toBe(false); + }); + + it('keeps only cells every judge graded, so a judge that skipped hard cells cannot look lenient', () => { + // j2 is missing the hard cell (ex-1). Including it would credit j2 with a + // higher mean purely from absence. + const input: JudgeOverlapInput[] = [ + { judgeId: 'j1', cells: cells({ a: [0.8, 0.2] }) }, + { + judgeId: 'j2', + cells: [{ modelId: 'a', exampleId: 'ex-0', score: 0.8 }], + }, + ]; + + const report = analyzeJudgeOverlap(input); + + expect(report.commonCellCount).toBe(1); + // Both judges scored the shared cell identically -> no severity difference. + expect(report.severities.every((s) => Math.abs(s.offset) < 1e-9)).toBe(true); + }); + + it('identifies models whose rank is identical under every judge', () => { + const input: JudgeOverlapInput[] = [ + { judgeId: 'j1', cells: cells({ top: [0.9, 0.9], mid: [0.5, 0.7], low: [0.2, 0.2] }) }, + { judgeId: 'j2', cells: cells({ top: [0.8, 0.8], mid: [0.7, 0.5], low: [0.1, 0.1] }) }, + ]; + + const report = analyzeJudgeOverlap(input); + + // top and low hold their positions; mid is only stable because it sits + // between two robust anchors. + expect(report.stableRanks.map((s) => s.modelId)).toContain('top'); + expect(report.stableRanks.map((s) => s.modelId)).toContain('low'); + expect(report.stableRanks.find((s) => s.modelId === 'top')!.rank).toBe(1); + }); + + it('refuses to compare judges that share no cell', () => { + const input: JudgeOverlapInput[] = [ + { judgeId: 'j1', cells: [{ modelId: 'a', exampleId: 'ex-0', score: 0.5 }] }, + { judgeId: 'j2', cells: [{ modelId: 'a', exampleId: 'ex-9', score: 0.5 }] }, + ]; + + expect(() => analyzeJudgeOverlap(input)).toThrow(/graded by all 2 judges/); + }); + + it('refuses a single judge, which cannot separate judge effect from model effect', () => { + expect(() => analyzeJudgeOverlap([{ judgeId: 'only', cells: cells({ a: [0.5] }) }])).toThrow( + /at least 2 judges/ + ); + }); +}); + +describe('realModelIdFromSourceExecution', () => { + it('recovers the real model id from a blind run', () => { + expect( + realModelIdFromSourceExecution( + 'sweep-1788679167-rja-s1of3::security-persona-matrix::anthropic-claude-4.8-opus' + ) + ).toBe('anthropic-claude-4.8-opus'); + }); + + it('throws rather than returning a blind alias when the id is malformed', () => { + expect(() => realModelIdFromSourceExecution('Model A')).toThrow(/Cannot recover a model id/); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_overlap.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_overlap.ts new file mode 100644 index 0000000000000..31eda555e8025 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_overlap.ts @@ -0,0 +1,213 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Quantify how much of a matrix column is the model and how much is the judge. + * + * A matrix whose columns were each graded by a different judge cannot be + * ranked: a model's score moves with judge severity, and severity is not + * constant. Re-judging one common subset of cells with several judges makes + * the two separable, because every judge sees an identical set of cells. + * + * The decomposition that matters: + * + * - OFFSET is a judge's uniform leniency. It shifts every model equally, so + * it cancels out of any within-judge ranking and is harmless to ordering. + * - RESIDUAL is what is left once the offset is removed: per-cell + * disagreement that moves models relative to each other. This is what + * reorders a board. + * + * A ranking is only trustworthy when residual disagreement is small relative + * to the spread between the models being ranked. Comparing residual against + * model spread is therefore the headline output, not the raw judge means -- + * two judges can differ wildly in mean while agreeing perfectly on order. + */ + +export interface OverlapCellScore { + /** Real model id. Blind runs alias this, so callers must de-anonymize first. */ + modelId: string; + exampleId: string; + /** Mean of the evaluator scores for this cell, already normalized to 0..1. */ + score: number; +} + +export interface JudgeOverlapInput { + judgeId: string; + cells: OverlapCellScore[]; +} + +export interface JudgeSeverity { + judgeId: string; + mean: number; + /** Uniform leniency relative to the mean judge. Cancels in ranking. */ + offset: number; +} + +export interface JudgeOverlapReport { + /** Cells graded by every judge. Anything less is not a fair comparison. */ + commonCellCount: number; + models: string[]; + judges: string[]; + severities: JudgeSeverity[]; + /** Per-model mean under each judge, keyed by model then judge. */ + perModel: Record>; + /** Best->worst model order under each judge. */ + rankings: Record; + /** Mean/median/p90 of per-cell judge disagreement after offsets are removed. */ + residual: { mean: number; median: number; p90: number }; + /** Widest gap between model means under the same judge. */ + modelSpread: number; + /** + * True when residual disagreement is smaller than the spread it must + * resolve. When false, the board orders judges rather than models and no + * ranking below the top slot should be published. + */ + rankable: boolean; + /** Models holding the same rank under every judge -- the robust claims. */ + stableRanks: Array<{ modelId: string; rank: number }>; +} + +const mean = (xs: number[]): number => xs.reduce((a, b) => a + b, 0) / xs.length; + +const quantile = (xs: number[], q: number): number => { + if (xs.length === 0) return NaN; + const sorted = [...xs].sort((a, b) => a - b); + const idx = Math.min(sorted.length - 1, Math.floor(q * sorted.length)); + return sorted[idx]; +}; + +const median = (xs: number[]): number => { + const sorted = [...xs].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; +}; + +/** NUL cannot occur in a model or example id, so it cannot collide. */ +const SEP = '\u0000'; +const cellKey = (c: { modelId: string; exampleId: string }): string => + `${c.modelId}${SEP}${c.exampleId}`; + +/** + * Compare judges over the cells they all graded. + * + * Cells missing from any judge are dropped rather than imputed: a judge that + * failed on the hard cells would otherwise look lenient purely from absence, + * which is the exact confound this analysis exists to remove. + */ +export function analyzeJudgeOverlap(inputs: JudgeOverlapInput[]): JudgeOverlapReport { + if (inputs.length < 2) { + throw new Error( + `Judge overlap needs at least 2 judges to separate judge effect from model effect; got ${inputs.length}.` + ); + } + + const byJudge = new Map>(); + for (const input of inputs) { + const m = new Map(); + for (const cell of input.cells) m.set(cellKey(cell), cell); + byJudge.set(input.judgeId, m); + } + + const judges = inputs.map((i) => i.judgeId); + const [firstJudge, ...restJudges] = judges; + const commonKeys = [...byJudge.get(firstJudge)!.keys()].filter((k) => + restJudges.every((j) => byJudge.get(j)!.has(k)) + ); + + if (commonKeys.length === 0) { + throw new Error( + `No cell was graded by all ${judges.length} judges, so judge severity and model quality cannot be separated.` + ); + } + + const scoreOf = (judgeId: string, key: string) => byJudge.get(judgeId)!.get(key)!.score; + const meansByJudge = new Map( + judges.map((judgeId) => [judgeId, mean(commonKeys.map((k) => scoreOf(judgeId, k)))]) + ); + const grandMean = mean([...meansByJudge.values()]); + const severities: JudgeSeverity[] = judges.map((judgeId) => ({ + judgeId, + mean: meansByJudge.get(judgeId)!, + offset: meansByJudge.get(judgeId)! - grandMean, + })); + + const keysByModel = new Map(); + for (const key of commonKeys) { + const modelId = key.split(SEP)[0]; + keysByModel.set(modelId, [...(keysByModel.get(modelId) ?? []), key]); + } + const models = [...keysByModel.keys()].sort(); + + const perModel: Record> = {}; + for (const modelId of models) { + const keys = keysByModel.get(modelId)!; + perModel[modelId] = Object.fromEntries( + judges.map((judgeId) => [judgeId, mean(keys.map((k) => scoreOf(judgeId, k)))]) + ); + } + + const rankings: Record = {}; + for (const judgeId of judges) { + rankings[judgeId] = [...models].sort((a, b) => perModel[b][judgeId] - perModel[a][judgeId]); + } + + // Residual: strip each judge's uniform offset, then measure what disagreement + // survives on each cell. Offsets are removed first precisely because they do + // not reorder anything. + const offsetOf = new Map(severities.map((s) => [s.judgeId, s.offset])); + const perCellSpread = commonKeys.map((k) => { + const adjusted = judges.map((j) => scoreOf(j, k) - offsetOf.get(j)!); + return Math.max(...adjusted) - Math.min(...adjusted); + }); + + const modelMeansPerJudge = judges.map((j) => models.map((m) => perModel[m][j])); + const modelSpread = Math.max( + ...modelMeansPerJudge.map((ms) => Math.max(...ms) - Math.min(...ms)) + ); + + const residual = { + mean: mean(perCellSpread), + median: median(perCellSpread), + p90: quantile(perCellSpread, 0.9), + }; + + const stableRanks: Array<{ modelId: string; rank: number }> = []; + for (const modelId of models) { + const ranks = judges.map((j) => rankings[j].indexOf(modelId)); + if (new Set(ranks).size === 1) stableRanks.push({ modelId, rank: ranks[0] + 1 }); + } + + return { + commonCellCount: commonKeys.length, + models, + judges, + severities, + perModel, + rankings, + residual, + modelSpread, + rankable: residual.mean < modelSpread, + stableRanks: stableRanks.sort((a, b) => a.rank - b.rank), + }; +} + +/** + * Recover the real model id from a blind rejudge result. + * + * Blind runs replace `modelId` with an alias so the judge cannot recognize the + * contestant, but `sourceExecutionId` keeps the original coordinates. Reading + * the alias instead would group every model under "Model A". + */ +export function realModelIdFromSourceExecution(sourceExecutionId: string): string { + const parts = sourceExecutionId.split('::'); + if (parts.length < 3) { + throw new Error( + `Cannot recover a model id from executionId "${sourceExecutionId}"; expected ::::.` + ); + } + return parts[2]; +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_provenance.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_provenance.test.ts new file mode 100644 index 0000000000000..68b1528ab2c6a --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_provenance.test.ts @@ -0,0 +1,293 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + auditJudges, + checkJudge, + classifyFamily, + describeJudge, + isEisBacked, + deriveJudgeProvenance, +} from './judge_provenance'; +import type { AggregatedModelScores } from './query_matrix_scores'; + +describe('classifyFamily', () => { + it.each([ + ['eis-anthropic-claude-4.6-sonnet', 'anthropic'], + ['anthropic-claude-4.6-sonnet-chat_completion', 'anthropic'], + ['eis-openai-gpt-5-4', 'openai'], + ['google-gemini-3.1-pro', 'google'], + ['Qwen/Qwen3-Coder-30B-A3B-Instruct', 'qwen'], + ['mistralai/Mistral-Small-24B-Instruct-2501', 'mistral'], + ['LiteLLM gpt-oss-20b', 'openai'], + ])('classifies %s as %s', (id, expected) => { + expect(classifyFamily(id)).toBe(expected); + }); + + it('attributes a Nous finetune of a Llama base to nous, not meta', () => { + // The id names both vendors; misattributing it to meta would overstate + // how much cross-family coverage a panel actually has. + expect(classifyFamily('NousResearch/Hermes-3-Llama-3.1-70B')).toBe('nous'); + }); + + it('returns unknown for an unrecognised or empty id', () => { + expect(classifyFamily('some-internal-endpoint')).toBe('unknown'); + expect(classifyFamily('')).toBe('unknown'); + expect(classifyFamily(undefined)).toBe('unknown'); + }); +}); + +describe('isEisBacked', () => { + it('accepts eis-prefixed connectors', () => { + expect(isEisBacked('eis-anthropic-claude-4.5-haiku')).toBe(true); + expect(isEisBacked('eis-google-gemini-3-1-pro')).toBe(true); + }); + + it('accepts vendor-canonical ids without a repo path', () => { + expect(isEisBacked('anthropic-claude-4.6-sonnet')).toBe(true); + }); + + it.each([ + ['Qwen/Qwen3-Coder-30B-A3B-Instruct'], + ['LiteLLM Qwen3-Coder-30B-A3B-Instruct-AWQ'], + ['NousResearch/Hermes-3-Llama-3.1-70B'], + ['cyankiwi/Qwen3-Coder-30B-A3B-Instruct-AWQ-4bit'], + ['gghfez/Mistral-Small-3.2-24B-Instruct-hf-AWQ'], + ])('rejects self-hosted endpoint %s', (id) => { + expect(isEisBacked(id)).toBe(false); + }); + + it('rejects empty and whitespace ids', () => { + expect(isEisBacked('')).toBe(false); + expect(isEisBacked(' ')).toBe(false); + expect(isEisBacked(undefined)).toBe(false); + }); +}); + +describe('describeJudge', () => { + it('flags a model grading itself', () => { + const p = describeJudge('eis-anthropic-claude-4.6-sonnet', 'eis-anthropic-claude-4.6-sonnet'); + expect(p.selfJudged).toBe(true); + expect(p.sameFamily).toBe(true); + }); + + it('treats case and padding differences as the same model', () => { + const p = describeJudge(' EIS-Anthropic-Claude-4.6-Sonnet ', 'eis-anthropic-claude-4.6-sonnet'); + expect(p.selfJudged).toBe(true); + }); + + it('separates same-family from self-judged', () => { + const p = describeJudge('eis-anthropic-claude-4.6-sonnet', 'eis-anthropic-claude-4.8-opus'); + expect(p.selfJudged).toBe(false); + expect(p.sameFamily).toBe(true); + }); + + it('does not call two unknown-family models the same family', () => { + // Both classify as `unknown`; treating that as a family match would + // fabricate a same-family violation between unrelated endpoints. + const p = describeJudge('mystery-endpoint-a', 'mystery-endpoint-b'); + expect(p.sameFamily).toBe(false); + }); +}); + +describe('checkJudge', () => { + it('reports a non-EIS judge by default', () => { + const v = checkJudge('Qwen/Qwen3-Coder-30B-A3B-Instruct', 'eis-openai-gpt-5-4'); + expect(v.map((x) => x.kind)).toEqual(['non-eis-judge']); + }); + + it('reports self-judging by default', () => { + const v = checkJudge('eis-anthropic-claude-4.6-sonnet', 'eis-anthropic-claude-4.6-sonnet'); + expect(v.map((x) => x.kind)).toEqual(['self-judged']); + }); + + it('does NOT report same-family unless explicitly enabled', () => { + // Measured same-family bias was not significant, so this must stay opt-in. + const v = checkJudge('eis-anthropic-claude-4.6-sonnet', 'eis-anthropic-claude-4.8-opus'); + expect(v).toEqual([]); + }); + + it('reports same-family when the policy asks for it', () => { + const v = checkJudge('eis-anthropic-claude-4.6-sonnet', 'eis-anthropic-claude-4.8-opus', { + forbidSameFamily: true, + }); + expect(v.map((x) => x.kind)).toEqual(['same-family']); + }); + + it('passes a clean cross-family EIS pairing', () => { + expect(checkJudge('eis-google-gemini-3-1-pro', 'eis-anthropic-claude-4.8-opus')).toEqual([]); + }); + + it('can report several violations for one pairing', () => { + const v = checkJudge('Qwen/Qwen3-Coder-30B-A3B-Instruct', 'Qwen/Qwen3-Coder-30B-A3B-Instruct', { + forbidSameFamily: true, + }); + expect(v.map((x) => x.kind).sort()).toEqual(['non-eis-judge', 'same-family', 'self-judged']); + }); +}); + +describe('auditJudges', () => { + it('weights counts by docCount, not by row', () => { + const summary = auditJudges([ + { + judgeId: 'eis-anthropic-claude-4.6-sonnet', + taskModelId: 'eis-openai-gpt-5-4', + docCount: 1000, + }, + { + judgeId: 'Qwen/Qwen3-Coder-30B-A3B-Instruct', + taskModelId: 'eis-openai-gpt-5-4', + docCount: 50, + }, + ]); + expect(summary.totalDocs).toBe(1050); + expect(summary.nonEisDocs).toBe(50); + }); + + it('defaults docCount to 1', () => { + const summary = auditJudges([ + { judgeId: 'eis-anthropic-claude-4.6-sonnet', taskModelId: 'eis-openai-gpt-5-4' }, + ]); + expect(summary.totalDocs).toBe(1); + }); + + it('deduplicates violations per judge/candidate pairing', () => { + const rows = Array.from({ length: 5 }, () => ({ + judgeId: 'Qwen/Qwen3-Coder-30B-A3B-Instruct', + taskModelId: 'eis-openai-gpt-5-4', + docCount: 10, + })); + const summary = auditJudges(rows); + expect(summary.violations).toHaveLength(1); + expect(summary.nonEisDocs).toBe(50); + }); + + it('counts self-judged docs separately from non-EIS docs', () => { + const summary = auditJudges([ + { + judgeId: 'anthropic-claude-4.6-sonnet', + taskModelId: 'anthropic-claude-4.6-sonnet', + docCount: 30, + }, + ]); + expect(summary.selfJudgedDocs).toBe(30); + expect(summary.nonEisDocs).toBe(0); + }); + + it('lists the distinct judge families present', () => { + const summary = auditJudges([ + { judgeId: 'eis-anthropic-claude-4.6-sonnet', taskModelId: 'eis-openai-gpt-5-4' }, + { judgeId: 'eis-google-gemini-3-1-pro', taskModelId: 'eis-openai-gpt-5-4' }, + { judgeId: 'eis-anthropic-claude-4.5-haiku', taskModelId: 'eis-openai-gpt-5-4' }, + ]); + expect(summary.judgeFamilies).toEqual(['anthropic', 'google']); + }); + + it('handles an empty audit', () => { + const summary = auditJudges([]); + expect(summary).toMatchObject({ totalDocs: 0, nonEisDocs: 0, violations: [] }); + }); +}); + +const model = (modelId: string, judges: Array): AggregatedModelScores => + ({ + modelId, + suites: judges.map((judgeModelId, i) => ({ + suiteId: `suite-${i}`, + judgeModelId, + datasets: [], + })), + } as unknown as AggregatedModelScores); + +describe('deriveJudgeProvenance', () => { + it('reports the bare id only when every admitted run shares one judge', () => { + const result = deriveJudgeProvenance([ + model('openai-gpt-5.5', ['google-gemini-3.1-pro']), + model('anthropic-claude-5-sonnet', ['google-gemini-3.1-pro']), + ]); + + expect(result.judgeModelId).toBe('google-gemini-3.1-pro'); + expect(result.judgeBreakdown).toEqual([ + { judgeModelId: 'google-gemini-3.1-pro', suites: 2, share: 100 }, + ]); + }); + + it('refuses to name a single judge when the board is actually mixed', () => { + // The regression that motivated this: the artifact declared a unified + // gemini judge while the golden rows were mostly haiku-graded. A mixed + // board must never render as a bare id. + const result = deriveJudgeProvenance([ + model('a', ['anthropic-claude-4.5-haiku']), + model('b', ['anthropic-claude-4.5-haiku']), + model('c', ['anthropic-claude-4.5-haiku']), + model('d', ['google-gemini-3.1-pro']), + ]); + + expect(result.judgeModelId).not.toBe('google-gemini-3.1-pro'); + expect(result.judgeModelId).toMatch(/^mixed: /); + expect(result.judgeModelId).toContain('anthropic-claude-4.5-haiku 75.0%'); + expect(result.judgeModelId).toContain('google-gemini-3.1-pro 25.0%'); + }); + + // Regression: attack-discovery runs each slice as its own experiment, so a + // suite can carry several judges. Counting only the single-judge field made + // those runs vanish from the breakdown -- the mixed column disappeared from + // the figure whose whole job is to expose it. + it('counts every judge on a suite whose columns ran as separate experiments', () => { + const mixed: AggregatedModelScores[] = [ + { + modelId: 'a', + suites: [ + { + suiteId: 'attack-discovery-agent-builder', + judgeModelIds: ['anthropic-claude-4.6-sonnet', 'google-gemini-3.1-pro'], + datasets: [], + }, + ], + } as unknown as AggregatedModelScores, + ]; + + const result = deriveJudgeProvenance(mixed); + + expect(result.judgeBreakdown.map((j) => j.judgeModelId).sort()).toEqual([ + 'anthropic-claude-4.6-sonnet', + 'google-gemini-3.1-pro', + ]); + expect(result.judgeModelId).toMatch(/^mixed: /); + }); + + it('orders the breakdown by how much of the board each judge graded', () => { + const result = deriveJudgeProvenance([ + model('a', ['gemini', 'haiku', 'haiku']), + model('b', ['haiku']), + ]); + + expect(result.judgeBreakdown.map((j) => j.judgeModelId)).toEqual(['haiku', 'gemini']); + expect(result.judgeBreakdown[0]).toEqual({ judgeModelId: 'haiku', suites: 3, share: 75 }); + expect(result.judgeBreakdown[1]).toEqual({ judgeModelId: 'gemini', suites: 1, share: 25 }); + }); + + it('says the judge is unknown rather than implying one when none is recorded', () => { + const result = deriveJudgeProvenance([model('a', [undefined]), model('b', [undefined])]); + + expect(result.judgeModelId).toMatch(/^unknown/); + expect(result.judgeBreakdown).toEqual([]); + }); + + it('ignores runs with no judge instead of counting them as a judge', () => { + const result = deriveJudgeProvenance([model('a', ['gemini', undefined, undefined])]); + + expect(result.judgeModelId).toBe('gemini'); + expect(result.judgeBreakdown).toEqual([{ judgeModelId: 'gemini', suites: 1, share: 100 }]); + }); + + it('handles a model that contributed no suites at all', () => { + const result = deriveJudgeProvenance([model('a', ['gemini']), model('b', [])]); + + expect(result.judgeModelId).toBe('gemini'); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_provenance.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_provenance.ts new file mode 100644 index 0000000000000..0d5e2b3af7376 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/judge_provenance.ts @@ -0,0 +1,314 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AggregatedModelScores } from './query_matrix_scores'; + +/** + * Judge provenance: who graded a cell, on what backend, and from which family. + * + * Two properties make a judged score trustworthy enough to rank on: + * + * 1. The judge ran on a known inference backend (EIS), not an ad-hoc + * locally-hosted endpoint whose weights/quantisation are unpinned. + * 2. The judge is not scoring a candidate from its own model family, and + * above all is not scoring *itself*. + * + * Violations are reported, never silently corrected — a matrix that quietly + * drops cells is worse than one that shows why a cell is untrustworthy. + */ + +/** Model families we can recognise from a connector or model id. */ +export type ModelFamily = + | 'anthropic' + | 'openai' + | 'google' + | 'meta' + | 'mistral' + | 'qwen' + | 'deepseek' + | 'nous' + | 'unknown'; + +/** + * Family patterns, most-specific first. + * + * `nous` precedes `meta` deliberately: `NousResearch/Hermes-3-Llama-3.1-70B` + * names both vendors, and attributing it to Meta would overstate how much + * cross-family coverage a judge panel actually has. + */ +const FAMILY_PATTERNS: Array<[ModelFamily, RegExp]> = [ + ['nous', /hermes|nousresearch/i], + ['anthropic', /claude|anthropic/i], + ['openai', /gpt|openai|o[13]-|oss/i], + ['google', /gemini|google|gemma/i], + ['mistral', /mistral|mixtral|magistral/i], + ['qwen', /qwen/i], + ['deepseek', /deepseek/i], + ['meta', /llama|meta-/i], +]; + +export function classifyFamily(modelId: string | undefined | null): ModelFamily { + const id = String(modelId ?? ''); + if (!id) { + return 'unknown'; + } + for (const [family, pattern] of FAMILY_PATTERNS) { + if (pattern.test(id)) { + return family; + } + } + return 'unknown'; +} + +/** + * EIS-backed connectors are registered with an `eis-` prefix. Anything else — + * a raw provider id, a LiteLLM alias, a HuggingFace repo path — is a + * self-hosted or third-party endpoint whose exact weights are not pinned by + * the eval infrastructure. + */ +export function isEisBacked(judgeId: string | undefined | null): boolean { + const id = String(judgeId ?? '').trim(); + if (!id) { + return false; + } + if (/^eis[-_]/i.test(id)) { + return true; + } + // Vendor-canonical ids used by EIS connectors, e.g. `anthropic-claude-4.6-sonnet`. + return /^(anthropic|openai|google)-/i.test(id) && !id.includes('/'); +} + +export interface JudgeProvenance { + judgeId: string; + taskModelId: string; + eisBacked: boolean; + judgeFamily: ModelFamily; + taskFamily: ModelFamily; + /** Judge and candidate are literally the same model. */ + selfJudged: boolean; + /** Judge and candidate come from the same model family. */ + sameFamily: boolean; +} + +export function describeJudge(judgeId: string, taskModelId: string): JudgeProvenance { + const judgeFamily = classifyFamily(judgeId); + const taskFamily = classifyFamily(taskModelId); + const norm = (s: string) => + String(s ?? '') + .trim() + .toLowerCase(); + return { + judgeId, + taskModelId, + eisBacked: isEisBacked(judgeId), + judgeFamily, + taskFamily, + selfJudged: norm(judgeId) === norm(taskModelId) && norm(judgeId) !== '', + sameFamily: judgeFamily === taskFamily && judgeFamily !== 'unknown', + }; +} + +export type JudgeViolationKind = 'non-eis-judge' | 'self-judged' | 'same-family'; + +export interface JudgeViolation { + kind: JudgeViolationKind; + judgeId: string; + taskModelId: string; + detail: string; +} + +export interface JudgePolicy { + /** Reject judges that are not EIS-backed. Default true. */ + requireEis?: boolean; + /** Reject a model grading itself. Default true. */ + forbidSelfJudging?: boolean; + /** + * Reject a judge from the candidate's own family. Off by default: measured + * same-family bias on the persona matrix was not statistically significant + * (n=431 paired cells, z=-0.88), so this is opt-in rather than assumed. + */ + forbidSameFamily?: boolean; +} + +const DEFAULT_POLICY: Required = { + requireEis: true, + forbidSelfJudging: true, + forbidSameFamily: false, +}; + +/** Check one judge/candidate pairing against the policy. */ +export function checkJudge( + judgeId: string, + taskModelId: string, + policy: JudgePolicy = {} +): JudgeViolation[] { + const effective = { ...DEFAULT_POLICY, ...policy }; + const p = describeJudge(judgeId, taskModelId); + const violations: JudgeViolation[] = []; + + if (effective.requireEis && !p.eisBacked) { + violations.push({ + kind: 'non-eis-judge', + judgeId, + taskModelId, + detail: `judge "${judgeId}" is not an EIS-backed connector; its weights and quantisation are not pinned by the eval infrastructure`, + }); + } + if (effective.forbidSelfJudging && p.selfJudged) { + violations.push({ + kind: 'self-judged', + judgeId, + taskModelId, + detail: `model "${taskModelId}" graded its own output`, + }); + } + if (effective.forbidSameFamily && p.sameFamily) { + violations.push({ + kind: 'same-family', + judgeId, + taskModelId, + detail: `judge "${judgeId}" and candidate "${taskModelId}" are both in the "${p.judgeFamily}" family`, + }); + } + return violations; +} + +export interface JudgeAuditRow { + judgeId: string; + taskModelId: string; + docCount?: number; +} + +export interface JudgeAuditSummary { + totalDocs: number; + nonEisDocs: number; + selfJudgedDocs: number; + sameFamilyDocs: number; + violations: JudgeViolation[]; + /** Distinct judge families that graded at least one cell. */ + judgeFamilies: ModelFamily[]; +} + +/** + * Audit a whole matrix worth of judge pairings so a report can state, up front, + * how much of its data came from judges that meet the policy. + */ +export function auditJudges(rows: JudgeAuditRow[], policy: JudgePolicy = {}): JudgeAuditSummary { + let totalDocs = 0; + let nonEisDocs = 0; + let selfJudgedDocs = 0; + let sameFamilyDocs = 0; + const violations: JudgeViolation[] = []; + const seen = new Set(); + const families = new Set(); + + for (const row of rows) { + const docs = row.docCount ?? 1; + totalDocs += docs; + const p = describeJudge(row.judgeId, row.taskModelId); + families.add(p.judgeFamily); + if (!p.eisBacked) { + nonEisDocs += docs; + } + if (p.selfJudged) { + selfJudgedDocs += docs; + } + if (p.sameFamily) { + sameFamilyDocs += docs; + } + + const key = `${row.judgeId}::${row.taskModelId}`; + if (!seen.has(key)) { + seen.add(key); + violations.push(...checkJudge(row.judgeId, row.taskModelId, policy)); + } + } + + return { + totalDocs, + nonEisDocs, + selfJudgedDocs, + sameFamilyDocs, + violations, + judgeFamilies: [...families].sort(), + }; +} + +export interface JudgeShare { + judgeModelId: string; + /** Number of admitted (model, suite) runs this judge graded. */ + suites: number; + /** Percentage of admitted runs, to one decimal. */ + share: number; +} + +export interface DerivedJudgeProvenance { + /** + * A single judge id when every admitted run shares one, otherwise a + * `mixed: ...` summary. Never a bare id when the board is not unanimous — + * the whole point is that a reader cannot mistake a mixed board for a + * unified one. + */ + judgeModelId: string; + judgeBreakdown: JudgeShare[]; +} + +/** + * Derive which judge graded the admitted runs, counted from the aggregated + * scores rather than asserted by the caller. + * + * A hardcoded judge id silently survives a rejudge that never landed: the + * board then claims one shared instrument while the rows were graded by + * several. That is precisely the assumption the published spread/CI figures + * rest on, so it must be measured, not declared. + */ +export function deriveJudgeProvenance(aggregated: AggregatedModelScores[]): DerivedJudgeProvenance { + const counts = new Map(); + for (const model of aggregated) { + for (const suite of model.suites ?? []) { + // A suite whose columns ran as separate experiments carries every judge + // that graded it. Counting only the single-judge `judgeModelId` would + // drop those runs from the breakdown entirely, so a mixed column would + // disappear from the very figure meant to expose it. + const judges = suite.judgeModelIds?.length + ? suite.judgeModelIds + : suite.judgeModelId + ? [suite.judgeModelId] + : []; + for (const judge of judges) { + counts.set(judge, (counts.get(judge) ?? 0) + 1); + } + } + } + + const byFrequency = [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])); + const total = byFrequency.reduce((sum, [, n]) => sum + n, 0); + + const judgeBreakdown: JudgeShare[] = byFrequency.map(([judgeModelId, suites]) => ({ + judgeModelId, + suites, + share: Number(((100 * suites) / total).toFixed(1)), + })); + + if (byFrequency.length === 0) { + return { + judgeModelId: 'unknown (no judge recorded on any admitted suite)', + judgeBreakdown, + }; + } + + if (byFrequency.length === 1) { + return { judgeModelId: byFrequency[0][0], judgeBreakdown }; + } + + return { + judgeModelId: `mixed: ${judgeBreakdown + .map((j) => `${j.judgeModelId} ${j.share.toFixed(1)}%`) + .join(', ')}`, + judgeBreakdown, + }; +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/jury.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/jury.test.ts new file mode 100644 index 0000000000000..e892e3d96c1a5 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/jury.test.ts @@ -0,0 +1,178 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + aggregateJury, + FACTUALITY_LADDER, + GROUNDEDNESS_LADDER, + RELEVANCE_LADDER, + scoreVerdict, + type JuryVote, +} from './jury'; + +const vote = (judgeId: string, score: number, verdict?: string): JuryVote => ({ + judgeId, + score, + verdict, +}); + +describe('aggregateJury', () => { + it('takes the median of one vote per family', () => { + const result = aggregateJury([ + vote('eis-anthropic-claude-4.6-sonnet', 0.9), + vote('eis-google-gemini-3-1-pro', 0.5), + vote('eis-openai-gpt-5-4', 0.7), + ]); + expect(result.score).toBe(0.7); + expect(result.decided).toBe(true); + expect(result.families).toEqual(['anthropic', 'google', 'openai']); + }); + + it('ignores a single broken judge that returns 0', () => { + // The whole point of a median: one judge failing to parse an output must + // not be able to drag a good cell down to a failing-looking score. + const result = aggregateJury([ + vote('eis-anthropic-claude-4.6-sonnet', 0.9), + vote('eis-google-gemini-3-1-pro', 0.92), + vote('eis-openai-gpt-5-4', 0), + ]); + expect(result.score).toBe(0.9); + // The disagreement is still surfaced rather than hidden. + expect(result.disagreement).toBeCloseTo(0.92); + }); + + it('caps an over-represented family to one vote by default', () => { + // Five Anthropic judges are one family voting five times, not a panel. + const result = aggregateJury([ + vote('eis-anthropic-claude-4.6-sonnet', 0.9), + vote('eis-anthropic-claude-4.8-opus', 0.95), + vote('eis-anthropic-claude-4.5-haiku', 0.85), + vote('eis-google-gemini-3-1-pro', 0.2), + ]); + expect(result.counted).toHaveLength(2); + expect(result.dropped).toHaveLength(2); + expect(result.families).toEqual(['anthropic', 'google']); + expect(result.score).toBeCloseTo(0.55); + }); + + it('honours a raised per-family cap', () => { + const result = aggregateJury( + [ + vote('eis-anthropic-claude-4.6-sonnet', 0.9), + vote('eis-anthropic-claude-4.8-opus', 0.8), + vote('eis-google-gemini-3-1-pro', 0.4), + ], + { maxVotesPerFamily: 2 } + ); + expect(result.counted).toHaveLength(3); + expect(result.score).toBe(0.8); + }); + + it('reports disagreement 0 when judges agree exactly', () => { + const result = aggregateJury([ + vote('eis-anthropic-claude-4.6-sonnet', 0.75), + vote('eis-google-gemini-3-1-pro', 0.75), + ]); + expect(result.disagreement).toBe(0); + }); + + it('marks a single-vote panel undecided', () => { + const result = aggregateJury([vote('eis-anthropic-claude-4.6-sonnet', 0.9)]); + expect(result.decided).toBe(false); + expect(result.score).toBe(0.9); + }); + + it('discards non-finite scores instead of poisoning the median', () => { + const result = aggregateJury([ + vote('eis-anthropic-claude-4.6-sonnet', Number.NaN), + vote('eis-google-gemini-3-1-pro', 0.6), + vote('eis-openai-gpt-5-4', 0.8), + ]); + expect(result.counted).toHaveLength(2); + expect(result.score).toBeCloseTo(0.7); + }); + + it('returns an undecided null result when every vote is unusable', () => { + const result = aggregateJury([ + vote('eis-anthropic-claude-4.6-sonnet', Number.NaN), + vote('eis-google-gemini-3-1-pro', Number.POSITIVE_INFINITY), + ]); + expect(result.score).toBeNull(); + expect(result.decided).toBe(false); + }); + + it('handles an empty panel', () => { + const result = aggregateJury([]); + expect(result.score).toBeNull(); + expect(result.decided).toBe(false); + expect(result.families).toEqual([]); + }); + + it('averages the two middle votes for an even panel', () => { + const result = aggregateJury([ + vote('eis-anthropic-claude-4.6-sonnet', 0.4), + vote('eis-google-gemini-3-1-pro', 0.6), + ]); + expect(result.score).toBeCloseTo(0.5); + }); + + it('detects unanimous categorical verdicts', () => { + const result = aggregateJury([ + vote('eis-anthropic-claude-4.6-sonnet', 1, 'GROUNDED'), + vote('eis-google-gemini-3-1-pro', 0.85, 'GROUNDED'), + ]); + expect(result.verdictUnanimous).toBe(true); + }); + + it('reports a split verdict even when the scores are close', () => { + const result = aggregateJury([ + vote('eis-anthropic-claude-4.6-sonnet', 0.9, 'GROUNDED'), + vote('eis-google-gemini-3-1-pro', 0.85, 'GROUNDED_WITH_DISCLOSURE'), + ]); + expect(result.verdictUnanimous).toBe(false); + }); + + it('does not claim unanimity when no verdicts were supplied', () => { + const result = aggregateJury([ + vote('eis-anthropic-claude-4.6-sonnet', 0.9), + vote('eis-google-gemini-3-1-pro', 0.9), + ]); + expect(result.verdictUnanimous).toBe(false); + }); +}); + +describe('scoreVerdict', () => { + it('maps the groundedness ladder', () => { + expect(scoreVerdict('GROUNDED', GROUNDEDNESS_LADDER)).toBe(1); + expect(scoreVerdict('GROUNDED_WITH_DISCLOSURE', GROUNDEDNESS_LADDER)).toBe(0.85); + expect(scoreVerdict('MINOR_HALLUCINATIONS', GROUNDEDNESS_LADDER)).toBe(0.5); + expect(scoreVerdict('MAJOR_HALLUCINATIONS', GROUNDEDNESS_LADDER)).toBe(0); + }); + + it('maps the factuality and relevance ladders', () => { + expect(scoreVerdict('ACCURATE', FACTUALITY_LADDER)).toBe(1); + expect(scoreVerdict('MAJOR_INACCURACIES', FACTUALITY_LADDER)).toBe(0); + expect(scoreVerdict('PARTIALLY_RELEVANT', RELEVANCE_LADDER)).toBe(0.5); + }); + + it('is case and padding insensitive', () => { + expect(scoreVerdict(' grounded ', GROUNDEDNESS_LADDER)).toBe(1); + }); + + it('returns null for an unrecognised verdict rather than scoring it 0', () => { + // "No correctness analysis available" appears in real data; scoring it 0 + // would look identical to a hallucinating answer. + expect(scoreVerdict('No correctness analysis available', FACTUALITY_LADDER)).toBeNull(); + expect(scoreVerdict('', GROUNDEDNESS_LADDER)).toBeNull(); + expect(scoreVerdict(undefined, GROUNDEDNESS_LADDER)).toBeNull(); + }); + + it('does not resolve inherited object properties as verdicts', () => { + expect(scoreVerdict('constructor', GROUNDEDNESS_LADDER)).toBeNull(); + expect(scoreVerdict('toString', GROUNDEDNESS_LADDER)).toBeNull(); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/jury.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/jury.ts new file mode 100644 index 0000000000000..46f0db6082a07 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/jury.ts @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { classifyFamily, type ModelFamily } from './judge_provenance'; + +/** + * Jury scoring: combine several judges' verdicts on the SAME candidate output + * into one score plus an explicit measure of how much they disagreed. + * + * Why a median and not a mean: a single judge that fails to parse an output and + * returns 0 drags a mean of three from ~0.9 to ~0.6, which is indistinguishable + * from a genuine quality problem. The median of three ignores one outlier + * entirely, so a lone broken judge cannot move the published number. + * + * Why cap per family: with five Anthropic judges and one Google judge, a + * "panel" is really one family voting six times. Capping the contribution of + * each family keeps the panel from inheriting a single family's idiosyncrasies. + */ + +export interface JuryVote { + judgeId: string; + score: number; + /** Categorical verdict, when the evaluator emits one. */ + verdict?: string; +} + +export interface JuryOptions { + /** + * Maximum votes counted from any one model family. Extra votes from an + * over-represented family are dropped (lowest-variance-first is not worth the + * complexity; we drop from the end of the family's vote list). + * Defaults to 1 — one vote per family. + */ + maxVotesPerFamily?: number; + /** + * Minimum number of counted votes for the result to be considered decided. + * Defaults to 2. + */ + minVotes?: number; +} + +export interface JuryResult { + /** Median of the counted votes; null when there were not enough votes. */ + score: number | null; + /** Votes actually counted after per-family capping. */ + counted: JuryVote[]; + /** Votes discarded because their family was already at its cap. */ + dropped: JuryVote[]; + /** max - min across counted votes. 0 means unanimous. */ + disagreement: number; + /** True when every counted vote carries the same categorical verdict. */ + verdictUnanimous: boolean; + /** Distinct families represented among counted votes. */ + families: ModelFamily[]; + /** False when fewer than `minVotes` votes were available. */ + decided: boolean; +} + +function median(values: number[]): number { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; +} + +/** + * Aggregate a panel's votes on one cell. + * + * Votes with a non-finite score are discarded before anything else: a judge + * that errored should not be able to vote, and letting NaN through would + * silently poison the median. + */ +export function aggregateJury(votes: JuryVote[], options: JuryOptions = {}): JuryResult { + const maxPerFamily = Math.max(1, options.maxVotesPerFamily ?? 1); + const minVotes = Math.max(1, options.minVotes ?? 2); + + const usable = votes.filter((v) => Number.isFinite(v.score)); + const perFamily = new Map(); + const counted: JuryVote[] = []; + const dropped: JuryVote[] = []; + + for (const vote of usable) { + const family = classifyFamily(vote.judgeId); + const used = perFamily.get(family) ?? 0; + if (used < maxPerFamily) { + perFamily.set(family, used + 1); + counted.push(vote); + } else { + dropped.push(vote); + } + } + + if (counted.length === 0) { + return { + score: null, + counted, + dropped, + disagreement: 0, + verdictUnanimous: false, + families: [], + decided: false, + }; + } + + const scores = counted.map((v) => v.score); + const verdicts = counted.map((v) => v.verdict).filter((v): v is string => v !== undefined); + + return { + score: median(scores), + counted, + dropped, + disagreement: Math.max(...scores) - Math.min(...scores), + verdictUnanimous: verdicts.length > 0 && new Set(verdicts).size === 1, + families: [...new Set(counted.map((v) => classifyFamily(v.judgeId)))].sort(), + decided: counted.length >= minVotes, + }; +} + +/** + * An ordinal ladder over a judge's categorical verdict. + * + * Measured on the persona matrix: scoring Groundedness by its summary verdict + * flips across identical repetitions 51.9% of the time, versus 90.4% for the + * geometric mean over a claim list the judge re-extracts each run. The verdict + * is the more reproducible instrument, so it is the one worth ranking on. + */ +export type VerdictLadder = Record; + +export const GROUNDEDNESS_LADDER: VerdictLadder = { + GROUNDED: 1, + GROUNDED_WITH_DISCLOSURE: 0.85, + MINOR_HALLUCINATIONS: 0.5, + MAJOR_HALLUCINATIONS: 0, +}; + +export const FACTUALITY_LADDER: VerdictLadder = { + ACCURATE: 1, + MINOR_INACCURACIES: 0.5, + MAJOR_INACCURACIES: 0, +}; + +export const RELEVANCE_LADDER: VerdictLadder = { + RELEVANT: 1, + PARTIALLY_RELEVANT: 0.5, + IRRELEVANT: 0, +}; + +/** + * Verdict ladders keyed by evaluator name, for callers mapping stored score + * documents. Evaluators absent from this map have no verdict vocabulary and + * keep their continuous score. + */ +export const VERDICT_LADDERS: Record = { + Groundedness: GROUNDEDNESS_LADDER, + Factuality: FACTUALITY_LADDER, + Relevance: RELEVANCE_LADDER, +}; + +/** + * Map a categorical verdict onto its ordinal score. + * + * Returns null for an unrecognised verdict rather than guessing: an unknown + * verdict string means the judge returned something the ladder was not built + * for, and scoring it as 0 would look like a failing answer. + */ +export function scoreVerdict( + verdict: string | undefined | null, + ladder: VerdictLadder +): number | null { + const key = String(verdict ?? '') + .trim() + .toUpperCase(); + if (!key) { + return null; + } + return Object.prototype.hasOwnProperty.call(ladder, key) ? ladder[key] : null; +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/jury_adapters.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/jury_adapters.test.ts new file mode 100644 index 0000000000000..895b6f194a224 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/jury_adapters.test.ts @@ -0,0 +1,218 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + selectJury, + checkJuryCoverage, + personaMatrixJury, + attackDiscoveryJury, + JURY_ADAPTERS, +} from './jury_adapters'; +import type { ReplayCell } from './replay_plan'; + +const cell = (overrides: Partial = {}): ReplayCell => ({ + executionId: 'exec-1', + exampleId: '0', + modelId: 'model-a', + question: 'What happened?', + expected: 'A reference answer.', + agentResponse: 'An agent answer.', + steps: [], + recordedAt: '2026-09-08T00:00:00.000Z', + ...overrides, +}); + +describe('selectJury', () => { + it('resolves a jury by suite id', () => { + expect(selectJury('attack-discovery-agent-builder')?.name).toBe('attack-discovery'); + expect(selectJury('security-persona-matrix')?.name).toBe('persona-matrix'); + }); + + it('resolves a jury by adapter name', () => { + expect(selectJury('attack-discovery')?.name).toBe('attack-discovery'); + }); + + it('returns undefined for an unregistered suite rather than defaulting', () => { + // Defaulting to the persona jury is precisely the bug this registry fixes: + // it produced Factuality/Relevance verdicts for an AD replay. + expect(selectJury('security-automatic-migrations')).toBeUndefined(); + expect(selectJury('totally-unknown-suite')).toBeUndefined(); + }); +}); + +describe('personaMatrixJury.toArgs', () => { + it('builds args from a complete cell', () => { + const args = personaMatrixJury.toArgs(cell()); + expect(args).not.toBeNull(); + expect(args!.input).toEqual({ question: 'What happened?' }); + expect(args!.expected).toEqual({ expected: 'A reference answer.' }); + }); + + it('rejects a cell with no agent response', () => { + expect(personaMatrixJury.toArgs(cell({ agentResponse: '' }))).toBeNull(); + }); + + it('rejects a cell with no prose reference', () => { + expect(personaMatrixJury.toArgs(cell({ expected: '' }))).toBeNull(); + }); +}); + +describe('attackDiscoveryJury.toArgs', () => { + const adCell = (output: unknown, expectedStructured: unknown = { criteria: ['c1'] }) => + cell({ + // AD grades the structured payload, not the transcript, so a cell with + // no final message is still replayable. + agentResponse: '', + taskOutput: output, + expectedStructured, + }); + + it('grades insights even when the transcript has no final message', () => { + const args = attackDiscoveryJury.toArgs( + adCell({ insights: [{ title: 'Suspicious curl', alertIds: ['a1'] }] }) + ); + expect(args).not.toBeNull(); + expect((args!.output as { insights: unknown[] }).insights).toHaveLength(1); + }); + + it('rejects a cell whose insights are empty', () => { + expect(attackDiscoveryJury.toArgs(adCell({ insights: [] }))).toBeNull(); + }); + + it('rejects a cell with no ground truth to grade against', () => { + expect( + attackDiscoveryJury.toArgs(adCell({ insights: [{ title: 't' }] }, { criteria: [] })) + ).toBeNull(); + }); +}); + +describe('attackDiscoveryJury.criteriaFor', () => { + const args = { + input: { question: 'Run attack discovery' }, + output: { insights: [{ title: 'Suspicious curl', alertIds: ['a1'] }], errors: [] }, + expected: { criteria: ['c1', 'c2'], attackDiscoveries: [{ title: 'ref', alertIds: ['a1'] }] }, + metadata: {}, + }; + + it('emits both Criteria and Rubric invocations', () => { + const specs = attackDiscoveryJury.criteriaFor!(args); + expect(specs.map((s) => s.name)).toEqual(['Criteria', 'Rubric']); + }); + + it('passes the suite criteria through verbatim', () => { + const specs = attackDiscoveryJury.criteriaFor!(args); + expect(specs.find((s) => s.name === 'Criteria')!.criteria).toEqual(['c1', 'c2']); + }); + + it('passes each of the 7 rubric items as its own criterion', () => { + // One criterion per requirement is what lets the shared criteria judge + // return partial credit. Collapsing them into a single "5 of 7 -> Y or N" + // question, as this adapter used to, scored 95.6% of cells at exactly 1.0 + // and left the AD column unable to rank any model against another. + const rubric = attackDiscoveryJury.criteriaFor!(args).find( + (s) => s.name === 'Rubric' + )!.criteria; + + expect(rubric).toHaveLength(7); + expect(rubric.some((c) => c.includes('alertIds'))).toBe(true); + expect(rubric.some((c) => c.includes('MITRE'))).toBe(true); + // The threshold must not come back: a pass rule on top of per-item scores + // re-collapses them to binary. + expect(rubric.some((c) => c.includes('at least 5 of the 7'))).toBe(false); + // Every criterion carries the reference so it can be judged standalone. + expect(rubric.every((c) => c.includes('Reference:'))).toBe(true); + }); + + it('prefers rubric items injected by the caller over its own copy', () => { + // The suite owns the rubric; this adapter keeps a fallback. When the CLI + // supplies the suite's list, a rubric change in the suite must reach the + // replay rather than being shadowed by the stale local copy. + const specs = attackDiscoveryJury.criteriaFor!({ + ...args, + metadata: { ...args.metadata, rubricCriteria: ['only item A', 'only item B'] }, + } as typeof args); + const rubric = specs.find((s) => s.name === 'Rubric')!.criteria; + + expect(rubric).toHaveLength(2); + expect(rubric[0]).toContain('only item A'); + expect(rubric.every((c) => c.includes('Reference:'))).toBe(true); + }); + + it('omits the Rubric invocation when there is no reference discovery', () => { + const specs = attackDiscoveryJury.criteriaFor!({ + ...args, + expected: { criteria: ['c1'], attackDiscoveries: [] }, + }); + expect(specs.map((s) => s.name)).toEqual(['Criteria']); + }); +}); + +describe('checkJuryCoverage', () => { + it('accepts results carrying the jury evaluators', () => { + const result = checkJuryCoverage(attackDiscoveryJury, [ + { name: 'Criteria', score: 1 }, + { name: 'Rubric', score: 0.5 }, + ]); + expect(result.ok).toBe(true); + expect(result.missing).toEqual([]); + }); + + it('rejects persona verdicts returned for an AD replay', () => { + // The exact failure that produced 253 unusable cells. + const result = checkJuryCoverage(attackDiscoveryJury, [ + { name: 'Factuality', score: 0 }, + { name: 'Relevance', score: 0.5 }, + ]); + expect(result.ok).toBe(false); + expect(result.unexpected).toEqual(['Factuality', 'Relevance']); + }); + + it('reports a partially-covered jury without failing it', () => { + const result = checkJuryCoverage(attackDiscoveryJury, [{ name: 'Criteria', score: 1 }]); + expect(result.ok).toBe(true); + expect(result.missing).toEqual(['Rubric']); + }); + + it('does not accept a run that produced no verdicts at all', () => { + // Failure mode 2: the jury resolves without throwing but grades nothing. + // The run exits 0 with no failures, so only an assertion on the evaluator + // names present distinguishes it from a successful rejudge. + const result = checkJuryCoverage(attackDiscoveryJury, []); + expect(result.ok).toBe(false); + expect(result.missing).toEqual(['Criteria', 'Rubric']); + }); + + it('does not accept verdicts whose scores are all null', () => { + // A judge that answers but cannot parse its own output yields named + // evaluators carrying no score. Counting the name as coverage would let an + // ungraded column read as refreshed. + const result = checkJuryCoverage(attackDiscoveryJury, [ + { name: 'Criteria', score: null }, + { name: 'Rubric', score: null }, + ]); + expect(result.ok).toBe(false); + expect(result.missing).toEqual(['Criteria', 'Rubric']); + }); +}); + +describe('JURY_ADAPTERS registry', () => { + it('declares evaluator names for every jury', () => { + for (const jury of JURY_ADAPTERS) { + expect(jury.evaluatorNames.length).toBeGreaterThan(0); + } + }); + + it('never maps one suite id to two juries', () => { + const seen = new Set(); + for (const jury of JURY_ADAPTERS) { + for (const suiteId of jury.suiteIds) { + expect(seen.has(suiteId)).toBe(false); + seen.add(suiteId); + } + } + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/jury_adapters.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/jury_adapters.ts new file mode 100644 index 0000000000000..c7715d8203f14 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/jury_adapters.ts @@ -0,0 +1,296 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Per-suite jury adapters for judge replay. + * + * `ext rejudge` originally hardcoded the persona-matrix jury: it always built + * the correctness/groundedness evaluators and always emitted Factuality, + * Relevance, Groundedness and Sequence Accuracy. Pointing it at another suite + * therefore produced a confident, exit-0 result that measured the wrong thing -- + * an Attack Discovery replay came back MAJOR_INACCURACIES on 244/253 cells + * because a prose-comparison judge was grading structured discoveries against a + * reference it had never been given. The scores looked like model quality and + * were really an instrument mismatch. + * + * A jury adapter answers two questions per suite: + * 1. which evaluator names the suite's judged column consists of, and + * 2. how a stored golden cell is reshaped into the args those evaluators read. + * + * The second half matters as much as the first. Golden records a cell as + * `task.output`, but each suite's evaluators read a different slice of it: + * persona-matrix wants the message transcript, Attack Discovery wants + * `insights`. Passing the persona shape to an AD evaluator yields a null score, + * which aggregates into a silently empty column rather than a loud failure. + */ + +import type { ReplayCell } from './replay_plan'; +import type { RejudgeScore } from './run_rejudge'; + +/** A judged cell reshaped into the argument object a suite's evaluators expect. */ +export interface JuryArgs { + input: Record; + output: Record; + expected: Record; + metadata: Record; +} + +export interface JuryAdapter { + /** Adapter id, reported in logs so a replay names the jury it used. */ + name: string; + /** + * Golden `metadata.suite_id` values this jury serves. Matching on the suite + * id recorded in the score document -- rather than on the dataset path -- is + * deliberate: the dataset file is a CLI argument that can point anywhere, + * while the suite id is what the published matrix column is keyed by. + */ + suiteIds: string[]; + /** + * Evaluator names this jury recomputes, i.e. the judged evaluators of the + * suite's column. Used to verify a replay refreshed the column it claims to. + */ + evaluatorNames: string[]; + /** Reshape a stored cell into evaluator args, or null when unreplayable. */ + toArgs: (cell: ReplayCell) => JuryArgs | null; + /** + * Criteria-judge invocations this jury needs, when its evaluators are all + * built on the shared criteria judge. + * + * Suites whose evaluators live in a private, solutions-side package cannot be + * imported by this platform package. Where those evaluators are thin wrappers + * over `createCriteriaEvaluator`, the jury restates the same criteria and args + * so the replay runs the identical judge rather than importing across the + * boundary or, worse, substituting a different rubric. + */ + criteriaFor?: (args: JuryArgs) => Array<{ + name: string; + criteria: string[]; + args: JuryArgs; + }>; +} + +const isRecord = (v: unknown): v is Record => + typeof v === 'object' && v !== null && !Array.isArray(v); + +/** + * persona-matrix: prose answers graded by the correctness/groundedness pair. + * + * `steps` are forwarded because the groundedness judge checks each claim + * against the tool-call history; replaying with only the final message makes + * every specific claim unverifiable. + */ +export const personaMatrixJury: JuryAdapter = { + name: 'persona-matrix', + suiteIds: ['security-persona-matrix'], + evaluatorNames: ['Factuality', 'Relevance', 'Groundedness', 'Sequence Accuracy'], + toArgs: (cell) => { + // Both a final message and a prose reference are required: the correctness + // judge grades one against the other, and replaying with either missing + // manufactures a MAJOR_INACCURACIES verdict from absent data rather than + // from a weak answer. + if (!cell.agentResponse || !cell.expected) { + return null; + } + return { + input: { question: cell.question }, + output: { messages: [{ message: cell.agentResponse }], steps: cell.steps }, + expected: { expected: cell.expected }, + metadata: {}, + }; + }, +}; + +/** + * Attack Discovery: `Criteria` and `Rubric` are the LLM-judged evaluators, and + * both read the structured discoveries under `output.insights`, not the message + * transcript. + * + * A cell whose insights are absent or empty is NOT replayable. Both evaluators + * short-circuit to a null score when their reference or submission is empty, so + * replaying such a cell would overwrite a real score with N/A -- the emptiness + * would then read as a model that discovered nothing, when in fact the + * trajectory was never captured. + */ +export const attackDiscoveryJury: JuryAdapter = { + name: 'attack-discovery', + suiteIds: ['attack-discovery-agent-builder', 'attack-discovery'], + evaluatorNames: ['Criteria', 'Rubric'], + toArgs: (cell) => { + const output = isRecord(cell.taskOutput) ? cell.taskOutput : undefined; + const insights = output?.insights; + if (!Array.isArray(insights) || insights.length === 0) { + return null; + } + // Ground truth drives both evaluators: Criteria needs `criteria[]` and + // Rubric needs `attackDiscoveries`. With neither, both short-circuit to a + // null score, and replaying would overwrite a real verdict with N/A. + const expected = isRecord(cell.expectedStructured) ? cell.expectedStructured : undefined; + const hasCriteria = Array.isArray(expected?.criteria) && expected!.criteria.length > 0; + const hasDiscoveries = + Array.isArray(expected?.attackDiscoveries) && expected!.attackDiscoveries.length > 0; + if (!expected || (!hasCriteria && !hasDiscoveries)) { + return null; + } + return { + input: { question: cell.question }, + output: { insights, errors: Array.isArray(output?.errors) ? output.errors : [] }, + // AD ground truth is structured (criteria[] + attackDiscoveries); the + // reference adapter renders a prose form for the correctness judge, but + // these evaluators need the original objects. + expected, + metadata: {}, + }; + }, + /** + * Mirrors the suite's Criteria and Rubric evaluators. + * + * Both serialise the graded artefact to JSON and hand it to the shared + * criteria judge -- Criteria against the annotated `criteria[]`, Rubric + * against the 7 requirements compared with the reference discoveries. + * + * Each rubric requirement is passed as its own criterion, matching the + * suite's evaluator. The previous form collapsed all 7 into one string + * ending "5 of 7 -> Y or N", which scored 95.6% of cells at exactly 1.0 + * and could not rank. Prefer `metadata.rubricCriteria` when the caller + * supplies the suite's own list, so the two definitions cannot drift. + */ + criteriaFor: (args) => { + const specs: Array<{ name: string; criteria: string[]; args: JuryArgs }> = []; + const expected = args.expected as { + criteria?: unknown; + attackDiscoveries?: unknown; + }; + const insights = (args.output.insights as unknown[]) ?? []; + const errors = (args.output.errors as unknown[]) ?? []; + + const serializedOutput = JSON.stringify({ insights, errors }, null, 2); + + const criteria = Array.isArray(expected?.criteria) ? (expected.criteria as string[]) : []; + if (criteria.length > 0) { + specs.push({ + name: 'Criteria', + criteria, + args: { + input: args.input, + expected: { expected: serializedOutput }, + output: { messages: [{ message: serializedOutput }], steps: [], errors }, + metadata: args.metadata, + }, + }); + } + + const referenceInsights = Array.isArray(expected?.attackDiscoveries) + ? (expected.attackDiscoveries as Array>) + : []; + if (referenceInsights.length > 0) { + const toRubricShape = (list: Array>) => + list.map((insight) => ({ + title: insight.title ?? '', + summaryMarkdown: insight.summaryMarkdown ?? '', + detailsMarkdown: insight.detailsMarkdown ?? '', + entitySummaryMarkdown: insight.entitySummaryMarkdown ?? '', + mitreAttackTactics: insight.mitreAttackTactics ?? [], + alertIds: insight.alertIds ?? [], + })); + + const reference = JSON.stringify( + { attackDiscoveries: toRubricShape(referenceInsights) }, + null, + 2 + ); + const submission = JSON.stringify( + { attackDiscoveries: toRubricShape(insights as Array>) }, + null, + 2 + ); + + // The rubric items come from the suite module when the CLI could load + // them (`--dataset` resolves the suite's own evaluator), so a rubric + // change in the suite reaches a rejudge instead of being silently + // shadowed by a stale copy here. The inline list below is the fallback + // for callers that supply no rubric, and is deliberately the SAME + // per-item form -- not the old collapsed Y/N question. + const injected = (args.metadata as { rubricCriteria?: unknown } | undefined)?.rubricCriteria; + if (Array.isArray(injected) && injected.length > 0) { + specs.push({ + name: 'Rubric', + criteria: (injected as string[]).map((item) => `${item} Reference: ${reference}`), + args: { + input: args.input, + expected: { expected: reference }, + output: { messages: [{ message: submission }], steps: [], errors }, + metadata: args.metadata, + }, + }); + return specs; + } + + const rubricItems = [ + 'Is the submission non-empty and well-formed JSON with an array of attackDiscoveries?', + 'Do the detailsMarkdown values capture the overall essence of the reference, allowing slight differences in wording but not omitting or misrepresenting key incidents?', + 'Does the submission mention at least half of the same entities (host or user) as the reference?', + 'Are the summaryMarkdown values at least partially similar and summarizing the same incidents?', + 'Are the title values at least partially similar and mentioning the same incidents?', + 'Do more than half of the alertIds in the submission overlap with the alertIds in the reference?', + 'Are the MITRE tactics consistent with the reference?', + ]; + + specs.push({ + name: 'Rubric', + criteria: rubricItems.map((item) => `${item} Reference: ${reference}`), + args: { + input: args.input, + expected: { expected: reference }, + output: { messages: [{ message: submission }], steps: [], errors }, + metadata: args.metadata, + }, + }); + } + + return specs; + }, +}; + +export const JURY_ADAPTERS: JuryAdapter[] = [personaMatrixJury, attackDiscoveryJury]; + +/** + * Resolve the jury for a suite. + * + * Returns undefined for an unregistered suite rather than falling back to the + * persona jury. That fallback is exactly what produced a plausible-looking but + * meaningless Attack Discovery replay, so an unknown suite must fail loudly at + * the CLI boundary instead of being silently mis-scored. + */ +export function selectJury(suiteId: string | undefined): JuryAdapter | undefined { + if (!suiteId) { + return undefined; + } + return JURY_ADAPTERS.find((jury) => jury.suiteIds.includes(suiteId)); +} + +/** + * Check that a replay refreshed the evaluators the suite's column is built from. + * + * A replay whose verdicts all fall outside the jury's evaluator set has + * measured something other than the column it claims to update -- the exact + * failure that made an exit-0 Attack Discovery rejudge unusable. + */ +export function checkJuryCoverage( + jury: JuryAdapter, + scores: RejudgeScore[] +): { ok: boolean; unexpected: string[]; missing: string[] } { + // A named evaluator carrying no score has not graded anything: the judge + // answered but its verdict could not be parsed into a number. Counting the + // name alone as coverage would let an ungraded column read as refreshed. + const produced = new Set(scores.filter((s) => s.score !== null).map((s) => s.name)); + const expected = new Set(jury.evaluatorNames); + return { + ok: [...produced].some((name) => expected.has(name)), + unexpected: [...produced].filter((name) => !expected.has(name)), + missing: [...expected].filter((name) => !produced.has(name)), + }; +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/load_matrix_config.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/load_matrix_config.test.ts new file mode 100644 index 0000000000000..3c6e7c8b54557 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/load_matrix_config.test.ts @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + parseMatrixConfig, + DEFAULT_EXCLUDED_EVALUATORS, + applyModelOverrides, + parseModelOverride, +} from './load_matrix_config'; + +describe('parseMatrixConfig', () => { + const minimalConfig = { + columns: [{ id: 'alert_triage', label: 'Alert Triage', suites: ['security-alert-triage'] }], + models: [{ id: 'eis/foo', label: 'Foo' }], + }; + + it('applies defaults for optional fields', () => { + const config = parseMatrixConfig(minimalConfig); + + expect(config.branch).toBe('main'); + expect(config.defaultScale).toBe(10); + expect(config.decimals).toBe(2); + expect(config.notRecommendedBelow).toBe(0); + expect(config.notRecommendedLabel).toBe('Not recommended'); + expect(config.notRecommendedCountsAsZeroInOverall).toBe(true); + expect(config.overall).toEqual({ + label: 'Overall', + mode: 'weighted', + excludeSaturatedEvaluators: false, + }); + expect(config.showOverall).toBe(true); + expect(config.composites).toEqual([]); + expect(config.layout).toBeUndefined(); + expect(config.columns[0].weight).toBe(1); + expect(config.columns[0].group).toBeUndefined(); + expect(config.models[0].openSource).toBe(false); + expect(config.excludeEvaluators).toEqual([...DEFAULT_EXCLUDED_EVALUATORS]); + }); + + it('accepts grouped columns, composites, a layout, and showOverall', () => { + const config = parseMatrixConfig({ + ...minimalConfig, + showOverall: false, + columns: [ + { id: 'a', label: 'A', group: 'Agent Builder', suites: ['s-a'] }, + { id: 'b', label: 'B', group: 'Agent Builder', suites: ['s-b'] }, + ], + composites: [{ id: 'ab', label: 'AB Score', from: ['a', 'b'] }], + layout: ['a', 'b', 'ab'], + }); + + expect(config.showOverall).toBe(false); + expect(config.columns[0].group).toBe('Agent Builder'); + expect(config.composites).toEqual([{ id: 'ab', label: 'AB Score', from: ['a', 'b'] }]); + expect(config.layout).toEqual(['a', 'b', 'ab']); + }); + + it('throws when a composite has no source columns', () => { + expect(() => + parseMatrixConfig({ + ...minimalConfig, + composites: [{ id: 'ab', label: 'AB', from: [] }], + }) + ).toThrow(); + }); + + it('allows overriding the evaluator exclusion list (including emptying it)', () => { + expect( + parseMatrixConfig({ ...minimalConfig, excludeEvaluators: [] }).excludeEvaluators + ).toEqual([]); + expect( + parseMatrixConfig({ ...minimalConfig, excludeEvaluators: ['Latency'] }).excludeEvaluators + ).toEqual(['Latency']); + }); + + it('throws when a column has no suites', () => { + expect(() => + parseMatrixConfig({ + ...minimalConfig, + columns: [{ id: 'x', label: 'X', suites: [] }], + }) + ).toThrow(); + }); + + it('throws when there are no columns or models', () => { + expect(() => parseMatrixConfig({ columns: [], models: [] })).toThrow(); + }); + + it('rejects an invalid overall mode', () => { + expect(() => parseMatrixConfig({ ...minimalConfig, overall: { mode: 'nope' } })).toThrow(); + }); +}); + +describe('applyModelOverrides', () => { + const base = parseMatrixConfig({ + title: 'Weekly', + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'], weight: 1 }], + models: [ + { id: 'weekly-1', label: 'Weekly One' }, + { id: 'weekly-2', label: 'Weekly Two' }, + ], + }); + + it('returns the config untouched when no overrides are given', () => { + expect(applyModelOverrides(base, [])).toBe(base); + }); + + it('replaces rather than appends, so an on-demand run shows only what was asked for', () => { + const result = applyModelOverrides(base, ['custom-a']); + expect(result.models).toEqual([{ id: 'custom-a', label: 'custom-a', openSource: false }]); + }); + + it('does not mutate the weekly config', () => { + applyModelOverrides(base, ['custom-a']); + expect(base.models.map((m) => m.id)).toEqual(['weekly-1', 'weekly-2']); + }); + + it('parses label and explicit open-source marker', () => { + expect(applyModelOverrides(base, ['qwen3-72b:Qwen3 72B:open-source']).models[0]).toEqual({ + id: 'qwen3-72b', + label: 'Qwen3 72B', + openSource: true, + }); + }); + + it('defaults the label to the id and openSource to false', () => { + expect(parseModelOverride('gpt-5')).toEqual({ + id: 'gpt-5', + label: 'gpt-5', + openSource: false, + }); + }); + + it('rejects a bogus third segment instead of silently treating it as proprietary', () => { + expect(() => parseModelOverride('gpt-5:GPT-5:oss')).toThrow(/literal "open-source"/); + }); + + it('rejects too many segments', () => { + expect(() => parseModelOverride('a:b:open-source:c')).toThrow(/at most 3/); + }); + + it('rejects an empty id', () => { + expect(() => parseModelOverride(':Label')).toThrow(/model id is required/); + }); + + it('rejects duplicate ids', () => { + expect(() => applyModelOverrides(base, ['dup', 'dup:Other'])).toThrow(/Duplicate --model id/); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/load_matrix_config.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/load_matrix_config.ts new file mode 100644 index 0000000000000..26a180ae1189b --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/load_matrix_config.ts @@ -0,0 +1,419 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Fs from 'fs'; +import { schema, type TypeOf } from '@kbn/config-schema'; + +/** + * Upper bounds for schema fields. The config is a static repo-controlled JSON + * file (not request input), so these exist to satisfy bounded-input validation + * rather than to mitigate a real DoS vector; the limits are generous enough that + * any realistic matrix config stays well within them. + */ +const MAX_STRING_LENGTH = 1024; +const MAX_ARRAY_SIZE = 1000; + +/** + * Schema for the LLM performance matrix configuration file. + * + * The matrix engine is domain-agnostic: a config file maps human-facing matrix + * columns onto the eval `suite.id` / `example.dataset.id` / `evaluator.name` + * values stored in the `kibana-evaluations` data stream, declares the model + * allowlist (with display names + open-source classification), and describes how + * raw evaluator scores are normalized onto the published 0-10 scale. + */ +const columnSchema = schema.object({ + /** Stable identifier for the column (used as the CSV/JSON key). */ + id: schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), + /** Human-facing column header (e.g. "Alert Triage"). */ + label: schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), + /** + * Optional grouped-header label (e.g. "Agent Builder"). Columns sharing a + * `group` render under one spanning header in the published docs page. Carried + * through to the JSON artifact; the flat CSV/markdown keep one header row. + */ + group: schema.maybe(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH })), + /** `suite.id` values whose scores contribute to this column. */ + suites: schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { + minSize: 1, + maxSize: MAX_ARRAY_SIZE, + }), + /** Optional restriction to specific `example.dataset.id` values. */ + datasetIds: schema.maybe( + schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { + maxSize: MAX_ARRAY_SIZE, + }) + ), + /** + * Optional restriction to `example.id` prefixes. Splits a single dataset + * (all examples share one `example.dataset.id`) into per-category columns: + * e.g. ['alert-analysis'] matches examples `alert-analysis-a/b/c`. When set, + * the matrix query fetches per-example scores (stripped experiment-scores + * route) and buckets them by prefix instead of the dataset-level stats. + */ + examplePrefixes: schema.maybe( + schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { + minSize: 1, + maxSize: MAX_ARRAY_SIZE, + }) + ), + /** Optional restriction to specific `evaluator.name` values. */ + evaluators: schema.maybe( + schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { + maxSize: MAX_ARRAY_SIZE, + }) + ), + /** + * Multiplier applied to the (weighted) mean evaluator score before rounding. + * Defaults to `defaultScale` (10) so 0-1 evaluator scores map onto the 0-10 + * scale. Set to 1 for evaluators that already emit a 0-10 score. + */ + scale: schema.maybe(schema.number({ min: 0 })), + /** + * Git branch this column's experiments are read from, overriding the + * top-level `branch`. Suites do not all publish on the same branch, and a + * single global branch renders every column that is absent from it blank. + */ + branch: schema.maybe( + schema.oneOf([ + schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), + // A suite whose models are split across branches needs every branch + // queried and unioned; a single string silently drops the others' rows. + schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { + minSize: 1, + }), + ]) + ), + /** + * Opt this column's suites out of the global `scoring.excludeSelfJudged`. + * + * Set only where self-preference has been measured and found absent: a judge + * that also appears as a ranked model normally has its own row dropped, which + * blanks a real cell. Leave unset to keep the strict global policy. + */ + allowSelfJudged: schema.maybe(schema.boolean()), + /** Relative weight of this column in the legacy Overall score. Defaults to 1. */ + weight: schema.number({ defaultValue: 1, min: 0 }), +}); + +/** + * A derived ("composite") column whose cell is the equal-weighted mean of other + * columns' cells. `from` may reference base columns or earlier-defined + * composites, so composites can be layered (e.g. "Overall Score" averages the + * "Agent Builder Score" composite alongside two standalone feature columns). + * + * Aggregation mirrors the legacy Overall: "Not recommended" sources count as 0 + * (when `notRecommendedCountsAsZeroInOverall` is set) and missing sources are + * skipped, so a composite reflects the data that actually exists. + */ +const compositeSchema = schema.object({ + /** Stable identifier for the composite (used as the CSV/JSON key). */ + id: schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), + /** Human-facing column header (e.g. "Agent Builder Score"). */ + label: schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), + /** Optional grouped-header label (see `columnSchema.group`). */ + group: schema.maybe(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH })), + /** Column/composite ids whose cells are averaged into this composite. */ + from: schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { + minSize: 1, + maxSize: MAX_ARRAY_SIZE, + }), +}); + +const modelSchema = schema.object({ + /** Primary `task.model.id` value to match against. */ + id: schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), + /** Display name shown in the published matrix (e.g. "Claude Sonnet 4"). */ + label: schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), + /** Additional `task.model.id` values that should map to the same row. */ + matchIds: schema.maybe( + schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { + maxSize: MAX_ARRAY_SIZE, + }) + ), + /** Renders the model under the "Open-source models" table when true. */ + openSource: schema.boolean({ defaultValue: false }), +}); + +/** + * Default `evaluator.name` values excluded from column aggregation. + * + * Security eval suites register an "observability" tier of trace-based + * evaluators (latency, token counts, tool-call counts, skill invocation) + * alongside the 0-1 quality evaluators. Those emit raw magnitudes (thousands of + * tokens, milliseconds) rather than a 0-1 score, so averaging them into a column + * and multiplying by the 0-10 scale produces wildly inflated cells. We exclude + * them by default; a config may override `excludeEvaluators` to opt back in. + * + * Matching is name-prefix based so dynamically-named evaluators such as + * `Skill Invoked (alert-analysis)` are caught by the `Skill Invoked` entry. + */ +export const DEFAULT_EXCLUDED_EVALUATORS: readonly string[] = [ + 'Latency', + 'Tool Calls', + 'Input Tokens', + 'Output Tokens', + 'Cached Tokens', + 'Skill Invoked', +]; + +export const matrixConfigSchema = schema.object({ + /** Page/table title (informational; used in the markdown artifact). */ + title: schema.string({ defaultValue: 'LLM performance matrix', maxLength: MAX_STRING_LENGTH }), + /** Default git branch to pull experiments from (CLI `--branch` overrides). */ + branch: schema.string({ defaultValue: 'main', maxLength: MAX_STRING_LENGTH }), + /** Only consider experiments newer than `now-d`. */ + lookbackDays: schema.number({ defaultValue: 45, min: 1 }), + /** + * Scoring policy for judged (LLM-graded) evaluators. + * + * Measured on the persona matrix (8,482 golden score documents): reading the + * judge's categorical verdict instead of its continuous score, and dropping + * grades from judges that were neither EIS-pinned nor independent of the + * graded model, cuts the rerun flip rate from 83.3% to 33.3%. + * + * Defaults are off so existing matrices keep their published numbers; a + * matrix opts in explicitly and its scores change. + */ + scoring: schema.maybe( + schema.object({ + /** + * Score judged evaluators by their categorical verdict (SUPPORTED / + * PARTIALLY_SUPPORTED / ...) rather than the geometric mean over a + * per-run claim list. Contract evaluators are unaffected — they are + * already deterministic and expose no verdict. + */ + useVerdictLadder: schema.boolean({ defaultValue: false }), + /** + * Drop scores produced by judges that are not EIS-pinned (LiteLLM + * aliases, HuggingFace repo paths, local quantisations). Those judges + * cannot be re-run to reproduce a number. + */ + requireEisJudge: schema.boolean({ defaultValue: false }), + /** Drop scores where a model graded its own output. */ + excludeSelfJudged: schema.boolean({ defaultValue: false }), + }) + ), + /** Default multiplier applied to evaluator means when a column omits `scale`. */ + defaultScale: schema.number({ defaultValue: 10, min: 0 }), + /** Decimal places used when rounding cell values. */ + decimals: schema.number({ defaultValue: 2, min: 0, max: 6 }), + /** Cells at/under this value (after scaling) render as `notRecommendedLabel`. */ + notRecommendedBelow: schema.number({ defaultValue: 0, min: 0 }), + /** + * Tool-call count above which a cell is reported as a possible runaway loop. + * Observability only: thrashing cells do not score worse, so this must not + * become a score penalty. It exists because the cost is otherwise invisible + * (one cell burned 115 calls / 3.78M input tokens). 0 disables the check. + */ + toolCallWarnAbove: schema.number({ defaultValue: 0, min: 0 }), + /** + * Minimum scored columns a model needs before `Overall` is published as a + * number. A model scored on 2 of 24 prompts can average 10.0 and outrank every + * frontier model, so below this floor `Overall` becomes `insufficient-coverage`, + * which ranks last. Default 0 preserves existing behaviour. + */ + minCoverage: schema.number({ defaultValue: 0, min: 0 }), + /** Text rendered when a model fails / lacks data for a column. */ + notRecommendedLabel: schema.string({ + defaultValue: 'Not recommended', + maxLength: MAX_STRING_LENGTH, + }), + /** + * When true, "Not recommended" cells count as 0 in the Overall score and in + * composite columns (matches the published matrix behavior where failures drag + * the average down). + */ + notRecommendedCountsAsZeroInOverall: schema.boolean({ defaultValue: true }), + /** + * `evaluator.name` values (matched by prefix) excluded from every column's + * aggregation. Defaults to the observability-tier evaluators, which emit raw + * magnitudes rather than 0-1 quality scores. Set to `[]` to include everything. + */ + excludeEvaluators: schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { + defaultValue: [...DEFAULT_EXCLUDED_EVALUATORS], + maxSize: MAX_ARRAY_SIZE, + }), + overall: schema.object({ + label: schema.string({ defaultValue: 'Overall', maxLength: MAX_STRING_LENGTH }), + mode: schema.oneOf([schema.literal('weighted'), schema.literal('mean')], { + defaultValue: 'weighted', + }), + /** + * Run-to-run standard deviation of the overall score, measured by + * re-running one model on an unchanged commit. Set it and rows within + * 2x the 95% interval are grouped into a tie tier instead of being + * presented as ranked. Omit to keep the raw ordering. + */ + runStdev: schema.maybe(schema.number({ min: 0, max: 10 })), + /** + * Drop evaluators that score every model almost identically from the + * Overall aggregate. Such an evaluator carries no ranking information but + * still takes an equal share of the mean, compressing the spread between + * models and hiding the evaluators that do separate them. Detection is + * mechanical (see `evaluator_saturation.ts`); saturated evaluators are + * still rendered in their own columns, just not folded into Overall. + */ + excludeSaturatedEvaluators: schema.boolean({ defaultValue: false }), + }), + /** + * Renders the legacy single "Overall" column (weighted/mean over every base + * column) at the far right. Set to `false` when the layout expresses its own + * Overall via a composite, to avoid a duplicate trailing column. + */ + showOverall: schema.boolean({ defaultValue: true }), + columns: schema.arrayOf(columnSchema, { minSize: 1, maxSize: MAX_ARRAY_SIZE }), + /** Derived columns averaged from base columns / earlier composites. */ + composites: schema.arrayOf(compositeSchema, { defaultValue: [], maxSize: MAX_ARRAY_SIZE }), + /** + * Explicit left-to-right display order of base + composite column ids. When + * omitted, base columns render first (config order), then composites. The + * legacy Overall column (when `showOverall`) is always appended last. + */ + layout: schema.maybe( + schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { + maxSize: MAX_ARRAY_SIZE, + }) + ), + models: schema.arrayOf(modelSchema, { minSize: 1, maxSize: MAX_ARRAY_SIZE }), + /** + * Opt-in token/cost axis. The quality matrix deliberately drops the + * observability-tier evaluators (see {@link DEFAULT_EXCLUDED_EVALUATORS}) + * because their raw magnitudes would blow out the 0-10 scale. This block + * re-admits them on a *separate* axis: instead of being folded into a column + * mean, the named evaluators are aggregated per (model, column) into + * `matrix.tokenCost`, preserving mean/min/max in native units. + * + * Omitted by default, so existing configs are unaffected. + */ + tokenCost: schema.maybe( + schema.object({ + /** Evaluator name (prefix-matched) contributing input-token magnitudes. */ + inputEvaluator: schema.string({ + defaultValue: 'Input Tokens', + maxLength: MAX_STRING_LENGTH, + }), + /** Evaluator name (prefix-matched) contributing output-token magnitudes. */ + outputEvaluator: schema.string({ + defaultValue: 'Output Tokens', + maxLength: MAX_STRING_LENGTH, + }), + /** + * Column ids the token axis is aggregated over. Defaults to every base + * column in the config when omitted. + */ + columns: schema.maybe( + schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { + maxSize: MAX_ARRAY_SIZE, + }) + ), + }) + ), + /** + * Opt-in provenance extras rendered into the HTML footer. The fixture + * fingerprint pins which dataset/tool seed revision the scores came from + * (drift between benchmark fixture generations is a known divergence + * source), and methodology notes document scoring-semantics changes a + * reader must know before comparing against older matrices. + */ + provenance: schema.maybe( + schema.object({ + fixtureFingerprint: schema.maybe(schema.string({ maxLength: MAX_STRING_LENGTH })), + methodologyNotes: schema.maybe( + schema.arrayOf(schema.string({ minLength: 1, maxLength: 2000 }), { maxSize: 20 }) + ), + }) + ), +}); + +export type MatrixConfig = TypeOf; +export type MatrixTokenCostConfig = NonNullable; +export type MatrixColumnConfig = TypeOf; +export type MatrixCompositeConfig = TypeOf; +export type MatrixModelConfig = TypeOf; + +export const parseMatrixConfig = (raw: unknown): MatrixConfig => matrixConfigSchema.validate(raw); + +/** + * Parses a `--model` CLI value into a model config entry. + * + * Format: `id[:label][:open-source]`, e.g. + * `gpt-5-preview` + * `gpt-5-preview:GPT-5 Preview` + * `qwen3-72b:Qwen3 72B:open-source` + * + * Labels may contain spaces but not colons; the third segment is an explicit + * open-source marker rather than a substring guess at the model name. + */ +export const parseModelOverride = (raw: string): MatrixModelConfig => { + const segments = raw.split(':').map((segment) => segment.trim()); + const [id, label, openSourceFlag] = segments; + + if (!id) { + throw new Error( + `Invalid --model value "${raw}": model id is required (format: id[:label][:open-source]).` + ); + } + if (segments.length > 3) { + throw new Error( + `Invalid --model value "${raw}": expected at most 3 colon-separated segments (id[:label][:open-source]).` + ); + } + if (openSourceFlag !== undefined && openSourceFlag !== 'open-source') { + throw new Error( + `Invalid --model value "${raw}": third segment must be the literal "open-source", got "${openSourceFlag}".` + ); + } + + return { id, label: label || id, openSource: openSourceFlag === 'open-source' }; +}; + +/** + * Replaces the config's model set with an ad-hoc one for on-demand runs. + * + * The weekly matrix is a fixed, reviewed model set that must stay stable + * across runs, so this deliberately does not mutate the config file — an + * on-demand run with `--model` is a throwaway view over the same score data. + */ +export const applyModelOverrides = ( + config: MatrixConfig, + rawModels: readonly string[] +): MatrixConfig => { + if (rawModels.length === 0) { + return config; + } + + const models = rawModels.map(parseModelOverride); + const seen = new Set(); + for (const model of models) { + if (seen.has(model.id)) { + throw new Error(`Duplicate --model id "${model.id}".`); + } + seen.add(model.id); + } + + return { ...config, models }; +}; + +export const loadMatrixConfig = (configPath: string): MatrixConfig => { + if (!Fs.existsSync(configPath)) { + throw new Error(`Matrix config not found at: ${configPath}`); + } + + let parsed: unknown; + try { + parsed = JSON.parse(Fs.readFileSync(configPath, 'utf-8')); + } catch (error) { + throw new Error( + `Failed to parse matrix config at ${configPath}: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + + return parseMatrixConfig(parsed); +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/local_git_state.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/local_git_state.ts new file mode 100644 index 0000000000000..2be5a3eb491b3 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/local_git_state.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { execFileSync } from 'child_process'; +import type { ToolingLog } from '@kbn/tooling-log'; + +export interface LocalGitState { + sha?: string; + dirty?: boolean; +} + +/** + * Read the generator's own git state so the artifact says which code produced + * it. `BUILDKITE_COMMIT` covers CI; locally it is unset, which is precisely + * where an artifact gets regenerated, compared against an older one, and + * mistaken for a reproduction of it. + * + * Never throws: provenance is a label on the artifact, not a reason to fail a + * run that otherwise succeeded. + */ +export function readLocalGitState(repoRoot: string, log: ToolingLog): LocalGitState { + const git = (args: string[]) => + execFileSync('git', args, { + cwd: repoRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + + try { + return { + sha: git(['rev-parse', 'HEAD']).trim(), + // Untracked files are excluded: they cannot change generator behaviour, + // and scratch files in a worktree would otherwise flag every local run. + dirty: git(['status', '--porcelain', '--untracked-files=no']).trim().length > 0, + }; + } catch (error) { + log.debug( + `Could not read local git state for provenance: ${ + error instanceof Error ? error.message : String(error) + }` + ); + return {}; + } +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/merge_rejudged_scores.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/merge_rejudged_scores.test.ts new file mode 100644 index 0000000000000..f175c97ec9415 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/merge_rejudged_scores.test.ts @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mergeRejudgedScores, type GoldenCell, type RejudgedCell } from './merge_rejudged_scores'; + +const golden = ( + experimentId: string, + datasetId: string, + evaluatorName: string, + score: number, + judgeModelId?: string +): GoldenCell => ({ experimentId, datasetId, evaluatorName, score, judgeModelId }); + +const rejudged = ( + experimentId: string, + datasetId: string, + evaluatorName: string, + score: number, + judgeModelId: string +): RejudgedCell => ({ experimentId, datasetId, evaluatorName, score, judgeModelId }); + +describe('mergeRejudgedScores', () => { + it('replaces the score and the judge of a re-judged cell', () => { + const result = mergeRejudgedScores( + [golden('exp-1', 'linux-curl', 'Criteria', 0.4, 'anthropic-claude-4.6-sonnet')], + [rejudged('exp-1', 'linux-curl', 'Criteria', 0.9, 'google-gemini-3.1-pro')] + ); + + expect(result.replaced).toBe(1); + expect(result.cells[0].score).toBe(0.9); + expect(result.cells[0].judgeModelId).toBe('google-gemini-3.1-pro'); + expect(result.unmatched).toEqual([]); + }); + + it('leaves a cell the rejudge did not cover completely untouched', () => { + const untouched = golden('exp-1', 'wmi-lateral', 'Rubric', 0.5, 'anthropic-claude-4.6-sonnet'); + const result = mergeRejudgedScores( + [golden('exp-1', 'linux-curl', 'Criteria', 0.4, 'anthropic-claude-4.6-sonnet'), untouched], + [rejudged('exp-1', 'linux-curl', 'Criteria', 0.9, 'google-gemini-3.1-pro')] + ); + + expect(result.replaced).toBe(1); + expect(result.cells[1]).toEqual(untouched); + }); + + it('does not overwrite a different execution of the same model and column', () => { + // Keying on anything less specific than experiment_id would let a rejudge + // of exp-1 clobber exp-2, which is a different run of the same board cell. + const other = golden('exp-2', 'linux-curl', 'Criteria', 0.5, 'anthropic-claude-4.6-sonnet'); + const result = mergeRejudgedScores( + [golden('exp-1', 'linux-curl', 'Criteria', 0.4, 'anthropic-claude-4.6-sonnet'), other], + [rejudged('exp-1', 'linux-curl', 'Criteria', 0.9, 'google-gemini-3.1-pro')] + ); + + expect(result.replaced).toBe(1); + expect(result.cells[1]).toEqual(other); + }); + + it('reports a re-judged cell with no golden match instead of appending it', () => { + const result = mergeRejudgedScores( + [golden('exp-1', 'linux-curl', 'Criteria', 0.4)], + [rejudged('exp-1', 'bits-mshta', 'Criteria', 0.9, 'google-gemini-3.1-pro')] + ); + + expect(result.replaced).toBe(0); + expect(result.cells).toHaveLength(1); + expect(result.cells[0].score).toBe(0.4); + expect(result.unmatched).toHaveLength(1); + expect(result.unmatched[0].datasetId).toBe('bits-mshta'); + }); + + it('distinguishes evaluators within the same execution and column', () => { + const result = mergeRejudgedScores( + [ + golden('exp-1', 'linux-curl', 'Criteria', 0.4), + golden('exp-1', 'linux-curl', 'Rubric', 0.4), + ], + [rejudged('exp-1', 'linux-curl', 'Rubric', 0.9, 'google-gemini-3.1-pro')] + ); + + expect(result.cells[0].score).toBe(0.4); + expect(result.cells[1].score).toBe(0.9); + expect(result.replaced).toBe(1); + }); + + it('preserves board order and length so a partial merge cannot drop a row', () => { + const board = [ + golden('exp-1', 'a', 'Criteria', 0.1), + golden('exp-1', 'b', 'Criteria', 0.2), + golden('exp-1', 'c', 'Criteria', 0.3), + ]; + const result = mergeRejudgedScores(board, [ + rejudged('exp-1', 'b', 'Criteria', 0.9, 'google-gemini-3.1-pro'), + ]); + + expect(result.cells).toHaveLength(3); + expect(result.cells.map((c) => c.datasetId)).toEqual(['a', 'b', 'c']); + }); + + it('is a no-op for an empty rejudge artifact', () => { + const board = [golden('exp-1', 'a', 'Criteria', 0.1, 'anthropic-claude-4.5-haiku')]; + const result = mergeRejudgedScores(board, []); + + expect(result.replaced).toBe(0); + expect(result.cells).toEqual(board); + expect(result.unmatched).toEqual([]); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/merge_rejudged_scores.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/merge_rejudged_scores.ts new file mode 100644 index 0000000000000..a8225a621a6b2 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/merge_rejudged_scores.ts @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Merging re-judged scores back onto a golden board. + * + * A rejudge run grades an existing set of model outputs with a different judge. + * It produces an artifact, not a golden write: the scores live in a local JSON + * file and the published board still shows whatever judge originally graded + * those cells. This module is the missing step — it decides, deterministically + * and without touching the network, which golden cells a rejudge artifact + * replaces. + * + * The merge is keyed on `experiment_id` because that is the only field that + * identifies the *run* whose outputs were re-graded. Keying on model id alone + * would let a rejudge of one execution silently overwrite a different (possibly + * newer) execution of the same model. + */ + +/** One re-graded cell, as produced by a rejudge run. */ +export interface RejudgedCell { + /** The golden execution whose outputs were re-graded. */ + experimentId: string; + /** Dataset/column key within that execution. */ + datasetId: string; + evaluatorName: string; + /** The re-computed score. */ + score: number; + /** The judge that produced `score`. */ + judgeModelId: string; +} + +/** One existing golden cell, as read off the published board. */ +export interface GoldenCell { + experimentId: string; + datasetId: string; + evaluatorName: string; + score: number; + judgeModelId?: string; +} + +export interface MergeOutcome { + /** The merged board: same length and order as the input `golden`. */ + cells: GoldenCell[]; + /** Cells whose score/judge were replaced by a re-judged value. */ + replaced: number; + /** + * Re-judged cells that matched no golden cell. These are NOT appended: a + * rejudge can only re-grade outputs that already exist on the board, so a + * non-matching cell means the artifact and the board disagree about what was + * run, and silently adding it would fabricate a cell the board never had. + */ + unmatched: RejudgedCell[]; +} + +const keyOf = (c: { experimentId: string; datasetId: string; evaluatorName: string }): string => + `${c.experimentId}\u0000${c.datasetId}\u0000${c.evaluatorName}`; + +/** + * Apply a rejudge artifact to a golden board. + * + * Pure and order-preserving: the caller decides whether to persist the result. + * Cells the rejudge did not cover are returned untouched, so a partial rejudge + * (the common case — a rejudge usually covers one column) leaves the rest of + * the board exactly as it was. + */ +export function mergeRejudgedScores(golden: GoldenCell[], rejudged: RejudgedCell[]): MergeOutcome { + const byKey = new Map(); + for (const cell of rejudged) { + // Last write wins within a single artifact: a rejudge that graded the same + // cell twice has no principled tie-break, and taking the last keeps the + // merge deterministic for a given artifact ordering. + byKey.set(keyOf(cell), cell); + } + + const consumed = new Set(); + let replaced = 0; + + const cells = golden.map((cell) => { + const key = keyOf(cell); + const update = byKey.get(key); + if (!update) return cell; + + consumed.add(key); + replaced += 1; + return { + ...cell, + score: update.score, + judgeModelId: update.judgeModelId, + }; + }); + + const unmatched = rejudged.filter((cell) => !consumed.has(keyOf(cell))); + + return { cells, replaced, unmatched }; +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/merge_shard_experiments.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/merge_shard_experiments.test.ts new file mode 100644 index 0000000000000..40b0d88d4a3be --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/merge_shard_experiments.test.ts @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mergeShardDatasets, pickShardExperiments } from './merge_shard_experiments'; +import type { EvaluationExperimentSummary } from '@kbn/evals-common'; +import type { AggregatedDatasetScores } from './query_matrix_scores'; + +const experiment = ( + executionId: string, + timestamp: string, + modelId = 'glm-5.3-flash' +): EvaluationExperimentSummary => + ({ + experiment_id: executionId, + execution_id: executionId, + timestamp, + task_model: { id: modelId }, + } as EvaluationExperimentSummary); + +describe('pickShardExperiments', () => { + it('keeps every shard of the same sweep, not just the newest', () => { + const picked = pickShardExperiments([ + experiment('sweep-100-s1of2::suite::glm-5.3-flash', '2026-09-03T10:00:00Z'), + experiment('sweep-100-s2of2::suite::glm-5.3-flash', '2026-09-03T10:30:00Z'), + ]); + + expect(picked.map((e) => e.execution_id)).toEqual([ + 'sweep-100-s1of2::suite::glm-5.3-flash', + 'sweep-100-s2of2::suite::glm-5.3-flash', + ]); + }); + + it('prefers the newest sweep and does not blend shards across sweeps', () => { + // Mixing an old shard with a new one would silently report stale scores + // as part of the current run. + // + // The old sweep's LAST shard must finish AFTER the new sweep's FIRST one, + // otherwise comparing experiments individually still happens to pick the + // right sweep and this test proves nothing. + const picked = pickShardExperiments([ + experiment('sweep-100-s1of2::suite::glm-5.3-flash', '2026-09-03T10:00:00Z'), + experiment('sweep-100-s2of2::suite::glm-5.3-flash', '2026-09-03T21:00:00Z'), + experiment('sweep-200-s1of2::suite::glm-5.3-flash', '2026-09-03T20:00:00Z'), + experiment('sweep-200-s2of2::suite::glm-5.3-flash', '2026-09-03T22:00:00Z'), + ]); + + expect(picked.map((e) => e.execution_id)).toEqual([ + 'sweep-200-s1of2::suite::glm-5.3-flash', + 'sweep-200-s2of2::suite::glm-5.3-flash', + ]); + }); + + it('ranks a sweep by its last-finishing shard, not its last-listed one', () => { + // Shards finish out of order. If a sweep is ranked by whichever shard + // happens to be seen last, a sweep whose final shard finished EARLY is + // undervalued and the stale sweep wins. + const picked = pickShardExperiments([ + experiment('sweep-100-s1of2::suite::glm-5.3-flash', '2026-09-03T10:00:00Z'), + experiment('sweep-100-s2of2::suite::glm-5.3-flash', '2026-09-03T19:00:00Z'), + // Newest sweep: its max (21:00) beats sweep-100's max (19:00), but its + // LAST-listed shard (12:00) does not. + experiment('sweep-200-s1of2::suite::glm-5.3-flash', '2026-09-03T21:00:00Z'), + experiment('sweep-200-s2of2::suite::glm-5.3-flash', '2026-09-03T12:00:00Z'), + ]); + + expect(picked.map((e) => e.execution_id)).toEqual([ + 'sweep-200-s1of2::suite::glm-5.3-flash', + 'sweep-200-s2of2::suite::glm-5.3-flash', + ]); + }); + + it('leaves an unsharded run exactly as-is', () => { + const picked = pickShardExperiments([ + experiment('plain-run::suite::glm-5.3-flash', '2026-09-03T10:00:00Z'), + ]); + + expect(picked.map((e) => e.execution_id)).toEqual(['plain-run::suite::glm-5.3-flash']); + }); + + it('returns nothing for no experiments', () => { + expect(pickShardExperiments([])).toEqual([]); + }); +}); + +describe('mergeShardDatasets', () => { + const ds = ( + datasetId: string, + evaluatorName: string, + mean: number, + count: number + ): AggregatedDatasetScores => ({ + datasetId, + datasetName: datasetId, + evaluators: [{ evaluatorName, mean, count }], + }); + + it('combines disjoint datasets from different shards', () => { + const merged = mergeShardDatasets([ + [ds('prefix:alerts', 'Factuality', 8, 6)], + [ds('prefix:hunting', 'Factuality', 6, 5)], + ]); + + expect(merged).toHaveLength(2); + expect(merged.map((d) => d.datasetId).sort()).toEqual(['prefix:alerts', 'prefix:hunting']); + }); + + it('weights the mean by count when shards share a dataset', () => { + // 6 examples at 8 and 4 examples at 3 is 6.0, NOT the 5.5 a naive + // mean-of-means would report. + const merged = mergeShardDatasets([ + [ds('prefix:alerts', 'Factuality', 8, 6)], + [ds('prefix:alerts', 'Factuality', 3, 4)], + ]); + + expect(merged).toHaveLength(1); + expect(merged[0].evaluators).toHaveLength(1); + expect(merged[0].evaluators[0].count).toBe(10); + expect(merged[0].evaluators[0].mean).toBeCloseTo(6.0, 10); + }); + + it('keeps distinct evaluators of a shared dataset separate', () => { + const merged = mergeShardDatasets([ + [ds('prefix:alerts', 'Factuality', 8, 6)], + [ds('prefix:alerts', 'Groundedness', 4, 6)], + ]); + + expect(merged).toHaveLength(1); + const names = merged[0].evaluators.map((e) => e.evaluatorName).sort(); + expect(names).toEqual(['Factuality', 'Groundedness']); + }); + + it('carries min/max across shards', () => { + const merged = mergeShardDatasets([ + [ + { + datasetId: 'd', + datasetName: 'd', + evaluators: [{ evaluatorName: 'F', mean: 8, count: 2, min: 7, max: 9 }], + }, + ], + [ + { + datasetId: 'd', + datasetName: 'd', + evaluators: [{ evaluatorName: 'F', mean: 4, count: 2, min: 1, max: 5 }], + }, + ], + ]); + + expect(merged[0].evaluators[0].min).toBe(1); + expect(merged[0].evaluators[0].max).toBe(9); + }); + + it('ignores a shard that produced no datasets', () => { + const merged = mergeShardDatasets([[ds('prefix:alerts', 'Factuality', 8, 6)], []]); + + expect(merged).toHaveLength(1); + expect(merged[0].evaluators[0].count).toBe(6); + }); + + it('returns nothing when every shard is empty', () => { + expect(mergeShardDatasets([[], []])).toEqual([]); + }); + + it('does not mutate its input', () => { + const shard = [ds('prefix:alerts', 'Factuality', 8, 6)]; + mergeShardDatasets([shard, [ds('prefix:alerts', 'Factuality', 2, 2)]]); + + expect(shard[0].evaluators[0].mean).toBe(8); + expect(shard[0].evaluators[0].count).toBe(6); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/merge_shard_experiments.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/merge_shard_experiments.ts new file mode 100644 index 0000000000000..6fd6c567b35ed --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/merge_shard_experiments.ts @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EvaluationExperimentSummary } from '@kbn/evals-common'; +import type { AggregatedDatasetScores } from './query_matrix_scores'; + +/** + * A sharded sweep runs one model on several VMs, each covering a stride of the + * dataset and writing its OWN execution_id (`-sof::::`). + * + * Picking only the newest experiment per model therefore renders a single + * shard and blanks every example the other shards covered. These helpers keep + * all shards of one sweep and fold their scores into a single row. + */ + +/** `sweep-123-s2of4::suite::model` -> `sweep-123`. Undefined when unsharded. */ +const shardBase = (executionId: string | undefined): string | undefined => { + if (!executionId) { + return undefined; + } + const runId = executionId.split('::')[0]; + const match = /^(.*)-s\d+of\d+$/.exec(runId); + return match ? match[1] : undefined; +}; + +/** + * Returns every experiment belonging to the newest sweep for these inputs. + * + * Shards of one sweep share a base run id, so recency is decided per SWEEP + * rather than per experiment -- shards finish at different times and comparing + * them individually would drop the slower ones. An unsharded run has no base + * and is returned alone, preserving existing behaviour. + */ +export const pickShardExperiments = ( + experiments: EvaluationExperimentSummary[] +): EvaluationExperimentSummary[] => { + if (experiments.length === 0) { + return []; + } + + const bySweep = new Map(); + + for (const candidate of experiments) { + const base = shardBase(candidate.execution_id); + // Unsharded runs are their own group, keyed by execution id so two of them + // never merge into one another. + const key = base ?? `unsharded:${candidate.execution_id ?? candidate.experiment_id}`; + const at = Date.parse(candidate.timestamp); + if (!Number.isFinite(at)) { + continue; + } + + const group = bySweep.get(key); + if (!group) { + bySweep.set(key, { members: [candidate], at }); + continue; + } + group.members.push(candidate); + // A sweep is as recent as its LAST finishing shard. + group.at = Math.max(group.at, at); + } + + const newest = [...bySweep.values()].reduce<{ + members: EvaluationExperimentSummary[]; + at: number; + } | null>((best, group) => (!best || group.at > best.at ? group : best), null); + + if (!newest) { + return []; + } + + return [...newest.members].sort((a, b) => + (a.execution_id ?? '').localeCompare(b.execution_id ?? '') + ); +}; + +/** + * Folds per-shard dataset scores into one set. + * + * Means are combined WEIGHTED BY COUNT. Averaging the shard means directly + * would misreport any sweep whose shards cover different numbers of examples + * -- and stride sharding produces exactly that (21 examples over 2 shards is + * 11 and 10). + */ +export const mergeShardDatasets = ( + shards: AggregatedDatasetScores[][] +): AggregatedDatasetScores[] => { + const byDataset = new Map< + string, + { + datasetId: string; + datasetName: string; + evaluators: Map< + string, + { sum: number; count: number; min: number | undefined; max: number | undefined } + >; + } + >(); + + for (const shard of shards) { + for (const dataset of shard) { + let entry = byDataset.get(dataset.datasetId); + if (!entry) { + entry = { + datasetId: dataset.datasetId, + datasetName: dataset.datasetName, + evaluators: new Map(), + }; + byDataset.set(dataset.datasetId, entry); + } + + for (const evaluator of dataset.evaluators) { + const agg = entry.evaluators.get(evaluator.evaluatorName) ?? { + sum: 0, + count: 0, + min: undefined, + max: undefined, + }; + // Reconstruct the score total from mean x count so the combined mean + // stays weighted. + agg.sum += evaluator.mean * evaluator.count; + agg.count += evaluator.count; + agg.min = + evaluator.min === undefined ? agg.min : Math.min(agg.min ?? evaluator.min, evaluator.min); + agg.max = + evaluator.max === undefined ? agg.max : Math.max(agg.max ?? evaluator.max, evaluator.max); + entry.evaluators.set(evaluator.evaluatorName, agg); + } + } + } + + return [...byDataset.values()].map((entry) => ({ + datasetId: entry.datasetId, + datasetName: entry.datasetName, + evaluators: [...entry.evaluators.entries()].map(([evaluatorName, agg]) => ({ + evaluatorName, + mean: agg.count === 0 ? 0 : agg.sum / agg.count, + count: agg.count, + ...(agg.min === undefined ? {} : { min: agg.min }), + ...(agg.max === undefined ? {} : { max: agg.max }), + })), + })); +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/model_comparison.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/model_comparison.test.ts new file mode 100644 index 0000000000000..68847aecff93b --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/model_comparison.test.ts @@ -0,0 +1,387 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + bootstrapInterval, + pairedComparison, + buildTiers, + buildFamilyRecommendations, + buildEfficiency, + buildComparisonReport, + type ModelObservation, + type ComparisonConfig, +} from './model_comparison'; + +const COLUMNS = [ + 'alert-analysis-a', + 'alert-analysis-b', + 'alert-analysis-c', + 'threat-hunting-a', + 'threat-hunting-b', + 'threat-hunting-c', +]; + +const familyOf = (columnId: string): string => columnId.replace(/-[abc]$/, ''); + +const config: ComparisonConfig = { + columns: COLUMNS, + familyOf, + minColumns: 6, + resamples: 600, + seed: 42, +}; + +const observation = ( + modelId: string, + modelLabel: string, + values: number[], + extra: Partial = {} +): ModelObservation => ({ + modelId, + modelLabel, + scores: Object.fromEntries(COLUMNS.map((c, i) => [c, values[i]])), + ...extra, +}); + +const rngOf = (seed: number): (() => number) => { + const modulus = 2147483647; + let state = Math.floor(Math.abs(seed)) % modulus; + if (state === 0) state = 1; + return () => { + state = (state * 16807) % modulus; + return (state - 1) / (modulus - 1); + }; +}; + +describe('bootstrapInterval', () => { + it('brackets the observed mean', () => { + const ci = bootstrapInterval([5, 6, 7, 8, 5, 6], 500, rngOf(1)); + expect(ci.mean).toBeCloseTo(6.1667, 3); + expect(ci.low).toBeLessThanOrEqual(ci.mean); + expect(ci.high).toBeGreaterThanOrEqual(ci.mean); + }); + + it('reports a zero-width interval for a constant model', () => { + const ci = bootstrapInterval([7, 7, 7, 7], 500, rngOf(2)); + expect(ci.low).toBeCloseTo(7, 6); + expect(ci.high).toBeCloseTo(7, 6); + }); + + it('widens the interval as the model becomes more erratic', () => { + const steady = bootstrapInterval([6, 6.1, 5.9, 6, 6.1, 5.9], 800, rngOf(3)); + const erratic = bootstrapInterval([1, 10, 2, 9, 3, 8], 800, rngOf(3)); + expect(erratic.high - erratic.low).toBeGreaterThan(steady.high - steady.low); + }); + + it('cannot form an interval from a single observation', () => { + const ci = bootstrapInterval([8], 500, rngOf(4)); + expect(ci.low).toBe(8); + expect(ci.high).toBe(8); + }); +}); + +describe('pairedComparison', () => { + it('separates models that differ on every prompt', () => { + const result = pairedComparison([9, 9, 9, 9, 9, 9], [3, 3, 3, 3, 3, 3], 500, rngOf(5)); + expect(result.probability).toBe(1); + expect(result.low).toBeGreaterThan(0); + }); + + it('leaves 0 inside the interval for models that trade wins', () => { + const result = pairedComparison([9, 2, 9, 2, 9, 2], [2, 9, 2, 9, 2, 9], 800, rngOf(6)); + expect(result.low).toBeLessThan(0); + expect(result.high).toBeGreaterThan(0); + }); + + it('detects a small but consistent edge that an unpaired view would bury', () => { + // Prompt difficulty swamps the gap; pairing cancels it out. + const a = [9.4, 2.4, 7.4, 1.4, 8.4, 3.4]; + const b = [9.0, 2.0, 7.0, 1.0, 8.0, 3.0]; + const result = pairedComparison(a, b, 1000, rngOf(7)); + expect(result.probability).toBe(1); + expect(result.low).toBeGreaterThan(0); + }); + + it('returns an undecided result for mismatched inputs', () => { + expect(pairedComparison([1, 2], [1], 100, rngOf(8)).probability).toBe(0.5); + expect(pairedComparison([], [], 100, rngOf(8)).probability).toBe(0.5); + }); +}); + +describe('buildTiers', () => { + it('groups models that cannot be told apart into one tier', () => { + const { tiers } = buildTiers( + [ + observation('a', 'Model A', [7, 6, 8, 7, 6, 8]), + observation('b', 'Model B', [6.9, 6.1, 7.9, 7.1, 6.1, 7.9]), + observation('c', 'Model C', [7.1, 5.9, 8.1, 6.9, 5.9, 8.1]), + ], + config + ); + expect(tiers).toHaveLength(1); + expect(tiers[0].members.map((m) => m.modelLabel)).toEqual(['Model A', 'Model B', 'Model C']); + }); + + it('splits a clearly weaker model into its own tier', () => { + const { tiers } = buildTiers( + [ + observation('a', 'Strong', [9, 9, 9, 9, 9, 9]), + observation('b', 'Weak', [2, 2, 2, 2, 2, 2]), + ], + config + ); + expect(tiers).toHaveLength(2); + expect(tiers[0].members[0].modelLabel).toBe('Strong'); + expect(tiers[1].members[0].modelLabel).toBe('Weak'); + }); + + it('orders within a tier alphabetically, never by an insignificant decimal', () => { + const { tiers } = buildTiers( + [ + observation('z', 'Zeta', [7.01, 7.01, 7.01, 7.01, 7.01, 7.01]), + observation('a', 'Alpha', [7, 7, 7, 7, 7, 7]), + ], + config + ); + expect(tiers).toHaveLength(1); + expect(tiers[0].members.map((m) => m.modelLabel)).toEqual(['Alpha', 'Zeta']); + }); + + it('does not split a tier on a statistically clean but trivial gap', () => { + // Zero-variance rows separate on any gap at all, so statistical + // significance alone would publish a 0.01 "win" as a tier boundary. + const { tiers } = buildTiers( + [ + observation('z', 'Zeta', [7.01, 7.01, 7.01, 7.01, 7.01, 7.01]), + observation('a', 'Alpha', [7, 7, 7, 7, 7, 7]), + ], + { ...config, minMeaningfulDifference: 0.25 } + ); + expect(tiers).toHaveLength(1); + }); + + it('still splits once the gap clears the practical floor', () => { + const { tiers } = buildTiers( + [observation('z', 'Zeta', [8, 8, 8, 8, 8, 8]), observation('a', 'Alpha', [7, 7, 7, 7, 7, 7])], + { ...config, minMeaningfulDifference: 0.25 } + ); + expect(tiers).toHaveLength(2); + }); + + it('refuses to rank a partial row against complete ones', () => { + const partial: ModelObservation = { + modelId: 'p', + modelLabel: 'Partial', + // A 10.0 on 2 of 6 columns must not outrank a complete 7.0 row. + scores: { 'alert-analysis-a': 10, 'alert-analysis-b': 10 }, + }; + const { tiers, excluded } = buildTiers( + [observation('a', 'Complete', [7, 7, 7, 7, 7, 7]), partial], + config + ); + expect(excluded).toEqual([ + { modelId: 'p', modelLabel: 'Partial', scoredColumns: 2, requiredColumns: 6 }, + ]); + expect(tiers.flatMap((t) => t.members.map((m) => m.modelId))).not.toContain('p'); + }); + + it('is deterministic across runs so a published report does not drift', () => { + const models = [ + observation('a', 'A', [7, 6, 8, 7, 6, 8]), + observation('b', 'B', [5, 5, 6, 5, 5, 6]), + observation('c', 'C', [9, 9, 8, 9, 9, 9]), + ]; + expect(JSON.stringify(buildTiers(models, config))).toEqual( + JSON.stringify(buildTiers(models, config)) + ); + }); +}); + +describe('buildFamilyRecommendations', () => { + it('names a different leader per family when models specialise', () => { + const recs = buildFamilyRecommendations( + [ + observation('a', 'AlertSpecialist', [9, 9, 9, 3, 3, 3]), + observation('b', 'HuntSpecialist', [3, 3, 3, 9, 9, 9]), + ], + config + ); + const alert = recs.find((r) => r.family === 'alert-analysis')!; + const hunt = recs.find((r) => r.family === 'threat-hunting')!; + expect(alert.leaders).toEqual(['AlertSpecialist']); + expect(hunt.leaders).toEqual(['HuntSpecialist']); + }); + + it('surfaces a model that leads one family and trails another', () => { + const recs = buildFamilyRecommendations( + [ + observation('a', 'Lopsided', [9, 9, 9, 2, 2, 2]), + observation('b', 'Even', [6, 6, 6, 6, 6, 6]), + ], + config + ); + expect(recs.find((r) => r.family === 'alert-analysis')!.leaders).toContain('Lopsided'); + expect(recs.find((r) => r.family === 'threat-hunting')!.laggards).toContain('Lopsided'); + }); + + it('reports co-leaders rather than inventing a single winner', () => { + const recs = buildFamilyRecommendations( + [observation('a', 'A', [8, 8, 8, 5, 5, 5]), observation('b', 'B', [8.1, 8.1, 8.1, 5, 5, 5])], + config + ); + expect(recs.find((r) => r.family === 'alert-analysis')!.leaders).toEqual(['A', 'B']); + }); + + it('skips models missing part of a family', () => { + const partial: ModelObservation = { + modelId: 'p', + modelLabel: 'Partial', + scores: { 'alert-analysis-a': 10 }, + }; + const recs = buildFamilyRecommendations( + [observation('a', 'Full', [7, 7, 7, 7, 7, 7]), partial], + config + ); + expect(recs.find((r) => r.family === 'alert-analysis')!.leaders).toEqual(['Full']); + }); +}); + +describe('buildEfficiency', () => { + it('marks a model beaten on quality, latency and tokens at once', () => { + const rows = buildEfficiency( + [ + observation('fast', 'Fast', [7, 7, 7, 7, 7, 7], { + latencySeconds: 25, + inputTokens: 130000, + }), + observation('slow', 'Slow', [6.9, 6.9, 6.9, 6.9, 6.9, 6.9], { + latencySeconds: 144, + inputTokens: 696000, + }), + ], + config + ); + expect(rows.find((r) => r.modelId === 'slow')!.dominatedBy).toBe('Fast'); + expect(rows.find((r) => r.modelId === 'fast')!.dominatedBy).toBeUndefined(); + }); + + it('keeps a slower model that buys real quality on the frontier', () => { + const rows = buildEfficiency( + [ + observation('fast', 'Fast', [5, 5, 5, 5, 5, 5], { + latencySeconds: 20, + inputTokens: 100000, + }), + observation('good', 'Good', [9, 9, 9, 9, 9, 9], { + latencySeconds: 90, + inputTokens: 500000, + }), + ], + config + ); + expect(rows.every((r) => r.dominatedBy === undefined)).toBe(true); + }); + + it('never reports a model as efficient just because cost data is missing', () => { + const rows = buildEfficiency( + [ + observation('measured', 'Measured', [7, 7, 7, 7, 7, 7], { + latencySeconds: 30, + inputTokens: 100000, + }), + observation('unmeasured', 'Unmeasured', [4, 4, 4, 4, 4, 4]), + ], + config + ); + const unmeasured = rows.find((r) => r.modelId === 'unmeasured')!; + expect(unmeasured.dominatedBy).toBeUndefined(); + expect(unmeasured.latencySeconds).toBeUndefined(); + }); + + it('does not let an unmeasured model dominate or be dominated on absent axes', () => { + // Removing the missing-data guards makes JS compare `undefined` with `<=`, + // which is always false -- so an unmeasured model silently becomes an + // undominated "frontier" entry. Ordering it FIRST makes that visible: + // without the guard it would dominate the measured row behind it. + const rows = buildEfficiency( + [ + observation('unmeasured', 'Unmeasured', [9, 9, 9, 9, 9, 9]), + observation('measured', 'Measured', [7, 7, 7, 7, 7, 7], { + latencySeconds: 30, + inputTokens: 100000, + }), + ], + config + ); + expect(rows.find((r) => r.modelId === 'unmeasured')!.dominatedBy).toBeUndefined(); + // The measured model must not be dominated BY the unmeasured one: we never + // measured its cost, so we cannot claim it is cheaper or faster. + expect(rows.find((r) => r.modelId === 'measured')!.dominatedBy).toBeUndefined(); + }); + + it('treats latency as a real axis, not just tokens', () => { + // Same quality and tokens, but one model is 4x slower: it is dominated only + // if the latency axis is actually evaluated. + const rows = buildEfficiency( + [ + observation('quick', 'Quick', [7, 7, 7, 7, 7, 7], { + latencySeconds: 25, + inputTokens: 100000, + }), + observation('sluggish', 'Sluggish', [7, 7, 7, 7, 7, 7], { + latencySeconds: 100, + inputTokens: 100000, + }), + ], + config + ); + expect(rows.find((r) => r.modelId === 'sluggish')!.dominatedBy).toBe('Quick'); + expect(rows.find((r) => r.modelId === 'quick')!.dominatedBy).toBeUndefined(); + }); + + it('treats token cost as a real axis, not just latency', () => { + // Same quality and latency, but one model burns 5x the tokens: it is + // dominated only if the token axis is actually evaluated. + const rows = buildEfficiency( + [ + observation('lean', 'Lean', [7, 7, 7, 7, 7, 7], { + latencySeconds: 40, + inputTokens: 100000, + }), + observation('greedy', 'Greedy', [7, 7, 7, 7, 7, 7], { + latencySeconds: 40, + inputTokens: 500000, + }), + ], + config + ); + expect(rows.find((r) => r.modelId === 'greedy')!.dominatedBy).toBe('Lean'); + expect(rows.find((r) => r.modelId === 'lean')!.dominatedBy).toBeUndefined(); + }); +}); + +describe('buildComparisonReport', () => { + it('warns when the whole field lands in a single tier', () => { + const report = buildComparisonReport( + [ + observation('a', 'A', [7, 6, 8, 7, 6, 8]), + observation('b', 'B', [6.9, 6.1, 7.9, 7.1, 6.1, 7.9]), + ], + config + ); + expect(report.allTiedWarning).toBe(true); + }); + + it('does not warn once a real separation exists', () => { + const report = buildComparisonReport( + [observation('a', 'A', [9, 9, 9, 9, 9, 9]), observation('b', 'B', [2, 2, 2, 2, 2, 2])], + config + ); + expect(report.allTiedWarning).toBe(false); + expect(report.tiers).toHaveLength(2); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/model_comparison.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/model_comparison.ts new file mode 100644 index 0000000000000..2412002482635 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/model_comparison.ts @@ -0,0 +1,400 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Decision-oriented comparison over an already-built matrix. + * + * The per-prompt grid is sound evidence; the ranked `Overall` mean laid over it + * is not. Two properties of the real data drive everything here: + * + * 1. Adjacent `Overall` ranks are not distinguishable. Paired bootstrap over + * the prompt columns puts 0 inside the 95% CI for every adjacent pair, so + * a sorted list invents precision the evidence does not carry. + * 2. Task families measure independent skills (mean inter-family rank + * correlation ~0.04, and a third of family pairs anti-correlate). There is + * no single "good at security" axis for a mean to summarise, and the + * `Overall` leader wins only one of ten families. + * + * So this module answers the questions a reader actually has -- which model for + * MY task, at what cost, and how much should I trust it -- instead of ordering + * models by a decimal that is inside its own noise floor. + */ + +/** Quality/cost facts for one model, already reduced from the matrix + traces. */ +export interface ModelObservation { + readonly modelId: string; + readonly modelLabel: string; + readonly openSource?: boolean; + /** Per-column scores. Only complete rows are comparable; see `minColumns`. */ + readonly scores: Readonly>; + /** Mean end-to-end seconds, when the instrument reported it. */ + readonly latencySeconds?: number; + /** Mean input tokens, when the instrument reported it. */ + readonly inputTokens?: number; +} + +export interface ComparisonConfig { + /** Columns that define a complete row. */ + readonly columns: readonly string[]; + /** Family id per column, e.g. `alert-analysis-a` -> `alert-analysis`. */ + readonly familyOf: (columnId: string) => string; + /** Rows below this many scored columns are reported but never ranked. */ + readonly minColumns: number; + /** Bootstrap resamples. */ + readonly resamples?: number; + /** Deterministic seed -- a published report must not move between renders. */ + readonly seed?: number; + /** + * Smallest score difference worth a tier boundary. Statistical separation is + * necessary but not sufficient: two models with near-zero variance separate + * on a 0.01 gap that no reader should act on, which is the same false + * precision this module exists to remove. Defaults to 0.25 on the 0-10 scale. + */ + readonly minMeaningfulDifference?: number; +} + +export interface ConfidenceInterval { + readonly mean: number; + readonly low: number; + readonly high: number; +} + +export interface TierEntry { + readonly modelId: string; + readonly modelLabel: string; + readonly interval: ConfidenceInterval; +} + +export interface Tier { + readonly rank: number; + readonly members: readonly TierEntry[]; +} + +export interface FamilyRecommendation { + readonly family: string; + /** Every model whose CI overlaps the best mean -- co-leaders, not one winner. */ + readonly leaders: readonly string[]; + readonly bestMean: number; + /** Models notably weak here; surfaces "great at X, terrible at Y" tradeoffs. */ + readonly laggards: readonly string[]; + readonly worstMean: number; +} + +export interface EfficiencyEntry { + readonly modelId: string; + readonly modelLabel: string; + readonly quality: number; + readonly latencySeconds?: number; + readonly inputTokens?: number; + /** Set when another model is >= quality and <= cost on every axis. */ + readonly dominatedBy?: string; +} + +export interface ExcludedRow { + readonly modelId: string; + readonly modelLabel: string; + readonly scoredColumns: number; + readonly requiredColumns: number; +} + +export interface ComparisonReport { + readonly tiers: readonly Tier[]; + readonly families: readonly FamilyRecommendation[]; + readonly efficiency: readonly EfficiencyEntry[]; + /** Rows too partial to rank. Reported so they are not silently dropped. */ + readonly excluded: readonly ExcludedRow[]; + /** True when no adjacent tier pair separated -- the field is a single blob. */ + readonly allTiedWarning: boolean; +} + +/** + * Small deterministic PRNG (Lehmer / MINSTD). + * + * Bootstrap resampling must be reproducible: a published comparison that + * reshuffles its tiers between renders is not auditable. Uses modular + * arithmetic rather than the usual bit-mixing so it stays within the repo's + * no-bitwise rule, and stays well inside float53 precision. + */ +const createRng = (seed: number): (() => number) => { + const modulus = 2147483647; + let state = Math.floor(Math.abs(seed)) % modulus; + if (state === 0) state = 1; + return () => { + state = (state * 16807) % modulus; + return (state - 1) / (modulus - 1); + }; +}; + +const mean = (xs: readonly number[]): number => + xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length; + +const quantile = (sorted: readonly number[], q: number): number => { + if (sorted.length === 0) return 0; + const idx = Math.min(sorted.length - 1, Math.max(0, Math.floor(q * sorted.length))); + return sorted[idx]; +}; + +/** + * Bootstrap CI for one model's mean across its own columns. + * + * Resampling columns (not repetitions) is deliberate: it answers "how much does + * this average depend on which tasks we happened to pick?", which is the + * question a reader has when a leaderboard claims a 0.05 lead. + */ +export const bootstrapInterval = ( + values: readonly number[], + resamples: number, + rng: () => number +): ConfidenceInterval => { + const observed = mean(values); + if (values.length < 2) return { mean: observed, low: observed, high: observed }; + const means: number[] = []; + for (let i = 0; i < resamples; i++) { + let total = 0; + for (let j = 0; j < values.length; j++) { + total += values[Math.floor(rng() * values.length)]; + } + means.push(total / values.length); + } + means.sort((a, b) => a - b); + return { mean: observed, low: quantile(means, 0.025), high: quantile(means, 0.975) }; +}; + +/** + * Paired bootstrap: resample COLUMNS once, score both models on that same + * resample. Models answer identical prompts, so the unpaired test would inflate + * the variance with per-prompt difficulty that cancels between them, and hide + * differences that are real. + * + * Returns the probability that A > B and the CI of the difference. A CI that + * spans 0 means the ordering is not evidence. + */ +export const pairedComparison = ( + a: readonly number[], + b: readonly number[], + resamples: number, + rng: () => number +): { probability: number; low: number; high: number } => { + if (a.length !== b.length || a.length === 0) { + return { probability: 0.5, low: 0, high: 0 }; + } + const diffs: number[] = []; + let wins = 0; + for (let i = 0; i < resamples; i++) { + let sa = 0; + let sb = 0; + for (let j = 0; j < a.length; j++) { + const pick = Math.floor(rng() * a.length); + sa += a[pick]; + sb += b[pick]; + } + const d = (sa - sb) / a.length; + diffs.push(d); + if (d > 0) wins++; + } + diffs.sort((x, y) => x - y); + return { + probability: wins / resamples, + low: quantile(diffs, 0.025), + high: quantile(diffs, 0.975), + }; +}; + +const isSeparated = (result: { low: number; high: number }): boolean => + !(result.low <= 0 && 0 <= result.high); + +/** + * Group models into tiers, splitting only where a paired test separates a model + * from the current tier's weakest member. Within a tier order is alphabetical, + * never by an insignificant decimal -- sorting inside a tie is exactly the + * false precision this replaces. + */ +export const buildTiers = ( + observations: readonly ModelObservation[], + config: ComparisonConfig +): { tiers: Tier[]; excluded: ExcludedRow[] } => { + const resamples = config.resamples ?? 2000; + const complete: ModelObservation[] = []; + const excluded: ExcludedRow[] = []; + + for (const obs of observations) { + const scored = config.columns.filter((c) => typeof obs.scores[c] === 'number'); + if (scored.length < config.minColumns) { + excluded.push({ + modelId: obs.modelId, + modelLabel: obs.modelLabel, + scoredColumns: scored.length, + requiredColumns: config.minColumns, + }); + } else { + complete.push(obs); + } + } + + const vectorOf = (obs: ModelObservation): number[] => + config.columns.map((c) => obs.scores[c]).filter((v): v is number => typeof v === 'number'); + + const ranked = [...complete].sort((x, y) => { + const d = mean(vectorOf(y)) - mean(vectorOf(x)); + return d !== 0 ? d : x.modelLabel.localeCompare(y.modelLabel); + }); + + const tiers: Tier[] = []; + let current: ModelObservation[] = []; + + const flush = () => { + if (current.length === 0) return; + const rng = createRng((config.seed ?? 1337) + tiers.length * 7919); + const members = current + .map((obs) => ({ + modelId: obs.modelId, + modelLabel: obs.modelLabel, + interval: bootstrapInterval(vectorOf(obs), resamples, rng), + })) + .sort((x, y) => x.modelLabel.localeCompare(y.modelLabel)); + tiers.push({ rank: tiers.length + 1, members }); + current = []; + }; + + for (const obs of ranked) { + if (current.length === 0) { + current.push(obs); + continue; + } + // Compare against the tier's weakest member: a model only starts a new tier + // when it is separated from everything already grouped above it. + const weakest = current.reduce((lo, c) => (mean(vectorOf(c)) < mean(vectorOf(lo)) ? c : lo)); + const rng = createRng((config.seed ?? 1337) + ranked.indexOf(obs) * 104729); + const result = pairedComparison(vectorOf(weakest), vectorOf(obs), resamples, rng); + const gap = mean(vectorOf(weakest)) - mean(vectorOf(obs)); + const floor = config.minMeaningfulDifference ?? 0.25; + if (isSeparated(result) && Math.abs(gap) >= floor) { + flush(); + } + current.push(obs); + } + flush(); + + return { tiers, excluded }; +}; + +/** + * Per-family leaders and laggards. + * + * Reports every model within `tolerance` of the best mean rather than a single + * winner: with three prompts per family, naming one winner would overstate the + * evidence in exactly the way the `Overall` column already does. + */ +export const buildFamilyRecommendations = ( + observations: readonly ModelObservation[], + config: ComparisonConfig, + tolerance = 0.5 +): FamilyRecommendation[] => { + const families = new Map(); + for (const column of config.columns) { + const family = config.familyOf(column); + const list = families.get(family); + if (list) list.push(column); + else families.set(family, [column]); + } + + const recommendations: FamilyRecommendation[] = []; + for (const [family, columns] of families) { + const means: Array<{ id: string; label: string; value: number }> = []; + for (const obs of observations) { + const vals = columns + .map((c) => obs.scores[c]) + .filter((v): v is number => typeof v === 'number'); + // Require the whole family: a model scored on 1 of 3 prompts is not + // comparable to one scored on all 3. + if (vals.length === columns.length && vals.length > 0) { + means.push({ id: obs.modelId, label: obs.modelLabel, value: mean(vals) }); + } + } + if (means.length === 0) continue; + const best = Math.max(...means.map((m) => m.value)); + const worst = Math.min(...means.map((m) => m.value)); + recommendations.push({ + family, + leaders: means + .filter((m) => m.value >= best - tolerance) + .map((m) => m.label) + .sort(), + bestMean: best, + laggards: means + .filter((m) => m.value <= worst + tolerance) + .map((m) => m.label) + .sort(), + worstMean: worst, + }); + } + return recommendations.sort((a, b) => a.family.localeCompare(b.family)); +}; + +/** + * Pareto frontier over quality (up), latency (down) and input tokens (down). + * + * A dominated model is one no reader should choose: something else is at least + * as accurate while being both faster and cheaper. Models missing cost data are + * returned undominated -- absent instrumentation must never read as "efficient". + */ +export const buildEfficiency = ( + observations: readonly ModelObservation[], + config: ComparisonConfig +): EfficiencyEntry[] => { + const entries: EfficiencyEntry[] = observations + .filter((obs) => { + const scored = config.columns.filter((c) => typeof obs.scores[c] === 'number'); + return scored.length >= config.minColumns; + }) + .map((obs) => { + const vals = config.columns + .map((c) => obs.scores[c]) + .filter((v): v is number => typeof v === 'number'); + return { + modelId: obs.modelId, + modelLabel: obs.modelLabel, + quality: mean(vals), + latencySeconds: obs.latencySeconds, + inputTokens: obs.inputTokens, + }; + }); + + return entries.map((entry) => { + if (entry.latencySeconds === undefined || entry.inputTokens === undefined) { + return entry; + } + const dominator = entries.find( + (other) => + other.modelId !== entry.modelId && + other.latencySeconds !== undefined && + other.inputTokens !== undefined && + other.quality >= entry.quality && + other.latencySeconds <= entry.latencySeconds! && + other.inputTokens <= entry.inputTokens! && + (other.quality > entry.quality || + other.latencySeconds < entry.latencySeconds! || + other.inputTokens < entry.inputTokens!) + ); + return dominator ? { ...entry, dominatedBy: dominator.modelLabel } : entry; + }); +}; + +/** Full decision report. */ +export const buildComparisonReport = ( + observations: readonly ModelObservation[], + config: ComparisonConfig +): ComparisonReport => { + const { tiers, excluded } = buildTiers(observations, config); + return { + tiers, + families: buildFamilyRecommendations(observations, config), + efficiency: buildEfficiency(observations, config), + excluded, + allTiedWarning: tiers.length === 1 && tiers[0]?.members.length > 1, + }; +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/model_comparison_golden.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/model_comparison_golden.test.ts new file mode 100644 index 0000000000000..af861f41ac32e --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/model_comparison_golden.test.ts @@ -0,0 +1,169 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import fs from 'fs'; +import path from 'path'; +import { + buildComparisonReport, + type ModelObservation, + type ComparisonConfig, +} from './model_comparison'; + +/** + * End-to-end contract against a real rendered matrix bundle. + * + * Unit tests prove the statistics on synthetic input; this proves the module + * reproduces the findings that justify v2 on the actual published data. It is + * skipped when the bundle is absent so CI stays hermetic. + */ +const BUNDLE = + process.env.MATRIX_BUNDLE ?? + path.resolve(__dirname, '../../../../../../../target/llm_matrix_latest/matrix.json'); + +const describeIfBundle = fs.existsSync(BUNDLE) ? describe : describe.skip; + +describeIfBundle('model_comparison against a real matrix bundle', () => { + let report: ReturnType; + let config: ComparisonConfig; + let observations: ModelObservation[]; + + beforeAll(() => { + const matrix = JSON.parse(fs.readFileSync(BUNDLE, 'utf8')); + const columns: string[] = matrix.columns.map((c: { id: string }) => c.id); + const rows = [...(matrix.proprietary ?? []), ...(matrix.openSource ?? [])]; + + const latency = new Map(); + const inputTokens = new Map(); + for (const [key, trace] of Object.entries>(matrix.traces ?? {})) { + const modelId = key.split(':')[0]; + const scores = (trace as { scores?: Record }).scores; + if (!scores) continue; + if (typeof scores.Latency === 'number') { + latency.set(modelId, (latency.get(modelId) ?? 0) + scores.Latency); + } + if (typeof scores['Input Tokens'] === 'number') { + inputTokens.set(modelId, (inputTokens.get(modelId) ?? 0) + scores['Input Tokens']); + } + } + const counts = new Map(); + for (const key of Object.keys(matrix.traces ?? {})) { + const modelId = key.split(':')[0]; + counts.set(modelId, (counts.get(modelId) ?? 0) + 1); + } + + observations = rows.map( + (row: { + modelId: string; + modelLabel: string; + openSource?: boolean; + cells: Record; + }) => { + const scores: Record = {}; + for (const columnId of columns) { + const cell = row.cells[columnId]; + if (cell?.kind === 'score' && typeof cell.value === 'number') { + scores[columnId] = cell.value; + } + } + const n = counts.get(row.modelId) ?? 0; + return { + modelId: row.modelId, + modelLabel: row.modelLabel, + openSource: row.openSource, + scores, + latencySeconds: + n > 0 && latency.has(row.modelId) ? latency.get(row.modelId)! / n : undefined, + inputTokens: + n > 0 && inputTokens.has(row.modelId) ? inputTokens.get(row.modelId)! / n : undefined, + }; + } + ); + + config = { + columns, + familyOf: (columnId: string) => columnId.replace(/-[abc]$/, ''), + minColumns: columns.length, + resamples: 2000, + seed: 20260906, + }; + report = buildComparisonReport(observations, config); + }); + + it('loads a matrix with the expected shape', () => { + expect(observations.length).toBeGreaterThan(15); + expect(config.columns.length).toBe(24); + }); + + it('collapses the ranked leaderboard into far fewer tiers than models', () => { + const ranked = report.tiers.reduce((n, t) => n + t.members.length, 0); + expect(ranked).toBeGreaterThan(10); + // The v1 report ordered every model; the evidence supports only a handful + // of genuinely separated groups. + expect(report.tiers.length).toBeLessThan(ranked / 2); + }); + + it('keeps partial rows out of the ranking entirely', () => { + const rankedIds = report.tiers.flatMap((t) => t.members.map((m) => m.modelId)); + for (const excluded of report.excluded) { + expect(rankedIds).not.toContain(excluded.modelId); + expect(excluded.scoredColumns).toBeLessThan(excluded.requiredColumns); + } + }); + + it('finds genuine specialisation: no single model leads every family', () => { + const wins = new Map(); + for (const family of report.families) { + for (const leader of family.leaders) { + wins.set(leader, (wins.get(leader) ?? 0) + 1); + } + } + expect(report.families.length).toBeGreaterThanOrEqual(8); + // If one model led everywhere, a single ranked list would be adequate and + // v2 would not be justified. Real data: the best model leads a minority. + const mostWins = Math.max(...wins.values()); + expect(mostWins).toBeLessThan(report.families.length); + // And leadership is spread across several distinct models. + expect(wins.size).toBeGreaterThan(3); + }); + + it('reports the field as statistically tied rather than inventing an order', () => { + // The headline v2 finding: paired bootstrap cannot separate the complete + // rows, so any ranked leaderboard over them is presentation, not evidence. + const rankedCount = report.tiers.reduce((n, t) => n + t.members.length, 0); + if (report.tiers.length === 1) { + expect(report.allTiedWarning).toBe(true); + expect(report.tiers[0].members.length).toBe(rankedCount); + } + // Tier count must never approach one-tier-per-model, which would be the + // false precision v1 published. + expect(report.tiers.length).toBeLessThanOrEqual(Math.ceil(rankedCount / 3)); + }); + + it('identifies at least one model that leads one family and trails another', () => { + const lead = new Map(); + const trail = new Map(); + for (const family of report.families) { + for (const l of family.leaders) lead.set(l, (lead.get(l) ?? 0) + 1); + for (const l of family.laggards) trail.set(l, (trail.get(l) ?? 0) + 1); + } + const conflicted = [...lead.keys()].filter((m) => (trail.get(m) ?? 0) > 0); + expect(conflicted.length).toBeGreaterThan(0); + }); + + it('shows most models are dominated on the efficiency frontier', () => { + const measured = report.efficiency.filter((e) => e.latencySeconds !== undefined); + const dominated = measured.filter((e) => e.dominatedBy !== undefined); + expect(measured.length).toBeGreaterThan(10); + // If nearly everything is dominated, "pick the Overall winner" is bad advice. + expect(dominated.length).toBeGreaterThan(measured.length / 2); + }); + + it('produces byte-identical output across runs', () => { + const again = buildComparisonReport(observations, config); + expect(JSON.stringify(again)).toEqual(JSON.stringify(report)); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.test.ts new file mode 100644 index 0000000000000..3cf1a6019202d --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.test.ts @@ -0,0 +1,1001 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SomeDevLog } from '@kbn/some-dev-log'; +import { ToolingLog } from '@kbn/tooling-log'; +import type { EvaluationExperimentSummary, EvaluationScoreDocument } from '@kbn/evals-common'; +import type { EvalsClient, ExperimentStats } from '@kbn/evals'; +import { describeJudge } from './judge_provenance'; +import { + pickLatestExperimentPerModel, + experimentStatsToDatasets, + queryMatrixScores, + scoresByPrefixToDatasets, +} from './query_matrix_scores'; + +const experiment = ( + overrides: Partial & { modelId?: string } +): EvaluationExperimentSummary => { + const { modelId, ...rest } = overrides; + return { + experiment_id: 'exp', + timestamp: '2026-06-10T00:00:00.000Z', + task_model: modelId ? { id: modelId, family: 'fam', provider: 'prov' } : undefined, + ...rest, + } as EvaluationExperimentSummary; +}; + +describe('pickLatestExperimentPerModel', () => { + it('records self-judged per experiment from the judge and task ids', () => { + // D4: a hardcoded `true` here would flag every row in an opted-out column + // and no build_matrix test would notice -- the derivation only shows up + // against live golden data. Pin it where it is computed. + expect(describeJudge('google-gemini-3.1-pro', 'google-gemini-3.1-pro').selfJudged).toBe(true); + // Same family, different model: gemini-3.0-flash graded by gemini-3.1-pro + // is arm's-length and must NOT be disclosed as self-judged. + expect(describeJudge('google-gemini-3.1-pro', 'google-gemini-3.0-flash').selfJudged).toBe( + false + ); + expect(describeJudge('google-gemini-3.1-pro', 'openai-gpt-5.4').selfJudged).toBe(false); + }); + + it('keeps a self-judged experiment when allowSelfJudged is set', () => { + const experiments = [ + { + experiment_id: 'newer-self-judged', + task_model: { id: 'google-gemini-3.1-pro' }, + evaluator_model: { id: 'google-gemini-3.1-pro' }, + timestamp: '2026-09-01T00:00:00.000Z', + }, + ] as unknown as Parameters[0]; + + // Default policy drops it: a self-judged run must not silently win selection. + expect(pickLatestExperimentPerModel(experiments).size).toBe(0); + + // Opting in recovers the row rather than leaving the cell blank. + const kept = pickLatestExperimentPerModel(experiments, { allowSelfJudged: true }); + expect(kept.get('google-gemini-3.1-pro')?.experiment_id).toBe('newer-self-judged'); + }); + + it('ignores experiments newer than asOf for every model alike', () => { + const experiments = [ + experiment({ experiment_id: 'clean', modelId: 'm1', timestamp: '2026-08-22T00:00:00.000Z' }), + experiment({ experiment_id: 'bad', modelId: 'm1', timestamp: '2026-09-05T00:00:00.000Z' }), + experiment({ experiment_id: 'clean2', modelId: 'm2', timestamp: '2026-08-22T00:00:00.000Z' }), + experiment({ experiment_id: 'bad2', modelId: 'm2', timestamp: '2026-09-05T00:00:00.000Z' }), + ]; + + // Without a cutoff the newest run wins, even when its instrumentation was + // broken -- that is the whole reason the cutoff exists. + expect(pickLatestExperimentPerModel(experiments).get('m1')?.experiment_id).toBe('bad'); + + const asOf = Date.parse('2026-09-01T00:00:00.000Z'); + const selected = pickLatestExperimentPerModel(experiments, { now: asOf }); + // The cutoff must move BOTH models back, never one: a matrix that mixes a + // pre-cutoff row with a post-cutoff row is not comparable across rows. + expect(selected.get('m1')?.experiment_id).toBe('clean'); + expect(selected.get('m2')?.experiment_id).toBe('clean2'); + }); + + it('reports the judge that graded the surviving run', () => { + // Rejecting a self-judged run falls back to an older run -- which may have + // been graded by a different judge. Sonnet 4.6 published a fleet-leading + // 7.12 that way: its sonnet-judged runs were correctly rejected, and the + // haiku-judged survivor scored it ~1 point higher than the judge every + // other row was measured with. The swap has to be visible. + const experiments = [ + { + experiment_id: 'self-judged-newer', + task_model: { id: 'm1' }, + evaluator_model: { id: 'm1' }, + timestamp: '2026-08-29T00:00:00.000Z', + }, + { + experiment_id: 'other-judge-older', + task_model: { id: 'm1' }, + evaluator_model: { id: 'judge-b' }, + timestamp: '2026-08-22T00:00:00.000Z', + }, + ] as unknown as Parameters[0]; + + const selected = pickLatestExperimentPerModel(experiments); + expect(selected.get('m1')?.experiment_id).toBe('other-judge-older'); + expect(selected.get('m1')?.evaluator_model?.id).toBe('judge-b'); + }); + + it('keeps the most recent experiment per model', () => { + const result = pickLatestExperimentPerModel([ + experiment({ experiment_id: 'old', modelId: 'm1', timestamp: '2026-06-01T00:00:00.000Z' }), + experiment({ experiment_id: 'new', modelId: 'm1', timestamp: '2026-06-09T00:00:00.000Z' }), + experiment({ experiment_id: 'other', modelId: 'm2', timestamp: '2026-06-05T00:00:00.000Z' }), + ]); + + expect(result.get('m1')?.experiment_id).toBe('new'); + expect(result.get('m2')?.experiment_id).toBe('other'); + }); + + it('skips a self-judged experiment so an older independent run still counts', () => { + const result = pickLatestExperimentPerModel([ + experiment({ + experiment_id: 'clean', + modelId: 'm1', + timestamp: '2026-06-01T00:00:00.000Z', + evaluator_models: [{ id: 'judge', family: 'f', provider: 'p' }], + }), + experiment({ + experiment_id: 'self', + modelId: 'm1', + timestamp: '2026-06-09T00:00:00.000Z', + evaluator_models: [{ id: 'm1', family: 'f', provider: 'p' }], + }), + ]); + + expect(result.get('m1')?.experiment_id).toBe('clean'); + }); + + it('ignores experiments without a task model id', () => { + const result = pickLatestExperimentPerModel([ + experiment({ experiment_id: 'no-model', modelId: undefined }), + ]); + expect(result.size).toBe(0); + }); + + it('drops experiments older than the lookback window', () => { + const now = Date.parse('2026-06-15T00:00:00.000Z'); + const result = pickLatestExperimentPerModel( + [ + experiment({ + experiment_id: 'stale', + modelId: 'm1', + timestamp: '2026-05-01T00:00:00.000Z', + }), + experiment({ + experiment_id: 'fresh', + modelId: 'm2', + timestamp: '2026-06-14T00:00:00.000Z', + }), + ], + { lookbackDays: 14, now } + ); + + expect(result.has('m1')).toBe(false); + expect(result.get('m2')?.experiment_id).toBe('fresh'); + }); + + it('drops experiments with unparseable timestamps instead of treating them as epoch 0', () => { + const now = Date.parse('2026-06-15T00:00:00.000Z'); + const result = pickLatestExperimentPerModel( + [ + experiment({ experiment_id: 'broken', modelId: 'm1', timestamp: 'not-a-date' }), + experiment({ + experiment_id: 'fresh', + modelId: 'm1', + timestamp: '2026-06-14T00:00:00.000Z', + }), + ], + { lookbackDays: 14, now } + ); + + expect(result.get('m1')?.experiment_id).toBe('fresh'); + }); +}); + +describe('experimentStatsToDatasets', () => { + it('groups evaluator stats by dataset with mean + count', () => { + const stats: ExperimentStats = { + taskModel: { id: 'm1' }, + evaluatorModel: { id: 'judge' }, + totalRepetitions: 1, + stats: [ + { + datasetId: 'd1', + datasetName: 'D1', + evaluatorName: 'correctness', + stats: { mean: 0.9, median: 0.9, stdDev: 0, min: 0.9, max: 0.9, count: 10 }, + }, + { + datasetId: 'd1', + datasetName: 'D1', + evaluatorName: 'groundedness', + stats: { mean: 0.8, median: 0.8, stdDev: 0, min: 0.8, max: 0.8, count: 10 }, + }, + { + datasetId: 'd2', + datasetName: 'D2', + evaluatorName: 'correctness', + stats: { mean: 0.7, median: 0.7, stdDev: 0, min: 0.7, max: 0.7, count: 5 }, + }, + ], + }; + + expect(experimentStatsToDatasets(stats)).toEqual([ + { + datasetId: 'd1', + datasetName: 'D1', + evaluators: [ + { evaluatorName: 'correctness', mean: 0.9, count: 10, min: 0.9, max: 0.9 }, + { evaluatorName: 'groundedness', mean: 0.8, count: 10, min: 0.8, max: 0.8 }, + ], + }, + { + datasetId: 'd2', + datasetName: 'D2', + evaluators: [{ evaluatorName: 'correctness', mean: 0.7, count: 5, min: 0.7, max: 0.7 }], + }, + ]); + }); +}); + +describe('queryMatrixScores', () => { + const log = new ToolingLog() as unknown as SomeDevLog; + + const stats: ExperimentStats = { + taskModel: { id: 'm1' }, + evaluatorModel: { id: 'judge' }, + totalRepetitions: 1, + stats: [ + { + datasetId: 'd1', + datasetName: 'D1', + evaluatorName: 'correctness', + stats: { mean: 0.9, median: 0.9, stdDev: 0, min: 0.9, max: 0.9, count: 10 }, + }, + ], + }; + + const createClient = ( + experimentsByModel: Record + ): { client: EvalsClient; listExperiments: jest.Mock; getExperimentStats: jest.Mock } => { + const listExperiments = jest + .fn() + .mockImplementation(async ({ taskModelId }: { taskModelId?: string }) => + taskModelId ? experimentsByModel[taskModelId] ?? [] : [] + ); + const getExperimentStats = jest.fn().mockResolvedValue(stats); + const client = { listExperiments, getExperimentStats } as unknown as EvalsClient; + return { client, listExperiments, getExperimentStats }; + }; + + it('merges every shard of a sharded sweep into one row', async () => { + // A sharded sweep splits one model's examples across VMs, each writing its + // own execution_id. Fetching only one of them renders a single shard and + // blanks the examples the others covered. + const shardStats = (datasetId: string, mean: number, count: number): ExperimentStats => ({ + taskModel: { id: 'm1' }, + evaluatorModel: { id: 'judge' }, + totalRepetitions: 1, + stats: [ + { + datasetId, + datasetName: datasetId.toUpperCase(), + evaluatorName: 'correctness', + stats: { mean, median: mean, stdDev: 0, min: mean, max: mean, count }, + }, + ], + }); + + const listExperiments = jest.fn().mockResolvedValue([ + experiment({ + experiment_id: 'exp-s1', + execution_id: 'sweep-9-s1of2::suite-a::m1', + modelId: 'm1', + timestamp: '2026-06-10T00:00:00.000Z', + }), + experiment({ + experiment_id: 'exp-s2', + execution_id: 'sweep-9-s2of2::suite-a::m1', + modelId: 'm1', + timestamp: '2026-06-10T01:00:00.000Z', + }), + ]); + const getExperimentStats = jest + .fn() + .mockImplementation( + async (_experimentId: string, { executionId }: { executionId?: string }) => + executionId === 'sweep-9-s1of2::suite-a::m1' + ? shardStats('d1', 0.9, 10) + : shardStats('d2', 0.5, 5) + ); + const client = { listExperiments, getExperimentStats } as unknown as EvalsClient; + + const result = await queryMatrixScores(client, log, { + suiteIds: ['suite-a'], + modelIds: ['m1'], + branch: 'main', + }); + + // Both shards fetched, not just the newest. + expect(getExperimentStats).toHaveBeenCalledTimes(2); + + const datasets = result[0].suites[0].datasets; + expect(datasets.map((d) => d.datasetId).sort()).toEqual(['d1', 'd2']); + }); + + it('queries each (suite, model) pair through the route model_id filter', async () => { + const { client, listExperiments, getExperimentStats } = createClient({ + m1: [experiment({ experiment_id: 'exp-m1', modelId: 'm1' })], + m2: [experiment({ experiment_id: 'exp-m2', modelId: 'm2' })], + }); + + const result = await queryMatrixScores(client, log, { + suiteIds: ['suite-a'], + modelIds: ['m1', 'm2'], + branch: 'main', + }); + + expect(listExperiments).toHaveBeenCalledTimes(2); + expect(listExperiments).toHaveBeenCalledWith( + expect.objectContaining({ suiteId: 'suite-a', taskModelId: 'm1', branch: 'main' }) + ); + expect(listExperiments).toHaveBeenCalledWith( + expect.objectContaining({ suiteId: 'suite-a', taskModelId: 'm2', branch: 'main' }) + ); + expect(getExperimentStats).toHaveBeenCalledTimes(2); + expect(result.map((model) => model.modelId).sort()).toEqual(['m1', 'm2']); + }); + + it("carries the graded run's commit onto the suite so rows can be traced to a codebase", async () => { + // The artifact's top-level provenance is the GENERATOR's commit. When a + // model is appended to an existing board months later, the only honest + // answer to "which code was this row measured on" is the experiment's own + // git_commit_sha -- so it has to survive the query layer. + const { client } = createClient({ + m1: [ + experiment({ + experiment_id: 'exp-m1', + modelId: 'm1', + git_commit_sha: 'deadbeefcafe1234', + }), + ], + }); + + const result = await queryMatrixScores(client, log, { + suiteIds: ['suite-a'], + modelIds: ['m1'], + branch: 'main', + }); + + expect(result[0].suites[0].commitSha).toBe('deadbeefcafe1234'); + }); + + it('leaves the commit undefined when the experiment summary has none', async () => { + const { client } = createClient({ + m1: [experiment({ experiment_id: 'exp-m1', modelId: 'm1' })], + }); + + const result = await queryMatrixScores(client, log, { + suiteIds: ['suite-a'], + modelIds: ['m1'], + branch: 'main', + }); + + expect(result[0].suites[0].commitSha).toBeUndefined(); + }); + + it('reports self-judged exclusions so a rejected model is not mistaken for one that never ran', async () => { + // Reproduces the 2026-08-29 persona-matrix incident: claude-4.6-sonnet was + // its own judge, so every score was dropped by `excludeSelfJudged` and the + // row rendered blank — indistinguishable from a model that never ran. That + // ambiguity triggered a full re-sweep which could not, even in principle, + // fill the cells. + const selfJudgedScore = (index: number) => + ({ + task: { model: { id: 'm1' } }, + evaluator: { model: { id: 'm1' }, name: 'Factuality', score: 1 }, + example: { id: `entity-analytics-${index}` }, + } as unknown as EvaluationScoreDocument); + + const { client } = createClient({ + m1: [experiment({ experiment_id: 'exp-m1', modelId: 'm1' })], + }); + (client as unknown as { getExperimentScores: jest.Mock }).getExperimentScores = jest + .fn() + .mockResolvedValue([selfJudgedScore(1), selfJudgedScore(2)]); + + const warn = jest.spyOn(log, 'warning'); + const [model] = await queryMatrixScores(client, log, { + suiteIds: ['suite-a'], + modelIds: ['m1'], + branch: 'main', + prefixesBySuite: { 'suite-a': ['entity-analytics'] }, + scoring: { excludeSelfJudged: true }, + }); + + expect(model.excluded?.selfJudged).toBe(2); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('dropped 2 self-judged')); + + // No per-prefix dataset survived the policy, so the cells stay empty — but + // the exclusion tally proves the emptiness is a judge defect, not absent data. + expect(model.suites[0].datasets.some((d) => d.datasetId.startsWith('prefix:'))).toBe(false); + }); + + it('warns when a model ran fewer examples than its peers', async () => { + // A short run still produces a score, and that score renders identically to + // a complete one. 4.5-sonnet published 7.49 off 18 of 21 examples this way. + const score = (modelId: string, exampleIndex: number) => + ({ + task: { model: { id: modelId }, repetition_index: 0 }, + evaluator: { model: { id: 'judge' }, name: 'Factuality', score: 1 }, + example: { id: `ex-${exampleIndex}` }, + } as unknown as EvaluationScoreDocument); + + const { client } = createClient({ + complete: [experiment({ experiment_id: 'exp-complete', modelId: 'complete' })], + short: [experiment({ experiment_id: 'exp-short', modelId: 'short' })], + }); + (client as unknown as { getExperimentScores: jest.Mock }).getExperimentScores = jest + .fn() + .mockImplementation(async (experimentId: string) => + experimentId === 'exp-complete' + ? [score('complete', 1), score('complete', 2), score('complete', 3)] + : [score('short', 1)] + ); + + const warn = jest.spyOn(log, 'warning'); + await queryMatrixScores(client, log, { + suiteIds: ['suite-a'], + modelIds: ['complete', 'short'], + branch: 'main', + prefixesBySuite: { 'suite-a': ['ex'] }, + }); + + expect(warn).toHaveBeenCalledWith( + expect.stringContaining('short scored on 1 of 3 examples in suite-a') + ); + }); + + it('warns when one model was measured with more repetitions than the rest', async () => { + // Repetitions shrink judge variance, so a 3-rep row has a tighter error bar + // than a 1-rep row. Both render as one number, so the mismatch is invisible + // unless it is stated. + const score = (modelId: string, exampleIndex: number, repetitionIndex: number) => + ({ + task: { model: { id: modelId }, repetition_index: repetitionIndex }, + evaluator: { model: { id: 'judge' }, name: 'Factuality', score: 1 }, + example: { id: `ex-${exampleIndex}` }, + } as unknown as EvaluationScoreDocument); + + const { client } = createClient({ + once: [experiment({ experiment_id: 'exp-once', modelId: 'once' })], + thrice: [experiment({ experiment_id: 'exp-thrice', modelId: 'thrice' })], + }); + (client as unknown as { getExperimentScores: jest.Mock }).getExperimentScores = jest + .fn() + .mockImplementation(async (experimentId: string) => + experimentId === 'exp-once' + ? [score('once', 1, 0), score('once', 2, 0)] + : [ + score('thrice', 1, 0), + score('thrice', 1, 1), + score('thrice', 1, 2), + score('thrice', 2, 0), + ] + ); + + const warn = jest.spyOn(log, 'warning'); + await queryMatrixScores(client, log, { + suiteIds: ['suite-a'], + modelIds: ['once', 'thrice'], + branch: 'main', + prefixesBySuite: { 'suite-a': ['ex'] }, + }); + + expect(warn).toHaveBeenCalledWith(expect.stringContaining('Repetition imbalance in suite-a')); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('thrice')); + }); + + it('unions a suite across several branches so no branch-local model is lost', async () => { + // Real golden data for one suite is split across branches by model: the + // weekly-matrix branch holds six models, while a seventh (4.5-sonnet) only + // ever ran on the feature branch. Pinning to either single branch silently + // discards the other's rows, so a branch LIST has to be unioned. + const listExperiments = jest + .fn() + .mockImplementation( + async ({ taskModelId, branch }: { taskModelId?: string; branch?: string }) => { + if (branch === 'weekly' && taskModelId === 'm1') { + return [experiment({ experiment_id: 'exp-weekly-m1', modelId: 'm1' })]; + } + if (branch === 'feature' && taskModelId === 'm2') { + return [experiment({ experiment_id: 'exp-feature-m2', modelId: 'm2' })]; + } + return []; + } + ); + const client = { + listExperiments, + getExperimentStats: jest.fn().mockResolvedValue(stats), + } as unknown as EvalsClient; + + const result = await queryMatrixScores(client, log, { + suiteIds: ['migrations-suite'], + modelIds: ['m1', 'm2'], + branch: 'main', + branchBySuite: { 'migrations-suite': ['weekly', 'feature'] }, + }); + + // Both branch-local models survive the union. + expect(result.map((model) => model.modelId).sort()).toEqual(['m1', 'm2']); + }); + + it('prefers the newest run when the same model ran on several unioned branches', async () => { + // Union must not resurrect a stale run: when a model exists on both + // branches, selection still picks the most recent experiment. + const listExperiments = jest + .fn() + .mockImplementation(async ({ branch }: { branch?: string }) => { + if (branch === 'old') { + return [ + experiment({ + experiment_id: 'exp-old', + modelId: 'm1', + timestamp: '2026-01-01T00:00:00.000Z', + }), + ]; + } + return [ + experiment({ + experiment_id: 'exp-new', + modelId: 'm1', + timestamp: '2026-06-01T00:00:00.000Z', + }), + ]; + }); + const getExperimentStats = jest.fn().mockResolvedValue(stats); + const client = { listExperiments, getExperimentStats } as unknown as EvalsClient; + + await queryMatrixScores(client, log, { + suiteIds: ['migrations-suite'], + modelIds: ['m1'], + branch: 'main', + branchBySuite: { 'migrations-suite': ['old', 'new'] }, + }); + + // Only the newer run is fetched; the stale branch's run is never scored. + expect(getExperimentStats).toHaveBeenCalledTimes(1); + expect(getExperimentStats).toHaveBeenCalledWith( + 'exp-new', + expect.objectContaining({ executionId: 'exp-new' }) + ); + }); + + it('reports a fully self-judged suite as withheld, not as never-run', async () => { + // Selection drops the self-judged experiment BEFORE scores are fetched, + // so `latest` is undefined and the model silently vanishes from the + // suite. A build_matrix unit test with a hand-made suite record cannot + // catch this: the real pipeline never produces that record. Drive the + // query layer end to end instead. + const selfJudged = experiment({ + experiment_id: 'exp-self', + modelId: 'm1', + timestamp: '2026-06-10T00:00:00.000Z', + }) as EvaluationExperimentSummary & { evaluator_model?: { id: string } }; + selfJudged.evaluator_model = { id: 'm1' }; + + const { client, getExperimentStats } = createClient({ m1: [selfJudged] }); + getExperimentStats.mockResolvedValue({ + stats: [ + { + datasetId: 'd', + datasetName: 'd', + evaluatorName: 'Rubric', + stats: { mean: 0.7, median: 0.7, stdDev: 0, min: 0, max: 1, count: 4794 }, + }, + ], + taskModel: { id: 'm1' }, + totalRepetitions: 1, + }); + + const result = await queryMatrixScores(client, log, { + suiteIds: ['suite-a'], + modelIds: ['m1'], + branch: 'main', + scoring: { excludeSelfJudged: true }, + }); + + const suite = result[0]?.suites.find((entry) => entry.suiteId === 'suite-a'); + // The suite is present, carries no datasets, and states how much was + // withheld -- the three facts a cell needs to say 'excluded' not 'missing'. + expect(suite).toBeDefined(); + expect(suite!.datasets).toHaveLength(0); + expect(suite!.excludedSelfJudged).toBe(4794); + }); + + it('reads a suite from its branch override instead of the global branch', async () => { + const { client, listExperiments } = createClient({ + m1: [experiment({ experiment_id: 'exp-m1', modelId: 'm1' })], + }); + + await queryMatrixScores(client, log, { + suiteIds: ['persona-suite', 'migrations-suite'], + modelIds: ['m1'], + branch: 'main', + branchBySuite: { 'migrations-suite': 'feat/matrix-v3' }, + }); + + expect(listExperiments).toHaveBeenCalledWith( + expect.objectContaining({ suiteId: 'persona-suite', branch: 'main' }) + ); + expect(listExperiments).toHaveBeenCalledWith( + expect.objectContaining({ suiteId: 'migrations-suite', branch: 'feat/matrix-v3' }) + ); + }); + + it('picks the newest experiment within the lookback window per model', async () => { + const now = Date.now(); + const recent = new Date(now - 2 * 24 * 60 * 60 * 1000).toISOString(); + const stale = new Date(now - 60 * 24 * 60 * 60 * 1000).toISOString(); + const { client, getExperimentStats } = createClient({ + // Route returns newest first; the stale one would be picked by a naive per_page: 1 + // request if the newest ever fell outside the window. + m1: [ + experiment({ experiment_id: 'recent', modelId: 'm1', timestamp: recent }), + experiment({ experiment_id: 'stale', modelId: 'm1', timestamp: stale }), + ], + }); + + const result = await queryMatrixScores(client, log, { + suiteIds: ['suite-a'], + modelIds: ['m1'], + lookbackDays: 7, + }); + + expect(getExperimentStats).toHaveBeenCalledWith( + 'recent', + expect.objectContaining({ suiteId: 'suite-a', taskModelId: 'm1' }) + ); + expect(result[0].suites[0].experimentId).toBe('recent'); + }); + + it('omits models with no experiment inside the lookback window', async () => { + const stale = new Date(Date.now() - 60 * 24 * 60 * 60 * 1000).toISOString(); + const { client, getExperimentStats } = createClient({ + m1: [experiment({ experiment_id: 'stale', modelId: 'm1', timestamp: stale })], + }); + + const result = await queryMatrixScores(client, log, { + suiteIds: ['suite-a'], + modelIds: ['m1'], + lookbackDays: 7, + }); + + expect(getExperimentStats).not.toHaveBeenCalled(); + expect(result).toEqual([]); + }); +}); + +describe('scoresByPrefixToDatasets', () => { + const score = (exampleId: string, evaluatorName: string, s: number) => + ({ + example: { id: exampleId, index: 0, dataset: { id: 'ds', name: 'DS' } }, + task: { model: { id: 'm1' }, trace_id: 't' }, + evaluator: { name: evaluatorName, score: s }, + metadata: {}, + } as unknown as EvaluationScoreDocument); + + it('buckets docs by example.id prefix and computes per-evaluator means', () => { + const datasets = scoresByPrefixToDatasets( + [ + score('alert-analysis-a', 'correctness', 1), + score('alert-analysis-b', 'correctness', 0), + score('threat-hunting-a', 'correctness', 0.5), + score('threat-hunting-b', 'groundedness', 0.8), + ], + ['alert-analysis', 'threat-hunting'] + ); + + const byId = new Map(datasets.map((d) => [d.datasetId, d])); + expect(byId.get('prefix:alert-analysis')?.evaluators).toEqual([ + { evaluatorName: 'correctness', mean: 0.5, count: 2 }, + ]); + expect(byId.get('prefix:threat-hunting')?.evaluators).toEqual( + expect.arrayContaining([ + { evaluatorName: 'correctness', mean: 0.5, count: 1 }, + { evaluatorName: 'groundedness', mean: 0.8, count: 1 }, + ]) + ); + }); + + it('drops non-quality evaluators using evaluator.direction', () => { + // Latency is minimize and Tool Calls is neutral upstream (#284027). Averaging + // either into a 0-10 quality score is meaningless, and the name allowlist only + // approximates it -- a renamed evaluator silently slips back in. + const withDirection = (id: string, name: string, s: number, direction: string) => + ({ + example: { id, index: 0, dataset: { id: 'ds', name: 'DS' } }, + task: { model: { id: 'm1' }, trace_id: 't' }, + evaluator: { name, score: s, direction }, + metadata: {}, + } as unknown as EvaluationScoreDocument); + + const datasets = scoresByPrefixToDatasets( + [ + withDirection('alert-analysis-a', 'correctness', 1, 'maximize'), + withDirection('alert-analysis-b', 'Latency', 900, 'minimize'), + withDirection('alert-analysis-c', 'Tool Calls', 42, 'neutral'), + ], + ['alert-analysis'] + ); + + // Only the maximize evaluator survives; 900 and 42 would wreck the mean. + expect(datasets[0].evaluators).toEqual([{ evaluatorName: 'correctness', mean: 1, count: 1 }]); + }); + + it('matches exact example ids and prefix-dash boundaries only', () => { + const datasets = scoresByPrefixToDatasets( + [score('alert-analysis', 'correctness', 1), score('alert-analysisx', 'correctness', 0)], + ['alert-analysis'] + ); + // 'alert-analysisx' must NOT match prefix 'alert-analysis' + expect(datasets).toHaveLength(1); + expect(datasets[0].evaluators[0].count).toBe(1); + }); + + it('skips docs without evaluator score', () => { + const datasets = scoresByPrefixToDatasets( + [ + { + ...score('alert-analysis-a', 'correctness', 1), + evaluator: { name: 'x' }, + } as EvaluationScoreDocument, + ], + ['alert-analysis'] + ); + expect(datasets).toEqual([]); + }); +}); + +describe('queryMatrixScores with examplePrefixes', () => { + const log = new ToolingLog() as unknown as SomeDevLog; + + const stats: ExperimentStats = { + taskModel: { id: 'm1' }, + evaluatorModel: { id: 'judge' }, + totalRepetitions: 1, + stats: [ + { + datasetId: 'd1', + datasetName: 'D1', + evaluatorName: 'correctness', + stats: { mean: 0.9, median: 0.9, stdDev: 0, min: 0.9, max: 0.9, count: 10 }, + }, + ], + }; + + const createClient = (): { client: EvalsClient; getExperimentScores: jest.Mock } => { + const listExperiments = jest.fn().mockResolvedValue([ + { + experiment_id: 'e1', + execution_id: 'x1', + timestamp: new Date().toISOString(), + task_model: { id: 'm1' }, + }, + ]); + const getExperimentStats = jest.fn().mockResolvedValue(stats); + const getExperimentScores = jest.fn().mockResolvedValue([ + { + example: { id: 'alert-analysis-a', index: 0, dataset: { id: 'd1', name: 'D1' } }, + task: { model: { id: 'm1' }, trace_id: 't' }, + evaluator: { name: 'correctness', score: 0.6 }, + metadata: {}, + }, + ]); + const client = { + listExperiments, + getExperimentStats, + getExperimentScores, + } as unknown as EvalsClient; + return { client, getExperimentScores }; + }; + + it('fetches per-example scores and appends synthetic prefix datasets when prefixes requested', async () => { + const { client, getExperimentScores } = createClient(); + + const result = await queryMatrixScores(client, log, { + suiteIds: ['suite-a'], + modelIds: ['m1'], + prefixesBySuite: { 'suite-a': ['alert-analysis'] }, + }); + + expect(getExperimentScores).toHaveBeenCalledWith('e1', expect.anything()); + const datasetIds = result[0].suites[0].datasets.map((d) => d.datasetId); + expect(datasetIds).toContain('d1'); + expect(datasetIds).toContain('prefix:alert-analysis'); + }); + + it('applies the per-suite scoring policy, not the global one, to prefix scores', async () => { + // The graded model IS the judge. Globally self-judged scores are dropped; + // the audited suite opts out via scoringBySuite and must keep its cell. + const selfJudged = [ + { + example: { id: 'alert-analysis-a', index: 0, dataset: { id: 'd1', name: 'D1' } }, + task: { model: { id: 'm1' }, trace_id: 't' }, + evaluator: { name: 'correctness', score: 0.6, model: { id: 'm1' } }, + metadata: {}, + }, + ]; + + const build = () => { + const listExperiments = jest.fn().mockResolvedValue([ + { + experiment_id: 'e1', + execution_id: 'x1', + timestamp: new Date().toISOString(), + task_model: { id: 'm1' }, + // The judge IS the graded model here; the aggregation must derive + // that from these two ids so the artifact can disclose it per row. + evaluator_model: { id: 'm1' }, + }, + ]); + return { + listExperiments, + getExperimentStats: jest.fn().mockResolvedValue(stats), + getExperimentScores: jest.fn().mockResolvedValue(selfJudged), + } as unknown as EvalsClient; + }; + + const prefixIds = (r: Awaited>) => + r[0].suites[0].datasets.map((d) => d.datasetId); + + // Strict global policy: the self-judged experiment is dropped outright, + // so the model yields no scores at all. + const strict = await queryMatrixScores(build(), log, { + suiteIds: ['suite-a'], + modelIds: ['m1'], + prefixesBySuite: { 'suite-a': ['alert-analysis'] }, + scoring: { excludeSelfJudged: true }, + }); + // The suite is now RECORDED as withheld rather than vanishing, but it + // still yields no datasets, so no score can be published from it. + const strictSuite = strict[0]?.suites.find((entry) => entry.suiteId === 'suite-a'); + expect(strictSuite?.datasets ?? []).toHaveLength(0); + expect(strictSuite?.excludedSelfJudged).toBeGreaterThan(0); + + // Same global policy, but this suite opted out -> the cell survives. + const opted = await queryMatrixScores(build(), log, { + suiteIds: ['suite-a'], + modelIds: ['m1'], + prefixesBySuite: { 'suite-a': ['alert-analysis'] }, + scoring: { excludeSelfJudged: true }, + scoringBySuite: { 'suite-a': { excludeSelfJudged: false } }, + }); + expect(prefixIds(opted)).toContain('prefix:alert-analysis'); + // ...carrying the disclosure derived from the experiment's own ids. + expect(opted[0].suites[0].selfJudged).toBe(true); + + // An arm's-length judge in the same opted-out suite must NOT be flagged, + // or the artifact libels every other row in the column. + const armsLength = build() as unknown as { + listExperiments: jest.Mock; + }; + armsLength.listExperiments.mockResolvedValue([ + { + experiment_id: 'e1', + execution_id: 'x1', + timestamp: new Date().toISOString(), + task_model: { id: 'm1' }, + evaluator_model: { id: 'some-other-judge' }, + }, + ]); + const independent = await queryMatrixScores(armsLength as unknown as EvalsClient, log, { + suiteIds: ['suite-a'], + modelIds: ['m1'], + prefixesBySuite: { 'suite-a': ['alert-analysis'] }, + scoring: { excludeSelfJudged: true }, + scoringBySuite: { 'suite-a': { excludeSelfJudged: false } }, + }); + expect(independent[0].suites[0].selfJudged).toBe(false); + }); + + it('does not fetch per-example scores when no prefixes requested', async () => { + const { client, getExperimentScores } = createClient(); + + await queryMatrixScores(client, log, { suiteIds: ['suite-a'], modelIds: ['m1'] }); + + expect(getExperimentScores).not.toHaveBeenCalled(); + }); + + it('degrades gracefully when the scores route fails', async () => { + const listExperiments = jest.fn().mockResolvedValue([ + { + experiment_id: 'e1', + execution_id: 'x1', + timestamp: new Date().toISOString(), + task_model: { id: 'm1' }, + }, + ]); + const getExperimentStats = jest.fn().mockResolvedValue(stats); + const getExperimentScores = jest.fn().mockRejectedValue(new Error('route down')); + const client = { + listExperiments, + getExperimentStats, + getExperimentScores, + } as unknown as EvalsClient; + + const result = await queryMatrixScores(client, log, { + suiteIds: ['suite-a'], + modelIds: ['m1'], + prefixesBySuite: { 'suite-a': ['alert-analysis'] }, + }); + + expect(result[0].suites[0].datasets.map((d) => d.datasetId)).toEqual(['d1']); + }); +}); + +describe('scoresByPrefixToDatasets errored-out tracking', () => { + const doc = (exampleId: string, evaluatorName: string, s: number | undefined, label?: string) => + ({ + example: { id: exampleId, index: 0, dataset: { id: 'ds', name: 'DS' } }, + task: { model: { id: 'm1' }, trace_id: 't' }, + evaluator: { + name: evaluatorName, + ...(s !== undefined ? { score: s } : {}), + ...(label ? { label } : {}), + }, + metadata: {}, + } as unknown as EvaluationScoreDocument); + + it('names evaluators that errored on every example and never scored', () => { + // The DeepSeek alert-analysis-a failure shape: Trajectory and SkillInvoked + // wrote label=error docs for all examples, so they vanish from the mean. + const datasets = scoresByPrefixToDatasets( + [ + doc('alert-analysis-a', 'MinExpectedSteps', 1), + doc('alert-analysis-a', 'FinalAnswerPresent', 1), + doc('alert-analysis-a', 'Trajectory', undefined, 'error'), + doc('alert-analysis-b', 'Trajectory', undefined, 'error'), + doc('alert-analysis-a', 'SkillInvoked', undefined, 'error'), + ], + ['alert-analysis'] + ); + + expect(datasets[0].erroredOutEvaluators).toEqual( + expect.arrayContaining(['Trajectory', 'SkillInvoked']) + ); + // Saturated survivors still score — the guard flags the broken ones. + expect(datasets[0].evaluators).toHaveLength(2); + }); + + it('flags a trace evaluator that reported unavailable for every example', () => { + // The Claude Sonnet 4.5 shape: SkillInvoked found no tool spans and wrote + // label=unavailable with a null score for all 19 examples. That is not + // "error", so the cell used to publish an overall built only on the + // evaluators that survived -- ranked against peers graded on the full set. + const datasets = scoresByPrefixToDatasets( + [ + doc('alert-analysis-a', 'MinExpectedSteps', 1), + doc('alert-analysis-a', 'FinalAnswerPresent', 1), + doc('alert-analysis-a', 'SkillInvoked', undefined, 'unavailable'), + doc('alert-analysis-b', 'SkillInvoked', undefined, 'unavailable'), + ], + ['alert-analysis'] + ); + + expect(datasets[0].erroredOutEvaluators).toEqual(expect.arrayContaining(['SkillInvoked'])); + }); + + it('does not flag an evaluator that errored once but recovered', () => { + const datasets = scoresByPrefixToDatasets( + [ + doc('alert-analysis-a', 'Latency', undefined, 'error'), + doc('alert-analysis-b', 'Latency', 900), + doc('alert-analysis-a', 'correctness', 1), + ], + ['alert-analysis'] + ); + + expect(datasets[0].erroredOutEvaluators ?? []).toEqual([]); + }); + + it('omits the field when nothing errored out', () => { + const datasets = scoresByPrefixToDatasets( + [doc('alert-analysis-a', 'correctness', 1)], + ['alert-analysis'] + ); + expect(datasets[0].erroredOutEvaluators).toBeUndefined(); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.ts new file mode 100644 index 0000000000000..00f0c3fef8f4e --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.ts @@ -0,0 +1,802 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SomeDevLog } from '@kbn/some-dev-log'; +import type { EvaluationExperimentSummary, EvaluationScoreDocument } from '@kbn/evals-common'; +import { MAX_LIST_EXPERIMENTS, type EvalsClient, type ExperimentStats } from '@kbn/evals'; +import { mergeShardDatasets, pickShardExperiments } from './merge_shard_experiments'; +import { isEisBacked, describeJudge } from './judge_provenance'; +import { resolveVerdictScore } from './scoring_policy'; + +/** + * Counts of score documents dropped by the provenance/verdict policy, so the + * report can state what was excluded instead of silently shrinking its own + * sample. + */ +export interface ExcludedScoreCounts { + /** Dropped because evaluator.direction is minimize/neutral, not a quality score. */ + nonQuality: number; + nonEis: number; + selfJudged: number; + unmappedVerdict: number; +} + +export interface ScoreAggregationOptions { + /** Drop scores from judges that are not EIS-backed connectors. */ + requireEisJudge?: boolean; + /** Drop scores where the judge and the graded model are the same id. */ + excludeSelfJudged?: boolean; + /** + * Score the judge's categorical verdict via an ordinal ladder instead of its + * continuous value. Measured on the persona matrix: judged-score flip across + * identical repetitions drops from 83.3% to 33.3%, because the agent's prose + * changes every run while its verdict does not. + */ + useVerdictLadder?: boolean; + onExcluded?: (counts: ExcludedScoreCounts) => void; +} + +/** Aggregated evaluator score for a single dataset within a suite. */ +export interface AggregatedEvaluatorScore { + evaluatorName: string; + mean: number; + count: number; + /** Observed spread across the experiment's examples (used by the token axis). */ + min?: number; + max?: number; +} + +export interface AggregatedDatasetScores { + datasetId: string; + datasetName: string; + evaluators: AggregatedEvaluatorScore[]; + /** + * Evaluators that produced `label=error` documents for this dataset and no + * numeric score at all. Their absence from `evaluators` would otherwise be + * invisible, letting the dataset's mean rest on whichever evaluators + * survived a broken instrument. + */ + erroredOutEvaluators?: string[]; +} + +export interface AggregatedSuiteScores { + suiteId: string; + /** Primary execution id retained for backward compatibility. */ + experimentId: string; + /** + * Every execution contributing to this suite row. Sharded sweeps have one + * execution per VM; trace and per-prefix readers must query all of them. + */ + executionIds?: string[]; + timestamp?: string; + /** + * Commit the graded run executed against, straight from the experiment + * summary. The artifact's top-level provenance records the *generator's* + * commit; this records the *subject's*. They diverge as soon as a model is + * added to an existing matrix, which is the normal way this board grows. + */ + commitSha?: string; + /** + * True when the admitted scores for this suite were graded by the model + * being graded. Only possible when the suite opted out of + * `excludeSelfJudged`; carries the fact to the artifact so a published + * self-judged score can be disclosed per row rather than per column. + */ + selfJudged?: boolean; + /** + * Judge that graded this suite's run, for cross-row comparability checks. + * Left undefined when the run carries more than one judge -- see + * `judgeModelIds`, which is the honest form for suites whose columns ran as + * separate experiments. + */ + judgeModelId?: string; + /** Every judge that graded this suite's admitted runs. */ + judgeModelIds?: string[]; + /** + * Number of experiments withheld because the grader was the graded model. + * Set only when the withholding emptied the suite, so a cell can say + * "withheld for self-judging" instead of rendering identically to a model + * that never ran the suite at all. + */ + excludedSelfJudged?: number; + datasets: AggregatedDatasetScores[]; +} + +export interface AggregatedModelScores { + modelId: string; + family?: string; + provider?: string; + suites: AggregatedSuiteScores[]; + /** + * Scores that existed for this model but were rejected by judge policy. + * Present only when at least one score was dropped. A model with `suites` + * empty AND this set ran successfully — its grades were untrustworthy — so + * re-running it is wasted compute until the judge assignment is fixed. + */ + excluded?: ExcludedScoreCounts; +} + +export interface QueryMatrixScoresOptions { + suiteIds: string[]; + /** + * Task model ids to include (the config's `id` + `matchIds` per model). + * Each (suite, model) pair is queried separately via the route's `model_id` + * term filter, so a bounded single-page listing covers the full history for + * that pair instead of paging an ever-growing cross-model aggregation. + */ + modelIds: string[]; + branch?: string; + /** + * Per-suite branch overrides, keyed by suite id. A suite listed here is read + * from its mapped branch instead of the global `branch`. + */ + branchBySuite?: Record; + lookbackDays?: number; + /** + * Render the matrix as of this epoch-ms instant: experiments newer than it + * are ignored for every model alike. Reproduces an earlier matrix, or + * excludes a window of runs known to be instrumented wrong -- never a + * per-model choice of which run to score. + */ + asOf?: number; + /** + * When any config column sets `examplePrefixes`, per-example score documents + * are fetched (stripped experiment-scores route — unbounded fields excluded) + * and bucketed into synthetic per-prefix datasets alongside the dataset-level + * stats, so columns can slice a single dataset by example category. + */ + prefixesBySuite?: Record; + /** + * Judged-evaluator scoring policy. Forwarded to `scoresByPrefixToDatasets` + * for the per-prefix datasets. Omitted means the historical behaviour: + * continuous scores, every judge counted. + */ + scoring?: ScoreAggregationOptions; + /** + * Per-suite scoring overrides, keyed by suite id. A suite present here uses + * its own policy instead of the global `scoring`; suites absent from the map + * keep `scoring` unchanged. Mirrors `branchBySuite`. + */ + scoringBySuite?: Record; +} + +/** + * Buckets stripped per-example score documents into synthetic per-prefix + * datasets. Each doc carries `example.id` (e.g. `alert-analysis-b`) and + * `evaluator.{name,score}`; docs are grouped by prefix, then per-evaluator + * means are computed the same way the server-side stats route would. + * Pure for unit testing. + */ +export const scoresByPrefixToDatasets = ( + scores: EvaluationScoreDocument[], + prefixes: string[], + options: ScoreAggregationOptions = {} +): AggregatedDatasetScores[] => { + const byPrefix = new Map>(); + // Evaluators that produced an error label but no score for a prefix. A + // trace-metric evaluator erroring (Latency racing span ingestion) is noise; + // a cell-relevant judge evaluator erroring out entirely (Trajectory, + // SkillInvoked) means the cell's mean silently rests on the evaluators that + // survived. Track the names so the builder can refuse to publish that cell. + const erroredByPrefix = new Map>(); + const excluded: ExcludedScoreCounts = { + nonQuality: 0, + nonEis: 0, + selfJudged: 0, + unmappedVerdict: 0, + }; + + for (const doc of scores) { + const exampleId = doc.example?.id ?? ''; + const prefix = prefixes.find((p) => exampleId === p || exampleId.startsWith(`${p}-`)); + if (!prefix) { + continue; + } + // Upstream now persists evaluator polarity on the score doc (#284027): + // maximize | minimize | neutral. Prefer it over guessing from the name. + const evaluatorName = doc.evaluator?.name; + const direction = (doc.evaluator as { direction?: string } | undefined)?.direction; + if (!evaluatorName) { + continue; + } + + const judgeId = doc.evaluator?.model?.id; + const taskModelId = doc.task?.model?.id; + if (options.requireEisJudge && judgeId && !isEisBacked(judgeId)) { + excluded.nonEis += 1; + continue; + } + if ( + options.excludeSelfJudged && + judgeId && + taskModelId && + describeJudge(judgeId, taskModelId).selfJudged + ) { + excluded.selfJudged += 1; + continue; + } + + // A non-quality metric averaged into a 0-10 score is nonsense: Latency is + // minimize, Tool Calls is neutral. The name allowlist only approximates this; + // when the doc carries polarity, trust it and keep the average to maximize. + if (direction && direction !== 'maximize') { + excluded.nonQuality += 1; + continue; + } + const score = options.useVerdictLadder + ? resolveVerdictScore(evaluatorName, doc) + : doc.evaluator?.score; + + // Record whether this evaluator ever errored or ever scored for the + // prefix, so a judge evaluator that failed for every example is visible to + // the builder instead of silently absent from the mean's denominator. + let errTrack = erroredByPrefix.get(prefix); + if (!errTrack) { + errTrack = new Map(); + erroredByPrefix.set(prefix, errTrack); + } + const tally = errTrack.get(evaluatorName) ?? { errored: 0, scored: 0 }; + // 'unavailable' counts too: a trace evaluator that found no spans reports a + // null score without ever calling itself an error, so gating on 'error' + // alone let a model publish an overall built on the evaluators that did + // survive -- ranked against peers who were graded on the full set. + if (doc.evaluator?.label === 'error' || doc.evaluator?.label === 'unavailable') { + tally.errored += 1; + errTrack.set(evaluatorName, tally); + } + + if (typeof score !== 'number') { + if (options.useVerdictLadder && typeof doc.evaluator?.score === 'number') { + excluded.unmappedVerdict += 1; + } + continue; + } + + tally.scored += 1; + errTrack.set(evaluatorName, tally); + + let evaluators = byPrefix.get(prefix); + if (!evaluators) { + evaluators = new Map(); + byPrefix.set(prefix, evaluators); + } + const agg = evaluators.get(evaluatorName) ?? { sum: 0, count: 0 }; + agg.sum += score; + agg.count += 1; + evaluators.set(evaluatorName, agg); + } + + options.onExcluded?.(excluded); + + return [...byPrefix.entries()].map(([prefix, evaluators]) => { + // Only evaluators that errored AND never scored count: one that recovered + // on retry still produced a grade, so its partial history is noise, not a + // broken instrument. + const erroredOut = [...(erroredByPrefix.get(prefix)?.entries() ?? [])] + .filter(([, tally]) => tally.errored > 0 && tally.scored === 0) + .map(([name]) => name); + return { + datasetId: `prefix:${prefix}`, + datasetName: prefix, + evaluators: [...evaluators.entries()].map(([evaluatorName, agg]) => ({ + evaluatorName, + mean: agg.sum / agg.count, + count: agg.count, + })), + ...(erroredOut.length > 0 ? { erroredOutEvaluators: erroredOut } : {}), + }; + }); +}; + +/** + * Selects, per task model, the most recent experiment from a list (typically + * the newest experiments for one suite + model). Experiments without a model + * id or timestamp older than the lookback window are ignored. Pure for unit + * testing. + */ +export const pickLatestExperimentPerModel = ( + experiments: EvaluationExperimentSummary[], + { + lookbackDays, + now = Date.now(), + allowSelfJudged = false, + onSelfJudgedRejected, + }: { + lookbackDays?: number; + now?: number; + allowSelfJudged?: boolean; + /** + * Called for each experiment skipped because the grader was the graded + * model. Selection runs before scoring, so this is the ONLY place the + * fact is observable -- downstream the model simply has no experiment, + * indistinguishable from never having run. + */ + onSelfJudgedRejected?: (experiment: EvaluationExperimentSummary) => void; + } = {} +): Map => { + const cutoff = lookbackDays ? now - lookbackDays * 24 * 60 * 60 * 1000 : undefined; + const latestByModel = new Map(); + + for (const experiment of experiments) { + const modelId = experiment.task_model?.id; + if (!modelId) { + continue; + } + + const at = Date.parse(experiment.timestamp); + // An unparseable timestamp must not be treated as epoch 0, or stale + // experiments would silently survive the lookback cutoff. + if (!Number.isFinite(at) || (cutoff !== undefined && at < cutoff)) { + continue; + } + + // `now` doubles as the upper bound so a matrix can be rendered as of a + // point in time. Applied to every model identically -- it reproduces an + // older matrix, it does not let one model be scored on a different run + // than its neighbours. + if (at > now) { + continue; + } + + // Selection runs before scoring, so a self-judged experiment picked here + // blanks the model outright: its scores are dropped downstream and the + // older, independently judged runs are never reconsidered. Skip it now so + // recency cannot silently cost a model every cell it earned. + // + // `allowSelfJudged` is the audited escape hatch: on a suite where the judge + // demonstrably does not favour itself, dropping the run costs a real cell + // to prevent a bias that was measured not to occur. + const judges = experiment.evaluator_models?.length + ? experiment.evaluator_models + : [experiment.evaluator_model]; + if ( + !allowSelfJudged && + judges.some((judge) => judge?.id && describeJudge(judge.id, modelId).selfJudged) + ) { + onSelfJudgedRejected?.(experiment); + continue; + } + + const existing = latestByModel.get(modelId); + if (!existing || at > existing.at) { + latestByModel.set(modelId, { experiment, at }); + } + } + + return new Map([...latestByModel].map(([modelId, { experiment }]) => [modelId, experiment])); +}; + +/** + * Converts the per-experiment stats returned by the evals plugin into the + * dataset-grouped structure consumed by the matrix builder. Pure for testing. + */ +export const experimentStatsToDatasets = (stats: ExperimentStats): AggregatedDatasetScores[] => { + const byDataset = new Map(); + + for (const stat of stats.stats) { + let dataset = byDataset.get(stat.datasetId); + if (!dataset) { + dataset = { datasetId: stat.datasetId, datasetName: stat.datasetName, evaluators: [] }; + byDataset.set(stat.datasetId, dataset); + } + dataset.evaluators.push({ + evaluatorName: stat.evaluatorName, + mean: stat.stats.mean, + count: stat.stats.count, + min: stat.stats.min, + max: stat.stats.max, + }); + } + + return [...byDataset.values()]; +}; + +/** + * Queries the evals plugin for the latest experiment per (model, suite) and + * returns mean evaluator scores grouped by dataset, ready for `buildMatrix`. + * + * Each (suite, model) pair is listed separately through the route's `model_id` + * term filter: the route answers with a terms aggregation whose bucket size + * grows with `page * per_page`, so a bounded single page per pair is the only + * query shape that scales. The newest experiment within the lookback window is + * then picked client-side (a bare `per_page: 1` request could not express the + * lookback fallback). + */ +/** + * Normalises a branch override to a list. + * + * `branchBySuite` accepts either a single branch or several. A suite whose + * models are split across branches needs the union; a suite pinned to one + * branch keeps the plain-string form. + */ +const toBranchList = (branch: string | string[] | undefined): Array => { + if (Array.isArray(branch)) { + return branch.length > 0 ? branch : [undefined]; + } + return [branch]; +}; + +export const queryMatrixScores = async ( + evalsClient: EvalsClient, + log: SomeDevLog, + { + suiteIds, + modelIds, + branch, + branchBySuite, + lookbackDays, + asOf, + prefixesBySuite = {}, + scoring, + scoringBySuite, + }: QueryMatrixScoresOptions +): Promise => { + const byModel = new Map(); + /** + * Per-model tally of scores rejected by judge policy, keyed by model id. + * Lets the renderer distinguish a never-run cell from one whose grades were + * all thrown away — the two are indistinguishable otherwise. + */ + const excludedByModel = new Map(); + const exampleCoverage: Array<{ + modelId: string; + suiteId: string; + examples: number; + repetitions: number; + }> = []; + + for (const suiteId of suiteIds) { + const suiteBranches = toBranchList(branchBySuite?.[suiteId] ?? branch); + const suiteScoring = scoringBySuite?.[suiteId] ?? scoring; + for (const modelId of modelIds) { + // Golden data for one suite is split across branches by model: a weekly + // matrix branch may hold six models while a seventh only ever ran on a + // feature branch. Querying a single branch silently discards the rest, + // so every configured branch is queried and the results unioned before + // selection picks the newest run per model. + const experiments = ( + await Promise.all( + suiteBranches.map((suiteBranch) => + evalsClient.listExperiments({ + suiteId, + taskModelId: modelId, + branch: suiteBranch, + limit: MAX_LIST_EXPERIMENTS, + }) + ) + ) + ).flat(); + // Selection is where a self-judged run disappears, so record it here: + // downstream this model looks like it never ran the suite. + const selfJudgedRejected: EvaluationExperimentSummary[] = []; + const [latest] = [ + ...pickLatestExperimentPerModel(experiments, { + lookbackDays, + ...(asOf !== undefined ? { now: asOf } : {}), + // A suite that opted out of the exclusion must also keep its + // self-judged runs through selection, or the cell stays blank no + // matter what the scoring policy allows. + allowSelfJudged: suiteScoring?.excludeSelfJudged === false, + onSelfJudgedRejected: (rejected) => { + selfJudgedRejected.push(rejected); + }, + }).values(), + ]; + + log.debug( + `Suite ${suiteId}, model ${modelId}: ${experiments.length} experiment(s)` + + (latest ? '' : ', none within the lookback window') + ); + + if (!latest) { + if (selfJudgedRejected.length > 0) { + // The run exists and was deliberately withheld. Emit a suite record + // with no datasets so the cell renders as "excluded", not "missing". + // The renderer reports a SCORE count, so fetch the withheld run's + // real size rather than passing an experiment tally that would + // render as "1 score(s) rejected" for thousands of documents. + const newest = selfJudgedRejected.reduce((a, b) => + Date.parse(b.timestamp) > Date.parse(a.timestamp) ? b : a + ); + let withheldScores = 0; + try { + const withheldStats = await evalsClient.getExperimentStats(newest.experiment_id, { + suiteId, + taskModelId: modelId, + executionId: newest.execution_id ?? newest.experiment_id, + }); + // No total on the stats shape; sum the per-evaluator sample + // counts, which is what "score(s) rejected" means to a reader. + withheldScores = (withheldStats?.stats ?? []).reduce( + (total, entry) => total + (entry.stats?.count ?? 0), + 0 + ); + } catch (error) { + log.debug( + `Could not size withheld self-judged run for ${modelId}/${suiteId}: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + let withheldModel = byModel.get(modelId); + if (!withheldModel) { + withheldModel = { modelId, suites: [] }; + byModel.set(modelId, withheldModel); + } + withheldModel.suites.push({ + suiteId, + experimentId: newest.experiment_id, + excludedSelfJudged: withheldScores, + datasets: [], + }); + } + continue; + } + + // The experiments listing returns `execution_id` as its grouping key; the + // detail/stats route must be filtered by execution_id (+ suite + model), + // since a bare experiment_id path lookup targets a different field and 404s. + // A sharded sweep splits one model's examples across VMs, each with its + // own execution_id. Selecting a single experiment would render one shard + // and blank every example the others covered, so gather the whole sweep. + // Shard gathering re-scans the raw listing, so the cutoff has to be + // reapplied here. Without it selection honours `asOf` but the shard + // union pulls the excluded runs straight back in. + const shardMembers = pickShardExperiments( + experiments.filter((candidate) => { + if (candidate.task_model?.id !== modelId) { + return false; + } + if (asOf === undefined) { + return true; + } + const at = Date.parse(candidate.timestamp); + return Number.isFinite(at) && at <= asOf; + }) + ); + const shards = shardMembers.some( + (member) => member.execution_id === (latest.execution_id ?? latest.experiment_id) + ) + ? shardMembers + : [latest]; + + const perShardStats = await Promise.all( + shards.map((shard) => + evalsClient.getExperimentStats(shard.experiment_id, { + suiteId, + taskModelId: modelId, + executionId: shard.execution_id ?? shard.experiment_id, + }) + ) + ); + + if (!perShardStats.some((entry) => entry)) { + log.warning( + `No stats for experiment ${latest.experiment_id} (suite ${suiteId}, model ${modelId})` + ); + continue; + } + + let model = byModel.get(modelId); + if (!model) { + model = { + modelId, + family: latest.task_model?.family, + provider: latest.task_model?.provider, + suites: [], + }; + byModel.set(modelId, model); + } + + const datasets = mergeShardDatasets( + perShardStats + .filter((entry): entry is ExperimentStats => Boolean(entry)) + .map((entry) => experimentStatsToDatasets(entry)) + ); + // Per-prefix synthetic datasets: one extra stripped-scores fetch per + // (suite, model). Cheap — unbounded fields are excluded server-side. + const examplePrefixes = prefixesBySuite[suiteId] ?? []; + if (examplePrefixes.length > 0) { + try { + // Per-prefix bucketing must see every shard's docs: fetching only the + // newest execution would fill columns only for its stride of examples. + const scores = ( + await Promise.all( + shards.map((shard) => + evalsClient.getExperimentScores(shard.experiment_id, { + suiteId, + taskModelId: modelId, + executionId: shard.execution_id ?? shard.experiment_id, + }) + ) + ) + ).flat(); + // Capture WHY scores were dropped. Without this the caller cannot tell + // "model never ran" from "model ran and every grade was rejected" — + // they render identically as a blank cell and invite a pointless + // re-sweep. See references/self-judging-provenance-impact.md. + const before = datasets.length; + datasets.push( + ...scoresByPrefixToDatasets(scores, examplePrefixes, { + ...suiteScoring, + onExcluded: (counts) => { + if (counts.selfJudged + counts.nonEis + counts.unmappedVerdict > 0) { + excludedByModel.set(modelId, counts); + } + }, + }) + ); + if (datasets.length === before && excludedByModel.has(modelId)) { + const c = excludedByModel.get(modelId)!; + // A pure unmapped-verdict rejection is not a judge problem, so do + // not send the operator to the judge config. Measured cause on + // attack-discovery: every document carries `example.id` "0", so + // prefix bucketing matches no column and every score falls out. + const judgeIssue = c.selfJudged + c.nonEis; + const remedy = + judgeIssue === 0 + ? `no score carried a mappable verdict — check that this suite's example ids match the column's examplePrefixes (a suite that writes a constant example id cannot be bucketed) before blaming the judge.` + : `Re-running this model will NOT fill these cells — fix the judge assignment first.`; + log.warning( + `All per-prefix scores rejected for model ${modelId} (suite ${suiteId}): ` + + `${c.selfJudged} self-judged, ${c.nonEis} non-EIS judge, ${c.unmappedVerdict} unmapped verdict. ` + + remedy + ); + } + + // The sweep knows when a run stopped early (docs=252/294) but that + // fact never reaches the score docs, so an incomplete experiment + // publishes a headline score indistinguishable from a complete one: + // 4.5-sonnet ranked 7.49 off 18 of 21 examples on 2026-08-29 while + // its peers ran all 21. minCoverage only catches near-empty rows. + // Count the examples this experiment actually carries and say so. + const exampleIds = new Set( + scores + .map((doc) => doc.example?.id) + .filter((id): id is string => typeof id === 'string') + ); + if (exampleIds.size > 0) { + // Repetitions average out judge variance, so a model measured once + // carries a materially wider error bar than one measured three + // times -- comparing them as equals overstates the precision of the + // single-shot row. Track the reps actually present in the docs + // rather than the configured intent, which can silently not apply. + const repetitions = new Set( + scores + .map((doc) => doc.task?.repetition_index) + .filter((index): index is number => typeof index === 'number') + ); + exampleCoverage.push({ + modelId, + suiteId, + examples: exampleIds.size, + repetitions: repetitions.size, + }); + } + } catch (error) { + log.warning( + `Per-prefix scores unavailable for experiment ${ + latest.experiment_id + } (suite ${suiteId}, model ${modelId}): ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } + + model.suites.push({ + suiteId, + experimentId: latest.experiment_id, + executionIds: shards.map((s) => s.execution_id ?? s.experiment_id), + timestamp: latest.timestamp, + commitSha: latest.git_commit_sha ?? undefined, + // Derived from the experiment's own judge/task ids, NOT from the + // column's opt-out: a column that admits self-judged scores still + // contains rows the judge graded at arm's length, and flagging those + // too would be a false accusation. + selfJudged: + latest.evaluator_model?.id && latest.task_model?.id + ? describeJudge(latest.evaluator_model.id, latest.task_model.id).selfJudged + : undefined, + // Which judge graded this row. Rejecting a self-judged run silently + // falls back to an older run graded by someone else, so a model can + // end up measured with a different instrument than the rows it is + // ranked against. Carry the id so that is visible instead of implied. + judgeModelId: latest.evaluator_model?.id ?? undefined, + // A suite whose scores were ALL rejected for self-judging is not the + // same as a suite that never ran: the run exists and its size is + // known. Carry the count so the cell can say which it is. + excludedSelfJudged: + datasets.length === 0 && (excludedByModel.get(modelId)?.selfJudged ?? 0) > 0 + ? excludedByModel.get(modelId)!.selfJudged + : undefined, + datasets, + }); + } + } + + // Treat the count most models reached as the suite's full size rather than + // hardcoding it: the matrix reads score docs, and the expected example count + // lives in the eval suite. 18 of 20 models covered all 21 examples on + // 2026-08-29, so a model below that modal count ran short. + const bySuite = new Map(); + for (const entry of exampleCoverage) { + const list = bySuite.get(entry.suiteId) ?? []; + list.push(entry.examples); + bySuite.set(entry.suiteId, list); + } + for (const [suiteId, sizes] of bySuite) { + const tally = new Map(); + for (const n of sizes) tally.set(n, (tally.get(n) ?? 0) + 1); + let full = 0; + let best = 0; + for (const [n, c] of tally) { + if (c > best || (c === best && n > full)) { + full = n; + best = c; + } + } + for (const entry of exampleCoverage) { + if (entry.suiteId === suiteId && entry.examples < full) { + log.warning( + `${entry.modelId} scored on ${entry.examples} of ${full} examples in ${suiteId} -- its score rests on an incomplete run and is not comparable to models that ran all ${full}` + ); + } + } + + // Same modal-count logic for repetitions. A row measured once sits on a + // wider error bar than one measured three times, but both render as a + // single number, so the imbalance is invisible in the published artifact + // unless it is said out loud. + const repTally = new Map(); + for (const entry of exampleCoverage) { + if (entry.suiteId === suiteId && entry.repetitions > 0) { + repTally.set(entry.repetitions, (repTally.get(entry.repetitions) ?? 0) + 1); + } + } + let modalReps = 0; + let modalRepCount = 0; + for (const [reps, count] of repTally) { + // Tie-break toward the LOWER repetition count: with two models at 1 and 3 + // reps the baseline is the cheaper, more common shape, and the 3-rep row + // is the outlier worth flagging. Preferring the higher count here would + // make the advantaged row the baseline and silence the warning entirely. + const unset = modalRepCount === 0; + if (unset || count > modalRepCount || (count === modalRepCount && reps < modalReps)) { + modalReps = reps; + modalRepCount = count; + } + } + const better = [...repTally.keys()].filter((reps) => reps > modalReps); + if (better.length > 0) { + const maxReps = Math.max(...better); + const advantaged = exampleCoverage + .filter((entry) => entry.suiteId === suiteId && entry.repetitions === maxReps) + .map((entry) => entry.modelId); + log.warning( + `Repetition imbalance in ${suiteId}: most models were measured with ${modalReps} repetition(s), but ${advantaged.join( + ', ' + )} ran ${maxReps} -- the higher-repetition rows carry a narrower error bar, so ranking them against the rest compares estimates of unequal precision` + ); + } + } + + for (const [modelId, counts] of excludedByModel) { + if (counts.selfJudged > 0) { + log.warning( + `${modelId}: dropped ${counts.selfJudged} self-judged score doc(s). ` + + `Excluding them is correct, but a model judged by itself is not evidence ` + + `of quality — re-run it against an independent judge to fill those cells.` + ); + } + } + log.debug(`Matrix query resolved ${byModel.size} model(s) across ${suiteIds.length} suite(s)`); + return [...byModel.values()].map((model) => { + const excluded = excludedByModel.get(model.modelId); + return excluded ? { ...model, excluded } : model; + }); +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.test.ts new file mode 100644 index 0000000000000..9bd093e2cf380 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.test.ts @@ -0,0 +1,711 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EvaluationScoreDocument } from '@kbn/evals-common'; +import { + aliasTraceKeys, + countRepetitions, + exampleScoresByEvaluator, + exampleSpreadByEvaluator, + overlayRepeatedCacheTrails, + queryMatrixTraces, +} from './query_matrix_traces'; +import type { MatrixTraceData } from './trace_types'; + +const doc = (evaluatorName: string | undefined, score: number | null): EvaluationScoreDocument => + ({ + evaluator: { name: evaluatorName, score }, + } as EvaluationScoreDocument); + +describe('exampleScoresByEvaluator', () => { + it('means scores per evaluator across repetitions', () => { + const result = exampleScoresByEvaluator([ + doc('ExpectedToolCalled', 0.8), + doc('ExpectedToolCalled', 1), + doc('Correctness', 0.5), + ]); + expect(result).toEqual({ ExpectedToolCalled: 0.9, Correctness: 0.5 }); + }); + + it('skips documents without an evaluator name or a numeric score', () => { + const result = exampleScoresByEvaluator([ + doc('ExpectedToolCalled', 0.8), + doc('ExpectedToolCalled', null), + doc(undefined, 0.4), + ]); + expect(result).toEqual({ ExpectedToolCalled: 0.8 }); + }); + + it('returns an empty map when nothing is scorable', () => { + expect(exampleScoresByEvaluator([doc('ExpectedToolCalled', null)])).toEqual({}); + expect(exampleScoresByEvaluator([])).toEqual({}); + }); +}); + +describe('exampleSpreadByEvaluator', () => { + const repDoc = (name: string, score: number, repetition: number) => + ({ + evaluator: { name, score }, + task: { repetition_index: repetition }, + } as unknown as EvaluationScoreDocument); + + it('reports max - min per evaluator across repetitions', () => { + expect( + exampleSpreadByEvaluator([repDoc('Groundedness', 0, 0), repDoc('Groundedness', 20, 1)]) + ).toEqual({ Groundedness: 20 }); + }); + + it('distinguishes a volatile cell from a stable one with the same mean', () => { + // Both average to 10 — the mean alone cannot tell these apart, which is + // the entire reason this function exists. + const stable = [ + repDoc('Relevance', 10, 0), + repDoc('Relevance', 10, 1), + repDoc('Relevance', 10, 2), + ]; + const volatile = [ + repDoc('Relevance', 0, 0), + repDoc('Relevance', 10, 1), + repDoc('Relevance', 20, 2), + ]; + + expect(exampleScoresByEvaluator(stable)).toEqual(exampleScoresByEvaluator(volatile)); + expect(exampleSpreadByEvaluator(stable)).toEqual({ Relevance: 0 }); + expect(exampleSpreadByEvaluator(volatile)).toEqual({ Relevance: 20 }); + }); + + it('omits evaluators observed only once rather than claiming zero spread', () => { + // A single observation has no measured stability; emitting 0 would assert + // one that was never tested. + expect(exampleSpreadByEvaluator([repDoc('Factuality', 7, 0)])).toEqual({}); + }); + + it('ignores documents with no numeric score', () => { + const missing = { evaluator: { name: 'criteria' } } as unknown as EvaluationScoreDocument; + expect( + exampleSpreadByEvaluator([repDoc('criteria', 1, 0), missing, repDoc('criteria', 0, 1)]) + ).toEqual({ criteria: 1 }); + }); +}); + +describe('countRepetitions', () => { + const repDoc = (repetitionIndex: number | undefined): EvaluationScoreDocument => + ({ task: { repetition_index: repetitionIndex } } as EvaluationScoreDocument); + + it('counts distinct repetition indices', () => { + expect(countRepetitions([repDoc(0), repDoc(0), repDoc(1), repDoc(2)])).toBe(3); + }); + + it('treats a missing index as repetition 0', () => { + expect(countRepetitions([repDoc(undefined), repDoc(0)])).toBe(1); + }); +}); + +describe('queryMatrixTraces example fetching', () => { + const completeDoc = (executionId: string): EvaluationScoreDocument => + ({ + example: { id: 'example-1' }, + evaluator: { name: 'Correctness', score: 1 }, + metadata: { execution_id: executionId }, + task: { + model: { id: 'model-x' }, + output: { messages: [{ message: 'done' }] }, + repetition_index: 0, + }, + } as unknown as EvaluationScoreDocument); + + const makeClient = (opts: { filtered: boolean }) => { + const getExampleScores = jest.fn( + async (_exampleId: string, filters?: { executionId?: string }) => + // A server that knows the filters returns only that execution; an old + // server ignores them and returns every execution mixed together. + opts.filtered + ? [completeDoc(filters?.executionId ?? 'exec-a')] + : [completeDoc('exec-a'), completeDoc('exec-b')] + ); + const client = { + getExperimentScores: jest.fn( + async () => [{ example: { id: 'example-1' } }] as EvaluationScoreDocument[] + ), + getExampleScores, + }; + return client; + }; + + const aggregatedFor = (experimentId: string) => [ + { + modelId: 'model-x', + suites: [{ suiteId: 'suite-1', experimentId, datasets: [], evaluators: [] }], + }, + ]; + + const logStub = { + debug: jest.fn(), + warning: jest.fn(), + }; + + it('applies configured model aliases to the resolved trace keys', async () => { + // Wiring guard: aliasTraceKeys must be called by queryMatrixTraces itself. + // Without the call, aliased OpenRouter rows resolve zero trace cells even + // though the helper is correct in isolation. + const client = makeClient({ filtered: true }); + const traces = await queryMatrixTraces( + client as never, + logStub as never, + aggregatedFor('exec-a') as never, + undefined, + 0, + new Map([['openrouter-model-x', ['model-x']]]) + ); + expect(traces['model-x:example-1']).toBeDefined(); + expect(traces['openrouter-model-x:example-1']).toEqual(traces['model-x:example-1']); + }); + + it('arms the legacy fallback when a later response reveals unfiltered scores', async () => { + // Discriminates the latch specifically: the FIRST response is empty (proves + // nothing), the SECOND returns mixed executions (proves the server ignores + // filters). With the old `scores.length === 0 ||` latch the first response + // pinned serverSupportsFilter=true and this warning never fired. + let call = 0; + const getExampleScores = jest.fn(async () => { + call += 1; + return call === 1 ? [] : [completeDoc('exec-a'), completeDoc('exec-b')]; + }); + const client = { + getExperimentScores: jest.fn( + async () => + [ + { example: { id: 'example-1' } }, + { example: { id: 'example-2' } }, + ] as EvaluationScoreDocument[] + ), + getExampleScores, + }; + const log = { debug: jest.fn(), warning: jest.fn() }; + await queryMatrixTraces(client as never, log as never, aggregatedFor('exec-a') as never); + + expect(log.warning).toHaveBeenCalledWith( + expect.stringContaining('Example-scores route ignores execution filters') + ); + }); + + it('reports runaway tool loops above the configured threshold', async () => { + const heavy = (calls: number, trail: number): EvaluationScoreDocument => + ({ + example: { id: 'example-1' }, + evaluator: { name: 'Tool Calls', score: calls }, + metadata: { execution_id: 'exec-a' }, + task: { + model: { id: 'model-x' }, + output: { + messages: [{ message: 'done' }], + steps: Array.from({ length: trail }, () => ({ + type: 'tool_call', + tool_id: 'platform.core.search', + })), + }, + repetition_index: 0, + }, + } as unknown as EvaluationScoreDocument); + + const client = { + getExperimentScores: jest.fn( + async () => [{ example: { id: 'example-1' } }] as EvaluationScoreDocument[] + ), + getExampleScores: jest.fn(async () => [heavy(44, 44)]), + }; + const log = { debug: jest.fn(), warning: jest.fn() }; + await queryMatrixTraces( + client as never, + log as never, + aggregatedFor('exec-a') as never, + undefined, + 40 + ); + + expect(log.warning).toHaveBeenCalledWith( + expect.stringContaining('Possible runaway tool loops') + ); + }); + + it('does not cite a tool-call count the recorded trail cannot corroborate', async () => { + // The real shape from golden: openai-gpt-5.2:alert-analysis-c scored 115 + // against a 29-call trail (ratio 4.0) on 2026-08-22, while its five sibling + // executions of the same cell scored ~1.2x their trails. Reported as a loop + // length it became the headline "115 calls" in downstream summaries — a + // number no trace supports. + const scoredWithTrail = (calls: number, trail: number): EvaluationScoreDocument => + ({ + example: { id: 'example-1' }, + evaluator: { name: 'Tool Calls', score: calls }, + metadata: { execution_id: 'exec-a' }, + task: { + model: { id: 'model-x' }, + output: { + messages: [{ message: 'done' }], + steps: Array.from({ length: trail }, () => ({ + type: 'tool_call', + tool_id: 'platform.core.search', + })), + }, + repetition_index: 0, + }, + } as unknown as EvaluationScoreDocument); + + const client = { + getExperimentScores: jest.fn( + async () => [{ example: { id: 'example-1' } }] as EvaluationScoreDocument[] + ), + getExampleScores: jest.fn(async () => [scoredWithTrail(115, 29)]), + }; + const log = { debug: jest.fn(), warning: jest.fn() }; + await queryMatrixTraces( + client as never, + log as never, + aggregatedFor('exec-a') as never, + undefined, + 40 + ); + + expect(log.warning).toHaveBeenCalledWith( + expect.stringContaining("'Tool Calls' exceeds the recorded tool trail") + ); + expect(log.warning).toHaveBeenCalledWith(expect.stringContaining('=115 (trail 29)')); + // It must NOT also be presented as a genuine loop length. + expect(log.warning).not.toHaveBeenCalledWith( + expect.stringContaining('Possible runaway tool loops') + ); + }); + + it('still reports a high count when no trail is available to refute it', async () => { + // A cell with no cached steps cannot corroborate OR refute the count, so it + // must stay reportable rather than being silently dropped as suspect. + const noTrailDoc = { + example: { id: 'example-1' }, + evaluator: { name: 'Tool Calls', score: 115 }, + metadata: { execution_id: 'exec-a' }, + task: { + model: { id: 'model-x' }, + output: { messages: [{ message: 'done' }] }, + repetition_index: 0, + }, + } as unknown as EvaluationScoreDocument; + + const client = { + getExperimentScores: jest.fn( + async () => [{ example: { id: 'example-1' } }] as EvaluationScoreDocument[] + ), + getExampleScores: jest.fn(async () => [noTrailDoc]), + }; + const log = { debug: jest.fn(), warning: jest.fn() }; + await queryMatrixTraces( + client as never, + log as never, + aggregatedFor('exec-a') as never, + undefined, + 40 + ); + + expect(log.warning).toHaveBeenCalledWith( + expect.stringContaining('Possible runaway tool loops') + ); + }); + + it('does not report tool loops when the threshold is disabled', async () => { + const client = makeClient({ filtered: true }); + const log = { debug: jest.fn(), warning: jest.fn() }; + await queryMatrixTraces(client as never, log as never, aggregatedFor('exec-a') as never); + + expect(log.warning).not.toHaveBeenCalledWith( + expect.stringContaining('Possible runaway tool loops') + ); + }); + + it('does not treat an empty response as proof the server honours filters', async () => { + // The regression: an empty first response latched serverSupportsFilter=true, + // so the legacy fallback never armed and every later cell came back empty — + // 442/442 hollow traces while scores rendered perfectly. + const empty = jest.fn(async () => [] as EvaluationScoreDocument[]); + const client = { + getExperimentScores: jest.fn( + async () => [{ example: { id: 'example-1' } }] as EvaluationScoreDocument[] + ), + getExampleScores: empty, + }; + const log = { debug: jest.fn(), warning: jest.fn() }; + await queryMatrixTraces(client as never, log as never, aggregatedFor('exec-a') as never); + + expect(log.warning).toHaveBeenCalledWith( + expect.stringContaining('Trace fetch returned no documents') + ); + }); + + it('stays quiet about total trace loss when documents do come back', async () => { + const client = makeClient({ filtered: true }); + const log = { debug: jest.fn(), warning: jest.fn() }; + await queryMatrixTraces(client as never, log as never, aggregatedFor('exec-a') as never); + + expect(log.warning).not.toHaveBeenCalledWith( + expect.stringContaining('Trace fetch returned no documents') + ); + }); + + it('passes the execution filter to the example-scores route', async () => { + const client = makeClient({ filtered: true }); + const traces = await queryMatrixTraces( + client as never, + logStub as never, + aggregatedFor('exec-a') as never + ); + expect(client.getExampleScores).toHaveBeenCalledWith('example-1', { + executionId: 'exec-a', + modelId: 'model-x', + }); + expect(Object.keys(traces)).toContain('model-x:example-1'); + }); + + it('detects an unfiltered (legacy) server and reuses the shared fetch across runs', async () => { + const client = makeClient({ filtered: false }); + await queryMatrixTraces( + client as never, + logStub as never, + [...aggregatedFor('exec-a'), ...aggregatedFor('exec-b')] as never + ); + // Second run for the same example reuses the first fetch instead of + // re-downloading the full unfiltered payload. + expect(client.getExampleScores).toHaveBeenCalledTimes(1); + }); + + it('serves cells from the trace cache without touching the server', async () => { + const client = makeClient({ filtered: true }); + const traceCache = { + 'exec-a::example-1': [completeDoc('exec-a')], + }; + const traces = await queryMatrixTraces( + client as never, + logStub as never, + aggregatedFor('exec-a') as never, + traceCache as never + ); + expect(client.getExampleScores).not.toHaveBeenCalled(); + expect(traces['model-x:example-1']).toMatchObject({ + scores: { Correctness: 1 }, + repetitions: 1, + }); + }); + + it('does not duplicate the direct example key under prefix:', async () => { + const client = makeClient({ filtered: true }); + // Per-example columns set examplePrefixes to the full example id, which + // would emit prefix:example-1 as a byte-duplicate of example-1. + const aggregated = [ + { + modelId: 'model-x', + suites: [ + { + suiteId: 'suite-1', + experimentId: 'exec-a', + datasets: [{ datasetId: 'prefix:example-1' }], + evaluators: [], + }, + ], + }, + ]; + const traces = await queryMatrixTraces(client as never, logStub as never, aggregated as never); + expect(Object.keys(traces)).toContain('model-x:example-1'); + expect(Object.keys(traces)).not.toContain('model-x:prefix:example-1'); + }); + + it('fetches once per example on a legacy server even with many runs', async () => { + const client = makeClient({ filtered: false }); + const aggregated = Array.from({ length: 6 }, (_, i) => aggregatedFor(`exec-${i}`)).flat(); + await queryMatrixTraces(client as never, logStub as never, aggregated as never); + expect(client.getExampleScores).toHaveBeenCalledTimes(1); + }); + + it('fans out over every execution id of a sharded suite row', async () => { + // A sharded sweep splits one model's examples across executions. Trace + // enumeration that reads only experimentId sees ONE shard's examples and + // the other shard's cells render "trace unavailable" forever. + const shardedDoc = (executionId: string, exampleId: string): EvaluationScoreDocument => + ({ + example: { id: exampleId }, + metadata: { execution_id: executionId }, + evaluator: { name: 'Correctness', score: 1 }, + task: { + model: { id: 'model-x' }, + output: { messages: [{ message: 'done' }] }, + repetition_index: 0, + }, + } as unknown as EvaluationScoreDocument); + + const getExperimentScores = jest.fn( + async (_experimentId: string, { executionId }: { executionId?: string }) => + executionId === 'sweep-9-s1of2::suite::model-x' + ? [shardedDoc('sweep-9-s1of2::suite::model-x', 'example-1')] + : [shardedDoc('sweep-9-s2of2::suite::model-x', 'example-2')] + ); + const getExampleScores = jest.fn( + async (_exampleId: string, { executionId }: { executionId?: string }) => [ + shardedDoc(executionId ?? '?', _exampleId), + ] + ); + const client = { getExperimentScores, getExampleScores }; + const log = { debug: jest.fn(), warning: jest.fn() }; + + const aggregated = [ + { + modelId: 'model-x', + suites: [ + { + suiteId: 'suite-1', + experimentId: 'sweep-9-s1of2::suite::model-x', + executionIds: ['sweep-9-s1of2::suite::model-x', 'sweep-9-s2of2::suite::model-x'], + datasets: [], + }, + ], + }, + ]; + + await queryMatrixTraces(client as never, log as never, aggregated as never); + + // Both shards enumerated: each contributed its own example. + expect(getExperimentScores).toHaveBeenCalledTimes(2); + const enumeratedExecs = getExperimentScores.mock.calls.map((c) => c[1]?.executionId).sort(); + expect(enumeratedExecs).toEqual([ + 'sweep-9-s1of2::suite::model-x', + 'sweep-9-s2of2::suite::model-x', + ]); + // No "trace unavailable" for the second shard's example. + expect(log.warning).not.toHaveBeenCalledWith( + expect.stringContaining('No complete score documents found') + ); + }); + + it('fetches each (run, example) pair on a filtered server with no cross-run aliasing', async () => { + const client = makeClient({ filtered: true }); + await queryMatrixTraces( + client as never, + logStub as never, + [...aggregatedFor('exec-a'), ...aggregatedFor('exec-b')] as never + ); + expect(client.getExampleScores).toHaveBeenCalledTimes(2); + const executions = client.getExampleScores.mock.calls.map(([, f]) => f?.executionId).sort(); + expect(executions).toEqual(['exec-a', 'exec-b']); + }); + + it('bounds example-fetch concurrency and overlaps work across runs', async () => { + let inFlight = 0; + let maxInFlight = 0; + const getExampleScores = jest.fn( + async (_exampleId: string, filters?: { executionId?: string }) => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + await new Promise((resolve) => setTimeout(resolve, 5)); + inFlight -= 1; + return [completeDoc(filters?.executionId ?? 'exec-a')]; + } + ); + const client = { + getExperimentScores: jest.fn( + async () => [{ example: { id: 'example-1' } }] as EvaluationScoreDocument[] + ), + getExampleScores, + }; + const aggregated = Array.from({ length: 12 }, (_, i) => aggregatedFor(`exec-${i}`)).flat(); + await queryMatrixTraces(client as never, logStub as never, aggregated as never); + // First pair runs alone for filter detection; the remaining 11 pool at ≤8. + expect(maxInFlight).toBeGreaterThan(1); + expect(maxInFlight).toBeLessThanOrEqual(8); + expect(getExampleScores).toHaveBeenCalledTimes(12); + }); + + it('warns with model and example names when some scored cells lose their trace', async () => { + const client = { + getExperimentScores: jest.fn( + async () => [{ example: { id: 'example-1' } }, { example: { id: 'example-2' } }] as never + ), + // example-2 comes back with no documents at all: scores were aggregated + // for it upstream, but no trace doc exists for this execution. + getExampleScores: jest.fn(async (exampleId: string) => + exampleId === 'example-1' ? [completeDoc('exec-a')] : [] + ), + }; + const log = { debug: jest.fn(), warning: jest.fn() }; + const traces = await queryMatrixTraces( + client as never, + log as never, + aggregatedFor('exec-a') as never + ); + expect(Object.keys(traces)).toContain('model-x:example-1'); + expect(log.warning).toHaveBeenCalledWith(expect.stringContaining('Trace coverage incomplete')); + expect(log.warning).toHaveBeenCalledWith(expect.stringContaining('example-2')); + expect(log.warning).toHaveBeenCalledWith(expect.stringContaining('model-x')); + }); + + it('retries a transient fetch failure once before dropping the trace', async () => { + let calls = 0; + const getExampleScores = jest.fn(async () => { + calls += 1; + if (calls === 1) throw new Error('503 Service Unavailable'); + return [completeDoc('exec-a')]; + }); + const client = { + getExperimentScores: jest.fn( + async () => [{ example: { id: 'example-1' } }] as EvaluationScoreDocument[] + ), + getExampleScores, + }; + const traces = await queryMatrixTraces( + client as never, + logStub as never, + aggregatedFor('exec-a') as never + ); + expect(getExampleScores).toHaveBeenCalledTimes(2); + expect(Object.keys(traces)).toContain('model-x:example-1'); + }); + + it('enumerates experiments concurrently in phase 1', async () => { + let inFlight = 0; + let maxInFlight = 0; + const getExperimentScores = jest.fn(async () => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + await new Promise((resolve) => setTimeout(resolve, 5)); + inFlight -= 1; + return [{ example: { id: 'example-1' } }] as EvaluationScoreDocument[]; + }); + const client = { + getExperimentScores, + getExampleScores: jest.fn(async () => [completeDoc('exec-a')]), + }; + const aggregated = Array.from({ length: 8 }, (_, i) => aggregatedFor(`exec-${i}`)).flat(); + await queryMatrixTraces(client as never, logStub as never, aggregated as never); + expect(maxInFlight).toBeGreaterThan(1); + expect(maxInFlight).toBeLessThanOrEqual(6); + expect(getExperimentScores).toHaveBeenCalledTimes(8); + }); + + it('flags a broken Tool Calls metric when the trail is non-empty but the score is 0', async () => { + // Observed on real golden data: 4.5-sonnet scored Tool Calls=0 on 20/20 + // cells whose traces showed 43+ real calls. Unreported that publishes as + // "this model uses no tools" and hides genuine tool-loop failures. + const zeroToolCallDoc = { + example: { id: 'example-1' }, + evaluator: { name: 'Tool Calls', score: 0 }, + metadata: { execution_id: 'exec-a' }, + task: { + model: { id: 'model-x' }, + output: { + messages: [{ message: 'done' }], + steps: [ + { type: 'tool_call', tool_id: 'load_skill' }, + { type: 'tool_call', tool_id: 'load_skill' }, + ], + }, + repetition_index: 0, + }, + } as unknown as EvaluationScoreDocument; + + const log = { debug: jest.fn(), warning: jest.fn() }; + await queryMatrixTraces( + makeClient({ filtered: true }) as never, + log as never, + aggregatedFor('exec-a') as never, + { 'exec-a::example-1': [zeroToolCallDoc] } as never, + 40 + ); + + expect(log.warning).toHaveBeenCalledWith( + expect.stringContaining("'Tool Calls' reads 0 despite a non-empty tool trail") + ); + }); +}); + +describe('overlayRepeatedCacheTrails', () => { + const scored = (model: string, toolId: string, repetition: number): EvaluationScoreDocument => + ({ + task: { + model: { id: model }, + repetition_index: repetition, + output: { steps: [{ type: 'tool_call', tool_id: toolId }] }, + }, + } as unknown as EvaluationScoreDocument); + + it('overlays the genuine 3-rep execution and ignores sibling 1-rep weekly runs', () => { + const traces: MatrixTraceData = { + 'opus:workflow-authoring-a': { toolTrail: ['weekly'] }, + }; + overlayRepeatedCacheTrails(traces, { + 'old::security-persona-matrix::opus::workflow-authoring-a': [ + scored('opus', 'generate_workflow', 0), + scored('opus', 'generate_workflow', 1), + scored('opus', 'sml_search', 2), + ], + 'weekly::security-persona-matrix::opus::workflow-authoring-a': [ + scored('opus', 'execute_api', 0), + ], + }); + expect(traces['opus:workflow-authoring-a'].repTrails).toEqual([ + ['generate_workflow'], + ['generate_workflow'], + ['sml_search'], + ]); + }); + + it('does not invent repeats by concatenating two 1-rep executions', () => { + const traces: MatrixTraceData = {}; + overlayRepeatedCacheTrails(traces, { + 'run-a::security-persona-matrix::gpt::workflow-authoring-a': [scored('gpt', 'search', 0)], + 'run-b::security-persona-matrix::gpt::workflow-authoring-a': [scored('gpt', 'load_skill', 0)], + }); + expect(traces['gpt:workflow-authoring-a']).toBeUndefined(); + }); +}); + +describe('aliasTraceKeys', () => { + const entry = (stepCount: number) => ({ stepCount } as MatrixTraceData[string]); + + it('mirrors provider-keyed cells onto the row id for aliased models', () => { + // Regression: OpenRouter rows are keyed by connector id + // (openrouter-deepseek-v4-pro) while score docs report task.model.id as the + // upstream slug (deepseek/deepseek-v4-pro-0813), so every cell rendered + // "Trace unavailable" despite a full set of traces being present. + const traces: MatrixTraceData = { + 'deepseek/deepseek-v4-pro-0813:alert-analysis-a': entry(24), + 'deepseek/deepseek-v4-pro-0813:detection-rule-edit-b': entry(11), + }; + aliasTraceKeys( + traces, + new Map([['openrouter-deepseek-v4-pro', ['deepseek/deepseek-v4-pro-0813']]]) + ); + expect(traces['openrouter-deepseek-v4-pro:alert-analysis-a']).toEqual(entry(24)); + expect(traces['openrouter-deepseek-v4-pro:detection-rule-edit-b']).toEqual(entry(11)); + }); + + it('does not clobber a cell the row already resolved under its own id', () => { + const traces: MatrixTraceData = { + 'row:alert-analysis-a': entry(5), + 'provider/slug:alert-analysis-a': entry(99), + }; + aliasTraceKeys(traces, new Map([['row', ['provider/slug']]])); + expect(traces['row:alert-analysis-a']).toEqual(entry(5)); + }); + + it('leaves non-aliased (EIS) rows untouched', () => { + const traces: MatrixTraceData = { 'anthropic-claude-4.5-haiku:alert-analysis-a': entry(7) }; + const before = { ...traces }; + aliasTraceKeys(traces, new Map()); + expect(traces).toEqual(before); + }); + + it('only rewrites the example suffix, not example ids containing the alias', () => { + const traces: MatrixTraceData = { 'provider/slug:prefix:alert-analysis': entry(3) }; + aliasTraceKeys(traces, new Map([['row', ['provider/slug']]])); + expect(traces['row:prefix:alert-analysis']).toEqual(entry(3)); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.ts new file mode 100644 index 0000000000000..4af8c3d975c23 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.ts @@ -0,0 +1,752 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SomeDevLog } from '@kbn/some-dev-log'; +import type { EvalsClient } from '@kbn/evals'; +import type { EvaluationScoreDocument } from '@kbn/evals-common'; +import type { AggregatedModelScores } from './query_matrix_scores'; +import type { MatrixTraceData, MatrixTraceEntry, TraceStep } from './trace_types'; +import { traceKey } from './trace_types'; +import type { PathContract } from './trajectory_agreement'; +import { answersFromDocs, pathContractFromDocs, trailsFromDocs } from './trajectory_agreement'; + +/** + * Runs `fn` over `items` with at most `limit` in flight, preserving input + * order in the returned array. Package intentionally has zero dependencies, + * so this is a small local worker pool instead of p-limit. + */ +const mapWithConcurrency = async ( + items: T[], + limit: number, + fn: (item: T, index: number) => Promise +): Promise => { + const results = new Array(items.length); + let next = 0; + const worker = async (): Promise => { + while (next < items.length) { + const index = next++; + results[index] = await fn(items[index], index); + } + }; + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)); + return results; +}; + +/** + * Extracts trace data (initial question, tool trail, agent answer, step trace) + * from a single evaluation score document. + * + * Score documents store the full `task.output` which contains: + * - `steps`: array of `{ type: "reasoning"|"tool_call"|"relevant_skills", ... }` + * - `messages`: array of `{ message: string }` (the agent's final answer) + * And `example.input.question` holds the initial user question. + */ +const extractTraceFromScore = (score: EvaluationScoreDocument): MatrixTraceEntry => { + const question = (score.example?.input as { question?: string } | null)?.question; + const taskOutput = score.task?.output as + | { + steps?: Array>; + messages?: Array<{ message?: string }>; + } + | null + | undefined; + + const steps: TraceStep[] = []; + const toolTrail: string[] = []; + + for (const step of taskOutput?.steps ?? []) { + const stepType = step.type as string | undefined; + if (stepType === 'tool_call') { + const toolId = step.tool_id as string | undefined; + if (toolId) { + toolTrail.push(toolId); + } + // Live score docs store arguments under `params`. `args` is the name this + // extractor originally guessed; 0/363 tool_call steps on golden carried it. + const rawParams = step.params ?? step.args; + steps.push({ + type: 'tool', + toolId, + toolParams: rawParams ? JSON.stringify(rawParams).slice(0, 300) : undefined, + }); + } else if (stepType === 'reasoning') { + steps.push({ + type: 'reasoning', + text: (step.reasoning as string | undefined)?.slice(0, 500), + }); + } else if (stepType === 'relevant_skills') { + const skills = Array.isArray(step.skills) + ? (step.skills as Array<{ id?: string }>) + .map((s) => s.id) + .filter((id): id is string => Boolean(id)) + : undefined; + steps.push({ type: 'skill', skills }); + } + } + + // The final answer is the last non-empty message + let answer: string | undefined; + for (const msg of taskOutput?.messages ?? []) { + const content = msg.message; + if (content && content.length > 50) { + answer = content; + } + } + + return { + question, + toolTrail: toolTrail.length > 0 ? toolTrail : undefined, + answer: answer || undefined, + steps: steps.length > 0 ? steps : undefined, + stepCount: steps.length, + toolCount: toolTrail.length, + }; +}; + +/** + * A score document is considered "complete" when: + * - `evaluator.score` is non-null (the evaluator finished and produced a verdict) + * - `task.output` is non-null (the agent produced output — steps and/or messages) + * + * Incomplete runs (e.g. worker SIGKILL, OOM, timeout) may still write partial + * score documents with null scores or empty output. These should not pollute + * the matrix because they don't represent a real evaluation. + */ +const isCompleteScore = (score: EvaluationScoreDocument): boolean => { + if (score.evaluator?.score == null) return false; + return score.task?.output != null; +}; + +/** + * Per-evaluator mean scores for one example, computed over all of the + * example's score documents in the experiment (one doc per evaluator per + * repetition). Pure for unit testing. + */ +export const exampleScoresByEvaluator = ( + docs: EvaluationScoreDocument[] +): Record => { + const sums = new Map(); + for (const doc of docs) { + const name = doc.evaluator?.name; + const score = doc.evaluator?.score; + if (!name || typeof score !== 'number') { + continue; + } + const agg = sums.get(name) ?? { sum: 0, count: 0 }; + agg.sum += score; + agg.count += 1; + sums.set(name, agg); + } + return Object.fromEntries([...sums.entries()].map(([name, agg]) => [name, agg.sum / agg.count])); +}; + +/** + * Per-evaluator spread (max - min) across the repetitions of one example. + * + * The mean returned by `exampleScoresByEvaluator` hides volatility: 10/10/10 + * and 0/10/20 both average to 10. Reporting the spread alongside the mean is + * what makes a repeated run more informative than a single one. Returns only + * evaluators seen more than once — a single observation has no measurable + * spread, and emitting 0 for it would claim a stability that was never tested. + * Pure for unit testing. + */ +export const exampleSpreadByEvaluator = ( + docs: EvaluationScoreDocument[] +): Record => { + const seen = new Map(); + for (const doc of docs) { + const name = doc.evaluator?.name; + const score = doc.evaluator?.score; + if (!name || typeof score !== 'number') { + continue; + } + const agg = seen.get(name); + if (!agg) { + seen.set(name, { min: score, max: score, count: 1 }); + continue; + } + agg.min = Math.min(agg.min, score); + agg.max = Math.max(agg.max, score); + agg.count += 1; + } + return Object.fromEntries( + [...seen.entries()] + .filter(([, agg]) => agg.count > 1) + .map(([name, agg]) => [name, agg.max - agg.min]) + ); +}; + +/** + * Number of distinct repetitions present in a batch of score documents. + * Pure for unit testing. + */ +export const countRepetitions = (docs: EvaluationScoreDocument[]): number => + new Set(docs.map((doc) => doc.task?.repetition_index ?? 0)).size; + +/** + * The weekly board stores one repetition per (model, example). The 3-rep + * reliability pilots live on older execution ids in the same cache. Overlay + * those genuine repeats onto the example-id trace key without concatenating + * separate 1-rep weekly runs, which would invent false agreement. + * + * Cache keys are `${executionId}::${exampleId}`. executionId itself contains + * `::`, so the example id is the substring after the last `::`. + */ +export const overlayRepeatedCacheTrails = ( + traces: MatrixTraceData, + traceCache?: Record +): void => { + if (!traceCache) { + return; + } + interface RepeatedCell { + trails: string[][]; + answers: string[]; + pathContract?: PathContract; + executionId: string; + } + const best = new Map(); + for (const [cacheKey, docs] of Object.entries(traceCache)) { + const split = cacheKey.lastIndexOf('::'); + if (split < 0 || docs.length === 0) { + continue; + } + const exampleId = cacheKey.slice(split + 2); + const modelId = docs[0].task?.model?.id; + if (!modelId || !exampleId) { + continue; + } + const trails = trailsFromDocs(docs); + if (trails.length <= 1) { + continue; + } + const combo = `${modelId}\0${exampleId}`; + const previous = best.get(combo); + if (!previous || trails.length > previous.trails.length) { + best.set(combo, { + trails, + answers: answersFromDocs(docs), + pathContract: pathContractFromDocs(docs), + executionId: cacheKey.slice(0, split), + }); + } + } + for (const [combo, cell] of best) { + const sep = combo.indexOf('\0'); + const modelId = combo.slice(0, sep); + const exampleId = combo.slice(sep + 1); + const key = traceKey(modelId, exampleId); + const measured = { + repTrails: cell.trails, + repAnswers: cell.answers, + pathContract: cell.pathContract, + repExecutionIds: [cell.executionId], + }; + const existing = traces[key]; + if (existing) { + Object.assign(existing, measured); + } else { + traces[key] = measured; + } + } +}; + +/** + * Merges a batch of full (unstripped) score documents for one example into + * `traces`. Documents arrive in unknown order and span every experiment and + * repetition that ever scored this example, so this filters to the requested + * model + execution and keeps the newest complete document. + */ +const processExampleBatch = ( + scores: EvaluationScoreDocument[], + modelId: string, + suiteId: string, + executionId: string, + traces: MatrixTraceData, + examplePrefixes: ReadonlySet = new Set() +): boolean => { + // Filter to this model + this execution, newest first + const relevant = scores + .filter((score) => score.task?.model?.id === modelId) + .filter((score) => score.metadata?.execution_id === executionId) + .sort((a, b) => { + const ta = Date.parse(a['@timestamp'] ?? ''); + const tb = Date.parse(b['@timestamp'] ?? ''); + return tb - ta; + }); + + if (relevant.length === 0) return false; + + const entry = extractTraceFromScore(relevant[0]); + // Per-evaluator means over every repetition of this example — powers the + // per-prompt score on trace cards, so cards no longer repeat the column + // aggregate for every variant in a category. + entry.scores = exampleScoresByEvaluator(relevant); + // Spread across those same repetitions, so a volatile cell is visually + // distinct from a stable one instead of hiding behind an identical mean. + const spread = exampleSpreadByEvaluator(relevant); + if (Object.keys(spread).length > 0) { + entry.spread = spread; + } + // How many repetitions of this example fed the scores above — the report + // badges it so 1-rep cells are visually distinguishable from 3-rep cells. + entry.repetitions = countRepetitions(relevant); + const exampleId = relevant[0].example?.id; + + const complete = isCompleteScore(relevant[0]); + + // Key by model:exampleId for per-prompt detail. This key — and only this key + // — carries the per-repetition tool_id trails: the prefix and suite keys + // below alias the SAME entry object, so writing trails on `entry` would make + // one example's repetitions count once per alias in the reliability roll-up. + if (exampleId) { + const trails = trailsFromDocs(relevant); + traces[traceKey(modelId, exampleId)] = + trails.length > 0 ? { ...entry, repTrails: trails } : entry; + } + // Category columns slice the dataset by example.id prefix (examplePrefixes), + // synthesizing `prefix:` dataset ids. Emit one trace per matching + // prefix — mirroring scoresByPrefixToDatasets' semantics exactly (equality + // or boundary dash) — so every category card shows ITS OWN example's + // conversation instead of falling through to the suite key (which + // previously made all cards render the same, last-processed example). + if (exampleId) { + for (const prefix of examplePrefixes) { + // Equality writes `prefix:`, a byte-duplicate of the direct + // example key written above (also on the incomplete-fallback path + // below). Per-example columns (`examplePrefixes: ['']`) + // would otherwise double the traces map with unread duplicates. + if (prefix === exampleId) continue; + if (exampleId.startsWith(`${prefix}-`)) { + const key = traceKey(modelId, `prefix:${prefix}`); + // First variant wins: all matching variants are valid examples of the + // category, so keep the lookup deterministic across Set iteration order. + if (traces[key] === undefined) traces[key] = entry; + } + } + } + // Deliberately NOT keyed by `example.dataset.id`: that key is overwritten on + // every example, so it ends up holding whichever example happened to be + // processed last, and no renderer reads it (HTML resolves a column via + // examplePrefixes then `column.suites`). Writing it only added arbitrary + // `model:` entries that duplicate real cells' payload. + // For the suite-level key, only complete runs qualify — and the FIRST + // complete one wins. Overwriting on every example made the suite key + // "last example processed", which every non-matching card then inherited. + const suiteTraceKey = traceKey(modelId, suiteId); + if (complete && traces[suiteTraceKey] === undefined) { + traces[suiteTraceKey] = entry; + } + + // If the newest doc is incomplete, fall back to the newest complete one + if (!complete) { + const firstComplete = relevant.find(isCompleteScore); + if (firstComplete) { + const fallbackEntry = extractTraceFromScore(firstComplete); + fallbackEntry.scores = exampleScoresByEvaluator(relevant.filter(isCompleteScore)); + const fid = firstComplete.example?.id; + if (fid) { + const trails = trailsFromDocs(relevant.filter(isCompleteScore)); + traces[traceKey(modelId, fid)] = + trails.length > 0 ? { ...fallbackEntry, repTrails: trails } : fallbackEntry; + } + if (traces[suiteTraceKey] === undefined) { + traces[suiteTraceKey] = fallbackEntry; + } + } + } + + return complete; +}; + +/** + * Queries evaluation score documents from the golden cluster via the evals + * plugin and extracts trace data (initial question, tool trail, agent answer, + * step trace) for each (model, column) pair. + * + * The per-experiment scores route (`getExperimentScores`) strips unbounded + * fields (`task.output`, `example.input`, `example.metadata`) from responses, + * which are exactly the fields the trace detail needs. This function instead + * uses the per-example scores route (`getExampleScores`), which returns full + * documents, and filters client-side by model and execution. + * + * Example IDs are enumerated from the stripped per-experiment response (which + * still carries `example.id` and `example.dataset.id`), then each example is + * fetched once and reused across all models that ran it. + */ + +/** + * Trace cells are keyed by `task.model.id` -- the id the *provider* reports. + * For aliased rows (OpenRouter, gateways) that is the upstream slug + * (`deepseek/deepseek-v4-pro-0813`) while the matrix row is keyed by the + * connector id (`openrouter-deepseek-v4-pro`), so `traceKey(row.modelId, ...)` + * misses every cell and the row renders "Trace unavailable" despite having a + * full set of traces. Mirror each aliased cell onto the row's own id. EIS rows + * are unaffected: their connector id and task.model.id are the same string. + */ +export const aliasTraceKeys = ( + traces: MatrixTraceData, + modelAliases: ReadonlyMap +): void => { + for (const [rowId, aliases] of modelAliases) { + for (const alias of aliases) { + if (alias === rowId) { + continue; + } + const prefix = `${alias}:`; + for (const [key, entry] of Object.entries(traces)) { + if (!key.startsWith(prefix)) { + continue; + } + const rowKey = traceKey(rowId, key.slice(prefix.length)); + // Never clobber a cell the row already resolved under its own id. + if (traces[rowKey] === undefined) { + traces[rowKey] = entry; + } + } + } + } +}; + +export const queryMatrixTraces = async ( + evalsClient: EvalsClient, + log: SomeDevLog, + aggregated: AggregatedModelScores[], + traceCache?: Record, + toolCallWarnAbove: number = 0, + modelAliases: ReadonlyMap = new Map() +): Promise => { + const traces: MatrixTraceData = {}; + + // 1. Collect (suite, model, execution) tuples with the example IDs they ran, + // from the stripped experiment-scores responses (cheap, no heavy fields). + interface RunRef { + suiteId: string; + modelId: string; + executionId: string; + exampleIds: Set; + } + const runRefs: RunRef[] = []; + + const modelSuites: Array<{ modelId: string; suiteId: string; executionId: string }> = []; + for (const modelScores of aggregated) { + for (const suite of modelScores.suites) { + // A sharded suite row carries every shard's execution id; each shard + // owns a disjoint stride of examples and must be enumerated separately. + // Unsharded rows carry only experimentId. + const executionIds = + suite.executionIds && suite.executionIds.length > 0 + ? suite.executionIds + : [suite.experimentId]; + for (const executionId of executionIds) { + modelSuites.push({ modelId: modelScores.modelId, suiteId: suite.suiteId, executionId }); + } + } + } + + const enumerated = await mapWithConcurrency( + modelSuites, + 6, + async ({ modelId, suiteId, executionId }) => { + log.debug( + `Enumerating examples for experiment ${executionId} (model ${modelId}, suite ${suiteId})` + ); + + const stripped = await evalsClient.getExperimentScores(executionId, { + suiteId, + taskModelId: modelId, + executionId, + }); + + const exampleIds = new Set(); + for (const score of stripped) { + if (score.example?.id) exampleIds.add(score.example.id); + } + + if (exampleIds.size === 0) { + log.warning( + `No example IDs found for suite ${suiteId} (model ${modelId}) — trace will be unavailable` + ); + return null; + } + + return { suiteId, modelId, executionId, exampleIds }; + } + ); + + for (const ref of enumerated) { + if (ref) runRefs.push(ref); + } + + // 2. Fetch full score documents per (run, example) with the execution filter + // applied server-side. The unfiltered route returns EVERY historical run + // of an example (tens of MB), which outgrew the HTTP transport and made + // traces silently disappear; the filtered fetch is ~one execution's docs. + // All (run, example) pairs go through one bounded-concurrency pool. + // + // Backward compatibility: an older evals plugin ignores the unknown query + // params and returns all executions. The first pair is fetched alone so + // detection settles before fan-out; on a legacy server each example is + // then fetched exactly once (deduped while in flight) and shared across + // runs, instead of once per run. + const exampleScores = new Map(); + const cacheKey = (ref: RunRef, exampleId: string) => `${ref.executionId}::${exampleId}`; + let serverSupportsFilter: boolean | undefined; + // In-flight dedup: legacy servers get one request per example no matter how + // many runs need it; filtered servers dedupe only identical (run, example) + // repeats. Keyed differently per mode because the cache identity differs. + const inflight = new Map>(); + + const fetchScores = (ref: RunRef, exampleId: string): Promise => { + const key = serverSupportsFilter === false ? exampleId : cacheKey(ref, exampleId); + const pending = inflight.get(key); + if (pending) return pending; + // Concurrent heavy fetches can trip transient 502/503s on legacy servers; + // one bounded retry keeps a flake from silently costing a cell its trace. + const attempt = async (retriesLeft: number): Promise => { + try { + return await evalsClient.getExampleScores(exampleId, { + executionId: ref.executionId, + modelId: ref.modelId, + }); + } catch (error) { + if (retriesLeft === 0) throw error; + await new Promise((resolve) => setTimeout(resolve, 1000)); + return attempt(retriesLeft - 1); + } + }; + const request = attempt(1).finally(() => inflight.delete(key)); + inflight.set(key, request); + return request; + }; + + const fetchExample = async (ref: RunRef, exampleId: string): Promise => { + // Local trace cache: docs pulled ahead of time (e.g. directly from ES, + // bypassing an old evals plugin whose route ignores execution filters and + // trips the transport cap on heavy examples). A cache hit means no server + // round-trip at all for this cell. + const cached = traceCache?.[cacheKey(ref, exampleId)]; + if (cached) { + exampleScores.set(cacheKey(ref, exampleId), cached); + return; + } + if (serverSupportsFilter === false) { + const shared = exampleScores.get(exampleId); + if (shared) { + exampleScores.set(cacheKey(ref, exampleId), shared); + return; + } + } + const scores = await fetchScores(ref, exampleId); + if (serverSupportsFilter === undefined && scores.length > 0) { + // Only a NON-EMPTY response proves anything about filter support. An + // empty one is ambiguous — it happens when the route rejects the filter + // params, when the execution has no docs, or when the payload blew the + // transport cap. Latching `true` on it (the previous behaviour) declared + // the server healthy off the very response that signals it isn't, so the + // fallback never armed and EVERY later cell returned empty: 442/442 + // traces rendered hollow while scores stayed intact. + serverSupportsFilter = scores.every((s) => s.metadata?.execution_id === ref.executionId); + if (!serverSupportsFilter) { + log.warning( + 'Example-scores route ignores execution filters (older evals plugin) — falling back to shared per-example fetches; traces will be complete but slower' + ); + } + } + if (serverSupportsFilter === false) { + exampleScores.set(exampleId, scores); + } + exampleScores.set(cacheKey(ref, exampleId), scores); + }; + + const pairs: Array<{ ref: RunRef; exampleId: string }> = []; + for (const ref of runRefs) { + for (const exampleId of ref.exampleIds) { + pairs.push({ ref, exampleId }); + } + } + + const fetchPair = async ({ + ref, + exampleId, + }: { + ref: RunRef; + exampleId: string; + }): Promise => { + try { + await fetchExample(ref, exampleId); + } catch (error) { + // A single failed fetch must not abort the whole report: scores are + // already aggregated, only this cell's trace detail is lost. + log.warning( + `Skipping trace details for example ${exampleId} (execution ${ref.executionId}): ${ + error instanceof Error ? error.message : String(error) + }` + ); + exampleScores.set(cacheKey(ref, exampleId), []); + } + }; + + if (pairs.length > 0) { + await fetchPair(pairs[0]); + await mapWithConcurrency(pairs.slice(1), 8, fetchPair); + } + + // Completeness gate. Per-cell failures are deliberately swallowed above so one + // bad example cannot abort a report — but that also means a TOTAL fetch + // failure is silent, and the matrix renders every trace card empty while the + // score columns look perfect. Measured 2026-08-29: 442/442 cells hollow with + // no error surfaced. Fetching nothing at all is never a valid outcome. + const fetchedCells = [...exampleScores.values()].filter((docs) => docs.length > 0).length; + if (pairs.length > 0 && fetchedCells === 0) { + log.warning( + `Trace fetch returned no documents for any of ${pairs.length} (execution, example) pairs — ` + + `every trace card will render empty. The scores above are unaffected. ` + + `Re-run with --trace-cache to read score documents straight from ES.` + ); + } + + // Category prefixes come from the same synthetic `prefix:*` dataset ids the + // score aggregation created (scoresByPrefixToDatasets), so trace bucketing + // always matches score bucketing — no separate config source to keep in sync. + const examplePrefixes = new Set(); + for (const modelScores of aggregated) { + for (const suite of modelScores.suites) { + for (const dataset of suite.datasets) { + if (!dataset.datasetId?.startsWith('prefix:')) continue; + examplePrefixes.add(dataset.datasetId.slice('prefix:'.length)); + } + } + } + + // 3. For each run, pick the newest complete document per example and merge + // into the traces map. + const missingByRun: Array<{ ref: RunRef; missing: string[] }> = []; + for (const ref of runRefs) { + const missing: string[] = []; + for (const exampleId of ref.exampleIds) { + const scores = exampleScores.get(cacheKey(ref, exampleId)) ?? []; + const ok = processExampleBatch( + scores, + ref.modelId, + ref.suiteId, + ref.executionId, + traces, + examplePrefixes + ); + if (!ok) missing.push(exampleId); + } + if (missing.length === ref.exampleIds.size) { + log.warning( + `No complete score documents found for suite ${ref.suiteId} (model ${ref.modelId}, execution ${ref.executionId}) — trace will be unavailable` + ); + } else if (missing.length > 0) { + missingByRun.push({ ref, missing }); + } + } + + // Pass-through verification: every (model, example) that produced a score in + // the aggregation must also land in the traces map. A gap means the trace + // fetch silently lost a scored cell — name each one loudly instead of + // letting the HTML render "Trace unavailable" with no log trail. + if (missingByRun.length > 0) { + const details = missingByRun + .map( + ({ ref, missing }) => + `${ref.modelId} (execution ${ref.executionId}): ${missing.length}/${ + ref.exampleIds.size + } missing [${missing.join(', ')}]` + ) + .join('; '); + log.warning( + `Trace coverage incomplete — scores exist but no trace was resolved for: ${details}` + ); + } + + // Runaway tool-loop report. `Tool Calls` is deliberately excluded from + // quality scoring and thrashing cells do NOT score worse (measured 0.70 vs + // 0.62 trajectory), so this is a COST signal, never a penalty: without it a + // 44-call/3.78M-token cell is indistinguishable from an 8-call one in every + // rendered artifact. + if (toolCallWarnAbove > 0) { + // Corroborate the score against the trail the same document recorded. The + // evaluator counts OTel TOOL spans, which is a DIFFERENT source from + // `task.output.steps`, so a score far above the trail is measurement error + // (stale/duplicated spans), not a longer loop. Golden bears this out: over + // 313 recent scored docs the ratio is median 1.00 / mean 0.98, yet one + // 2026-08-22 cell reported 115 against a 29-call trail and became the + // headline "worst offender" in every report built off this warning. + // Reporting an uncorroborated count as a tool-loop length publishes a + // number no trace supports, so those cells are named separately. + const SUSPECT_RATIO = 2; + const withTrail = Object.entries(traces).map(([key, trace]) => ({ + key, + calls: trace.scores?.['Tool Calls'], + trail: trace.toolTrail?.length ?? 0, + })); + + const above = withTrail + .filter( + (c): c is { key: string; calls: number; trail: number } => typeof c.calls === 'number' + ) + .filter((c) => c.calls > toolCallWarnAbove) + .sort((a, b) => b.calls - a.calls); + + // A trail of 0 cannot corroborate or refute the count (the cell may simply + // have no cached steps), so it stays in the reportable set. + const suspect = above.filter((c) => c.trail > 0 && c.calls > c.trail * SUSPECT_RATIO); + const runaway = above.filter((c) => !suspect.includes(c)); + + if (runaway.length > 0) { + log.warning( + `Possible runaway tool loops (> ${toolCallWarnAbove} calls) in ${runaway.length} cell(s): ` + + runaway + .slice(0, 10) + .map((c) => `${c.key}=${c.calls}`) + .join(', ') + ); + } + + if (suspect.length > 0) { + log.warning( + `'Tool Calls' exceeds the recorded tool trail by more than ${SUSPECT_RATIO}x in ` + + `${suspect.length} cell(s): ` + + suspect + .slice(0, 10) + .map((c) => `${c.key}=${c.calls} (trail ${c.trail})`) + .join(', ') + + ` — the count is not corroborated by the trace; do not cite these as tool-loop lengths` + ); + } + + // A zero `Tool Calls` score on a cell whose trace shows real tool calls + // means the metric is broken for that model, not that it used no tools. + // Unreported, that renders as "this model uses no tools" in the published + // matrix — a fabricated claim, and it hides genuine tool-loop failures. + const brokenMetric = new Map(); + for (const [key, trace] of Object.entries(traces)) { + const trail = trace.toolTrail?.length ?? 0; + const scored = trace.scores?.['Tool Calls']; + if (trail > 0 && (scored === 0 || scored === undefined)) { + const modelId = key.split(':')[0]; + brokenMetric.set(modelId, (brokenMetric.get(modelId) ?? 0) + 1); + } + } + if (brokenMetric.size > 0) { + log.warning( + `'Tool Calls' reads 0 despite a non-empty tool trail for: ` + + [...brokenMetric].map(([modelId, n]) => `${modelId}=${n}`).join(', ') + + ` — the metric is broken for these models; do not publish their tool counts` + ); + } + } + + log.debug(`Matrix traces resolved ${Object.keys(traces).length} trace entries`); + overlayRepeatedCacheTrails(traces, traceCache); + aliasTraceKeys(traces, modelAliases); + return traces; +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/reference_adapters.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/reference_adapters.test.ts new file mode 100644 index 0000000000000..e886392304479 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/reference_adapters.test.ts @@ -0,0 +1,221 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + attackDiscoveryAdapter, + automaticMigrationsAdapter, + personaMatrixAdapter, + buildStructuredReferences, + collectExamples, + selectAdapter, + type DatasetExample, +} from './reference_adapters'; + +describe('reference adapters', () => { + describe('persona-matrix', () => { + const examples: DatasetExample[] = [ + { id: 'alert-analysis-a', output: { reference: 'Explain the alert.' } }, + { id: 'no-ref', output: {} }, + ]; + + it('keys prose references by example id', () => { + const refs = personaMatrixAdapter.build(examples); + expect(refs.get('alert-analysis-a')).toBe('Explain the alert.'); + }); + + it('omits examples without a reference rather than inventing one', () => { + expect(personaMatrixAdapter.build(examples).has('no-ref')).toBe(false); + }); + }); + + describe('attack-discovery', () => { + const examples: DatasetExample[] = [ + { + output: { + criteria: ['Insights mention encoded PowerShell.', 'Insights reference wks-alice-01.'], + attackDiscoveries: [ + { + title: 'Encoded PowerShell on wks-alice-01', + summaryMarkdown: 'Office spawned encoded PowerShell.', + mitreAttackTactics: ['Execution', 'Persistence'], + }, + ], + }, + metadata: { scenarioKey: 'encoded-powershell' }, + }, + { output: { criteria: ['Insights reference web-prod-07.'] } }, + ]; + + // Verified against 351 golden docs (2026-09-08): attack-discovery writes + // example.id = '0' on every document across 9 distinct scenario datasets, so + // a positional join matches one scenario and mis-grades the rest. The + // scenario key is the field golden actually varies. + it('joins by scenario key when the dataset supplies one', () => { + const refs = attackDiscoveryAdapter.build(examples); + expect([...refs.keys()]).toEqual(['encoded-powershell', '1']); + }); + + it('renders every criterion into the reference text', () => { + const ref = attackDiscoveryAdapter.build(examples).get('encoded-powershell')!; + expect(ref).toContain('Insights mention encoded PowerShell.'); + expect(ref).toContain('Insights reference wks-alice-01.'); + }); + + it('includes expected discovery detail and tactics', () => { + const ref = attackDiscoveryAdapter.build(examples).get('encoded-powershell')!; + expect(ref).toContain('Encoded PowerShell on wks-alice-01'); + expect(ref).toContain('Execution, Persistence'); + }); + + // The header is what tells the judge criteria are conjunctive, not a menu. + it('instructs the judge that all criteria must hold', () => { + const ref = attackDiscoveryAdapter.build(examples).get('encoded-powershell')!; + expect(ref).toContain('must satisfy all of the following'); + }); + + it('drops examples with no ground truth at all', () => { + const refs = attackDiscoveryAdapter.build([{ output: {} }]); + expect(refs.size).toBe(0); + }); + }); + + describe('automatic-migrations', () => { + const examples: DatasetExample[] = [ + { + id: 'splunk-simple-001', + output: { + translation_result: 'full', + esql_query: 'FROM logs | WHERE user == "root"', + index_pattern: 'logs-*', + has_lookup_join: false, + is_unsupported: false, + }, + }, + { + id: 'qradar-unsupported-001', + output: { translation_result: 'untranslatable', esql_query: null, is_unsupported: true }, + }, + ]; + + it('joins by real example id', () => { + const refs = automaticMigrationsAdapter.build(examples); + expect([...refs.keys()]).toEqual(['splunk-simple-001', 'qradar-unsupported-001']); + }); + + it('states the expected translation outcome in words', () => { + const ref = automaticMigrationsAdapter.build(examples).get('splunk-simple-001')!; + expect(ref).toContain('fully translated'); + expect(ref).toContain('FROM logs | WHERE user == "root"'); + }); + + // A null esql_query is an expectation ("none"), not a missing field. + it('distinguishes an expected-absent query from an unchecked one', () => { + const ref = automaticMigrationsAdapter.build(examples).get('qradar-unsupported-001')!; + expect(ref).toContain('No ES|QL query is expected'); + expect(ref).toContain('unsupported pattern'); + }); + }); + + describe('module collection and selection', () => { + it('collects examples split across several exports', () => { + const mod = { + splunkRules: [{ id: 's1', output: { translation_result: 'full' } }], + qradarRules: [{ id: 'q1', output: { translation_result: 'partial' } }], + }; + expect(collectExamples(mod)).toHaveLength(2); + }); + + // Regression: the agent-builder AD suite exports `goldenPathExamples`, which + // was absent from the adapter's export list, so rejudging the AD column + // failed with "does not export an examples array" -- the one suite the + // command most needed to reach. + it('collects the agent-builder attack-discovery fixtures', () => { + const mod = { + goldenPathExamples: [ + { output: { criteria: ['a'] }, metadata: { fixture: 'provided-alerts' } }, + { output: { criteria: ['b'] }, metadata: { fixture: 'live-retrieval' } }, + ], + }; + expect(collectExamples(mod)).toHaveLength(2); + expect(selectAdapter(collectExamples(mod))?.name).toBe('attack-discovery'); + }); + + it('unwraps a dataset object exposing an examples array', () => { + const mod = { dataset: { examples: [{ id: 'a', output: { reference: 'x' } }] } }; + expect(collectExamples(mod)).toHaveLength(1); + }); + + it('selects the adapter matching the example shape', () => { + expect(selectAdapter([{ output: { criteria: ['x'] } }])?.name).toBe('attack-discovery'); + expect(selectAdapter([{ id: 'a', output: { translation_result: 'full' } }])?.name).toBe( + 'automatic-migrations' + ); + expect(selectAdapter([{ id: 'a', output: { reference: 'x' } }])?.name).toBe('persona-matrix'); + }); + + it('returns no adapter for an unrecognised suite shape', () => { + expect(selectAdapter([{ id: 'a', output: { somethingElse: 1 } }])).toBeUndefined(); + }); + }); + + describe('attack-discovery join key', () => { + // Golden writes example.id = '0' on EVERY attack-discovery document because + // each scenario is registered as its own single-example dataset. Keying the + // references positionally therefore matches one scenario and grades the + // other eight against scenario 0's ground truth. + const scenarioExamples = [ + { + metadata: { scenarioKey: 'encoded-powershell' }, + output: { criteria: ['powershell criterion'] }, + }, + { metadata: { scenarioKey: 'wmi-lateral' }, output: { criteria: ['wmi criterion'] } }, + { metadata: { scenarioKey: 'linux-curl' }, output: { criteria: ['curl criterion'] } }, + ]; + + it('declares example.metadata.scenarioKey as its join field', () => { + expect(attackDiscoveryAdapter.joinField).toBe('example.metadata.scenarioKey'); + }); + + it('keys references by scenario key, not array index', () => { + const refs = attackDiscoveryAdapter.build(scenarioExamples); + + expect([...refs.keys()].sort()).toEqual(['encoded-powershell', 'linux-curl', 'wmi-lateral']); + // The positional contract would have produced '0', '1', '2'. + expect(refs.has('0')).toBe(false); + }); + + it('binds each scenario to its OWN criteria', () => { + const refs = attackDiscoveryAdapter.build(scenarioExamples); + + expect(refs.get('wmi-lateral')).toContain('wmi criterion'); + expect(refs.get('wmi-lateral')).not.toContain('powershell criterion'); + expect(refs.get('linux-curl')).toContain('curl criterion'); + }); + + it('falls back to id then index when a scenario key is absent', () => { + const refs = attackDiscoveryAdapter.build([ + { id: 'explicit-id', output: { criteria: ['a'] } }, + { output: { criteria: ['b'] } }, + ]); + + expect(refs.has('explicit-id')).toBe(true); + expect(refs.has('1')).toBe(true); + }); + + it('keys structured ground truth identically to the prose references', () => { + // A jury looks up prose and structured truth with ONE key. If the two maps + // derive keys differently, every structured lookup returns undefined and a + // jury that requires structured truth reports the cell as ungradable -- + // which reads as missing model output rather than as a key mismatch. + const prose = attackDiscoveryAdapter.build(scenarioExamples); + const structured = buildStructuredReferences(scenarioExamples); + + expect([...structured.keys()].sort()).toEqual([...prose.keys()].sort()); + expect(structured.get('wmi-lateral')).toBe(scenarioExamples[1].output); + }); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/reference_adapters.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/reference_adapters.ts new file mode 100644 index 0000000000000..f245e1d3f9556 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/reference_adapters.ts @@ -0,0 +1,288 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Per-suite reference adapters for judge replay. + * + * Golden score documents carry an empty `example.output`, so the ground truth a + * judge grades against lives only in the suite's dataset module. Every suite + * expresses that truth differently, and a single "read `output.reference`" + * rule silently yields nothing for most of them: + * + * persona-matrix : `output.reference` is already a prose string. + * attack-discovery : truth is a `criteria[]` array plus expected discoveries. + * Golden writes `example.id = '0'` for EVERY attack-discovery + * document (each scenario is its own single-example dataset), + * so the join is by `example.metadata.scenarioKey`, not by id. + * automatic-migrations : truth is structured translation fields + * (translation_result / esql_query / is_unsupported), + * most of them nullable, joined by a real example id. + * + * An adapter turns whatever a suite stores into the single thing the judge + * needs: a reference STRING keyed by the golden field named in `joinField`. + * Rendering structured truth as text is deliberate -- the correctness judge + * compares prose, so the adapter must state the expectation explicitly rather + * than hand the judge a JSON blob it has to reverse-engineer. + */ + +export interface DatasetExample { + id?: string; + input?: unknown; + output?: Record; + metadata?: Record; + reference?: unknown; +} + +export interface ReferenceAdapter { + /** Adapter id, reported in logs so a replay names the contract it used. */ + name: string; + /** Export names to look for in the dataset module, in priority order. */ + exportNames: string[]; + /** + * Golden field whose value the reference map is keyed by. Suites disagree: + * most record a usable `example.id`, but attack-discovery writes '0' for every + * document and varies `example.metadata.scenarioKey` instead. Defaults to + * `example.id` when omitted. + */ + joinField?: string; + /** True when this adapter recognises the module's examples. */ + matches: (examples: DatasetExample[]) => boolean; + /** joinField value -> reference string. */ + build: (examples: DatasetExample[]) => Map; +} + +/** Golden field used to join references when an adapter does not name one. */ +export const DEFAULT_JOIN_FIELD = 'example.id'; + +const isNonEmptyString = (v: unknown): v is string => typeof v === 'string' && v.length > 0; + +/** + * persona-matrix: the reference is already prose. + */ +export const personaMatrixAdapter: ReferenceAdapter = { + name: 'persona-matrix', + exportNames: ['personaMatrixDataset', 'PERSONA_MATRIX_EXAMPLES', 'default', 'dataset'], + matches: (examples) => + examples.some((e) => isNonEmptyString(e?.output?.reference ?? e?.reference)), + build: (examples) => { + const refs = new Map(); + for (const example of examples) { + const reference = example?.output?.reference ?? example?.reference; + if (isNonEmptyString(example?.id) && isNonEmptyString(reference)) { + refs.set(example.id, reference); + } + } + return refs; + }, +}; + +/** + * attack-discovery: criteria[] + expected discoveries, joined by scenario key. + * + * Golden does NOT store a positional `example.id` for this suite: every AD score + * document carries `example.id = '0'` regardless of scenario, because each AD + * scenario is registered as its own single-example dataset. Joining on the array + * index therefore matches only index 0 and -- worse -- applies scenario 0's + * ground truth to every other scenario, manufacturing confident wrong verdicts. + * + * `example.metadata.scenarioKey` is the field golden actually varies per + * scenario, so it is the join key. Verified 2026-09-08 against 351 golden docs: + * example.id was '0' for 351/351 across 9 distinct dataset names. + */ +export const attackDiscoveryAdapter: ReferenceAdapter = { + name: 'attack-discovery', + exportNames: [ + // The agent-builder AD suite exports its fixtures under this name, and the + // spec slices it per fixture. Omitting it made `--dataset src/dataset.ts` + // fail with "does not export an examples array" for the very suite whose + // column the rejudge exists to refresh. + 'goldenPathExamples', + 'cleanProfileProvidedAlertsExamples', + 'cleanProfileProvidedAlertsDataset', + 'default', + 'dataset', + ], + joinField: 'example.metadata.scenarioKey', + matches: (examples) => examples.some((e) => Array.isArray(e?.output?.criteria)), + build: (examples) => { + const refs = new Map(); + examples.forEach((example, index) => { + const output = example?.output ?? {}; + const criteria = (output.criteria as string[] | undefined) ?? []; + const discoveries = (output.attackDiscoveries as Array>) ?? []; + + const parts: string[] = []; + if (criteria.length > 0) { + parts.push( + `The attack discovery insights must satisfy all of the following:\n` + + criteria.map((c) => `- ${c}`).join('\n') + ); + } + for (const discovery of discoveries) { + const title = discovery.title; + const summary = discovery.summaryMarkdown; + const details = discovery.detailsMarkdown; + const tactics = discovery.mitreAttackTactics; + const lines: string[] = []; + if (isNonEmptyString(title)) lines.push(`Expected discovery: ${title}`); + if (isNonEmptyString(summary)) lines.push(`Summary: ${summary}`); + if (isNonEmptyString(details)) lines.push(`Details: ${details}`); + if (Array.isArray(tactics) && tactics.length > 0) { + lines.push(`MITRE ATT&CK tactics: ${tactics.join(', ')}`); + } + if (lines.length > 0) parts.push(lines.join('\n')); + } + + // An example with neither criteria nor discoveries has no ground truth; + // emitting a placeholder would grade answers against nothing. + if (parts.length === 0) return; + + // Same key derivation as the structured map, so both lookups agree. + const key = exampleKey(example, index); + refs.set(key, parts.join('\n\n')); + }); + return refs; + }, +}; + +const TRANSLATION_LABELS: Record = { + full: 'fully translated', + partial: 'partially translated', + untranslatable: 'not translatable', +}; + +/** + * automatic-migrations: structured translation expectations, joined by id. + * + * Fields are nullable by design (an untranslatable rule has no ESQL), so the + * adapter states absence explicitly instead of dropping the field -- "no ESQL + * query is expected" and "the ESQL query was not checked" grade differently. + */ +export const automaticMigrationsAdapter: ReferenceAdapter = { + name: 'automatic-migrations', + exportNames: ['splunkRules', 'qradarRules', 'default', 'dataset'], + matches: (examples) => examples.some((e) => e?.output && 'translation_result' in e.output), + build: (examples) => { + const refs = new Map(); + for (const example of examples) { + if (!isNonEmptyString(example?.id)) continue; + const o = example.output ?? {}; + const lines: string[] = []; + + const result = o.translation_result; + if (isNonEmptyString(result)) { + lines.push(`Expected translation result: ${TRANSLATION_LABELS[result] ?? result}.`); + } + if (o.is_unsupported === true) { + lines.push('The source rule uses an unsupported pattern and must be reported as such.'); + } + if (isNonEmptyString(o.esql_query)) { + lines.push(`Expected ES|QL query:\n${o.esql_query}`); + } else if (o.esql_query === null) { + lines.push('No ES|QL query is expected for this rule.'); + } + if (isNonEmptyString(o.index_pattern)) { + lines.push(`Expected index pattern: ${o.index_pattern}`); + } + if (isNonEmptyString(o.integration_id)) { + lines.push(`Expected integration: ${o.integration_id}`); + } + if (isNonEmptyString(o.prebuilt_rule_id)) { + lines.push(`Expected prebuilt rule match: ${o.prebuilt_rule_id}`); + } + if (o.has_lookup_join === true) { + lines.push('The translation must preserve a LOOKUP JOIN.'); + } + + if (lines.length === 0) continue; + refs.set(example.id, lines.join('\n')); + } + return refs; + }, +}; + +export const REFERENCE_ADAPTERS: ReferenceAdapter[] = [ + personaMatrixAdapter, + attackDiscoveryAdapter, + automaticMigrationsAdapter, +]; + +/** + * Collect the example arrays a module exports, across every adapter's known + * export names. Migrations splits its dataset over two exports + * (splunkRules + qradarRules), so a single-export lookup would silently + * replay half the suite. + */ +export function collectExamples(mod: Record): DatasetExample[] { + const seen = new Set(); + const examples: DatasetExample[] = []; + const names = [...new Set(REFERENCE_ADAPTERS.flatMap((a) => a.exportNames))]; + + for (const name of names) { + const value = mod[name]; + const arr = Array.isArray(value) + ? value + : Array.isArray((value as { examples?: unknown })?.examples) + ? ((value as { examples: DatasetExample[] }).examples as DatasetExample[]) + : undefined; + if (!arr || seen.has(arr)) continue; + seen.add(arr); + examples.push(...(arr as DatasetExample[])); + } + return examples; +} + +/** Pick the adapter whose contract the examples actually satisfy. */ +export function selectAdapter(examples: DatasetExample[]): ReferenceAdapter | undefined { + return REFERENCE_ADAPTERS.find((adapter) => adapter.matches(examples)); +} + +/** + * The key both reference maps use for an example. + * + * attack-discovery registers each scenario as its own single-example dataset, so + * golden records `example.id = '0'` on every document and only + * `example.metadata.scenarioKey` distinguishes them. Prefer that key, fall back + * to an explicit id, then to the position, so datasets that do carry ids or rely + * on order keep working. + */ +function exampleKey(example: DatasetExample, index: number): string { + const scenarioKey = (example as { metadata?: { scenarioKey?: unknown } })?.metadata?.scenarioKey; + if (isNonEmptyString(scenarioKey)) { + return scenarioKey; + } + return example.id ?? String(index); +} + +/** + * Build the STRUCTURED ground truth, keyed the same way as the prose reference. + * + * `build()` renders truth as prose because the correctness judge compares text. + * Suite-native evaluators do not: Attack Discovery's Criteria evaluator wants + * the `criteria[]` array and its Rubric evaluator wants the `attackDiscoveries` + * objects. Re-deriving those from the rendered prose would mean parsing back out + * of a lossy format, so the raw `output` is exposed under the same key the + * prose lookup uses. + * + * Both maps MUST derive their key identically. Keying this by index while + * `build()` keys by scenario key returns `undefined` for every lookup, and a + * jury that requires structured truth then reports the cell as ungradable -- + * which reads as missing model output rather than as a key mismatch. + */ +export function buildStructuredReferences( + examples: DatasetExample[] +): Map> { + const refs = new Map>(); + examples.forEach((example, index) => { + const output = example?.output; + if (!output || typeof output !== 'object') { + return; + } + refs.set(exampleKey(example, index), output as Record); + }); + return refs; +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix.test.ts new file mode 100644 index 0000000000000..58cf8a0319778 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix.test.ts @@ -0,0 +1,387 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { renderMatrix } from './render_matrix'; +import { parseMatrixConfig } from './load_matrix_config'; +import type { Matrix } from './build_matrix'; +import { buildMatrix, OVERALL_COLUMN_ID } from './build_matrix'; +import type { AggregatedModelScores } from './query_matrix_scores'; + +const config = parseMatrixConfig({ + title: 'Test Matrix', + columns: [ + { id: 'triage', label: 'Alert Triage', suites: ['a'] }, + { id: 'detect', label: 'Detection Engineering', suites: ['b'] }, + ], + models: [{ id: 'm', label: 'M' }], +}); + +const matrix: Matrix = { + columns: [ + { id: 'triage', label: 'Alert Triage' }, + { id: 'detect', label: 'Detection Engineering' }, + ], + composites: [], + displayColumns: [ + { id: 'triage', label: 'Alert Triage', kind: 'base' }, + { id: 'detect', label: 'Detection Engineering', kind: 'base' }, + { id: OVERALL_COLUMN_ID, label: 'Overall', kind: 'overall' }, + ], + overallLabel: 'Overall', + evaluatorSaturation: [], + proprietary: [ + { + modelId: 'claude', + modelLabel: 'Claude Sonnet 4', + openSource: false, + cells: { + triage: { kind: 'score', value: 9.2 }, + detect: { kind: 'not-recommended' }, + }, + overall: { kind: 'score', value: 4.6 }, + coverage: { covered: 2, total: 2 }, + }, + ], + openSource: [ + { + modelId: 'oss', + modelLabel: 'GPT OSS 120B', + openSource: true, + cells: { + triage: { kind: 'score', value: 7.6 }, + detect: { kind: 'missing' }, + }, + overall: { kind: 'score', value: 3.8 }, + coverage: { covered: 1, total: 2 }, + }, + ], +}; + +describe('renderMatrix', () => { + it('distinguishes a judge-rejected cell from a blank one in CSV', () => { + // A cell whose scores were all self-judged must not read as "no data". + // Both render empty otherwise, and a reader cannot tell that re-running the + // model is pointless until the judge is changed (2026-08-29 incident). + const withExcluded = { + ...matrix, + openSource: [ + { + ...matrix.openSource[0], + cells: { + triage: { kind: 'score' as const, value: 7.6 }, + detect: { kind: 'excluded' as const, reason: 'self-judged' as const, docs: 294 }, + }, + }, + ], + }; + + const { openSourceCsv } = renderMatrix(withExcluded, config); + + expect(openSourceCsv).toContain('excluded:self-judged'); + expect(openSourceCsv).not.toContain('GPT OSS 120B,7.6,,'); + }); + + it('renders CSV with a header row and one row per model', () => { + const { proprietaryCsv, openSourceCsv } = renderMatrix(matrix, config); + + expect(proprietaryCsv.split('\n')[0]).toBe('Model,Alert Triage,Detection Engineering,Overall'); + expect(proprietaryCsv).toContain('Claude Sonnet 4,9.2,Not recommended,4.6'); + // Missing cells render as empty fields. + expect(openSourceCsv).toContain('GPT OSS 120B,7.6,,3.8'); + }); + + it('renders markdown with proprietary and open-source sections', () => { + const { markdown } = renderMatrix(matrix, config); + + expect(markdown).toContain('# Test Matrix'); + expect(markdown).toContain('## Proprietary models'); + expect(markdown).toContain('## Open-source models'); + expect(markdown).toContain('| Claude Sonnet 4 | 9.2 | Not recommended | 4.6 |'); + }); + + it('produces valid JSON with the matrix structure', () => { + const { json } = renderMatrix(matrix, config); + const parsed = JSON.parse(json); + + expect(parsed.title).toBe('Test Matrix'); + expect(parsed.proprietary).toHaveLength(1); + expect(parsed.openSource[0].modelLabel).toBe('GPT OSS 120B'); + }); + + it('renders composite columns in displayColumns order (no trailing legacy overall)', () => { + const compositeMatrix: Matrix = { + columns: [ + { id: 'a', label: 'Alert Triage', group: 'Agent Builder' }, + { id: 'b', label: 'Investigation', group: 'Agent Builder' }, + ], + composites: [ + { id: 'ab', label: 'Agent Builder Score' }, + { id: 'overall_score', label: 'Overall Score' }, + ], + displayColumns: [ + { id: 'a', label: 'Alert Triage', group: 'Agent Builder', kind: 'base' }, + { id: 'b', label: 'Investigation', group: 'Agent Builder', kind: 'base' }, + { id: 'ab', label: 'Agent Builder Score', kind: 'composite' }, + { id: 'overall_score', label: 'Overall Score', kind: 'composite' }, + ], + overallLabel: 'Overall', + evaluatorSaturation: [], + proprietary: [ + { + modelId: 'm', + modelLabel: 'Claude', + openSource: false, + cells: { + a: { kind: 'score', value: 8.6 }, + b: { kind: 'score', value: 7.4 }, + ab: { kind: 'score', value: 8 }, + overall_score: { kind: 'score', value: 8 }, + }, + overall: { kind: 'score', value: 8 }, + coverage: { covered: 2, total: 2 }, + }, + ], + openSource: [], + }; + + const { proprietaryCsv } = renderMatrix(compositeMatrix, config); + // No trailing "Overall" column; composites appear in declared layout order. + expect(proprietaryCsv.split('\n')[0]).toBe( + 'Model,Alert Triage,Investigation,Agent Builder Score,Overall Score' + ); + expect(proprietaryCsv).toContain('Claude,8.6,7.4,8,8'); + }); + + it('escapes CSV fields that contain commas or quotes', () => { + const cfgWithComma = parseMatrixConfig({ + title: 'X', + columns: [{ id: 'c', label: 'Col, with comma', suites: ['a'] }], + models: [{ id: 'm', label: 'M' }], + }); + const m: Matrix = { + columns: [{ id: 'c', label: 'Col, with comma' }], + composites: [], + displayColumns: [ + { id: 'c', label: 'Col, with comma', kind: 'base' }, + { id: OVERALL_COLUMN_ID, label: 'Overall', kind: 'overall' }, + ], + overallLabel: 'Overall', + evaluatorSaturation: [], + proprietary: [ + { + modelId: 'm', + modelLabel: 'M', + openSource: false, + cells: { c: { kind: 'score', value: 1 } }, + overall: { kind: 'score', value: 1 }, + coverage: { covered: 1, total: 1 }, + }, + ], + openSource: [], + }; + + const { proprietaryCsv } = renderMatrix(m, cfgWithComma); + expect(proprietaryCsv.split('\n')[0]).toBe('Model,"Col, with comma",Overall'); + }); +}); + +describe('renderMatrix token axis', () => { + const tokenConfig = parseMatrixConfig({ + title: 'Token Matrix', + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'], weight: 1 }], + models: [{ id: 'm1', label: 'M1' }], + tokenCost: {}, + }); + + const withTokens: AggregatedModelScores[] = [ + { + modelId: 'm1', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'r1', + datasets: [ + { + datasetId: 'd1', + datasetName: 'D1', + evaluators: [ + { evaluatorName: 'correctness', mean: 0.9, count: 2 }, + { + evaluatorName: 'Input Tokens', + mean: 120_000, + count: 2, + min: 90_000, + max: 150_000, + }, + { evaluatorName: 'Output Tokens', mean: 3_000, count: 2, min: 2_000, max: 4_000 }, + ], + }, + ], + }, + ], + }, + ]; + + it('serializes the saturation verdict so the Overall exclusion is auditable', () => { + // Without this the artifact shows a score that silently changed because an + // evaluator was dropped, and the reader has no way to see which one. + const saturated = { + ...matrix, + evaluatorSaturation: [ + { + evaluatorName: 'MinExpectedSteps', + mean: 0.97, + stdev: 0.03, + range: 0.09, + observations: 20, + distinctValues: 5, + saturated: true, + }, + { + evaluatorName: 'Factuality', + mean: 0.35, + stdev: 0.12, + range: 0.58, + observations: 20, + distinctValues: 18, + saturated: false, + }, + ], + }; + + const parsed = JSON.parse(renderMatrix(saturated, config).json); + + expect(parsed.evaluatorSaturation).toHaveLength(2); + const flagged = parsed.evaluatorSaturation.filter( + (entry: { saturated: boolean }) => entry.saturated + ); + expect(flagged.map((entry: { evaluatorName: string }) => entry.evaluatorName)).toEqual([ + 'MinExpectedSteps', + ]); + expect(flagged[0].range).toBeCloseTo(0.09); + }); + + it('serializes tokenCost into matrix.json', () => { + const { json } = renderMatrix(buildMatrix(withTokens, tokenConfig), tokenConfig); + const parsed = JSON.parse(json); + + expect(parsed.tokenCost.models).toHaveLength(1); + expect(parsed.tokenCost.models[0].modelId).toBe('m1'); + expect(parsed.tokenCost.models[0].cells[0]).toEqual({ + columnId: 'triage', + inputTokens: { mean: 120_000, min: 90_000, max: 150_000, count: 2 }, + outputTokens: { mean: 3_000, min: 2_000, max: 4_000, count: 2 }, + totalMean: 123_000, + }); + }); + + it('omits the tokenCost key entirely when not configured', () => { + const plain = parseMatrixConfig({ + title: 'Plain', + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'], weight: 1 }], + models: [{ id: 'm1', label: 'M1' }], + }); + const { json } = renderMatrix(buildMatrix(withTokens, plain), plain); + expect(JSON.parse(json)).not.toHaveProperty('tokenCost'); + }); + + it('embeds traces into matrix.json when provided', () => { + const traces = { + 'm1:triage': { + question: 'What happened?', + answer: 'An alert fired.', + toolTrail: ['security.alerts'], + }, + }; + const { json } = renderMatrix(matrix, config, {}, traces as never); + const parsed = JSON.parse(json); + expect(parsed.traces['m1:triage'].question).toBe('What happened?'); + expect(parsed.traces['m1:triage'].toolTrail).toEqual(['security.alerts']); + }); + + it('omits the traces key when trace data was not queried', () => { + const { json } = renderMatrix(matrix, config); + expect(JSON.parse(json)).not.toHaveProperty('traces'); + }); +}); + +describe('renderMatrix provenance', () => { + const provConfig = parseMatrixConfig({ + title: 'Prov Matrix', + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'], weight: 1 }], + models: [{ id: 'm1', label: 'M1' }], + }); + + const scores: AggregatedModelScores[] = [ + { + modelId: 'm1', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'r1', + datasets: [ + { + datasetId: 'd1', + datasetName: 'D1', + evaluators: [{ evaluatorName: 'correctness', mean: 0.9, count: 2 }], + }, + ], + }, + ], + }, + ]; + + const render = (provenance?: Parameters[2]) => + renderMatrix(buildMatrix(scores, provConfig), provConfig, provenance); + + it('stamps the filters that produced the numbers into markdown and json', () => { + const { markdown, json } = render({ + branch: 'main', + lookbackDays: 14, + suiteIds: ['suite-a'], + commitSha: 'abc123', + buildUrl: 'https://buildkite.com/b/1', + }); + + expect(markdown).toContain('branch `main`'); + expect(markdown).toContain('14-day lookback'); + expect(markdown).toContain('commit `abc123`'); + expect(markdown).toContain('[build](https://buildkite.com/b/1)'); + + const parsed = JSON.parse(json); + expect(parsed.provenance).toEqual({ + branch: 'main', + lookbackDays: 14, + suiteIds: ['suite-a'], + commitSha: 'abc123', + buildUrl: 'https://buildkite.com/b/1', + }); + expect(parsed.generatedAt).toEqual(expect.any(String)); + }); + + it('omits unknown fields rather than stamping placeholders', () => { + const { markdown, json } = render({ branch: 'main', lookbackDays: 7 }); + + expect(markdown).toContain('branch `main`'); + expect(markdown).not.toContain('commit'); + expect(markdown).not.toContain('undefined'); + expect(JSON.parse(json).provenance).toEqual({ branch: 'main', lookbackDays: 7 }); + }); + + it('still renders a dated line when no provenance is supplied', () => { + const { markdown, json } = render(); + + expect(markdown).toContain('Generated '); + expect(markdown).not.toContain('undefined'); + expect(JSON.parse(json).provenance).toEqual({}); + }); + + it('uses one timestamp for both markdown and json', () => { + const { markdown, json } = render(); + expect(markdown).toContain(`Generated ${JSON.parse(json).generatedAt}`); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix.ts new file mode 100644 index 0000000000000..d515113ec7648 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix.ts @@ -0,0 +1,217 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MatrixConfig } from './load_matrix_config'; +import type { Matrix, MatrixCell, MatrixDisplayColumn, MatrixRow } from './build_matrix'; +import type { MatrixTraceData } from './trace_types'; + +/** + * Where the numbers came from. Without this, a published matrix is an + * undated table of scores with no way to tell which eval run, branch, or + * lookback window produced it — so a stale artifact is indistinguishable + * from a fresh one. + */ +export interface MatrixProvenance { + /** Branch filter applied to the query (undefined = any branch). */ + branch?: string; + /** Lookback window in days used to select experiments. */ + lookbackDays?: number; + /** + * Epoch-ms cutoff the matrix was rendered at, when not "now". A time-boxed + * matrix that does not say so reads as current data. + */ + asOf?: number; + /** Suite ids the scores were drawn from. */ + suiteIds?: string[]; + /** Commit the generator ran against, when known. */ + commitSha?: string; + /** + * True when the generator's working tree had uncommitted changes. An artifact + * built from a dirty tree cannot be reproduced from `commitSha` alone, and a + * regen at a later commit may not match it. + */ + dirtyWorkingTree?: boolean; + /** Trace-cache file the run loaded, or `none` when it fetched from the server. */ + traceCache?: string; + /** CI build URL that produced the artifact, when known. */ + buildUrl?: string; + /** sha256 of the dataset/tool seed files the runs were scored against. */ + fixtureFingerprint?: string; + /** Scoring-semantics notes a reader needs before comparing matrices. */ + methodologyNotes?: string[]; +} + +export interface RenderedMatrix { + /** CSV for the proprietary-models table (first row = header). */ + proprietaryCsv: string; + /** CSV for the open-source-models table (first row = header). */ + openSourceCsv: string; + /** Combined human-readable markdown document. */ + markdown: string; + /** Structured JSON artifact (machine-readable). */ + json: string; +} + +const cellToString = (cell: MatrixCell, notRecommendedLabel: string): string => { + switch (cell.kind) { + case 'score': + return String(cell.value); + case 'not-recommended': + return notRecommendedLabel; + // Rendered distinctly from 'missing': the model ran, but every grade was + // rejected. Publishing this as a blank invites a re-sweep that cannot fill it. + case 'excluded': + return `excluded:${cell.reason}`; + // Never a bare number: an average over too few columns is not comparable + // to a full row, and publishing one as a score invites a false ranking. + case 'insufficient-coverage': + return `insufficient-coverage:${cell.covered}/${cell.required}`; + // Same principle applied to the evaluator axis: an evaluator that errored + // on every example means the instrument failed, not that the model earned this. + case 'insufficient-evaluators': + return `insufficient-evaluators:${cell.evaluators.join('+')}`; + case 'missing': + default: + return ''; + } +}; + +/** Escapes a value for inclusion in a CSV field per RFC 4180. */ +const csvEscape = (value: string): string => { + if (/[",\n]/.test(value)) { + return `"${value.replace(/"/g, '""')}"`; + } + return value; +}; + +const cellForColumn = (row: MatrixRow, column: MatrixDisplayColumn): MatrixCell => + column.kind === 'overall' ? row.overall : row.cells[column.id] ?? { kind: 'missing' }; + +const buildHeader = (displayColumns: MatrixDisplayColumn[]): string[] => [ + 'Model', + ...displayColumns.map((column) => column.label), +]; + +const rowToValues = ( + displayColumns: MatrixDisplayColumn[], + row: MatrixRow, + notRecommendedLabel: string +): string[] => [ + row.modelLabel, + ...displayColumns.map((column) => cellToString(cellForColumn(row, column), notRecommendedLabel)), +]; + +const renderCsv = ( + displayColumns: MatrixDisplayColumn[], + rows: MatrixRow[], + notRecommendedLabel: string +): string => { + const lines = [ + buildHeader(displayColumns), + ...rows.map((row) => rowToValues(displayColumns, row, notRecommendedLabel)), + ]; + return lines.map((cells) => cells.map(csvEscape).join(',')).join('\n') + '\n'; +}; + +const renderMarkdownTable = ( + displayColumns: MatrixDisplayColumn[], + rows: MatrixRow[], + notRecommendedLabel: string +): string => { + const coverageOf = (row: MatrixRow): string => `${row.coverage.covered}/${row.coverage.total}`; + const header = [...buildHeader(displayColumns), 'Coverage']; + const separator = header.map(() => ':---'); + const body = rows.map((row) => [ + ...rowToValues(displayColumns, row, notRecommendedLabel), + coverageOf(row), + ]); + + const toRow = (cells: string[]): string => `| ${cells.join(' | ')} |`; + + return [toRow(header), toRow(separator), ...body.map(toRow)].join('\n'); +}; + +export const renderMatrix = ( + matrix: Matrix, + config: MatrixConfig, + provenance: MatrixProvenance = {}, + traces?: MatrixTraceData +): RenderedMatrix => { + const { notRecommendedLabel } = config; + const displayColumns = matrix.displayColumns; + const generatedAt = new Date().toISOString(); + + const proprietaryCsv = renderCsv(displayColumns, matrix.proprietary, notRecommendedLabel); + const openSourceCsv = renderCsv(displayColumns, matrix.openSource, notRecommendedLabel); + + // Rendered as a plain line rather than a comment so it survives into the + // published docs — a provenance footer nobody can see defeats the purpose. + const provenanceLine = [ + `Generated ${generatedAt}`, + provenance.branch ? `branch \`${provenance.branch}\`` : undefined, + provenance.lookbackDays !== undefined ? `${provenance.lookbackDays}-day lookback` : undefined, + provenance.asOf !== undefined + ? `as of ${new Date(provenance.asOf).toISOString().slice(0, 10)} (later runs excluded)` + : undefined, + provenance.commitSha ? `commit \`${provenance.commitSha}\`` : undefined, + provenance.buildUrl ? `[build](${provenance.buildUrl})` : undefined, + ] + .filter(Boolean) + .join(' · '); + + const markdown = [ + `# ${config.title}`, + '', + provenanceLine, + '', + 'Higher scores indicate better performance. A score of 10 on a task means the model met or exceeded all task-specific benchmarks. ' + + `Models with a score of "${notRecommendedLabel}" failed testing.`, + '', + '## Proprietary models', + '', + matrix.proprietary.length > 0 + ? renderMarkdownTable(displayColumns, matrix.proprietary, notRecommendedLabel) + : '_No proprietary models with results._', + '', + '## Open-source models', + '', + matrix.openSource.length > 0 + ? renderMarkdownTable(displayColumns, matrix.openSource, notRecommendedLabel) + : '_No open-source models with results._', + '', + ].join('\n'); + + const json = JSON.stringify( + { + title: config.title, + generatedAt, + provenance, + columns: matrix.columns, + composites: matrix.composites ?? [], + displayColumns, + overallLabel: matrix.overallLabel, + // Which evaluators were judged non-discriminating, and the numbers + // behind the verdict. Overall is computed WITHOUT the saturated ones + // when the config opts in, so the exclusion has to be auditable from + // the artifact alone -- otherwise a reader cannot tell why a score + // moved between two runs of the same data. + evaluatorSaturation: matrix.evaluatorSaturation ?? [], + proprietary: matrix.proprietary, + openSource: matrix.openSource, + ...(matrix.tokenCost ? { tokenCost: matrix.tokenCost } : {}), + // Traces are embedded at generation time so the artifact is + // reproducible: a reader can audit any cell's full conversation without + // re-querying the evals cluster. Present only when the caller queried + // trace data (the --html path). + ...(traces ? { traces } : {}), + }, + null, + 2 + ); + + return { proprietaryCsv, openSourceCsv, markdown, json }; +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.test.ts new file mode 100644 index 0000000000000..f9bd6e974e681 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.test.ts @@ -0,0 +1,610 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { renderMatrixHtml } from './render_matrix_html'; +import type { Matrix } from './build_matrix'; +import type { MatrixConfig } from './load_matrix_config'; +import type { MatrixTraceData } from './trace_types'; + +const mockConfig: MatrixConfig = { + title: 'Test Matrix', + branch: 'main', + lookbackDays: 30, + defaultScale: 10, + decimals: 2, + notRecommendedBelow: 0, + toolCallWarnAbove: 0, + minCoverage: 0, + notRecommendedLabel: 'Not recommended', + notRecommendedCountsAsZeroInOverall: true, + excludeEvaluators: [], + overall: { label: 'Overall', mode: 'weighted', excludeSaturatedEvaluators: false }, + showOverall: true, + columns: [ + { + id: 'alert', + label: 'Alert Analysis', + group: 'Agent Builder', + suites: ['suite-1'], + weight: 1, + }, + { + id: 'threat', + label: 'Threat Hunting', + group: 'Agent Builder', + suites: ['suite-2'], + weight: 1, + }, + ], + composites: [], + models: [{ id: 'test-model', label: 'Test Model', openSource: false }], +}; + +const mockMatrix: Matrix = { + columns: [ + { id: 'alert', label: 'Alert Analysis', group: 'Agent Builder' }, + { id: 'threat', label: 'Threat Hunting', group: 'Agent Builder' }, + ], + composites: [], + displayColumns: [ + { id: 'alert', label: 'Alert Analysis', kind: 'base' }, + { id: 'threat', label: 'Threat Hunting', kind: 'base' }, + { id: '__overall__', label: 'Overall', kind: 'overall' }, + ], + overallLabel: 'Overall', + evaluatorSaturation: [], + proprietary: [ + { + modelId: 'test-model', + modelLabel: 'Test Model', + openSource: false, + cells: { + alert: { kind: 'score', value: 8.5 }, + threat: { kind: 'score', value: 7.4 }, + }, + overall: { kind: 'score', value: 7.95 }, + coverage: { covered: 2, total: 2 }, + }, + ], + openSource: [], +}; + +describe('renderMatrixHtml', () => { + it('renders a self-contained HTML document', () => { + const html = renderMatrixHtml(mockMatrix, mockConfig); + expect(html).toContain(''); + expect(html).toContain(' + + +
    +

    ${esc(config.title)}

    +

    ${provenanceLine}

    +${methodologyBlock} +${summaryTable} +${tokenCostTable} +

    Each cell shows the model's score (0–10). Expand a prompt below to read the agent's full answer, tool trail, and reasoning trace.

    +${modelCards} +
    + +`; +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_reliability_html.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_reliability_html.test.ts new file mode 100644 index 0000000000000..ebc30936fbfad --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_reliability_html.test.ts @@ -0,0 +1,194 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Matrix } from './build_matrix'; +import { renderReliabilityHtml } from './render_reliability_html'; + +const matrix: Matrix = { + columns: [], + composites: [], + displayColumns: [], + overallLabel: 'Overall', + evaluatorSaturation: [], + proprietary: [ + { + modelId: 'measured', + modelLabel: 'Measured', + openSource: false, + cells: {}, + overall: { kind: 'score', value: 8 }, + capability: { kind: 'score', value: 9 }, + judgedQuality: { kind: 'score', value: 7 }, + coverage: { covered: 8, total: 8 }, + tier: 1, + }, + { + modelId: 'single', + modelLabel: 'Single run', + openSource: false, + cells: {}, + overall: { kind: 'score', value: 7 }, + capability: { kind: 'score', value: 6 }, + judgedQuality: { kind: 'score', value: 8 }, + coverage: { covered: 8, total: 8 }, + tier: 1, + }, + ], + openSource: [], +}; + +describe('renderReliabilityHtml', () => { + it('renders a separate reliability artifact without treating unmeasured as zero', () => { + const html = renderReliabilityHtml(matrix, { + 'measured:example-a': { repTrails: [['search'], ['search']] }, + 'measured:example-b': { repTrails: [['search'], ['load_skill']] }, + 'single:example-a': { repTrails: [['search']] }, + }); + expect(html).toContain('Capability · Reliability · Judged quality'); + expect(html).toContain('50%'); + expect(html).toContain('9.00'); + expect(html).toContain('7.00'); + expect(html).toContain('Unmeasured'); + expect(html).not.toContain('Unmeasured0'); + }); + + it('excludes probe examples from the identical-path rate', () => { + const html = renderReliabilityHtml(matrix, { + 'measured:alert-analysis-a': { repTrails: [['search'], ['load_skill']] }, + 'measured:workflow-authoring-a': { repTrails: [['search'], ['search']] }, + }); + expect(html).toContain('100%'); + expect(html).not.toContain('50%'); + }); + + it('reports the pair count and interval instead of a bare rate', () => { + const html = renderReliabilityHtml(matrix, { + 'measured:example-a': { repTrails: [['search'], ['search']] }, + 'measured:example-b': { repTrails: [['search'], ['load_skill']] }, + }); + // 2 pairs is a near-useless sample; the page must say so rather than + // presenting 50% as a finding. + expect(html).toContain('2 pairs'); + expect(html).toMatch(/\(\d+%–\d+%\)/); + }); + + it('marks measured rows as tied when their intervals overlap', () => { + const html = renderReliabilityHtml(matrix, { + 'measured:example-a': { repTrails: [['search'], ['search']] }, + 'measured:example-b': { repTrails: [['search'], ['load_skill']] }, + 'single:example-a': { repTrails: [['search'], ['search']] }, + 'single:example-b': { repTrails: [['search'], ['load_skill']] }, + }); + expect(html).toContain('statistically tied'); + }); + + it('prefers the declared path contract over the legacy prefix guess', () => { + const html = renderReliabilityHtml(matrix, { + // Hunt-prefixed but declared rankable: it must count, and the page must + // not claim any cell was legacy-classified. + 'measured:alert-analysis-a': { + repTrails: [['search'], ['load_skill']], + pathContract: 'rankable', + }, + }); + expect(html).toContain('0%'); + expect(html).not.toContain('legacy example-prefix list'); + }); + + it('discloses legacy classification for corpora predating pathContract', () => { + const html = renderReliabilityHtml(matrix, { + 'measured:workflow-authoring-a': { repTrails: [['search'], ['search']] }, + }); + expect(html).toContain('legacy example-prefix list'); + }); + + it('reports answer similarity separately from path agreement', () => { + const answer = 'the host was compromised via a scheduled task '.repeat(3); + const html = renderReliabilityHtml(matrix, { + 'measured:example-a': { + repTrails: [['search'], ['search']], + repAnswers: [answer, 'a completely different conclusion entirely here now'], + }, + }); + // Identical paths, divergent answers: the board must not imply one from + // the other. + expect(html).toContain('100%'); + expect(html).toContain('answer similarity'); + }); + + it('says what an unmeasured row needs instead of leaving it blank', () => { + const html = renderReliabilityHtml(matrix, { + 'measured:example-a': { repTrails: [['search'], ['search']] }, + }); + expect(html).toContain('needs k≥5'); + }); + + it('discloses a dirty working tree in provenance', () => { + const html = renderReliabilityHtml( + matrix, + { 'measured:example-a': { repTrails: [['search'], ['search']] } }, + { commitSha: 'abc123', dirtyWorkingTree: true } + ); + expect(html).toContain('uncommitted changes present'); + }); + + describe('judge agreement column', () => { + const verdict = ( + modelId: string, + judgeId: string, + example: string, + evaluator: string, + score: number + ) => ({ modelId, judgeId, example, repetition: 0, evaluator, score }); + + it('renders Unmeasured when no judge verdicts are supplied', () => { + const html = renderReliabilityHtml(matrix, {}, {}); + expect(html).toContain('Judge agreement'); + expect(html).toContain('no verdicts'); + }); + + it('renders Single judge, never a percentage, for one-judge models', () => { + const html = renderReliabilityHtml(matrix, {}, {}, [ + verdict('measured', 'gemini', 'ex-1', 'Relevance', 1), + verdict('measured', 'gemini', 'ex-2', 'Relevance', 1), + ]); + expect(html).toContain('Single judge'); + expect(html).toContain('no second opinion'); + // The regression that matters: a lone judge must not read as consensus. + expect(html).not.toContain('100.0%'); + }); + + it('renders agreement with its interval and pair count', () => { + const html = renderReliabilityHtml(matrix, {}, {}, [ + verdict('measured', 'gemini', 'ex-1', 'Relevance', 1), + verdict('measured', 'sonnet', 'ex-1', 'Relevance', 1), + verdict('measured', 'gemini', 'ex-2', 'Relevance', 1), + verdict('measured', 'sonnet', 'ex-2', 'Relevance', 0), + ]); + expect(html).toContain('50.0%'); + expect(html).toContain('2 paired verdicts'); + expect(html).toContain('95% CI'); + }); + + it('names the worst evaluator with its flip interval', () => { + const html = renderReliabilityHtml(matrix, {}, {}, [ + verdict('measured', 'gemini', 'ex-1', 'Relevance', 1), + verdict('measured', 'sonnet', 'ex-1', 'Relevance', 0), + ]); + expect(html).toContain('worst:'); + expect(html).toContain('Relevance'); + }); + + it('states that agreement is not correctness', () => { + const html = renderReliabilityHtml(matrix, {}, {}, [ + verdict('measured', 'gemini', 'ex-1', 'Relevance', 1), + verdict('measured', 'sonnet', 'ex-1', 'Relevance', 1), + ]); + expect(html).toContain('both judges can agree and both be wrong'); + }); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_reliability_html.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_reliability_html.ts new file mode 100644 index 0000000000000..c5f00389d4f2b --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_reliability_html.ts @@ -0,0 +1,275 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Matrix, MatrixCell, MatrixRow } from './build_matrix'; +import type { MatrixProvenance } from './render_matrix'; +import type { MatrixTraceData } from './trace_types'; +import { + judgeAgreementForModel, + type JudgeAgreementRow, + type JudgeVerdict, +} from './judge_agreement'; +import { + intervalsOverlap, + resolveProbe, + rowAgreement, + type ReliabilityRow, + type TrajectoryCell, +} from './trajectory_agreement'; + +const esc = (value: string): string => + value.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); + +const cellValue = (cell: MatrixCell): string => { + switch (cell.kind) { + case 'score': + return cell.value.toFixed(2); + case 'not-recommended': + return 'Not recommended'; + case 'excluded': + return `Excluded: ${cell.reason}`; + case 'insufficient-coverage': + return `Insufficient: ${cell.covered}/${cell.required}`; + case 'insufficient-evaluators': + return `Unmeasured (${cell.evaluators.join(', ')} errored)`; + case 'missing': + return 'Unmeasured'; + } +}; + +/** + * Converts only direct example trace keys into reliability cells. Prefix and + * suite aliases do not carry repTrails, preventing double-counting. + */ +export const reliabilityCellsFromTraces = (traces: MatrixTraceData = {}): TrajectoryCell[] => + Object.entries(traces).flatMap(([key, trace]) => { + if (!trace.repTrails) { + return []; + } + const split = key.indexOf(':'); + if (split < 1) { + return []; + } + const example = key.slice(split + 1); + // Prefer the contract the dataset declared; fall back to the legacy prefix + // list only for corpora recorded before that field existed, and carry the + // source through so the page can disclose how many cells were guessed. + const { probe, source } = resolveProbe(example, trace.pathContract); + return [ + { + model: key.slice(0, split), + example, + trails: trace.repTrails, + answers: trace.repAnswers, + probe, + probeSource: source, + }, + ]; + }); + +const pct = (value: number): string => `${(value * 100).toFixed(0)}%`; + +/** + * Renders the agreement rate with its Wilson interval and pair count. The + * point estimate alone invites ordering two models that the data cannot + * separate, so the interval and sample size are not optional decoration. + */ +const reliabilityHtml = (agreement: ReliabilityRow, tied: boolean): string => { + if (agreement.status === 'unmeasured') { + return ( + 'Unmeasured' + + `needs k≥5 repeats · ${agreement.cells} rankable cells at 1 rep` + ); + } + const interval = agreement.interval; + const range = interval ? ` (${pct(interval.low)}–${pct(interval.high)})` : ''; + const answer = + agreement.answerSimilarity === undefined + ? '' + : ` · ${pct(agreement.answerSimilarity)} answer similarity`; + const hotspot = agreement.divergenceHotspot + ? `first diverges at ${esc(agreement.divergenceHotspot.tool)} in ${ + agreement.divergenceHotspot.cells + }/${agreement.measuredCells} cells` + : ''; + const tiedNote = tied ? 'tied — intervals overlap' : ''; + const orderOnly = + agreement.toolSetRate !== undefined && agreement.identicalRate !== undefined + ? ` · ${pct(agreement.toolSetRate)} same tool set` + : ''; + return ( + `${pct(agreement.identicalRate ?? 0)}${range}` + + `${agreement.pairs ?? 0} pairs · ${agreement.measuredCells} repeated cells · ${pct( + agreement.sequenceSimilarity ?? 0 + )} path similarity${orderOnly}${answer}` + + hotspot + + tiedNote + ); +}; + +const judgeHtml = (row: JudgeAgreementRow): string => { + if (row.status === 'unmeasured') { + return 'Unmeasuredno verdicts'; + } + if (row.status === 'single-judge') { + // Never render this as agreement. One judge scoring a model is an absence + // of corroboration, and a percentage here would read as its opposite. + return `Single judge${esc( + row.judges.join(', ') + )} · no second opinion`; + } + + const agreementPct = `${((row.verdictAgreement ?? 0) * 100).toFixed(1)}%`; + const ci = row.interval + ? ` 95% CI ${(row.interval.low * 100).toFixed(0)}–${(row.interval.high * 100).toFixed( + 0 + )}` + : ''; + const bias = + row.bias !== undefined && row.biasJudges + ? `bias ${row.bias >= 0 ? '+' : ''}${row.bias.toFixed(3)} (${esc( + row.biasJudges[0] + )} − ${esc(row.biasJudges[1])})` + : ''; + const worst = row.worstEvaluators.length + ? `worst: ${row.worstEvaluators + .slice(0, 2) + .map( + (e) => + `${esc(e.evaluator)} ${((e.flips / e.pairs) * 100).toFixed(0)}% [${( + e.interval.low * 100 + ).toFixed(0)}–${(e.interval.high * 100).toFixed(0)}]` + ) + .join(', ')}` + : ''; + // Disclose one-sided coverage. Without this a row computed over 395 of 531 + // cells reads identically to one where both judges scored everything. + const coverage = + row.unpaired > 0 + ? `${row.unpaired} cell${ + row.unpaired === 1 ? '' : 's' + } scored by one judge only, excluded` + : ''; + return `${agreementPct}${ci}${row.pairs} paired verdicts${coverage}${bias}${worst}`; +}; + +const rowHtml = ( + row: MatrixRow, + agreement: ReliabilityRow, + tied: boolean, + judge: JudgeAgreementRow +): string => { + const tier = row.tier === undefined ? '—' : `Tier ${row.tier}`; + return ` + ${esc(row.modelLabel)}${esc(row.modelId)} + ${esc( + cellValue(row.capability ?? { kind: 'missing' }) + )}deterministic contract evaluators + ${reliabilityHtml(agreement, tied)} + ${judgeHtml(judge)} + ${esc(cellValue(row.judgedQuality ?? { kind: 'missing' }))}${esc( + tier + )} · ${row.coverage.covered}/${row.coverage.total} columns + `; +}; + +/** + * Separate artifact. It deliberately does not replace matrix.html: reliability + * is currently measured for only a subset of models, and absence must remain + * visibly "Unmeasured" rather than silently ranking as 0. + */ +export const renderReliabilityHtml = ( + matrix: Matrix, + traces: MatrixTraceData = {}, + provenance: MatrixProvenance = {}, + judgeVerdicts: readonly JudgeVerdict[] = [] +): string => { + const cells = reliabilityCellsFromTraces(traces); + const rows = [...matrix.proprietary, ...matrix.openSource]; + const agreements = new Map(rows.map((row) => [row.modelId, rowAgreement(cells, row.modelId)])); + const measured = [...agreements.values()].filter((a) => a.status === 'measured'); + const judgeRows = new Map( + rows.map((row) => [row.modelId, judgeAgreementForModel(judgeVerdicts, row.modelId)]) + ); + const judgeMeasured = [...judgeRows.values()].filter((j) => j.status === 'measured'); + + // A row is "tied" when its interval overlaps any other measured row's. With + // ~27 pairs per model the intervals span roughly 30pp, so two point + // estimates 4pp apart are the same measurement, not a ranking. + const tiedModels = new Set(); + for (const a of measured) { + for (const b of measured) { + if (a.modelId !== b.modelId && a.interval && b.interval) { + if (intervalsOverlap(a.interval, b.interval)) { + tiedModels.add(a.modelId); + } + } + } + } + + const legacyCells = cells.filter((c) => c.probeSource === 'legacy-prefix').length; + const probeCells = cells.filter((c) => c.probe).length; + const generated = new Date().toISOString(); + const prov = [ + `Generated ${generated}`, + provenance.branch ? `branch ${provenance.branch}` : undefined, + provenance.commitSha ? `commit ${provenance.commitSha}` : undefined, + // An artifact built from a dirty tree does not correspond to any commit; + // saying so is the difference between provenance and decoration. + provenance.dirtyWorkingTree ? 'uncommitted changes present' : undefined, + ] + .filter(Boolean) + .join(' · '); + + return ` + +Matrix reliability view

    Capability · Reliability · Judged quality

    ${esc( + prov + )}

    +
    ${measured.length}/${ + rows.length + } models have repeated cells. Reliability is blank by design for the rest. A single run is neither stable nor unstable.${ + measured.length > 1 && tiedModels.size === measured.length + ? ' Every measured row is statistically tied — the intervals overlap, so this column does not order them.' + : '' + }${ + legacyCells > 0 + ? ` ${legacyCells} of ${cells.length} cells were classified by the legacy example-prefix list because their score documents predate the pathContract field.` + : '' + }${probeCells > 0 ? ` ${probeCells} probe cells are excluded from agreement.` : ''}${ + judgeMeasured.length > 0 + ? ` ${judgeMeasured.length}/${rows.length} models were scored by two judge families; agreement below is verdict-level concordance, not correctness — both judges can agree and both be wrong.` + : '' + }
    +${rows + .map((row) => + rowHtml( + row, + agreements.get(row.modelId) ?? { + modelId: row.modelId, + status: 'unmeasured', + cells: 0, + measuredCells: 0, + }, + tiedModels.has(row.modelId), + judgeRows.get(row.modelId) ?? { + modelId: row.modelId, + status: 'unmeasured', + judges: [], + pairs: 0, + unpaired: 0, + worstEvaluators: [], + } + ) + ) + .join('\n')}
    ModelCapabilityReliabilityJudge agreementJudged quality
    +

    Capability is the mean of deterministic contract evaluators (ExpectedToolCalled, FinalAnswerPresent, MinExpectedSteps, and SkillInvoked). Reliability is pairwise exact agreement of ordered tool_id sequences; provider-generated tool_call_id is never compared. Rates carry a Wilson 95% interval and their pair count, and rows whose intervals overlap are marked tied rather than ordered. Answer similarity is reported separately because path stability barely predicts it (r=0.14 on the pilot corpus). Judge agreement is pass/fail concordance between two judge families on the identical example, repetition, and evaluator; cost and latency instruments are excluded because they are not verdicts. A model scored by one judge reads Single judge, never 100% — an absent second opinion is not consensus. Agreement measures reproducibility, not correctness. Judged quality is the mean of the remaining maximize-direction evaluators. The four axes are never averaged into a rank.

    +
    `; +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/replay_plan.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/replay_plan.test.ts new file mode 100644 index 0000000000000..2dced90afb76f --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/replay_plan.test.ts @@ -0,0 +1,382 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EvaluationScoreDocument } from '@kbn/evals-common'; +import { + agentSteps, + lastAgentMessage, + planReplay, + replayExecutionId, + summarizePlan, +} from './replay_plan'; + +/** + * Shaped after a real golden document (run sweep-1788679175-gj): the judge + * inputs live on example.input.question, example.output.expected and + * task.output.messages[].message. + */ +/** Dataset reference lookup, standing in for the suite's prompts file. */ +const refs = (exampleId: string) => + (( + { + 'persona-001': 'host-42', + 'persona-002': 'A2', + 'persona-003': 'ref-3', + 'persona-broken': 'e', + p2: 'e', + } as Record + )[exampleId]); + +function doc(overrides: Record = {}): EvaluationScoreDocument { + const base: any = { + '@timestamp': '2026-09-06T09:00:00.000Z', + experiment_id: 'exp-1', + example: { + id: 'persona-001', + input: { question: 'Which host triggered the alert?' }, + output: {}, + }, + task: { + model: { id: 'eis-openai-gpt-5-4-nano' }, + output: { + messages: [ + { message: { content: 'Looking into it' } }, + { message: { content: 'host-42 triggered the alert.' } }, + ], + }, + }, + evaluator: { name: 'Factuality', kind: 'llm' }, + metadata: { + execution_id: 'sweep-1788679175-gj-s1of3::security-persona-matrix::nano', + total_repetitions: 1, + }, + }; + return { ...base, ...overrides } as EvaluationScoreDocument; +} + +describe('lastAgentMessage', () => { + it('reads the final message content', () => { + expect(lastAgentMessage(doc().task!.output)).toBe('host-42 triggered the alert.'); + }); + + it('accepts a plain string message', () => { + expect(lastAgentMessage({ messages: [{ message: 'plain' }] })).toBe('plain'); + }); + + it('returns undefined rather than inventing an empty answer', () => { + // Grading "" would produce a confident MAJOR_INACCURACIES verdict for a + // cell whose trajectory simply failed to load. + expect(lastAgentMessage({ messages: [] })).toBeUndefined(); + expect(lastAgentMessage(undefined)).toBeUndefined(); + expect(lastAgentMessage({})).toBeUndefined(); + }); +}); + +describe('agentSteps', () => { + it('returns the tool-call history the groundedness judge grades against', () => { + const steps = [{ type: 'tool_call', name: 'search' }, { type: 'relevant_skills' }]; + expect(agentSteps({ messages: [], steps })).toEqual(steps); + }); + + it('returns an empty array when a trajectory recorded no steps', () => { + expect(agentSteps({ messages: [] })).toEqual([]); + expect(agentSteps(undefined)).toEqual([]); + expect(agentSteps({ steps: 'not-an-array' })).toEqual([]); + }); +}); + +describe('planReplay', () => { + it('builds one cell carrying the judge inputs', () => { + const plan = planReplay([doc()], refs); + expect(plan.skipped).toEqual([]); + expect(plan.cells).toHaveLength(1); + expect(plan.cells[0]).toEqual( + expect.objectContaining({ + exampleId: 'persona-001', + question: 'Which host triggered the alert?', + expected: 'host-42', + agentResponse: 'host-42 triggered the alert.', + modelId: 'eis-openai-gpt-5-4-nano', + }) + ); + }); + + it('carries the tool-call history so grounding is graded against evidence', () => { + // The groundedness judge reads output.steps as `tool_call_history` and + // verifies each claim against it. Dropping steps on replay left that + // history empty, so a row the judge had scored 0.86 grounded came back + // 0.25 with 17/21 cells labelled MAJOR_HALLUCINATIONS -- a harness + // artifact that reads exactly like a model regression. + const steps = [{ type: 'tool_call', name: 'search', result: 'host-42' }]; + const plan = planReplay( + [ + doc({ + task: { + model: { id: 'eis-openai-gpt-5-4-nano' }, + output: { + messages: [{ message: { content: 'host-42 triggered the alert.' } }], + steps, + }, + }, + }), + ], + refs + ); + expect(plan.cells[0].steps).toEqual(steps); + }); + + it('dedupes the evaluator documents that share one trajectory', () => { + // A cell emits ~7 evaluator docs. Without dedupe a re-judge would call the + // judge 7x per cell and write conflicting analyses for the same trajectory. + const docs = ['Factuality', 'Relevance', 'Groundedness', 'Latency'].map((name) => + doc({ evaluator: { name, kind: 'llm' } }) + ); + const plan = planReplay(docs, refs); + expect(plan.cells).toHaveLength(1); + }); + + it('grades one deterministic trajectory per cell', () => { + // Evaluator documents for a cell all carry the same trajectory, but a + // partial re-export can leave a differing copy. Without an explicit + // first-wins rule the graded answer depends on document arrival order, + // so the same golden data could yield two different verdicts. + const first = doc(); + const later = doc({ + task: { + model: { id: 'eis-openai-gpt-5-4-nano' }, + output: { messages: [{ message: { content: 'a different answer' } }] }, + }, + }); + expect(planReplay([first, later], refs).cells[0].agentResponse).toBe( + 'host-42 triggered the alert.' + ); + expect(planReplay([first, later], refs).cells).toHaveLength(1); + }); + + it('skips a cell whose example is absent from the dataset', () => { + // Golden documents carry no reference answer (example.output is empty on + // every stored document). If a replay silently graded against undefined, + // every answer would score inaccurate and the matrix would look like a + // model collapse rather than a missing join. + const unknown = doc({ + example: { id: 'not-in-dataset', input: { question: 'q' }, output: {} }, + }); + const plan = planReplay([unknown], refs); + expect(plan.cells).toHaveLength(0); + expect(plan.skipped[0].reason).toContain('dataset reference'); + }); + + it('takes the reference from the dataset, not the document', () => { + const misleading = doc({ + example: { + id: 'persona-001', + input: { question: 'Which host triggered the alert?' }, + output: { expected: 'WRONG-from-doc' }, + }, + }); + expect(planReplay([misleading], refs).cells[0].expected).toBe('host-42'); + }); + + it('keeps distinct examples and distinct executions apart', () => { + const other = doc({ + example: { + id: 'persona-002', + input: { question: 'Q2' }, + output: { expected: 'A2' }, + }, + }); + const otherRun = doc({ + metadata: { execution_id: 'sweep-other::suite::model', total_repetitions: 1 }, + }); + expect(planReplay([doc(), other, otherRun], refs).cells).toHaveLength(3); + }); + + it('skips ungradeable cells instead of grading empty strings', () => { + const noAnswer = doc({ task: { model: { id: 'm' }, output: { messages: [] } } }); + const plan = planReplay([noAnswer], refs); + expect(plan.cells).toHaveLength(0); + expect(plan.skipped).toHaveLength(1); + expect(plan.skipped[0].reason).toContain('agent response'); + }); + + it('reports every missing judge input by name', () => { + const bare = doc({ + // Id absent from the dataset, so all three judge inputs are missing. + example: { id: 'not-in-dataset', input: {}, output: {} }, + task: { model: { id: 'm' }, output: { messages: [] } }, + }); + const plan = planReplay([bare], refs); + expect(plan.skipped[0].reason).toBe('missing question, dataset reference, agent response'); + }); + + it('reports an unreplayable cell once, not once per evaluator', () => { + const broken = () => doc({ task: { model: { id: 'm' }, output: { messages: [] } } }); + expect(planReplay([broken(), broken(), broken()], refs).skipped).toHaveLength(1); + }); + + it('does not both replay and skip the same cell', () => { + // Evaluator documents for one cell arrive in no guaranteed order. If the + // incomplete one is seen first, the cell must still count as replayable + // exactly once -- never as a cell AND a skip, which would double-count it. + const incomplete = doc({ task: { model: { id: 'm' }, output: { messages: [] } } }); + const forward = planReplay([incomplete, doc()], refs); + const reverse = planReplay([doc(), incomplete], refs); + expect(forward.cells).toHaveLength(1); + expect(forward.skipped).toHaveLength(0); + expect(forward).toEqual(reverse); + }); + + it('returns an empty plan for no input', () => { + expect(planReplay([], refs)).toEqual({ cells: [], skipped: [] }); + }); +}); + +describe('replayExecutionId', () => { + it('namespaces re-judged scores under their judge', () => { + // Two judges must never merge into one execution: the matrix aggregates by + // execution, so a mixed execution silently averages disagreeing verdicts. + expect(replayExecutionId('sweep-1::suite::model', 'haiku')).toBe( + 'sweep-1::suite::model::rejudge-haiku' + ); + expect(replayExecutionId('sweep-1::suite::model', 'gemini')).not.toBe( + replayExecutionId('sweep-1::suite::model', 'haiku') + ); + }); + + it('refuses an empty judge tag', () => { + expect(() => replayExecutionId('sweep-1', '')).toThrow(/judge tag/); + }); +}); + +describe('summarizePlan', () => { + it('counts cells, models and executions', () => { + const plan = planReplay( + [doc(), doc({ example: { id: 'p2', input: { question: 'q' }, output: { expected: 'e' } } })], + refs + ); + expect(summarizePlan(plan)).toBe('2 cell(s) across 1 model(s), 1 execution(s)'); + }); + + it('surfaces unreplayable cells in the summary', () => { + const broken = doc({ + example: { id: 'persona-broken', input: { question: 'q' }, output: { expected: 'e' } }, + task: { model: { id: 'm' }, output: { messages: [] } }, + }); + expect(summarizePlan(planReplay([doc(), broken], refs))).toContain('1 unreplayable'); + }); +}); + +describe('planReplay with a suite jury', () => { + /** Minimal AD-shaped jury: grades `output.insights`, ignores the transcript. */ + const adJury = { + name: 'attack-discovery', + suiteIds: ['attack-discovery-agent-builder'], + evaluatorNames: ['Criteria', 'Rubric'], + toArgs: (cell: any) => + Array.isArray(cell.taskOutput?.insights) && cell.taskOutput.insights.length > 0 + ? { input: {}, output: {}, expected: {}, metadata: {} } + : null, + }; + + /** An AD cell: structured insights, but no final agent message. */ + const adDoc = doc({ + example: { id: '0', input: { question: 'Run attack discovery' }, output: {} }, + metadata: { execution_id: 'exec-ad', suite_id: 'attack-discovery-agent-builder' }, + task: { + model: { id: 'm' }, + output: { messages: [], insights: [{ title: 'Suspicious curl' }] }, + }, + }); + + it('replays a cell the persona contract would have skipped', () => { + // Without a jury this cell is "missing agent response" -- the defect that + // made 433 of 800 AD cells look unreplayable. + expect(planReplay([adDoc], refs).cells).toHaveLength(0); + + const plan = planReplay([adDoc], refs, { jury: adJury as any }); + expect(plan.cells).toHaveLength(1); + expect(plan.skipped).toHaveLength(0); + }); + + it('carries the raw task output and suite id onto the cell', () => { + const [cell] = planReplay([adDoc], refs, { jury: adJury as any }).cells; + expect((cell.taskOutput as any).insights).toHaveLength(1); + expect(cell.suiteId).toBe('attack-discovery-agent-builder'); + }); + + it('skips a cell the jury cannot grade, naming the jury', () => { + const empty = doc({ + example: { id: '0', input: { question: 'q' }, output: {} }, + metadata: { execution_id: 'exec-ad', suite_id: 'attack-discovery-agent-builder' }, + task: { model: { id: 'm' }, output: { messages: [], insights: [] } }, + }); + const plan = planReplay([empty], refs, { jury: adJury as any }); + expect(plan.cells).toHaveLength(0); + expect(plan.skipped[0].reason).toContain('attack-discovery'); + }); + + it('passes structured references through to the cell', () => { + const [cell] = planReplay([adDoc], refs, { + jury: adJury as any, + structuredReferenceFor: () => ({ criteria: ['c1'] }), + }).cells; + expect(cell.expectedStructured).toEqual({ criteria: ['c1'] }); + }); + + describe('join field', () => { + // Every attack-discovery document carries example.id = '0'. Keying cells on + // the id therefore collapses nine distinct scenarios into a single cell and + // grades eight of them against the wrong scenario's ground truth. + const scenarioDocs = ['wmi-lateral', 'linux-curl', 'encoded-powershell'].map((key) => + doc({ + example: { + id: '0', + metadata: { scenarioKey: key }, + input: { question: `q-${key}` }, + output: {}, + }, + metadata: { execution_id: 'exec-ad', suite_id: 'attack-discovery-agent-builder' }, + task: { model: { id: 'm' }, output: { insights: [{ title: key }] } }, + }) + ); + + it('collapses distinct scenarios into one cell when keyed on example.id', () => { + const plan = planReplay(scenarioDocs, () => 'ref', { jury: adJury as any }); + + // The defect this guards against: three scenarios, one surviving cell. + expect(plan.cells).toHaveLength(1); + }); + + it('keeps one cell per scenario when keyed on the scenario key', () => { + const plan = planReplay(scenarioDocs, () => 'ref', { + jury: adJury as any, + joinField: 'example.metadata.scenarioKey', + }); + + expect(plan.cells).toHaveLength(3); + expect(plan.cells.map((c) => c.exampleId).sort()).toEqual([ + 'encoded-powershell', + 'linux-curl', + 'wmi-lateral', + ]); + }); + + it('looks up each scenario reference by its own key', () => { + const seen: string[] = []; + planReplay( + scenarioDocs, + (id) => { + seen.push(id); + return `ref-${id}`; + }, + { jury: adJury as any, joinField: 'example.metadata.scenarioKey' } + ); + + expect(seen.sort()).toEqual(['encoded-powershell', 'linux-curl', 'wmi-lateral']); + }); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/replay_plan.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/replay_plan.ts new file mode 100644 index 0000000000000..267b681da2ac7 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/replay_plan.ts @@ -0,0 +1,277 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EvaluationScoreDocument } from '@kbn/evals-common'; +import type { JuryAdapter } from './jury_adapters'; +import { DEFAULT_JOIN_FIELD } from './reference_adapters'; + +/** + * Reads a dotted golden field (e.g. `example.metadata.scenarioKey`) off a score + * document. Suites disagree on which field identifies an example, so the join + * key is a path rather than a fixed property. + */ +function joinValue(doc: unknown, path: string): string { + let node: unknown = doc; + for (const segment of path.split('.')) { + if (node === null || typeof node !== 'object') return ''; + node = (node as Record)[segment]; + } + return typeof node === 'string' || typeof node === 'number' ? String(node) : ''; +} + +/** + * Replay planning for judge-only re-scoring. + * + * A re-judge changes which model grades an already-recorded trajectory. It does + * NOT need the agent, a Kibana, an Elasticsearch, or seeded data -- every input + * the judges read (the user question, the agent's messages, the tool calls it + * made, the ground truth) is already durable in the golden score documents. + * + * That durability is load-bearing and easy to lose: judges differ in WHICH + * parts of the trajectory they read. The correctness judges read the final + * message; the groundedness judge reads `task.output.steps` to check claims + * against retrieved evidence. A replay that forwards only the final message + * silently reduces the grounding judge to grading unsupported assertions, which + * looks like a model or judge regression but is a harness defect. When adding a + * judge here, forward the whole trajectory, not just the part today's judges + * happen to use. + * + * Re-running the full sweep to change a judge cost ~68min of wall clock and 25 + * VMs on 2026-09-06, of which ~83% was VM provisioning and stack boot. Replay + * reduces that to the judge's own latency. + */ + +/** + * Ground truth for one example, supplied by the caller from the suite's + * dataset. + * + * Golden score documents do NOT carry the reference answer: `example.output` + * is empty on every stored document (verified across the whole index). The + * suite mirrors its dataset's `output.reference` into `expected.expected` at + * run time, so a replay must re-join the dataset by example id. Grading + * against a missing reference would score every answer as inaccurate. + */ +export type ReferenceLookup = (exampleId: string) => string | undefined; + +/** One unit of replayable work: a single (execution, example) trajectory. */ +export interface ReplayCell { + executionId: string; + exampleId: string; + modelId: string; + /** The question put to the agent. */ + question: string; + /** Ground-truth answer the correctness judge compares against. */ + expected: string; + /** The agent's final message, i.e. what gets graded. */ + agentResponse: string; + /** + * The agent's intermediate steps (tool calls and their results). + * + * The groundedness judge verifies each claim against the evidence the agent + * actually retrieved: it reads `output.steps` and passes it to the prompt as + * `tool_call_history`. Replaying with only the final message leaves that + * history empty, so every specific claim becomes unverifiable and the judge + * returns MAJOR_HALLUCINATIONS for answers it had graded as grounded moments + * earlier -- a property of the harness, not of the model or the judge. + * + * Steps are `_source`-only on the score documents (not indexed), so they can + * be read back but never filtered on. + */ + steps: unknown[]; + /** + * The raw `task.output` object, retained verbatim. + * + * Suites other than persona-matrix grade a different slice of the output than + * the message transcript: Attack Discovery's Criteria and Rubric evaluators + * read `output.insights`. Flattening every cell to question/response/steps + * discards that payload, so a non-persona jury would receive an empty + * submission and score it N/A. + */ + taskOutput?: unknown; + /** + * Structured ground truth, when the suite has one. + * + * `expected` is the prose rendering the correctness judge compares against; + * suites whose evaluators consume the original objects (AD's `criteria[]` and + * `attackDiscoveries`) need them unflattened. + */ + expectedStructured?: unknown; + /** Golden `metadata.suite_id`, used to resolve the jury for this cell. */ + suiteId?: string; + /** Source document's timestamp, retained for provenance. */ + recordedAt: string; +} + +export interface PlanIssue { + executionId: string; + exampleId: string; + reason: string; +} + +export interface ReplayPlan { + cells: ReplayCell[]; + /** Cells that cannot be replayed, with why. Never silently dropped. */ + skipped: PlanIssue[]; +} + +/** Extract the graded text from a task output's message list. */ +export function lastAgentMessage(output: unknown): string | undefined { + const messages = (output as { messages?: Array<{ message?: unknown }> })?.messages; + if (!Array.isArray(messages) || messages.length === 0) { + return undefined; + } + const last = messages[messages.length - 1]?.message; + if (typeof last === 'string') { + return last; + } + if (last && typeof last === 'object') { + const content = (last as { content?: unknown }).content; + if (typeof content === 'string') { + return content; + } + } + return undefined; +} + +/** Extract the agent's intermediate steps (tool calls + results) from a task output. */ +export function agentSteps(output: unknown): unknown[] { + const steps = (output as { steps?: unknown })?.steps; + return Array.isArray(steps) ? steps : []; +} + +/** + * Build a replay plan from golden score documents. + * + * Deduplicates by (executionId, exampleId): a cell carries one trajectory but + * many evaluator documents, and re-judging the same trajectory once per + * evaluator would multiply judge cost by the evaluator count and write + * conflicting analyses for a single cell. + * + * A document missing the question, the ground truth, or the agent's response + * cannot be graded; it is reported in `skipped` rather than being replayed + * against empty strings, which would silently manufacture MAJOR_INACCURACIES + * verdicts for cells whose data merely failed to load. + */ +export function planReplay( + docs: EvaluationScoreDocument[], + referenceFor: ReferenceLookup, + options: { + /** + * Jury for the suite being replayed. When supplied it defines + * replayability; when omitted the persona-matrix contract + * (question + prose reference + final message) applies. + */ + jury?: JuryAdapter; + /** Structured ground truth lookup, for juries that grade objects. */ + structuredReferenceFor?: (exampleId: string) => unknown; + /** + * Golden field the reference keys correspond to, matching the adapter's + * `joinField`. attack-discovery documents all carry `example.id = '0'`, so + * keying cells on the id there merges nine scenarios into one and grades + * eight of them against the wrong ground truth. + */ + joinField?: string; + } = {} +): ReplayPlan { + const { jury, structuredReferenceFor, joinField = DEFAULT_JOIN_FIELD } = options; + const cells = new Map(); + const skipped: PlanIssue[] = []; + const seenSkips = new Set(); + + for (const doc of docs) { + const executionId = doc.metadata?.execution_id ?? ''; + const exampleId = joinValue(doc, joinField); + const key = `${executionId}::${exampleId}`; + if (cells.has(key)) { + continue; + } + + const question = (doc.example?.input as { question?: unknown })?.question; + // Reference comes from the dataset, not the document (see ReferenceLookup). + const expected = exampleId ? referenceFor(exampleId) : undefined; + const agentResponse = lastAgentMessage(doc.task?.output); + + const candidate: ReplayCell = { + executionId, + exampleId, + modelId: doc.task?.model?.id ?? '', + question: typeof question === 'string' ? question : '', + expected: typeof expected === 'string' ? expected : '', + agentResponse: agentResponse ?? '', + steps: agentSteps(doc.task?.output), + taskOutput: doc.task?.output, + expectedStructured: exampleId ? structuredReferenceFor?.(exampleId) : undefined, + suiteId: doc.metadata?.suite_id, + recordedAt: doc['@timestamp'], + }; + + const missing: string[] = []; + if (typeof question !== 'string' || !question) missing.push('question'); + + if (jury) { + // The jury decides what a replayable cell looks like for its suite. + // Attack Discovery grades `output.insights`, so requiring a final agent + // message here would skip cells that are perfectly gradable -- the + // defect that made 433 of 800 AD cells look unreplayable. + if (!jury.toArgs(candidate)) { + missing.push(`gradable ${jury.name} output`); + } + } else { + if (typeof expected !== 'string' || !expected) missing.push('dataset reference'); + if (!agentResponse) missing.push('agent response'); + } + + if (missing.length > 0) { + if (!seenSkips.has(key)) { + seenSkips.add(key); + skipped.push({ + executionId, + exampleId, + reason: `missing ${missing.join(', ')}`, + }); + } + continue; + } + + cells.set(key, candidate); + } + + // A cell whose first document was incomplete but whose later documents carry + // the trajectory is replayable: report it as a cell, not as both a cell and a + // skip. Evaluator documents for one cell arrive in no guaranteed order, so + // resolving this by document order would make the plan order-dependent. + const resolved = new Set(cells.keys()); + return { + cells: [...cells.values()], + skipped: skipped.filter((s) => !resolved.has(`${s.executionId}::${s.exampleId}`)), + }; +} + +/** + * Derive the execution id a replay writes under. + * + * Re-judged scores MUST NOT be written back under the source execution id: + * the matrix aggregates by execution, so mixing two judges' verdicts into one + * execution produces a cell that is silently an average of disagreeing judges. + */ +export function replayExecutionId(sourceExecutionId: string, judgeTag: string): string { + if (!judgeTag) { + throw new Error('replay requires a judge tag so re-judged scores stay separable'); + } + return `${sourceExecutionId}::rejudge-${judgeTag}`; +} + +/** Estimated cost of a replay, for the pre-flight summary. */ +export function summarizePlan(plan: ReplayPlan): string { + const models = new Set(plan.cells.map((c) => c.modelId)); + const executions = new Set(plan.cells.map((c) => c.executionId)); + return ( + `${plan.cells.length} cell(s) across ${models.size} model(s), ` + + `${executions.size} execution(s)` + + (plan.skipped.length > 0 ? `; ${plan.skipped.length} unreplayable` : '') + ); +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/run_rejudge.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/run_rejudge.test.ts new file mode 100644 index 0000000000000..e4473e37ccf0b --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/run_rejudge.test.ts @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { runRejudge, describeJudgeFailure, type CellJudge } from './run_rejudge'; +import type { ReplayCell } from './replay_plan'; + +const cell = (overrides: Partial = {}): ReplayCell => ({ + executionId: 'exec-1', + exampleId: 'ex-a', + modelId: 'model-x', + question: 'q', + expected: 'e', + agentResponse: 'a', + steps: [], + recordedAt: '2026-08-22T16:24:55.232Z', + ...overrides, +}); + +const okJudge: CellJudge = async (c) => ({ + scores: [{ name: 'Factuality', score: 1, label: 'ACCURATE' }], + analyses: { correctness: { example: c.exampleId } }, +}); + +describe('runRejudge', () => { + it('judges every cell and tags results with the judge execution id', async () => { + const result = await runRejudge({ + cells: [cell(), cell({ exampleId: 'ex-b' })], + judge: okJudge, + judgeTag: 'haiku', + concurrency: 2, + }); + + expect(result.results).toHaveLength(2); + expect(result.results[0].executionId).toBe('exec-1::rejudge-haiku'); + expect(result.failures).toEqual([]); + }); + + it('reports a failing cell instead of dropping it', async () => { + // A judge that throws on some cells must not silently shrink the matrix: + // a model whose hard examples all failed would otherwise publish an + // average over only its easy ones, which reads as a better model. + const judge: CellJudge = async (c) => { + if (c.exampleId === 'ex-b') throw new Error('judge 500'); + return okJudge(c); + }; + + const result = await runRejudge({ + cells: [cell(), cell({ exampleId: 'ex-b' })], + judge, + judgeTag: 'haiku', + concurrency: 2, + }); + + expect(result.results).toHaveLength(1); + expect(result.failures).toEqual([ + expect.objectContaining({ exampleId: 'ex-b', reason: expect.stringContaining('judge 500') }), + ]); + }); + + it('respects the concurrency limit', async () => { + // Unbounded fan-out over ~400 cells rate-limits the judge connector and + // turns a cheap replay into a retry storm. + let inFlight = 0; + let peak = 0; + const judge: CellJudge = async (c) => { + inFlight++; + peak = Math.max(peak, inFlight); + await new Promise((r) => setTimeout(r, 5)); + inFlight--; + return okJudge(c); + }; + + await runRejudge({ + cells: Array.from({ length: 12 }, (_, i) => cell({ exampleId: `ex-${i}` })), + judge, + judgeTag: 'haiku', + concurrency: 3, + }); + + expect(peak).toBeLessThanOrEqual(3); + expect(peak).toBeGreaterThan(1); + }); + + it('refuses an empty judge tag', async () => { + // Without a tag the replay writes under the source execution id and mixes + // two judges' verdicts into one cell. + await expect( + runRejudge({ cells: [cell()], judge: okJudge, judgeTag: '', concurrency: 1 }) + ).rejects.toThrow(/judge tag/i); + }); + + it('preserves source provenance on each result', async () => { + const result = await runRejudge({ + cells: [cell()], + judge: okJudge, + judgeTag: 'haiku', + concurrency: 1, + }); + + expect(result.results[0]).toEqual( + expect.objectContaining({ + sourceExecutionId: 'exec-1', + modelId: 'model-x', + exampleId: 'ex-a', + }) + ); + }); +}); + +describe('describeJudgeFailure', () => { + // Regression: a whole 99-cell judge run reported one indistinguishable reason + // ("LLM could not complete task successfully in 4 attempts") because + // AggregateError.message hides the per-attempt causes that name the real + // problem. Without them there is nothing to act on. + it('unwraps the causes an AggregateError hides behind its summary', () => { + const error = new AggregateError( + [ + new Error('Tool call did not match schema: criteria is required'), + new Error('rate limited'), + ], + 'LLM could not complete task successfully in 4 attempts' + ); + + const reason = describeJudgeFailure(error); + + expect(reason).toContain('LLM could not complete task successfully in 4 attempts'); + expect(reason).toContain('Tool call did not match schema: criteria is required'); + expect(reason).toContain('rate limited'); + }); + + it('collapses repeated identical causes instead of repeating them per attempt', () => { + const error = new AggregateError( + [new Error('same failure'), new Error('same failure'), new Error('same failure')], + 'summary' + ); + + expect(describeJudgeFailure(error)).toBe('summary: same failure'); + }); + + it('passes an ordinary Error message through unchanged', () => { + expect(describeJudgeFailure(new Error('plain failure'))).toBe('plain failure'); + }); + + it('stringifies a non-Error rejection', () => { + expect(describeJudgeFailure('string failure')).toBe('string failure'); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/run_rejudge.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/run_rejudge.ts new file mode 100644 index 0000000000000..477a2768b9cce --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/run_rejudge.ts @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import pLimit from 'p-limit'; +import type { ReplayCell, PlanIssue } from './replay_plan'; +import { replayExecutionId } from './replay_plan'; + +/** One evaluator verdict produced by a re-judge. */ +export interface RejudgeScore { + name: string; + score: number | null; + label?: string; + explanation?: string; +} + +/** Grades a single trajectory. Supplied by the CLI, faked in tests. */ +export type CellJudge = (cell: ReplayCell) => Promise<{ + scores: RejudgeScore[]; + analyses?: Record; +}>; + +export interface RejudgeResult { + /** Execution id the re-judged scores belong to, never the source's. */ + executionId: string; + sourceExecutionId: string; + modelId: string; + exampleId: string; + recordedAt: string; + scores: RejudgeScore[]; + analyses?: Record; +} + +export interface RejudgeRunResult { + results: RejudgeResult[]; + /** Cells the judge could not grade. Reported, never silently dropped. */ + failures: PlanIssue[]; +} + +/** + * Re-grade planned cells with a new judge. + * + * Failures are collected rather than thrown: one judge error must not discard + * the other several hundred cells, but it also must not vanish — a model whose + * hardest examples all failed would otherwise average only its easy ones and + * look better than it is. + */ +/** + * An AggregateError's `message` is a fixed summary ("LLM could not complete task + * successfully in 4 attempts") that hides the per-attempt causes in `errors`. + * Reporting only the summary makes every cell fail with one indistinguishable + * string, which is what a whole judge run looked like before the real cause -- + * schema validation rejecting the judge's tool output -- could be seen at all. + */ +export function describeJudgeFailure(error: unknown): string { + if (error instanceof AggregateError) { + const causes = [ + ...new Set( + error.errors.map((inner) => (inner instanceof Error ? inner.message : String(inner))) + ), + ]; + return causes.length ? `${error.message}: ${causes.join('; ')}` : error.message; + } + return error instanceof Error ? error.message : String(error); +} + +export async function runRejudge({ + cells, + judge, + judgeTag, + concurrency = 5, +}: { + cells: ReplayCell[]; + judge: CellJudge; + judgeTag: string; + concurrency?: number; +}): Promise { + // Fail before spending judge calls: replayExecutionId rejects an empty tag, + // and without a tag the results would overwrite the source execution's cell. + replayExecutionId(cells[0]?.executionId ?? 'probe', judgeTag); + + const limit = pLimit(Math.max(1, concurrency)); + const results: RejudgeResult[] = []; + const failures: PlanIssue[] = []; + + await Promise.all( + cells.map((cell) => + limit(async () => { + try { + const { scores, analyses } = await judge(cell); + results.push({ + executionId: replayExecutionId(cell.executionId, judgeTag), + sourceExecutionId: cell.executionId, + modelId: cell.modelId, + exampleId: cell.exampleId, + recordedAt: cell.recordedAt, + scores, + analyses, + }); + } catch (error) { + failures.push({ + executionId: cell.executionId, + exampleId: cell.exampleId, + reason: describeJudgeFailure(error), + }); + } + }) + ) + ); + + // Deterministic order regardless of completion order, so artifacts diff cleanly. + results.sort((a, b) => `${a.modelId}${a.exampleId}`.localeCompare(`${b.modelId}${b.exampleId}`)); + failures.sort((a, b) => + `${a.executionId}${a.exampleId}`.localeCompare(`${b.executionId}${b.exampleId}`) + ); + + return { results, failures }; +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/saturation.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/saturation.test.ts new file mode 100644 index 0000000000000..06b02ab46ec1a --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/saturation.test.ts @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { analyzeSaturation } from './saturation'; + +const judges = (a: number, b: number, c: number) => ({ gemini: a, sonnet: b, gpt: c }); + +describe('analyzeSaturation', () => { + it('names saturation when judges agree but the rubric pins models at the ceiling', () => { + // The attack-discovery case: 93-98% judge agreement, yet most models score a + // perfect 1.0. Calling this "judge noise" would invite a rejudge that cannot + // possibly separate the models. + const cells = [ + { modelId: 'a', cellKey: 'x', scoresByJudge: judges(1, 1, 1) }, + { modelId: 'b', cellKey: 'x', scoresByJudge: judges(1, 1, 1) }, + { modelId: 'c', cellKey: 'x', scoresByJudge: judges(1, 1, 1) }, + { modelId: 'd', cellKey: 'x', scoresByJudge: judges(0.5, 0.5, 0.5) }, + ]; + + const report = analyzeSaturation({ cells, ceiling: 1 }); + + expect(report.limitingFactor).toBe('saturation'); + expect(report.saturatedModels).toBe(3); + expect(report.ceilingShare).toBeCloseTo(0.75); + expect(report.verdict).toMatch(/rubric, not the judge/); + }); + + // Precedence regression: a saturated column can ALSO have judgeSpread >= + // modelSpread, because a ceiling squeezes the model spread toward zero. + // Testing judge noise first would then report 'judge-noise' for a column no + // rejudge can fix -- exactly the wrong instruction. + it('reports saturation, not judge noise, when a ceiling squeezes the model spread', () => { + const cells = [ + { modelId: 'a', cellKey: 'x', scoresByJudge: judges(1, 1, 1) }, + { modelId: 'b', cellKey: 'x', scoresByJudge: judges(1, 1, 0.9) }, + { modelId: 'c', cellKey: 'x', scoresByJudge: judges(1, 0.9, 1) }, + { modelId: 'd', cellKey: 'x', scoresByJudge: judges(1, 1, 1) }, + ]; + + const report = analyzeSaturation({ cells, ceiling: 1 }); + + expect(report.judgeSpread).toBeGreaterThanOrEqual(report.modelSpread); + expect(report.limitingFactor).toBe('saturation'); + expect(report.verdict).toMatch(/rubric, not the judge/); + }); + + it('names judge noise when disagreement swamps the model differences', () => { + const cells = [ + { modelId: 'a', cellKey: 'x', scoresByJudge: judges(0.1, 0.9, 0.5) }, + { modelId: 'b', cellKey: 'x', scoresByJudge: judges(0.2, 0.8, 0.4) }, + ]; + + const report = analyzeSaturation({ cells, ceiling: 1 }); + + expect(report.limitingFactor).toBe('judge-noise'); + expect(report.verdict).toMatch(/ensembling judges/); + }); + + it('reports none when models separate by more than the judges disagree', () => { + const cells = [ + { modelId: 'a', cellKey: 'x', scoresByJudge: judges(0.9, 0.9, 0.88) }, + { modelId: 'b', cellKey: 'x', scoresByJudge: judges(0.5, 0.52, 0.5) }, + { modelId: 'c', cellKey: 'x', scoresByJudge: judges(0.1, 0.12, 0.1) }, + ]; + + const report = analyzeSaturation({ cells, ceiling: 1 }); + + expect(report.limitingFactor).toBe('none'); + }); + + it('counts distinct judges and models rather than raw cells', () => { + const cells = [ + { modelId: 'a', cellKey: 'x', scoresByJudge: judges(1, 1, 1) }, + { modelId: 'a', cellKey: 'y', scoresByJudge: judges(1, 1, 1) }, + { modelId: 'b', cellKey: 'x', scoresByJudge: judges(0.2, 0.2, 0.2) }, + ]; + + const report = analyzeSaturation({ cells, ceiling: 1 }); + + expect(report.cellCount).toBe(3); + expect(report.modelCount).toBe(2); + expect(report.judgeCount).toBe(3); + }); + + it('refuses to report on an empty column instead of returning NaN', () => { + expect(() => analyzeSaturation({ cells: [], ceiling: 1 })).toThrow(/at least one cell/); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/saturation.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/saturation.ts new file mode 100644 index 0000000000000..af091c006294a --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/saturation.ts @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * A column can fail to rank for two very different reasons, and the fix differs: + * + * - **Judge noise**: judges disagree per cell by more than the models differ. + * Harmonising or ensembling judges helps. + * - **Rubric saturation**: judges agree almost perfectly, but the rubric puts + * nearly every model at the ceiling. No amount of rejudging helps; the rubric + * cannot express the differences being asked of it. + * + * Reporting "not rankable" without saying which one is actively misleading: it + * invites a rejudge that cannot possibly work. This measures both. + */ + +export interface SaturationInput { + /** Score keyed by judge, for one (model, example, evaluator) cell. */ + cells: Array<{ modelId: string; cellKey: string; scoresByJudge: Record }>; + /** Highest score the rubric can award; scores at this value are saturated. */ + ceiling: number; + /** How close to the ceiling still counts as saturated. */ + tolerance?: number; +} + +export interface SaturationReport { + cellCount: number; + modelCount: number; + judgeCount: number; + /** Fraction of all (cell, judge) scores sitting at the ceiling. */ + ceilingShare: number; + /** Models whose ensemble mean is within `tolerance` of the ceiling. */ + saturatedModels: number; + /** Mean per-cell disagreement between judges (max - min). */ + judgeSpread: number; + /** Spread of ensemble means across models (max - min). */ + modelSpread: number; + /** + * `saturation` when the rubric ceiling is the binding constraint, + * `judge-noise` when disagreement exceeds the model differences, + * `none` when the column can support a ranking. + */ + limitingFactor: 'saturation' | 'judge-noise' | 'none'; + /** Plain-language statement of what would actually change the outcome. */ + verdict: string; +} + +export function analyzeSaturation({ + cells, + ceiling, + tolerance = 0.02, +}: SaturationInput): SaturationReport { + if (cells.length === 0) { + throw new Error('analyzeSaturation requires at least one cell; refusing to report on nothing.'); + } + + const judges = [...new Set(cells.flatMap((c) => Object.keys(c.scoresByJudge)))]; + const scores = cells.flatMap((c) => Object.values(c.scoresByJudge)); + const atCeiling = scores.filter((s) => Math.abs(s - ceiling) <= Number.EPSILON).length; + + const spreads = cells.map((c) => { + const v = Object.values(c.scoresByJudge); + return Math.max(...v) - Math.min(...v); + }); + const judgeSpread = mean(spreads); + + const byModel = new Map(); + for (const cell of cells) { + const ensemble = mean(Object.values(cell.scoresByJudge)); + byModel.set(cell.modelId, [...(byModel.get(cell.modelId) ?? []), ensemble]); + } + const modelMeans = [...byModel.values()].map(mean); + const modelSpread = Math.max(...modelMeans) - Math.min(...modelMeans); + const saturatedModels = modelMeans.filter((m) => m >= ceiling - tolerance).length; + + const saturatedShare = saturatedModels / byModel.size; + // Saturation is judged first: when most models sit at the ceiling, the judges + // can agree perfectly and the column still cannot rank. + const limitingFactor: SaturationReport['limitingFactor'] = + saturatedShare >= 0.5 ? 'saturation' : judgeSpread >= modelSpread ? 'judge-noise' : 'none'; + + const verdict = + limitingFactor === 'saturation' + ? `${saturatedModels}/${byModel.size} models sit within ${tolerance} of the ${ceiling} ceiling ` + + `while judges disagree by only ${judgeSpread.toFixed( + 3 + )} per cell. The rubric, not the judge, ` + + `is the binding constraint -- rejudging cannot separate these models, and a harder or ` + + `finer-grained rubric is the only thing that would.` + : limitingFactor === 'judge-noise' + ? `Judges disagree by ${judgeSpread.toFixed(3)} per cell against a model spread of ` + + `${modelSpread.toFixed(3)}. Judge noise dominates the differences being measured, so ` + + `harmonising or ensembling judges is what would make this column rankable.` + : `Judge spread ${judgeSpread.toFixed(3)} is below the model spread ${modelSpread.toFixed( + 3 + )} ` + `and only ${saturatedModels}/${byModel.size} models are near the ceiling.`; + + return { + cellCount: cells.length, + modelCount: byModel.size, + judgeCount: judges.length, + ceilingShare: atCeiling / scores.length, + saturatedModels, + judgeSpread, + modelSpread, + limitingFactor, + verdict, + }; +} + +function mean(values: number[]): number { + return values.reduce((sum, v) => sum + v, 0) / values.length; +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/scoring_config.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/scoring_config.test.ts new file mode 100644 index 0000000000000..d85959c18ab37 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/scoring_config.test.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { matrixConfigSchema } from './load_matrix_config'; + +describe('matrixConfigSchema — scoring policy', () => { + const base = { + columns: [{ id: 'c', label: 'C', suites: ['s'] }], + models: [{ id: 'm', label: 'M' }], + overall: { label: 'Overall' }, + }; + + it('leaves scoring unset when a matrix does not opt in, so published numbers are unchanged', () => { + const config = matrixConfigSchema.validate(base); + + expect(config.scoring).toBeUndefined(); + }); + + it('accepts a fully opted-in policy', () => { + const config = matrixConfigSchema.validate({ + ...base, + scoring: { + useVerdictLadder: true, + requireEisJudge: true, + excludeSelfJudged: true, + }, + }); + + expect(config.scoring).toEqual({ + useVerdictLadder: true, + requireEisJudge: true, + excludeSelfJudged: true, + }); + }); + + it('allows opting into the ladder without the provenance gate', () => { + const config = matrixConfigSchema.validate({ + ...base, + scoring: { useVerdictLadder: true }, + }); + + expect(config.scoring?.useVerdictLadder).toBe(true); + expect(config.scoring?.requireEisJudge).toBe(false); + }); + + it('rejects a non-boolean flag rather than coercing it', () => { + expect(() => + matrixConfigSchema.validate({ + ...base, + scoring: { useVerdictLadder: 'yes' }, + }) + ).toThrow(/useVerdictLadder/); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/scoring_passthrough.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/scoring_passthrough.test.ts new file mode 100644 index 0000000000000..6325033b8a267 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/scoring_passthrough.test.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EvalsClient } from '@kbn/evals'; +import type { SomeDevLog } from '@kbn/some-dev-log'; +import { queryMatrixScores } from './query_matrix_scores'; + +/** + * The policy flags are only worth anything if a config value survives the trip + * to the aggregator. A unit test on `scoresByPrefixToDatasets` cannot see a + * dropped `scoring:` line in the call site — this one can. + */ +describe('queryMatrixScores — scoring policy passthrough', () => { + const log = { + debug: jest.fn(), + info: jest.fn(), + warning: jest.fn(), + error: jest.fn(), + } as unknown as SomeDevLog; + + const experiment = { + experiment_id: 'exp-1', + execution_id: 'exec-1', + timestamp: new Date().toISOString(), + task_model: { id: 'model-a', family: 'anthropic', provider: 'eis' }, + }; + + /** One judged score per judge: EIS-pinned, self-hosted, and self-judged. */ + const scores = [ + { + example: { id: 'alert-analysis-a' }, + evaluator: { + name: 'Groundedness', + score: 0.9, + model: { id: 'eis-anthropic-claude-4.6-sonnet' }, + metadata: { groundednessAnalysis: { summary_verdict: 'GROUNDED' } }, + }, + task: { model: { id: 'model-a' } }, + }, + { + example: { id: 'alert-analysis-b' }, + evaluator: { + name: 'Groundedness', + score: 0.1, + model: { id: 'Qwen/Qwen3-Coder-30B-A3B-Instruct' }, + metadata: { groundednessAnalysis: { summary_verdict: 'MAJOR_HALLUCINATIONS' } }, + }, + task: { model: { id: 'model-a' } }, + }, + ]; + + const clientFor = (docs = scores) => + ({ + listExperiments: jest.fn().mockResolvedValue([experiment]), + getExperimentStats: jest.fn().mockResolvedValue({ stats: [] }), + getExperimentScores: jest.fn().mockResolvedValue(docs), + } as unknown as EvalsClient); + + const prefixMean = async (scoring?: Parameters[2]['scoring']) => { + const [model] = await queryMatrixScores(clientFor(), log, { + suiteIds: ['suite-a'], + modelIds: ['model-a'], + prefixesBySuite: { 'suite-a': ['alert-analysis'] }, + scoring, + }); + + const dataset = model?.suites[0]?.datasets.find((d) => d.datasetId === 'prefix:alert-analysis'); + return dataset?.evaluators.find((e) => e.evaluatorName === 'Groundedness'); + }; + + it('counts every judge and uses continuous scores when no policy is given', async () => { + const evaluator = await prefixMean(); + + expect(evaluator?.count).toBe(2); + expect(evaluator?.mean).toBeCloseTo(0.5, 5); + }); + + it('drops the non-EIS judge when the config requires EIS-pinned judges', async () => { + const evaluator = await prefixMean({ requireEisJudge: true }); + + expect(evaluator?.count).toBe(1); + expect(evaluator?.mean).toBeCloseTo(0.9, 5); + }); + + it('reads the categorical verdict when the config enables the ladder', async () => { + // Continuous score and verdict disagree, so only the ladder path yields 1. + const supported = [{ ...scores[0], evaluator: { ...scores[0].evaluator, score: 0.42 } }]; + + const read = async (scoring?: { useVerdictLadder: boolean }) => { + const [model] = await queryMatrixScores(clientFor(supported), log, { + suiteIds: ['suite-a'], + modelIds: ['model-a'], + prefixesBySuite: { 'suite-a': ['alert-analysis'] }, + scoring, + }); + return model?.suites[0]?.datasets + .find((d) => d.datasetId === 'prefix:alert-analysis') + ?.evaluators.find((e) => e.evaluatorName === 'Groundedness')?.mean; + }; + + expect(await read()).toBeCloseTo(0.42, 5); + expect(await read({ useVerdictLadder: true })).toBeCloseTo(1, 5); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/scoring_policy.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/scoring_policy.test.ts new file mode 100644 index 0000000000000..7454b1a261a18 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/scoring_policy.test.ts @@ -0,0 +1,182 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + applyScoringPolicy, + resolveVerdictScore, + emptyExclusionCounts, + tallyRejection, + type PolicyScoreDoc, +} from './scoring_policy'; + +const noneExcluded = () => false; +const defaultExcluded = (name: string) => + ['Latency', 'Tool Calls', 'Input Tokens', 'Output Tokens', 'Cached Tokens', 'Skill Invoked'].some( + (p) => name.startsWith(p) + ); + +const doc = ( + over: Partial = {}, + taskModelId = 'model-a' +): PolicyScoreDoc => ({ + task: { model: { id: taskModelId } }, + evaluator: { name: 'Groundedness', score: 7, ...over }, +}); + +describe('scoring policy', () => { + describe('provenance filters', () => { + it('drops a non-EIS judge only when requireEisJudge is set', () => { + const d = doc({ model: { id: 'NousResearch/Hermes-3-Llama-3.1-70B' } }); + + expect(applyScoringPolicy(d, {}, noneExcluded).score).toBe(7); + expect(applyScoringPolicy(d, { requireEisJudge: true }, noneExcluded)).toEqual({ + score: null, + rejected: 'non-eis', + }); + }); + + it('keeps an EIS-backed judge under requireEisJudge', () => { + const d = doc({ model: { id: 'eis-gemini-3-1-pro' } }); + expect(applyScoringPolicy(d, { requireEisJudge: true }, noneExcluded).score).toBe(7); + }); + + it('drops a self-judged score only when excludeSelfJudged is set', () => { + const d = doc({ model: { id: 'model-a' } }, 'model-a'); + + expect(applyScoringPolicy(d, {}, noneExcluded).score).toBe(7); + expect(applyScoringPolicy(d, { excludeSelfJudged: true }, noneExcluded)).toEqual({ + score: null, + rejected: 'self-judged', + }); + }); + + it('keeps a cross-model judge under excludeSelfJudged', () => { + const d = doc({ model: { id: 'model-b' } }, 'model-a'); + expect(applyScoringPolicy(d, { excludeSelfJudged: true }, noneExcluded).score).toBe(7); + }); + }); + + describe('non-quality exclusion', () => { + it('drops evaluators excluded by name', () => { + const d = doc({ name: 'Latency', score: 1200 }); + expect(applyScoringPolicy(d, {}, defaultExcluded)).toEqual({ + score: null, + rejected: 'non-quality', + }); + }); + + it('drops dynamically-named excluded evaluators by prefix', () => { + const d = doc({ name: 'Skill Invoked (alert-analysis)', score: 0 }); + expect(applyScoringPolicy(d, {}, defaultExcluded).rejected).toBe('non-quality'); + }); + + it('drops a non-maximize direction even when the name is allowed', () => { + const d = doc({ name: 'Groundedness', direction: 'minimize' }); + expect(applyScoringPolicy(d, {}, noneExcluded).rejected).toBe('non-quality'); + }); + + it('keeps a maximize direction', () => { + const d = doc({ direction: 'maximize' }); + expect(applyScoringPolicy(d, {}, noneExcluded).score).toBe(7); + }); + }); + + describe('verdict ladder', () => { + it('ladders a categorical verdict instead of the continuous score', () => { + const d = doc({ + name: 'Groundedness', + score: 3, + metadata: { groundednessAnalysis: { summary_verdict: 'GROUNDED' } }, + }); + const laddered = applyScoringPolicy(d, { useVerdictLadder: true }, noneExcluded).score; + + // GROUNDED sits at the top of the ladder (1), not the stored 3. + expect(laddered).toBe(1); + }); + + it('maps a mid-ladder verdict to its ordinal value', () => { + const d = doc({ + name: 'Groundedness', + score: 9, + metadata: { groundednessAnalysis: { summary_verdict: 'MINOR_HALLUCINATIONS' } }, + }); + expect(applyScoringPolicy(d, { useVerdictLadder: true }, noneExcluded).score).toBe(0.5); + }); + + it('reads the nested correctness block for Factuality', () => { + const d = doc({ + name: 'Factuality', + score: 2, + metadata: { correctnessAnalysis: { summary: { factual_accuracy_summary: 'ACCURATE' } } }, + }); + // ACCURATE ladders to 1, replacing the stored continuous 2. + expect(applyScoringPolicy(d, { useVerdictLadder: true }, noneExcluded).score).toBe(1); + }); + + it('falls back to the numeric grade when metadata was stripped server-side', () => { + // 64% of golden persona docs retain metadata; rejecting the rest would + // blank whole columns and report valid grades as excluded. + const d = doc({ name: 'Groundedness', score: 7 }); + delete (d.evaluator as { metadata?: unknown }).metadata; + + expect(applyScoringPolicy(d, { useVerdictLadder: true }, noneExcluded).score).toBe(7); + }); + + it('passes through evaluators that have no verdict vocabulary', () => { + const d = doc({ name: 'Trajectory', score: 5, metadata: {} }); + expect(resolveVerdictScore('Trajectory', d)).toBe(5); + }); + + it('reports an unmappable verdict rather than scoring it', () => { + const d = doc({ + name: 'Groundedness', + score: undefined, + metadata: { groundednessAnalysis: { summary_verdict: 'not-a-known-verdict' } }, + }); + expect(applyScoringPolicy(d, { useVerdictLadder: true }, noneExcluded)).toEqual({ + score: null, + rejected: 'unmapped-verdict', + }); + }); + }); + + describe('exclusion tallies', () => { + it('counts each rejection kind separately', () => { + const counts = emptyExclusionCounts(); + tallyRejection(counts, 'non-quality'); + tallyRejection(counts, 'non-eis'); + tallyRejection(counts, 'self-judged'); + tallyRejection(counts, 'self-judged'); + tallyRejection(counts, 'unmapped-verdict'); + tallyRejection(counts, undefined); + + expect(counts).toEqual({ + nonQuality: 1, + nonEis: 1, + selfJudged: 2, + unmappedVerdict: 1, + }); + }); + }); + + describe('policy ordering', () => { + it('rejects on provenance before laddering, so a dropped judge never scores', () => { + const d = doc( + { + name: 'Groundedness', + model: { id: 'model-a' }, + metadata: { groundednessAnalysis: { summary_verdict: 'GROUNDED' } }, + }, + 'model-a' + ); + + expect( + applyScoringPolicy(d, { excludeSelfJudged: true, useVerdictLadder: true }, noneExcluded) + ).toEqual({ score: null, rejected: 'self-judged' }); + }); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/scoring_policy.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/scoring_policy.ts new file mode 100644 index 0000000000000..749543c2abc9e --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/scoring_policy.ts @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { VERDICT_LADDERS, scoreVerdict } from './jury'; +import { isEisBacked, describeJudge } from './judge_provenance'; + +/** + * Scoring policy shared by the two paths that turn raw score documents into + * matrix cells: the CLI transport (`query_matrix_scores`) and the golden + * driver (`scripts/extract_golden_aggregate.ts`). + * + * Both used to implement this independently, and only the CLI implemented it + * at all -- so a board rendered from a golden extract silently disagreed with + * the published board by ~2 points per cell while exiting 0. Keeping the + * per-document decision here means the two paths cannot drift again. + */ + +/** + * Where each judged evaluator stores its categorical verdict. Taken from live + * score documents, not inferred: Groundedness writes + * `groundednessAnalysis.summary_verdict`, while Factuality and Relevance both + * hang off the shared `correctnessAnalysis.summary` block under different keys. + */ +export const VERDICT_PATHS: Record = { + Groundedness: ['groundednessAnalysis', 'summary_verdict'], + Factuality: ['correctnessAnalysis', 'factual_accuracy_summary'], + Relevance: ['correctnessAnalysis', 'relevance_summary'], +}; + +export interface ScoringPolicy { + /** Drop scores from judges that are not EIS-backed connectors. */ + requireEisJudge?: boolean; + /** Drop scores where the judge and the graded model are the same id. */ + excludeSelfJudged?: boolean; + /** Score the judge's categorical verdict via an ordinal ladder. */ + useVerdictLadder?: boolean; +} + +/** Minimal shape the policy reads; both callers' document types satisfy it. */ +export interface PolicyScoreDoc { + task?: { model?: { id?: string | null } | null } | null; + evaluator?: { + name?: string | null; + score?: number | null; + label?: string | null; + direction?: string | null; + model?: { id?: string | null } | null; + metadata?: unknown; + } | null; +} + +export type PolicyRejection = 'non-quality' | 'non-eis' | 'self-judged' | 'unmapped-verdict'; + +export interface PolicyDecision { + /** Effective score for the cell, or null when the document is dropped. */ + score: number | null; + /** Why the document was dropped, when it was. */ + rejected?: PolicyRejection; +} + +export interface PolicyExclusionCounts { + nonQuality: number; + nonEis: number; + selfJudged: number; + unmappedVerdict: number; +} + +export const emptyExclusionCounts = (): PolicyExclusionCounts => ({ + nonQuality: 0, + nonEis: 0, + selfJudged: 0, + unmappedVerdict: 0, +}); + +/** + * Ladder score for a judged evaluator, or the stored continuous score for + * evaluators with no verdict vocabulary. + * + * The scores route strips `evaluator.metadata` server-side (UNBOUNDED_SCORE_FIELDS, + * #286691). When the block is absent there is no verdict to ladder, but the + * numeric grade is still trustworthy -- fall back to it rather than rejecting a + * valid score. Treating this as "unmapped" silently blanked per-prefix columns. + * Measured on golden: only 64% of persona score docs retain `evaluator.metadata`, + * so this fallback is the common path, not an edge case. + */ +export function resolveVerdictScore( + evaluatorName: string, + doc: PolicyScoreDoc +): number | null | undefined { + const ladder = VERDICT_LADDERS[evaluatorName]; + const path = VERDICT_PATHS[evaluatorName]; + if (!ladder || !path) { + return doc.evaluator?.score; + } + + const [blockKey, verdictKey] = path; + const metadata = doc.evaluator?.metadata as Record | undefined; + if (metadata === undefined) { + return doc.evaluator?.score; + } + const block = metadata?.[blockKey] as Record | undefined; + // Groundedness puts its verdict at the top of the block; the correctness + // evaluators nest theirs one level deeper under `summary`. + const summary = (block?.summary as Record | undefined) ?? block; + const verdict = summary?.[verdictKey]; + + const mapped = scoreVerdict(typeof verdict === 'string' ? verdict : undefined, ladder); + return mapped ?? undefined; +} + +/** + * Apply the scoring policy to a single score document. + * + * `isExcludedName` reports whether an evaluator name is excluded by the + * config's evaluator allowlist (Latency, Tool Calls, ...). Upstream now + * persists evaluator polarity (#284027), but only 2.3% of persona score docs + * carry `evaluator.direction`, so the name allowlist remains the primary + * signal and polarity is used only when present. + */ +export function applyScoringPolicy( + doc: PolicyScoreDoc, + policy: ScoringPolicy, + isExcludedName: (evaluatorName: string) => boolean +): PolicyDecision { + const evaluatorName = doc.evaluator?.name; + if (!evaluatorName) { + return { score: null, rejected: 'non-quality' }; + } + + const judgeId = doc.evaluator?.model?.id; + const taskModelId = doc.task?.model?.id; + + if (policy.requireEisJudge && judgeId && !isEisBacked(judgeId)) { + return { score: null, rejected: 'non-eis' }; + } + if ( + policy.excludeSelfJudged && + judgeId && + taskModelId && + describeJudge(judgeId, taskModelId).selfJudged + ) { + return { score: null, rejected: 'self-judged' }; + } + + const direction = doc.evaluator?.direction; + if (direction && direction !== 'maximize') { + return { score: null, rejected: 'non-quality' }; + } + if (isExcludedName(evaluatorName)) { + return { score: null, rejected: 'non-quality' }; + } + + const score = policy.useVerdictLadder + ? resolveVerdictScore(evaluatorName, doc) + : doc.evaluator?.score; + + if (typeof score !== 'number') { + return { score: null, rejected: policy.useVerdictLadder ? 'unmapped-verdict' : undefined }; + } + return { score }; +} + +/** Tally a rejection into the running exclusion counts. */ +export function tallyRejection( + counts: PolicyExclusionCounts, + rejected: PolicyRejection | undefined +): void { + if (rejected === 'non-quality') counts.nonQuality += 1; + else if (rejected === 'non-eis') counts.nonEis += 1; + else if (rejected === 'self-judged') counts.selfJudged += 1; + else if (rejected === 'unmapped-verdict') counts.unmappedVerdict += 1; +} diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/stability_diff.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/stability_diff.test.ts new file mode 100644 index 0000000000000..d6e93309af907 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/stability_diff.test.ts @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EvaluationScoreDocument } from '@kbn/evals-common'; +import { diffStability, toCells } from './stability_diff'; + +const doc = ( + model: string, + example: string, + evaluator: string, + score: number, + repetition = 0 +): EvaluationScoreDocument => + ({ + evaluator: { name: evaluator, score }, + example: { id: example }, + task: { model: { id: model }, repetition_index: repetition }, + } as unknown as EvaluationScoreDocument); + +describe('toCells', () => { + it('averages repetitions into one cell and records the count', () => { + const cells = toCells([ + doc('haiku', 'alert-analysis-a', 'criteria', 0, 0), + doc('haiku', 'alert-analysis-a', 'criteria', 1, 1), + ]); + expect([...cells.values()]).toEqual([ + { + model: 'haiku', + example: 'alert-analysis-a', + evaluator: 'criteria', + score: 0.5, + repetitions: 2, + }, + ]); + }); + + it('skips documents missing model, example, evaluator or score', () => { + expect(toCells([{ evaluator: { name: 'criteria' } } as EvaluationScoreDocument]).size).toBe(0); + }); +}); + +describe('diffStability', () => { + it('flags the real run5 -> run6 Haiku flip that a single sweep hides', () => { + // run5: Haiku failed alert-analysis-a and multi-step-b (19/21). + // run6: same commit, same stack, both passed (21/21). + const run5 = [ + doc('haiku', 'alert-analysis-a', 'criteria', 0), + doc('haiku', 'multi-step-b', 'criteria', 0), + doc('haiku', 'entity-analytics-a', 'criteria', 1), + ]; + const run6 = [ + doc('haiku', 'alert-analysis-a', 'criteria', 1), + doc('haiku', 'multi-step-b', 'criteria', 1), + doc('haiku', 'entity-analytics-a', 'criteria', 1), + ]; + + const diff = diffStability(run5, run6); + + expect(diff.unchanged).toBe(1); + expect(diff.flips).toHaveLength(2); + expect(diff.flips.map((f) => f.example).sort()).toEqual(['alert-analysis-a', 'multi-step-b']); + // Neither sweep repeated, so these are noise-suspect, not proven regressions. + expect(diff.flips.every((f) => f.singleSampled)).toBe(true); + }); + + it('sorts regressions before improvements', () => { + const diff = diffStability( + [doc('m', 'a', 'criteria', 1), doc('m', 'b', 'criteria', 0)], + [doc('m', 'a', 'criteria', 0), doc('m', 'b', 'criteria', 1)] + ); + expect(diff.flips[0]).toMatchObject({ example: 'a', delta: -1 }); + expect(diff.flips[1]).toMatchObject({ example: 'b', delta: 1 }); + }); + + it('does not mark a flip single-sampled when repetitions back it', () => { + const diff = diffStability( + [doc('m', 'a', 'Relevance', 8, 0), doc('m', 'a', 'Relevance', 8, 1)], + [doc('m', 'a', 'Relevance', 3, 0), doc('m', 'a', 'Relevance', 3, 1)] + ); + expect(diff.flips[0].singleSampled).toBe(false); + }); + + it('ignores wobble within tolerance', () => { + const diff = diffStability( + [doc('m', 'a', 'Groundedness', 8.0)], + [doc('m', 'a', 'Groundedness', 8.2)], + { tolerance: 0.5 } + ); + expect(diff.flips).toHaveLength(0); + expect(diff.unchanged).toBe(1); + }); + + it('counts cells that exist in only one sweep instead of reporting them as flips', () => { + const diff = diffStability([doc('m', 'gone', 'criteria', 1)], [doc('m', 'new', 'criteria', 1)]); + expect(diff.flips).toHaveLength(0); + expect(diff.onlyBefore).toBe(1); + expect(diff.onlyAfter).toBe(1); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/stability_diff.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/stability_diff.ts new file mode 100644 index 0000000000000..a9c95d1c89ed8 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/stability_diff.ts @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EvaluationScoreDocument } from '@kbn/evals-common'; + +/** One evaluator's outcome for one example, in one sweep. */ +export interface StabilityCell { + model: string; + example: string; + evaluator: string; + /** Mean score across whatever repetitions that sweep ran. */ + score: number; + /** How many repetitions produced `score`. 1 means the value is a single sample. */ + repetitions: number; +} + +/** A cell whose score moved between two sweeps of the same suite. */ +export interface StabilityFlip { + model: string; + example: string; + evaluator: string; + before: number; + after: number; + delta: number; + /** True when neither side had more than one repetition backing it. */ + singleSampled: boolean; +} + +export interface StabilityDiff { + flips: StabilityFlip[]; + /** Cells present in both sweeps with identical scores. */ + unchanged: number; + /** Cells that exist in only one of the two sweeps. */ + onlyBefore: number; + onlyAfter: number; +} + +const cellKey = (c: { model: string; example: string; evaluator: string }) => + `${c.model}\u0000${c.example}\u0000${c.evaluator}`; + +/** + * Collapse raw score documents into one cell per (model, example, evaluator), + * averaging over repetitions. Exported for unit testing. + */ +export const toCells = (docs: EvaluationScoreDocument[]): Map => { + const acc = new Map(); + for (const doc of docs) { + const evaluator = doc.evaluator?.name; + const score = doc.evaluator?.score; + const example = doc.example?.id; + const model = (doc as { task?: { model?: { id?: string } } }).task?.model?.id; + if (!evaluator || !example || !model || typeof score !== 'number') { + continue; + } + const cell: StabilityCell = { model, example, evaluator, score: 0, repetitions: 0 }; + const key = cellKey(cell); + const prev = acc.get(key) ?? { sum: 0, n: 0, cell }; + prev.sum += score; + prev.n += 1; + acc.set(key, prev); + } + return new Map( + [...acc.entries()].map(([key, { sum, n, cell }]) => [ + key, + { ...cell, score: sum / n, repetitions: n }, + ]) + ); +}; + +/** + * Compare two sweeps of the same suite and report every cell whose score moved. + * + * This is the guard against silent drift: a model that scored 21/21 last sweep + * and 19/21 this sweep looks fine in isolation on both, and the regression is + * only visible by diffing them. `singleSampled` marks flips where neither side + * had repetitions behind it — those are as likely to be run-to-run noise as a + * real regression, and should be re-run at a higher repetition tier before + * anyone treats them as a finding. + * + * `tolerance` ignores moves at or below its size, for graded judge evaluators + * that wobble by fractions without changing any conclusion. + */ +export const diffStability = ( + before: EvaluationScoreDocument[], + after: EvaluationScoreDocument[], + { tolerance = 0 }: { tolerance?: number } = {} +): StabilityDiff => { + const a = toCells(before); + const b = toCells(after); + + const flips: StabilityFlip[] = []; + let unchanged = 0; + let onlyBefore = 0; + + for (const [key, cellA] of a) { + const cellB = b.get(key); + if (!cellB) { + onlyBefore += 1; + continue; + } + const delta = cellB.score - cellA.score; + if (Math.abs(delta) <= tolerance) { + unchanged += 1; + continue; + } + flips.push({ + model: cellA.model, + example: cellA.example, + evaluator: cellA.evaluator, + before: cellA.score, + after: cellB.score, + delta, + singleSampled: cellA.repetitions <= 1 && cellB.repetitions <= 1, + }); + } + + let onlyAfter = 0; + for (const key of b.keys()) { + if (!a.has(key)) { + onlyAfter += 1; + } + } + + // Largest regressions first — that is what a reader needs to triage. + flips.sort((x, y) => x.delta - y.delta); + + return { flips, unchanged, onlyBefore, onlyAfter }; +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/trace_types.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/trace_types.ts new file mode 100644 index 0000000000000..433862276fcb7 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/trace_types.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** A single step in the agent's reasoning + tool-call trace. */ +export interface TraceStep { + type: 'reasoning' | 'tool' | 'skill'; + /** Reasoning text (for `type: 'reasoning'`). */ + text?: string; + /** Tool call ID (for `type: 'tool'`). */ + toolId?: string; + /** Tool call parameters (for `type: 'tool'`). */ + toolParams?: string; + /** + * Skills selected (for `type: 'skill'`). + * + * Agent Builder emits skill entries as objects (`{ id, name, path, description }`), + * not bare strings. This was typed `string[]`, so `.join()` in the renderer produced + * `[object Object]` for every skill step and TypeScript could not catch it. Accept + * both shapes and normalise at the render boundary via `skillLabel`. + */ + skills?: Array; +} + +/** Trace data for a single (model, column) pair. */ +export interface MatrixTraceEntry { + /** The initial user question from the eval dataset. */ + question?: string; + /** Ordered list of tool IDs the agent called. */ + toolTrail?: string[]; + /** The agent's final answer (markdown). */ + answer?: string; + /** Full reasoning + tool-call step trace. */ + steps?: TraceStep[]; + /** Number of steps (cached for summary table). */ + stepCount?: number; + /** Number of tool calls (cached for summary table). */ + toolCount?: number; + /** + * Per-evaluator mean scores for this example within the experiment + * (evaluator name → mean over repetitions). Lets the report render a + * per-prompt score instead of repeating the column aggregate on every card. + */ + scores?: Record; + /** Number of repetitions aggregated into this entry's scores. */ + repetitions?: number; + /** + * Per-evaluator spread across repetitions (evaluator name → max - min). + * `scores` alone cannot distinguish a stable cell from a volatile one: a + * cell scoring 10/10/10 and one scoring 0/10/20 both report a mean of 10. + * Only populated when `repetitions > 1`; an absent entry means the cell was + * measured once and its stability is unknown, not that it is stable. + */ + spread?: Record; + /** + * Ordered tool identifiers observed in each repetition. Invocation ids are + * excluded because providers generate a new id for every call. + */ + repTrails?: string[][]; + /** + * Final answer text per repetition, index-aligned with `repTrails`. Path + * churn and answer churn are only weakly related (r=0.14 on the pilot), so + * the board must measure them separately rather than implying one from the + * other. + */ + repAnswers?: string[]; + /** + * The example's declared path contract, read from + * `example.metadata.pathContract`. Absent on corpora predating the field. + */ + pathContract?: 'rankable' | 'candidate' | 'probe'; + /** Execution ids that contributed the repetitions above, for auditability. */ + repExecutionIds?: string[]; +} + +/** + * Map of trace entries keyed by `${modelId}:${columnId}`. + * Lookups use the same model/column IDs as the matrix config. + */ +export type MatrixTraceData = Record; + +/** Build the trace-data lookup key. */ +export const traceKey = (modelId: string, columnId: string): string => `${modelId}:${columnId}`; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/trajectory_agreement.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/trajectory_agreement.test.ts new file mode 100644 index 0000000000000..742a3ebc678e1 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/trajectory_agreement.test.ts @@ -0,0 +1,290 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + answerSimilarity, + answersFromDocs, + cellAgreement, + firstDivergence, + intervalsOverlap, + pathContractFromDocs, + resolveProbe, + rowAgreement, + sequenceSimilarity, + trailsEqual, + trailSetsEqual, + trailsFromDocs, + wilsonInterval, +} from './trajectory_agreement'; + +describe('trailsEqual / sequenceSimilarity', () => { + it('treats identical tool_id sequences as equal', () => { + expect(trailsEqual(['load_skill', 'search'], ['load_skill', 'search'])).toBe(true); + }); + + it('does not treat reorder as equal — order is the path', () => { + expect(trailsEqual(['a', 'b'], ['b', 'a'])).toBe(false); + }); + + it('reports 1 for two empty trails and 0 for empty vs non-empty', () => { + expect(sequenceSimilarity([], [])).toBe(1); + expect(sequenceSimilarity([], ['a'])).toBe(0); + }); + + it('scores a shared prefix below 1', () => { + const sim = sequenceSimilarity(['a', 'b', 'c'], ['a', 'b', 'd']); + expect(sim).toBeCloseTo(0.666, 2); + }); +}); + +describe('cellAgreement', () => { + it('marks a single trail unmeasured, not 0 and not 1', () => { + expect(cellAgreement([['a']])).toEqual({ status: 'unmeasured', repetitions: 1 }); + expect(cellAgreement([])).toEqual({ status: 'unmeasured', repetitions: 0 }); + }); + + it('reports identicalRate 1 when every pair matches', () => { + const a = cellAgreement([ + ['load_skill', 'search'], + ['load_skill', 'search'], + ['load_skill', 'search'], + ]); + expect(a.status).toBe('measured'); + expect(a.identicalRate).toBe(1); + expect(a.sequenceSimilarity).toBe(1); + }); + + it('reports identicalRate 0 when no pair matches, even if similar', () => { + const a = cellAgreement([['generate_workflow'], ['sml_search', 'discover_apis']]); + expect(a.identicalRate).toBe(0); + expect(a.sequenceSimilarity).toBeLessThan(1); + }); +}); + +describe('trailsFromDocs', () => { + const step = (toolId: string) => ({ type: 'tool_call', tool_id: toolId }); + + it('keeps one trail per repetition_index, first doc wins', () => { + const trails = trailsFromDocs([ + { task: { repetition_index: 0, output: { steps: [step('a'), step('b')] } } }, + // same rep, different evaluator — must not become a second trail + { task: { repetition_index: 0, output: { steps: [step('a'), step('b')] } } }, + { task: { repetition_index: 1, output: { steps: [step('a')] } } }, + ]); + expect(trails).toEqual([['a', 'b'], ['a']]); + }); + + it('never keys on tool_call_id — only tool_id is collected', () => { + const trails = trailsFromDocs([ + { + task: { + repetition_index: 0, + output: { + steps: [{ type: 'tool_call', tool_id: 'search', tool_call_id: 'toolu_AAA' } as never], + }, + }, + }, + { + task: { + repetition_index: 1, + output: { + steps: [{ type: 'tool_call', tool_id: 'search', tool_call_id: 'toolu_BBB' } as never], + }, + }, + }, + ]); + expect(trails).toEqual([['search'], ['search']]); + expect(cellAgreement(trails).identicalRate).toBe(1); + }); +}); + +describe('rowAgreement', () => { + it('stays unmeasured when every cell is a single rep', () => { + const row = rowAgreement( + [ + { model: 'opus', example: 'a', trails: [['x']] }, + { model: 'opus', example: 'b', trails: [['y']] }, + ], + 'opus' + ); + expect(row.status).toBe('unmeasured'); + expect(row.identicalRate).toBeUndefined(); + expect(row.measuredCells).toBe(0); + }); + + it('averages only measured cells and ignores other models', () => { + const row = rowAgreement( + [ + { + model: 'opus', + example: 'a', + trails: [['x'], ['x']], + }, + { + model: 'opus', + example: 'b', + trails: [['y'], ['z']], + }, + { model: 'gpt', example: 'a', trails: [['x'], ['x']] }, + ], + 'opus' + ); + expect(row.status).toBe('measured'); + expect(row.measuredCells).toBe(2); + expect(row.identicalRate).toBe(0.5); + }); +}); + +describe('answersFromDocs / pathContractFromDocs', () => { + const doc = (rep: number, message: string, pathContract?: string) => ({ + task: { repetition_index: rep, output: { messages: [{ message }] } }, + example: pathContract ? { metadata: { pathContract } } : {}, + }); + + it('returns one answer per repetition, ordered by repetition index', () => { + const long = 'a'.repeat(60); + expect(answersFromDocs([doc(1, `${long}-second`), doc(0, `${long}-first`)])).toEqual([ + `${long}-first`, + `${long}-second`, + ]); + }); + + it('aligns with trailsFromDocs so answer pairs match path pairs', () => { + const long = 'a'.repeat(60); + const docs = [ + { + ...doc(0, `${long}-x`), + task: { repetition_index: 0, output: { steps: [], messages: [{ message: `${long}-x` }] } }, + }, + { + ...doc(1, `${long}-y`), + task: { repetition_index: 1, output: { steps: [], messages: [{ message: `${long}-y` }] } }, + }, + ]; + expect(answersFromDocs(docs).length).toBe(trailsFromDocs(docs).length); + }); + + it('reads the declared contract and returns undefined for pre-field corpora', () => { + expect(pathContractFromDocs([doc(0, 'x', 'probe')])).toBe('probe'); + expect(pathContractFromDocs([doc(0, 'x')])).toBeUndefined(); + }); +}); + +describe('trailSetsEqual', () => { + it('treats a reordered trail as the same tool set', () => { + expect(trailSetsEqual(['a', 'b'], ['b', 'a'])).toBe(true); + // ...while exact-sequence equality does not — that gap is the order-only churn. + expect(trailsEqual(['a', 'b'], ['b', 'a'])).toBe(false); + }); + + it('ignores repetition of the same tool', () => { + expect(trailSetsEqual(['a', 'a', 'b'], ['b', 'a'])).toBe(true); + }); + + it('separates a genuinely different tool from a reordering', () => { + expect(trailSetsEqual(['a', 'b'], ['a', 'c'])).toBe(false); + }); +}); + +describe('cellAgreement toolSetRate', () => { + it('scores order-only churn as full tool-set agreement but zero exact agreement', () => { + const agreement = cellAgreement([ + ['search', 'load_skill'], + ['load_skill', 'search'], + ]); + expect(agreement.identicalRate).toBe(0); + expect(agreement.toolSetRate).toBe(1); + }); + + it('scores a different tool as disagreement on both metrics', () => { + const agreement = cellAgreement([['search'], ['execute_esql']]); + expect(agreement.identicalRate).toBe(0); + expect(agreement.toolSetRate).toBe(0); + }); +}); + +describe('resolveProbe', () => { + it('prefers the declared contract over the example id', () => { + // A hunt-prefixed example the dataset declares rankable must be treated as + // rankable: the dataset is the source of truth, not the id. + expect(resolveProbe('threat-hunting-a', 'rankable')).toEqual({ + probe: false, + source: 'declared', + }); + expect(resolveProbe('workflow-authoring-a', 'probe')).toEqual({ + probe: true, + source: 'declared', + }); + }); + + it('falls back to the legacy prefix list and reports it, for pre-field corpora', () => { + expect(resolveProbe('alert-analysis-a')).toEqual({ probe: true, source: 'legacy-prefix' }); + expect(resolveProbe('entity-analytics-c')).toEqual({ probe: true, source: 'legacy-prefix' }); + expect(resolveProbe('workflow-authoring-a')).toEqual({ + probe: false, + source: 'legacy-prefix', + }); + }); +}); + +describe('wilsonInterval', () => { + it('brackets the point estimate and stays inside [0, 1]', () => { + const wide = wilsonInterval(7, 27); + expect(wide.low).toBeGreaterThan(0.1); + expect(wide.high).toBeLessThan(0.5); + expect(wide.low).toBeLessThan(7 / 27); + expect(wide.high).toBeGreaterThan(7 / 27); + }); + + it('does not run below zero at a zero rate', () => { + const zero = wilsonInterval(0, 10); + expect(zero.low).toBe(0); + expect(zero.high).toBeGreaterThan(0); + }); + + it('narrows as the sample grows', () => { + const small = wilsonInterval(5, 20); + const large = wilsonInterval(50, 200); + expect(large.high - large.low).toBeLessThan(small.high - small.low); + }); +}); + +describe('intervalsOverlap', () => { + it('treats two small-sample rates a few points apart as indistinguishable', () => { + // 26% and 22% over ~27 pairs each — the case the board must not order. + expect(intervalsOverlap(wilsonInterval(7, 27), wilsonInterval(6, 27))).toBe(true); + }); + + it('separates rates that genuinely differ at adequate sample size', () => { + expect(intervalsOverlap(wilsonInterval(10, 200), wilsonInterval(180, 200))).toBe(false); + }); +}); + +describe('answerSimilarity', () => { + it('scores identical text 1 and disjoint text 0', () => { + expect(answerSimilarity('the host was compromised', 'the host was compromised')).toBe(1); + expect(answerSimilarity('alpha bravo', 'charlie delta')).toBe(0); + }); + + it('ignores case and punctuation', () => { + expect(answerSimilarity('Host: compromised!', 'host compromised')).toBe(1); + }); +}); + +describe('firstDivergence', () => { + it('reports the index and tool where two paths split', () => { + expect(firstDivergence(['a', 'b', 'c'], ['a', 'x', 'c'])).toEqual({ step: 1, tool: 'b' }); + }); + + it('reports the extra step when one path is a prefix of the other', () => { + expect(firstDivergence(['a', 'b'], ['a', 'b', 'c'])).toEqual({ step: 2, tool: 'c' }); + }); + + it('returns undefined for identical paths', () => { + expect(firstDivergence(['a', 'b'], ['a', 'b'])).toBeUndefined(); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/trajectory_agreement.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/trajectory_agreement.ts new file mode 100644 index 0000000000000..ac258da2e864e --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/trajectory_agreement.ts @@ -0,0 +1,438 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Trajectory agreement across repetitions of the same (model, example). + * + * Key on `tool_id`, never `tool_call_id`. Provider invocation ids are unique + * per call, so comparing them reports 100% divergence by construction even + * when the agent issued the same tools. + * + * A cell with one repetition is `unmeasured`, not 0 and not 1. Collapsing + * those three is how a 21-of-23 empty reliability column would look "broken" + * on the current corpus. + */ + +export type ReliabilityStatus = 'unmeasured' | 'measured'; + +export interface TrajectoryCell { + model: string; + example: string; + /** One tool_id sequence per repetition. Empty sequences are kept. */ + trails: string[][]; + /** + * True when the example is an open-ended capability probe with no repeatable + * path. Set from dataset metadata (`pathContract`), never inferred from the + * example id — the dataset is the single source of truth. + */ + probe?: boolean; + /** Whether `probe` came from declared metadata or the legacy prefix guess. */ + probeSource?: 'declared' | 'legacy-prefix'; + /** Final answer per repetition, index-aligned with `trails`. */ + answers?: string[]; +} + +/** Wilson score interval for a binomial proportion. */ +export interface ConfidenceInterval { + low: number; + high: number; +} + +export interface TrajectoryAgreement { + status: ReliabilityStatus; + /** Distinct repetitions that produced a trail. 1 → unmeasured. */ + repetitions: number; + /** + * Pairwise identical-sequence rate in [0, 1]. Undefined when unmeasured. + * Identity, not similarity: order-preserving exact match of tool_id lists. + */ + identicalRate?: number; + /** + * Pairwise same-tool-set rate in [0, 1], ignoring order and repetition. Always + * >= identicalRate; the gap between them is order-only churn, which is a much + * weaker claim of instability than reaching for different tools. + */ + toolSetRate?: number; + /** Mean pairwise LCS / max(len_a, len_b). Undefined when unmeasured. */ + sequenceSimilarity?: number; + /** Pairs compared (n choose 2 over repetitions) — the sample behind the rate. */ + pairs?: number; + /** + * Mean pairwise similarity of final answers in [0, 1]. Reported beside path + * agreement, never derived from it: on the pilot corpus path similarity ran + * 0.69-0.73 while answer similarity was 0.17 (r=0.14), so a stable path does + * not imply a stable answer. + */ + answerSimilarity?: number; + /** Pairs that had an answer on both sides — may be fewer than `pairs`. */ + answerPairs?: number; + /** + * Index of the first step where two trails diverge, minimum over pairs. + * 0 means the runs picked different first tools. Undefined when identical. + */ + firstDivergenceStep?: number; + /** The tool_id at `firstDivergenceStep` on the shorter-prefix side. */ + firstDivergenceTool?: string; +} + +export interface ReliabilityRow { + modelId: string; + status: ReliabilityStatus; + cells: number; + measuredCells: number; + identicalRate?: number; + /** + * Pooled same-tool-set rate. Reported beside identicalRate so a low exact rate + * driven purely by ordering is visible as such rather than read as tool churn. + */ + toolSetRate?: number; + sequenceSimilarity?: number; + /** Total pairs pooled across this model's measured cells. */ + pairs?: number; + /** + * Wilson 95% interval on `identicalRate`. Nine cells at 3 reps is ~27 pairs, + * which puts the interval near +/-15pp — wide enough that two models cannot be + * ordered on the point estimate alone. + */ + interval?: ConfidenceInterval; + /** Pooled answer similarity across measured cells, if any answers were kept. */ + answerSimilarity?: number; + /** How many of this row's cells were classified by the legacy prefix guess. */ + legacyClassifiedCells?: number; + /** Most common first-divergence tool across measured cells, with its count. */ + divergenceHotspot?: { tool: string; cells: number }; +} + +const lcsLength = (a: string[], b: string[]): number => { + if (a.length === 0 || b.length === 0) { + return 0; + } + const prev = new Array(b.length + 1).fill(0); + const curr = new Array(b.length + 1).fill(0); + for (let i = 1; i <= a.length; i++) { + for (let j = 1; j <= b.length; j++) { + curr[j] = a[i - 1] === b[j - 1] ? prev[j - 1] + 1 : Math.max(prev[j], curr[j - 1]); + } + for (let j = 0; j <= b.length; j++) { + prev[j] = curr[j]; + } + } + return prev[b.length]; +}; + +/** SequenceMatcher-style ratio: 2 * LCS / (len_a + len_b). */ +export const sequenceSimilarity = (a: string[], b: string[]): number => { + if (a.length === 0 && b.length === 0) { + return 1; + } + const denom = a.length + b.length; + if (denom === 0) { + return 1; + } + return (2 * lcsLength(a, b)) / denom; +}; + +const pairwiseMean = (trails: string[][], pairFn: (a: string[], b: string[]) => number): number => { + let sum = 0; + let n = 0; + for (let i = 0; i < trails.length; i++) { + for (let j = i + 1; j < trails.length; j++) { + sum += pairFn(trails[i], trails[j]); + n += 1; + } + } + return n === 0 ? 0 : sum / n; +}; + +/** + * Wilson score interval at 95%. Chosen over the normal approximation because + * the rates here sit near 0 and the samples are small (tens of pairs), where + * the normal interval runs off the end of [0, 1] and understates uncertainty. + */ +export const wilsonInterval = (successes: number, total: number): ConfidenceInterval => { + if (total <= 0) { + return { low: 0, high: 1 }; + } + const z = 1.96; + const p = successes / total; + const z2 = z * z; + const denom = 1 + z2 / total; + const centre = p + z2 / (2 * total); + const spread = z * Math.sqrt((p * (1 - p)) / total + z2 / (4 * total * total)); + return { + low: Math.max(0, (centre - spread) / denom), + high: Math.min(1, (centre + spread) / denom), + }; +}; + +/** + * True when two intervals overlap, i.e. the data cannot order the two rows. + * The board must not rank measured models whose intervals overlap. + */ +export const intervalsOverlap = (a: ConfidenceInterval, b: ConfidenceInterval): boolean => + a.low <= b.high && b.low <= a.high; + +/** Word-level similarity of two answers, using the same LCS ratio as paths. */ +export const answerSimilarity = (a: string, b: string): number => { + const tokens = (text: string) => + text + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter(Boolean); + return sequenceSimilarity(tokens(a), tokens(b)); +}; + +/** + * Index of the first differing step between two trails, or undefined when one + * is a prefix of the other and they otherwise agree. Locating divergence is + * what turns a low rate into an actionable lead: on the pilot corpus 70% of + * divergences occurred by step 2, most at skill selection. + */ +export const firstDivergence = ( + a: string[], + b: string[] +): { step: number; tool?: string } | undefined => { + const shared = Math.min(a.length, b.length); + for (let i = 0; i < shared; i++) { + if (a[i] !== b[i]) { + return { step: i, tool: a[i] }; + } + } + if (a.length === b.length) { + return undefined; + } + return { step: shared, tool: (a.length > b.length ? a : b)[shared] }; +}; + +export const trailsEqual = (a: string[], b: string[]): boolean => + a.length === b.length && a.every((id, i) => id === b[i]); + +/** + * Whether two trails used the same distinct tools, ignoring order and repetition. + * + * Exact-sequence equality conflates two different behaviours: reaching for a different tool, and + * reaching for the same tools in a different order. Measured on the pilot corpus, ~6% of repeated + * cells differ ONLY in ordering, so reporting exact agreement alone charges those cells the full + * penalty of a genuinely divergent path. + */ +export const trailSetsEqual = (a: string[], b: string[]): boolean => { + const left = new Set(a); + const right = new Set(b); + if (left.size !== right.size) { + return false; + } + for (const id of left) { + if (!right.has(id)) { + return false; + } + } + return true; +}; + +export const cellAgreement = (trails: string[][], answers?: string[]): TrajectoryAgreement => { + if (trails.length <= 1) { + return { status: 'unmeasured', repetitions: trails.length }; + } + + let answerSum = 0; + let answerPairs = 0; + let earliest: { step: number; tool?: string } | undefined; + for (let i = 0; i < trails.length; i++) { + for (let j = i + 1; j < trails.length; j++) { + const left = answers?.[i]; + const right = answers?.[j]; + if (left && right) { + answerSum += answerSimilarity(left, right); + answerPairs += 1; + } + const divergence = firstDivergence(trails[i], trails[j]); + if (divergence && (earliest === undefined || divergence.step < earliest.step)) { + earliest = divergence; + } + } + } + + return { + status: 'measured', + repetitions: trails.length, + identicalRate: pairwiseMean(trails, (a, b) => (trailsEqual(a, b) ? 1 : 0)), + toolSetRate: pairwiseMean(trails, (a, b) => (trailSetsEqual(a, b) ? 1 : 0)), + sequenceSimilarity: pairwiseMean(trails, sequenceSimilarity), + pairs: (trails.length * (trails.length - 1)) / 2, + answerSimilarity: answerPairs > 0 ? answerSum / answerPairs : undefined, + answerPairs: answerPairs > 0 ? answerPairs : undefined, + firstDivergenceStep: earliest?.step, + firstDivergenceTool: earliest?.tool, + }; +}; + +export const rowAgreement = (cells: TrajectoryCell[], modelId: string): ReliabilityRow => { + // Probes are open-ended by contract; including them would report a low rate + // for prompts that were never meant to have one repeatable path. + const mine = cells.filter((c) => c.model === modelId && !c.probe); + const measured = mine + .map((c) => cellAgreement(c.trails, c.answers)) + .filter((a) => a.status === 'measured'); + const legacyClassifiedCells = cells.filter( + (c) => c.model === modelId && c.probeSource === 'legacy-prefix' + ).length; + if (measured.length === 0) { + return { + modelId, + status: 'unmeasured', + cells: mine.length, + measuredCells: 0, + legacyClassifiedCells: legacyClassifiedCells || undefined, + }; + } + // Pool pairs rather than averaging per-cell rates: a mean of means hides the + // sample size, and the sample size is the whole caveat on this column. + const pairs = measured.reduce((s, a) => s + (a.pairs ?? 0), 0); + const matches = measured.reduce((s, a) => s + (a.identicalRate ?? 0) * (a.pairs ?? 0), 0); + const identicalRate = pairs === 0 ? 0 : matches / pairs; + const toolSetRate = + pairs === 0 + ? 0 + : measured.reduce((s, a) => s + (a.toolSetRate ?? 0) * (a.pairs ?? 0), 0) / pairs; + const sequenceSim = + measured.reduce((s, a) => s + (a.sequenceSimilarity ?? 0) * (a.pairs ?? 0), 0) / (pairs || 1); + + const answerPairs = measured.reduce((s, a) => s + (a.answerPairs ?? 0), 0); + const answerSum = measured.reduce( + (s, a) => s + (a.answerSimilarity ?? 0) * (a.answerPairs ?? 0), + 0 + ); + + const hotspots = new Map(); + for (const cell of measured) { + if (cell.firstDivergenceTool) { + hotspots.set(cell.firstDivergenceTool, (hotspots.get(cell.firstDivergenceTool) ?? 0) + 1); + } + } + const topHotspot = [...hotspots.entries()].sort((a, b) => b[1] - a[1])[0]; + + return { + modelId, + status: 'measured', + cells: mine.length, + measuredCells: measured.length, + identicalRate, + toolSetRate, + sequenceSimilarity: sequenceSim, + pairs, + interval: wilsonInterval(matches, pairs), + answerSimilarity: answerPairs > 0 ? answerSum / answerPairs : undefined, + legacyClassifiedCells: legacyClassifiedCells || undefined, + divergenceHotspot: topHotspot ? { tool: topHotspot[0], cells: topHotspot[1] } : undefined, + }; +}; + +/** + * Group per-rep tool_id sequences from score documents of one example. + * Docs sharing a repetition_index are the same agent run (one per evaluator); + * the first complete trail wins. Key on tool_id only. + */ +export type PathContract = 'rankable' | 'candidate' | 'probe'; + +/** + * Legacy classification, derived from the 3-rep pilot where every + * hunt/investigation example scored 0/5 on exact path agreement. + * + * @deprecated The dataset now declares `pathContract` per example and that is + * the single source of truth. This list only classifies score documents + * produced BEFORE that field existed, so an old cached corpus still renders. + * Callers must report how many cells fell back here — see `probeSource`. + * Delete once every corpus in use carries `example.metadata.pathContract`. + */ +export const LEGACY_PROBE_EXAMPLE_PREFIXES = [ + 'alert-analysis-', + 'entity-analytics-', + 'multi-step-', + 'threat-hunting-', +] as const; + +const isLegacyProbeExample = (exampleId: string): boolean => + LEGACY_PROBE_EXAMPLE_PREFIXES.some((prefix) => exampleId.startsWith(prefix)); + +/** Reads the declared contract off a score document's example metadata. */ +export const pathContractFromDocs = ( + docs: ReadonlyArray<{ example?: unknown }> +): PathContract | undefined => { + for (const doc of docs) { + const example = doc.example as { metadata?: { pathContract?: PathContract } } | undefined; + const declared = example?.metadata?.pathContract; + if (declared) { + return declared; + } + } + return undefined; +}; + +/** + * Resolves whether an example is a probe, preferring the declared contract and + * reporting which mechanism answered so the board can disclose the fallback + * rather than presenting a legacy guess as measured metadata. + */ +export const resolveProbe = ( + exampleId: string, + declared?: PathContract +): { probe: boolean; source: 'declared' | 'legacy-prefix' } => + declared + ? { probe: declared === 'probe', source: 'declared' } + : { probe: isLegacyProbeExample(exampleId), source: 'legacy-prefix' }; + +/** + * Final answer per repetition, index-aligned with `trailsFromDocs` output. + * Uses the same repetition grouping so answer pairs line up with path pairs. + */ +export const answersFromDocs = (docs: ReadonlyArray<{ task?: unknown }>): string[] => { + const byRep = new Map(); + for (const doc of docs) { + const task = doc.task as + | { + repetition_index?: number; + output?: { messages?: Array<{ message?: string }> }; + } + | undefined; + const rep = task?.repetition_index ?? 0; + if (byRep.has(rep)) { + continue; + } + let answer = ''; + for (const msg of task?.output?.messages ?? []) { + if (msg.message && msg.message.length > 50) { + answer = msg.message; + } + } + byRep.set(rep, answer); + } + return [...byRep.entries()].sort((a, b) => a[0] - b[0]).map(([, answer]) => answer); +}; + +export const trailsFromDocs = (docs: ReadonlyArray<{ task?: unknown }>): string[][] => { + const byRep = new Map(); + for (const doc of docs) { + const task = doc.task as + | { + repetition_index?: number; + output?: { steps?: Array<{ type?: string; tool_id?: string }> }; + } + | undefined; + const rep = task?.repetition_index ?? 0; + if (byRep.has(rep)) { + continue; + } + const trail: string[] = []; + for (const step of task?.output?.steps ?? []) { + if (step.type === 'tool_call' && step.tool_id) { + trail.push(step.tool_id); + } + } + byRep.set(rep, trail); + } + return [...byRep.entries()].sort((a, b) => a[0] - b[0]).map(([, trail]) => trail); +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/verdict_metadata_stripped.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/verdict_metadata_stripped.test.ts new file mode 100644 index 0000000000000..ca8fee3aeff63 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/verdict_metadata_stripped.test.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { scoresByPrefixToDatasets } from './query_matrix_scores'; + +/** + * Regression guard for a silent upstream behavior change. + * + * `GET /internal/evals/experiments/{id}/scores` applies `_source_excludes: + * UNBOUNDED_SCORE_FIELDS`, and that list includes `evaluator.metadata` + * (added upstream in #286691, e7a90532a446). The verdict ladder reads the + * verdict from `evaluator.metadata.correctnessAnalysis.summary`, so every + * ladder-scored document arrives verdict-less on this route and is counted + * as `unmappedVerdict` — i.e. the counter reports SUCCESSFUL grades as + * rejected, and per-prefix columns silently lose their scores. + * + * These tests pin the shape the server actually returns. + */ +describe('verdict scoring when the server strips evaluator.metadata', () => { + const makeStrippedDoc = (evaluatorName: string, score: number) => + ({ + example: { id: 'alert-analysis-a', index: 0 }, + task: { model: { id: 'anthropic-claude-4.8-opus' } }, + // NOTE: no `evaluator.metadata` — this is what the scores route returns. + evaluator: { name: evaluatorName, score }, + } as never); + + it('does not count a stripped ladder doc as an unmapped verdict', () => { + let counts: { unmappedVerdict: number } | undefined; + + scoresByPrefixToDatasets( + [makeStrippedDoc('Factuality', 0.75), makeStrippedDoc('Relevance', 0.5)], + ['alert-analysis'], + { + useVerdictLadder: true, + onExcluded: (c: { unmappedVerdict: number }) => { + counts = c; + }, + } as never + ); + + // Before the fix this was 2: every successfully-graded doc was rejected. + expect(counts?.unmappedVerdict ?? 0).toBe(0); + }); + + it('still produces a dataset for stripped ladder scores', () => { + const datasets = scoresByPrefixToDatasets( + [makeStrippedDoc('Factuality', 0.75)], + ['alert-analysis'], + { useVerdictLadder: true } as never + ); + + // Before the fix the score fell out entirely and the column went blank. + expect(datasets.length).toBeGreaterThan(0); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/verdict_scoring.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/verdict_scoring.test.ts new file mode 100644 index 0000000000000..139bff70e56eb --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/verdict_scoring.test.ts @@ -0,0 +1,232 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EvaluationScoreDocument } from '@kbn/evals-common'; +import { scoresByPrefixToDatasets } from './query_matrix_scores'; + +/** + * Document shapes mirror live persona-matrix score docs: Groundedness stores + * `groundednessAnalysis.summary_verdict`, Factuality and Relevance share the + * `correctnessAnalysis.summary` block. + */ +const doc = (over: { + example: string; + evaluator: string; + score: number; + judge?: string; + task?: string; + metadata?: Record; +}): EvaluationScoreDocument => + ({ + example: { id: over.example }, + task: { model: { id: over.task ?? 'eis-openai-gpt-5-4' } }, + evaluator: { + name: over.evaluator, + score: over.score, + model: { id: over.judge ?? 'eis-anthropic-claude-4.6-sonnet' }, + metadata: over.metadata, + }, + } as unknown as EvaluationScoreDocument); + +const grounded = (verdict: string) => ({ + groundednessAnalysis: { summary_verdict: verdict }, +}); + +const correctness = (factual: string, relevance: string) => ({ + correctnessAnalysis: { + summary: { factual_accuracy_summary: factual, relevance_summary: relevance }, + }, +}); + +const meanOf = ( + datasets: ReturnType, + dataset: string, + evaluator: string +) => + datasets + .find((d) => d.datasetName === dataset) + ?.evaluators.find((e) => e.evaluatorName === evaluator)?.mean; + +describe('scoresByPrefixToDatasets — default behaviour', () => { + it('averages the stored continuous score when no options are passed', () => { + const datasets = scoresByPrefixToDatasets( + [ + doc({ example: 'alert-analysis-a', evaluator: 'Groundedness', score: 0.4 }), + doc({ example: 'alert-analysis-b', evaluator: 'Groundedness', score: 0.8 }), + ], + ['alert-analysis'] + ); + + expect(meanOf(datasets, 'alert-analysis', 'Groundedness')).toBeCloseTo(0.6, 5); + }); + + it('keeps non-EIS and self-judged docs when the policy is off', () => { + const datasets = scoresByPrefixToDatasets( + [ + doc({ + example: 'alert-analysis-a', + evaluator: 'Groundedness', + score: 1, + judge: 'Qwen/Qwen3-Coder-30B-A3B-Instruct', + }), + doc({ + example: 'alert-analysis-b', + evaluator: 'Groundedness', + score: 0, + judge: 'eis-openai-gpt-5-4', + task: 'eis-openai-gpt-5-4', + }), + ], + ['alert-analysis'] + ); + + expect(datasets.find((d) => d.datasetName === 'alert-analysis')?.evaluators[0].count).toBe(2); + }); +}); + +describe('scoresByPrefixToDatasets — provenance policy', () => { + const mixed = [ + doc({ example: 'alert-analysis-a', evaluator: 'Groundedness', score: 1 }), + doc({ + example: 'alert-analysis-b', + evaluator: 'Groundedness', + score: 0, + judge: 'Qwen/Qwen3-Coder-30B-A3B-Instruct', + }), + doc({ + example: 'alert-analysis-c', + evaluator: 'Groundedness', + score: 0, + judge: 'eis-openai-gpt-5-4', + task: 'eis-openai-gpt-5-4', + }), + ]; + + it('drops non-EIS judges and reports the count', () => { + const excluded: Array> = []; + const datasets = scoresByPrefixToDatasets(mixed, ['alert-analysis'], { + requireEisJudge: true, + onExcluded: (c) => excluded.push({ ...c }), + }); + + expect(excluded[0].nonEis).toBe(1); + // The self-judged doc survives because that gate is off. + expect(meanOf(datasets, 'alert-analysis', 'Groundedness')).toBeCloseTo(0.5, 5); + }); + + it('drops self-judged docs and reports the count', () => { + const excluded: Array> = []; + scoresByPrefixToDatasets(mixed, ['alert-analysis'], { + excludeSelfJudged: true, + onExcluded: (c) => excluded.push({ ...c }), + }); + + expect(excluded[0].selfJudged).toBe(1); + }); + + it('applies both gates together, leaving only the clean doc', () => { + const datasets = scoresByPrefixToDatasets(mixed, ['alert-analysis'], { + requireEisJudge: true, + excludeSelfJudged: true, + }); + + const evaluator = datasets.find((d) => d.datasetName === 'alert-analysis')?.evaluators[0]; + expect(evaluator?.count).toBe(1); + expect(evaluator?.mean).toBe(1); + }); +}); + +describe('scoresByPrefixToDatasets — verdict ladder', () => { + it('scores the Groundedness verdict, ignoring the continuous value', () => { + // Continuous score is 0.13 but the verdict is GROUNDED: the ladder must + // read the verdict, which is the whole point of the change. + const datasets = scoresByPrefixToDatasets( + [ + doc({ + example: 'alert-analysis-a', + evaluator: 'Groundedness', + score: 0.13, + metadata: grounded('GROUNDED'), + }), + ], + ['alert-analysis'], + { useVerdictLadder: true } + ); + + expect(meanOf(datasets, 'alert-analysis', 'Groundedness')).toBe(1); + }); + + it('reads Factuality and Relevance from the shared correctness block', () => { + const metadata = correctness('MINOR_INACCURACIES', 'IRRELEVANT'); + const datasets = scoresByPrefixToDatasets( + [ + doc({ example: 'alert-analysis-a', evaluator: 'Factuality', score: 0.9, metadata }), + doc({ example: 'alert-analysis-a', evaluator: 'Relevance', score: 0.9, metadata }), + ], + ['alert-analysis'], + { useVerdictLadder: true } + ); + + expect(meanOf(datasets, 'alert-analysis', 'Factuality')).toBeCloseTo(0.5, 5); + expect(meanOf(datasets, 'alert-analysis', 'Relevance')).toBe(0); + }); + + it('separates two verdicts that the continuous score conflates', () => { + const datasets = scoresByPrefixToDatasets( + [ + doc({ + example: 'alert-analysis-a', + evaluator: 'Groundedness', + score: 0.5, + metadata: grounded('GROUNDED'), + }), + doc({ + example: 'alert-analysis-b', + evaluator: 'Groundedness', + score: 0.5, + metadata: grounded('MAJOR_HALLUCINATIONS'), + }), + ], + ['alert-analysis'], + { useVerdictLadder: true } + ); + + // Identical continuous scores, opposite verdicts -> mean lands between. + expect(meanOf(datasets, 'alert-analysis', 'Groundedness')).toBeCloseTo(0.5, 5); + }); + + it('excludes an unmapped verdict rather than scoring it zero', () => { + // "No correctness analysis available" appears in real data; scoring it 0 + // is indistinguishable from a hallucinating answer. + const excluded: Array> = []; + const datasets = scoresByPrefixToDatasets( + [ + doc({ + example: 'alert-analysis-a', + evaluator: 'Factuality', + score: 0.7, + metadata: correctness('No correctness analysis available', 'RELEVANT'), + }), + ], + ['alert-analysis'], + { useVerdictLadder: true, onExcluded: (c) => excluded.push({ ...c }) } + ); + + expect(excluded[0].unmappedVerdict).toBe(1); + expect(datasets).toHaveLength(0); + }); + + it('leaves contract evaluators on their continuous score', () => { + const datasets = scoresByPrefixToDatasets( + [doc({ example: 'alert-analysis-a', evaluator: 'Sequence Accuracy', score: 0.25 })], + ['alert-analysis'], + { useVerdictLadder: true } + ); + + expect(meanOf(datasets, 'alert-analysis', 'Sequence Accuracy')).toBe(0.25); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/tsconfig.json b/x-pack/platform/packages/shared/kbn-evals-extensions/tsconfig.json index ca398649905c9..c62e7529eaec4 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/tsconfig.json +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/tsconfig.json @@ -11,6 +11,13 @@ "@kbn/evals-common", "@kbn/dev-cli-runner", "@kbn/dev-cli-errors", - "@kbn/tooling-log" + "@kbn/tooling-log", + "@kbn/config-schema", + "@kbn/some-dev-log", + "@kbn/kbn-client", + "@kbn/repo-info", + "@kbn/core", + "@kbn/inference-plugin", + "@kbn/inference-common" ] } diff --git a/x-pack/platform/packages/shared/kbn-evals/index.ts b/x-pack/platform/packages/shared/kbn-evals/index.ts index b09f46f261ffe..a9ad0db06f3a8 100644 --- a/x-pack/platform/packages/shared/kbn-evals/index.ts +++ b/x-pack/platform/packages/shared/kbn-evals/index.ts @@ -76,6 +76,9 @@ export type { export type { DefaultEvaluators, EvaluatorKind, ReportDisplayOptions } from './src/types'; export type { Direction } from '@kbn/evals-common'; export type { EvaluationCriterion, EvaluationCriterionStructured } from './src/evaluators/criteria'; +// Exposed so judge-replay tooling can rebuild a suite's criteria evaluator +// outside an eval run, where `DefaultEvaluators` is not constructed. +export { createCriteriaEvaluator } from './src/evaluators/criteria'; export { createPlaywrightEvalsConfig } from './src/config/create_playwright_eval_config'; export type { Example, @@ -91,14 +94,20 @@ export type { OnExperimentStart, } from './src/types'; export { KibanaEvalsClient } from './src/kibana_evals_executor/client'; -export { createQuantitativeCorrectnessEvaluators } from './src/evaluators/correctness'; +export { + createQuantitativeCorrectnessEvaluators, + createCorrectnessAnalysisEvaluator, +} from './src/evaluators/correctness'; export { LlmCorrectnessEvaluationPrompt } from './src/evaluators/correctness/prompt'; export type { CorrectnessAnalysis } from './src/evaluators/correctness/types'; export { calculateFactualScore, calculateRelevanceScore, } from './src/evaluators/correctness/scoring'; -export { createQuantitativeGroundednessEvaluator } from './src/evaluators/groundedness'; +export { + createQuantitativeGroundednessEvaluator, + createGroundednessAnalysisEvaluator, +} from './src/evaluators/groundedness'; export type { EvaluationDataset, EvaluationWorkerFixtures, EvaluationReport } from './src/types'; export { withEvaluatorSpan, withTaskSpan, getCurrentTraceId } from './src/utils/tracing'; export { withRetry, type RetryOptions } from './src/utils/retry_utils'; @@ -125,11 +134,20 @@ export type { export { createTable } from './src/utils/reporting/report_table'; export { EvalsClient, + MAX_LIST_EXPERIMENTS, type EvaluatorStats, type ExperimentStats, type UpsertDatasetInput, type DatasetWithId, + type ListExperimentsFilters, } from './src/utils/evals_client'; +export { + createEvaluationsEvalsClient, + getEvaluationsKbnClient, + DEFAULT_EVALUATIONS_KBN_URL, + type CreateEvaluationsEvalsClientParams, +} from './src/utils/evaluations_kbn_client'; +export { envFromDatasetsProfile } from './src/cli/profiles'; export { EvaluatorApiClient, type MapContextFn } from './src/utils/evaluator_api_client'; export { getBuildkiteCiMetadataFromEnv, type BuildkiteCiMetadata } from './src/utils/ci_metadata'; export { buildIngestRequest } from './src/utils/build_ingest_request'; @@ -152,6 +170,7 @@ export { createSkillInvocationEvaluator, createChatCallsEvaluator, createToolCallsEvaluator, + TRACE_INDEX_PATTERN, } from './src/evaluators/trace_based'; export { getGitMetadata, type GitMetadata } from './src/utils/git_metadata'; diff --git a/x-pack/platform/packages/shared/kbn-evals/src/cli/commands/init.test.ts b/x-pack/platform/packages/shared/kbn-evals/src/cli/commands/init.test.ts new file mode 100644 index 0000000000000..eaf720bcbfd97 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals/src/cli/commands/init.test.ts @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Fs from 'fs'; +import Os from 'os'; +import Path from 'path'; +import { ToolingLog } from '@kbn/tooling-log'; +import { ensureLocalConfig } from './init'; + +// Scout binds 5620/9220; `yarn start` uses 5601/9200. A config carrying the +// `yarn start` values against a Scout stack fails every request, which is how +// this shipped: `evals start` wrote 5601 while Scout served 5620. +const SCOUT_KBN = 'http://localhost:5620'; +const SCOUT_ES = 'http://localhost:9220'; + +const writeScoutConfig = (repoRoot: string) => { + const dir = Path.join(repoRoot, '.scout/servers'); + Fs.mkdirSync(dir, { recursive: true }); + Fs.writeFileSync( + Path.join(dir, 'local.json'), + JSON.stringify({ hosts: { kibana: SCOUT_KBN, elasticsearch: SCOUT_ES } }) + ); +}; + +const vaultConfigPath = (repoRoot: string) => + Path.join(repoRoot, 'x-pack/platform/packages/shared/kbn-evals/scripts/vault/config.local.json'); + +const readVaultConfig = (repoRoot: string) => + JSON.parse(Fs.readFileSync(vaultConfigPath(repoRoot), 'utf-8')); + +const seedVaultConfig = (repoRoot: string, config: Record) => { + const target = vaultConfigPath(repoRoot); + Fs.mkdirSync(Path.dirname(target), { recursive: true }); + Fs.writeFileSync(target, JSON.stringify(config, null, 2)); +}; + +describe('ensureLocalConfig', () => { + let repoRoot: string; + let log: ToolingLog; + + beforeEach(() => { + repoRoot = Fs.mkdtempSync(Path.join(Os.tmpdir(), 'kbn-evals-init-')); + log = new ToolingLog(); + writeScoutConfig(repoRoot); + }); + + afterEach(() => { + Fs.rmSync(repoRoot, { recursive: true, force: true }); + }); + + describe('when no config exists', () => { + it('writes Scout ports rather than the yarn start defaults', async () => { + await ensureLocalConfig(repoRoot, log); + + const config = readVaultConfig(repoRoot); + expect(config.evaluationsKbn.url).toContain('localhost:5620'); + expect(config.evaluationsEs.url).toContain('localhost:9220'); + expect(config.evaluationsKbn.url).not.toContain('5601'); + expect(config.evaluationsEs.url).not.toContain('9200'); + }); + + it('does not append the /dev base path, which 404s on Scout', async () => { + await ensureLocalConfig(repoRoot, log); + + expect(readVaultConfig(repoRoot).evaluationsKbn.url).not.toContain('/dev'); + }); + + it('keeps credentials in the URL', async () => { + await ensureLocalConfig(repoRoot, log); + + expect(readVaultConfig(repoRoot).evaluationsKbn.url).toContain('elastic:changeme@'); + }); + + it('falls back to defaults when Scout has not written a config', async () => { + Fs.rmSync(Path.join(repoRoot, '.scout'), { recursive: true, force: true }); + + await ensureLocalConfig(repoRoot, log); + + expect(readVaultConfig(repoRoot).evaluationsKbn.url).toContain('5601'); + }); + }); + + describe('when a stale config already exists', () => { + // The original bug: existence alone was treated as validity, so a config + // pointing at a dead port survived every subsequent run. + beforeEach(() => { + seedVaultConfig(repoRoot, { + description: 'kbn-evals local config', + owner: 'someone@example.com', + environment: 'local', + evaluationsKbn: { url: 'http://elastic:changeme@localhost:5601/dev', apiKey: '' }, + evaluationsEs: { url: 'http://elastic:changeme@localhost:9200', apiKey: '' }, + tracingEs: { url: 'http://elastic:changeme@localhost:9200', apiKey: '' }, + }); + }); + + it('realigns stale hosts to the running Scout stack', async () => { + await ensureLocalConfig(repoRoot, log); + + const config = readVaultConfig(repoRoot); + expect(config.evaluationsKbn.url).toContain('localhost:5620'); + expect(config.evaluationsEs.url).toContain('localhost:9220'); + expect(config.tracingEs.url).toContain('localhost:9220'); + }); + + it('strips the /dev base path that Scout does not serve', async () => { + await ensureLocalConfig(repoRoot, log); + + expect(readVaultConfig(repoRoot).evaluationsKbn.url).not.toContain('/dev'); + }); + + it('preserves credentials and unrelated fields', async () => { + await ensureLocalConfig(repoRoot, log); + + const config = readVaultConfig(repoRoot); + expect(config.evaluationsKbn.url).toContain('elastic:changeme@'); + expect(config.owner).toBe('someone@example.com'); + expect(config.description).toBe('kbn-evals local config'); + }); + + it('leaves an already-correct config untouched', async () => { + seedVaultConfig(repoRoot, { + owner: 'someone@example.com', + environment: 'local', + evaluationsKbn: { url: `http://elastic:changeme@localhost:5620`, apiKey: '' }, + evaluationsEs: { url: `http://elastic:changeme@localhost:9220`, apiKey: '' }, + tracingEs: { url: `http://elastic:changeme@localhost:9220`, apiKey: '' }, + }); + const before = Fs.readFileSync(vaultConfigPath(repoRoot), 'utf-8'); + + await ensureLocalConfig(repoRoot, log); + + expect(Fs.readFileSync(vaultConfigPath(repoRoot), 'utf-8')).toBe(before); + }); + + it('leaves the config alone when Scout is not running', async () => { + Fs.rmSync(Path.join(repoRoot, '.scout'), { recursive: true, force: true }); + + await ensureLocalConfig(repoRoot, log); + + expect(readVaultConfig(repoRoot).evaluationsKbn.url).toContain('5601'); + }); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals/src/cli/commands/init.ts b/x-pack/platform/packages/shared/kbn-evals/src/cli/commands/init.ts index 5764959cad667..1ce52e61355a9 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/cli/commands/init.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/cli/commands/init.ts @@ -45,16 +45,112 @@ const buildKbnUrl = (basePath: string): string => { : 'http://elastic:changeme@localhost:5601'; }; +/** + * Scout writes the ports it actually bound to `.scout/servers/local.json`. They do + * not match the `yarn start` defaults baked into LOCAL_DEFAULTS (5620/9220 vs + * 5601/9200), so a config written from those defaults points the eval client at a + * stack that isn't there and every dataset fetch fails. + */ +const readScoutHosts = (repoRoot: string): { kibana?: string; elasticsearch?: string } => { + try { + const raw = Fs.readFileSync(Path.join(repoRoot, '.scout/servers/local.json'), 'utf-8'); + const parsed = JSON.parse(raw) as { hosts?: { kibana?: string; elasticsearch?: string } }; + return parsed.hosts ?? {}; + } catch { + return {}; + } +}; + +const withCredentials = (url: string): string => + url.replace(/^(https?:\/\/)(?!.*@)/, '$1elastic:changeme@'); + +const isConfigEntry = (value: unknown): value is { url: string } => + typeof value === 'object' && + value !== null && + typeof (value as { url?: unknown }).url === 'string'; + +const hostPortOf = (url: string): string => { + try { + const parsed = new URL(url); + return `${parsed.hostname}:${parsed.port}`; + } catch { + return ''; + } +}; + +/** + * Rewrite an existing local config whose host:port no longer matches the running + * Scout stack. Only the origin is replaced -- credentials, base path and every + * unrelated field are preserved. + */ +const repairStaleScoutHosts = ( + configPath: string, + scoutHosts: { kibana?: string; elasticsearch?: string }, + log: ToolingLog +): void => { + if (!scoutHosts.kibana && !scoutHosts.elasticsearch) return; + + let config: Record; + try { + config = JSON.parse(Fs.readFileSync(configPath, 'utf-8')); + } catch { + return; + } + + const repaired: string[] = []; + const realign = (key: 'evaluationsKbn' | 'evaluationsEs' | 'tracingEs', target?: string) => { + if (!target) return; + const entry = config[key]; + if (!isConfigEntry(entry)) return; + if (hostPortOf(entry.url) === hostPortOf(target)) return; + + try { + const current = new URL(entry.url); + const scout = new URL(target); + current.hostname = scout.hostname; + current.port = scout.port; + // Scout serves at the root: a `/dev` base path from `yarn start` 404s here. + if (key === 'evaluationsKbn') current.pathname = scout.pathname; + config[key] = { ...entry, url: current.toString().replace(/\/$/, '') }; + repaired.push(key); + } catch { + // leave the entry untouched if it isn't a parsable URL + } + }; + + realign('evaluationsKbn', scoutHosts.kibana); + realign('evaluationsEs', scoutHosts.elasticsearch); + realign('tracingEs', scoutHosts.elasticsearch); + + if (repaired.length === 0) return; + Fs.writeFileSync(configPath, JSON.stringify(config, null, 2) + '\n'); + log.info(`[init] realigned ${repaired.join(', ')} to the running Scout stack`); +}; + export const ensureLocalConfig = async (repoRoot: string, log: ToolingLog): Promise => { const configPath = resolveVaultConfigPath(repoRoot, 'local'); - if (Fs.existsSync(configPath)) return; + // Prefer the ports Scout actually bound over the `yarn start` defaults. + const scoutHosts = readScoutHosts(repoRoot); + + if (Fs.existsSync(configPath)) { + // An existing config is NOT necessarily a correct one: a config written before + // Scout booted (or by a `yarn start` run) points at 5601/9200 and silently + // fails every request against a Scout stack on 5620/9220. Repair in place + // rather than returning early on mere existence. + repairStaleScoutHosts(configPath, scoutHosts, log); + return; + } + + const evaluationsEs = scoutHosts.elasticsearch + ? { url: withCredentials(scoutHosts.elasticsearch), apiKey: '' } + : { ...LOCAL_DEFAULTS.evaluationsEs }; const config: Record = { description: 'kbn-evals local config', owner: resolveUserIdentifier(), environment: 'local', - evaluationsEs: { ...LOCAL_DEFAULTS.evaluationsEs }, - tracingEs: { ...LOCAL_DEFAULTS.tracingEs }, + evaluationsEs, + tracingEs: { ...evaluationsEs }, tracingExporters: [...LOCAL_DEFAULTS.tracingExporters], }; @@ -107,9 +203,15 @@ export const ensureLocalConfig = async (repoRoot: string, log: ToolingLog): Prom } } } else { - config.evaluationsKbn = { ...LOCAL_DEFAULTS.evaluationsKbn }; + // Non-interactive (CI, background shells): no prompt for a base path, so take + // Scout's URL verbatim. Scout serves at the root -- appending the `/dev` + // basePath that `yarn start` uses would 404 every request. + config.evaluationsKbn = scoutHosts.kibana + ? { url: withCredentials(scoutHosts.kibana), apiKey: '' } + : { ...LOCAL_DEFAULTS.evaluationsKbn }; } + Fs.mkdirSync(Path.dirname(configPath), { recursive: true }); Fs.writeFileSync(configPath, JSON.stringify(config, null, 2) + '\n'); log.info(`Written local config to ${configPath}`); log.info(''); diff --git a/x-pack/platform/packages/shared/kbn-evals/src/cli/commands/run.ts b/x-pack/platform/packages/shared/kbn-evals/src/cli/commands/run.ts index 91094039f6ff6..a0aa872191d65 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/cli/commands/run.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/cli/commands/run.ts @@ -57,9 +57,9 @@ export const runSuiteCmd: Command = { 'evaluations-kbn-url', 'evaluations-kbn-api-key', ], - boolean: ['dry-run'], + boolean: ['dry-run', 'require-eis-judge'], alias: { model: 'project', judge: 'evaluation-connector-id' }, - default: { 'dry-run': false }, + default: { 'dry-run': false, 'require-eis-judge': false }, }, run: async ({ log, flagsReader }) => { const repoRoot = process.cwd(); diff --git a/x-pack/platform/packages/shared/kbn-evals/src/cli/eval_stack.ts b/x-pack/platform/packages/shared/kbn-evals/src/cli/eval_stack.ts index 9f7cdeb721935..08cd55a23a2cf 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/cli/eval_stack.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/cli/eval_stack.ts @@ -27,7 +27,9 @@ import { probeHttp } from './profiles'; const SCOUT_LOCAL_CONFIG = '.scout/servers/local.json'; const SCOUT_READY_POLL_INTERVAL_MS = 3000; -const SCOUT_READY_TIMEOUT_MS = 180_000; +// Overridable via SCOUT_READY_TIMEOUT_MS env: cold-boot rspack compile (303 +// bundles) can exceed the 180s default on loaded eval VMs. +const SCOUT_READY_TIMEOUT_MS = Number(process.env.SCOUT_READY_TIMEOUT_MS) || 180_000; const waitForScoutReady = async (repoRoot: string, log: ToolingLog): Promise => { const configPath = Path.join(repoRoot, SCOUT_LOCAL_CONFIG); @@ -155,6 +157,7 @@ export interface EnsureScoutOptions { log: ToolingLog; gcsCredentials: string | undefined; tracingExporters: string | undefined; + agentBuilderTracingExporters?: string | undefined; serverConfigSet?: string; } @@ -167,6 +170,7 @@ export const ensureScout = async ({ log, gcsCredentials, tracingExporters, + agentBuilderTracingExporters, serverConfigSet = 'evals_tracing', }: EnsureScoutOptions): Promise => { const scoutEnv: Record = {}; @@ -176,6 +180,9 @@ export const ensureScout = async ({ if (tracingExporters) { scoutEnv.TRACING_EXPORTERS = tracingExporters; } + if (agentBuilderTracingExporters) { + scoutEnv.AGENT_BUILDER_TRACING_EXPORTERS = agentBuilderTracingExporters; + } const scoutAlive = isServiceRunning(repoRoot, 'scout'); const staleCheck = scoutAlive @@ -293,6 +300,7 @@ export const ensureEvalStack = async ({ log, gcsCredentials: profileEnvOverrides.GCS_CREDENTIALS, tracingExporters: profileEnvOverrides.TRACING_EXPORTERS, + agentBuilderTracingExporters: profileEnvOverrides.AGENT_BUILDER_TRACING_EXPORTERS, serverConfigSet, }); diff --git a/x-pack/platform/packages/shared/kbn-evals/src/cli/profiles.test.ts b/x-pack/platform/packages/shared/kbn-evals/src/cli/profiles.test.ts new file mode 100644 index 0000000000000..5917277ab8308 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals/src/cli/profiles.test.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Fs from 'fs'; +import Os from 'os'; +import Path from 'path'; + +import { envFromExportProfile } from './profiles'; + +const VAULT_DIR = 'x-pack/platform/packages/shared/kbn-evals/scripts/vault'; + +describe('envFromExportProfile', () => { + let repoRoot: string; + + const writeConfig = (profile: string, config: unknown) => { + const dir = Path.resolve(repoRoot, VAULT_DIR); + Fs.mkdirSync(dir, { recursive: true }); + const fileName = profile === 'config' ? 'config.json' : `config.${profile}.json`; + Fs.writeFileSync(Path.resolve(dir, fileName), JSON.stringify(config)); + }; + + beforeEach(() => { + repoRoot = Fs.mkdtempSync(Path.join(Os.tmpdir(), 'kbn-evals-profiles-')); + }); + + afterEach(() => { + Fs.rmSync(repoRoot, { recursive: true, force: true }); + }); + + it('points score ingest at the export profile target', () => { + writeConfig('config', { + evaluationsKbn: { url: 'https://golden.example.com', apiKey: 'golden-key' }, + }); + + // Regression: an explicit export profile used to set only tracing vars, so + // scores kept going to the default local Kibana while the run exited 0. + expect(envFromExportProfile(repoRoot, 'config')).toMatchObject({ + EVAL_KBN_URL: 'https://golden.example.com', + EVAL_KBN_API_KEY: 'golden-key', + }); + }); + + it('still carries tracing settings alongside the score target', () => { + writeConfig('config', { + evaluationsKbn: { url: 'https://golden.example.com', apiKey: 'golden-key' }, + tracingEs: { url: 'https://tracing.example.com', apiKey: 'tracing-key' }, + }); + + expect(envFromExportProfile(repoRoot, 'config')).toMatchObject({ + EVAL_KBN_URL: 'https://golden.example.com', + TRACING_ES_URL: 'https://tracing.example.com', + TRACING_ES_API_KEY: 'tracing-key', + }); + }); + + it('ignores placeholder credentials rather than exporting to a bogus target', () => { + writeConfig('config', { + evaluationsKbn: { url: '', apiKey: '' }, + }); + + const env = envFromExportProfile(repoRoot, 'config'); + + expect(env).not.toHaveProperty('EVAL_KBN_URL'); + expect(env).not.toHaveProperty('EVAL_KBN_API_KEY'); + }); + + it('returns nothing when no export profile is selected', () => { + writeConfig('config', { + evaluationsKbn: { url: 'https://golden.example.com', apiKey: 'golden-key' }, + }); + + // Not selecting a profile must never implicitly export to the golden cluster. + expect(envFromExportProfile(repoRoot, undefined)).toEqual({}); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals/src/cli/profiles.ts b/x-pack/platform/packages/shared/kbn-evals/src/cli/profiles.ts index ab26aa7b9ba35..253af9389a92c 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/cli/profiles.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/cli/profiles.ts @@ -48,6 +48,7 @@ interface VaultConfig { evaluationsKbn?: { url?: string; apiKey?: string }; tracingEs?: { url?: string; apiKey?: string }; tracingExporters?: unknown; + agentBuilderTracingExporters?: unknown; gcsDatasetAccessCredentials?: unknown; } @@ -154,6 +155,19 @@ export const envFromExportProfile = ( const next: Record = {}; + // Score ingest targets EVAL_KBN_*, which otherwise only the datasets profile + // sets. Without this an explicit `--export-profile ` silently leaves + // scores pointed at the default local Kibana, so a run reports success while + // the intended export target receives nothing. + if (cfg.evaluationsKbn) { + if (isNonEmptyString(cfg.evaluationsKbn.url) && !isPlaceholder(cfg.evaluationsKbn.url)) { + next.EVAL_KBN_URL = cfg.evaluationsKbn.url; + } + if (isNonEmptyString(cfg.evaluationsKbn.apiKey) && !isPlaceholder(cfg.evaluationsKbn.apiKey)) { + next.EVAL_KBN_API_KEY = cfg.evaluationsKbn.apiKey; + } + } + if (isNonEmptyString(cfg.tracingEs?.url) && !isPlaceholder(cfg.tracingEs.url)) { next.TRACING_ES_URL = cfg.tracingEs.url; } @@ -167,6 +181,17 @@ export const envFromExportProfile = ( next.TRACING_EXPORTERS = JSON.stringify([{ http: { url: 'http://localhost:4318/v1/traces' } }]); } + // Agent Builder runs its own tracer provider (see register_tracing.ts); its + // spans are what trace-based evaluators query and they do NOT flow through + // telemetry.tracing.exporters. Kept separate so the built-in local-ES + // exporter stays intact and this only ADDS a remote destination. + if ( + Array.isArray(cfg.agentBuilderTracingExporters) && + cfg.agentBuilderTracingExporters.length > 0 + ) { + next.AGENT_BUILDER_TRACING_EXPORTERS = JSON.stringify(cfg.agentBuilderTracingExporters); + } + maybeSetGcsCredentialsEnv(cfg, next); return next; }; diff --git a/x-pack/platform/packages/shared/kbn-evals/src/cli/run_helpers.test.ts b/x-pack/platform/packages/shared/kbn-evals/src/cli/run_helpers.test.ts new file mode 100644 index 0000000000000..268797a3f5444 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals/src/cli/run_helpers.test.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ToolingLog } from '@kbn/tooling-log'; +import type { FlagsReader } from '@kbn/dev-cli-runner'; +import { resolveEvaluationConnectorId, evalRunFlags } from './run_helpers'; + +const log = new ToolingLog(); + +/** + * Minimal FlagsReader stub: only the accessors this code path touches. + */ +const flags = (values: Record): FlagsReader => + ({ + string: (name: string) => values[name] as string | undefined, + boolean: (name: string) => Boolean(values[name]), + } as unknown as FlagsReader); + +describe('resolveEvaluationConnectorId', () => { + const original = { ...process.env }; + + afterEach(() => { + process.env = { ...original }; + }); + + it('returns the connector id from the flag', async () => { + await expect( + resolveEvaluationConnectorId('/repo', log, flags({ 'evaluation-connector-id': 'eis-x' })) + ).resolves.toBe('eis-x'); + }); + + it('falls back to EVAL_CONNECTOR_ID', async () => { + process.env.EVAL_CONNECTOR_ID = 'eis-from-env'; + await expect(resolveEvaluationConnectorId('/repo', log, flags({}))).resolves.toBe( + 'eis-from-env' + ); + }); + + describe('--require-eis-judge', () => { + it('rejects a self-hosted judge that graded golden data before', async () => { + // This exact id appears 714 times in the persona-matrix golden index. + await expect( + resolveEvaluationConnectorId( + '/repo', + log, + flags({ + 'evaluation-connector-id': 'Qwen/Qwen3-Coder-30B-A3B-Instruct', + 'require-eis-judge': true, + }) + ) + ).rejects.toThrow(/not EIS-backed/); + }); + + it('rejects a LiteLLM alias', async () => { + await expect( + resolveEvaluationConnectorId( + '/repo', + log, + flags({ + 'evaluation-connector-id': 'LiteLLM Qwen3-Coder-30B-A3B-Instruct-AWQ', + 'require-eis-judge': true, + }) + ) + ).rejects.toThrow(/not EIS-backed/); + }); + + it('accepts an eis-* connector', async () => { + await expect( + resolveEvaluationConnectorId( + '/repo', + log, + flags({ + 'evaluation-connector-id': 'eis-anthropic-claude-4-6-sonnet', + 'require-eis-judge': true, + }) + ) + ).resolves.toBe('eis-anthropic-claude-4-6-sonnet'); + }); + + it('is opt-in: a non-EIS judge passes when the flag is absent', async () => { + await expect( + resolveEvaluationConnectorId( + '/repo', + log, + flags({ 'evaluation-connector-id': 'Qwen/Qwen3-Coder-30B-A3B-Instruct' }) + ) + ).resolves.toBe('Qwen/Qwen3-Coder-30B-A3B-Instruct'); + }); + + it('can be enabled from the environment for CI', async () => { + process.env.EVAL_REQUIRE_EIS_JUDGE = 'true'; + await expect( + resolveEvaluationConnectorId( + '/repo', + log, + flags({ 'evaluation-connector-id': 'NousResearch/Hermes-3-Llama-3.1-70B' }) + ) + ).rejects.toThrow(/not EIS-backed/); + }); + }); +}); + +describe('evalRunFlags', () => { + // resolveEvaluationConnectorId reads `require-eis-judge` via flagsReader.boolean(), + // and FlagsReader throws on any flag absent from the command's declaration. + // `start` and `run` both call it, so the flag must live in the SHARED set -- + // declaring it on `run` alone made every `evals start` die with + // "expected --require-eis-judge to be a boolean". + it.each(['require-eis-judge', 'skip-server', 'dry-run', 'skip-init'])( + 'declares %s so both start and run can read it', + (flag) => { + expect(evalRunFlags.boolean).toContain(flag); + expect(evalRunFlags.default).toHaveProperty(flag, false); + } + ); + + it('aliases judge to evaluation-connector-id', () => { + expect(evalRunFlags.alias).toMatchObject({ judge: 'evaluation-connector-id' }); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals/src/cli/run_helpers.ts b/x-pack/platform/packages/shared/kbn-evals/src/cli/run_helpers.ts index 068e52e9b6eb7..f079aae2e8a52 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/cli/run_helpers.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/cli/run_helpers.ts @@ -257,15 +257,28 @@ export const resolveEvaluationConnectorId = async ( const evaluationConnectorId = flagsReader.string('evaluation-connector-id') ?? process.env.EVAL_CONNECTOR_ID; - if (evaluationConnectorId) { - return evaluationConnectorId; + const resolved = + evaluationConnectorId ?? (isTTY() ? await promptForConnector(repoRoot, log) : undefined); + + if (!resolved) { + throw createFlagError('EVAL_CONNECTOR_ID is required. Set --evaluation-connector-id or env.'); } - if (isTTY()) { - return promptForConnector(repoRoot, log); + // Judged scores are only comparable across runs when the judge's weights and + // quantisation are pinned by the eval infra. Ad-hoc endpoints (LiteLLM + // aliases, HuggingFace repo paths, local 4-bit quants) have graded golden + // data before, which makes those scores unusable for ranking. + if ( + (flagsReader.boolean('require-eis-judge') || process.env.EVAL_REQUIRE_EIS_JUDGE === 'true') && + !isEisConnectorId(resolved) + ) { + throw createFlagError( + `Evaluation connector ${resolved} is not EIS-backed. Pass an eis-* connector, ` + + `or drop --require-eis-judge to allow an unpinned judge.` + ); } - throw createFlagError('EVAL_CONNECTOR_ID is required. Set --evaluation-connector-id or env.'); + return resolved; }; const isEisConnectorId = (id: string): boolean => id.startsWith('eis-'); @@ -478,7 +491,12 @@ export const evalRunFlags: FlagOptions = { 'evaluations-kbn-url', 'evaluations-kbn-api-key', ], - boolean: ['skip-server', 'dry-run', 'skip-init'], + boolean: ['skip-server', 'dry-run', 'skip-init', 'require-eis-judge'], alias: { model: 'project', judge: 'evaluation-connector-id' }, - default: { 'skip-server': false, 'dry-run': false, 'skip-init': false }, + default: { + 'skip-server': false, + 'dry-run': false, + 'skip-init': false, + 'require-eis-judge': false, + }, }; diff --git a/x-pack/platform/packages/shared/kbn-evals/src/evaluate.ts b/x-pack/platform/packages/shared/kbn-evals/src/evaluate.ts index 81758a643ca39..70773a162b256 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/evaluate.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/evaluate.ts @@ -35,6 +35,11 @@ import { EvalsClient } from './utils/evals_client'; import { EvaluatorApiClient } from './utils/evaluator_api_client'; import { getBuildkiteCiMetadataFromEnv } from './utils/ci_metadata'; import { getSpaceIdsFromEnv } from './utils/space_ids'; +import { + classifyIngestOutcome, + createIngestOutcome, + recordIngestFailure, +} from './utils/ingest_outcome'; import { buildIngestRequest, toScoreModel } from './utils/build_ingest_request'; import { buildModelFromConnector } from './utils/build_model_from_connector'; import type { @@ -285,6 +290,11 @@ export const evaluate = base.extend<{}, EvaluationSpecificWorkerFixtures>({ const gitMetadata = getGitMetadata(); const hostName = osHostname(); + // Score ingest is best-effort per example so one bad document cannot abort a + // long sweep. Tally outcomes so a run that exported nothing can be told apart + // from one that succeeded; see `classifyIngestOutcome`. + const ingestOutcome = createIngestOutcome(); + const executorClient = new KibanaEvalsClient({ log, model, @@ -322,15 +332,17 @@ export const evaluate = base.extend<{}, EvaluationSpecificWorkerFixtures>({ ingestRequests.map((ingestRequest) => evalsClient.ingestScores(ingestRequest)) ); for (const result of results) { + ingestOutcome.ingested += result.ingested; if (result.failed.length > 0) { + const reasons = result.failed.map((f) => f.reason).join(', '); + recordIngestFailure(ingestOutcome, reasons); log.warning( - `Score ingest partially failed for example ${event.exampleId}: ${result.failed - .map((f) => f.reason) - .join(', ')}` + `Score ingest partially failed for example ${event.exampleId}: ${reasons}` ); } } } catch (error) { + recordIngestFailure(ingestOutcome, String(error)); log.warning(`Score ingest failed for example ${event.exampleId}: ${error}`); } }, @@ -338,6 +350,14 @@ export const evaluate = base.extend<{}, EvaluationSpecificWorkerFixtures>({ await use(executorClient); + const ingestVerdict = classifyIngestOutcome(ingestOutcome); + if (ingestVerdict.kind === 'total-failure') { + throw new Error(ingestVerdict.message); + } + if (ingestVerdict.kind === 'partial') { + log.error(ingestVerdict.message); + } + const datasetRunResults = await executorClient.getDatasetRunResults(); if (datasetRunResults.length > 0 && executionId) { await reportModelScore(evalsClient, datasetRunResults[0].id, log, { diff --git a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/correctness/system_prompt.text b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/correctness/system_prompt.text index 22803f64775a4..a86493d10d78c 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/correctness/system_prompt.text +++ b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/correctness/system_prompt.text @@ -61,7 +61,9 @@ Follow these steps rigorously: 7. Enforce Strict Phrasing for Procedural Steps: For claims that describe specific UI elements or actions, the agent's wording must be an exact match or very close to the `[Ground Truth Response]`. Using a functionally different synonym should be marked as `CONTRADICTED`. For example, if the ground truth says "click the `Apply` button to see a preview," an agent claiming "click the `Save` button to see a preview" is a contradiction, as `Apply` and `Save` often have distinct functions in a user interface. [OUTPUT FORMAT] -Your final output MUST be a single, valid JSON object. Do not include any text or explanations outside of this JSON object. +Your final output MUST be a single tool call to `analyze` whose arguments are a single, valid JSON object matching the schema EXACTLY. Do not include any text or explanations outside of this tool call. + +CRITICAL: the `summary` argument MUST be a JSON OBJECT with exactly three string fields (`factual_accuracy_summary`, `relevance_summary`, `sequence_accuracy_summary`) — never a free-text string. The `analysis` argument MUST be an array of claim objects. Use ONLY the field names and enum values shown below; any other key or value makes the whole evaluation invalid. [JSON STRUCTURE] diff --git a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/chat_calls.ts b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/chat_calls.ts index 45bde9592ea62..3c2bbe2ba6953 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/chat_calls.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/chat_calls.ts @@ -8,7 +8,7 @@ import type { Client as EsClient } from '@elastic/elasticsearch'; import type { ToolingLog } from '@kbn/tooling-log'; import type { Evaluator } from '../../types'; -import { createTraceBasedEvaluator } from './factory'; +import { TRACE_INDEX_PATTERN, createTraceBasedEvaluator } from './factory'; /** * Counts the LLM round-trips in a trace. Agentic flows re-send the whole @@ -29,7 +29,7 @@ export function createChatCallsEvaluator({ config: { name: 'Chat Calls', direction: 'neutral', - buildQuery: (traceId) => `FROM traces-* + buildQuery: (traceId) => `FROM ${TRACE_INDEX_PATTERN} | WHERE trace.id == "${traceId}" AND attributes.gen_ai.operation.name == "chat" | STATS chat_calls = COUNT(*)`, diff --git a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/factory.test.ts b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/factory.test.ts index 5953511daac7e..943c17ce2f063 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/factory.test.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/factory.test.ts @@ -36,8 +36,9 @@ describe('createTraceBasedEvaluator', () => { debug: jest.fn(), } as any; - // Longer than the factory's full 62s backoff, so the retry budget is always drained. - exhaustRetries = () => jest.advanceTimersByTimeAsync(300_000); + // Longer than the factory's full backoff (~515s with 8 retries, 5s min / + // 120s max), so the retry budget is always drained. + exhaustRetries = () => jest.advanceTimersByTimeAsync(600_000); mockConfig = { name: 'Test Evaluator', @@ -205,6 +206,37 @@ describe('createTraceBasedEvaluator', () => { expect(result.metadata).toEqual({ incomplete: true }); }); + it('should not republish a rejected value as the retry fallback', async () => { + // The exact shape that published 25 fabricated `Tool Calls: 0` cells: TOOL spans are + // not indexed yet, so every attempt reads 0, `isResultValid` rejects all of them, and + // retries exhaust. The fallback must NOT resurrect the rejected 0 as a real score -- + // an unscored cell is honest, a zero next to a trace full of tool calls is not. + const query = mockEsClient.esql.query as jest.Mock; + query.mockResolvedValue({ + columns: [{ name: 'r', type: 'number' }], + values: [[0]], + }); + + const evaluator = createTraceBasedEvaluator({ + traceEsClient: mockEsClient, + log: mockLog, + config: { + ...mockConfig, + isResultValid: (result) => result !== null && result > 0, + }, + }); + + const promise = evaluateWith(evaluator, VALID_TRACE_ID); + await exhaustRetries(); + const result = await promise; + + // The rejected 0 must not survive as a score. With no validated value to fall back + // on, this is an honestly-unscored cell rather than `potentially_incomplete: 0`. + expect(result.score).not.toBe(0); + expect(result.score ?? null).toBeNull(); + expect(result.label).toBe('error'); + }); + it('should not log an error when a usable result is still returned', async () => { const query = mockEsClient.esql.query as jest.Mock; query.mockResolvedValue({ diff --git a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/factory.ts b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/factory.ts index 2aba7330c65df..c3610bc5dc82b 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/factory.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/factory.ts @@ -21,6 +21,24 @@ interface EsqlResponse { // "no value" case distinct from a legitimate score. const NOT_REPORTED = Symbol('notReported'); +/** + * Index pattern every trace-based evaluator reads spans from. + * + * `traces-*` alone is not enough. On a remote tracing cluster the API key is + * granted on the datastream BACKING indices (`.ds-traces-generic*`, + * `.ds-traces-agent_builder*`), which `traces-*` does not match -- so the + * pattern resolves to zero authorized indices and ES|QL reports + * `Unknown column [trace.id]`. That reads like "this model emitted no spans" + * but means "this key may not see any index under that pattern", and it + * silently nulled every trace metric (Tool Calls, Latency, tokens, + * SkillInvoked) for runs pointed at the golden cluster. + * + * Both forms are listed so the evaluators work against a local Scout cluster + * (plain datastream name) and a remote one (backing indices) without knowing + * which they were handed. + */ +export const TRACE_INDEX_PATTERN = 'traces-*,.ds-traces-*'; + export interface TraceBasedEvaluatorConfig { name: string; buildQuery: (traceId: string) => string; @@ -140,22 +158,38 @@ export function createTraceBasedEvaluator({ } const result = extractResult(response); - lastResult = result; const valid = isResultValid ? isResultValid(result) : result !== null; if (!valid) { + // A value rejected by a custom `isResultValid` must not become the retry + // fallback below, which republishes the last value seen. Remembering it would + // resurrect exactly what validation rejected: a `Tool Calls: 0` produced by + // unindexed TOOL spans is indistinguishable from a real zero, and 25 published + // cells read `0` next to a trace that plainly shows tools running. An unscored + // cell is honest; a fabricated zero is not. A plain `null` (no custom validator) + // still records, so the existing `potentially_incomplete` path is unchanged. + if (!isResultValid) { + lastResult = result; + } throw new Error(`${name} result looks incomplete (value: ${result}), retrying`); } + lastResult = result; + return result as number; } try { const score = await pRetry(fetchStats, { - retries: 5, + // The budget must exceed the OTel collector's flush lag (~3-7 min on + // sweep VMs): `Unknown column [trace.id]` means no traces-* index + // exists yet, and a 62s budget (5 x factor-2 retries) burns entirely + // inside that empty window, erroring every trace evaluator on + // otherwise healthy runs (observed on Azure shard 1/2, run 9). + retries: 8, factor: 2, - minTimeout: 2000, - maxTimeout: 60000, + minTimeout: 5000, + maxTimeout: 120000, onFailedAttempt: (error) => { log.debug( `${name} query failed on attempt ${error.attemptNumber}, ${error.retriesLeft} retries left (traceId: ${traceId}): ${error.message}` diff --git a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/index.ts b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/index.ts index 72d2297865f0f..b3fed641c9671 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/index.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/index.ts @@ -5,7 +5,11 @@ * 2.0. */ -export { createTraceBasedEvaluator, type TraceBasedEvaluatorConfig } from './factory'; +export { + createTraceBasedEvaluator, + TRACE_INDEX_PATTERN, + type TraceBasedEvaluatorConfig, +} from './factory'; export { createInputTokensEvaluator, createOutputTokensEvaluator, diff --git a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/index_pattern.test.ts b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/index_pattern.test.ts new file mode 100644 index 0000000000000..c5bff94d6d1b3 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/index_pattern.test.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Client as EsClient } from '@elastic/elasticsearch'; +import type { ToolingLog } from '@kbn/tooling-log'; +import { TRACE_INDEX_PATTERN } from './factory'; +import { createChatCallsEvaluator } from './chat_calls'; +import { createLatencyEvaluator } from './latency'; +import { createSkillInvocationEvaluator } from './skill_invocation'; +import { createCachedTokensEvaluator } from './tokens'; +import { createToolCallsEvaluator } from './tool_calls'; + +const VALID_TRACE_ID = '0af7651916cd43dd8448eb211c80319c'; + +/** + * Regression guard for the index pattern the trace evaluators query. + * + * A remote tracing cluster grants the eval API key on datastream BACKING + * indices (`.ds-traces-generic*`, `.ds-traces-agent_builder*`). A bare + * `traces-*` matches none of them, so ES|QL resolves zero authorized indices + * and fails with `Unknown column [trace.id]` -- which looks like "the model + * emitted no spans" but is really "this key cannot see that pattern". That + * silently nulled every trace metric for every model on the golden cluster. + */ +describe('trace evaluator index pattern', () => { + let mockEsClient: jest.Mocked; + let mockLog: jest.Mocked; + + beforeEach(() => { + mockEsClient = { + esql: { query: jest.fn().mockResolvedValue({ columns: [], values: [] }) }, + } as unknown as jest.Mocked; + mockLog = { + debug: jest.fn(), + warning: jest.fn(), + error: jest.fn(), + info: jest.fn(), + } as unknown as jest.Mocked; + }); + + const queriesFor = async (evaluator: { evaluate: Function }) => { + // The assertion only needs the query TEXT, which is fixed at the first + // call. Awaiting the full evaluate() would sit through the factory's + // retry/backoff loop, so kick it off, let the first query issue, and + // inspect the recorded call instead. + const pending = evaluator + .evaluate({ + input: {}, + output: { traceId: VALID_TRACE_ID }, + expected: {}, + metadata: {}, + }) + .catch(() => undefined); + + await Promise.resolve(); + await Promise.resolve(); + + const queries = (mockEsClient.esql.query as jest.Mock).mock.calls.map( + ([args]) => args.query as string + ); + + void pending; + return queries; + }; + + const evaluators = () => + [ + ['Chat Calls', createChatCallsEvaluator({ traceEsClient: mockEsClient, log: mockLog })], + ['Latency', createLatencyEvaluator({ traceEsClient: mockEsClient, log: mockLog })], + [ + 'SkillInvoked', + createSkillInvocationEvaluator({ + traceEsClient: mockEsClient, + log: mockLog, + skillName: 'some-skill', + }), + ], + ['Cached Tokens', createCachedTokensEvaluator({ traceEsClient: mockEsClient, log: mockLog })], + ['Tool Calls', createToolCallsEvaluator({ traceEsClient: mockEsClient, log: mockLog })], + ] as Array<[string, { evaluate: Function }]>; + + it('includes the datastream backing indices so a remote-cluster key can resolve spans', () => { + expect(TRACE_INDEX_PATTERN).toContain('.ds-traces-*'); + }); + + it('still covers the plain datastream name for local Scout clusters', () => { + expect(TRACE_INDEX_PATTERN).toContain('traces-*'); + }); + + describe.each(evaluators().map(([name]) => name))('%s', (name) => { + it('queries the shared pattern and never a bare traces-*', async () => { + const evaluator = evaluators().find(([n]) => n === name)![1]; + const queries = await queriesFor(evaluator); + + expect(queries.length).toBeGreaterThan(0); + + for (const query of queries) { + expect(query).toContain(`FROM ${TRACE_INDEX_PATTERN}`); + // The bug: a FROM clause naming only the un-dotted pattern. + expect(query).not.toMatch(/FROM\s+traces-\*\s*$/m); + } + }); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/index_pattern_repo.test.ts b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/index_pattern_repo.test.ts new file mode 100644 index 0000000000000..78af00870e411 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/index_pattern_repo.test.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { execFileSync } from 'child_process'; +import path from 'path'; + +/** + * Repo-wide guard for the trace index pattern. + * + * The per-evaluator test next to this one only exercises the evaluators built by + * the shared factory. Suites are free to hand-roll their own ES|QL against + * `traceEsClient`, and two of them did -- the persona-matrix `SkillInvoked` + * check and the endpoint suite's skill-invocation evaluator both kept a bare + * `FROM traces-*` after the shared package was fixed. On the golden cluster the + * eval key is granted on datastream BACKING indices, so a bare `traces-*` + * resolves zero authorized indices and ES|QL reports `Unknown column + * [trace.id]`. Those cells then read as "the model emitted no spans". + * + * Grep the sources instead of the evaluator objects so a new suite that writes + * its own query is covered the day it lands. + */ +describe('trace ES|QL index pattern (repo-wide)', () => { + const repoRoot = path.resolve(__dirname, '../../../../../../../..'); + + const matches = (): string[] => { + try { + // -F: the pattern is literal. Restrict to the eval packages so an unrelated + // doc or example query cannot fail this suite. + const out = execFileSync( + 'grep', + [ + '-rn', + '-F', + 'FROM traces-*', + '--include=*.ts', + 'x-pack/platform/packages/shared/kbn-evals/src', + 'x-pack/solutions/security/packages', + ], + { cwd: repoRoot, encoding: 'utf8' } + ); + return out.split('\n').filter(Boolean); + } catch (error: unknown) { + // grep exits 1 with no output when nothing matches, which is the pass case. + const status = (error as { status?: number }).status; + if (status === 1) { + return []; + } + throw error; + } + }; + + it('has no bare FROM traces-* left in eval sources', () => { + // A line is only a violation when the pattern is NOT the shared constant, + // i.e. the literal is followed by something other than `,.ds-traces-*`. + // Test files are excluded: they legitimately spell out a bare pattern as a + // fixture or, like this file, quote it to describe the bug. + const offenders = matches() + .filter((line) => !line.includes('FROM traces-*,.ds-traces-*')) + .filter((line) => !/\.test\.ts:/.test(line)); + + expect(offenders).toEqual([]); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/latency.ts b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/latency.ts index f0802ed8f5700..ef2a650cd648a 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/latency.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/latency.ts @@ -8,7 +8,7 @@ import type { Client as EsClient } from '@elastic/elasticsearch'; import type { ToolingLog } from '@kbn/tooling-log'; import type { Evaluator } from '../../types'; -import { createTraceBasedEvaluator } from './factory'; +import { TRACE_INDEX_PATTERN, createTraceBasedEvaluator } from './factory'; export function createLatencyEvaluator({ traceEsClient, @@ -23,7 +23,7 @@ export function createLatencyEvaluator({ config: { name: 'Latency', direction: 'minimize', - buildQuery: (traceId) => `FROM traces-* + buildQuery: (traceId) => `FROM ${TRACE_INDEX_PATTERN} | WHERE trace.id == "${traceId}" | STATS total_duration_ns = MAX(duration) | EVAL latency_seconds = TO_DOUBLE(total_duration_ns) / 1000000000 @@ -63,7 +63,7 @@ export function createSpanLatencyEvaluator({ config: { name: 'Latency', direction: 'minimize', - buildQuery: (traceId) => `FROM traces-* + buildQuery: (traceId) => `FROM ${TRACE_INDEX_PATTERN} | WHERE trace.id == "${traceId}" AND ${spanFilter} | STATS total_duration_ns = SUM(duration) | EVAL latency_seconds = TO_DOUBLE(total_duration_ns) / 1000000000 diff --git a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/skill_invocation.test.ts b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/skill_invocation.test.ts index 2e1e25f332b18..3b0079d55e7e6 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/skill_invocation.test.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/skill_invocation.test.ts @@ -272,7 +272,7 @@ describe('createSkillInvocationEvaluator', () => { (mockEsClient.esql.query as jest.Mock).mockRejectedValue(new Error('Network failure')); const promise = evaluateWith(evaluator, VALID_TRACE_ID); - await jest.advanceTimersByTimeAsync(300_000); + await jest.advanceTimersByTimeAsync(600_000); const result = await promise; expect(result.label).toBe('error'); diff --git a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/skill_invocation.ts b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/skill_invocation.ts index 021ef6f814438..9e5e1fae3f1f7 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/skill_invocation.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/skill_invocation.ts @@ -8,7 +8,7 @@ import type { Client as EsClient } from '@elastic/elasticsearch'; import type { ToolingLog } from '@kbn/tooling-log'; import type { Evaluator } from '../../types'; -import { createTraceBasedEvaluator } from './factory'; +import { TRACE_INDEX_PATTERN, createTraceBasedEvaluator } from './factory'; const VALID_SKILL_NAME = /^[a-zA-Z0-9_-]+$/; @@ -33,7 +33,7 @@ export function createSkillInvocationEvaluator({ config: { name: `Skill Invoked (${skillName})`, direction: 'maximize', - buildQuery: (traceId) => `FROM traces-* + buildQuery: (traceId) => `FROM ${TRACE_INDEX_PATTERN} | WHERE trace.id == "${traceId}" | STATS total_spans = COUNT(*), diff --git a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/tokens.test.ts b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/tokens.test.ts index 92df36614ece8..46d86ef609733 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/tokens.test.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/tokens.test.ts @@ -136,12 +136,13 @@ describe('createCachedTokensEvaluator', () => { query.mockRejectedValue(new Error('verification_exception: Unknown column [typo_column]')); const promise = evaluate(); - await jest.advanceTimersByTimeAsync(300_000); + await jest.advanceTimersByTimeAsync(600_000); const result = await promise; expect(result.label).toBe('error'); - // Only the main query runs; the probe is never reached for an unrelated column. - expect(query).toHaveBeenCalledTimes(6); + // Only the main query runs (8 retries + initial attempt); the probe is + // never reached for an unrelated column. + expect(query).toHaveBeenCalledTimes(9); expect(mockLog.error).toHaveBeenCalled(); }); }); diff --git a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/tokens.ts b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/tokens.ts index 1a50545362276..e7c942a806f03 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/tokens.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/tokens.ts @@ -8,7 +8,7 @@ import type { Client as EsClient } from '@elastic/elasticsearch'; import type { ToolingLog } from '@kbn/tooling-log'; import type { Evaluator } from '../../types'; -import { createTraceBasedEvaluator } from './factory'; +import { TRACE_INDEX_PATTERN, createTraceBasedEvaluator } from './factory'; // The ES client folds the root-cause reason into `message`. Pinned to this one column: any other // unknown column is a query bug and must stay a hard failure. @@ -29,7 +29,7 @@ export function createOutputTokensEvaluator({ name: 'Output Tokens', direction: 'minimize', // TO_LONG resolves union types (integer vs long across trace index generations). - buildQuery: (traceId) => `FROM traces-* + buildQuery: (traceId) => `FROM ${TRACE_INDEX_PATTERN} | WHERE trace.id == "${traceId}" | STATS output_tokens = SUM(TO_LONG(attributes.gen_ai.usage.output_tokens))`, @@ -58,7 +58,7 @@ export function createInputTokensEvaluator({ name: 'Input Tokens', direction: 'minimize', // TO_LONG resolves union types (integer vs long across trace index generations). - buildQuery: (traceId) => `FROM traces-* + buildQuery: (traceId) => `FROM ${TRACE_INDEX_PATTERN} | WHERE trace.id == "${traceId}" | STATS input_tokens = SUM(TO_LONG(attributes.gen_ai.usage.input_tokens))`, @@ -88,7 +88,7 @@ export function createCachedTokensEvaluator({ direction: 'neutral', // `input_tokens` is a liveness probe: providers that never report caching (most EIS models) // omit cache_read entirely, which otherwise looks like a trace that has not finished indexing. - buildQuery: (traceId) => `FROM traces-* + buildQuery: (traceId) => `FROM ${TRACE_INDEX_PATTERN} | WHERE trace.id == "${traceId}" | STATS cached_tokens = SUM(TO_LONG(attributes.gen_ai.usage.cache_read.input_tokens)), @@ -108,7 +108,7 @@ export function createCachedTokensEvaluator({ notReportedProbe: { matchesQueryError: (error) => error instanceof Error && CACHE_READ_COLUMN_MISSING.test(error.message), - buildQuery: (traceId) => `FROM traces-* + buildQuery: (traceId) => `FROM ${TRACE_INDEX_PATTERN} | WHERE trace.id == "${traceId}" | STATS input_tokens = SUM(TO_LONG(attributes.gen_ai.usage.input_tokens))`, isTraceComplete: (response) => { diff --git a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/tool_calls.ts b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/tool_calls.ts index 636ac3ac14292..981b74f1af447 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/tool_calls.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/evaluators/trace_based/tool_calls.ts @@ -8,7 +8,7 @@ import type { Client as EsClient } from '@elastic/elasticsearch'; import type { ToolingLog } from '@kbn/tooling-log'; import type { Evaluator } from '../../types'; -import { createTraceBasedEvaluator } from './factory'; +import { TRACE_INDEX_PATTERN, createTraceBasedEvaluator } from './factory'; export function createToolCallsEvaluator({ traceEsClient, @@ -23,13 +23,21 @@ export function createToolCallsEvaluator({ config: { name: 'Tool Calls', direction: 'neutral', - buildQuery: (traceId) => `FROM traces-* + buildQuery: (traceId) => `FROM ${TRACE_INDEX_PATTERN} | WHERE trace.id == "${traceId}" AND attributes.elastic.inference.span.kind == "TOOL" | STATS tool_calls = COUNT(*)`, extractResult: (response) => { return response.values[0][0] as number; }, + // A count of 0 is indistinguishable from "the TOOL spans are not indexed + // yet": this evaluator reads OTel traces, not the agent's tool trail, so + // it races span ingestion. Treating that race as a real zero published 19 + // cells for 4.5-sonnet reading `Tool Calls: 0` while their trace clearly + // showed load_skill and platform.core.cases.manage having run. Retry + // instead, and let the factory fall back to unreported if the count is + // still 0 once the trace is complete. + isResultValid: (result) => result !== null && result > 0, }, }); } diff --git a/x-pack/platform/packages/shared/kbn-evals/src/kibana_evals_executor/client.test.ts b/x-pack/platform/packages/shared/kbn-evals/src/kibana_evals_executor/client.test.ts index d39140e5376b7..de0fb1f45e14c 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/kibana_evals_executor/client.test.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/kibana_evals_executor/client.test.ts @@ -754,4 +754,103 @@ describe('KibanaEvalsClient', () => { expect(exp.evaluationRuns).toHaveLength(4); }); }); + + describe('error isolation', () => { + const dataset: EvaluationDataset = { + name: 'ds', + description: 'desc', + examples: [ + { id: 'ex-1', input: { q: 1 }, output: { expected: 1 } }, + { id: 'ex-2', input: { q: 2 }, output: { expected: 2 } }, + ], + }; + + const evaluators: Array> = [ + { + name: 'AlwaysOne', + kind: 'CODE', + direction: 'maximize', + evaluate: async () => ({ score: 1 }), + }, + ]; + + it('records a task failure, keeps other examples, and rejects after the run', async () => { + const client = createClient({ repetitions: 1 }); + + await expect( + client.runExperiment( + { + datasets: [dataset], + task: async (example) => { + if ((example.input as { q: number }).q === 2) { + throw new Error('converse 500: Request timed out'); + } + return { value: 1 }; + }, + }, + evaluators + ) + ).rejects.toThrow('errored run: exampleIndex=1, repetition=0: converse 500'); + + // The partial result is still accumulated, with the failure explicit on the run + const [exp] = await client.getDatasetRunResults(); + const recordedRuns = Object.values(exp.runs); + expect(recordedRuns).toHaveLength(2); + const failed = recordedRuns.find((run) => run.exampleIndex === 1); + expect(failed?.error).toBe('converse 500: Request timed out'); + expect(failed?.output).toBeNull(); + + // Only the healthy example was evaluated + expect(exp.evaluationRuns).toHaveLength(1); + expect(exp.evaluationRuns[0].exampleId).toBe('ex-1'); + }); + + it('records an evaluator failure as an explicit error result and completes', async () => { + const client = createClient({ repetitions: 1 }); + + const flakyJudge: Evaluator = { + name: 'FlakyJudge', + kind: 'LLM', + direction: 'maximize', + evaluate: async ({ input }) => { + if ((input as { q: number }).q === 2) { + throw new Error('toolValidationError'); + } + return { score: 1 }; + }, + }; + + const [exp] = await client.runExperiment( + { datasets: [dataset], task: async () => ({ value: 1 }) }, + [flakyJudge] + ); + + expect(exp.evaluationRuns).toHaveLength(2); + const errorRun = exp.evaluationRuns.find((run) => run.exampleId === 'ex-2'); + expect(errorRun?.result).toEqual({ + score: null, + label: 'error', + explanation: 'Evaluator threw: toolValidationError', + }); + }); + + it('fails the experiment when an evaluator errors on every run', async () => { + const client = createClient({ repetitions: 1 }); + + const brokenJudge: Evaluator = { + name: 'BrokenJudge', + kind: 'LLM', + direction: 'maximize', + evaluate: async () => { + throw new Error('connector dead'); + }, + }; + + await expect( + client.runExperiment({ datasets: [dataset], task: async () => ({ value: 1 }) }, [ + brokenJudge, + ]) + ).rejects.toThrow('evaluator "BrokenJudge" failed on every run'); + }); + }); }); diff --git a/x-pack/platform/packages/shared/kbn-evals/src/kibana_evals_executor/client.ts b/x-pack/platform/packages/shared/kbn-evals/src/kibana_evals_executor/client.ts index c6846611e1b29..7894d5037c305 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/kibana_evals_executor/client.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/kibana_evals_executor/client.ts @@ -184,6 +184,8 @@ export class KibanaEvalsClient implements EvalsExecutorClient { const evaluationRuns: DatasetRunResult['evaluationRuns'] = []; const runs: DatasetRunResult['runs'] = {}; + const erroredRuns: string[] = []; + const evaluatorAttempts = new Map(); const runJobs: Array> = []; @@ -201,23 +203,51 @@ export class KibanaEvalsClient implements EvalsExecutorClient { `🔧 Running task "${resolvedDataset.name}" on dataset "${datasetId}" (exampleIndex=${exampleIndex}, repetition=${rep})` ); - const { taskOutput, traceId } = await withTaskSpan( - resolvedDataset.name, - { - attributes: { - 'dataset.name': resolvedDataset.name, - 'dataset.id': datasetId, + const runTask = () => + withTaskSpan( + resolvedDataset.name, + { + attributes: { + 'dataset.name': resolvedDataset.name, + 'dataset.id': datasetId, + }, }, - }, - async () => { - const _traceId = getCurrentTraceId(); - const _taskOutput = await task(example); - return { - taskOutput: _taskOutput, - traceId: _traceId, - }; - } - ); + async () => { + const _traceId = getCurrentTraceId(); + const _taskOutput = await task(example); + return { + taskOutput: _taskOutput, + traceId: _traceId, + }; + } + ); + + let taskSpanResult: Awaited>; + try { + taskSpanResult = await runTask(); + } catch (error) { + // One example failing (e.g. converse 500) must not abort the whole + // experiment: record the run as errored, skip its evaluators, and + // keep measuring the remaining examples. The aggregate check after + // all jobs complete still fails the run so this never goes green. + const message = error instanceof Error ? error.message : String(error); + this.options.log.error( + `❌ Task failed on dataset "${datasetId}" (exampleIndex=${exampleIndex}, repetition=${rep}); continuing with remaining examples: ${message}` + ); + runs[runKey] = { + exampleIndex, + repetition: rep, + input: example.input, + expected: example.output ?? null, + metadata: example.metadata ?? {}, + output: null, + traceId: null, + error: message, + }; + erroredRuns.push(`exampleIndex=${exampleIndex}, repetition=${rep}: ${message}`); + return; + } + const { taskOutput, traceId } = taskSpanResult; // Prefer the trace id the task itself surfaced (e.g. converse's response // trace_id) over the eval client's own task-span trace id. See #276308. @@ -242,40 +272,67 @@ export class KibanaEvalsClient implements EvalsExecutorClient { this.options.log.info( `🧠 Evaluating run (exampleIndex=${exampleIndex}, repetition=${rep}) with evaluator "${evaluator.name}"` ); - const { result, evaluatorTraceId } = await withEvaluatorSpan( - evaluator.name, - {}, - async () => { - const _traceId = getCurrentTraceId(); - const _result = await evaluator.evaluate({ - input: example.input, - output: { - ...taskOutput, - traceId: taskOrClientTraceId, - }, - expected: example.output ?? null, - metadata: example.metadata ?? {}, - }); - return { - result: _result, - evaluatorTraceId: _traceId, - }; - } - ); - this.options.log.info( - `✅ Evaluator "${evaluator.name}" on run (exampleIndex=${exampleIndex}, repetition=${rep}) completed` - ); - return { - evaluatorName: evaluator.name, - direction: evaluator.direction, - result, - evaluatorTraceId, - kind: evaluator.kind, - // Read after `evaluate` so evaluators that learn their model from - // the `_evaluate` response have it by now. - model: evaluator.getModel?.(), - version: evaluator.getVersion?.(), - }; + const attempts = evaluatorAttempts.get(evaluator.name) ?? { errors: 0, total: 0 }; + attempts.total += 1; + evaluatorAttempts.set(evaluator.name, attempts); + try { + const { result, evaluatorTraceId } = await withEvaluatorSpan( + evaluator.name, + {}, + async () => { + const _traceId = getCurrentTraceId(); + const _result = await evaluator.evaluate({ + input: example.input, + output: { + ...taskOutput, + traceId: taskOrClientTraceId, + }, + expected: example.output ?? null, + metadata: example.metadata ?? {}, + }); + return { + result: _result, + evaluatorTraceId: _traceId, + }; + } + ); + this.options.log.info( + `✅ Evaluator "${evaluator.name}" on run (exampleIndex=${exampleIndex}, repetition=${rep}) completed` + ); + return { + evaluatorName: evaluator.name, + result, + evaluatorTraceId, + kind: evaluator.kind, + direction: evaluator.direction, + // Read after `evaluate` so evaluators that learn their model from + // the `_evaluate` response have it by now. + model: evaluator.getModel?.(), + version: evaluator.getVersion?.(), + }; + } catch (error) { + // A single evaluator failing (e.g. an LLM judge's inference call + // erroring out) must not take down the run's other measurements: + // record an explicit error result so the gap is visible in the + // exported scores instead of silently dropping the document. + const message = error instanceof Error ? error.message : String(error); + attempts.errors += 1; + this.options.log.error( + `❌ Evaluator "${evaluator.name}" failed on run (exampleIndex=${exampleIndex}, repetition=${rep}): ${message}` + ); + return { + evaluatorName: evaluator.name, + result: { + score: null, + label: 'error', + explanation: `Evaluator threw: ${message}`, + }, + evaluatorTraceId: undefined, + kind: evaluator.kind, + model: evaluator.getModel?.(), + version: evaluator.getVersion?.(), + }; + } }) ); @@ -295,7 +352,7 @@ export class KibanaEvalsClient implements EvalsExecutorClient { experimentRunId: runKey, traceId: evaluatorTraceId, exampleId: example.id, - direction, + direction: direction ?? 'neutral', kind, ...(model && { model }), }; @@ -325,7 +382,6 @@ export class KibanaEvalsClient implements EvalsExecutorClient { } await Promise.all(runJobs); - this.options.log.info(`✅ Experiment ${experimentId} completed`); const result: DatasetRunResult = { id: experimentId, @@ -343,6 +399,31 @@ export class KibanaEvalsClient implements EvalsExecutorClient { }; this.datasetRunResults.push(result); + + const fullyBrokenEvaluators = [...evaluatorAttempts.entries()] + .filter(([, { errors, total }]) => total > 0 && errors === total) + .map(([name]) => name); + if (erroredRuns.length > 0 || fullyBrokenEvaluators.length > 0) { + // Every completed measurement is recorded and exported by now; failing the + // experiment afterwards keeps errored examples visible instead of either + // aborting the whole run (previous behavior) or silently going green. An + // evaluator that failed on every single run is a broken instrument, not a + // measurement, so that fails the experiment too. + const details = [ + ...erroredRuns.map((run) => `errored run: ${run}`), + ...fullyBrokenEvaluators.map( + (name) => + `evaluator "${name}" failed on every run (broken instrument, not a measurement)` + ), + ]; + throw new Error( + `Experiment "${experimentName}" finished with ${ + details.length + } failure(s):\n${details.join('\n')}` + ); + } + + this.options.log.info(`✅ Experiment ${experimentId} completed`); return result; }); } diff --git a/x-pack/platform/packages/shared/kbn-evals/src/types.ts b/x-pack/platform/packages/shared/kbn-evals/src/types.ts index 2f8af04297b4e..d3173e32f5e90 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/types.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/types.ts @@ -181,6 +181,8 @@ export interface TaskRun { metadata: Example['metadata']; output: TaskOutput; traceId?: string | null; + /** Set when the task itself threw; evaluators are skipped for such runs. */ + error?: string; } export interface EvaluationRun { diff --git a/x-pack/platform/packages/shared/kbn-evals/src/utils/evals_client.ts b/x-pack/platform/packages/shared/kbn-evals/src/utils/evals_client.ts index 8bb6fd837fd09..4a80ccd796847 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/utils/evals_client.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/utils/evals_client.ts @@ -18,11 +18,13 @@ import { EVALS_EXPERIMENT_SCORES_URL, EVALS_EXPERIMENT_URL, EVALS_EXPERIMENTS_URL, + EVALS_EXAMPLE_SCORES_URL, EVALS_SCORES_URL, GetEvaluationDatasetResponse, GetEvaluationExperimentResponse, GetEvaluationExperimentScoresResponse, GetEvaluationExperimentsResponse, + GetExampleScoresResponse, IngestScoresRequestBody, IngestScoresResponse, MAX_SCORES_PER_QUERY, @@ -31,6 +33,7 @@ import { UpsertEvaluationDatasetResponse, getDatasetId, type DatasetMaturity, + type EvaluationExperimentSummary, type EvaluationScoreDocument, type IngestScoresRequestBodyInput, type Model as EvalsModel, @@ -188,6 +191,18 @@ const buildExperimentQuery = (options?: GetExperimentFilters) => ({ const VERSIONED_HEADERS = { 'elastic-api-version': API_VERSIONS.internal.v1 }; +export interface ListExperimentsFilters { + suiteId?: string; + taskModelId?: string; + branch?: string; + datasetId?: string; + buildId?: string; + /** Maximum number of experiments to return (newest first). Defaults to and capped at 100 (the route's per_page maximum). */ + limit?: number; +} + +export const MAX_LIST_EXPERIMENTS = 100; + export class EvalsClient { /** The spaces this run writes to, in the order they were listed. */ private readonly spaceIds: string[]; @@ -300,6 +315,51 @@ export class EvalsClient { } } + /** + * Retrieves scores for a single example across all experiments that include + * it. Unlike {@link getExperimentScores}, the response is NOT stripped of + * unbounded fields (`task.output`, `example.input`, `example.metadata`), + * because this route does not apply `_source_excludes`. + */ + async getExampleScores( + exampleId: string, + filters?: { executionId?: string; modelId?: string } + ): Promise { + try { + const query: Record = {}; + if (filters?.executionId) { + query.execution_id = filters.executionId; + } + if (filters?.modelId) { + query.model_id = filters.modelId; + } + const response = await this.kbnClient.request({ + path: this.path( + EVALS_EXAMPLE_SCORES_URL.replace('{exampleId}', encodeURIComponent(exampleId)) + ), + method: 'GET', + headers: VERSIONED_HEADERS, + ...(Object.keys(query).length > 0 ? { query } : {}), + }); + const parsed = GetExampleScoresResponse.parse(getResponseData(response)); + + if (parsed.total > MAX_SCORES_PER_QUERY) { + throw new Error( + `Example ${exampleId} returned ${parsed.total} scores, which exceeds MAX_SCORES_PER_QUERY (${MAX_SCORES_PER_QUERY})` + ); + } + + return parsed.scores; + } catch (error: unknown) { + this.log.error( + `Failed to retrieve scores for example ID ${exampleId}: ${ + error instanceof Error ? error.message : String(error) + }` + ); + return []; + } + } + /** * Creates or updates a dataset and returns the id the server assigned it. Ids * derive from the owning space, so the caller can't compute one. @@ -577,4 +637,28 @@ export class EvalsClient { return undefined; } } + + async listExperiments(filters?: ListExperimentsFilters): Promise { + const limit = Math.min(filters?.limit ?? MAX_LIST_EXPERIMENTS, MAX_LIST_EXPERIMENTS); + const response = await this.kbnClient.request({ + path: EVALS_EXPERIMENTS_URL, + method: 'GET', + query: { + suite_id: filters?.suiteId, + model_id: filters?.taskModelId, + branch: filters?.branch, + dataset_id: filters?.datasetId, + build_id: filters?.buildId, + page: 1, + per_page: limit, + }, + headers: VERSIONED_HEADERS, + }); + + const parsed = GetEvaluationExperimentsResponse.parse(getResponseData(response)); + if (!filters?.branch) { + return parsed.experiments; + } + return parsed.experiments.filter((experiment) => experiment.git_branch === filters.branch); + } } diff --git a/x-pack/platform/packages/shared/kbn-evals/src/utils/evaluations_kbn_client.test.ts b/x-pack/platform/packages/shared/kbn-evals/src/utils/evaluations_kbn_client.test.ts index 3078dc9a6f2be..2aa8205be6ead 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/utils/evaluations_kbn_client.test.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/utils/evaluations_kbn_client.test.ts @@ -79,6 +79,21 @@ describe('withKbnClientApiKeyAuth', () => { }); }); describe('getEvaluationsKbnClient', () => { + let originalEvalKbnUrl: string | undefined; + + beforeEach(() => { + originalEvalKbnUrl = process.env.EVAL_KBN_URL; + delete process.env.EVAL_KBN_URL; + }); + + afterEach(() => { + if (originalEvalKbnUrl === undefined) { + delete process.env.EVAL_KBN_URL; + } else { + process.env.EVAL_KBN_URL = originalEvalKbnUrl; + } + }); + it('returns default kbnClient when EVAL_KBN_URL is not set', () => { const defaultKbnClient = createMockKbnClient(); const createKbnClient = jest.fn(); diff --git a/x-pack/platform/packages/shared/kbn-evals/src/utils/evaluations_kbn_client.ts b/x-pack/platform/packages/shared/kbn-evals/src/utils/evaluations_kbn_client.ts index 01688e7100050..d552d08e260f4 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/utils/evaluations_kbn_client.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/utils/evaluations_kbn_client.ts @@ -10,6 +10,42 @@ import type { KbnClient } from '@kbn/kbn-client'; import { KbnClient as TestKbnClient } from '@kbn/kbn-client'; import { wrapKbnClientWithRetries } from './kbn_client_with_retries'; +import { EvalsClient } from './evals_client'; + +/** + * Default target used when no evaluations Kibana URL is provided. Keeps + * `@kbn/kbn-client` an implementation detail of `@kbn/evals` so callers (e.g. + * `@kbn/evals-extensions`) can build a client without depending on it directly. + */ +export const DEFAULT_EVALUATIONS_KBN_URL = 'http://elastic:changeme@localhost:5601'; + +export interface CreateEvaluationsEvalsClientParams { + log: ToolingLog; + /** Evaluations Kibana URL (falls back to {@link DEFAULT_EVALUATIONS_KBN_URL}). */ + url?: string; + /** API key used to authenticate against a non-local target. */ + apiKey?: string; +} + +/** + * Thin factory wiring the default {@link TestKbnClient} through + * {@link getEvaluationsKbnClient} (URL/API-key/version/retry handling). + */ +export function createEvaluationsEvalsClient({ + log, + url, + apiKey, +}: CreateEvaluationsEvalsClientParams): EvalsClient { + const defaultKbnClient = new TestKbnClient({ log, url: DEFAULT_EVALUATIONS_KBN_URL }); + const kbnClient = getEvaluationsKbnClient({ + kbnClient: defaultKbnClient, + log, + evaluationsKbnUrl: url, + evaluationsKbnApiKey: apiKey, + }); + return new EvalsClient(kbnClient, log); +} + export interface GetEvaluationsKbnClientParams { kbnClient: KbnClient; log: ToolingLog; diff --git a/x-pack/platform/packages/shared/kbn-evals/src/utils/http_handler_from_kbn_client.test.ts b/x-pack/platform/packages/shared/kbn-evals/src/utils/http_handler_from_kbn_client.test.ts new file mode 100644 index 0000000000000..888f6a0b80208 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals/src/utils/http_handler_from_kbn_client.test.ts @@ -0,0 +1,236 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ToolingLog } from '@kbn/tooling-log'; +import { httpHandlerFromKbnClient } from './http_handler_from_kbn_client'; + +/** + * A dead transport arrives with no HTTP status, so the retry predicate used to + * skip it entirely: glm-5-2 lost 19 of 21 examples 58 minutes into a sweep when + * Kibana stopped answering and every remaining call failed as + * `Status: N/A, Cause: fetch failed`. + */ +describe('httpHandlerFromKbnClient transport retries', () => { + const log = new ToolingLog({ level: 'silent', writeTo: process.stdout }); + + const makeError = (message: string, status?: number) => + Object.assign(new Error(message), { status, headers: undefined }); + + const kbnClient = (request: jest.Mock) => ({ request } as never); + + beforeEach(() => { + process.env.KBN_EVALS_HTTP_RETRIES = '2'; + }); + + afterEach(() => { + delete process.env.KBN_EVALS_HTTP_RETRIES; + delete process.env.KBN_EVALS_HTTP_TIMEOUT_MS; + }); + + it('retries a status-less "fetch failed" and succeeds', async () => { + const request = jest + .fn() + .mockRejectedValueOnce(makeError('request failed -- Status: N/A, Cause: fetch failed')) + .mockResolvedValueOnce({ data: { ok: true }, status: 200, statusText: 'OK', headers: {} }); + + const handler = httpHandlerFromKbnClient({ kbnClient: kbnClient(request), log }); + await handler('/api/agent_builder/converse', { method: 'POST' }); + + expect(request).toHaveBeenCalledTimes(2); + }); + + it('retries a socket hang up', async () => { + const request = jest + .fn() + .mockRejectedValueOnce(makeError('socket hang up')) + .mockResolvedValueOnce({ data: {}, status: 200, statusText: 'OK', headers: {} }); + + const handler = httpHandlerFromKbnClient({ kbnClient: kbnClient(request), log }); + await handler('/api/agent_builder/converse', { method: 'POST' }); + + expect(request).toHaveBeenCalledTimes(2); + }); + + it('does NOT retry a status-less programming error', async () => { + const request = jest.fn().mockRejectedValue(new TypeError('x.map is not a function')); + + const handler = httpHandlerFromKbnClient({ kbnClient: kbnClient(request), log }); + await expect(handler('/api/agent_builder/converse', { method: 'POST' })).rejects.toThrow( + 'x.map is not a function' + ); + + expect(request).toHaveBeenCalledTimes(1); + }); + + it('still retries a 503 and still refuses a 400', async () => { + const ok = { data: {}, status: 200, statusText: 'OK', headers: {} }; + const retried = jest + .fn() + .mockRejectedValueOnce(makeError('service unavailable', 503)) + .mockResolvedValueOnce(ok); + const handler = httpHandlerFromKbnClient({ kbnClient: kbnClient(retried), log }); + await handler('/api/x', { method: 'POST' }); + expect(retried).toHaveBeenCalledTimes(2); + + const refused = jest.fn().mockRejectedValue(makeError('bad request', 400)); + const handler2 = httpHandlerFromKbnClient({ kbnClient: kbnClient(refused), log }); + await expect(handler2('/api/x', { method: 'POST' })).rejects.toThrow('bad request'); + expect(refused).toHaveBeenCalledTimes(1); + }); + it('retries an EIS-shaped 500 and still refuses a 501', async () => { + // EIS surfaces transient upstream provider faults as a Kibana 500, not a 503. + // On 2026-09-02 this exact shape failed 21/21 examples on two VMs at the same + // repetition because 500 was absent from retryStatuses. + const ok = { data: {}, status: 200, statusText: 'OK', headers: {} }; + const eisFault = jest + .fn() + .mockRejectedValueOnce( + makeError( + 'Received a server error status code for request from inference entity id ' + + '[.anthropic-claude-4.7-opus-chat_completion] status [500]. Error message: [Internal error]', + 500 + ) + ) + .mockResolvedValueOnce(ok); + const handler = httpHandlerFromKbnClient({ kbnClient: kbnClient(eisFault), log }); + await handler('/api/agent_builder/converse', { method: 'POST' }); + expect(eisFault).toHaveBeenCalledTimes(2); + + // 501 is a real "server will never do this" — must stay terminal so a genuine + // bug is not retried into a slow failure. + const refused = jest.fn().mockRejectedValue(makeError('not implemented', 501)); + const handler2 = httpHandlerFromKbnClient({ kbnClient: kbnClient(refused), log }); + await expect(handler2('/api/x', { method: 'POST' })).rejects.toThrow('not implemented'); + expect(refused).toHaveBeenCalledTimes(1); + }); + it('aborts a hung request and retries it', async () => { + process.env.KBN_EVALS_HTTP_RETRIES = '1'; + process.env.KBN_EVALS_HTTP_TIMEOUT_MS = '150'; + let calls = 0; + const request = jest.fn().mockImplementation(async ({ signal }: { signal?: AbortSignal }) => { + calls += 1; + if (calls === 1) { + // Never resolves on its own -- only the timeout signal ends it. + return new Promise((_resolve, reject) => { + signal?.addEventListener('abort', () => { + const err: any = new Error('The operation was aborted'); + err.name = 'AbortError'; + reject(err); + }); + }); + } + return { data: { ok: true } }; + }); + + const handler = httpHandlerFromKbnClient({ kbnClient: { request } as any, log }); + const result = await handler({ path: '/api/agent_builder/converse', method: 'POST' } as any); + + expect(result).toEqual({ ok: true }); + expect(calls).toBe(2); + }); + + it('bounds a hung request even when KBN_EVALS_HTTP_TIMEOUT_MS is unset', async () => { + // The wedge shape: with the old `?? '0'` default no AbortController was built, so + // nothing could abort and the attempt parked forever. Assert a controller reaches + // the request rather than waiting out the real 1,500,000ms default. + delete process.env.KBN_EVALS_HTTP_TIMEOUT_MS; + const request = jest.fn().mockResolvedValue({ data: { ok: true } }); + + const handler = httpHandlerFromKbnClient({ kbnClient: kbnClient(request), log }); + await handler({ path: '/api/agent_builder/converse', method: 'POST' } as any); + + const { signal } = request.mock.calls[0][0]; + expect(signal).toBeInstanceOf(AbortSignal); + expect(signal.aborted).toBe(false); + }); + + it('keeps the timeout active when the caller supplies its own signal', async () => { + // `signal || timeoutController?.signal` returned the caller's signal and discarded + // the timeout, so the abort timer fired into nothing and the hang came back. + process.env.KBN_EVALS_HTTP_RETRIES = '1'; + process.env.KBN_EVALS_HTTP_TIMEOUT_MS = '150'; + const callerController = new AbortController(); + let calls = 0; + + const request = jest.fn().mockImplementation(async ({ signal }: { signal?: AbortSignal }) => { + calls += 1; + if (calls === 1) { + return new Promise((_resolve, reject) => { + signal?.addEventListener('abort', () => { + const err: any = new Error('The operation was aborted'); + err.name = 'AbortError'; + reject(err); + }); + }); + } + return { data: { ok: true } }; + }); + + const handler = httpHandlerFromKbnClient({ kbnClient: kbnClient(request), log }); + const result = await handler({ + path: '/api/agent_builder/converse', + method: 'POST', + signal: callerController.signal, + } as any); + + // The caller never aborted; only the timeout did, and it still took effect. + expect(callerController.signal.aborted).toBe(false); + expect(result).toEqual({ ok: true }); + expect(calls).toBe(2); + }); + + /** + * Regression gate for the too-eager retry window. On sweep-1788696599-g53 + * (2026-09-06) Kibana went unreachable and two examples burned all four + * attempts in SEVEN SECONDS (1s + 2s + 4s), losing work that costs ~5 + * minutes per example while the endpoint recovered minutes later. Retry + * patience must be proportional to the cost of the work it protects. + */ + it('waits at least 30s once the cheap early attempts are exhausted', async () => { + process.env.KBN_EVALS_HTTP_RETRIES = '4'; + const delays: number[] = []; + const realSetTimeout = global.setTimeout; + const spy = jest.spyOn(global, 'setTimeout').mockImplementation((( + fn: () => void, + ms?: number + ) => { + // Record backoff sleeps, then fire immediately so the test stays fast. + // The request timeout uses a multi-minute bound; only short sleeps are + // backoff. Firing it would abort the request under test, so leave the + // timeout handle pending instead. + if (typeof ms === 'number' && ms < 1_000_000) { + delays.push(ms); + return realSetTimeout(fn, 0); + } + return realSetTimeout(() => {}, 0); + }) as never); + + try { + const request = jest + .fn() + .mockRejectedValue(makeError('request failed -- Status: N/A, Cause: fetch failed')); + + const handler = httpHandlerFromKbnClient({ kbnClient: kbnClient(request), log }); + await expect( + handler('/api/agent_builder/converse', { method: 'POST' }) + ).rejects.toBeDefined(); + + // Attempts 0 and 1 stay cheap; from attempt 2 the floor holds at 30s. + expect(delays.length).toBe(4); + expect(delays[0]).toBeLessThan(2_000); + expect(delays[1]).toBeLessThan(3_000); + expect(delays[2]).toBeGreaterThanOrEqual(30_000); + expect(delays[3]).toBeGreaterThanOrEqual(30_000); + + // The whole window must outlast a real outage, not seven seconds. + const total = delays.reduce((a, b) => a + b, 0); + expect(total).toBeGreaterThan(60_000); + } finally { + spy.mockRestore(); + } + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals/src/utils/http_handler_from_kbn_client.ts b/x-pack/platform/packages/shared/kbn-evals/src/utils/http_handler_from_kbn_client.ts index 2d5ad6e9b2efb..20b8989b06b95 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/utils/http_handler_from_kbn_client.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/utils/http_handler_from_kbn_client.ts @@ -21,6 +21,19 @@ type HttpHandlerArgs = * Creates a function that matches the HttpHandler interface from Core's * API, using the KbnClient from @kbn/kbn-client */ +/** + * Combine an optional caller signal with an optional timeout signal so an abort from + * either one takes effect. Picking one (`a || b`) silently disables the other. + */ +function combineSignals( + callerSignal: AbortSignal | null | undefined, + timeoutSignal: AbortSignal | undefined +): AbortSignal | undefined { + if (!callerSignal) return timeoutSignal; + if (!timeoutSignal) return callerSignal; + return AbortSignal.any([callerSignal, timeoutSignal]); +} + export function httpHandlerFromKbnClient({ kbnClient, log, @@ -64,7 +77,16 @@ export function httpHandlerFromKbnClient({ const finalHeaders = Object.keys(nextHeaders).length ? nextHeaders : undefined; const maxRetries = Number(process.env.KBN_EVALS_HTTP_RETRIES ?? '0') || 0; - const retryStatuses = new Set([429, 503, 504]); + // 500 belongs here: EIS surfaces transient upstream provider faults as a + // Kibana 500 ("Received a server error status code for request from inference + // entity id [...] status [500]"), not a 502/503. Observed 2026-09-02: a + // provider-side blip failed 21/21 examples on two independent VMs at the same + // repetition and discarded two good repetitions with them. These are retryable + // by nature — a non-retryable 500 just fails again and costs one extra call. + const retryStatuses = new Set([429, 500, 502, 503, 504]); + // Transport-level deaths, which arrive with no HTTP status at all. + const RETRYABLE_TRANSPORT_ERRORS = + /fetch failed|aborted|AbortError|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EPIPE|EAI_AGAIN|socket hang up|other side closed/i; async function sleep(ms: number) { await new Promise((r) => setTimeout(r, ms)); @@ -94,9 +116,31 @@ export function httpHandlerFromKbnClient({ return seconds * 1000; } + // A hung endpoint is worse than a failing one: retries need a request that + // FAILS, and a converse call that never returns just parks the worker in + // ep_poll forever. Observed on 2026-08-29: a glm-5-2 run sat 45 minutes with + // 4 seconds of CPU and six open sockets while /api/status still answered 200. + // Bound each attempt so a dead endpoint becomes a retryable failure. Set it + // ABOVE the slowest legitimate call: golden shows a real glm-5-2 example at + // 1198s, so too tight a bound aborts healthy work and the retry aborts again. + // + // Default to a bound rather than 0. With no bound no AbortController is created, + // so nothing can ever abort and attempt 4 parks forever: measured 2026-09-02 at + // concurrency 1, 2 and 5 alike (Kibana 0.0% CPU, zero established sockets), + // which is what ruled concurrency out as the cause. + const DEFAULT_REQUEST_TIMEOUT_MS = 1_500_000; + const rawTimeout = process.env.KBN_EVALS_HTTP_TIMEOUT_MS; + const requestTimeoutMs = + rawTimeout === undefined || rawTimeout === '' + ? DEFAULT_REQUEST_TIMEOUT_MS + : Number(rawTimeout) || 0; let lastError: unknown; for (let attempt = 0; attempt <= maxRetries; attempt++) { + const timeoutController = requestTimeoutMs > 0 ? new AbortController() : undefined; + const timeoutHandle = timeoutController + ? setTimeout(() => timeoutController.abort(), requestTimeoutMs) + : undefined; try { const response = await kbnClient.request({ path: options.path, @@ -105,10 +149,14 @@ export function httpHandlerFromKbnClient({ query, responseType: rawResponse ? 'stream' : undefined, headers: finalHeaders, - signal: signal || undefined, + // Compose rather than choose: `signal || timeoutController?.signal` silently + // drops the timeout the moment a caller supplies its own signal, leaving the + // abort timer firing into nothing and restoring the unbounded hang. + signal: combineSignals(signal, timeoutController?.signal), // We implement retries here so we can retry only on specific status codes. retries: 0, }); + if (timeoutHandle) clearTimeout(timeoutHandle); // success if (asResponse) { // `HttpResponse.request` is required by Core's type. We don't have access to undici's @@ -136,14 +184,29 @@ export function httpHandlerFromKbnClient({ } return response.data as any; } catch (err) { + if (timeoutHandle) clearTimeout(timeoutHandle); // `kbnClient.request` only ever throws `KbnClientRequesterError`. const error = err as KbnClientRequesterError; const status = error.status; lastError = error; + // A dead transport carries no HTTP status: kbnClient surfaces it as + // `Status: N/A, Cause: fetch failed` (undici) or a bare socket errno. + // Those are exactly the blips a long sweep must survive -- glm-5-2 lost + // 19 of 21 examples 58 minutes in when Kibana stopped answering and + // every remaining example failed this way. Retry them like a 503, but + // stay narrow: a status-less TypeError from our own code is a bug, not + // a blip, and must still fail fast. + const transportCause = `${error.message ?? ''} ${ + (error as { cause?: { code?: string; message?: string } }).cause?.code ?? '' + } ${(error as { cause?: { message?: string } }).cause?.message ?? ''}`; + const isTransportFailure = + typeof status !== 'number' && RETRYABLE_TRANSPORT_ERRORS.test(transportCause); + const shouldRetry = - attempt < maxRetries && typeof status === 'number' && retryStatuses.has(status); + attempt < maxRetries && + ((typeof status === 'number' && retryStatuses.has(status)) || isTransportFailure); if (!shouldRetry) { throw error; @@ -154,17 +217,32 @@ export function httpHandlerFromKbnClient({ parseRetryAfterMsFromMessage(error.message); // Exponential backoff (1s, 2s, 4s, ...) with jitter, but never sooner than retry-after. + // + // Cap the growth, not the patience: an uncapped 2^attempt reaches 8.5 + // minutes by attempt 6, but a floor that is too eager is worse. Measured + // 2026-09-06 on sweep-1788696599-g53: Kibana went unreachable and + // examples 3 and 4 burned all four attempts inside SEVEN SECONDS + // (1s + 2s + 4s), losing two examples of a shard whose examples cost + // ~5 minutes each. The endpoint answered 200 again minutes later, so the + // outage outlived the retry window by orders of magnitude. Retry patience + // must be proportional to the cost of the work it protects, so hold a + // floor of 30s once the cheap early attempts are exhausted. const baseBackoffMs = 1000 * Math.pow(2, attempt); - const baseDelayMs = retryAfterMs ? Math.max(baseBackoffMs, retryAfterMs) : baseBackoffMs; + const patientBackoffMs = attempt >= 2 ? Math.max(baseBackoffMs, 30_000) : baseBackoffMs; + const baseDelayMs = retryAfterMs + ? Math.max(patientBackoffMs, retryAfterMs) + : patientBackoffMs; const jitterMs = Math.floor( Math.random() * Math.min(1000, Math.max(100, baseDelayMs * 0.15)) ); const delayMs = baseDelayMs + jitterMs; log.warning( - `HTTP ${status} from Kibana; retrying in ${Math.round(delayMs / 1000)}s (attempt ${ - attempt + 1 - }/${maxRetries + 1})` + `${ + typeof status === 'number' ? `HTTP ${status}` : 'Transport failure' + } from Kibana; retrying in ${Math.round(delayMs / 1000)}s (attempt ${attempt + 1}/${ + maxRetries + 1 + })` ); await sleep(delayMs); } diff --git a/x-pack/platform/packages/shared/kbn-evals/src/utils/ingest_outcome.test.ts b/x-pack/platform/packages/shared/kbn-evals/src/utils/ingest_outcome.test.ts new file mode 100644 index 0000000000000..e17fcb971667a --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals/src/utils/ingest_outcome.test.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { classifyIngestOutcome, createIngestOutcome, recordIngestFailure } from './ingest_outcome'; + +describe('ingest outcome', () => { + describe('recordIngestFailure', () => { + it('keeps the first failure reason so later noise cannot mask the original cause', () => { + const outcome = createIngestOutcome(); + + recordIngestFailure(outcome, 'unauthorized for API key'); + recordIngestFailure(outcome, 'some later unrelated error'); + + expect(outcome.rejected).toBe(2); + expect(outcome.firstFailure).toBe('unauthorized for API key'); + }); + }); + + describe('classifyIngestOutcome', () => { + it('passes a run where nothing was rejected', () => { + const outcome = createIngestOutcome(); + outcome.ingested = 42; + + expect(classifyIngestOutcome(outcome)).toEqual({ kind: 'ok' }); + }); + + it('treats a run where every document was rejected as a total failure', () => { + // The regression this guards: a dead or unprivileged export key rejects + // every score while the run still exits 0, producing no durable results. + const outcome = createIngestOutcome(); + recordIngestFailure(outcome, 'action [indices:data/write/bulk[s]] is unauthorized'); + + const verdict = classifyIngestOutcome(outcome); + + expect(verdict.kind).toBe('total-failure'); + expect(verdict).toMatchObject({ + message: expect.stringContaining('no durable results'), + }); + expect(verdict).toMatchObject({ + message: expect.stringContaining('is unauthorized'), + }); + }); + + it('reports partial rejection without failing a run that still exported scores', () => { + const outcome = createIngestOutcome(); + outcome.ingested = 10; + recordIngestFailure(outcome, 'mapper_parsing_exception'); + + const verdict = classifyIngestOutcome(outcome); + + expect(verdict.kind).toBe('partial'); + expect(verdict).toMatchObject({ + message: expect.stringContaining('10 ingested, 1 rejected'), + }); + }); + + it('does not fail a run that exported nothing because it had nothing to export', () => { + // Zero ingested with zero rejected means no scores were produced at all + // (e.g. an empty suite) — that is not an export failure. + expect(classifyIngestOutcome(createIngestOutcome())).toEqual({ kind: 'ok' }); + }); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals/src/utils/ingest_outcome.ts b/x-pack/platform/packages/shared/kbn-evals/src/utils/ingest_outcome.ts new file mode 100644 index 0000000000000..f306adb660aa1 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals/src/utils/ingest_outcome.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface IngestOutcome { + ingested: number; + rejected: number; + firstFailure: string; +} + +export type IngestVerdict = + | { kind: 'ok' } + | { kind: 'partial'; message: string } + | { kind: 'total-failure'; message: string }; + +export const createIngestOutcome = (): IngestOutcome => ({ + ingested: 0, + rejected: 0, + firstFailure: '', +}); + +export const recordIngestFailure = (outcome: IngestOutcome, reason: string): void => { + outcome.rejected += 1; + if (!outcome.firstFailure) { + outcome.firstFailure = reason; + } +}; + +/** + * Classifies a run's score-export result. + * + * Score ingest is best-effort per example so one bad document cannot abort a long + * sweep. That tolerance hides total failure: an expired or unprivileged export key + * makes every document fail while the run still exits 0, which has silently + * destroyed whole sweeps. Rejections with zero successes are therefore fatal — + * the run produced nothing durable and must not be reported as a pass. + */ +export const classifyIngestOutcome = (outcome: IngestOutcome): IngestVerdict => { + if (outcome.rejected === 0) { + return { kind: 'ok' }; + } + + const summary = + `Score export: ${outcome.ingested} ingested, ${outcome.rejected} rejected. ` + + `First failure: ${outcome.firstFailure}`; + + if (outcome.ingested === 0) { + return { + kind: 'total-failure', + message: `No evaluation scores could be exported — this run produced no durable results. ${summary}`, + }; + } + + return { kind: 'partial', message: summary }; +}; diff --git a/x-pack/platform/packages/shared/kbn-evals/src/utils/kbn_client_with_retries.test.ts b/x-pack/platform/packages/shared/kbn-evals/src/utils/kbn_client_with_retries.test.ts index d34b32e324dde..cc25ac4258732 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/utils/kbn_client_with_retries.test.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/utils/kbn_client_with_retries.test.ts @@ -73,9 +73,12 @@ describe('wrapKbnClientWithRetries', () => { expect(log.warning).toHaveBeenCalledTimes(1); }); - it('does NOT retry on HTTP 500', async () => { + it('retries an EIS-shaped HTTP 500, inheriting the retry_utils policy', async () => { + // Policy owned by retry_utils (RETRYABLE_SERVER_STATUSES), which this wrapper + // delegates to; changed 2026-09-02 because EIS reports transient upstream + // provider faults as a Kibana 500 rather than a 502/503. See retry_utils.ts. const err = makeStatusError(500); - const request = jest.fn().mockRejectedValue(err); + const request = jest.fn().mockRejectedValueOnce(err).mockResolvedValueOnce('ok'); const inner = { request } as unknown as KbnClient; const log = createLog(); @@ -83,8 +86,8 @@ describe('wrapKbnClientWithRetries', () => { await expect( wrapped.request({ path: '/x', method: 'POST' } as Parameters[0]) - ).rejects.toBe(err); - expect(request).toHaveBeenCalledTimes(1); + ).resolves.toBe('ok'); + expect(request).toHaveBeenCalledTimes(2); expect(log.error).not.toHaveBeenCalled(); }); diff --git a/x-pack/platform/packages/shared/kbn-evals/src/utils/retry_integration.test.ts b/x-pack/platform/packages/shared/kbn-evals/src/utils/retry_integration.test.ts new file mode 100644 index 0000000000000..81c10818b7cdf --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals/src/utils/retry_integration.test.ts @@ -0,0 +1,259 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * End-to-end proof that the 500 retry actually fires. + * + * The unit tests around `retry_utils` assert the POLICY (is 500 retryable?). + * They cannot show that a 500 travelling the real path -- undici -> KbnClient + * -> KbnClientRequesterError -> our retry layer -- is recognised and retried, + * because every layer in between is mocked away. + * + * The production sweep never proved it either: attempt 4 passed with zero 500s, + * so `retried` stayed 0 and the branch never executed against a real socket. + * A fix that has never run is not a fix. + * + * So: a real HTTP server returning the exact EIS 500 body observed in + * production, a real KbnClient pointed at it, and assertions that the request + * ultimately SUCCEEDS after N failures. + * + * Note KbnClientRequester has a retry loop of its OWN (delay(1000 * attempt)). + * A single blip is absorbed there and never reaches us, so the test that + * targets our layer must out-last the inner budget to prove anything. + */ + +import http from 'http'; +import type { AddressInfo } from 'net'; +import { ToolingLog } from '@kbn/tooling-log'; +import { KbnClient } from '@kbn/kbn-client'; +import { httpHandlerFromKbnClient } from './http_handler_from_kbn_client'; +import { wrapKbnClientWithRetries } from './kbn_client_with_retries'; + +// The verbatim shape EIS returns when an upstream provider blips. It is a +// Kibana 500, not a 502/503 -- the whole reason 500 had to become retryable. +const EIS_500_BODY = JSON.stringify({ + statusCode: 500, + error: 'Internal Server Error', + message: + 'Received a server error status code for request from inference entity id [eis-gemini-3-1-pro] status [500]', +}); + +interface Scenario { + failures: number; + status: number; + body?: string; +} + +/** Fails the first `failures` requests with `status`, then returns 200. */ +function startFlakyServer(scenario: Scenario) { + let hits = 0; + const server = http.createServer((req, res) => { + hits += 1; + if (hits <= scenario.failures) { + res.writeHead(scenario.status, { 'content-type': 'application/json' }); + res.end(scenario.body ?? EIS_500_BODY); + return; + } + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ ok: true, servedOnAttempt: hits })); + }); + + return new Promise<{ url: string; getHits: () => number; close: () => Promise }>( + (resolve) => { + server.listen(0, '127.0.0.1', () => { + const { port } = server.address() as AddressInfo; + resolve({ + url: `http://127.0.0.1:${port}`, + getHits: () => hits, + close: () => + new Promise((done) => { + server.close(() => done()); + }), + }); + }); + } + ); +} + +describe('500 retry against a real HTTP server', () => { + let log: ToolingLog; + let warnings: string[]; + + beforeEach(() => { + warnings = []; + // Capture via a real writer: ToolingLog routes messages through its writers, + // so stubbing the .warning() method would silently observe nothing. + log = new ToolingLog(); + log.setWriters([ + { + write: (msg: any) => { + if (msg?.type === 'warning' || msg?.type === 'error') { + warnings.push(msg.args.map(String).join(' ')); + } + return true; + }, + }, + ]); + }); + + describe('httpHandlerFromKbnClient', () => { + const previous = process.env.KBN_EVALS_HTTP_RETRIES; + afterEach(() => { + if (previous === undefined) delete process.env.KBN_EVALS_HTTP_RETRIES; + else process.env.KBN_EVALS_HTTP_RETRIES = previous; + }); + + it('recovers from two real EIS 500s and returns the success payload', async () => { + process.env.KBN_EVALS_HTTP_RETRIES = '4'; + const server = await startFlakyServer({ failures: 2, status: 500 }); + + try { + const kbnClient = new KbnClient({ url: server.url, log }); + const fetch = httpHandlerFromKbnClient({ kbnClient, log }); + + const result: any = await fetch('/internal/eis/converse', { method: 'POST' }); + + // Proof the retry ran: the server was hit 3 times and the caller + // still got a success rather than an exception. + expect(server.getHits()).toBe(3); + expect(result).toEqual({ ok: true, servedOnAttempt: 3 }); + } finally { + await server.close(); + } + }, 30000); + + it('still gives up on a 500 that never clears, instead of hanging', async () => { + process.env.KBN_EVALS_HTTP_RETRIES = '2'; + const server = await startFlakyServer({ failures: Infinity, status: 500 }); + + try { + const kbnClient = new KbnClient({ url: server.url, log }); + const fetch = httpHandlerFromKbnClient({ kbnClient, log }); + + await expect(fetch('/internal/eis/converse', { method: 'POST' })).rejects.toThrow(); + // initial attempt + 2 retries + expect(server.getHits()).toBe(3); + } finally { + await server.close(); + } + }, 30000); + + it('does NOT retry a 400, so real bugs still fail fast', async () => { + process.env.KBN_EVALS_HTTP_RETRIES = '4'; + const server = await startFlakyServer({ + failures: 1, + status: 400, + body: JSON.stringify({ statusCode: 400, message: 'bad request' }), + }); + + try { + const kbnClient = new KbnClient({ url: server.url, log }); + const fetch = httpHandlerFromKbnClient({ kbnClient, log }); + + // Only the FIRST request 400s; a retry would be served a 200 and the + // call would resolve. Requiring a rejection therefore proves we did + // not retry -- if 400 ever becomes retryable this test goes green-to-red. + await expect(fetch('/internal/eis/converse', { method: 'POST' })).rejects.toThrow(); + expect(server.getHits()).toBe(1); + } finally { + await server.close(); + } + }, 30000); + }); + + describe('wrapKbnClientWithRetries', () => { + it('recovers from a real EIS 500 on the kbnClient.request path', async () => { + const server = await startFlakyServer({ failures: 1, status: 500 }); + + try { + const raw = new KbnClient({ url: server.url, log }); + const wrapped = wrapKbnClientWithRetries({ kbnClient: raw, log }); + + const response = await wrapped.request({ + path: '/internal/eis/converse', + method: 'POST', + } as any); + + expect(server.getHits()).toBe(2); + expect(response.data).toEqual({ ok: true, servedOnAttempt: 2 }); + } finally { + await server.close(); + } + }, 40000); + + it('retries a 500 in OUR layer once KbnClient has exhausted its own retries', async () => { + // `retries: 1` disables KbnClientRequester's internal loop so the 500 + // propagates out to withRetry. Two failures then force OUR layer to + // re-drive the request, which the inner loop can no longer explain. + const server = await startFlakyServer({ failures: 2, status: 500 }); + + try { + const raw = new KbnClient({ url: server.url, log }); + const wrapped = wrapKbnClientWithRetries({ kbnClient: raw, log }); + + const response = await wrapped.request({ + path: '/internal/eis/converse', + method: 'POST', + retries: 1, + } as any); + + expect(server.getHits()).toBe(3); + expect(response.data).toEqual({ ok: true, servedOnAttempt: 3 }); + expect( + warnings.some( + (w) => + /kbnClient\.request POST \/internal\/eis\/converse/.test(w) && /attempt 1\//.test(w) + ) + ).toBe(true); + } finally { + await server.close(); + } + }, 60000); + + it('gives up immediately on a 404, proving the policy actually refuses', async () => { + // Guards the "retry everything" failure mode. With retries:1 the inner + // KbnClient loop is disabled, so the only thing that could re-drive this + // request is our layer. One 404 then a 200: if we wrongly treated 404 as + // retryable the call would succeed on hit 2. + const server = await startFlakyServer({ + failures: 1, + status: 404, + body: JSON.stringify({ statusCode: 404, message: 'not found' }), + }); + + try { + const raw = new KbnClient({ url: server.url, log }); + const wrapped = wrapKbnClientWithRetries({ kbnClient: raw, log }); + + await expect( + wrapped.request({ path: '/internal/eis/converse', method: 'POST', retries: 1 } as any) + ).rejects.toThrow(); + + expect(server.getHits()).toBe(1); + } finally { + await server.close(); + } + }, 30000); + + it('honours retries:0 and does not retry a 500', async () => { + const server = await startFlakyServer({ failures: Infinity, status: 500 }); + + try { + const raw = new KbnClient({ url: server.url, log }); + const wrapped = wrapKbnClientWithRetries({ kbnClient: raw, log }); + + await expect( + wrapped.request({ path: '/internal/eis/converse', method: 'POST', retries: 0 } as any) + ).rejects.toThrow(); + + expect(server.getHits()).toBe(1); + } finally { + await server.close(); + } + }, 30000); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals/src/utils/retry_utils.test.ts b/x-pack/platform/packages/shared/kbn-evals/src/utils/retry_utils.test.ts index 07fe9f2b4e4f9..fab19630d8a2a 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/utils/retry_utils.test.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/utils/retry_utils.test.ts @@ -45,11 +45,25 @@ describe('withRetry', () => { expect(fn).toHaveBeenCalledTimes(2); }); - it('does NOT retry on HTTP 500 (treated as deterministic in this stack)', async () => { + it('retries an EIS-shaped HTTP 500 (transient upstream provider fault)', async () => { + // Policy change 2026-09-02, overturning "500 is deterministic in this stack". + // EIS wraps transient upstream provider faults as a Kibana 500, not a 502/503: + // "Received a server error status code for request from inference entity id + // [.anthropic-claude-4.7-opus-chat_completion] status [500]" + // Observed: 27 such 500s per model failed 21/21 examples on two independent + // VMs at the same repetition, discarding two good repetitions with them. + // A truly deterministic 500 costs one extra call; a transient one cost a sweep. + const fn = jest.fn().mockRejectedValueOnce(makeStatusError(500)).mockResolvedValueOnce('ok'); + const result = await withRetry(fn, fastRetryOptions); + expect(result).toBe('ok'); + expect(fn).toHaveBeenCalledTimes(2); + }); + + it('gives up on a persistent HTTP 500 instead of retrying forever', async () => { const err = makeStatusError(500); const fn = jest.fn().mockRejectedValue(err); - await expect(withRetry(fn, fastRetryOptions)).rejects.toBe(err); - expect(fn).toHaveBeenCalledTimes(1); + await expect(withRetry(fn, { ...fastRetryOptions, maxAttempts: 3 })).rejects.toBe(err); + expect(fn).toHaveBeenCalledTimes(3); }); it('does NOT retry on HTTP 413 (payload too large)', async () => { diff --git a/x-pack/platform/packages/shared/kbn-evals/src/utils/retry_utils.ts b/x-pack/platform/packages/shared/kbn-evals/src/utils/retry_utils.ts index 57bf9d0e7018d..e1ae03d6639fe 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/utils/retry_utils.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/utils/retry_utils.ts @@ -97,7 +97,13 @@ function computeDelayMs({ return base + Math.floor(Math.random() * extra); } -const RETRYABLE_SERVER_STATUSES = new Set([502, 503, 504]); +// 500 is included deliberately. EIS surfaces transient upstream provider faults +// as a Kibana 500 ("Received a server error status code for request from inference +// entity id [...] status [500]"), not a 502/503. Observed 2026-09-02: 27 such 500s +// per model failed 21/21 examples on two independent VMs at the same repetition, +// destroying two good repetitions with them. A genuinely non-retryable 500 just +// fails again and costs one extra call; a transient one costs a whole sweep. +const RETRYABLE_SERVER_STATUSES = new Set([500, 502, 503, 504]); function isRetryable(error: any): { retry: boolean; retryAfterMs?: number } { const status = getStatusCode(error); diff --git a/x-pack/platform/plugins/shared/evals/server/routes/examples/get_example_scores.test.ts b/x-pack/platform/plugins/shared/evals/server/routes/examples/get_example_scores.test.ts index 300bab9e7091e..7d1b91ca7f21e 100644 --- a/x-pack/platform/plugins/shared/evals/server/routes/examples/get_example_scores.test.ts +++ b/x-pack/platform/plugins/shared/evals/server/routes/examples/get_example_scores.test.ts @@ -49,12 +49,12 @@ describe('GET /internal/evals/examples/{exampleId}/scores', () => { return { handler, context, evaluationScoreService, logger }; }; - const makeRequest = (exampleId = 'example-123') => + const makeRequest = (exampleId = 'example-123', query: Record = {}) => httpServerMock.createKibanaRequest({ method: 'get', path: EVALS_EXAMPLE_SCORES_URL.replace('{exampleId}', exampleId), params: { exampleId }, - query: {}, + query, }); it('uses the correct query parameters', async () => { @@ -86,6 +86,33 @@ describe('GET /internal/evals/examples/{exampleId}/scores', () => { ); }); + it('passes execution_id and model_id query params into the ES query', async () => { + const { handler, context, evaluationScoreService } = setup(); + evaluationScoreService.search.mockResolvedValueOnce({ hits: { hits: [] } } as any); + + await handler( + context, + makeRequest('example-123', { execution_id: 'run-abc', model_id: 'openai-gpt-5.4' }), + kibanaResponseFactory + ); + + expect(evaluationScoreService.search).toHaveBeenCalledWith( + expect.objectContaining({ + query: { + bool: { + must: [ + { term: { 'example.id': 'example-123' } }, + { term: { 'metadata.execution_id': 'run-abc' } }, + { term: { 'task.model.id': 'openai-gpt-5.4' } }, + buildSpaceFilter('default'), + ], + }, + }, + size: 10000, + }) + ); + }); + it('adds a dataset filter when dataset_id is provided', async () => { const { handler, context, evaluationScoreService } = setup(); evaluationScoreService.search.mockResolvedValueOnce({ hits: { hits: [] } } as any); diff --git a/x-pack/platform/plugins/shared/evals/server/routes/examples/get_example_scores.ts b/x-pack/platform/plugins/shared/evals/server/routes/examples/get_example_scores.ts index 39fd76e9ab978..b9150803e8f01 100644 --- a/x-pack/platform/plugins/shared/evals/server/routes/examples/get_example_scores.ts +++ b/x-pack/platform/plugins/shared/evals/server/routes/examples/get_example_scores.ts @@ -55,12 +55,21 @@ export const registerGetExampleScoresRoute = ({ async (context, request, response) => { try { const { exampleId } = request.params; - const { dataset_id: datasetId } = request.query; + const { + dataset_id: datasetId, + execution_id: executionId, + model_id: modelId, + } = request.query; const evalsContext = await context.evals; const spaceId = getSpaceId ? await getSpaceId(request) : DEFAULT_SPACE_ID; const searchResponse = await evalsContext.evaluationScoreService.search({ - query: buildExampleScoresQuery(exampleId, { spaceId, datasetId }), + query: buildExampleScoresQuery(exampleId, { + spaceId, + datasetId, + executionId, + modelId, + }), sort: EXAMPLE_SCORES_SORT_ORDER, size: MAX_SCORES_PER_QUERY, _source_excludes: UNBOUNDED_SCORE_FIELDS, diff --git a/x-pack/platform/plugins/shared/inference/server/chat_complete/utils/inference_endpoint_executor.ts b/x-pack/platform/plugins/shared/inference/server/chat_complete/utils/inference_endpoint_executor.ts index 3919245de51f8..71425a662867f 100644 --- a/x-pack/platform/plugins/shared/inference/server/chat_complete/utils/inference_endpoint_executor.ts +++ b/x-pack/platform/plugins/shared/inference/server/chat_complete/utils/inference_endpoint_executor.ts @@ -20,6 +20,19 @@ export interface InferenceEndpointExecutor { invoke(options: InferenceEndpointInvokeOptions): Promise; } +/** + * Default per-request timeout for an inference endpoint call. + * + * Agent Builder eval runs drive much longer single turns than product traffic + * (multi-step skills against large alert sets), where the 180s default aborts + * a turn mid-flight and the example is scored as a failure of the model rather + * than of the harness. The eval VMs raise it via AGENT_BUILDER_INFERENCE_TIMEOUT_MS; + * unset, the product default is unchanged. + */ +const AGENT_BUILDER_INFERENCE_TIMEOUT_MS = Number( + process.env.AGENT_BUILDER_INFERENCE_TIMEOUT_MS ?? 180_000 +); + export const createInferenceEndpointExecutor = ({ inferenceId, esClient, @@ -28,7 +41,12 @@ export const createInferenceEndpointExecutor = ({ esClient: ElasticsearchClient; }): InferenceEndpointExecutor => { return { - async invoke({ body, signal, metadata, timeout = 180_000 }): Promise { + async invoke({ + body, + signal, + metadata, + timeout = AGENT_BUILDER_INFERENCE_TIMEOUT_MS, + }): Promise { const response = await esClient.transport.request( { method: 'POST', diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/src/dataset.ts b/x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/src/dataset.ts index 409e519ce2777..04a6b2b9f2879 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/src/dataset.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/src/dataset.ts @@ -57,6 +57,7 @@ export const goldenPathExamples: AttackDiscoveryAgentBuilderExample[] = [ metadata: { alertCount: providedAlertFixture.alertCount, fixture: 'provided-alerts', + scenarioKey: 'provided-alerts', }, }, { @@ -88,6 +89,7 @@ export const goldenPathExamples: AttackDiscoveryAgentBuilderExample[] = [ metadata: { alertCount: liveRetrievalFixture.alertCount, fixture: 'live-retrieval', + scenarioKey: 'live-retrieval', }, }, { @@ -111,6 +113,7 @@ export const goldenPathExamples: AttackDiscoveryAgentBuilderExample[] = [ metadata: { alertCount: multipleAlertSetsFixture.alertCount, fixture: 'multiple-alert-sets', + scenarioKey: 'multiple-alert-sets', }, }, { @@ -135,6 +138,7 @@ export const goldenPathExamples: AttackDiscoveryAgentBuilderExample[] = [ metadata: { alertCount: missingAlertRetrievalFixture.alertCount, fixture: 'missing-alert-retrieval', + scenarioKey: 'missing-alert-retrieval', }, }, { @@ -154,6 +158,7 @@ export const goldenPathExamples: AttackDiscoveryAgentBuilderExample[] = [ metadata: { alertCount: statusOnlyFixture.alertCount, fixture: 'status-only', + scenarioKey: 'status-only', }, }, ]; diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/src/evaluators/attack_discovery_rubric_evaluator.test.ts b/x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/src/evaluators/attack_discovery_rubric_evaluator.test.ts index f2ddacaa23513..30810dcedc6af 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/src/evaluators/attack_discovery_rubric_evaluator.test.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/src/evaluators/attack_discovery_rubric_evaluator.test.ts @@ -52,7 +52,7 @@ const expectedFor = ( describe('createAttackDiscoveryRubricEvaluator', () => { const judge = jest.fn(); - const criteria = jest.fn(() => ({ + const criteria = jest.fn((_criteria: string[]) => ({ name: 'criteria', kind: 'LLM' as const, direction: 'maximize', @@ -87,6 +87,27 @@ describe('createAttackDiscoveryRubricEvaluator', () => { expect(result.score).toBe(1); }); + // The saturation guard. Each rubric item must reach the judge as its own + // criterion, because `evaluators.criteria` scores criteria independently + // and returns the weighted pass rate. One combined criterion carrying a + // "5 of 7 -> Y/N" threshold can only ever return 0 or 1, which is what + // pinned 95.6% of regraded attack-discovery cells at the ceiling. + it('passes each rubric item as a separate criterion so partial credit survives', async () => { + await evaluate({ insights: [insight], expected: expectedFor([insight]) }); + + const passedCriteria = criteria.mock.calls[0][0]; + expect(passedCriteria).toHaveLength(7); + // No item may smuggle the old aggregate threshold back in: that is the + // exact construct that collapsed 7 signals into one binary verdict. + for (const criterion of passedCriteria) { + expect(criterion).not.toMatch(/at least 5 of the 7/i); + expect(criterion).not.toMatch(/single character/i); + } + // Every item keeps the reference, otherwise a judge scoring one item in + // isolation has nothing to compare the submission against. + expect(passedCriteria.every((c) => c.includes('Reference:'))).toBe(true); + }); + // Guards against the N/A branch becoming a blanket exemption: an example // that has a reference but produced nothing must still reach the judge and // keep whatever score the judge gives it. diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/src/evaluators/attack_discovery_rubric_evaluator.ts b/x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/src/evaluators/attack_discovery_rubric_evaluator.ts index bc7d6dabf2978..11952bb230e98 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/src/evaluators/attack_discovery_rubric_evaluator.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/src/evaluators/attack_discovery_rubric_evaluator.ts @@ -12,6 +12,29 @@ import type { AttackDiscoveryAgentBuilderTaskOutput, } from '../types'; +/** + * The seven rubric requirements, one per criterion. + * + * Exported because the matrix rejudge harness grades recorded AD runs through + * its own jury and previously restated this list verbatim. Two copies drift: + * the first per-item version of this rubric shipped while the harness copy + * still collapsed all seven into a single "5 of 7 -> Y or N" question, so a + * rejudge silently scored a different thing under the same column name. + */ +export const ATTACK_DISCOVERY_RUBRIC_ITEMS = [ + 'Is the submission non-empty and well-formed JSON with an array of attackDiscoveries?', + 'Do the detailsMarkdown values capture the overall essence of the reference, allowing slight differences in wording but not omitting or misrepresenting key incidents?', + 'Does the submission mention at least half of the same entities (host or user) as the reference?', + 'Are the summaryMarkdown values at least partially similar and summarizing the same incidents?', + 'Are the title values at least partially similar and mentioning the same incidents?', + 'Do more than half of the alertIds in the submission overlap with the alertIds in the reference?', + 'Are the MITRE tactics consistent with the reference?', +] as const; + +/** Appends the serialized reference to each item so criteria stay self-contained. */ +export const buildAttackDiscoveryRubricCriteria = (reference: string): string[] => + ATTACK_DISCOVERY_RUBRIC_ITEMS.map((item) => `${item} Reference: ${reference}`); + const truncateInsightsForRubric = ( insights: AttackDiscovery[] | null | undefined ): Array<{ @@ -64,21 +87,17 @@ export const createAttackDiscoveryRubricEvaluator = ({ const submission = JSON.stringify({ attackDiscoveries: submissionInsights }, null, 2); const reference = JSON.stringify({ attackDiscoveries: referenceInsights }, null, 2); - const rubric = [ - 'Evaluate the submission against the reference using these 7 rubric items:', - '1. Is the submission non-empty and well-formed JSON with an array of attackDiscoveries?', - '2. Do the detailsMarkdown values capture the overall essence of the reference, allowing slight differences in wording but not omitting or misrepresenting key incidents?', - '3. Does the submission mention at least half of the same entities (host or user) as the reference?', - '4. Are the summaryMarkdown values at least partially similar and summarizing the same incidents?', - '5. Are the title values at least partially similar and mentioning the same incidents?', - '6. Do more than half of the alertIds in the submission overlap with the alertIds in the reference?', - '7. Are the MITRE tactics consistent with the reference?', - `Reference: ${reference}`, - 'Score the submission as passing if at least 5 of the 7 rubric items are correct. Explain your reasoning briefly and end with a single character: Y or N.', - ].join('\n'); + // Each rubric item is passed as its own criterion so the judge scores it + // independently and `evaluators.criteria` returns the weighted pass rate. + // Collapsing all 7 into one string with a "5 of 7 -> Y/N" threshold, as + // this evaluator used to, discards every partial result: a submission + // that misses two items scores identically to a perfect one. Measured on + // 295 regraded cells that produced 95.6% perfect scores (sd 0.205, + // effectively binary) and left the column unable to rank. + const rubricCriteria = buildAttackDiscoveryRubricCriteria(reference); try { - return await evaluators.criteria([rubric]).evaluate({ + return await evaluators.criteria(rubricCriteria).evaluate({ input, expected: { expected: reference }, output: { diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/src/replay_join_keys.test.ts b/x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/src/replay_join_keys.test.ts new file mode 100644 index 0000000000000..f46e2a15b2011 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/src/replay_join_keys.test.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { goldenPathExamples } from './dataset'; +import { cleanProfileProvidedAlertsExamples } from './datasets/clean_profile_provided_alerts'; + +/** + * A rejudge joins recorded score documents back to dataset references on + * `example.metadata.scenarioKey`. Golden documents carry `example.id = '0'` for + * 11,512 of 11,993 attack-discovery rows, so the id cannot distinguish one + * scenario from another. + * + * The five golden-path slices recorded no scenarioKey at all, which made them + * permanently unreplayable: a rejudge would have graded all five against the + * first scenario's ground truth. These tests pin the join key so a future + * scenario cannot be added without one. + */ +describe('attack-discovery replay join keys', () => { + const allExamples = [...goldenPathExamples, ...cleanProfileProvidedAlertsExamples]; + + it('gives every golden-path example a scenarioKey', () => { + const missing = goldenPathExamples.filter((example) => !example.metadata?.scenarioKey); + expect(missing).toEqual([]); + }); + + it('keys every example uniquely across the whole suite', () => { + // Two scenarios sharing a key silently merge into one replay cell, and the + // loser gets graded against the winner's reference. + const keys = allExamples.map((example) => example.metadata?.scenarioKey); + expect(new Set(keys).size).toBe(allExamples.length); + }); + + it('matches the scenarioKey to the fixture each golden-path example exercises', () => { + // The suite already sliced these by `metadata.fixture`; the join key has to + // agree with that slicing or a replay reunites slices the spec separated. + const pairs = goldenPathExamples.map((example) => [ + example.metadata?.fixture, + example.metadata?.scenarioKey, + ]); + expect(pairs).toEqual([ + ['provided-alerts', 'provided-alerts'], + ['live-retrieval', 'live-retrieval'], + ['multiple-alert-sets', 'multiple-alert-sets'], + ['missing-alert-retrieval', 'missing-alert-retrieval'], + ['status-only', 'status-only'], + ]); + }); + + it('keeps every scenarioKey a non-empty string', () => { + const bad = allExamples.filter( + (example) => + typeof example.metadata?.scenarioKey !== 'string' || + (example.metadata?.scenarioKey as string).trim() === '' + ); + expect(bad).toEqual([]); + }); +}); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/src/types.ts b/x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/src/types.ts index 9b92d550a8a6b..c1d3cda10583f 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/src/types.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery-agent-builder/src/types.ts @@ -44,7 +44,16 @@ export interface AttackDiscoveryAgentBuilderMetadata extends Record => { const response = (await traceEsClient.esql.query({ - query: `FROM traces-*\n| WHERE ${where} AND ${TOOL_KIND}\n| STATS tool_calls = COUNT(*),\n required_tool_calls = COUNT(CASE(attributes.gen_ai.tool.name == "${RULE_CREATION_TOOL_ID}", 1, NULL))`, + query: `FROM ${TRACE_INDEX_PATTERN}\n| WHERE ${where} AND ${TOOL_KIND}\n| STATS tool_calls = COUNT(*),\n required_tool_calls = COUNT(CASE(attributes.gen_ai.tool.name == "${RULE_CREATION_TOOL_ID}", 1, NULL))`, })) as unknown as EsqlResponse; const row = response.values?.[0]; if (!row) return undefined; @@ -139,7 +139,7 @@ export function createToolRoutingEvaluator({ let diagnosis = 'probe did not run'; try { const probe = (await traceEsClient.esql.query({ - query: `FROM traces-* + query: `FROM ${TRACE_INDEX_PATTERN} | WHERE attributes.elastic.inference.span.kind == "TOOL" | STATS tool_spans = COUNT(*)`, })) as unknown as EsqlResponse; @@ -196,7 +196,7 @@ export const assertToolSpansReachable = async ({ for (const clause of clauses) { try { const response = (await traceEsClient.esql.query({ - query: `FROM traces-*\n| WHERE ${clause.where} AND ${TOOL_KIND}\n| STATS tool_spans = COUNT(*)`, + query: `FROM ${TRACE_INDEX_PATTERN}\n| WHERE ${clause.where} AND ${TOOL_KIND}\n| STATS tool_spans = COUNT(*)`, })) as unknown as EsqlResponse; if (Number(response.values?.[0]?.[0] ?? 0) > 0) { log.info(`Tool spans reachable via ${clause.name}`); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-endpoint/src/security_skill_invocation_evaluator.ts b/x-pack/solutions/security/packages/kbn-evals-suite-endpoint/src/security_skill_invocation_evaluator.ts index ed8cee6b4cefc..bbfd1dfcfd9dd 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-endpoint/src/security_skill_invocation_evaluator.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-endpoint/src/security_skill_invocation_evaluator.ts @@ -6,7 +6,7 @@ */ import type { Client as EsClient } from '@elastic/elasticsearch'; -import { createTraceBasedEvaluator, type Evaluator } from '@kbn/evals'; +import { createTraceBasedEvaluator, TRACE_INDEX_PATTERN, type Evaluator } from '@kbn/evals'; import type { ToolingLog } from '@kbn/tooling-log'; const VALID_SKILL_NAME = /^[a-zA-Z0-9_-]+$/; @@ -32,7 +32,7 @@ export function createSecuritySkillInvocationEvaluator({ config: { name: `Skill Invoked (${skillName})`, direction: 'maximize', - buildQuery: (traceId) => `FROM traces-* + buildQuery: (traceId) => `FROM ${TRACE_INDEX_PATTERN} | WHERE trace.id == "${traceId}" | STATS total_spans = COUNT(*), diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-automatic-migrations/src/dashboards/evaluators/panel_count_preservation.test.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-automatic-migrations/src/dashboards/evaluators/panel_count_preservation.test.ts new file mode 100644 index 0000000000000..66c19f55fd0cf --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-automatic-migrations/src/dashboards/evaluators/panel_count_preservation.test.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createPanelCountPreservationEvaluator } from './panel_count_preservation'; +import type { MigrationResult } from '../migration_client'; +import type { DashboardExample, DashboardExpected } from '../../../datasets/dashboards/types'; + +function makeMigrationResult(panelCount: number): MigrationResult { + const panelObjects = Array.from({ length: panelCount }, (_, i) => ({ + type: 'lens', + panelIndex: `panel-${i}`, + title: `Panel ${i}`, + gridData: { x: 0, y: i * 6, w: 24, h: 6, i: `panel-${i}` }, + embeddableConfig: {}, + })); + + return { + migrationId: 'test', + dashboards: [ + { + id: 'd1', + migration_id: 'test', + original_dashboard: { id: 'orig1', title: 'Original Dashboard' }, + elastic_dashboard: { + title: 'Test Dashboard', + description: '', + data: JSON.stringify({ + attributes: { + title: 'Test Dashboard', + description: '', + panelsJSON: JSON.stringify(panelObjects), + }, + type: 'dashboard', + }), + }, + status: 'completed', + translation_result: 'full', + comments: '', + }, + ], + } as unknown as MigrationResult; +} + +function evaluate(actualPanels: number, expectedPanelCount: number | undefined) { + const evaluator = createPanelCountPreservationEvaluator(); + return evaluator.evaluate({ + input: { + original_dashboard_export: '', + resources: [], + } as unknown as DashboardExample['input'], + output: makeMigrationResult(actualPanels), + expected: { panel_count: expectedPanelCount } as unknown as DashboardExpected, + metadata: { + category: 'standard', + has_lookups: false, + has_markdown_panels: false, + panel_count: expectedPanelCount ?? 0, + complexity: 'low', + } as unknown as DashboardExample['metadata'], + }); +} + +describe('Panel Count Preservation evaluator', () => { + it('scores 1 when the panel count matches', async () => { + const result = await evaluate(4, 4); + + expect(result.score).toBe(1); + expect(result.explanation).toContain('Panel count matches: 4'); + }); + + it('scores 0 on a genuine mismatch between two non-zero counts', async () => { + const result = await evaluate(3, 4); + + expect(result.score).toBe(0); + expect(result.explanation).toContain('expected 4, got 3'); + }); + + it('scores 1 when the source dashboard genuinely has no panels', async () => { + const result = await evaluate(0, 0); + + expect(result.score).toBe(1); + }); + + it('returns null when panel_count is absent from the dataset', async () => { + const result = await evaluate(4, undefined); + + expect(result.score).toBeNull(); + expect(result.explanation).toBe('No expected panel_count in dataset'); + }); + + // The biting case: 3 of the 5 standard-dashboard dataset entries ship + // `panel_count: 0` while their source Splunk XML defines real panels, so every + // run scored a hard 0 regardless of migration quality. An unpopulated ground + // truth must not be reported as a model failure. + it('returns null instead of 0 when ground truth is unpopulated', async () => { + const result = await evaluate(20, 0); + + expect(result.score).toBeNull(); + expect(result.explanation).toContain('produced 20 panels'); + expect(result.explanation).toContain('unscored'); + expect(result.metadata).toMatchObject({ + expectedCount: 0, + actualCount: 20, + unpopulatedGroundTruth: true, + }); + }); +}); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-automatic-migrations/src/dashboards/evaluators/panel_count_preservation.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-automatic-migrations/src/dashboards/evaluators/panel_count_preservation.ts index 692b3371c854d..74c3ca3085caa 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-security-automatic-migrations/src/dashboards/evaluators/panel_count_preservation.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-automatic-migrations/src/dashboards/evaluators/panel_count_preservation.ts @@ -32,6 +32,25 @@ export const createPanelCountPreservationEvaluator = (): Evaluator< const actualCount = countTranslatedPanels(output); const expectedCount = expected.panel_count; + + // A dataset entry that expects zero panels while the source dashboard defines + // some is unpopulated ground truth, not a migration failure: scoring it 0 makes + // the metric report failure no matter how well the model performed. Report it as + // unscored so the gap stays visible instead of masquerading as a real result. + if (expectedCount === 0 && actualCount > 0) { + return { + score: null, + explanation: + `Expected panel_count is 0 but the migration produced ${actualCount} panels. ` + + `Treating as unscored: the dataset entry has no populated ground truth.`, + metadata: { + expectedCount, + actualCount, + unpopulatedGroundTruth: true, + }, + }; + } + const matches = actualCount === expectedCount; return { diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/docs/tool_surface_mapping.md b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/docs/tool_surface_mapping.md new file mode 100644 index 0000000000000..009bd5e928822 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/docs/tool_surface_mapping.md @@ -0,0 +1,47 @@ +# Tool-surface mapping: original EIS benchmark → current eval stacks + +The original benchmark (June 2026, `eis-benchmark` handoff bundle) ran against a +different agent harness and tool surface than the current persona-matrix suite +(August 2026, golden cluster). Tool-trail comparison between the two requires +this mapping — a raw string diff is meaningless without it. + +## Renamed / replaced tools + +| Original tool | Current equivalent | Notes | +|---|---|---| +| `create.case` | `platform.core.cases.manage` | Cases tooling rebuilt; `security_solution_setup.create_case` alias also seen | +| `create.channel`, `check.on.call.schedule`, `get.time` | `platform.core.generate_workflow` inputs | Workflow authoring folds these primitives into generated workflow steps | +| `filestore.read /skills/.md` | `platform.core.load_skill` + `platform.core.search_relevant_skills` | Skill activation moved from raw file reads to a first-class tool | +| `entity.store.search` / `entity.store.get_analytics` | `platform.core.search` (`.entities.v1.latest.*`) or ES|QL via `generate_esql`/`execute_esql` | Entities query surface consolidated | +| `attack_discovery.analyze` | Attack-discovery skill flow | Moved behind skill instructions | + +## Absent on current eval stacks + +| Original tool | Status | +|---|---| +| `vt.hash.lookup` | VirusTotal connector not provisioned on eval stacks (230 calls in the original run) | + +## New on current eval stacks (no original counterpart) + +| Current tool | Purpose | +|---|---| +| `platform.core.write_todos` | Agent framework todo tracking (321 calls in the 2026-08-21 sweep) | +| `platform.core.execute_api` / `discover_apis` / `describe_api` | Generic API discovery layer (197 calls combined) | +| `platform.core.sml_search` | Semantic search | +| `platform.core.run_subagent` | Sub-agent fan-out | +| `platform.core.list_files`, `platform.core.list_attachments` | Listing primitives | + +## Known divergences caused by the surface change + +1. **Tool-set similarity between benchmark generations is structurally low** + (mean Jaccard ≈ 0.13 across 315 paired cells). This is expected and does not + indicate model regression. +2. **Categories hitting `vt.*` flows** (alert-analysis, threat-hunting) lose an + enrichment step on current stacks. ExpectedTools annotations must reference + the current surface, not the original one. +3. **Skill-load accounting**: original `filestore.read /skills/...` and current + `load_skill` must both be treated as skill activation when comparing + SkillInvoked-style metrics across generations. + +Fixture provenance for the current generation is pinned via +`provenance.fixtureFingerprint` in `persona_matrix.config.json`. diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/evals/persona_matrix.spec.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/evals/persona_matrix.spec.ts index e06a0499ba6f7..488bafd83e472 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/evals/persona_matrix.spec.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/evals/persona_matrix.spec.ts @@ -10,10 +10,13 @@ import type { Client as EsClient } from '@elastic/elasticsearch'; import { evaluate } from '../src/evaluate'; import { personaMatrixDataset } from '../src/datasets'; import { seedChrysalisAlerts, cleanupChrysalisAlerts } from '../src/fixtures/chrysalis_seed'; +import { seedPersonaMatrixEnvironment, cleanupEnvSeeds } from '../src/fixtures/env_seeds'; import { seedPersonaMatrixTools, + attachPersonaMatrixToolsToAgent, cleanupPersonaMatrixTools, } from '../src/fixtures/persona_matrix_tools_seed'; +import { assertPersonaMatrixToolsRegistered } from '../src/fixtures/tool_registration_check'; const DATASET_NAME = 'security: security-persona-matrix'; const DATASET_DESCRIPTION = @@ -23,18 +26,36 @@ evaluate.describe('Security Persona Matrix', { tag: tags.stateful.classic }, () evaluate.beforeAll(async ({ esClient, kbnClient, log }) => { await seedChrysalisAlerts({ esClient: esClient as unknown as EsClient, log, count: 3 }); log.info('[persona-matrix] seeded Chrysalis alerts'); + await seedPersonaMatrixEnvironment({ + esClient: esClient as unknown as EsClient, + kbnClient, + log, + }); + log.info('[persona-matrix] seeded environment-truth data (endpoint, labs, ti-mock, entity)'); await seedPersonaMatrixTools({ kbnClient, log }); + // Registry creation alone leaves the tools invisible to the model — the + // default agent ships `tools: []` and only sees `defaultAgentToolIds`. + await attachPersonaMatrixToolsToAgent({ kbnClient, log }); log.info('[persona-matrix] seeded virustotal_lookup + on_call_lookup tools'); }); evaluate.afterAll(async ({ esClient, kbnClient, log }) => { await cleanupChrysalisAlerts({ esClient: esClient as unknown as EsClient, log }); + await cleanupEnvSeeds({ + esClient: esClient as unknown as EsClient, + kbnClient, + log, + }); await cleanupPersonaMatrixTools({ kbnClient, log }); }); - evaluate('all 21 examples', async ({ evaluateDataset, log }) => { + evaluate('all 21 examples', async ({ evaluateDataset, kbnClient, log }) => { log.info(`Running persona matrix evaluation with ${personaMatrixDataset.length} examples`); + // Pre-flight: fail fast if an expected custom tool isn't registered, rather + // than silently scoring a tool that doesn't exist. + await assertPersonaMatrixToolsRegistered({ kbnClient, log }); + await evaluateDataset({ dataset: { name: DATASET_NAME, diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/moon.yml b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/moon.yml index a7ad9a606f3e7..e60047af6f164 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/moon.yml +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/moon.yml @@ -20,11 +20,13 @@ dependsOn: - '@kbn/agent-builder-common' - '@kbn/core' - '@kbn/evals' + - '@kbn/evals-extensions' - '@kbn/evals-suite-attack-discovery' - '@kbn/scout' - '@kbn/security-evals-alerts-snapshot' - '@kbn/tooling-log' - '@kbn/kbn-client' + - '@kbn/evals-common' tags: - functional-tests - package @@ -35,6 +37,7 @@ tags: fileGroups: src: - '**/*.ts' + - persona_matrix.config.json - '!target/**/*' jest-config: - jest.config.js diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/persona_matrix.config.json b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/persona_matrix.config.json new file mode 100644 index 0000000000000..256ee9f42fce9 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/persona_matrix.config.json @@ -0,0 +1,449 @@ +{ + "title": "Persona Matrix \u2014 per-prompt proprietary sweep", + "branch": "main", + "lookbackDays": 45, + "scoring": { + "useVerdictLadder": true, + "requireEisJudge": true, + "excludeSelfJudged": true + }, + "defaultScale": 10, + "decimals": 2, + "notRecommendedBelow": 0, + "minCoverage": 12, + "overall": { + "runStdev": 0.478, + "excludeSaturatedEvaluators": true + }, + "toolCallWarnAbove": 40, + "notRecommendedLabel": "Not recommended", + "showOverall": true, + "columns": [ + { + "id": "alert-analysis-a", + "label": "Alert Analysis A", + "group": "Alert Analysis", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "alert-analysis-a" + ], + "weight": 1 + }, + { + "id": "alert-analysis-b", + "label": "Alert Analysis B", + "group": "Alert Analysis", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "alert-analysis-b" + ], + "weight": 1 + }, + { + "id": "alert-analysis-c", + "label": "Alert Analysis C", + "group": "Alert Analysis", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "alert-analysis-c" + ], + "weight": 1 + }, + { + "id": "entity-analytics-a", + "label": "Entity Analytics A", + "group": "Entity Analytics", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "entity-analytics-a" + ], + "weight": 1 + }, + { + "id": "entity-analytics-b", + "label": "Entity Analytics B", + "group": "Entity Analytics", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "entity-analytics-b" + ], + "weight": 1 + }, + { + "id": "entity-analytics-c", + "label": "Entity Analytics C", + "group": "Entity Analytics", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "entity-analytics-c" + ], + "weight": 1 + }, + { + "id": "threat-hunting-a", + "label": "Threat Hunting A", + "group": "Threat Hunting", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "threat-hunting-a" + ], + "weight": 1 + }, + { + "id": "threat-hunting-b", + "label": "Threat Hunting B", + "group": "Threat Hunting", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "threat-hunting-b" + ], + "weight": 1 + }, + { + "id": "threat-hunting-c", + "label": "Threat Hunting C", + "group": "Threat Hunting", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "threat-hunting-c" + ], + "weight": 1 + }, + { + "id": "detection-rule-edit-a", + "label": "Detection Rules A", + "group": "Detection Rules", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "detection-rule-edit-a" + ], + "weight": 1 + }, + { + "id": "detection-rule-edit-b", + "label": "Detection Rules B", + "group": "Detection Rules", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "detection-rule-edit-b" + ], + "weight": 1 + }, + { + "id": "detection-rule-edit-c", + "label": "Detection Rules C", + "group": "Detection Rules", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "detection-rule-edit-c" + ], + "weight": 1 + }, + { + "id": "workflow-authoring-a", + "label": "Workflow Authoring A", + "group": "Workflow Authoring", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "workflow-authoring-a" + ], + "weight": 1 + }, + { + "id": "workflow-authoring-b", + "label": "Workflow Authoring B", + "group": "Workflow Authoring", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "workflow-authoring-b" + ], + "weight": 1 + }, + { + "id": "workflow-authoring-c", + "label": "Workflow Authoring C", + "group": "Workflow Authoring", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "workflow-authoring-c" + ], + "weight": 1 + }, + { + "id": "workflow-execution-a", + "label": "Triggering Workflows A", + "group": "Triggering Workflows", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "workflow-execution-a" + ], + "weight": 1 + }, + { + "id": "workflow-execution-b", + "label": "Triggering Workflows B", + "group": "Triggering Workflows", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "workflow-execution-b" + ], + "weight": 1 + }, + { + "id": "workflow-execution-c", + "label": "Triggering Workflows C", + "group": "Triggering Workflows", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "workflow-execution-c" + ], + "weight": 1 + }, + { + "id": "multi-step-a", + "label": "Multi-Step Executions A", + "group": "Multi-Step Executions", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "multi-step-a" + ], + "weight": 1 + }, + { + "id": "multi-step-b", + "label": "Multi-Step Executions B", + "group": "Multi-Step Executions", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "multi-step-b" + ], + "weight": 1 + }, + { + "id": "multi-step-c", + "label": "Multi-Step Executions C", + "group": "Multi-Step Executions", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "multi-step-c" + ], + "weight": 1 + }, + { + "id": "attack-discovery", + "label": "Kill-Chain Discovery", + "group": "Attack Discovery", + "suites": [ + "attack-discovery-agent-builder" + ], + "evaluators": [ + "AttackDiscoveryBasic", + "Rubric", + "Criteria" + ], + "weight": 1, + "allowSelfJudged": true, + "branch": ["main", "patrykkopycinski:feat/attack-discovery-agent-builder-evals", "feat/evals-extensions-matrix-v3"] + }, + { + "id": "migrations-rules", + "label": "Rule Translation", + "group": "Automatic Migrations", + "suites": [ + "security-automatic-migrations" + ], + "evaluators": [ + "Translation Result", + "Translated ESQL Validity", + "Custom Query Accuracy", + "Prebuilt Rule Match", + "Integration Match", + "Hallucination Detection", + "Unsupported Pattern Detection", + "LOOKUP JOIN Preservation", + "NL Description Faithfulness" + ], + "weight": 1, + "branch": ["main", "elastic:fix/weekly-evals-matrix", "elastic:feat/siem-migrations-invoke-endpoint", "feat/evals-extensions-matrix-v3"] + }, + { + "id": "migrations-dashboards", + "label": "Dashboard Translation", + "group": "Automatic Migrations", + "suites": [ + "security-automatic-migrations" + ], + "evaluators": [ + "Translation Completeness", + "ES|QL Completeness", + "Index Pattern Validity", + "Panel Count Preservation", + "Lookup Join Presence", + "Markdown Error Detection", + "translation_fidelity" + ], + "weight": 1, + "branch": ["main", "elastic:fix/weekly-evals-matrix", "elastic:feat/siem-migrations-invoke-endpoint", "feat/evals-extensions-matrix-v3"] + } + ], + "models": [ + { + "id": "anthropic-claude-4.5-haiku", + "label": "Claude Haiku 4.5" + }, + { + "id": "anthropic-claude-4.5-sonnet", + "label": "Claude Sonnet 4.5" + }, + { + "id": "anthropic-claude-4.5-opus", + "label": "Claude Opus 4.5" + }, + { + "id": "anthropic-claude-4.6-sonnet", + "label": "Claude Sonnet 4.6", + "matchIds": [ + "anthropic-claude-4.6-sonnet-chat_completion" + ] + }, + { + "id": "anthropic-claude-4.6-opus", + "label": "Claude Opus 4.6" + }, + { + "id": "anthropic-claude-4.7-opus", + "label": "Claude Opus 4.7" + }, + { + "id": "anthropic-claude-4.8-opus", + "label": "Claude Opus 4.8" + }, + { + "id": "openai-gpt-5.2", + "label": "GPT-5.2" + }, + { + "id": "openai-gpt-5.4", + "label": "GPT-5.4" + }, + { + "id": "openai-gpt-5.4-mini", + "label": "GPT-5.4 Mini" + }, + { + "id": "openai-gpt-5.4-nano", + "label": "GPT-5.4 Nano" + }, + { + "id": "google-gemini-2.5-flash", + "label": "Gemini 2.5 Flash" + }, + { + "id": "google-gemini-2.5-pro", + "label": "Gemini 2.5 Pro" + }, + { + "id": "google-gemini-3.0-flash", + "label": "Gemini 3.0 Flash" + }, + { + "id": "google-gemini-3.1-flash-lite", + "label": "Gemini 3.1 Flash Lite" + }, + { + "id": "google-gemini-3.1-pro", + "label": "Gemini 3.1 Pro" + }, + { + "id": "google-gemini-3.5-flash", + "label": "Gemini 3.5 Flash" + }, + { + "id": "anthropic-claude-5-sonnet", + "label": "Claude Sonnet 5" + }, + { + "id": "openai-gpt-5.5", + "label": "GPT-5.5" + }, + { + "id": "openai-gpt-oss-120b", + "label": "GPT-OSS 120B", + "openSource": true + }, + { + "id": "zai-glm-5-2", + "label": "GLM-5.2", + "openSource": true + }, + { + "id": "openrouter-zai-glm-5-3-flash", + "label": "GLM-5.3 Flash", + "openSource": true, + "matchIds": ["z-ai/glm-5.3-flash"] + }, + { + "id": "openrouter-deepseek-v4-pro", + "label": "DeepSeek V4 Pro", + "openSource": true, + "matchIds": ["deepseek/deepseek-v4-pro-0813"] + } + ], + "tokenCost": {}, + "provenance": { + "fixtureFingerprint": "sha256:cf9c1d22 (env_seeds.ts + persona_matrix_tools_seed.ts, persona-matrix-env-truth worktree)", + "methodologyNotes": [ + "ExpectedToolCalled is all-or-nothing over the FULL declared expectedTools set since 2026-08-21 (previously only expectedTools[0] was checked). Categories whose skill contract changed (e.g. threat-hunting requiring generate_esql before execute_esql) score lower than in pre-fix matrices for identical traces.", + "FinalAnswerPresent evaluator added 2026-08-22: scores 0 when a run ends on a tool call with no user-facing final message (observed in 62% of detection-rule-edit runs). Folded into every cell mean.", + "MinExpectedSteps evaluator added 2026-08-22: scores 0 when a run made fewer tool calls than the declared expectedTools count \u2014 flags premature-termination / 'gave up without trying' (complements FinalAnswerPresent). Folded into every cell mean.", + "Repetition depth varies by model: determinism-sweep models ran 3 reps/example, batch models 1 rep. Trace cards carry an N reps badge; 1-rep cells have proportionally higher variance.", + "Scores are means over 10 evaluators (8 score-typed + FinalAnswerPresent + MinExpectedSteps), scaled to 0-10. Token/latency/tool-count observability evaluators are excluded from cell means and shown separately in the token-cost table.", + "Judged evaluators (Groundedness, Factuality, Relevance) are scored from their categorical verdict via an ordinal ladder since 2026-08-24, not the per-run geometric mean over a re-extracted claim list. Repetitions re-run the agent, which emits semantically equivalent but textually different answers (inter-rep output similarity 0.172, zero identical across 8,482 measured docs); the judge then extracts a different claim list from different prose and the continuous mean wobbles. Measured on 312 multi-repetition cells, rank flip between repetitions falls from 83.3% to 33.3%. Contract evaluators are deterministic and unaffected.", + "Scores from judges that are not EIS-pinned are excluded since 2026-08-24 (2,077 docs, 5.7%), as are 807 self-judged docs where a model graded its own output. A non-EIS judge id (LiteLLM alias, HuggingFace repo path, local quantisation) cannot be re-run to reproduce a number. Cells whose only grades came from excluded judges are absent rather than estimated.", + "Scores in this matrix are NOT comparable to matrices generated before 2026-08-24: the judged-evaluator scoring basis changed from continuous mean to verdict ladder, and the judge population changed. Compare model-to-model within this matrix, not cell-to-cell across generations.", + "Fixture fingerprint pins the dataset/tool seeds this matrix was scored against; comparing across fingerprints is a cross-generation comparison, not a model delta." + ] + } +} diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/persona_matrix.production.config.json b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/persona_matrix.production.config.json new file mode 100644 index 0000000000000..630d51a676418 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/persona_matrix.production.config.json @@ -0,0 +1,464 @@ +{ + "title": "Persona Matrix \u2014 per-prompt proprietary sweep", + "branch": "main", + "lookbackDays": 45, + "scoring": { + "useVerdictLadder": true, + "requireEisJudge": true, + "excludeSelfJudged": true + }, + "defaultScale": 10, + "decimals": 2, + "notRecommendedBelow": 0, + "minCoverage": 12, + "overall": { + "runStdev": 0.478, + "excludeSaturatedEvaluators": true + }, + "toolCallWarnAbove": 40, + "notRecommendedLabel": "Not recommended", + "showOverall": true, + "columns": [ + { + "id": "alert-analysis-a", + "label": "Alert Analysis A", + "group": "Alert Analysis", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "alert-analysis-a" + ], + "weight": 1 + }, + { + "id": "alert-analysis-b", + "label": "Alert Analysis B", + "group": "Alert Analysis", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "alert-analysis-b" + ], + "weight": 1 + }, + { + "id": "alert-analysis-c", + "label": "Alert Analysis C", + "group": "Alert Analysis", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "alert-analysis-c" + ], + "weight": 1 + }, + { + "id": "entity-analytics-a", + "label": "Entity Analytics A", + "group": "Entity Analytics", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "entity-analytics-a" + ], + "weight": 1 + }, + { + "id": "entity-analytics-b", + "label": "Entity Analytics B", + "group": "Entity Analytics", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "entity-analytics-b" + ], + "weight": 1 + }, + { + "id": "entity-analytics-c", + "label": "Entity Analytics C", + "group": "Entity Analytics", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "entity-analytics-c" + ], + "weight": 1 + }, + { + "id": "threat-hunting-a", + "label": "Threat Hunting A", + "group": "Threat Hunting", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "threat-hunting-a" + ], + "weight": 1 + }, + { + "id": "threat-hunting-b", + "label": "Threat Hunting B", + "group": "Threat Hunting", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "threat-hunting-b" + ], + "weight": 1 + }, + { + "id": "threat-hunting-c", + "label": "Threat Hunting C", + "group": "Threat Hunting", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "threat-hunting-c" + ], + "weight": 1 + }, + { + "id": "detection-rule-edit-a", + "label": "Detection Rules A", + "group": "Detection Rules", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "detection-rule-edit-a" + ], + "weight": 1 + }, + { + "id": "detection-rule-edit-b", + "label": "Detection Rules B", + "group": "Detection Rules", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "detection-rule-edit-b" + ], + "weight": 1 + }, + { + "id": "detection-rule-edit-c", + "label": "Detection Rules C", + "group": "Detection Rules", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "detection-rule-edit-c" + ], + "weight": 1 + }, + { + "id": "workflow-authoring-a", + "label": "Workflow Authoring A", + "group": "Workflow Authoring", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "workflow-authoring-a" + ], + "weight": 1 + }, + { + "id": "workflow-authoring-b", + "label": "Workflow Authoring B", + "group": "Workflow Authoring", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "workflow-authoring-b" + ], + "weight": 1 + }, + { + "id": "workflow-authoring-c", + "label": "Workflow Authoring C", + "group": "Workflow Authoring", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "workflow-authoring-c" + ], + "weight": 1 + }, + { + "id": "workflow-execution-a", + "label": "Triggering Workflows A", + "group": "Triggering Workflows", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "workflow-execution-a" + ], + "weight": 1 + }, + { + "id": "workflow-execution-b", + "label": "Triggering Workflows B", + "group": "Triggering Workflows", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "workflow-execution-b" + ], + "weight": 1 + }, + { + "id": "workflow-execution-c", + "label": "Triggering Workflows C", + "group": "Triggering Workflows", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "workflow-execution-c" + ], + "weight": 1 + }, + { + "id": "multi-step-a", + "label": "Multi-Step Executions A", + "group": "Multi-Step Executions", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "multi-step-a" + ], + "weight": 1 + }, + { + "id": "multi-step-b", + "label": "Multi-Step Executions B", + "group": "Multi-Step Executions", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "multi-step-b" + ], + "weight": 1 + }, + { + "id": "multi-step-c", + "label": "Multi-Step Executions C", + "group": "Multi-Step Executions", + "suites": [ + "security-persona-matrix" + ], + "examplePrefixes": [ + "multi-step-c" + ], + "weight": 1 + }, + { + "id": "attack-discovery", + "label": "Kill-Chain Discovery", + "group": "Attack Discovery", + "suites": [ + "attack-discovery-agent-builder" + ], + "evaluators": [ + "AttackDiscoveryBasic", + "Rubric", + "Criteria" + ], + "weight": 1, + "allowSelfJudged": true, + "branch": [ + "main", + "patrykkopycinski:feat/attack-discovery-agent-builder-evals", + "feat/evals-extensions-matrix-v3" + ] + }, + { + "id": "migrations-rules", + "label": "Rule Translation", + "group": "Automatic Migrations", + "suites": [ + "security-automatic-migrations" + ], + "evaluators": [ + "Translation Result", + "Translated ESQL Validity", + "Custom Query Accuracy", + "Prebuilt Rule Match", + "Integration Match", + "Hallucination Detection", + "Unsupported Pattern Detection", + "LOOKUP JOIN Preservation", + "NL Description Faithfulness" + ], + "weight": 1, + "branch": [ + "main", + "elastic:fix/weekly-evals-matrix", + "elastic:feat/siem-migrations-invoke-endpoint", + "feat/evals-extensions-matrix-v3" + ] + }, + { + "id": "migrations-dashboards", + "label": "Dashboard Translation", + "group": "Automatic Migrations", + "suites": [ + "security-automatic-migrations" + ], + "evaluators": [ + "Translation Completeness", + "ES|QL Completeness", + "Index Pattern Validity", + "Panel Count Preservation", + "Lookup Join Presence", + "Markdown Error Detection", + "translation_fidelity" + ], + "weight": 1, + "branch": [ + "main", + "elastic:fix/weekly-evals-matrix", + "elastic:feat/siem-migrations-invoke-endpoint", + "feat/evals-extensions-matrix-v3" + ] + } + ], + "models": [ + { + "id": "anthropic-claude-4.5-haiku", + "label": "Claude Haiku 4.5" + }, + { + "id": "anthropic-claude-4.5-sonnet", + "label": "Claude Sonnet 4.5" + }, + { + "id": "anthropic-claude-4.5-opus", + "label": "Claude Opus 4.5" + }, + { + "id": "anthropic-claude-4.6-sonnet", + "label": "Claude Sonnet 4.6", + "matchIds": [ + "anthropic-claude-4.6-sonnet-chat_completion" + ] + }, + { + "id": "anthropic-claude-4.6-opus", + "label": "Claude Opus 4.6" + }, + { + "id": "anthropic-claude-4.7-opus", + "label": "Claude Opus 4.7" + }, + { + "id": "anthropic-claude-4.8-opus", + "label": "Claude Opus 4.8" + }, + { + "id": "openai-gpt-5.2", + "label": "GPT-5.2" + }, + { + "id": "openai-gpt-5.4", + "label": "GPT-5.4" + }, + { + "id": "openai-gpt-5.4-mini", + "label": "GPT-5.4 Mini" + }, + { + "id": "openai-gpt-5.4-nano", + "label": "GPT-5.4 Nano" + }, + { + "id": "google-gemini-2.5-flash", + "label": "Gemini 2.5 Flash" + }, + { + "id": "google-gemini-2.5-pro", + "label": "Gemini 2.5 Pro" + }, + { + "id": "google-gemini-3.0-flash", + "label": "Gemini 3.0 Flash" + }, + { + "id": "google-gemini-3.1-flash-lite", + "label": "Gemini 3.1 Flash Lite" + }, + { + "id": "google-gemini-3.1-pro", + "label": "Gemini 3.1 Pro" + }, + { + "id": "google-gemini-3.5-flash", + "label": "Gemini 3.5 Flash" + }, + { + "id": "anthropic-claude-5-sonnet", + "label": "Claude Sonnet 5" + }, + { + "id": "openai-gpt-5.5", + "label": "GPT-5.5" + }, + { + "id": "openai-gpt-oss-120b", + "label": "GPT-OSS 120B", + "openSource": true + }, + { + "id": "openrouter-deepseek-v4-pro", + "label": "DeepSeek V4 Pro", + "openSource": true, + "matchIds": [ + "deepseek/deepseek-v4-pro-0813" + ] + }, + { + "id": "selfhost-qwen38", + "label": "Qwen3.8-27B (self-hosted A100)", + "openSource": true, + "matchIds": [ + "qwen3.8-27b" + ] + } + ], + "tokenCost": {}, + "provenance": { + "fixtureFingerprint": "sha256:cf9c1d22 (env_seeds.ts + persona_matrix_tools_seed.ts, persona-matrix-env-truth worktree)", + "methodologyNotes": [ + "ExpectedToolCalled is all-or-nothing over the FULL declared expectedTools set since 2026-08-21 (previously only expectedTools[0] was checked). Categories whose skill contract changed (e.g. threat-hunting requiring generate_esql before execute_esql) score lower than in pre-fix matrices for identical traces.", + "FinalAnswerPresent evaluator added 2026-08-22: scores 0 when a run ends on a tool call with no user-facing final message (observed in 62% of detection-rule-edit runs). Folded into every cell mean.", + "MinExpectedSteps evaluator added 2026-08-22: scores 0 when a run made fewer tool calls than the declared expectedTools count \u2014 flags premature-termination / 'gave up without trying' (complements FinalAnswerPresent). Folded into every cell mean.", + "Repetition depth varies by model: determinism-sweep models ran 3 reps/example, batch models 1 rep. Trace cards carry an N reps badge; 1-rep cells have proportionally higher variance.", + "Scores are means over 10 evaluators (8 score-typed + FinalAnswerPresent + MinExpectedSteps), scaled to 0-10. Token/latency/tool-count observability evaluators are excluded from cell means and shown separately in the token-cost table.", + "Judged evaluators (Groundedness, Factuality, Relevance) are scored from their categorical verdict via an ordinal ladder since 2026-08-24, not the per-run geometric mean over a re-extracted claim list. Repetitions re-run the agent, which emits semantically equivalent but textually different answers (inter-rep output similarity 0.172, zero identical across 8,482 measured docs); the judge then extracts a different claim list from different prose and the continuous mean wobbles. Measured on 312 multi-repetition cells, rank flip between repetitions falls from 83.3% to 33.3%. Contract evaluators are deterministic and unaffected.", + "Scores from judges that are not EIS-pinned are excluded since 2026-08-24 (2,077 docs, 5.7%), as are 807 self-judged docs where a model graded its own output. A non-EIS judge id (LiteLLM alias, HuggingFace repo path, local quantisation) cannot be re-run to reproduce a number. Cells whose only grades came from excluded judges are absent rather than estimated.", + "Scores in this matrix are NOT comparable to matrices generated before 2026-08-24: the judged-evaluator scoring basis changed from continuous mean to verdict ladder, and the judge population changed. Compare model-to-model within this matrix, not cell-to-cell across generations.", + "Fixture fingerprint pins the dataset/tool seeds this matrix was scored against; comparing across fingerprints is a cross-generation comparison, not a model delta.", + "Rendered as of 2026-09-01: Agent Builder eval stacks on the Azure sweep fleet ran with agentBuilder:tracing:includeToolDetails=false from 2026-09-04, which strips gen_ai.tool.call.arguments from tool spans and makes the SkillInvoked evaluator unscoreable. The cutoff is applied identically to every model.", + "GLM models (zai-glm-5-2, z-ai/glm-5.3-flash) are excluded from this matrix: neither has a complete, correctly-instrumented 21-example run. This is a data-availability exclusion, not a quality judgement." + ] + } +} \ No newline at end of file diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/playwright.config.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/playwright.config.ts index f8f0c4c8e69e7..32c936c11f820 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/playwright.config.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/playwright.config.ts @@ -7,10 +7,24 @@ import { createPlaywrightEvalsConfig } from '@kbn/evals'; +// Default 30min covers a single-pass 21-example run (frontier models finish in +// single-digit minutes; a local vLLM L4 can take 45+). Determinism runs with +// EVAL_REPETITIONS=3 triple the workload (~90min); allow raising the ceiling +// from the environment instead of editing the config per-run. +const TIMEOUT_MINUTES = Number(process.env.PERSONA_MATRIX_TIMEOUT_MINUTES) || 30; + +// Examples are network-bound on the model under test, so the default single +// worker leaves the box mostly idle. Opt in per-run rather than raising the +// default: `beforeAll` seeds shared indices that `afterAll` deletes, so extra +// workers inside one suite tear down data the others are still using. Safe +// only when each worker gets its own stack. +const WORKERS = Number(process.env.PERSONA_MATRIX_WORKERS); + export default createPlaywrightEvalsConfig({ testDir: `${__dirname}/evals`, // The co-located Attack Discovery spec runs under its own config // (attack_discovery.playwright.config.ts) with the evals_tracing server config. testIgnore: '**/attack_discovery.spec.ts', - timeout: 30 * 60_000, + timeout: TIMEOUT_MINUTES * 60_000, + workers: WORKERS === 2 || WORKERS === 3 ? WORKERS : 1, }); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/__fixtures__/golden_scores_sample.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/__fixtures__/golden_scores_sample.ts new file mode 100644 index 0000000000000..fff6db1fd4a90 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/__fixtures__/golden_scores_sample.ts @@ -0,0 +1,1325 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Real golden score documents captured 2026-09-06 from the persona-matrix + * re-judge waves (executions sweep-1788685888-rj3 and siblings), trimmed to the + * fields a replay reads. Used to pin the replay planner against the schema the + * sweep actually writes -- synthetic fixtures cannot catch that drift. + */ +import type { EvaluationScoreDocument } from '@kbn/evals-common/impl/schemas/common_attributes.gen'; + +export const GOLDEN_SCORES_SAMPLE = [ + { + '@timestamp': '2026-09-06T10:05:46.442Z', + experiment_id: 'fixture', + example: { + id: 'workflow-authoring-a', + input: { + question: + "Use the workflow authoring skill to author a runnable Elastic workflow (YAML) that posts a Chrysalis triage summary to Slack. Give it a manual trigger and a 'me", + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-2.5-pro', + }, + output: { + messages: [ + { + message: + "I've created a workflow that will post a triage summary to Slack. When you run it, you'll be prompted to provide the summary message.\n\n\nKey details:\n- Manual tri', + }, + ], + }, + }, + evaluator: { + name: 'Latency', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s1of3::security-persona-matrix::openai-gpt-5.5', + }, + }, + { + '@timestamp': '2026-09-06T10:05:20.908Z', + experiment_id: 'fixture', + example: { + id: 'workflow-authoring-a', + input: { + question: + "Use the workflow authoring skill to author a runnable Elastic workflow (YAML) that posts a Chrysalis triage summary to Slack. Give it a manual trigger and a 'me", + }, + output: {}, + }, + task: { + model: { + id: 'openai-gpt-5.5', + }, + output: { + messages: [ + { + message: + 'Created and validated the runnable Elastic workflow YAML.\n\n\n\nKey details:\n- Manual tri', + }, + ], + }, + }, + evaluator: { + name: 'Tool Calls', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s1of3::security-persona-matrix::openai-gpt-5.5', + }, + }, + { + '@timestamp': '2026-09-06T10:05:15.906Z', + experiment_id: 'fixture', + example: { + id: 'workflow-authoring-a', + input: { + question: + "Use the workflow authoring skill to author a runnable Elastic workflow (YAML) that posts a Chrysalis triage summary to Slack. Give it a manual trigger and a 'me", + }, + output: {}, + }, + task: { + model: { + id: 'openai-gpt-5.5', + }, + output: { + messages: [ + { + message: + 'Created and validated the runnable Elastic workflow YAML.\n\n\n\nKey details:\n- Manual tri', + }, + ], + }, + }, + evaluator: { + name: 'Output Tokens', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s1of3::security-persona-matrix::openai-gpt-5.5', + }, + }, + { + '@timestamp': '2026-09-06T10:05:15.904Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-a', + input: { + question: + 'The Chrysalis loader hash is 275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f. Check this hash against VirusTotal and tell me the verdict.', + }, + output: {}, + }, + task: { + model: { + id: 'openai-gpt-5.5', + }, + output: { + messages: [ + { + message: + 'VirusTotal returned **no matching verdict** for the hash:\n\n`275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f`\n\nSo I don\u2019t have a malicious/benig', + }, + ], + }, + }, + evaluator: { + name: 'Latency', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s1of3::security-persona-matrix::openai-gpt-5.5', + }, + }, + { + '@timestamp': '2026-09-06T10:05:10.905Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-a', + input: { + question: + 'The Chrysalis loader hash is 275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f. Check this hash against VirusTotal and tell me the verdict.', + }, + output: {}, + }, + task: { + model: { + id: 'openai-gpt-5.5', + }, + output: { + messages: [ + { + message: + 'VirusTotal returned **no matching verdict** for the hash:\n\n`275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f`\n\nSo I don\u2019t have a malicious/benig', + }, + ], + }, + }, + evaluator: { + name: 'Tool Calls', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s1of3::security-persona-matrix::openai-gpt-5.5', + }, + }, + { + '@timestamp': '2026-09-06T10:05:05.897Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-a', + input: { + question: + 'The Chrysalis loader hash is 275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f. Check this hash against VirusTotal and tell me the verdict.', + }, + output: {}, + }, + task: { + model: { + id: 'openai-gpt-5.5', + }, + output: { + messages: [ + { + message: + 'VirusTotal returned **no matching verdict** for the hash:\n\n`275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f`\n\nSo I don\u2019t have a malicious/benig', + }, + ], + }, + }, + evaluator: { + name: 'Output Tokens', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s1of3::security-persona-matrix::openai-gpt-5.5', + }, + }, + { + '@timestamp': '2026-09-06T10:03:29.487Z', + experiment_id: 'fixture', + example: { + id: 'workflow-authoring-b', + input: { + question: + "Author an Elastic workflow in YAML that, on a manual trigger, posts a fixed one-line message 'Chrysalis hunt complete \u2014 see case for details' to Slack channel #", + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.1-flash-lite', + }, + output: { + messages: [ + { + message: + 'I have authored the requested workflow for you. \n\nIt uses a manual trigger and the `slack2.sendMessage` step to post "Chrysalis hunt complete \u2014 see case for det', + }, + ], + }, + }, + evaluator: { + name: 'Latency', + }, + metadata: { + execution_id: + 'sweep-1788685888-rj3-s2of3::security-persona-matrix::google-gemini-3.1-flash-lite', + }, + }, + { + '@timestamp': '2026-09-06T10:03:24.482Z', + experiment_id: 'fixture', + example: { + id: 'workflow-authoring-b', + input: { + question: + "Author an Elastic workflow in YAML that, on a manual trigger, posts a fixed one-line message 'Chrysalis hunt complete \u2014 see case for details' to Slack channel #", + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.1-flash-lite', + }, + output: { + messages: [ + { + message: + 'I have authored the requested workflow for you. \n\nIt uses a manual trigger and the `slack2.sendMessage` step to post "Chrysalis hunt complete \u2014 see case for det', + }, + ], + }, + }, + evaluator: { + name: 'Tool Calls', + }, + metadata: { + execution_id: + 'sweep-1788685888-rj3-s2of3::security-persona-matrix::google-gemini-3.1-flash-lite', + }, + }, + { + '@timestamp': '2026-09-06T10:03:19.475Z', + experiment_id: 'fixture', + example: { + id: 'workflow-authoring-b', + input: { + question: + "Author an Elastic workflow in YAML that, on a manual trigger, posts a fixed one-line message 'Chrysalis hunt complete \u2014 see case for details' to Slack channel #", + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.1-flash-lite', + }, + output: { + messages: [ + { + message: + 'I have authored the requested workflow for you. \n\nIt uses a manual trigger and the `slack2.sendMessage` step to post "Chrysalis hunt complete \u2014 see case for det', + }, + ], + }, + }, + evaluator: { + name: 'Output Tokens', + }, + metadata: { + execution_id: + 'sweep-1788685888-rj3-s2of3::security-persona-matrix::google-gemini-3.1-flash-lite', + }, + }, + { + '@timestamp': '2026-09-06T10:03:29.485Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-b', + input: { + question: + 'Who is currently on call to own a Chrysalis incident response? Look up the on-call schedule and tell me the primary responder.', + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.1-flash-lite', + }, + output: { + messages: [ + { + message: + 'The on-call schedule appears to be outdated, as the most recent entry for the primary responder, Dana Whitfield, ended on July 27, 2026.\n\nI am unable to identif', + }, + ], + }, + }, + evaluator: { + name: 'Latency', + }, + metadata: { + execution_id: + 'sweep-1788685888-rj3-s2of3::security-persona-matrix::google-gemini-3.1-flash-lite', + }, + }, + { + '@timestamp': '2026-09-06T10:03:24.480Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-b', + input: { + question: + 'Who is currently on call to own a Chrysalis incident response? Look up the on-call schedule and tell me the primary responder.', + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.1-flash-lite', + }, + output: { + messages: [ + { + message: + 'The on-call schedule appears to be outdated, as the most recent entry for the primary responder, Dana Whitfield, ended on July 27, 2026.\n\nI am unable to identif', + }, + ], + }, + }, + evaluator: { + name: 'Tool Calls', + }, + metadata: { + execution_id: + 'sweep-1788685888-rj3-s2of3::security-persona-matrix::google-gemini-3.1-flash-lite', + }, + }, + { + '@timestamp': '2026-09-06T10:03:19.474Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-b', + input: { + question: + 'Who is currently on call to own a Chrysalis incident response? Look up the on-call schedule and tell me the primary responder.', + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.1-flash-lite', + }, + output: { + messages: [ + { + message: + 'The on-call schedule appears to be outdated, as the most recent entry for the primary responder, Dana Whitfield, ended on July 27, 2026.\n\nI am unable to identif', + }, + ], + }, + }, + evaluator: { + name: 'Output Tokens', + }, + metadata: { + execution_id: + 'sweep-1788685888-rj3-s2of3::security-persona-matrix::google-gemini-3.1-flash-lite', + }, + }, + { + '@timestamp': '2026-09-06T09:58:34.991Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-c', + input: { + question: + "Open a Security case for the confirmed Chrysalis incident on srv-win-defend-01. Title it 'Chrysalis backdoor \u2014 srv-win-defend-01', set severity to critical, and", + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.5-flash', + }, + output: { + messages: [ + { + message: + 'I have opened a critical Security case for the confirmed Chrysalis backdoor incident on `srv-win-defend-01`.\n\n### Summary of Side-Loading Activity\n- **Malware f', + }, + ], + }, + }, + evaluator: { + name: 'Latency', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s3of3::security-persona-matrix::google-gemini-3.5-flash', + }, + }, + { + '@timestamp': '2026-09-06T09:58:29.986Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-c', + input: { + question: + "Open a Security case for the confirmed Chrysalis incident on srv-win-defend-01. Title it 'Chrysalis backdoor \u2014 srv-win-defend-01', set severity to critical, and", + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.5-flash', + }, + output: { + messages: [ + { + message: + 'I have opened a critical Security case for the confirmed Chrysalis backdoor incident on `srv-win-defend-01`.\n\n### Summary of Side-Loading Activity\n- **Malware f', + }, + ], + }, + }, + evaluator: { + name: 'Tool Calls', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s3of3::security-persona-matrix::google-gemini-3.5-flash', + }, + }, + { + '@timestamp': '2026-09-06T09:58:24.982Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-c', + input: { + question: + "Open a Security case for the confirmed Chrysalis incident on srv-win-defend-01. Title it 'Chrysalis backdoor \u2014 srv-win-defend-01', set severity to critical, and", + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.5-flash', + }, + output: { + messages: [ + { + message: + 'I have opened a critical Security case for the confirmed Chrysalis backdoor incident on `srv-win-defend-01`.\n\n### Summary of Side-Loading Activity\n- **Malware f', + }, + ], + }, + }, + evaluator: { + name: 'Output Tokens', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s3of3::security-persona-matrix::google-gemini-3.5-flash', + }, + }, + { + '@timestamp': '2026-09-06T09:58:21.724Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-b', + input: { + question: + 'Who is currently on call to own a Chrysalis incident response? Look up the on-call schedule and tell me the primary responder.', + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.5-flash', + }, + output: { + messages: [ + { + message: + "Let's look at the fields we have in the `on-call-schedule` index:\nWe have:\n- Dana Whitfield (Primary) - Shift: 2026-07-20 to 2026-07-27.\n- Ravi Osei (Secondary)", + }, + ], + }, + }, + evaluator: { + name: 'Latency', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s2of3::security-persona-matrix::google-gemini-3.5-flash', + }, + }, + { + '@timestamp': '2026-09-06T09:58:16.722Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-b', + input: { + question: + 'Who is currently on call to own a Chrysalis incident response? Look up the on-call schedule and tell me the primary responder.', + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.5-flash', + }, + output: { + messages: [ + { + message: + "Let's look at the fields we have in the `on-call-schedule` index:\nWe have:\n- Dana Whitfield (Primary) - Shift: 2026-07-20 to 2026-07-27.\n- Ravi Osei (Secondary)", + }, + ], + }, + }, + evaluator: { + name: 'Tool Calls', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s2of3::security-persona-matrix::google-gemini-3.5-flash', + }, + }, + { + '@timestamp': '2026-09-06T09:58:11.718Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-b', + input: { + question: + 'Who is currently on call to own a Chrysalis incident response? Look up the on-call schedule and tell me the primary responder.', + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.5-flash', + }, + output: { + messages: [ + { + message: + "Let's look at the fields we have in the `on-call-schedule` index:\nWe have:\n- Dana Whitfield (Primary) - Shift: 2026-07-20 to 2026-07-27.\n- Ravi Osei (Secondary)", + }, + ], + }, + }, + evaluator: { + name: 'Output Tokens', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s2of3::security-persona-matrix::google-gemini-3.5-flash', + }, + }, + { + '@timestamp': '2026-09-06T09:58:25.372Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-c', + input: { + question: + "Open a Security case for the confirmed Chrysalis incident on srv-win-defend-01. Title it 'Chrysalis backdoor \u2014 srv-win-defend-01', set severity to critical, and", + }, + output: {}, + }, + task: { + model: { + id: 'openai-gpt-5.4-mini', + }, + output: { + messages: [ + { + message: + 'Created the Security case and set it to critical.\n\n', + }, + ], + }, + }, + evaluator: { + name: 'Latency', + }, + metadata: { + execution_id: 'sweep-1788685878-rjc-s3of3::security-persona-matrix::openai-gpt-5.4-mini', + }, + }, + { + '@timestamp': '2026-09-06T09:58:20.370Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-c', + input: { + question: + "Open a Security case for the confirmed Chrysalis incident on srv-win-defend-01. Title it 'Chrysalis backdoor \u2014 srv-win-defend-01', set severity to critical, and", + }, + output: {}, + }, + task: { + model: { + id: 'openai-gpt-5.4-mini', + }, + output: { + messages: [ + { + message: + 'Created the Security case and set it to critical.\n\n', + }, + ], + }, + }, + evaluator: { + name: 'Tool Calls', + }, + metadata: { + execution_id: 'sweep-1788685878-rjc-s3of3::security-persona-matrix::openai-gpt-5.4-mini', + }, + }, + { + '@timestamp': '2026-09-06T09:58:15.368Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-c', + input: { + question: + "Open a Security case for the confirmed Chrysalis incident on srv-win-defend-01. Title it 'Chrysalis backdoor \u2014 srv-win-defend-01', set severity to critical, and", + }, + output: {}, + }, + task: { + model: { + id: 'openai-gpt-5.4-mini', + }, + output: { + messages: [ + { + message: + 'Created the Security case and set it to critical.\n\n', + }, + ], + }, + }, + evaluator: { + name: 'Output Tokens', + }, + metadata: { + execution_id: 'sweep-1788685878-rjc-s3of3::security-persona-matrix::openai-gpt-5.4-mini', + }, + }, + { + '@timestamp': '2026-09-06T09:58:15.367Z', + experiment_id: 'fixture', + example: { + id: 'workflow-authoring-c', + input: { + question: + "I want to automate Chrysalis incident notifications. Author a workflow (YAML) with a manual trigger that takes a 'summary' string input and posts it to Slack #g", + }, + output: {}, + }, + task: { + model: { + id: 'openai-gpt-5.4-mini', + }, + output: { + messages: [ + { + message: + 'Done \u2014 I created the workflow draft.\n\nIf you want, I can also:\n- render the YAML diff/workflow attachment here, or\n- validate the workflow before you save/use i', + }, + ], + }, + }, + evaluator: { + name: 'Latency', + }, + metadata: { + execution_id: 'sweep-1788685878-rjc-s3of3::security-persona-matrix::openai-gpt-5.4-mini', + }, + }, + { + '@timestamp': '2026-09-06T09:58:10.362Z', + experiment_id: 'fixture', + example: { + id: 'workflow-authoring-c', + input: { + question: + "I want to automate Chrysalis incident notifications. Author a workflow (YAML) with a manual trigger that takes a 'summary' string input and posts it to Slack #g", + }, + output: {}, + }, + task: { + model: { + id: 'openai-gpt-5.4-mini', + }, + output: { + messages: [ + { + message: + 'Done \u2014 I created the workflow draft.\n\nIf you want, I can also:\n- render the YAML diff/workflow attachment here, or\n- validate the workflow before you save/use i', + }, + ], + }, + }, + evaluator: { + name: 'Tool Calls', + }, + metadata: { + execution_id: 'sweep-1788685878-rjc-s3of3::security-persona-matrix::openai-gpt-5.4-mini', + }, + }, + { + '@timestamp': '2026-09-06T09:58:05.359Z', + experiment_id: 'fixture', + example: { + id: 'workflow-authoring-c', + input: { + question: + "I want to automate Chrysalis incident notifications. Author a workflow (YAML) with a manual trigger that takes a 'summary' string input and posts it to Slack #g", + }, + output: {}, + }, + task: { + model: { + id: 'openai-gpt-5.4-mini', + }, + output: { + messages: [ + { + message: + 'Done \u2014 I created the workflow draft.\n\nIf you want, I can also:\n- render the YAML diff/workflow attachment here, or\n- validate the workflow before you save/use i', + }, + ], + }, + }, + evaluator: { + name: 'Output Tokens', + }, + metadata: { + execution_id: 'sweep-1788685878-rjc-s3of3::security-persona-matrix::openai-gpt-5.4-mini', + }, + }, + { + '@timestamp': '2026-09-06T09:58:16.899Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-b', + input: { + question: + 'Who is currently on call to own a Chrysalis incident response? Look up the on-call schedule and tell me the primary responder.', + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.1-pro', + }, + output: { + messages: [ + { + message: + "I checked the on-call schedule, but there doesn't seem to be an active shift logged for the current date (September 6, 2026). The most recent primary on-call re", + }, + ], + }, + }, + evaluator: { + name: 'Latency', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s2of3::security-persona-matrix::google-gemini-3.1-pro', + }, + }, + { + '@timestamp': '2026-09-06T09:58:11.898Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-b', + input: { + question: + 'Who is currently on call to own a Chrysalis incident response? Look up the on-call schedule and tell me the primary responder.', + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.1-pro', + }, + output: { + messages: [ + { + message: + "I checked the on-call schedule, but there doesn't seem to be an active shift logged for the current date (September 6, 2026). The most recent primary on-call re", + }, + ], + }, + }, + evaluator: { + name: 'Tool Calls', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s2of3::security-persona-matrix::google-gemini-3.1-pro', + }, + }, + { + '@timestamp': '2026-09-06T09:58:06.891Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-b', + input: { + question: + 'Who is currently on call to own a Chrysalis incident response? Look up the on-call schedule and tell me the primary responder.', + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.1-pro', + }, + output: { + messages: [ + { + message: + "I checked the on-call schedule, but there doesn't seem to be an active shift logged for the current date (September 6, 2026). The most recent primary on-call re", + }, + ], + }, + }, + evaluator: { + name: 'Output Tokens', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s2of3::security-persona-matrix::google-gemini-3.1-pro', + }, + }, + { + '@timestamp': '2026-09-06T09:58:01.889Z', + experiment_id: 'fixture', + example: { + id: 'workflow-authoring-b', + input: { + question: + "Author an Elastic workflow in YAML that, on a manual trigger, posts a fixed one-line message 'Chrysalis hunt complete \u2014 see case for details' to Slack channel #", + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.1-pro', + }, + output: { + messages: [ + { + message: + 'I have created the requested workflow. It uses a manual trigger and an HTTP step configured to use a Slack connector to post your message to `#general`. \n\nHere ', + }, + ], + }, + }, + evaluator: { + name: 'Latency', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s2of3::security-persona-matrix::google-gemini-3.1-pro', + }, + }, + { + '@timestamp': '2026-09-06T09:57:56.885Z', + experiment_id: 'fixture', + example: { + id: 'workflow-authoring-b', + input: { + question: + "Author an Elastic workflow in YAML that, on a manual trigger, posts a fixed one-line message 'Chrysalis hunt complete \u2014 see case for details' to Slack channel #", + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.1-pro', + }, + output: { + messages: [ + { + message: + 'I have created the requested workflow. It uses a manual trigger and an HTTP step configured to use a Slack connector to post your message to `#general`. \n\nHere ', + }, + ], + }, + }, + evaluator: { + name: 'Tool Calls', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s2of3::security-persona-matrix::google-gemini-3.1-pro', + }, + }, + { + '@timestamp': '2026-09-06T09:57:51.881Z', + experiment_id: 'fixture', + example: { + id: 'workflow-authoring-b', + input: { + question: + "Author an Elastic workflow in YAML that, on a manual trigger, posts a fixed one-line message 'Chrysalis hunt complete \u2014 see case for details' to Slack channel #", + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.1-pro', + }, + output: { + messages: [ + { + message: + 'I have created the requested workflow. It uses a manual trigger and an HTTP step configured to use a Slack connector to post your message to `#general`. \n\nHere ', + }, + ], + }, + }, + evaluator: { + name: 'Output Tokens', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s2of3::security-persona-matrix::google-gemini-3.1-pro', + }, + }, + { + '@timestamp': '2026-09-06T09:56:08.937Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-a', + input: { + question: + 'The Chrysalis loader hash is 275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f. Check this hash against VirusTotal and tell me the verdict.', + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.0-flash', + }, + output: { + messages: [ + { + message: + 'The VirusTotal verdict for the Chrysalis loader hash (`275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f`) is **malicious**.\n\n**Details:**\n* **', + }, + ], + }, + }, + evaluator: { + name: 'Latency', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s1of3::security-persona-matrix::google-gemini-3.0-flash', + }, + }, + { + '@timestamp': '2026-09-06T09:56:03.933Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-a', + input: { + question: + 'The Chrysalis loader hash is 275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f. Check this hash against VirusTotal and tell me the verdict.', + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.0-flash', + }, + output: { + messages: [ + { + message: + 'The VirusTotal verdict for the Chrysalis loader hash (`275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f`) is **malicious**.\n\n**Details:**\n* **', + }, + ], + }, + }, + evaluator: { + name: 'Tool Calls', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s1of3::security-persona-matrix::google-gemini-3.0-flash', + }, + }, + { + '@timestamp': '2026-09-06T09:55:58.929Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-a', + input: { + question: + 'The Chrysalis loader hash is 275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f. Check this hash against VirusTotal and tell me the verdict.', + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.0-flash', + }, + output: { + messages: [ + { + message: + 'The VirusTotal verdict for the Chrysalis loader hash (`275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f`) is **malicious**.\n\n**Details:**\n* **', + }, + ], + }, + }, + evaluator: { + name: 'Output Tokens', + }, + metadata: { + execution_id: 'sweep-1788685888-rj3-s1of3::security-persona-matrix::google-gemini-3.0-flash', + }, + }, + { + '@timestamp': '2026-09-06T09:56:02.371Z', + experiment_id: 'fixture', + example: { + id: 'workflow-execution-c', + input: { + question: + "Open a Security case for the confirmed Chrysalis incident on srv-win-defend-01. Title it 'Chrysalis backdoor \u2014 srv-win-defend-01', set severity to critical, and", + }, + output: {}, + }, + task: { + model: { + id: 'google-gemini-3.0-flash', + }, + output: { + messages: [ + { + message: + 'I\'ve opened a new critical Security case to track the Chrysalis incident on **srv-win-defend-01**.\n\n { + it('persists null sampling controls and hashes tool_id + params, not provider call ids', async () => { + const fetch = jest.fn().mockResolvedValue({ + response: { message: 'done' }, + steps: [ + { type: 'tool_call', tool_id: 'search', tool_call_id: 'toolu_A', params: { q: 'x' } }, + ], + }); + const log = { warning: jest.fn() }; + const client = new PersonaMatrixChatClient(fetch, log as never, 'connector-1'); + + const first = await client.query('q'); + fetch.mockResolvedValue({ + response: { message: 'done' }, + steps: [ + { type: 'tool_call', tool_id: 'search', tool_call_id: 'toolu_B', params: { q: 'x' } }, + ], + }); + const second = await client.query('q'); + + expect(first.sampling).toEqual({ + connectorId: 'connector-1', + temperature: null, + topP: null, + seed: null, + }); + expect(first.trajectoryFingerprint).toMatch(/^[a-f0-9]{64}$/); + expect(second.trajectoryFingerprint).toBe(first.trajectoryFingerprint); + }); + + it('falls back to the last non-empty assistant step when the response message is blank', async () => { + // Models that end on a tool call return response.message === "". Without + // the fallback, judges see an empty answer and the matrix renders + // "No final answer message captured" (51 cells across all models, + // observed 2026-09-03). The fallback must take the LAST non-empty + // assistant reasoning/output step verbatim and disclose its source. + const fetch = jest.fn().mockResolvedValue({ + response: { message: '' }, + steps: [ + { type: 'reasoning', reasoning: 'I will check the alert first.' }, + { type: 'tool_call', tool_id: 'search', params: { q: 'x' } }, + { type: 'reasoning', reasoning: ' ' }, // blank — must be skipped + { type: 'output', output: 'The alert is benign.' }, + { type: 'tool_call', tool_id: 'search', params: { q: 'y' } }, // trailing tool call, no closing turn + ], + }); + const log = { warning: jest.fn() }; + const client = new PersonaMatrixChatClient(fetch, log as never, 'connector-1'); + + const res = await client.query('q'); + expect(res.messages[0].message).toBe('The alert is benign.'); + expect(res.messageSource).toBe('last_assistant_step'); + }); + + it('keeps the response message verbatim when present', async () => { + const fetch = jest.fn().mockResolvedValue({ + response: { message: 'final answer' }, + steps: [{ type: 'output', output: 'not the answer' }], + }); + const client = new PersonaMatrixChatClient(fetch, { warning: jest.fn() } as never, 'c'); + const res = await client.query('q'); + expect(res.messages[0].message).toBe('final answer'); + expect(res.messageSource).toBe('response'); + }); + + it('leaves the message empty when no assistant step has text either', async () => { + const fetch = jest.fn().mockResolvedValue({ + response: {}, + steps: [{ type: 'tool_call', tool_id: 'search', params: {} }], + }); + const client = new PersonaMatrixChatClient(fetch, { warning: jest.fn() } as never, 'c'); + const res = await client.query('q'); + expect(res.messages[0].message).toBe(''); + expect(res.messageSource).toBe('response'); + }); +}); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/chat_client.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/chat_client.ts index ec8c0b87d3752..d008efec139d4 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/chat_client.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/chat_client.ts @@ -7,6 +7,9 @@ import type { HttpHandler } from '@kbn/core/public'; import type { ToolingLog } from '@kbn/tooling-log'; +// Evals run in Node; SHA-256 is needed for stable trajectory provenance. +// eslint-disable-next-line import/no-nodejs-modules +import { createHash } from 'crypto'; // Eval-harness capture side-channel: writes the verbatim model answer to a // local gitignored JSONL for offline report rendering. // eslint-disable-next-line @kbn/eslint/require_kbn_fs, import/no-nodejs-modules @@ -36,6 +39,22 @@ export interface ConverseResponse { errors: Array<{ error: { message: string; stack?: string }; type: 'error' }>; conversationId?: string; traceId?: string | null; + /** + * Where `messages[last].message` came from. 'last_assistant_step' means the + * converse response carried no final message and the last non-empty + * assistant reasoning/output step was used verbatim instead — surfaced so + * reports can label the fallback instead of presenting it as a true answer. + */ + messageSource: 'response' | 'last_assistant_step'; + /** Reproducibility metadata persisted in task.output on every score doc. */ + sampling: { + connectorId: string; + temperature: number | null; + topP: number | null; + seed: number | null; + }; + /** sha256 of the ordered tool_id + params sequence (provider ids excluded). */ + trajectoryFingerprint: string; } interface ConverseApiResponse { @@ -74,14 +93,54 @@ export class PersonaMatrixChatClient { body: JSON.stringify(body), }); - const message = resp.response?.message ?? ''; + const steps = resp.steps ?? []; + // Final-answer fallback: some models end on a tool call (no closing + // assistant turn), so response.message is "". Downstream, a blank final + // message renders "No final answer message captured" and the answer- + // based LLM judges see an empty answer. Use the LAST non-empty assistant + // reasoning/output step verbatim — never synthesized — and mark it as a + // fallback so the trace card can label it honestly. + let message = resp.response?.message ?? ''; + let messageSource: 'response' | 'last_assistant_step' = 'response'; + if (!message.trim()) { + const lastAssistant = [...steps].reverse().find((step) => { + const text = + step.type === 'reasoning' ? step.reasoning : step.type === 'output' ? step.output : ''; + return typeof text === 'string' && text.trim().length > 0; + }); + if (lastAssistant) { + const text = + lastAssistant.type === 'reasoning' ? lastAssistant.reasoning : lastAssistant.output; + message = typeof text === 'string' ? text : ''; + messageSource = message.trim() ? 'last_assistant_step' : 'response'; + } + } + const trajectoryFingerprint = createHash('sha256') + .update( + JSON.stringify( + steps + .filter((step) => step.type === 'tool_call' && step.tool_id) + .map((step) => ({ tool_id: step.tool_id, params: step.params ?? null })) + ) + ) + .digest('hex'); const result: ConverseResponse = { messages: [{ message }], - steps: resp.steps ?? [], + steps, errors: [], + messageSource, conversationId: resp.conversation_id, traceId: resp.trace_id ?? null, + sampling: { + connectorId: this.connectorId, + // The converse API exposes no sampling controls. Persist null rather + // than inventing provider defaults; absence is itself evidence. + temperature: null, + topP: null, + seed: null, + }, + trajectoryFingerprint, }; // Capture the verbatim answer when PERSONA_MATRIX_CAPTURE is set. diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/datasets/persona_matrix_prompts.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/datasets/persona_matrix_prompts.ts index bcc0ff777fbc4..7a643a6a0e69b 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/datasets/persona_matrix_prompts.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/datasets/persona_matrix_prompts.ts @@ -38,6 +38,23 @@ export interface PersonaMatrixExample { */ expectedSkill?: string; expectedTools?: string[]; + /** + * Connector id that the authored artifact must actually reference, checked + * deterministically by the `ConnectorInvoked` evaluator. Calling the + * workflow-generation tool is not evidence that the produced workflow + * targets the connector the prompt demanded. + */ + expectedConnectorId?: string; + /** + * Step type (e.g. `http`) that must appear as a YAML key in the authored + * workflow, so prose describing a step cannot score as a built one. + */ + expectedStepType?: string; + /** + * Whether the declared tool sequence is a rankable contract. "probe" is + * intentionally open-ended and returns N/A from the trajectory evaluator. + */ + pathContract?: 'rankable' | 'candidate' | 'probe'; /** * Additional registered skill ids that also satisfy the skill assertion. * Scored as a union with `expectedSkill`: loading any one of them passes. @@ -89,11 +106,16 @@ export const PERSONA_MATRIX_EXAMPLES: PersonaMatrixExample[] = [ 'than dismissal, citing entity risk context and any Security Labs research on the technique.', }, metadata: { + pathContract: 'probe', expectedSkill: 'alert-analysis', + // security.entity_risk_score is force-disabled under + // agentBuilder:experimentalFeatures (skills), which this suite always + // enables; the entity-risk context for point-in-time triage comes from + // security.get_entity's profile instead. expectedTools: [ 'attachments.read', 'security.alerts', - 'security.entity_risk_score', + 'security.get_entity', 'security.security_labs_search', ], severity: 'high', @@ -118,8 +140,11 @@ export const PERSONA_MATRIX_EXAMPLES: PersonaMatrixExample[] = [ 'using entity risk scoring to justify the disposition rather than treating each alert in isolation.', }, metadata: { + pathContract: 'probe', expectedSkill: 'alert-analysis', - expectedTools: ['security.alerts', 'security.entity_risk_score'], + // entity_risk_score is unavailable under the skills flag (see above); + // disposition risk context comes from the get_entity profile. + expectedTools: ['security.alerts', 'security.get_entity'], severity: 'high', tags: ['triage', 'host-queue'], }, @@ -141,12 +166,11 @@ export const PERSONA_MATRIX_EXAMPLES: PersonaMatrixExample[] = [ 'technique/malware family to ground the false-positive-vs-true-positive call in external evidence.', }, metadata: { + pathContract: 'probe', expectedSkill: 'alert-analysis', - expectedTools: [ - 'security.alerts', - 'security.security_labs_search', - 'security.entity_risk_score', - ], + // entity_risk_score is unavailable under the skills flag (see above); + // the baseline-vs-pattern risk check comes from the get_entity profile. + expectedTools: ['security.alerts', 'security.security_labs_search', 'security.get_entity'], severity: 'medium', tags: ['triage', 'noise'], }, @@ -238,8 +262,14 @@ export const PERSONA_MATRIX_EXAMPLES: PersonaMatrixExample[] = [ 'summarizes recent risk-contributing activity for srv-win-defend-01.', }, metadata: { + pathContract: 'probe', expectedSkill: 'entity-analytics', - expectedTools: ['security.get_entity', 'security.entity_risk_score'], + // security.entity_risk_score is force-disabled whenever + // agentBuilder:experimentalFeatures (skills) is on — which the persona + // matrix always runs with — so it can never be invoked here. Risk data + // comes from get_entity plus, for the "unusual behavior tied to it" + // phrasing, the risk score history tool. + expectedTools: ['security.get_entity', 'security.get_entity_risk_score_history'], severity: 'medium', tags: ['entity', 'host-profile'], }, @@ -259,6 +289,7 @@ export const PERSONA_MATRIX_EXAMPLES: PersonaMatrixExample[] = [ 'entity, then looks up that entity in detail to produce a profile.', }, metadata: { + pathContract: 'probe', expectedSkill: 'entity-analytics', expectedTools: ['security.search_entities', 'security.get_entity'], severity: 'medium', @@ -282,8 +313,12 @@ export const PERSONA_MATRIX_EXAMPLES: PersonaMatrixExample[] = [ 'generic answer.', }, metadata: { + pathContract: 'probe', expectedSkill: 'entity-analytics', - expectedTools: ['security.get_entity', 'security.entity_risk_score'], + // Same skills-precedence constraint as entity-analytics-a: risk history + // comes from get_entity_risk_score_history, which stays available under + // the experimental-features flag that the suite needs for skills. + expectedTools: ['security.get_entity', 'security.get_entity_risk_score_history'], severity: 'medium', tags: ['entity', 'history'], }, @@ -316,11 +351,14 @@ export const PERSONA_MATRIX_EXAMPLES: PersonaMatrixExample[] = [ 'recommendation with the IOCs called out.', }, metadata: { + pathContract: 'probe', expectedSkill: 'alert-analysis', expectedTools: [ 'attachments.read', 'security.security_labs_search', - 'security.entity_risk_score', + // entity_risk_score is unavailable under the skills flag (see + // alert-analysis-a); the risk signal comes from the get_entity profile. + 'security.get_entity', ], severity: 'critical', tags: ['multi-step', 'orchestration'], @@ -346,6 +384,7 @@ export const PERSONA_MATRIX_EXAMPLES: PersonaMatrixExample[] = [ 'each step taken.', }, metadata: { + pathContract: 'probe', expectedSkill: 'alert-analysis', allowSkills: ['cases-management'], expectedTools: ['security.security_labs_search', 'platform.core.cases'], @@ -372,13 +411,16 @@ export const PERSONA_MATRIX_EXAMPLES: PersonaMatrixExample[] = [ 'the hunt confirms a true positive — explicitly stating when it does not escalate.', }, metadata: { + pathContract: 'probe', expectedSkill: 'alert-analysis', allowSkills: ['threat-hunting'], expectedTools: [ 'security.alerts', 'platform.core.generate_esql', 'platform.core.execute_esql', - 'security.entity_risk_score', + // entity_risk_score is unavailable under the skills flag (see + // alert-analysis-a); the risk signal comes from the get_entity profile. + 'security.get_entity', ], severity: 'critical', tags: ['multi-step', 'conditional-escalation'], @@ -400,6 +442,7 @@ export const PERSONA_MATRIX_EXAMPLES: PersonaMatrixExample[] = [ 'load events across hosts, then reports which hosts/processes show the pattern (or that none do).', }, metadata: { + pathContract: 'probe', expectedSkill: 'threat-hunting', expectedTools: ['platform.core.generate_esql', 'platform.core.execute_esql'], severity: 'high', @@ -422,6 +465,7 @@ export const PERSONA_MATRIX_EXAMPLES: PersonaMatrixExample[] = [ 'telemetry for the named IOAs, and narrates each query and its result as it builds the picture.', }, metadata: { + pathContract: 'probe', expectedSkill: 'threat-hunting', expectedTools: ['platform.core.generate_esql', 'platform.core.execute_esql'], severity: 'high', @@ -445,6 +489,7 @@ export const PERSONA_MATRIX_EXAMPLES: PersonaMatrixExample[] = [ 'a specific IOC upfront.', }, metadata: { + pathContract: 'probe', expectedSkill: 'threat-hunting', expectedTools: ['platform.core.generate_esql', 'platform.core.execute_esql'], severity: 'medium', @@ -471,6 +516,11 @@ export const PERSONA_MATRIX_EXAMPLES: PersonaMatrixExample[] = [ metadata: { expectedSkill: 'workflow-authoring', expectedTools: ['platform.core.generate_workflow', 'platform.workflows.validate_workflow'], + // Verified deterministically by ConnectorInvoked: calling + // generate_workflow is not evidence that the authored workflow actually + // targets Slack. The prompt names this connector id explicitly. + expectedConnectorId: 'd7306385-cbe6-4541-9726-49afdff59ba5', + expectedStepType: 'http', severity: 'medium', tags: ['workflow', 'authoring'], }, @@ -496,6 +546,10 @@ export const PERSONA_MATRIX_EXAMPLES: PersonaMatrixExample[] = [ // `workflow-authoring` documents that it is NOT required for creating or // editing a workflow — call `platform.core.generate_workflow` directly. expectedTools: ['platform.core.generate_workflow', 'platform.workflows.validate_workflow'], + // This prompt names no connector id, so only the http step is a fair + // deterministic assertion. Asserting an id the prompt never gave would + // false-fail a correct answer. + expectedStepType: 'http', severity: 'low', tags: ['workflow', 'authoring-fixed'], }, @@ -520,6 +574,9 @@ export const PERSONA_MATRIX_EXAMPLES: PersonaMatrixExample[] = [ // No expectedSkill: see `workflow-authoring-b` — the prompt does not ask // for the skill and generate_workflow is the documented direct path. expectedTools: ['platform.core.generate_workflow', 'platform.workflows.validate_workflow'], + // Prompt names the connector id explicitly, so both are assertable. + expectedConnectorId: 'd7306385-cbe6-4541-9726-49afdff59ba5', + expectedStepType: 'http', severity: 'medium', tags: ['workflow', 'authoring-parameterized'], }, diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/datasets/select_shard.test.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/datasets/select_shard.test.ts new file mode 100644 index 0000000000000..ffe3e0309f135 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/datasets/select_shard.test.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PERSONA_MATRIX_EXAMPLES } from './persona_matrix_prompts'; +import { selectShard } from './select_shard'; + +describe('selectShard', () => { + const examples = PERSONA_MATRIX_EXAMPLES; + + it('returns every example when no shard is requested', () => { + expect(selectShard(examples, undefined)).toHaveLength(21); + expect(selectShard(examples, '')).toHaveLength(21); + }); + + it('is the identity for 1/1', () => { + expect(selectShard(examples, '1/1').map((e) => e.id)).toEqual(examples.map((e) => e.id)); + }); + + it('partitions the dataset: shards are disjoint and cover everything', () => { + const shards = [1, 2, 3, 4].map((i) => selectShard(examples, `${i}/4`)); + const ids = shards.flat().map((e) => e.id); + + // No example runs twice — a duplicate would double-bill the slowest models + // and break the union doc-count gate. + expect(new Set(ids).size).toBe(ids.length); + // No example is dropped — a silent gap reads as "model scored fewer points" + // rather than "we never asked it that question". + expect(new Set(ids)).toEqual(new Set(examples.map((e) => e.id))); + }); + + it('balances shards to within one example', () => { + // Stride assignment, not contiguous slicing: the per-example cost is skewed + // (measured median 75 model calls, max 280), so adjacent examples must not + // pile onto one shard. + const sizes = [1, 2, 3, 4].map((i) => selectShard(examples, `${i}/4`).length); + expect(Math.max(...sizes) - Math.min(...sizes)).toBeLessThanOrEqual(1); + }); + + it.each(['0/4', '5/4', 'x/y', '1/0', '3', '-1/4', '1/4/9'])( + 'throws on malformed shard %p rather than silently running all 21', + (bad) => { + // Silently falling back to the full dataset is the dangerous failure: + // every shard would run every example and the run would look "complete". + expect(() => selectShard(examples, bad)).toThrow(/PERSONA_MATRIX_SHARD/); + } + ); +}); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/datasets/select_shard.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/datasets/select_shard.ts new file mode 100644 index 0000000000000..78453600645d4 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/datasets/select_shard.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PersonaMatrixExample } from './persona_matrix_prompts'; + +/** + * Select this run's slice of the dataset from a `"/"` spec. + * + * Slow models need ~5-20 min per example, so a 21-example run serialises into + * hours on one stack. Splitting the dataset across VMs (one Kibana each — the + * suite's beforeAll/afterAll share seed indices, so extra in-process workers + * would tear down live fixtures) is the only safe parallelism. + * + * Assignment is by stride (`i % total`), not contiguous slicing: per-example + * cost is heavily skewed, so neighbouring examples must land on different + * shards to keep wall clock balanced. + */ +export function selectShard( + examples: PersonaMatrixExample[], + spec: string | undefined +): PersonaMatrixExample[] { + if (!spec) { + return examples; + } + + const match = /^(\d+)\/(\d+)$/.exec(spec.trim()); + const index = match ? Number(match[1]) : NaN; + const total = match ? Number(match[2]) : NaN; + + // Reject rather than fall back to the full dataset: a silent fallback makes + // every shard run all 21 examples while the sweep still reports "complete". + if (!match || total < 1 || index < 1 || index > total) { + throw new Error( + `PERSONA_MATRIX_SHARD must be "/" with 1 <= index <= total, got "${spec}"` + ); + } + + return examples.filter((_, position) => position % total === index - 1); +} diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.test.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.test.ts index 73ebdc31fd1a1..705643c20be02 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.test.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.test.ts @@ -7,14 +7,19 @@ import type { Client as EsClient } from '@elastic/elasticsearch'; import type { ToolingLog } from '@kbn/tooling-log'; -import type { TaskOutput } from '@kbn/evals'; +import type { DefaultEvaluators, EvalsExecutorClient, TaskOutput } from '@kbn/evals'; import { toDatasetExample, + createEvaluatePersonaMatrixDataset, createPersonaMatrixTrajectoryEvaluator, createPersonaMatrixExpectedToolCalledEvaluator, + createPersonaMatrixFinalAnswerPresentEvaluator, + createPersonaMatrixMinExpectedStepsEvaluator, createPersonaMatrixSkillInvokedEvaluator, + isRankablePathContract, type PersonaMatrixDatasetExample, } from './evaluate_dataset'; +import type { PersonaMatrixChatClient } from './chat_client'; import { PERSONA_MATRIX_EXAMPLES, type PersonaMatrixExample, @@ -43,6 +48,29 @@ const buildLog = (): ToolingLog => debug: jest.fn(), } as unknown as ToolingLog); +describe('path contract classification', () => { + it('keeps probes diagnostic and leaves candidate/rankable paths measurable', () => { + expect(isRankablePathContract({ pathContract: 'probe' })).toBe(false); + expect(isRankablePathContract({ pathContract: 'candidate' })).toBe(true); + expect(isRankablePathContract({ pathContract: 'rankable' })).toBe(true); + expect(isRankablePathContract(undefined)).toBe(true); + }); + + it('marks every measured 0/5 hunt example as a probe', () => { + const probePrefixes = [ + 'alert-analysis-', + 'entity-analytics-', + 'multi-step-', + 'threat-hunting-', + ]; + const probes = PERSONA_MATRIX_EXAMPLES.filter((example) => + probePrefixes.some((prefix) => example.id.startsWith(prefix)) + ); + expect(probes).toHaveLength(12); + expect(probes.every((example) => example.metadata.pathContract === 'probe')).toBe(true); + }); +}); + describe('toDatasetExample', () => { it('resolves the golden path from metadata.expectedTools into output.tool_sequence', () => { // The trajectory evaluator's goldenPathExtractor only receives `expected` @@ -162,21 +190,29 @@ describe('createPersonaMatrixTrajectoryEvaluator', () => { })) ) ); - expect(result.score).not.toBeNull(); - const metadata = result.metadata as { expected: string[] } | undefined; - expect(metadata?.expected).toEqual(wrapped.output.tool_sequence); + if (example.metadata.pathContract === 'probe') { + expect(result.score).toBeNull(); + expect(result.metadata).toMatchObject({ pathContract: 'probe' }); + } else { + expect(result.score).not.toBeNull(); + const metadata = result.metadata as { expected: string[] } | undefined; + expect(metadata?.expected).toEqual(wrapped.output.tool_sequence); + } } }); }); describe('createPersonaMatrixExpectedToolCalledEvaluator', () => { - it('scores 1 when the primary expected tool was called', async () => { + it('scores 1 when every declared expected tool was called', async () => { const evaluator = createPersonaMatrixExpectedToolCalledEvaluator(); const result = await evaluator.evaluate({ input: { question: 'q' }, expected: toDatasetExample(baseExample).output, output: { - steps: [{ type: 'tool_call', tool_id: 'security.alerts' }], + steps: [ + { type: 'tool_call', tool_id: 'security.alerts' }, + { type: 'tool_call', tool_id: 'security.get_related_alerts' }, + ], } as unknown as TaskOutput, metadata: baseExample.metadata, } as unknown as Parameters['evaluate']>[0]); @@ -198,6 +234,191 @@ describe('createPersonaMatrixExpectedToolCalledEvaluator', () => { expect(result.score).toBeNull(); expect(result.label).toBe('N/A'); }); + + // Regression: only `expectedTools[0]` used to be checked, so a run that + // skipped every later declared tool still scored a full 1. + it('scores 0 when a non-primary expected tool was skipped', async () => { + const evaluator = createPersonaMatrixExpectedToolCalledEvaluator(); + const multiTool: PersonaMatrixExample = { + ...baseExample, + metadata: { + ...baseExample.metadata, + expectedTools: ['platform.core.generate_esql', 'platform.core.execute_esql'], + }, + }; + const result = await evaluator.evaluate({ + input: { question: 'q' }, + expected: toDatasetExample(multiTool).output, + output: { + steps: [{ type: 'tool_call', tool_id: 'platform.core.generate_esql' }], + } as unknown as TaskOutput, + metadata: multiTool.metadata, + } as unknown as Parameters['evaluate']>[0]); + expect(result.score).toBe(0); + expect((result.metadata as { missingToolIds: string[] }).missingToolIds).toEqual([ + 'platform.core.execute_esql', + ]); + }); + + it('scores 1 only when every declared expected tool was called', async () => { + const evaluator = createPersonaMatrixExpectedToolCalledEvaluator(); + const multiTool: PersonaMatrixExample = { + ...baseExample, + metadata: { + ...baseExample.metadata, + expectedTools: ['platform.core.generate_esql', 'platform.core.execute_esql'], + }, + }; + const result = await evaluator.evaluate({ + input: { question: 'q' }, + expected: toDatasetExample(multiTool).output, + output: { + steps: [ + { type: 'tool_call', tool_id: 'platform.core.generate_esql' }, + { type: 'tool_call', tool_id: 'platform.core.list_indices' }, + { type: 'tool_call', tool_id: 'platform.core.execute_esql' }, + ], + } as unknown as TaskOutput, + metadata: multiTool.metadata, + } as unknown as Parameters['evaluate']>[0]); + expect(result.score).toBe(1); + expect((result.metadata as { missingToolIds: string[] }).missingToolIds).toEqual([]); + }); +}); + +describe('createPersonaMatrixFinalAnswerPresentEvaluator', () => { + const evaluate = (output: unknown) => + createPersonaMatrixFinalAnswerPresentEvaluator().evaluate({ + input: { question: 'q' }, + expected: undefined, + output, + metadata: {}, + } as unknown as Parameters['evaluate']>[0]); + + it('scores 1 when the run produced a non-empty final message', async () => { + const result = await evaluate({ messages: [{ message: 'Rule created: ...' }] }); + expect(result.score).toBe(1); + }); + + // Regression: 62% of detection-rule-edit runs in the 2026-08-21 sweep ended + // on a tool call with no user-facing closing text. + it('scores 0 when every message is empty', async () => { + const result = await evaluate({ messages: [{ message: '' }] }); + expect(result.score).toBe(0); + }); + + it('scores 0 when messages are missing from the output', async () => { + const result = await evaluate({ steps: [] }); + expect(result.score).toBe(0); + }); + + const ruleToolStep = (data: unknown, toolId = 'security.create_detection_rule') => ({ + type: 'tool_call', + tool_id: toolId, + results: [{ type: 'other', data }], + }); + + // The detection-rule-edit references ask the agent to render the created rule + // attachment inline "rather than describing the rule in prose only", so a run + // that ends on a successful rule tool call HAS answered the user. Scoring it 0 + // measured the harness, not the model: 86% of FinalAnswerPresent=0 runs in the + // family had successfully created a rule. + it('scores 1 when the run ended on a rendered rule artifact instead of prose', async () => { + const result = await evaluate({ + messages: [{ message: '' }], + steps: [ruleToolStep({ success: true, rule: { id: 'abc', severity: 'high' } })], + }); + expect(result.score).toBe(1); + expect(result.explanation).toContain('artifact'); + }); + + it('scores 1 for an update_detection_rule artifact as well', async () => { + const result = await evaluate({ + messages: [], + steps: [ + ruleToolStep({ success: true, rule: { id: 'abc' } }, 'security.update_detection_rule'), + ], + }); + expect(result.score).toBe(1); + }); + + // The gate must keep biting real silent failures: 27 of 196 FinalAnswerPresent=0 + // runs in the family produced no rule at all. + it('still scores 0 when the rule tool call failed', async () => { + const result = await evaluate({ + messages: [{ message: '' }], + steps: [ruleToolStep({ success: false, rule: { id: 'abc' } })], + }); + expect(result.score).toBe(0); + }); + + it('still scores 0 when a rule tool reported success without a rule payload', async () => { + const result = await evaluate({ + messages: [{ message: '' }], + steps: [ruleToolStep({ success: true })], + }); + expect(result.score).toBe(0); + }); + + it('does not count a non-artifact tool call as an answer', async () => { + const result = await evaluate({ + messages: [{ message: '' }], + steps: [ruleToolStep({ success: true, rule: { id: 'abc' } }, 'security.run_rule_preview')], + }); + expect(result.score).toBe(0); + }); + + it('returns N/A when there is no task output at all', async () => { + const result = await evaluate(undefined); + expect(result.score).toBeNull(); + expect(result.label).toBe('N/A'); + }); +}); + +describe('createPersonaMatrixMinExpectedStepsEvaluator', () => { + const toolStep = (id: string) => ({ type: 'tool_call', tool_id: id }); + const evaluate = (output: unknown, expectedTools?: string[]) => + createPersonaMatrixMinExpectedStepsEvaluator().evaluate({ + input: { question: 'q' }, + expected: expectedTools ? { tool_sequence: expectedTools } : {}, + output, + metadata: expectedTools ? { expectedTools } : {}, + } as unknown as Parameters['evaluate']>[0]); + + it('scores 1 when tool calls meet the expected minimum', async () => { + const result = await evaluate( + { steps: [toolStep('platform.core.generate_esql'), toolStep('platform.core.execute_esql')] }, + ['platform.core.generate_esql', 'platform.core.execute_esql'] + ); + expect(result.score).toBe(1); + }); + + // Regression: ~90 original-sweep runs produced an answer in <3 steps having + // called nothing — premature termination that FinalAnswerPresent alone misses. + it('scores 0 when the agent gave up without trying (fewer calls than expected)', async () => { + const result = await evaluate({ steps: [] }, ['on_call_lookup']); + expect(result.score).toBe(0); + }); + + it('scores 0 when only some of the expected tools were called', async () => { + const result = await evaluate({ steps: [toolStep('platform.core.generate_esql')] }, [ + 'platform.core.generate_esql', + 'platform.core.execute_esql', + ]); + expect(result.score).toBe(0); + }); + + it('returns N/A when the example declares no expectedTools', async () => { + const result = await evaluate({ steps: [toolStep('x')] }); + expect(result.score).toBeNull(); + expect(result.label).toBe('N/A'); + }); + + it('returns N/A when there is no task output', async () => { + const result = await evaluate(undefined, ['on_call_lookup']); + expect(result.score).toBeNull(); + expect(result.label).toBe('N/A'); + }); }); describe('createPersonaMatrixSkillInvokedEvaluator', () => { @@ -236,14 +457,9 @@ describe('createPersonaMatrixSkillInvokedEvaluator', () => { expect(result.label).toBe('unavailable'); }); - it('matches the load_skill tool, not just the retired filestore.read', async () => { - // Regression guard. The agent loads skills via the `load_skill` tool: - // {"skill":"/skills///SKILL.md"} - // A predicate pinned to `filestore.read` can never match, so `skill_invoked` - // stays 0 while `total_tool_spans` is non-zero -- meaning the "unavailable" - // guard does NOT trip and the evaluator reports a confident false 0 for - // every model. Verified against the golden cluster: over 7 days, - // filestore.read = 0 spans, load_skill = 7,991 spans. + it('matches load_skill by skill ID while retaining the legacy SKILL.md path', async () => { + // `load_skill` accepts an ID or path. Current traces store {"skill":""}; + // older traces may contain a SKILL.md path. const query = jest.fn().mockResolvedValue({ columns: [{ name: 'total_tool_spans' }, { name: 'skill_invoked' }], values: [[2, 1]], @@ -253,13 +469,15 @@ describe('createPersonaMatrixSkillInvokedEvaluator', () => { log: buildLog(), }); - const result = await evaluator.evaluate( + await evaluator.evaluate( buildEvaluatorArgs(baseExample.metadata, '0af7651916cd43dd8448eb211c80319c') ); const sent = query.mock.calls[0][0].query as string; expect(sent).toContain('load_skill'); - expect(result.score).toBe(1); + expect(sent).toContain('filestore.read'); + expect(sent).toContain('*\\"skill\\":\\"alert-analysis\\"*'); + expect(sent).toContain('*/alert-analysis/SKILL.md*'); }); }); @@ -278,4 +496,113 @@ describe('task output shape', () => { const latestMessage = messages[messages.length - 1]?.message; expect(latestMessage).toBe('the real answer'); }); + + it('forwards messageSource so a fallback answer is distinguishable from a real final turn', async () => { + // chat_client falls back to the last non-empty reasoning/output step when a + // model ends its turn on a tool call, and tags which one it used via + // `messageSource`. If the task drops that tag, every stored cell looks like + // a genuine closing answer and mid-run narration ("I will now preview the + // detection rule...") is scored as if it were the model's final response. + // Measured 2026-09-07: 0 of 2422 Sep-4+ detection-rule-edit score docs + // carried messageSource, because taskOutput never forwarded it. + const log = buildLog(); + let capturedTask: ((example: unknown) => Promise) | undefined; + + const evaluateDataset = createEvaluatePersonaMatrixDataset({ + chatClient: { + query: jest.fn().mockResolvedValue({ + messages: [{ message: 'I will now preview the detection rule.' }], + messageSource: 'last_assistant_step', + steps: [], + errors: [], + traceId: 'trace-1', + }), + } as unknown as PersonaMatrixChatClient, + evaluators: { + traceBasedEvaluators: { + inputTokens: { name: 'inputTokens' }, + outputTokens: { name: 'outputTokens' }, + toolCalls: { name: 'toolCalls' }, + latency: { name: 'latency' }, + }, + criteria: jest.fn(), + correctnessAnalysis: () => ({ evaluate: jest.fn().mockResolvedValue({ metadata: {} }) }), + groundednessAnalysis: () => ({ evaluate: jest.fn().mockResolvedValue({ metadata: {} }) }), + } as unknown as DefaultEvaluators, + executorClient: { + runExperiment: jest.fn(async (params: { task: unknown }) => { + capturedTask = params.task as (example: unknown) => Promise; + }), + } as unknown as EvalsExecutorClient, + traceEsClient: {} as EsClient, + log, + }); + + await evaluateDataset({ + dataset: { name: 'ds', description: 'desc', examples: [baseExample] }, + }); + + const output = (await capturedTask!(toDatasetExample(baseExample))) as Record; + + expect(output.messageSource).toBe('last_assistant_step'); + }); +}); + +describe('task judge failure isolation', () => { + it('keeps the trajectory and degrades qualitative scores when a judge call rejects', async () => { + // Regression guard for the determinism-sweep suite deaths: a single judge + // call failing (e.g. inference 500 toolValidationError) must not throw out + // of the task and take down all remaining examples. + const log = buildLog(); + let capturedTask: ((example: unknown) => Promise) | undefined; + + const evaluateDataset = createEvaluatePersonaMatrixDataset({ + chatClient: { + query: jest.fn().mockResolvedValue({ + messages: [{ message: 'the real answer' }], + steps: [], + errors: [], + traceId: 'trace-1', + }), + } as unknown as PersonaMatrixChatClient, + evaluators: { + traceBasedEvaluators: { + inputTokens: { name: 'inputTokens' }, + outputTokens: { name: 'outputTokens' }, + toolCalls: { name: 'toolCalls' }, + latency: { name: 'latency' }, + }, + criteria: jest.fn(), + correctnessAnalysis: () => ({ + evaluate: jest.fn().mockRejectedValue(new Error('toolValidationError')), + }), + groundednessAnalysis: () => ({ + evaluate: jest.fn().mockResolvedValue({ metadata: { verdict: 'grounded' } }), + }), + } as unknown as DefaultEvaluators, + executorClient: { + runExperiment: jest.fn(async (params: { task: unknown }) => { + capturedTask = params.task as (example: unknown) => Promise; + }), + } as unknown as EvalsExecutorClient, + traceEsClient: {} as EsClient, + log, + }); + + await evaluateDataset({ + dataset: { name: 'ds', description: 'desc', examples: [baseExample] }, + }); + + expect(capturedTask).toBeDefined(); + const output = (await capturedTask!(toDatasetExample(baseExample))) as Record; + + // The failed judge degrades to an absent analysis (quantitative evaluators + // then report "unavailable"); the successful judge and the agent's real + // trajectory are preserved. + expect(output.correctnessAnalysis).toBeUndefined(); + expect(output.groundednessAnalysis).toEqual({ verdict: 'grounded' }); + expect((output.messages as Array<{ message: string }>)[0].message).toBe('the real answer'); + expect(log.error).toHaveBeenCalledWith(expect.stringContaining('CorrectnessAnalysis failed')); + expect((log.error as jest.Mock).mock.calls.flat().join(' ')).toContain('toolValidationError'); + }); }); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.ts index f2d51d08415a0..a7fa1254a1b13 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.ts @@ -18,12 +18,15 @@ import { type EvaluationDataset, type Evaluator, type TaskOutput, + TRACE_INDEX_PATTERN, } from '@kbn/evals'; import type { ToolingLog } from '@kbn/tooling-log'; import type { PersonaMatrixExample, PersonaMatrixExampleInput, } from './datasets/persona_matrix_prompts'; +import { selectShard } from './datasets/select_shard'; +import { createConnectorInvokedEvaluator } from './evaluators/connector_invoked_evaluator'; import type { PersonaMatrixChatClient } from './chat_client'; /** @@ -67,9 +70,19 @@ export const toDatasetExample = (ex: PersonaMatrixExample): PersonaMatrixDataset }; /** - * ExpectedToolCalled — verifies the primary expected tool was invoked. - * Reads `expectedTools` from example metadata (first entry) or `tool_sequence` - * from the expected output. + * ExpectedToolCalled — verifies every declared expected tool was invoked. + * Reads `expectedTools` from example metadata, or `tool_sequence` from the + * expected output. + * + * Scores the whole declared set, not just `expectedTools[0]`. 16 of the 21 + * examples declare more than one expected tool, so reading only the first + * entry left the rest unenforced: an example annotated + * `['platform.core.generate_esql', 'platform.core.execute_esql']` scored 1 for + * a run that generated a query and never executed it. + * + * All-or-nothing rather than a partial ratio: `Trajectory` already reports + * graded per-tool overlap. `missingToolIds` names what was skipped so a 0 is + * diagnosable without re-reading the trace. */ export const createPersonaMatrixExpectedToolCalledEvaluator = (): Evaluator => ({ name: 'ExpectedToolCalled', @@ -89,14 +102,143 @@ export const createPersonaMatrixExpectedToolCalledEvaluator = (): Evaluator => ( }; } - const expectedToolId = expectedTools[0]; const usedToolIds = getToolCallSteps(output as TaskOutput) .map((step) => step.tool_id) .filter((id): id is string => Boolean(id)); + const usedToolIdSet = new Set(usedToolIds); + const missingToolIds = expectedTools.filter((toolId) => !usedToolIdSet.has(toolId)); + + return { + score: missingToolIds.length === 0 ? 1 : 0, + explanation: missingToolIds.length + ? `Expected tools not called: ${missingToolIds.join(', ')}.` + : `All expected tools called: ${expectedTools.join(', ')}.`, + metadata: { expectedToolIds: expectedTools, missingToolIds, usedToolIds }, + }; + }, +}); + +/** + * Tools whose successful result is itself the user-facing deliverable: they + * render an artifact (a rule) inline in the conversation. A run that ends on + * one of these has answered the user even with no closing prose. + */ +const ARTIFACT_PRODUCING_TOOL_IDS = new Set([ + 'security.create_detection_rule', + 'security.update_detection_rule', +]); + +/** + * True when the run produced a rendered artifact: an artifact-producing tool + * returned `success: true` with a `rule` payload. Mirrors the shape asserted + * by the dataset references ("renders the created rule attachment inline"). + */ +const hasRenderedArtifact = (output: TaskOutput): boolean => + getToolCallSteps(output).some((step) => { + if (!step.tool_id || !ARTIFACT_PRODUCING_TOOL_IDS.has(step.tool_id)) { + return false; + } + return (step.results ?? []).some((result) => { + const data = (result as { data?: { success?: unknown; rule?: unknown } } | undefined)?.data; + return Boolean(data?.success === true && data?.rule); + }); + }); + +/** + * Regression gate for the empty-final-message failure mode: 62% of + * detection-rule-edit runs in the 2026-08-21 sweep ended on a tool call with + * no user-facing closing text, leaving judges (and users) with nothing to + * read. + * + * Scores 1 when the run leaves the user something to read: either non-empty + * closing prose, or a rendered artifact (a successfully created/updated rule). + * The artifact clause exists because `detection-rule-edit` references + * explicitly ask the agent to render the rule attachment inline "rather than + * describing the rule in prose only" — scoring those runs 0 measured the + * harness, not the model. Runs that end silently *without* producing an + * artifact still score 0: that is the real premature-termination failure. + * + * N/A only when the task produced no output at all (harness failure — already + * surfaced by every other evaluator). + */ +export const createPersonaMatrixFinalAnswerPresentEvaluator = (): Evaluator => ({ + name: 'FinalAnswerPresent', + kind: 'CODE', + direction: 'maximize', + evaluate: async ({ output }) => { + const taskOutput = output as { messages?: Array<{ message?: unknown }> } | undefined; + if (!taskOutput) { + return { + score: null, + label: 'N/A', + explanation: 'No task output — skipping FinalAnswerPresent.', + }; + } + const hasMessage = (taskOutput.messages ?? []).some( + (msg) => typeof msg?.message === 'string' && msg.message.trim().length > 0 + ); + if (hasMessage) { + return { score: 1, explanation: 'Final user-facing message present.' }; + } + if (hasRenderedArtifact(output as TaskOutput)) { + return { + score: 1, + explanation: 'No closing prose, but the run rendered a rule artifact inline.', + }; + } return { - score: usedToolIds.includes(expectedToolId) ? 1 : 0, - metadata: { expectedToolId, usedToolIds }, + score: 0, + explanation: 'Run ended without a user-facing final message or rendered artifact.', + }; + }, +}); + +/** + * MinExpectedSteps — flags "gave up without trying": the agent produced a + * (possibly non-empty) answer but performed fewer tool calls than the example + * declares in `expectedTools`. Distinct from FinalAnswerPresent (which only + * checks that *some* text exists) — a model can write a confident answer having + * called nothing, which is exactly the premature-termination failure mode seen + * in the original sweep (~90 runs finished in <3 steps). + * + * Scores 1 when the run made at least `expectedTools.length` tool calls, + * otherwise 0. N/A when the example declares no expectedTools (nothing to + * compare against) or the task produced no output. + */ +export const createPersonaMatrixMinExpectedStepsEvaluator = (): Evaluator => ({ + name: 'MinExpectedSteps', + kind: 'CODE', + direction: 'maximize', + evaluate: async ({ output, expected, metadata }) => { + const toolSequence = (expected as PersonaMatrixDatasetExpected | undefined)?.tool_sequence; + const meta = metadata as { expectedTools?: string[] } | undefined; + const expectedTools = meta?.expectedTools ?? toolSequence ?? []; + const minToolCalls = expectedTools.length; + if (minToolCalls === 0) { + return { + score: null, + label: 'N/A', + explanation: 'No expectedTools annotation — skipping MinExpectedSteps.', + }; + } + const taskOutput = output as TaskOutput | undefined; + if (!taskOutput) { + return { + score: null, + label: 'N/A', + explanation: 'No task output — skipping MinExpectedSteps.', + }; + } + const actualToolCalls = getToolCallSteps(taskOutput).length; + const met = actualToolCalls >= minToolCalls; + return { + score: met ? 1 : 0, + explanation: met + ? `Made ${actualToolCalls} tool call(s), meeting the expected minimum of ${minToolCalls}.` + : `Made ${actualToolCalls} tool call(s) but expected at least ${minToolCalls} (${expectedTools.join( + ', ' + )}) — agent may have given up without trying.`, }; }, }); @@ -112,6 +254,10 @@ export const createPersonaMatrixExpectedToolCalledEvaluator = (): Evaluator => ( */ const FILESTORE_READ_TOOL_ID = 'filestore.read'; +export const isRankablePathContract = ( + metadata: { pathContract?: 'rankable' | 'candidate' | 'probe' } | undefined +): boolean => metadata?.pathContract !== 'probe'; + export const createPersonaMatrixTrajectoryEvaluator = (): Evaluator => { const inner = createTrajectoryEvaluator({ extractToolCalls: (output) => @@ -129,6 +275,17 @@ export const createPersonaMatrixTrajectoryEvaluator = (): Evaluator => { name: 'Trajectory', evaluate: async (args) => { const exp = args.expected as PersonaMatrixDatasetExpected | undefined; + const meta = args.metadata as + | { pathContract?: 'rankable' | 'candidate' | 'probe' } + | undefined; + if (!isRankablePathContract(meta)) { + return { + score: null, + label: 'N/A', + explanation: 'Open-ended capability probe — trajectory is diagnostic, not rankable.', + metadata: { pathContract: 'probe' }, + }; + } if (!exp?.tool_sequence || exp.tool_sequence.length === 0) { return { score: null, @@ -196,10 +353,13 @@ export const createPersonaMatrixSkillInvokedEvaluator = ({ } const skillPredicate = acceptedSkills - .map((skillName) => `attributes.gen_ai.tool.call.arguments LIKE "*/${skillName}/SKILL.md*"`) + .flatMap((skillName) => [ + `attributes.gen_ai.tool.call.arguments LIKE "*\\\"skill\\\":\\\"${skillName}\\\"*"`, + `attributes.gen_ai.tool.call.arguments LIKE "*/${skillName}/SKILL.md*"`, + ]) .join(' OR '); - const query = `FROM traces-* + const query = `FROM ${TRACE_INDEX_PATTERN} | WHERE trace.id == "${traceId}" | STATS total_tool_spans = COUNT( @@ -269,7 +429,18 @@ export function createEvaluatePersonaMatrixDataset({ }: { dataset: EvaluationDataset; }): Promise { - const wrappedExamples = dataset.examples.map(toDatasetExample); + // Shard before wrapping so a sharded run seeds and grades only its slice. + // Slow models need hours for all 21 examples on one stack; the sweeper fans + // shards out to one VM each. + const shardedExamples = selectShard(dataset.examples, process.env.PERSONA_MATRIX_SHARD); + const wrappedExamples = shardedExamples.map(toDatasetExample); + + if (process.env.PERSONA_MATRIX_SHARD) { + log.info( + `[persona-matrix] shard ${process.env.PERSONA_MATRIX_SHARD}: ` + + `${shardedExamples.length}/${dataset.examples.length} examples` + ); + } const skillInvokedEvaluator = createPersonaMatrixSkillInvokedEvaluator({ traceEsClient, @@ -278,6 +449,12 @@ export function createEvaluatePersonaMatrixDataset({ const trajectoryEvaluator = createPersonaMatrixTrajectoryEvaluator(); const expectedToolCalledEvaluator = createPersonaMatrixExpectedToolCalledEvaluator(); + const finalAnswerPresentEvaluator = createPersonaMatrixFinalAnswerPresentEvaluator(); + const minExpectedStepsEvaluator = createPersonaMatrixMinExpectedStepsEvaluator(); + // Deterministic proof that an authored workflow targets the connector the + // prompt demanded. ExpectedToolCalled only proves generate_workflow was + // called, never what it produced -- see connector_invoked_evaluator.ts. + const connectorInvokedEvaluator = createConnectorInvokedEvaluator(); const { inputTokens, outputTokens, toolCalls, latency } = evaluators.traceBasedEvaluators; @@ -288,6 +465,9 @@ export function createEvaluatePersonaMatrixDataset({ skillInvokedEvaluator, trajectoryEvaluator, expectedToolCalledEvaluator, + finalAnswerPresentEvaluator, + minExpectedStepsEvaluator, + connectorInvokedEvaluator, ...createQuantitativeCorrectnessEvaluators(), createQuantitativeGroundednessEvaluator(), evaluators.criteria([ @@ -304,6 +484,12 @@ export function createEvaluatePersonaMatrixDataset({ await executorClient.runExperiment( { + // Reasoning models (GLM, Qwen-thinking) wedge a single-node Kibana + // event loop at the default concurrency of 5: `converse` calls time out + // or fail outright with `fetch failed`, losing whole examples. Allow the + // runner to dial it back per model instead of hardcoding one value that + // is either too slow for frontier models or too aggressive for these. + concurrency: Number(process.env.PERSONA_MATRIX_CONCURRENCY) || undefined, datasets: [ { name: dataset.name, @@ -320,9 +506,17 @@ export function createEvaluatePersonaMatrixDataset({ const taskOutput: TaskOutput = { messages: response.messages, + // Which turn the answer came from: 'response' is the model's real + // closing message, 'last_assistant_step' is chat_client's fallback + // to an interior reasoning/output step (models that end on a tool + // call return an empty response.message). Without this tag, mid-run + // narration is indistinguishable from a final answer once stored. + messageSource: response.messageSource, steps: response.steps, errors: response.errors, traceId: response.traceId ?? null, + sampling: response.sampling, + trajectoryFingerprint: response.trajectoryFingerprint, }; // Precompute the qualitative analyses inside the task once, so the @@ -331,7 +525,12 @@ export function createEvaluatePersonaMatrixDataset({ // result and correctnessAnalysis() is invoked exactly once per // example (was: once here + once again as a registered evaluator). const expected = example.output as PersonaMatrixDatasetExpected; - const [correctnessResult, groundednessResult] = await Promise.all([ + + // The judges already retry internally; if they still fail, degrade this + // example's qualitative scores to "unavailable" (the quantitative + // evaluators handle a missing analysis) rather than discarding the + // agent's real trajectory, which the deterministic evaluators can score. + const [correctnessSettled, groundednessSettled] = await Promise.allSettled([ withEvaluatorSpan('CorrectnessAnalysis', {}, () => evaluators.correctnessAnalysis().evaluate({ input, @@ -350,6 +549,25 @@ export function createEvaluatePersonaMatrixDataset({ ), ]); + for (const [name, settled] of [ + ['CorrectnessAnalysis', correctnessSettled], + ['GroundednessAnalysis', groundednessSettled], + ] as const) { + if (settled.status === 'rejected') { + const reason = settled.reason; + log.error( + `[persona-matrix] ${name} failed for example "${example.id ?? question}": ${ + reason instanceof Error ? reason.message : String(reason) + }` + ); + } + } + + const correctnessResult = + correctnessSettled.status === 'fulfilled' ? correctnessSettled.value : undefined; + const groundednessResult = + groundednessSettled.status === 'fulfilled' ? groundednessSettled.value : undefined; + return { ...(taskOutput as object), correctnessAnalysis: correctnessResult?.metadata, diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluators/connector_invoked_evaluator.test.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluators/connector_invoked_evaluator.test.ts new file mode 100644 index 0000000000000..9812fa29dee3c --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluators/connector_invoked_evaluator.test.ts @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + createConnectorInvokedEvaluator, + collectProducedText, +} from './connector_invoked_evaluator'; + +const SLACK_CONNECTOR_ID = 'd7306385-cbe6-4541-9726-49afdff59ba5'; + +/** A workflow that genuinely posts to Slack — the passing baseline. */ +const WORKFLOW_WITH_SLACK = ` +name: chrysalis-triage-summary +triggers: + - type: manual +inputs: + - name: message + type: string +steps: + - name: post-to-slack + type: http + connector_id: ${SLACK_CONNECTOR_ID} + with: + body: "{{ inputs.message }}" +`; + +/** Same workflow with the Slack step removed — the mutation. */ +const WORKFLOW_WITHOUT_SLACK = ` +name: chrysalis-triage-summary +triggers: + - type: manual +inputs: + - name: message + type: string +steps: + - name: log-only + type: console + with: + body: "{{ inputs.message }}" +`; + +const expectation = { + expectedConnectorId: SLACK_CONNECTOR_ID, + expectedStepType: 'http', +}; + +const run = async (output: unknown, metadata: unknown = expectation) => { + const evaluator = createConnectorInvokedEvaluator(); + return evaluator.evaluate({ output, metadata } as never); +}; + +describe('ConnectorInvoked evaluator', () => { + it('scores 1 when the authored workflow targets the Slack connector', async () => { + const result = await run({ messages: [{ message: WORKFLOW_WITH_SLACK }] }); + expect(result.score).toBe(1); + expect(result.metadata).toMatchObject({ connectorPresent: true, stepPresent: true }); + }); + + // The mutation that matters: this is exactly the false-green the evaluator + // exists to catch. ExpectedToolCalled scores this 1.0; we must score it 0. + it('scores 0 when the Slack step is removed from the workflow', async () => { + const result = await run({ messages: [{ message: WORKFLOW_WITHOUT_SLACK }] }); + expect(result.score).toBe(0); + expect(result.explanation).toContain(SLACK_CONNECTOR_ID); + }); + + it('scores 0 when the workflow points at a different connector', async () => { + const wrong = WORKFLOW_WITH_SLACK.replace( + SLACK_CONNECTOR_ID, + '00000000-dead-beef-0000-000000000000' + ); + const result = await run({ messages: [{ message: wrong }] }); + expect(result.score).toBe(0); + expect(result.metadata).toMatchObject({ connectorPresent: false }); + }); + + it('does not accept prose that merely describes using an http step', async () => { + const prose = + `I would create a workflow that uses an http step against the Slack connector ` + + `${SLACK_CONNECTOR_ID}, posting the triage summary to #general.`; + const result = await run({ messages: [{ message: prose }] }); + // Connector id is mentioned, but there is no `type: http` key — narration + // must not score as an authored workflow. + expect(result.score).toBe(0); + expect(result.metadata).toMatchObject({ connectorPresent: true, stepPresent: false }); + }); + + it('returns N/A when the example declares no connector expectation', async () => { + const result = await run({ messages: [{ message: WORKFLOW_WITH_SLACK }] }, {}); + expect(result.score).toBeNull(); + expect(result.label).toBe('N/A'); + }); + + it('finds the workflow whether it is in the final answer or a tool result', async () => { + const inToolResult = { + messages: [{ message: 'Here is your workflow.' }], + steps: [{ tool_id: 'platform.core.generate_workflow', result: WORKFLOW_WITH_SLACK }], + }; + const result = await run(inToolResult); + expect(result.score).toBe(1); + }); + + it('treats regex metacharacters in the step type literally', async () => { + // Unescaped, `a.b` would match `type: axb`. The value comes from dataset + // metadata, so it must be escaped before being spliced into a RegExp. + const result = await run( + { messages: [{ message: 'steps:\n - type: axb\n' }] }, + { expectedStepType: 'a.b' } + ); + expect(result.score).toBe(0); + }); + + describe('collectProducedText', () => { + it('gathers strings from nested structures', () => { + const text = collectProducedText({ a: 'one', b: [{ c: 'two' }], d: { e: ['three'] } }); + expect(text).toContain('one'); + expect(text).toContain('two'); + expect(text).toContain('three'); + }); + }); +}); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluators/connector_invoked_evaluator.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluators/connector_invoked_evaluator.ts new file mode 100644 index 0000000000000..97f954e93f5a3 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluators/connector_invoked_evaluator.ts @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Evaluator } from '@kbn/evals'; + +export const CONNECTOR_INVOKED_EVALUATOR_NAME = 'ConnectorInvoked'; + +/** + * Deterministic check that an authored workflow targets the connector the + * prompt demanded. + * + * `ExpectedToolCalled` only proves the agent CALLED `generate_workflow`, never + * what came back — a workflow with no http step, or one aimed at the wrong + * connector, still scores 1.0. Being CODE rather than LLM-judged, a + * plausible-sounding answer cannot talk its way past this. + */ + +export interface ConnectorExpectation { + /** Connector id that must appear in the authored workflow. */ + expectedConnectorId?: string; + /** Step type that must appear, e.g. `http`. */ + expectedStepType?: string; +} + +/** + * Pull every string the agent produced: the final answer plus any tool results. + * The workflow YAML can surface in either depending on whether the model + * rendered it inline or left it in the generate_workflow response. + */ +export const collectProducedText = (output: unknown): string => { + const parts: string[] = []; + const visit = (node: unknown): void => { + if (typeof node === 'string') { + parts.push(node); + } else if (Array.isArray(node)) { + node.forEach(visit); + } else if (node && typeof node === 'object') { + Object.values(node as Record).forEach(visit); + } + }; + visit(output); + return parts.join('\n'); +}; + +export const createConnectorInvokedEvaluator = (): Evaluator => ({ + name: CONNECTOR_INVOKED_EVALUATOR_NAME, + kind: 'CODE', + direction: 'maximize', + evaluate: async ({ output, metadata }) => { + const meta = metadata as ConnectorExpectation | undefined; + const expectedConnectorId = meta?.expectedConnectorId; + const expectedStepType = meta?.expectedStepType; + + if (!expectedConnectorId && !expectedStepType) { + return { + score: null, + label: 'N/A', + explanation: 'No connector expectation declared — ConnectorInvoked does not apply.', + metadata: {}, + }; + } + + const produced = collectProducedText(output); + + // Connector ids are opaque identifiers; a substring match is exact enough + // and avoids depending on YAML indentation or quoting style. + const connectorPresent = expectedConnectorId ? produced.includes(expectedConnectorId) : true; + + // Step type must appear as a YAML key (`type: http`), not merely in prose, + // so a model narrating "I would use an http step" does not pass. Escaped: + // the value comes from dataset metadata, not a trusted literal. + const escapedStepType = expectedStepType?.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const stepPresent = escapedStepType + ? new RegExp(`type\\s*:\\s*["']?${escapedStepType}\\b`, 'i').test(produced) + : true; + + const passed = connectorPresent && stepPresent; + + const missing: string[] = []; + if (!connectorPresent) missing.push(`connector id ${expectedConnectorId}`); + if (!stepPresent) missing.push(`step type ${expectedStepType}`); + + return { + score: Number(passed), + explanation: passed + ? 'Authored workflow targets the required connector.' + : `Authored workflow is missing: ${missing.join(', ')}.`, + metadata: { + expectedConnectorId, + expectedStepType, + connectorPresent, + stepPresent, + }, + }; + }, +}); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/fixtures/env_seeds.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/fixtures/env_seeds.ts new file mode 100644 index 0000000000000..72576df1da541 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/fixtures/env_seeds.ts @@ -0,0 +1,481 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under the + * Elastic License 2.0. Licensed under the Elastic License 2.0"; you may not use + * this file except in compliance with, at your election, the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", or the "Server Side + * Public License v 1". + */ + +import type { Client as EsClient } from '@elastic/elasticsearch'; +import type { KbnClient } from '@kbn/kbn-client'; +import type { ToolingLog } from '@kbn/tooling-log'; + +/** + * Environment-truth seeds for the persona matrix. + * + * Every category prompt must be satisfiable by data/tools that actually exist + * in the eval stack. Before this module, four categories measured "model gives + * up gracefully on a missing backend" instead of the capability under test: + * + * - Threat Hunting hunts `log.dll` across process/file telemetry → we seed + * `logs-endpoint.events.process-default` with the side-load evidence. + * - Multi-Step checks the Chrysalis hash against Elastic Security Labs → we + * seed the Labs knowledge index with a T1574.002 entry for that hash. + * - Triggering Workflows checks the hash against VirusTotal → VirusTotal is + * NOT shipped with the stack; we seed a mock threat-intel verdict index and + * (in persona_matrix_tools_seed) point the `virustotal_lookup` tool at it. + * - Entity Analytics profiles srv-win-defend-01 → we seed a watchlist and an + * entity-store record so watchlist/entity lookups return data. + * + * All seeds are idempotent (skip when the marker doc already exists) and + * cleaned up by `cleanupEnvSeeds`. + */ + +export const ENV_SEED_MARKER_INDEX = 'persona-matrix-env-seeds'; +const ENDPOINT_INDEX = 'logs-endpoint.events.process-default'; +const LABS_INDEX = 'logs-security_labs.research-default'; +const TI_INDEX = 'ti-mock-default'; +const WATCHLIST_SO_TYPE = 'security-tanker-watchlist'; +const ONCALL_INDEX = 'on-call-schedule'; + +const CHRYSALIS_HASH = '275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f'; +const HOST = 'srv-win-defend-01'; + +// Entity docs in `entities-latest-default` are keyed by sha256(euid) — the +// same digest as @kbn/entity-store's hashEuid — so re-seeding overwrites +// instead of duplicating. The four euids below are fixed dataset constants, +// so their digests are precomputed at author time rather than hashing at +// runtime (which would need the Node crypto builtin, disallowed here): +// sha256('host-default-srv-win-defend-01') = 8956913254bdbf2d6070b7cfc851c691f653b150050013b4bea74c9f9b93457f +// sha256('host-default-srv-linux-web-02') = 1d3a97ba8f44d13e1c6e249923b7994d14c7f36e5223d29832113403c0d24a6c +// sha256('host-default-srv-mac-dev-03') = c7d2277442521af9efbf039522b13067d5444d1dbdb890a1e759ff5527553ed9 +// sha256('user-default-SYSTEM-srv-win-defend-01') = 4e1e3960d8231ed8b67f2f3788f4a5602b93a44a9ab96cd25e8f0a2c6e2bb9d2 +const SEED_DOC_IDS: Record = { + 'host-default-srv-win-defend-01': + '8956913254bdbf2d6070b7cfc851c691f653b150050013b4bea74c9f9b93457f', + 'host-default-srv-linux-web-02': + '1d3a97ba8f44d13e1c6e249923b7994d14c7f36e5223d29832113403c0d24a6c', + 'host-default-srv-mac-dev-03': 'c7d2277442521af9efbf039522b13067d5444d1dbdb890a1e759ff5527553ed9', + 'user-default-SYSTEM-srv-win-defend-01': + '4e1e3960d8231ed8b67f2f3788f4a5602b93a44a9ab96cd25e8f0a2c6e2bb9d2', +}; + +interface SeedOptions { + esClient: EsClient; + kbnClient: KbnClient; + log: ToolingLog; +} + +async function ensureIndexWithDocs( + esClient: EsClient, + index: string, + docs: Array>, + log: ToolingLog, + markerId: string +): Promise { + const marker = await esClient.exists({ index, id: markerId }).catch(() => ({ body: false })); + // exists() throws on missing index; treat that as "not seeded" + if (marker && (marker as { body?: boolean }).body === true) { + log.info(`[env-seed] ${index} already seeded, skipping`); + return; + } + await esClient.indices.create({ index }).catch((err) => { + // 400 resource_already_exists is fine (concurrent boot) + const status = + (err as { statusCode?: number })?.statusCode ?? + (err as { meta?: { statusCode?: number } })?.meta?.statusCode; + if (status !== 400) throw err; + }); + // The bulk can transiently fail right after boot (ES still initializing, + // index going yellow, master flap). A single-shot failure here kills the + // whole eval attempt (~60min of model work on slow models), so retry the + // seed with backoff instead of letting a 5-second blip fail the run. + const bulkOperations = [ + ...docs.flatMap((doc) => [{ create: {} }, doc] as const), + // marker doc so idempotent reruns skip + { create: { _id: markerId } }, + { seeded: true, seeded_at: new Date().toISOString() }, + ]; + const MAX_SEED_ATTEMPTS = 5; + let lastError: unknown; + for (let attempt = 1; attempt <= MAX_SEED_ATTEMPTS; attempt++) { + try { + await esClient.bulk({ + index, + refresh: 'wait_for', + operations: bulkOperations, + }); + lastError = undefined; + break; + } catch (err) { + lastError = err; + const isLast = attempt === MAX_SEED_ATTEMPTS; + log.warning( + `[env-seed] bulk into ${index} failed (attempt ${attempt}/${MAX_SEED_ATTEMPTS}): ${String( + err + )}` + ); + if (isLast) throw err; + await new Promise((r) => setTimeout(r, attempt * 10_000)); + } + } + if (lastError !== undefined) throw lastError; + log.info(`[env-seed] seeded ${docs.length} docs into ${index}`); +} + +export async function seedPersonaMatrixEnvironment({ + esClient, + kbnClient, + log, +}: SeedOptions): Promise { + const markerId = 'persona-matrix-env-seed-v1'; + + // A2: endpoint process telemetry with the log.dll side-load evidence. + // + // Fixture density matters here: a bare 3-doc evidence-only index made every + // model's legitimate first move (a 30d baseline / 24h window ESQL query) + // return empty, burning 6+ diagnostic tool calls rediscovering what data + // exists (observed on every model in the matrix, worst on slow ones). + // Seeded shape: + // - ~40 benign baseline process events spread over the last 30 days for + // HOST and two neighbors, timestamps computed relative to seed time so + // the fixture never rots. + // - The malicious chain stamped ~50 minutes ago, i.e. inside any sane + // recent-activity window, and anomalous against the baseline by + // construction (SYSTEM user, Users\Public path, unsigned DLL). + const now = Date.now(); + const minutesAgo = (m: number) => new Date(now - m * 60_000).toISOString(); + const daysAgo = (d: number, jitterMin = 0) => + new Date(now - d * 86_400_000 - jitterMin * 60_000).toISOString(); + + const benignExecutables = [ + { name: 'svchost.exe', exe: 'C:\\Windows\\System32\\svchost.exe', user: 'SYSTEM' }, + { + name: 'MsMpEng.exe', + exe: 'C:\\Program Files\\Windows Defender\\MsMpEng.exe', + user: 'SYSTEM', + }, + { name: 'RuntimeBroker.exe', exe: 'C:\\Windows\\System32\\RuntimeBroker.exe', user: 'DANEL' }, + { + name: 'chrome.exe', + exe: 'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe', + user: 'DANEL', + }, + { + name: 'powershell.exe', + exe: 'C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe', + user: 'DANEL', + }, + ]; + const baselineHosts = [HOST, 'srv-linux-web-02', 'srv-mac-dev-03']; + const baselineDocs: Array> = []; + benignExecutables.forEach((bin, i) => { + baselineHosts.forEach((host, j) => { + // ~13 events per host over 30 days: enough for COUNT/STATS baselines, + // small enough to keep the seed bulk cheap. + for (let k = 0; k < 3; k++) { + const day = 1 + ((i * 7 + j * 3 + k * 5) % 29); // deterministic spread 1..29 days ago + baselineDocs.push({ + '@timestamp': daysAgo(day, i * 10 + j * 5 + k), + 'event.category': ['process'], + 'event.type': ['start'], + 'event.dataset': 'endpoint.events.process', + 'host.name': host, + 'user.name': bin.user, + 'process.name': bin.name, + 'process.executable': bin.exe, + 'process.parent.name': host === HOST ? 'services.exe' : 'init', + 'process.command_line': `"${bin.exe}"`, + }); + } + }); + }); + + await ensureIndexWithDocs( + esClient, + ENDPOINT_INDEX, + [ + ...baselineDocs, + { + '@timestamp': minutesAgo(50), + 'event.category': ['process'], + 'event.type': ['start'], + 'event.dataset': 'endpoint.events.process', + 'host.name': HOST, + 'user.name': 'SYSTEM', + 'process.name': 'BluetoothService.exe', + 'process.executable': 'C:\\Windows\\BluetoothService.exe', + 'process.parent.name': 'services.exe', + 'process.command_line': 'C:\\Windows\\BluetoothService.exe -embed', + }, + { + '@timestamp': minutesAgo(49), + 'event.category': ['file'], + 'event.type': ['creation'], + 'event.dataset': 'endpoint.events.file', + 'host.name': HOST, + 'user.name': 'SYSTEM', + 'process.name': 'BluetoothService.exe', + 'file.name': 'log.dll', + 'file.path': 'C:\\Users\\Public\\log.dll', + 'file.hash.sha256': CHRYSALIS_HASH, + 'file.code_signature.status': 'unsigned', + }, + { + '@timestamp': minutesAgo(48), + 'event.category': ['dll'], + 'event.type': ['start'], + 'event.action': 'dll_loaded', + 'event.dataset': 'endpoint.events.library', + 'host.name': HOST, + 'process.name': 'BluetoothService.exe', + 'dll.name': 'log.dll', + 'dll.path': 'C:\\Users\\Public\\log.dll', + 'process.executable': 'C:\\Windows\\BluetoothService.exe', + }, + ], + log, + markerId + ); + + // A3: Elastic Security Labs research entry so security_labs_search returns content. + await ensureIndexWithDocs( + esClient, + LABS_INDEX, + [ + { + '@timestamp': '2026-07-14T00:00:00.000Z', + title: 'Chrysalis backdoor: DLL side-loading via BluetoothService.exe', + 'security_labs.id': 'SL-2026-0142', + 'security_labs.tags': ['malware', 'chrysalis', 'dll-side-loading'], + 'security_labs.threat_technique_id': 'T1574.002', + 'security_labs.ioc.hash.sha256': [CHRYSALIS_HASH], + 'security_labs.ioc.file.name': ['log.dll'], + 'security_labs.ioc.process.name': ['BluetoothService.exe'], + content: + `Elastic Security Labs tracks the Chrysalis backdoor as side-loading log.dll ` + + `(sha256 ${CHRYSALIS_HASH}) from a world-writable path via the legitimate BluetoothService.exe binary. ` + + `The hash is a known-bad indicator: flagged by 45/72 vendors on VirusTotal at ` + + `time of research. Recommended detection: DLL load from C:\\Users\\Public by a ` + + `service binary (T1574.002).`, + }, + ], + log, + markerId + ); + + // A1: mock VirusTotal verdict so the stubbed virustotal_lookup tool returns a + // coherent answer. VirusTotal itself is NOT part of the stack — this index is + // the eval-only stub the tool queries. + await ensureIndexWithDocs( + esClient, + TI_INDEX, + [ + { + '@timestamp': '2026-07-20T00:00:00.000Z', + 'threat_intel.provider': 'virustotal-mock', + 'threat_intel.indicator.type': 'hash.sha256', + 'threat_intel.indicator.value': CHRYSALIS_HASH, + 'threat_intel.verdict': 'malicious', + 'threat_intel.detection_ratio': '45/72', + 'threat_intel.classification': 'trojan/chrysalis', + 'threat_intel.first_seen': '2026-07-02T00:00:00.000Z', + }, + ], + log, + markerId + ); + + // A4: Entity Store V2 engines + entities in the latest alias so entity + // lookups return data. Entity Analytics Agent Builder tools gate on the + // `entities-latest-` alias existing (entity_analytics_availability), + // which only the installed V2 engines create — a bare `.entities-v1` index + // (previous seed) satisfies no gate and leaves every tool unavailable. + // Pattern ported from kbn-evals-suite-entity-analytics setup_helpers. + try { + // Warm-stack fast path: engines already installed and running from a + // previous run — skip straight to re-seeding docs. + const initial = (await kbnClient.request({ + method: 'GET', + path: '/api/security/entity_store/status', + })) as unknown as { data?: { status?: string } }; + if (initial.data?.status !== 'running') { + log.info(`[env-seed] installing entity store v2`); + // Fire-and-poll: on a cold cluster the install endpoint can hold the + // connection open for the entire transform-init duration (observed + // >9 min, wedging beforeAll past any await deadline). The status poll + // below is the real completion signal. + kbnClient + .request({ + method: 'POST', + path: '/api/security/entity_store/install', + body: { entityTypes: ['user', 'host'] }, + }) + .catch((err) => log.warning(`[env-seed] entity store install call errored: ${err}`)); + // First-install on a cold stack initializes ES transforms and can take + // several minutes (observed >120s locally); the entity-analytics suite + // helper defaults to the same poll but is equally tunable. + const deadline = Date.now() + 600_000; + for (;;) { + const statusRes = (await kbnClient.request({ + method: 'GET', + path: '/api/security/entity_store/status', + })) as unknown as { data?: { status?: string } }; + const body = statusRes.data; + // Visible poll progress: a silent loop here previously masked a + // response-shape bug for entire runs. + log.info(`[env-seed] entity store status: ${body?.status ?? 'unknown'}`); + if (body?.status === 'running') break; + if (body?.status === 'error') { + throw new Error(`entity store v2 error state: ${JSON.stringify(body)}`); + } + if (Date.now() > deadline) { + throw new Error('entity store v2 did not reach running within 600s'); + } + await new Promise((r) => setTimeout(r, 2_000)); + } + } + log.info(`[env-seed] entity store v2 running`); + + const latestAlias = 'entities-latest-default'; + const seededAt = new Date().toISOString(); + const seedEntities = [ + { + euid: `host-default-${HOST}`, + type: 'host', + name: HOST, + riskLevel: 'high' as const, + riskScoreNorm: 73, + assetCriticality: 'high_impact' as const, + }, + { + euid: 'host-default-srv-linux-web-02', + type: 'host', + name: 'srv-linux-web-02', + riskLevel: 'medium' as const, + riskScoreNorm: 41, + assetCriticality: 'normal' as const, + }, + { + euid: 'host-default-srv-mac-dev-03', + type: 'host', + name: 'srv-mac-dev-03', + riskLevel: 'low' as const, + riskScoreNorm: 18, + assetCriticality: 'normal' as const, + }, + { + euid: `user-default-SYSTEM-${HOST}`, + type: 'user', + name: 'SYSTEM', + riskLevel: 'critical' as const, + riskScoreNorm: 88, + assetCriticality: 'high_impact' as const, + }, + ]; + const operations = seedEntities.flatMap((e) => { + const doc: Record = { + '@timestamp': seededAt, + entity: { + id: e.euid, + EngineMetadata: { Type: e.type }, + risk: { + calculated_level: e.riskLevel, + calculated_score_norm: e.riskScoreNorm, + }, + }, + [e.type]: { name: e.name }, + asset: { criticality: e.assetCriticality }, + }; + const seedDocId = SEED_DOC_IDS[e.euid]; + if (!seedDocId) { + throw new Error( + `No precomputed doc id for ${e.euid} — add its sha256 to SEED_DOC_IDS in env_seeds.ts` + ); + } + return [{ index: { _index: latestAlias, _id: seedDocId } }, doc] as const; + }); + await esClient.bulk({ refresh: true, operations }); + log.info(`[env-seed] seeded ${seedEntities.length} entities into ${latestAlias}`); + } catch (err) { + // Non-fatal by design (same contract as the watchlist seed): a missing + // entity store degrades entity-analytics coverage but the suite still + // measures the other six categories. + log.warning(`[env-seed] entity store v2 seed failed (non-fatal): ${err}`); + } + + // A5: on-call schedule so on_call_lookup (re-pointed at this index) can answer + // "who is on call". The original run queried a dedicated on-call-schedule + // index; without it the tool queried the alerts index, which has no responder + // data, making workflow-execution-b structurally unanswerable. + await ensureIndexWithDocs( + esClient, + ONCALL_INDEX, + [ + { + '@timestamp': '2026-07-21T00:00:00.000Z', + name: 'Dana Whitfield', + email: 'dana.whitfield@example.com', + slack_handle: '@dana-w', + shift_start: '2026-07-20T00:00:00.000Z', + shift_end: '2026-07-27T00:00:00.000Z', + is_primary: true, + escalation_tier: 'primary', + }, + { + '@timestamp': '2026-07-21T00:00:00.000Z', + name: 'Ravi Osei', + email: 'ravi.osei@example.com', + slack_handle: '@ravi-o', + shift_start: '2026-07-20T00:00:00.000Z', + shift_end: '2026-07-27T00:00:00.000Z', + is_primary: false, + escalation_tier: 'secondary', + }, + ], + log, + markerId + ); + + // A4b: watchlist via saved object (kbnClient) — find API on the tankers type. + try { + await kbnClient.request({ + method: 'POST', + path: '/api/security/watchlists', + body: { + name: 'Privileged Users', + description: 'persona-matrix seed: privileged user monitoring watchlist', + filters: [{ field: 'user.name', value: 'SYSTEM' }], + }, + }); + log.info(`[env-seed] created watchlist 'Privileged Users'`); + } catch (err) { + const status = (err as { status?: number })?.status; + if (status === 409) { + log.info(`[env-seed] watchlist already exists, reusing`); + } else { + // Watchlist API shape may differ across versions; a missing watchlist + // degrades entity-analytics coverage but must not fail the suite. + log.warning(`[env-seed] watchlist seed failed (non-fatal): ${err}`); + } + } + void WATCHLIST_SO_TYPE; +} + +export async function cleanupEnvSeeds({ esClient, log }: SeedOptions): Promise { + // entities-latest-default is owned by the Entity Store V2 engines and + // outlives the run on purpose — uninstalling it would take the engine state + // with it. Plain seed indices are deleted idempotently. + for (const index of [ENDPOINT_INDEX, LABS_INDEX, TI_INDEX]) { + await esClient.indices.delete({ index }).catch(() => { + log.info(`[env-seed] cleanup: ${index} already gone`); + }); + } +} diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/fixtures/persona_matrix_tools_seed.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/fixtures/persona_matrix_tools_seed.ts index b5b3ededead8f..01059f0c11569 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/fixtures/persona_matrix_tools_seed.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/fixtures/persona_matrix_tools_seed.ts @@ -7,6 +7,7 @@ import type { KbnClient } from '@kbn/kbn-client'; import type { ToolingLog } from '@kbn/tooling-log'; +import { agentBuilderDefaultAgentId } from '@kbn/agent-builder-common'; /** * Seeds two real, registered Agent Builder tools (`virustotal_lookup` and @@ -31,8 +32,6 @@ const AGENT_BUILDER_TOOLS_HEADERS = { 'elastic-api-version': ELASTIC_API_VERSION, } as const; -const ALERT_INDEX = '.internal.alerts-security.alerts-default-000001'; - export const PERSONA_MATRIX_TOOL_IDS = ['virustotal_lookup', 'on_call_lookup'] as const; interface SeedToolsOptions { @@ -45,6 +44,7 @@ async function createToolIfMissing({ log, body, }: SeedToolsOptions & { body: Record }): Promise { + const toolPath = `/api/agent_builder/tools/${encodeURIComponent(String(body.id))}`; try { await kbnClient.request({ method: 'POST', @@ -54,12 +54,35 @@ async function createToolIfMissing({ }); log.info(`[persona-matrix] created tool '${body.id}'`); } catch (error) { - const status = (error as { status?: number })?.status; - if (status === 409) { - log.info(`[persona-matrix] tool '${body.id}' already exists, reusing`); - return; + const message = String((error as Error)?.message ?? error); + if (!/already exists/i.test(message)) { + throw error; + } + // Multi-model gate runs re-execute beforeAll per model worker against the + // same stack, so the tool can pre-date this call. Recover by reinstalling + // the CURRENT definition (delete-then-post); if the delete also fails, + // keep the stale-but-equivalent definition rather than failing the suite. + try { + await kbnClient.request({ + method: 'DELETE', + path: toolPath, + headers: AGENT_BUILDER_TOOLS_HEADERS, + }); + await kbnClient.request({ + method: 'POST', + path: '/api/agent_builder/tools', + headers: AGENT_BUILDER_TOOLS_HEADERS, + body, + }); + log.info(`[persona-matrix] removed stale tool '${body.id}', reinstalled`); + } catch (recoveryError) { + log.warning( + `[persona-matrix] tool '${body.id}' already exists and reinstall failed ` + + `(${String( + (recoveryError as Error)?.message ?? recoveryError + )}); keeping existing definition` + ); } - throw error; } } @@ -72,11 +95,22 @@ export async function seedPersonaMatrixTools({ kbnClient, log }: SeedToolsOption type: 'esql', description: 'Look up a file hash, URL, or domain against VirusTotal threat intelligence to check ' + - 'for known-malicious indicators. Use this to verify whether a given hash, URL, or ' + + 'for known-malicious indicators. Returns the verdict (benign/malicious), detection ' + + 'ratio, and classification. Use this to verify whether a given hash, URL, or ' + 'domain has been flagged by security vendors.', tags: ['persona-matrix', 'threat-intel'], configuration: { - query: `FROM ${ALERT_INDEX} | WHERE kibana.alert.rule.name LIKE "*Chrysalis*" | KEEP kibana.alert.rule.name, kibana.alert.reason | LIMIT 10`, + // Queries the eval-seeded mock verdict index (see env_seeds.ts) so the + // tool returns a coherent VirusTotal-style answer without any network + // access or real VirusTotal subscription. `params` is REQUIRED by the + // esql tool schema (verified live: omitting it fails with "[params]: + // expected value of type [object] but got [undefined]"; a params entry + // not referenced by a {{placeholder}} fails with "Defined parameters + // not used in query"). The {{hash}} placeholder drives the input + // schema; params stays empty. + query: + 'FROM ti-mock-default | WHERE threat_intel.indicator.value == "{{hash}}" ' + + '| KEEP threat_intel.verdict, threat_intel.detection_ratio, threat_intel.classification | LIMIT 5', params: {}, }, }, @@ -93,13 +127,53 @@ export async function seedPersonaMatrixTools({ kbnClient, log }: SeedToolsOption 'on-call responder to own or escalate a security incident.', tags: ['persona-matrix', 'incident-response'], configuration: { - query: `FROM ${ALERT_INDEX} | WHERE kibana.alert.rule.name LIKE "*Chrysalis*" | KEEP kibana.alert.rule.name, kibana.alert.severity | LIMIT 10`, + // Queries the eval-seeded on-call schedule index (see env_seeds.ts A5). + // Previously this pointed at the alerts index, which has no responder + // fields — making workflow-execution-b structurally unanswerable. + query: + `FROM on-call-schedule | WHERE is_primary == true ` + + `| KEEP name, email, slack_handle, shift_start, shift_end | LIMIT 5`, + // esql tool schema requires `params` present (even empty) — omitting + // it fails validation with "expected value of type [object] but got + // [undefined]" (verified live). params: {}, }, }, }); } +/** + * Attaches the seeded tools to the default agent's configuration. + * + * Creating a tool only puts it in the registry. `selectTools` (agent_builder + * server) exposes ONLY `agentConfiguration.tools` plus the hardcoded + * `defaultAgentToolIds` (all `platform.core.*`) — so a registry tool that is + * never attached is invisible to the model, and any example scoring + * "did it call virustotal_lookup" is structurally unanswerable. Verified live + * 2026-08-22: default agent ships `tools: []`, and every workflow-execution + * run scored ExpectedToolCalled=0 until this attach was added. + */ +export async function attachPersonaMatrixToolsToAgent({ + kbnClient, + log, +}: SeedToolsOptions): Promise { + await kbnClient.request({ + method: 'PUT', + path: `/api/agent_builder/agents/${agentBuilderDefaultAgentId}`, + headers: AGENT_BUILDER_TOOLS_HEADERS, + body: { + configuration: { + tools: [{ tool_ids: [...PERSONA_MATRIX_TOOL_IDS] }], + }, + }, + }); + log.info( + `[persona-matrix] attached ${PERSONA_MATRIX_TOOL_IDS.join( + ', ' + )} to '${agentBuilderDefaultAgentId}'` + ); +} + export async function cleanupPersonaMatrixTools({ kbnClient, log, diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/fixtures/tool_registration_check.test.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/fixtures/tool_registration_check.test.ts new file mode 100644 index 0000000000000..31fdff200bc77 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/fixtures/tool_registration_check.test.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + collectExpectedToolIds, + findUnregisteredToolIds, + findUnattachedToolIds, + selectListAssertableBuiltins, +} from './tool_registration_check'; +import type { KbnClient } from '@kbn/kbn-client'; + +describe('collectExpectedToolIds', () => { + it('collects, dedupes, and sorts expectedTools across examples', () => { + const dataset = [ + { metadata: { expectedTools: ['virustotal_lookup', 'platform.core.search'] } }, + { metadata: { expectedTools: ['on_call_lookup', 'virustotal_lookup'] } }, + { metadata: {} }, + ]; + expect(collectExpectedToolIds(dataset)).toEqual([ + 'on_call_lookup', + 'platform.core.search', + 'virustotal_lookup', + ]); + }); +}); + +describe('findUnregisteredToolIds', () => { + const makeKbnClient = (tools: Array<{ id: string }>) => + ({ + request: jest.fn().mockResolvedValue({ data: { results: tools } }), + } as unknown as KbnClient); + + it('returns tool ids not present in the registry', async () => { + const kbnClient = makeKbnClient([{ id: 'virustotal_lookup' }]); + await expect( + findUnregisteredToolIds(kbnClient, ['virustotal_lookup', 'on_call_lookup']) + ).resolves.toEqual(['on_call_lookup']); + }); + + it('lists tools with the date-formatted public API version (not "1")', async () => { + const kbnClient = makeKbnClient([]); + await findUnregisteredToolIds(kbnClient, ['virustotal_lookup']); + const request = (kbnClient as unknown as { request: jest.Mock }).request; + expect(request).toHaveBeenCalledWith( + expect.objectContaining({ + method: 'GET', + path: '/api/agent_builder/tools', + headers: expect.objectContaining({ + // 400 "Invalid version" regression: the public tools API expects a + // YYYY-MM-DD version string, like the seed client. + 'elastic-api-version': '2023-10-31', + }), + }) + ); + }); + + it('surfaces listing failures instead of silently passing the check', async () => { + const kbnClient = { + request: jest.fn().mockRejectedValue(new Error('boom')), + } as unknown as KbnClient; + await expect(findUnregisteredToolIds(kbnClient, ['virustotal_lookup'])).rejects.toThrow( + 'tool-registration pre-flight' + ); + }); + + it('flags availability-gated built-ins the tools list does not expose', async () => { + // Regression shape of the security.entity_risk_score bug: the tool is a + // registered built-in, but its availability handler (skills flag on) + // keeps it out of the public tools list, so the model can never call it. + const kbnClient = makeKbnClient([ + { id: 'security.get_entity' }, + { id: 'platform.core.generate_esql' }, + ]); + await expect( + findUnregisteredToolIds(kbnClient, [ + 'security.get_entity', + 'platform.core.generate_esql', + 'security.entity_risk_score', + ]) + ).resolves.toEqual(['security.entity_risk_score']); + }); + + it('exempts conversation-scoped built-ins from the list assertion', () => { + // attachments.* are injected into an agent run at execution time and never + // appear in the registry tools list — flagging them as availability-gated + // would fail every run spuriously (caught live on the first two-model + // gate run, 2026-08-23). + expect( + selectListAssertableBuiltins(['attachments.read', 'security.get_entity', 'virustotal_lookup']) + ).toEqual(['security.get_entity']); + }); +}); + +describe('findUnattachedToolIds', () => { + const makeAgentClient = (tools: Array<{ tool_ids?: string[] }>) => + ({ + request: jest.fn().mockResolvedValue({ data: { configuration: { tools } } }), + } as unknown as KbnClient); + + it('flags a registered tool that is not attached to the agent', async () => { + // The 2026-08-22 defect: both tools existed in the registry, but the + // default agent shipped `tools: []`, so the model never saw them and + // ExpectedToolCalled scored 0 across every workflow-execution run. + const kbnClient = makeAgentClient([]); + await expect( + findUnattachedToolIds(kbnClient, ['virustotal_lookup', 'on_call_lookup']) + ).resolves.toEqual(['virustotal_lookup', 'on_call_lookup']); + }); + + it('returns nothing when every tool is attached', async () => { + const kbnClient = makeAgentClient([{ tool_ids: ['virustotal_lookup', 'on_call_lookup'] }]); + await expect( + findUnattachedToolIds(kbnClient, ['virustotal_lookup', 'on_call_lookup']) + ).resolves.toEqual([]); + }); + + it('flattens tool_ids across multiple selection entries', async () => { + const kbnClient = makeAgentClient([ + { tool_ids: ['virustotal_lookup'] }, + { tool_ids: ['on_call_lookup'] }, + ]); + await expect( + findUnattachedToolIds(kbnClient, ['virustotal_lookup', 'on_call_lookup']) + ).resolves.toEqual([]); + }); +}); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/fixtures/tool_registration_check.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/fixtures/tool_registration_check.ts new file mode 100644 index 0000000000000..0804cc7889ffd --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/fixtures/tool_registration_check.ts @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { KbnClient } from '@kbn/kbn-client'; +import type { ToolingLog } from '@kbn/tooling-log'; +import { agentBuilderDefaultAgentId } from '@kbn/agent-builder-common'; +import { personaMatrixDataset } from '../datasets'; + +/** + * Collects every expectedTools entry declared across the dataset's examples. + */ +export function collectExpectedToolIds( + dataset: Array<{ metadata?: { expectedTools?: string[] } }> +): string[] { + const ids = new Set(); + for (const ex of dataset) { + for (const t of ex.metadata?.expectedTools ?? []) { + ids.add(t); + } + } + return [...ids].sort(); +} + +interface AgentBuilderTool { + id: string; +} + +/** + * Returns the subset of `toolIds` that are NOT registered in Agent Builder. + * Custom eval-seeded tools live in the Agent Builder registry; platform + * built-ins (platform.core.*, security.*) are always present and not listed + * here, so this check targets the eval-seeded surface only. + */ +export async function findUnregisteredToolIds( + kbnClient: KbnClient, + toolIds: string[] +): Promise { + let registered: AgentBuilderTool[] = []; + try { + const response = await kbnClient.request<{ + results?: AgentBuilderTool[]; + tools?: AgentBuilderTool[]; + }>({ + method: 'GET', + path: '/api/agent_builder/tools', + query: { per_page: 1000 }, + headers: { + 'kbn-xsrf': 'persona-matrix-tool-check', + // Public versioned API: date format required, like the seed client. + // '1' is rejected with 400 "Invalid version". + 'elastic-api-version': '2023-10-31', + 'x-elastic-internal-origin': 'kbn-evals', + }, + }); + const body = (response as { data?: unknown }).data ?? response; + const list = body as { results?: AgentBuilderTool[]; tools?: AgentBuilderTool[] }; + registered = list.results ?? list.tools ?? []; + } catch (error) { + // If we can't list tools, don't block the suite on a guess — return all as + // unverified-but-not-missing is wrong; surface the failure instead. + throw new Error(`tool-registration pre-flight: failed to list Agent Builder tools: ${error}`); + } + const present = new Set(registered.map((t) => t.id)); + return toolIds.filter((id) => !present.has(id)); +} + +/** + * Built-ins that are injected into an agent run's tool set at execution time + * (run_agent/utils/select_tools) and therefore never appear in the registry + * tools list. Their absence there is by design, not an availability gate. + */ +const CONVERSATION_SCOPED_TOOL_PREFIXES = ['attachments.']; + +/** + * The subset of expected built-in tool ids whose presence must be asserted via + * the public tools list — i.e. excluding conversation-scoped ones. + */ +export function selectListAssertableBuiltins(toolIds: string[]): string[] { + return toolIds.filter( + (id) => + id.includes('.') && !CONVERSATION_SCOPED_TOOL_PREFIXES.some((prefix) => id.startsWith(prefix)) + ); +} + +/** + * Pre-flight assertion: every expectedTools entry that is an eval-seeded + * custom tool (i.e. NOT a platform.* / security.* built-in) must be registered + * before the suite runs. A renamed or unseeded custom tool otherwise silently + * zeroes the ExpectedToolCalled evaluator and shows up as a confusing score + * drop rather than an actionable error. Fails fast with the missing ids. + */ +export async function assertPersonaMatrixToolsRegistered({ + kbnClient, + log, +}: { + kbnClient: KbnClient; + log: ToolingLog; +}): Promise { + const expected = collectExpectedToolIds(personaMatrixDataset); + // Only custom (eval-seeded) tools are listed in the Agent Builder registry; + // platform.* and security.* built-ins are always available and not registered + // as custom tools, so they're excluded from the registry check below. + const custom = expected.filter((id) => !id.includes('.')); + if (custom.length === 0) { + log.info('[persona-matrix] pre-flight: no custom expectedTools to verify'); + return; + } + const missing = await findUnregisteredToolIds(kbnClient, custom); + if (missing.length > 0) { + throw new Error( + `[persona-matrix] pre-flight tool-registration check failed: expected custom tools not registered: ${missing.join( + ', ' + )}. Seed them via seedPersonaMatrixTools before running the suite.` + ); + } + log.info( + `[persona-matrix] pre-flight: ${custom.length} custom tools registered (${custom.join(', ')})` + ); + + // BUILT-IN EXPECTED TOOLS: registration alone is not enough. The public + // tools list already excludes built-ins whose availability handler returned + // unavailable (skills flags, missing indices, license gates), and the model + // sees exactly that filtered list — so an expected built-in that is + // availability-gated off in this stack zeroes ExpectedToolCalled for every + // example declaring it. This is how security.entity_risk_score scored 0 on + // 34/34 runs before its availability contract was understood. + const builtins = selectListAssertableBuiltins(expected); + if (builtins.length > 0) { + const gatedOff = await findUnregisteredToolIds(kbnClient, builtins); + if (gatedOff.length > 0) { + throw new Error( + `[persona-matrix] pre-flight tool-availability check failed: expected built-in tools ` + + `not exposed by the Agent Builder tools list (availability-gated off in this stack): ` + + `${gatedOff.join(', ')}. Fix the environment or the expectedTools contract — ` + + `running anyway would silently score these examples 0.` + ); + } + log.info( + `[persona-matrix] pre-flight: ${builtins.length} built-in expectedTools exposed by the tools list` + ); + } + + // Registration is necessary but NOT sufficient: `selectTools` only exposes + // tools attached to the agent's configuration plus `defaultAgentToolIds`. + // A registered-but-unattached tool is invisible to the model, which zeroes + // ExpectedToolCalled while every other signal looks healthy. + const unattached = await findUnattachedToolIds(kbnClient, custom); + if (unattached.length > 0) { + throw new Error( + `[persona-matrix] pre-flight tool-attachment check failed: tools registered but not attached ` + + `to agent '${agentBuilderDefaultAgentId}': ${unattached.join(', ')}. ` + + `Call attachPersonaMatrixToolsToAgent after seeding.` + ); + } + log.info(`[persona-matrix] pre-flight: custom tools attached to '${agentBuilderDefaultAgentId}'`); +} + +export async function findUnattachedToolIds( + kbnClient: KbnClient, + toolIds: readonly string[] +): Promise { + const response = await kbnClient.request<{ + configuration?: { tools?: Array<{ tool_ids?: string[] }> }; + }>({ + method: 'GET', + path: `/api/agent_builder/agents/${agentBuilderDefaultAgentId}`, + headers: { + 'kbn-xsrf': 'persona-matrix-tool-check', + 'elastic-api-version': '2023-10-31', + }, + }); + const attached = new Set( + (response.data?.configuration?.tools ?? []).flatMap((entry) => entry.tool_ids ?? []) + ); + return toolIds.filter((id) => !attached.has(id)); +} diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/matrix_config.test.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/matrix_config.test.ts new file mode 100644 index 0000000000000..a139055d52346 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/matrix_config.test.ts @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import config from '../persona_matrix.config.json'; +import suitesMetadata from '../../../../../../.buildkite/pipelines/evals/evals.suites.json'; + +/** + * The shipped persona matrix config, not a fixture. + * + * The scoring policy decides whether a published cell can be reproduced, so a + * silent drop of this block during an edit has to fail the build rather than + * quietly restore the old scoring basis. + */ +describe('persona_matrix.config.json scoring policy', () => { + it('scores judged evaluators by verdict and admits only reproducible judges', () => { + expect(config.scoring).toEqual({ + useVerdictLadder: true, + requireEisJudge: true, + excludeSelfJudged: true, + }); + }); + + it('warns that these numbers are not comparable to older matrices', () => { + const notes: string[] = config.provenance?.methodologyNotes ?? []; + + expect(notes.some((note) => note.includes('NOT comparable'))).toBe(true); + expect(notes.some((note) => note.includes('verdict'))).toBe(true); + }); +}); + +/** + * A column whose `suites` entry does not match a registered eval suite id still + * renders — it just comes back permanently blank, because the matrix queries + * `listExperiments({ suiteId })` and silently gets nothing. That failure is + * invisible in a generated matrix (an empty cell looks like "not run yet"), so + * the wiring is asserted here instead. + */ +describe('persona_matrix.config.json column wiring', () => { + const registeredSuiteIds: string[] = suitesMetadata.suites.map((suite) => suite.id); + + it('references only suite ids that are registered in evals.suites.json', () => { + const referenced = [...new Set(config.columns.flatMap((column) => column.suites))]; + const unknown = referenced.filter((suiteId) => !registeredSuiteIds.includes(suiteId)); + + expect(unknown).toEqual([]); + }); + + it('covers attack discovery and automatic migrations alongside the persona prompts', () => { + const suitesByColumn = new Map(config.columns.map((column) => [column.id, column.suites])); + + // The live suite is attack-discovery-agent-builder: the legacy attack-discovery + // id still has docs but only ~2 per evaluator, which published a near-zero + // score for models that actually pass it 1.0 on the real suite. + expect(suitesByColumn.get('attack-discovery')).toEqual(['attack-discovery-agent-builder']); + expect(suitesByColumn.get('migrations-rules')).toEqual(['security-automatic-migrations']); + expect(suitesByColumn.get('migrations-dashboards')).toEqual(['security-automatic-migrations']); + }); + + it('gives every column a unique id so cells cannot overwrite each other', () => { + const ids = config.columns.map((column) => column.id); + + expect(ids).toHaveLength(new Set(ids).size); + }); +}); + +/** + * The extra suites do not run on the same branch as the persona prompts: attack + * discovery's weekly job lives on its feature branch and automatic migrations on + * the weekly-evals-matrix branch. Reading them from the global `branch` (main) + * returns nothing, which renders as an empty column rather than an error — the + * exact failure that published a matrix with 0/21 translation cells while ~2,500 + * scored documents sat in the golden cluster. + */ +describe('persona_matrix.config.json extra-suite branch pins', () => { + const branchByColumn = new Map(config.columns.map((column) => [column.id, column.branch])); + + it('unions main with the branch attack discovery runs were first exported on', () => { + // Sweep runs export on main; the historical rows live on the feature + // branch. Pinning only the feature branch filters every new run out, so + // the column reads the union. + expect(branchByColumn.get('attack-discovery')).toStrictEqual([ + 'main', + 'patrykkopycinski:feat/attack-discovery-agent-builder-evals', + 'feat/evals-extensions-matrix-v3', + ]); + }); + + it('unions every branch that holds migrations runs for a distinct model', () => { + // Golden migrations data is split across branches by model: the weekly + // branch holds six models, 4.6-sonnet has a newer run on the endpoint + // branch, and 4.5-sonnet only ever ran here. Sweep runs add main. Pinning + // one branch blanks the others' cells, so the column reads the union. + const expected = [ + 'main', + 'elastic:fix/weekly-evals-matrix', + 'elastic:feat/siem-migrations-invoke-endpoint', + 'feat/evals-extensions-matrix-v3', + ]; + + expect(branchByColumn.get('migrations-rules')).toStrictEqual(expected); + expect(branchByColumn.get('migrations-dashboards')).toStrictEqual(expected); + }); + + it('keeps both migration columns on one branch set so the suite is queried once', () => { + // branchBySuiteFromColumns throws on disagreement; asserting equality here + // names the constraint at the config instead of at a generator stack trace. + expect(branchByColumn.get('migrations-rules')).toStrictEqual( + branchByColumn.get('migrations-dashboards') + ); + }); + + it('keeps a lookback window wide enough to reach those runs', () => { + // The pinned branches last exported 2026-07-30 and 2026-08-11; a 14-day + // window silently drops them even with the branch pinned correctly. + expect(config.lookbackDays).toBeGreaterThanOrEqual(45); + }); +}); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/replay_plan.golden.test.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/replay_plan.golden.test.ts new file mode 100644 index 0000000000000..5d4217696fa7c --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/replay_plan.golden.test.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PERSONA_MATRIX_EXAMPLES } from './datasets/persona_matrix_prompts'; +import { GOLDEN_SCORES_SAMPLE } from './__fixtures__/golden_scores_sample'; +import { planReplay, summarizePlan } from '@kbn/evals-extensions'; + +/** + * Contract test against REAL golden documents (482 score docs, 40 cells, + * captured 2026-09-06 from the persona-matrix re-judge waves) joined against + * the REAL suite dataset. + * + * Synthetic fixtures cannot catch a schema drift between what the sweep writes + * and what a replay reads. The first version of this planner passed 21 + * synthetic tests and still produced ZERO replayable cells from production + * data, because it expected the reference answer inside the score document + * (golden never stores it). This test is what caught that. + */ +describe('planReplay against real golden documents', () => { + const docs = GOLDEN_SCORES_SAMPLE; + + const referenceFor = (exampleId: string) => + PERSONA_MATRIX_EXAMPLES.find((p) => p.id === exampleId)?.output.reference; + + it('replays real production cells', () => { + const plan = planReplay(docs, referenceFor); + // If the golden schema or the dataset ids drift, this drops to zero and a + // "fast" re-judge would silently grade nothing while reporting success. + expect(plan.cells.length).toBeGreaterThan(0); + // One cell per (execution, example) — far fewer than the raw document count, + // which carries one document per evaluator. + expect(plan.cells.length).toBeLessThan(docs.length); + }); + + it('every replayable cell carries the three judge inputs', () => { + const plan = planReplay(docs, referenceFor); + for (const cell of plan.cells) { + expect(typeof cell.question).toBe('string'); + expect(cell.question.length).toBeGreaterThan(0); + expect(cell.agentResponse.length).toBeGreaterThan(0); + expect(cell.expected.length).toBeGreaterThan(0); + expect(cell.modelId).toBeTruthy(); + } + }); + + it('joins every real example id to a dataset reference', () => { + // A miss here means the dataset moved under the stored runs: the replay + // would skip real work rather than grade it. + const plan = planReplay(docs, referenceFor); + const unjoined = plan.skipped.filter((s) => s.reason.includes('dataset reference')); + expect(unjoined).toHaveLength(0); + }); + + it('covers multiple models and executions', () => { + const plan = planReplay(docs, referenceFor); + expect(new Set(plan.cells.map((c) => c.modelId)).size).toBeGreaterThan(1); + expect(summarizePlan(plan)).toMatch(/\d+ cell\(s\) across \d+ model\(s\)/); + }); +}); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/tsconfig.json b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/tsconfig.json index fc4975cd0da29..417e3f719020b 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/tsconfig.json +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/tsconfig.json @@ -4,16 +4,18 @@ "outDir": "target/types", "types": ["jest", "node"] }, - "include": ["**/*.ts"], + "include": ["**/*.ts", "persona_matrix.config.json"], "exclude": ["target/**/*"], "kbn_references": [ "@kbn/agent-builder-common", "@kbn/core", "@kbn/evals", + "@kbn/evals-extensions", "@kbn/evals-suite-attack-discovery", "@kbn/scout", "@kbn/security-evals-alerts-snapshot", "@kbn/tooling-log", - "@kbn/kbn-client" + "@kbn/kbn-client", + "@kbn/evals-common" ] } diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/detection_rule_edit/index.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/detection_rule_edit/index.ts index 4674d3cfc69b0..9716bef9d8f1a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/detection_rule_edit/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/detection_rule_edit/index.ts @@ -196,6 +196,7 @@ Checklist before finishing the answer: - [ ] Did I run \`security.run_rule_preview\` after creating or modifying the rule query or schedule?` : '' } +- [ ] Did I end with a user-facing summary of what was created or changed — what the rule detects, its severity, and MITRE mapping? NEVER finish the turn on a tool call: after the last tool result, always write the closing message for the user. --- diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/threat_hunting/threat_hunting_skill.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/threat_hunting/threat_hunting_skill.ts index bbfea30b760dc..edf6d91df5438 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/threat_hunting/threat_hunting_skill.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/threat_hunting/threat_hunting_skill.ts @@ -42,7 +42,9 @@ Use this skill when: - Prefer ECS field names for cross-source portability ### 3. Explore Data Iteratively -- Start with broad queries to establish baselines using 'platform.core.generate_esql' and 'platform.core.execute_esql' +- Build every query with 'platform.core.generate_esql', then run it with 'platform.core.execute_esql'. This is required, not a shortcut to skip: 'generate_esql' validates syntax and resolves field names against the live mapping, so a hand-written query that looks correct can silently reference a field this deployment does not have. Do not hand-write ES|QL and pass it straight to 'platform.core.execute_esql'. +- This applies to the query templates below too. They are starting points for a hypothesis, not runnable answers — pass the pattern you want through 'platform.core.generate_esql' to bind it to the actual indices and fields in this environment before executing. +- Never report a finding from a query you did not execute. An unexecuted query is a hypothesis, not evidence. - Always scope queries with @timestamp ranges: WHERE @timestamp >= NOW() - 7 DAYS - Use STATS ... BY for aggregated views before drilling into raw events - Chain WHERE clauses to iteratively narrow down — avoid overly complex single queries @@ -68,7 +70,7 @@ Use this skill when: ## Query Templates -The following embedded query templates provide common hunting patterns (available as referenced content): +The following embedded query templates provide common hunting patterns (available as referenced content). They are hypothesis starting points, not runnable answers — rebuild the pattern through 'platform.core.generate_esql' so it binds to this deployment's indices and fields, then execute it: - lateral-movement: Detect lateral movement via remote service creation and suspicious logon types - c2-beaconing: Identify C2 beaconing through periodic network connection analysis - brute-force: Detect brute force and credential spraying attempts @@ -76,6 +78,7 @@ The following embedded query templates provide common hunting patterns (availabl ## Best Practices - Always start with a time-bounded hypothesis — do not explore without direction +- Generate queries with 'platform.core.generate_esql' and execute them with 'platform.core.execute_esql'; report findings only from executed results - Use STATS and aggregations before raw event queries to understand data volume - Validate findings against known-good baselines before escalating - Hunt on 7-30 day windows for behavioral patterns; use shorter windows for IOC sweeps