From 1c70fbaa75bfc673f95102f2158e48e509601aa6 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Tue, 18 Aug 2026 22:35:59 +0200 Subject: [PATCH 001/258] [Evals] Security LLM performance matrix generator in @kbn/evals-extensions Clean rebuild of PR #283586 against upstream/main, avoiding the stale globby v16 migration that upstream itself reverted. Includes the verified fix for EvalsClient that restores assertSpacesExist/deleteDataset/spaceIds while keeping the additive listExperiments API needed by the matrix commands. Files: - Matrix generation: build_matrix, query_matrix_scores, query_matrix_traces - HTML rendering: render_matrix_html, trace_types - CLI: matrix.ts command - Golden cluster privileges: create_doc -> create for serverless - EvalsClient: restore upstream API + listExperiments Verified: type_check 0 errors, matrix Jest 64/64, ESLint clean. --- .buildkite/pipelines/evals/llm_evals.yml | 49 ++ .../golden_cluster_privileges.ts | 18 +- .../shared/kbn-evals-extensions/moon.yml | 3 + .../src/cli/commands/matrix.test.ts | 58 +++ .../src/cli/commands/matrix.ts | 226 +++++++++ .../kbn-evals-extensions/src/cli/index.ts | 3 +- .../src/matrix/build_matrix.test.ts | 427 ++++++++++++++++++ .../src/matrix/build_matrix.ts | 420 +++++++++++++++++ .../src/matrix/load_matrix_config.test.ts | 149 ++++++ .../src/matrix/load_matrix_config.ts | 308 +++++++++++++ .../src/matrix/query_matrix_scores.test.ts | 229 ++++++++++ .../src/matrix/query_matrix_scores.ts | 189 ++++++++ .../src/matrix/query_matrix_traces.ts | 228 ++++++++++ .../src/matrix/render_matrix.test.ts | 299 ++++++++++++ .../src/matrix/render_matrix.ts | 168 +++++++ .../src/matrix/render_matrix_html.test.ts | 143 ++++++ .../src/matrix/render_matrix_html.ts | 351 ++++++++++++++ .../src/matrix/trace_types.ts | 44 ++ .../shared/kbn-evals-extensions/tsconfig.json | 5 +- .../packages/shared/kbn-evals/index.ts | 9 + .../kbn-evals/src/utils/evals_client.ts | 37 ++ .../src/utils/evaluations_kbn_client.ts | 36 ++ 22 files changed, 3382 insertions(+), 17 deletions(-) create mode 100644 x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.test.ts create mode 100644 x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.ts create mode 100644 x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.test.ts create mode 100644 x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.ts create mode 100644 x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/load_matrix_config.test.ts create mode 100644 x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/load_matrix_config.ts create mode 100644 x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.test.ts create mode 100644 x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.ts create mode 100644 x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.ts create mode 100644 x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix.test.ts create mode 100644 x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix.ts create mode 100644 x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.test.ts create mode 100644 x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.ts create mode 100644 x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/trace_types.ts diff --git a/.buildkite/pipelines/evals/llm_evals.yml b/.buildkite/pipelines/evals/llm_evals.yml index 6616c4d1e5872..51a5b51fadb52 100644 --- a/.buildkite/pipelines/evals/llm_evals.yml +++ b/.buildkite/pipelines/evals/llm_evals.yml @@ -590,6 +590,55 @@ steps: automatic: - exit_status: '-1' limit: 3 + + - label: "Evals: Security Persona Matrix" + key: kbn-evals-weekly-security-persona-matrix + command: bash .buildkite/scripts/steps/evals/run_suite.sh + env: + KBN_EVALS: '1' + FTR_EIS_CCM: '1' + EVAL_SUITE_ID: 'security-persona-matrix' + EVAL_FANOUT: '1' + EVAL_INCLUDE_EIS_MODELS: '1' + EVAL_MODEL_GROUPS: *weekly_eis_core_models + EVAL_SERVER_CONFIG_SET: 'evals_security_persona_matrix' + timeout_in_minutes: 90 + agents: + image: family/kibana-ubuntu-2404 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-8 + diskSizeGb: 130 + preemptible: true + retry: + automatic: + - exit_status: '-1' + limit: 3 + + - label: "Evals: Security Persona Matrix — Attack Discovery" + key: kbn-evals-weekly-security-persona-matrix-attack-discovery + command: bash .buildkite/scripts/steps/evals/run_suite.sh + env: + KBN_EVALS: '1' + FTR_EIS_CCM: '1' + EVAL_SUITE_ID: 'security-persona-matrix-attack-discovery' + EVAL_FANOUT: '1' + EVAL_INCLUDE_EIS_MODELS: '1' + EVAL_MODEL_GROUPS: *weekly_eis_core_models + EVAL_SERVER_CONFIG_SET: 'evals_tracing' + timeout_in_minutes: 90 + agents: + image: family/kibana-ubuntu-2404 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-8 + diskSizeGb: 130 + preemptible: true + retry: + automatic: + - exit_status: '-1' + limit: 3 + - wait: ~ continue_on_failure: true diff --git a/x-pack/platform/packages/shared/kbn-evals-common/golden_cluster_privileges.ts b/x-pack/platform/packages/shared/kbn-evals-common/golden_cluster_privileges.ts index 0cde60dbf14a9..51cfd89657f50 100644 --- a/x-pack/platform/packages/shared/kbn-evals-common/golden_cluster_privileges.ts +++ b/x-pack/platform/packages/shared/kbn-evals-common/golden_cluster_privileges.ts @@ -22,30 +22,18 @@ export const goldenClusterPrivileges = { indices: [ { names: [`${EvaluationIndices.SCORES}*`], - privileges: [ - 'auto_configure', - 'create_index', - 'create_doc', - 'read', - 'view_index_metadata', - ], + privileges: ['auto_configure', 'create_index', 'create', 'read', 'view_index_metadata'], }, { names: ['traces-*'], - privileges: [ - 'auto_configure', - 'create_index', - 'create_doc', - 'read', - 'view_index_metadata', - ], + privileges: ['auto_configure', 'create_index', 'create', 'read', 'view_index_metadata'], }, { names: [`${EvaluationIndices.DATASETS}*`, `${EvaluationIndices.DATASET_EXAMPLES}*`], privileges: [ 'auto_configure', 'create_index', - 'create_doc', + 'create', 'read', 'view_index_metadata', 'delete', diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/moon.yml b/x-pack/platform/packages/shared/kbn-evals-extensions/moon.yml index 219130985794a..46320f9c635ab 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/moon.yml +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/moon.yml @@ -22,6 +22,9 @@ dependsOn: - '@kbn/dev-cli-runner' - '@kbn/dev-cli-errors' - '@kbn/tooling-log' + - '@kbn/config-schema' + - '@kbn/some-dev-log' + - '@kbn/kbn-client' tags: - test-helper - package diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.test.ts new file mode 100644 index 0000000000000..25d87a005a9e8 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.test.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { buildMatrix } from '../../matrix/build_matrix'; +import { renderMatrix } from '../../matrix/render_matrix'; +import { parseMatrixConfig } from '../../matrix/load_matrix_config'; +import type { AggregatedModelScores } from '../../matrix/query_matrix_scores'; + +describe('matrix command empty-result guard', () => { + const config = parseMatrixConfig({ + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'], weight: 1 }], + models: [{ id: 'model-a', label: 'Model A' }], + }); + + it('renders header-only CSVs when no experiments match', () => { + const rendered = renderMatrix(buildMatrix([], config), config); + + expect(rendered.proprietaryCsv.trim().split('\n')).toHaveLength(1); + expect(rendered.openSourceCsv.trim().split('\n')).toHaveLength(1); + }); + + it('renders populated CSVs when experiments do match', () => { + const aggregated: AggregatedModelScores[] = [ + { + modelId: 'model-a', + provider: 'anthropic', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'experiment-a', + datasets: [ + { + datasetId: 'dataset-a-id', + datasetName: 'dataset-a', + evaluators: [{ evaluatorName: 'correctness', mean: 0.9, count: 10 }], + }, + ], + }, + ], + }, + ]; + + const rendered = renderMatrix(buildMatrix(aggregated, config), config); + + expect(rendered.proprietaryCsv.trim().split('\n').length).toBeGreaterThan(1); + }); + + it('produces no model rows when no experiments match', () => { + const matrix = buildMatrix([], config); + + expect(matrix.proprietary).toHaveLength(0); + expect(matrix.openSource).toHaveLength(0); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.ts new file mode 100644 index 0000000000000..462a2a0d4f682 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.ts @@ -0,0 +1,226 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Fs from 'fs'; +import Path from 'path'; +import { createFailError, createFlagError } from '@kbn/dev-cli-errors'; +import type { Command } from '@kbn/dev-cli-runner'; +import { + EvalsClient, + getEvaluationsKbnClient, + envFromDatasetsProfile, + DEFAULT_EVALUATIONS_KBN_URL, +} from '@kbn/evals'; +import { KbnClient } from '@kbn/kbn-client'; +import { loadMatrixConfig, applyModelOverrides } from '../../matrix/load_matrix_config'; +import type { MatrixConfig } from '../../matrix/load_matrix_config'; +import { queryMatrixScores } from '../../matrix/query_matrix_scores'; +import { buildMatrix } from '../../matrix/build_matrix'; +import { renderMatrix } from '../../matrix/render_matrix'; +import { renderMatrixHtml } from '../../matrix/render_matrix_html'; +import { queryMatrixTraces } from '../../matrix/query_matrix_traces'; +import type { MatrixTraceData } from '../../matrix/trace_types'; + +const DEFAULT_OUT_DIR = 'target/llm_matrix'; + +export const matrixCmd: Command = { + name: 'matrix', + description: ` + Generate an LLM performance matrix artifact from exported evaluation results. + + Reads the latest experiment per (model, suite) from the evals plugin on the + target Kibana, maps suites/datasets/evaluators onto matrix columns via a config + file, normalizes scores onto a 0-10 scale, and writes markdown + CSV + JSON. + + Configure target/auth with EVAL_KBN_URL and EVAL_KBN_API_KEY, + with --kbn-url/--kbn-api-key, or with --profile (e.g. dev-vault for the golden + cluster, or a config..json file). + + Example: + node scripts/evals ext matrix \\ + --config .buildkite/pipelines/evals/security_matrix.config.json \\ + --profile dev-vault --branch main --out target/llm_matrix + `, + flags: { + string: [ + 'config', + 'out', + 'branch', + 'lookback-days', + 'profile', + 'kbn-url', + 'kbn-api-key', + 'model', + ], + boolean: ['html'], + allowUnexpected: false, + help: ` + --config Path to the matrix config JSON (required). + --out Output directory for artifacts (default: ${DEFAULT_OUT_DIR}). + --branch Git branch filter override (default: config.branch). + --lookback-days Only consider experiments newer than now-d (default: config.lookbackDays). + --model Replace the config's model set for an on-demand run. + Format: id[:label][:open-source]. Repeatable. + e.g. --model gpt-5-preview:GPT-5 --model qwen3:Qwen3:open-source + --profile Golden-cluster config profile providing EVAL_KBN_URL/API_KEY + (e.g. 'dev-vault' for runtime Vault, or a config..json file). + --kbn-url Kibana URL override. + --kbn-api-key Kibana API key override. + --html Also generate a self-contained HTML report (matrix.html). + `, + }, + run: async ({ log, flagsReader }) => { + const configPath = flagsReader.string('config'); + if (!configPath) { + throw createFlagError('--config is required. Provide the path to a matrix config JSON.'); + } + + const repoRoot = process.cwd(); + const baseConfig = loadMatrixConfig(Path.resolve(repoRoot, configPath)); + + const modelOverrides = flagsReader.arrayOfStrings('model') ?? []; + let config: MatrixConfig; + try { + config = applyModelOverrides(baseConfig, modelOverrides); + } catch (error) { + throw createFlagError(error instanceof Error ? error.message : String(error)); + } + if (modelOverrides.length > 0) { + log.info( + `Overriding config model set with ${ + config.models.length + } on-demand model(s): ${config.models.map((model) => model.id).join(', ')}` + ); + } + + const profile = flagsReader.string('profile') ?? undefined; + const profileEnv = envFromDatasetsProfile(repoRoot, profile); + + const evaluationsKbnUrl = + flagsReader.string('kbn-url') ?? profileEnv.EVAL_KBN_URL ?? process.env.EVAL_KBN_URL; + if (!evaluationsKbnUrl) { + log.warning(`EVAL_KBN_URL not set; defaulting to ${DEFAULT_EVALUATIONS_KBN_URL}.`); + } + + const evaluationsKbnApiKey = + flagsReader.string('kbn-api-key') ?? + profileEnv.EVAL_KBN_API_KEY ?? + process.env.EVAL_KBN_API_KEY; + + const branch = flagsReader.string('branch') ?? config.branch; + const lookbackDaysFlag = flagsReader.string('lookback-days'); + const lookbackDays = lookbackDaysFlag ? Number(lookbackDaysFlag) : config.lookbackDays; + if (Number.isNaN(lookbackDays) || lookbackDays < 1) { + throw createFlagError('--lookback-days must be a positive number.'); + } + + const outDir = Path.resolve(repoRoot, flagsReader.string('out') ?? DEFAULT_OUT_DIR); + const suiteIds = [...new Set(config.columns.flatMap((column) => column.suites))]; + // Query per (suite, model) pair: the experiments route answers from a + // terms aggregation that grows with the page number, so the listing must + // stay bounded per pair. Include matchIds so aliased model rows are found. + const modelIds = [ + ...new Set(config.models.flatMap((model) => [model.id, ...(model.matchIds ?? [])])), + ]; + + const defaultKbnClient = new KbnClient({ log, url: DEFAULT_EVALUATIONS_KBN_URL }); + const kbnClient = getEvaluationsKbnClient({ + kbnClient: defaultKbnClient, + log, + evaluationsKbnUrl, + evaluationsKbnApiKey, + }); + const evalsClient = new EvalsClient(kbnClient, log); + + try { + await evalsClient.assertPluginEnabled(); + } catch (error) { + throw createFlagError( + [ + error instanceof Error ? error.message : String(error), + 'Set EVAL_KBN_URL to a Kibana instance with xpack.evals.enabled=true.', + 'Set EVAL_KBN_API_KEY when authenticating to a non-local target.', + ].join('\n') + ); + } + + log.info( + `Querying matrix scores from ${evaluationsKbnUrl ?? DEFAULT_EVALUATIONS_KBN_URL} (branch: ${ + branch ?? 'any' + })` + ); + + const aggregated = await queryMatrixScores(evalsClient, log, { + suiteIds, + modelIds, + branch, + lookbackDays, + }); + + if (aggregated.length === 0) { + // Empty CSVs would publish as a blank matrix in customer-facing docs. + throw createFailError( + [ + 'No experiments matched the configured filters, refusing to write an empty matrix.', + `Filters: suites=[${suiteIds.join(', ')}] models=[${modelIds.join(', ')}] branch=${ + branch ?? 'any' + } lookbackDays=${lookbackDays}`, + 'Check that the weekly eval run published results for these suites in the lookback window.', + ].join('\n') + ); + } + + const matrix = buildMatrix(aggregated, config); + const rendered = renderMatrix(matrix, config, { + branch, + lookbackDays, + suiteIds, + commitSha: process.env.BUILDKITE_COMMIT, + buildUrl: process.env.BUILDKITE_BUILD_URL, + }); + + Fs.mkdirSync(outDir, { recursive: true }); + const writes: Array<[string, string]> = [ + ['proprietary-models.csv', rendered.proprietaryCsv], + ['open-source-models.csv', rendered.openSourceCsv], + ['matrix.md', rendered.markdown], + ['matrix.json', rendered.json], + // Raw, pre-scaling per-evaluator means/counts, so reviewers can audit which + // evaluators feed a cell without re-querying. + ['scores.debug.json', `${JSON.stringify(aggregated, null, 2)}\n`], + ]; + for (const [fileName, contents] of writes) { + Fs.writeFileSync(Path.join(outDir, fileName), contents); + } + + const generateHtml = flagsReader.boolean('html'); + if (generateHtml) { + log.info('Querying trace data for HTML report...'); + const traces: MatrixTraceData = await queryMatrixTraces(evalsClient, log, aggregated); + const htmlContent = renderMatrixHtml( + matrix, + config, + { + branch, + lookbackDays, + suiteIds, + commitSha: process.env.BUILDKITE_COMMIT, + buildUrl: process.env.BUILDKITE_BUILD_URL, + }, + traces + ); + Fs.writeFileSync(Path.join(outDir, 'matrix.html'), htmlContent); + log.info(`Wrote matrix.html to ${outDir}`); + } + + log.info( + `Wrote matrix artifacts to ${outDir} ` + + `(${matrix.proprietary.length} proprietary, ${matrix.openSource.length} open-source models)` + ); + log.info(`\n${rendered.markdown}`); + }, +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/index.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/index.ts index 2f45e1481ac68..e8ea9f64007a7 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/index.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/index.ts @@ -6,6 +6,7 @@ */ import { RunWithCommands } from '@kbn/dev-cli-runner'; +import { matrixCmd } from './commands/matrix'; import { redTeamCmd } from './commands/red_team'; export async function run() { @@ -14,6 +15,6 @@ export async function run() { usage: 'node scripts/evals ext', description: 'Evals extensions CLI (experimental)', }, - [redTeamCmd] + [redTeamCmd, matrixCmd] ).execute(); } diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.test.ts new file mode 100644 index 0000000000000..d89ea1e4941a6 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.test.ts @@ -0,0 +1,427 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { buildMatrix } from './build_matrix'; +import { parseMatrixConfig, type MatrixConfig } from './load_matrix_config'; +import type { AggregatedModelScores } from './query_matrix_scores'; + +const config: MatrixConfig = parseMatrixConfig({ + columns: [ + { id: 'triage', label: 'Triage', suites: ['suite-a'], weight: 1 }, + { id: 'detect', label: 'Detect', suites: ['suite-b'], weight: 2 }, + ], + models: [ + { id: 'model-good', label: 'Good Model' }, + { id: 'model-oss', label: 'OSS Model', openSource: true }, + { id: 'model-missing', label: 'Absent Model' }, + ], +}); + +const evaluator = (mean: number, count = 10) => ({ evaluatorName: 'correctness', mean, count }); + +const aggregated: AggregatedModelScores[] = [ + { + modelId: 'model-good', + provider: 'anthropic', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'run-1', + datasets: [{ datasetId: 'd1', datasetName: 'D1', evaluators: [evaluator(0.9)] }], + }, + { + suiteId: 'suite-b', + experimentId: 'run-2', + datasets: [{ datasetId: 'd2', datasetName: 'D2', evaluators: [evaluator(0.8)] }], + }, + ], + }, + { + modelId: 'model-oss', + provider: 'meta', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'run-3', + datasets: [{ datasetId: 'd1', datasetName: 'D1', evaluators: [evaluator(0.5)] }], + }, + // No suite-b data -> "detect" column missing for this model. + ], + }, +]; + +describe('buildMatrix', () => { + it('scales evaluator means onto a 0-10 scale and splits proprietary/open-source', () => { + const matrix = buildMatrix(aggregated, config); + + expect(matrix.proprietary).toHaveLength(1); + expect(matrix.openSource).toHaveLength(1); + + const good = matrix.proprietary[0]; + expect(good.modelLabel).toBe('Good Model'); + expect(good.cells.triage).toEqual({ kind: 'score', value: 9 }); + expect(good.cells.detect).toEqual({ kind: 'score', value: 8 }); + // Weighted overall: (9*1 + 8*2) / 3 = 8.33 + expect(good.overall).toEqual({ kind: 'score', value: 8.33 }); + }); + + it('marks columns with no data as missing and counts them as 0 in the overall', () => { + const matrix = buildMatrix(aggregated, config); + const oss = matrix.openSource[0]; + + expect(oss.cells.triage).toEqual({ kind: 'score', value: 5 }); + expect(oss.cells.detect).toEqual({ kind: 'missing' }); + // detect is missing (excluded entirely), so overall = triage only = 5. + expect(oss.overall).toEqual({ kind: 'score', value: 5 }); + }); + + it('skips models absent from the aggregated data', () => { + const matrix = buildMatrix(aggregated, config); + const labels = [...matrix.proprietary, ...matrix.openSource].map((row) => row.modelLabel); + expect(labels).not.toContain('Absent Model'); + }); + + it('renders "not recommended" when a scaled score is at/under the threshold', () => { + const zeroConfig = parseMatrixConfig({ + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'] }], + models: [{ id: 'm', label: 'M' }], + }); + const matrix = buildMatrix( + [ + { + modelId: 'm', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'r', + datasets: [{ datasetId: 'd', datasetName: 'D', evaluators: [evaluator(0)] }], + }, + ], + }, + ], + zeroConfig + ); + + expect(matrix.proprietary[0].cells.triage).toEqual({ kind: 'not-recommended' }); + }); + + it('excludes observability-tier evaluators (latency/tokens/tool calls) by default', () => { + const matrix = buildMatrix( + [ + { + modelId: 'm', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'r', + datasets: [ + { + datasetId: 'd', + datasetName: 'D', + evaluators: [ + { evaluatorName: 'Factuality', mean: 0.8, count: 10 }, + { evaluatorName: 'Latency', mean: 4200, count: 10 }, + { evaluatorName: 'Input Tokens', mean: 51234, count: 10 }, + { evaluatorName: 'Tool Calls', mean: 7, count: 10 }, + { evaluatorName: 'Skill Invoked (alert-analysis)', mean: 1, count: 10 }, + ], + }, + ], + }, + ], + }, + ], + parseMatrixConfig({ + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'] }], + models: [{ id: 'm', label: 'M' }], + }) + ); + + // Only Factuality (0.8) contributes -> 0.8 * 10 = 8, not blown out by tokens/latency. + expect(matrix.proprietary[0].cells.triage).toEqual({ kind: 'score', value: 8 }); + }); + + it('honors a column evaluator allowlist over the global exclusion list', () => { + const matrix = buildMatrix( + [ + { + modelId: 'm', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'r', + datasets: [ + { + datasetId: 'd', + datasetName: 'D', + evaluators: [ + { evaluatorName: 'Factuality', mean: 0.8, count: 10 }, + { evaluatorName: 'Latency', mean: 0.2, count: 10 }, + ], + }, + ], + }, + ], + }, + ], + parseMatrixConfig({ + // Explicit allowlist including 'Latency' opts it back in despite the default exclusion. + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'], evaluators: ['Latency'] }], + models: [{ id: 'm', label: 'M' }], + }) + ); + + expect(matrix.proprietary[0].cells.triage).toEqual({ kind: 'score', value: 2 }); + }); + + describe('composites', () => { + const compositeConfig: MatrixConfig = parseMatrixConfig({ + showOverall: false, + columns: [ + { id: 'c1', label: 'C1', group: 'Group', suites: ['s1'] }, + { id: 'c2', label: 'C2', group: 'Group', suites: ['s2'] }, + { id: 'feat', label: 'Feat', suites: ['s3'] }, + ], + composites: [ + { id: 'group_score', label: 'Group Score', from: ['c1', 'c2'] }, + { id: 'overall_score', label: 'Overall Score', from: ['group_score', 'feat'] }, + ], + layout: ['c1', 'c2', 'group_score', 'feat', 'overall_score'], + models: [ + { id: 'm1', label: 'M1' }, + { id: 'm2', label: 'M2' }, + ], + }); + + const suite = (suiteId: string, mean: number) => ({ + suiteId, + experimentId: `e-${suiteId}`, + datasets: [{ datasetId: 'd', datasetName: 'D', evaluators: [evaluator(mean)] }], + }); + + it('averages base cells into a composite and layers composites of composites', () => { + const matrix = buildMatrix( + [{ modelId: 'm1', suites: [suite('s1', 0.8), suite('s2', 0.6)] }], + compositeConfig + ); + const row = matrix.proprietary[0]; + + expect(row.cells.group_score).toEqual({ kind: 'score', value: 7 }); + // feat has no data -> missing, so overall = group_score only = 7. + expect(row.cells.feat).toEqual({ kind: 'missing' }); + expect(row.cells.overall_score).toEqual({ kind: 'score', value: 7 }); + }); + + it('counts "Not recommended" sources as 0 inside a composite', () => { + const matrix = buildMatrix( + [{ modelId: 'm1', suites: [suite('s1', 0), suite('s2', 0.6)] }], + compositeConfig + ); + const row = matrix.proprietary[0]; + + expect(row.cells.c1).toEqual({ kind: 'not-recommended' }); + // mean(0, 6) = 3. + expect(row.cells.group_score).toEqual({ kind: 'score', value: 3 }); + }); + + it('marks a composite missing when none of its sources have data', () => { + const matrix = buildMatrix([{ modelId: 'm1', suites: [suite('s3', 0.9)] }], compositeConfig); + const row = matrix.proprietary[0]; + + expect(row.cells.group_score).toEqual({ kind: 'missing' }); + // overall = feat only = 9. + expect(row.cells.overall_score).toEqual({ kind: 'score', value: 9 }); + }); + + it('builds display columns in layout order and suppresses the legacy overall', () => { + const matrix = buildMatrix( + [{ modelId: 'm1', suites: [suite('s1', 0.8), suite('s2', 0.6)] }], + compositeConfig + ); + + expect(matrix.displayColumns?.map((column) => column.id)).toEqual([ + 'c1', + 'c2', + 'group_score', + 'feat', + 'overall_score', + ]); + expect(matrix.displayColumns?.find((column) => column.id === 'group_score')?.kind).toBe( + 'composite' + ); + expect(matrix.displayColumns?.some((column) => column.kind === 'overall')).toBe(false); + expect(matrix.displayColumns?.find((column) => column.id === 'c1')?.group).toBe('Group'); + }); + + it('ranks rows by the final composite (Overall Score) descending', () => { + const matrix = buildMatrix( + [ + { modelId: 'm1', suites: [suite('s1', 0.3), suite('s2', 0.3)] }, + { modelId: 'm2', suites: [suite('s1', 0.9), suite('s2', 0.9)] }, + ], + compositeConfig + ); + + expect(matrix.proprietary.map((row) => row.modelLabel)).toEqual(['M2', 'M1']); + }); + + it('throws when the layout references an unknown id', () => { + const badConfig = parseMatrixConfig({ + columns: [{ id: 'c1', label: 'C1', suites: ['s1'] }], + layout: ['c1', 'nope'], + models: [{ id: 'm1', label: 'M1' }], + }); + expect(() => buildMatrix([{ modelId: 'm1', suites: [suite('s1', 0.5)] }], badConfig)).toThrow( + /unknown column\/composite id/ + ); + }); + }); + + it('sorts rows by overall score descending', () => { + const matrix = buildMatrix( + [ + { + modelId: 'model-good', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'r1', + datasets: [{ datasetId: 'd', datasetName: 'D', evaluators: [evaluator(0.3)] }], + }, + ], + }, + { + modelId: 'model-missing', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'r2', + datasets: [{ datasetId: 'd', datasetName: 'D', evaluators: [evaluator(0.9)] }], + }, + ], + }, + ], + parseMatrixConfig({ + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'] }], + models: [ + { id: 'model-good', label: 'Lower' }, + { id: 'model-missing', label: 'Higher' }, + ], + }) + ); + + expect(matrix.proprietary.map((row) => row.modelLabel)).toEqual(['Higher', 'Lower']); + }); +}); + +describe('buildMatrix token axis', () => { + const tokenEvaluator = (name: string, mean: number, min: number, max: number, count = 3) => ({ + evaluatorName: name, + mean, + count, + min, + max, + }); + + const tokenAggregated: AggregatedModelScores[] = [ + { + modelId: 'model-good', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'run-1', + datasets: [ + { + datasetId: 'd1', + datasetName: 'D1', + evaluators: [ + evaluator(0.9), + tokenEvaluator('Input Tokens', 100_000, 50_000, 150_000), + tokenEvaluator('Output Tokens', 2_000, 1_000, 3_000), + ], + }, + ], + }, + ], + }, + ]; + + const tokenConfig: MatrixConfig = parseMatrixConfig({ + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'], weight: 1 }], + models: [{ id: 'model-good', label: 'Good Model' }], + tokenCost: {}, + }); + + it('is omitted entirely when the config does not opt in', () => { + expect(buildMatrix(tokenAggregated, config).tokenCost).toBeUndefined(); + }); + + it('aggregates token evaluators in native units with min/max preserved', () => { + const matrix = buildMatrix(tokenAggregated, tokenConfig); + const cell = matrix.tokenCost!.models[0].cells[0]; + + expect(cell.columnId).toBe('triage'); + expect(cell.inputTokens).toEqual({ mean: 100_000, min: 50_000, max: 150_000, count: 3 }); + expect(cell.outputTokens).toEqual({ mean: 2_000, min: 1_000, max: 3_000, count: 3 }); + expect(cell.totalMean).toBe(102_000); + }); + + it('does not let token evaluators leak into quality cells', () => { + const matrix = buildMatrix(tokenAggregated, tokenConfig); + // 0.9 * defaultScale(10) — unaffected by the 100k-magnitude token evaluators. + expect(matrix.proprietary[0].cells.triage).toEqual({ kind: 'score', value: 9 }); + }); + + it('weights the mean by sample count across suites', () => { + const twoSuite: MatrixConfig = parseMatrixConfig({ + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a', 'suite-b'], weight: 1 }], + models: [{ id: 'model-good', label: 'Good Model' }], + tokenCost: {}, + }); + const matrix = buildMatrix( + [ + { + modelId: 'model-good', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'r1', + datasets: [ + { + datasetId: 'd1', + datasetName: 'D1', + evaluators: [tokenEvaluator('Input Tokens', 100, 100, 100, 1)], + }, + ], + }, + { + suiteId: 'suite-b', + experimentId: 'r2', + datasets: [ + { + datasetId: 'd2', + datasetName: 'D2', + evaluators: [tokenEvaluator('Input Tokens', 200, 200, 200, 3)], + }, + ], + }, + ], + }, + ], + twoSuite + ); + // (100*1 + 200*3) / 4 = 175, not the unweighted 150. + expect(matrix.tokenCost!.models[0].cells[0].inputTokens!.mean).toBe(175); + expect(matrix.tokenCost!.models[0].cells[0].inputTokens!.min).toBe(100); + expect(matrix.tokenCost!.models[0].cells[0].inputTokens!.max).toBe(200); + }); + + it('omits cells with no token data', () => { + const matrix = buildMatrix(aggregated, tokenConfig); + expect(matrix.tokenCost!.models).toEqual([]); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.ts new file mode 100644 index 0000000000000..45f961ccd7251 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.ts @@ -0,0 +1,420 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + MatrixColumnConfig, + MatrixCompositeConfig, + MatrixConfig, + MatrixModelConfig, + MatrixTokenCostConfig, +} from './load_matrix_config'; +import type { AggregatedEvaluatorScore, AggregatedModelScores } from './query_matrix_scores'; + +/** A single matrix cell: either a numeric 0-10 score or "Not recommended". */ +export type MatrixCell = + | { kind: 'score'; value: number } + | { kind: 'not-recommended' } + | { kind: 'missing' }; + +/** Synthetic id for the legacy single "Overall" column. */ +export const OVERALL_COLUMN_ID = '__overall__'; + +/** A column as rendered, left-to-right, including derived composite columns. */ +export interface MatrixDisplayColumn { + id: string; + label: string; + group?: string; + kind: 'base' | 'composite' | 'overall'; +} + +export interface MatrixRow { + modelId: string; + modelLabel: string; + openSource: boolean; + /** Column/composite id -> cell. */ + cells: Record; + overall: MatrixCell; +} + +/** Aggregated token magnitudes for one (model, column) pair, in native units. */ +export interface TokenCostCell { + /** Base column id (matches `MatrixDisplayColumn.id`). */ + columnId: string; + inputTokens?: TokenStat; + outputTokens?: TokenStat; + /** Sum of the input + output means. */ + totalMean: number; +} + +export interface TokenStat { + mean: number; + min: number; + max: number; + count: number; +} + +export interface TokenCostModel { + modelId: string; + modelLabel: string; + openSource: boolean; + cells: TokenCostCell[]; +} + +export interface Matrix { + columns: Array<{ id: string; label: string; group?: string }>; + composites: Array<{ id: string; label: string; group?: string }>; + /** Full ordered render list (base + composite + legacy overall). */ + displayColumns: MatrixDisplayColumn[]; + overallLabel: string; + proprietary: MatrixRow[]; + openSource: MatrixRow[]; + /** Present only when the config opts into the token axis. */ + tokenCost?: { models: TokenCostModel[] }; +} + +const roundTo = (value: number, decimals: number): number => { + const factor = 10 ** decimals; + return Math.round(value * factor) / factor; +}; + +const matchesModel = (modelConfig: MatrixModelConfig, modelId: string): boolean => + modelConfig.id === modelId || (modelConfig.matchIds?.includes(modelId) ?? false); + +const isExcludedEvaluator = (evaluatorName: string, excluded: readonly string[]): boolean => + excluded.some((entry) => evaluatorName.startsWith(entry)); + +const toCell = (value: number, config: MatrixConfig): MatrixCell => + value <= config.notRecommendedBelow ? { kind: 'not-recommended' } : { kind: 'score', value }; + +/** Sample count doubles as the aggregation weight; zero-count evaluators still count once. */ +const weightOf = (evaluator: AggregatedEvaluatorScore): number => + evaluator.count > 0 ? evaluator.count : 1; + +/** Yields every evaluator contributing to a column, applying the suite/dataset filters. */ +function* columnEvaluators( + modelScores: AggregatedModelScores, + column: MatrixColumnConfig +): Generator { + const suiteSet = new Set(column.suites); + const datasetSet = column.datasetIds ? new Set(column.datasetIds) : undefined; + + for (const suite of modelScores.suites) { + if (!suiteSet.has(suite.suiteId)) { + continue; + } + for (const dataset of suite.datasets) { + if (!datasetSet || datasetSet.has(dataset.datasetId)) { + yield* dataset.evaluators; + } + } + } +} + +/** + * Weighted mean (by sample count) of the evaluator scores mapped to a column. + * Returns `undefined` when no scores contribute. + */ +const computeColumnMean = ( + modelScores: AggregatedModelScores, + column: MatrixColumnConfig, + excludeEvaluators: readonly string[] +): number | undefined => { + const evaluatorSet = column.evaluators ? new Set(column.evaluators) : undefined; + + let weightedSum = 0; + let totalCount = 0; + + for (const evaluator of columnEvaluators(modelScores, column)) { + // A column may opt into an explicit evaluator allowlist; otherwise the global + // exclusion list drops raw-magnitude evaluators that would blow out the 0-10 scale. + const skip = evaluatorSet + ? !evaluatorSet.has(evaluator.evaluatorName) + : isExcludedEvaluator(evaluator.evaluatorName, excludeEvaluators); + if (skip) { + continue; + } + + const weight = weightOf(evaluator); + weightedSum += evaluator.mean * weight; + totalCount += weight; + } + + return totalCount === 0 ? undefined : weightedSum / totalCount; +}; + +const buildCell = ( + mean: number | undefined, + column: MatrixColumnConfig, + config: MatrixConfig +): MatrixCell => { + if (mean === undefined) { + return { kind: 'missing' }; + } + + const scale = column.scale ?? config.defaultScale; + return toCell(roundTo(mean * scale, config.decimals), config); +}; + +/** + * Weighted mean of already-computed cells, shared by the legacy Overall column and + * composites: "Not recommended" sources contribute 0 (when configured) and missing + * sources are skipped, so the result reflects the data that exists rather than being + * dragged to "missing" by not-yet-wired columns. + */ +const aggregateCells = ( + sources: Array<{ cell: MatrixCell | undefined; weight: number }>, + config: MatrixConfig +): MatrixCell => { + let weightedSum = 0; + let totalWeight = 0; + let hasAnyData = false; + + for (const { cell, weight } of sources) { + if (!cell || cell.kind === 'missing') { + continue; + } + + hasAnyData = true; + + if (cell.kind === 'not-recommended') { + if (config.notRecommendedCountsAsZeroInOverall) { + totalWeight += weight; + } + continue; + } + + weightedSum += cell.value * weight; + totalWeight += weight; + } + + if (!hasAnyData || totalWeight === 0) { + return { kind: 'missing' }; + } + + return toCell(roundTo(weightedSum / totalWeight, config.decimals), config); +}; + +const computeOverall = (cells: Record, config: MatrixConfig): MatrixCell => + aggregateCells( + config.columns.map((column) => ({ + cell: cells[column.id], + weight: config.overall.mode === 'weighted' ? column.weight : 1, + })), + config + ); + +const computeComposite = ( + cells: Record, + composite: MatrixCompositeConfig, + config: MatrixConfig +): MatrixCell => + aggregateCells( + composite.from.map((refId) => ({ cell: cells[refId], weight: 1 })), + config + ); + +/** Resolves the left-to-right render order of base + composite (+ overall) columns. */ +const buildDisplayColumns = (config: MatrixConfig): MatrixDisplayColumn[] => { + const baseById = new Map(config.columns.map((column) => [column.id, column])); + const compositeById = new Map(config.composites.map((composite) => [composite.id, composite])); + + const declared: MatrixDisplayColumn[] = config.layout + ? config.layout.map((id): MatrixDisplayColumn => { + const base = baseById.get(id); + if (base) { + return { id, label: base.label, group: base.group, kind: 'base' }; + } + const composite = compositeById.get(id); + if (composite) { + return { id, label: composite.label, group: composite.group, kind: 'composite' }; + } + throw new Error(`Matrix config "layout" references unknown column/composite id: "${id}"`); + }) + : [ + ...config.columns.map( + (column): MatrixDisplayColumn => ({ + id: column.id, + label: column.label, + group: column.group, + kind: 'base', + }) + ), + ...config.composites.map( + (composite): MatrixDisplayColumn => ({ + id: composite.id, + label: composite.label, + group: composite.group, + kind: 'composite', + }) + ), + ]; + + return config.showOverall + ? [...declared, { id: OVERALL_COLUMN_ID, label: config.overall.label, kind: 'overall' }] + : declared; +}; + +/** + * Aggregates the raw-magnitude token evaluators for one (model, column) pair. Unlike + * the quality path these stay in native units and preserve the observed min/max spread. + */ +const computeTokenStat = ( + modelScores: AggregatedModelScores, + column: MatrixColumnConfig, + evaluatorPrefix: string +): TokenStat | undefined => { + let weightedSum = 0; + let totalCount = 0; + let min = Number.POSITIVE_INFINITY; + let max = Number.NEGATIVE_INFINITY; + + for (const evaluator of columnEvaluators(modelScores, column)) { + if (!evaluator.evaluatorName.startsWith(evaluatorPrefix)) { + continue; + } + + const weight = weightOf(evaluator); + weightedSum += evaluator.mean * weight; + totalCount += weight; + // Stats payloads may omit the per-experiment extremes; the mean is the only bound then. + min = Math.min(min, evaluator.min ?? evaluator.mean); + max = Math.max(max, evaluator.max ?? evaluator.mean); + } + + if (totalCount === 0) { + return undefined; + } + + return { mean: weightedSum / totalCount, min, max, count: totalCount }; +}; + +const buildTokenCost = ( + config: MatrixConfig, + tokenConfig: MatrixTokenCostConfig, + resolveScores: (modelConfig: MatrixModelConfig) => AggregatedModelScores | undefined +): { models: TokenCostModel[] } => { + const columnIds = tokenConfig.columns; + const tokenColumns = columnIds + ? config.columns.filter((column) => columnIds.includes(column.id)) + : config.columns; + + const models: TokenCostModel[] = []; + + for (const modelConfig of config.models) { + const modelScores = resolveScores(modelConfig); + if (!modelScores) { + continue; + } + + const cells: TokenCostCell[] = []; + for (const column of tokenColumns) { + const inputTokens = computeTokenStat(modelScores, column, tokenConfig.inputEvaluator); + const outputTokens = computeTokenStat(modelScores, column, tokenConfig.outputEvaluator); + if (!inputTokens && !outputTokens) { + continue; + } + cells.push({ + columnId: column.id, + inputTokens, + outputTokens, + totalMean: (inputTokens?.mean ?? 0) + (outputTokens?.mean ?? 0), + }); + } + + if (cells.length > 0) { + models.push({ + modelId: modelConfig.id, + modelLabel: modelConfig.label, + openSource: modelConfig.openSource, + cells, + }); + } + } + + return { models }; +}; + +/** + * Pure transform from aggregated eval scores + config into a renderable matrix. + * Models are emitted in config order; models absent from the data are skipped. + */ +export const buildMatrix = (aggregated: AggregatedModelScores[], config: MatrixConfig): Matrix => { + const byModelId = new Map(aggregated.map((entry) => [entry.modelId, entry])); + const resolveScores = (modelConfig: MatrixModelConfig) => + byModelId.get(modelConfig.id) ?? + aggregated.find((entry) => matchesModel(modelConfig, entry.modelId)); + + const proprietary: MatrixRow[] = []; + const openSource: MatrixRow[] = []; + + for (const modelConfig of config.models) { + const modelScores = resolveScores(modelConfig); + if (!modelScores) { + continue; + } + + const cells: Record = {}; + for (const column of config.columns) { + cells[column.id] = buildCell( + computeColumnMean(modelScores, column, config.excludeEvaluators), + column, + config + ); + } + + // Declared order, so a later composite can reference an earlier one (e.g. Overall + // Score <- Agent Builder Score); an unresolved reference contributes nothing. + for (const composite of config.composites) { + cells[composite.id] = computeComposite(cells, composite, config); + } + + const row: MatrixRow = { + modelId: modelConfig.id, + modelLabel: modelConfig.label, + openSource: modelConfig.openSource, + cells, + overall: computeOverall(cells, config), + }; + + (modelConfig.openSource ? openSource : proprietary).push(row); + } + + // Rank by the final composite (e.g. Overall Score) when composites exist, + // otherwise by the legacy Overall column. + const primaryId = + config.composites.length > 0 + ? config.composites[config.composites.length - 1].id + : OVERALL_COLUMN_ID; + + const sortValue = (row: MatrixRow): number => { + const cell = primaryId === OVERALL_COLUMN_ID ? row.overall : row.cells[primaryId]; + return cell && cell.kind === 'score' ? cell.value : -1; + }; + + const sortByPrimaryDesc = (a: MatrixRow, b: MatrixRow): number => sortValue(b) - sortValue(a); + + return { + columns: config.columns.map((column) => ({ + id: column.id, + label: column.label, + group: column.group, + })), + composites: config.composites.map((composite) => ({ + id: composite.id, + label: composite.label, + group: composite.group, + })), + displayColumns: buildDisplayColumns(config), + overallLabel: config.overall.label, + proprietary: proprietary.sort(sortByPrimaryDesc), + openSource: openSource.sort(sortByPrimaryDesc), + // Token magnitudes are meaningful only over base columns; composites are derived scores. + ...(config.tokenCost + ? { tokenCost: buildTokenCost(config, config.tokenCost, resolveScores) } + : {}), + }; +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/load_matrix_config.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/load_matrix_config.test.ts new file mode 100644 index 0000000000000..3357822ad1ec0 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/load_matrix_config.test.ts @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + parseMatrixConfig, + DEFAULT_EXCLUDED_EVALUATORS, + applyModelOverrides, + parseModelOverride, +} from './load_matrix_config'; + +describe('parseMatrixConfig', () => { + const minimalConfig = { + columns: [{ id: 'alert_triage', label: 'Alert Triage', suites: ['security-alert-triage'] }], + models: [{ id: 'eis/foo', label: 'Foo' }], + }; + + it('applies defaults for optional fields', () => { + const config = parseMatrixConfig(minimalConfig); + + expect(config.branch).toBe('main'); + expect(config.defaultScale).toBe(10); + expect(config.decimals).toBe(2); + expect(config.notRecommendedBelow).toBe(0); + expect(config.notRecommendedLabel).toBe('Not recommended'); + expect(config.notRecommendedCountsAsZeroInOverall).toBe(true); + expect(config.overall).toEqual({ label: 'Overall', mode: 'weighted' }); + expect(config.showOverall).toBe(true); + expect(config.composites).toEqual([]); + expect(config.layout).toBeUndefined(); + expect(config.columns[0].weight).toBe(1); + expect(config.columns[0].group).toBeUndefined(); + expect(config.models[0].openSource).toBe(false); + expect(config.excludeEvaluators).toEqual([...DEFAULT_EXCLUDED_EVALUATORS]); + }); + + it('accepts grouped columns, composites, a layout, and showOverall', () => { + const config = parseMatrixConfig({ + ...minimalConfig, + showOverall: false, + columns: [ + { id: 'a', label: 'A', group: 'Agent Builder', suites: ['s-a'] }, + { id: 'b', label: 'B', group: 'Agent Builder', suites: ['s-b'] }, + ], + composites: [{ id: 'ab', label: 'AB Score', from: ['a', 'b'] }], + layout: ['a', 'b', 'ab'], + }); + + expect(config.showOverall).toBe(false); + expect(config.columns[0].group).toBe('Agent Builder'); + expect(config.composites).toEqual([{ id: 'ab', label: 'AB Score', from: ['a', 'b'] }]); + expect(config.layout).toEqual(['a', 'b', 'ab']); + }); + + it('throws when a composite has no source columns', () => { + expect(() => + parseMatrixConfig({ + ...minimalConfig, + composites: [{ id: 'ab', label: 'AB', from: [] }], + }) + ).toThrow(); + }); + + it('allows overriding the evaluator exclusion list (including emptying it)', () => { + expect( + parseMatrixConfig({ ...minimalConfig, excludeEvaluators: [] }).excludeEvaluators + ).toEqual([]); + expect( + parseMatrixConfig({ ...minimalConfig, excludeEvaluators: ['Latency'] }).excludeEvaluators + ).toEqual(['Latency']); + }); + + it('throws when a column has no suites', () => { + expect(() => + parseMatrixConfig({ + ...minimalConfig, + columns: [{ id: 'x', label: 'X', suites: [] }], + }) + ).toThrow(); + }); + + it('throws when there are no columns or models', () => { + expect(() => parseMatrixConfig({ columns: [], models: [] })).toThrow(); + }); + + it('rejects an invalid overall mode', () => { + expect(() => parseMatrixConfig({ ...minimalConfig, overall: { mode: 'nope' } })).toThrow(); + }); +}); + +describe('applyModelOverrides', () => { + const base = parseMatrixConfig({ + title: 'Weekly', + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'], weight: 1 }], + models: [ + { id: 'weekly-1', label: 'Weekly One' }, + { id: 'weekly-2', label: 'Weekly Two' }, + ], + }); + + it('returns the config untouched when no overrides are given', () => { + expect(applyModelOverrides(base, [])).toBe(base); + }); + + it('replaces rather than appends, so an on-demand run shows only what was asked for', () => { + const result = applyModelOverrides(base, ['custom-a']); + expect(result.models).toEqual([{ id: 'custom-a', label: 'custom-a', openSource: false }]); + }); + + it('does not mutate the weekly config', () => { + applyModelOverrides(base, ['custom-a']); + expect(base.models.map((m) => m.id)).toEqual(['weekly-1', 'weekly-2']); + }); + + it('parses label and explicit open-source marker', () => { + expect(applyModelOverrides(base, ['qwen3-72b:Qwen3 72B:open-source']).models[0]).toEqual({ + id: 'qwen3-72b', + label: 'Qwen3 72B', + openSource: true, + }); + }); + + it('defaults the label to the id and openSource to false', () => { + expect(parseModelOverride('gpt-5')).toEqual({ + id: 'gpt-5', + label: 'gpt-5', + openSource: false, + }); + }); + + it('rejects a bogus third segment instead of silently treating it as proprietary', () => { + expect(() => parseModelOverride('gpt-5:GPT-5:oss')).toThrow(/literal "open-source"/); + }); + + it('rejects too many segments', () => { + expect(() => parseModelOverride('a:b:open-source:c')).toThrow(/at most 3/); + }); + + it('rejects an empty id', () => { + expect(() => parseModelOverride(':Label')).toThrow(/model id is required/); + }); + + it('rejects duplicate ids', () => { + expect(() => applyModelOverrides(base, ['dup', 'dup:Other'])).toThrow(/Duplicate --model id/); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/load_matrix_config.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/load_matrix_config.ts new file mode 100644 index 0000000000000..2edb830ff80ec --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/load_matrix_config.ts @@ -0,0 +1,308 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Fs from 'fs'; +import { schema, type TypeOf } from '@kbn/config-schema'; + +/** + * Upper bounds for schema fields. The config is a static repo-controlled JSON + * file (not request input), so these exist to satisfy bounded-input validation + * rather than to mitigate a real DoS vector; the limits are generous enough that + * any realistic matrix config stays well within them. + */ +const MAX_STRING_LENGTH = 1024; +const MAX_ARRAY_SIZE = 1000; + +/** + * Schema for the LLM performance matrix configuration file. + * + * The matrix engine is domain-agnostic: a config file maps human-facing matrix + * columns onto the eval `suite.id` / `example.dataset.id` / `evaluator.name` + * values stored in the `kibana-evaluations` data stream, declares the model + * allowlist (with display names + open-source classification), and describes how + * raw evaluator scores are normalized onto the published 0-10 scale. + */ +const columnSchema = schema.object({ + /** Stable identifier for the column (used as the CSV/JSON key). */ + id: schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), + /** Human-facing column header (e.g. "Alert Triage"). */ + label: schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), + /** + * Optional grouped-header label (e.g. "Agent Builder"). Columns sharing a + * `group` render under one spanning header in the published docs page. Carried + * through to the JSON artifact; the flat CSV/markdown keep one header row. + */ + group: schema.maybe(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH })), + /** `suite.id` values whose scores contribute to this column. */ + suites: schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { + minSize: 1, + maxSize: MAX_ARRAY_SIZE, + }), + /** Optional restriction to specific `example.dataset.id` values. */ + datasetIds: schema.maybe( + schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { + maxSize: MAX_ARRAY_SIZE, + }) + ), + /** Optional restriction to specific `evaluator.name` values. */ + evaluators: schema.maybe( + schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { + maxSize: MAX_ARRAY_SIZE, + }) + ), + /** + * Multiplier applied to the (weighted) mean evaluator score before rounding. + * Defaults to `defaultScale` (10) so 0-1 evaluator scores map onto the 0-10 + * scale. Set to 1 for evaluators that already emit a 0-10 score. + */ + scale: schema.maybe(schema.number({ min: 0 })), + /** Relative weight of this column in the legacy Overall score. Defaults to 1. */ + weight: schema.number({ defaultValue: 1, min: 0 }), +}); + +/** + * A derived ("composite") column whose cell is the equal-weighted mean of other + * columns' cells. `from` may reference base columns or earlier-defined + * composites, so composites can be layered (e.g. "Overall Score" averages the + * "Agent Builder Score" composite alongside two standalone feature columns). + * + * Aggregation mirrors the legacy Overall: "Not recommended" sources count as 0 + * (when `notRecommendedCountsAsZeroInOverall` is set) and missing sources are + * skipped, so a composite reflects the data that actually exists. + */ +const compositeSchema = schema.object({ + /** Stable identifier for the composite (used as the CSV/JSON key). */ + id: schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), + /** Human-facing column header (e.g. "Agent Builder Score"). */ + label: schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), + /** Optional grouped-header label (see `columnSchema.group`). */ + group: schema.maybe(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH })), + /** Column/composite ids whose cells are averaged into this composite. */ + from: schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { + minSize: 1, + maxSize: MAX_ARRAY_SIZE, + }), +}); + +const modelSchema = schema.object({ + /** Primary `task.model.id` value to match against. */ + id: schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), + /** Display name shown in the published matrix (e.g. "Claude Sonnet 4"). */ + label: schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), + /** Additional `task.model.id` values that should map to the same row. */ + matchIds: schema.maybe( + schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { + maxSize: MAX_ARRAY_SIZE, + }) + ), + /** Renders the model under the "Open-source models" table when true. */ + openSource: schema.boolean({ defaultValue: false }), +}); + +/** + * Default `evaluator.name` values excluded from column aggregation. + * + * Security eval suites register an "observability" tier of trace-based + * evaluators (latency, token counts, tool-call counts, skill invocation) + * alongside the 0-1 quality evaluators. Those emit raw magnitudes (thousands of + * tokens, milliseconds) rather than a 0-1 score, so averaging them into a column + * and multiplying by the 0-10 scale produces wildly inflated cells. We exclude + * them by default; a config may override `excludeEvaluators` to opt back in. + * + * Matching is name-prefix based so dynamically-named evaluators such as + * `Skill Invoked (alert-analysis)` are caught by the `Skill Invoked` entry. + */ +export const DEFAULT_EXCLUDED_EVALUATORS: readonly string[] = [ + 'Latency', + 'Tool Calls', + 'Input Tokens', + 'Output Tokens', + 'Cached Tokens', + 'Skill Invoked', +]; + +export const matrixConfigSchema = schema.object({ + /** Page/table title (informational; used in the markdown artifact). */ + title: schema.string({ defaultValue: 'LLM performance matrix', maxLength: MAX_STRING_LENGTH }), + /** Default git branch to pull experiments from (CLI `--branch` overrides). */ + branch: schema.string({ defaultValue: 'main', maxLength: MAX_STRING_LENGTH }), + /** Only consider experiments newer than `now-d`. */ + lookbackDays: schema.number({ defaultValue: 45, min: 1 }), + /** Default multiplier applied to evaluator means when a column omits `scale`. */ + defaultScale: schema.number({ defaultValue: 10, min: 0 }), + /** Decimal places used when rounding cell values. */ + decimals: schema.number({ defaultValue: 2, min: 0, max: 6 }), + /** Cells at/under this value (after scaling) render as `notRecommendedLabel`. */ + notRecommendedBelow: schema.number({ defaultValue: 0, min: 0 }), + /** Text rendered when a model fails / lacks data for a column. */ + notRecommendedLabel: schema.string({ + defaultValue: 'Not recommended', + maxLength: MAX_STRING_LENGTH, + }), + /** + * When true, "Not recommended" cells count as 0 in the Overall score and in + * composite columns (matches the published matrix behavior where failures drag + * the average down). + */ + notRecommendedCountsAsZeroInOverall: schema.boolean({ defaultValue: true }), + /** + * `evaluator.name` values (matched by prefix) excluded from every column's + * aggregation. Defaults to the observability-tier evaluators, which emit raw + * magnitudes rather than 0-1 quality scores. Set to `[]` to include everything. + */ + excludeEvaluators: schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { + defaultValue: [...DEFAULT_EXCLUDED_EVALUATORS], + maxSize: MAX_ARRAY_SIZE, + }), + overall: schema.object({ + label: schema.string({ defaultValue: 'Overall', maxLength: MAX_STRING_LENGTH }), + mode: schema.oneOf([schema.literal('weighted'), schema.literal('mean')], { + defaultValue: 'weighted', + }), + }), + /** + * Renders the legacy single "Overall" column (weighted/mean over every base + * column) at the far right. Set to `false` when the layout expresses its own + * Overall via a composite, to avoid a duplicate trailing column. + */ + showOverall: schema.boolean({ defaultValue: true }), + columns: schema.arrayOf(columnSchema, { minSize: 1, maxSize: MAX_ARRAY_SIZE }), + /** Derived columns averaged from base columns / earlier composites. */ + composites: schema.arrayOf(compositeSchema, { defaultValue: [], maxSize: MAX_ARRAY_SIZE }), + /** + * Explicit left-to-right display order of base + composite column ids. When + * omitted, base columns render first (config order), then composites. The + * legacy Overall column (when `showOverall`) is always appended last. + */ + layout: schema.maybe( + schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { + maxSize: MAX_ARRAY_SIZE, + }) + ), + models: schema.arrayOf(modelSchema, { minSize: 1, maxSize: MAX_ARRAY_SIZE }), + /** + * Opt-in token/cost axis. The quality matrix deliberately drops the + * observability-tier evaluators (see {@link DEFAULT_EXCLUDED_EVALUATORS}) + * because their raw magnitudes would blow out the 0-10 scale. This block + * re-admits them on a *separate* axis: instead of being folded into a column + * mean, the named evaluators are aggregated per (model, column) into + * `matrix.tokenCost`, preserving mean/min/max in native units. + * + * Omitted by default, so existing configs are unaffected. + */ + tokenCost: schema.maybe( + schema.object({ + /** Evaluator name (prefix-matched) contributing input-token magnitudes. */ + inputEvaluator: schema.string({ + defaultValue: 'Input Tokens', + maxLength: MAX_STRING_LENGTH, + }), + /** Evaluator name (prefix-matched) contributing output-token magnitudes. */ + outputEvaluator: schema.string({ + defaultValue: 'Output Tokens', + maxLength: MAX_STRING_LENGTH, + }), + /** + * Column ids the token axis is aggregated over. Defaults to every base + * column in the config when omitted. + */ + columns: schema.maybe( + schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { + maxSize: MAX_ARRAY_SIZE, + }) + ), + }) + ), +}); + +export type MatrixConfig = TypeOf; +export type MatrixTokenCostConfig = NonNullable; +export type MatrixColumnConfig = TypeOf; +export type MatrixCompositeConfig = TypeOf; +export type MatrixModelConfig = TypeOf; + +export const parseMatrixConfig = (raw: unknown): MatrixConfig => matrixConfigSchema.validate(raw); + +/** + * Parses a `--model` CLI value into a model config entry. + * + * Format: `id[:label][:open-source]`, e.g. + * `gpt-5-preview` + * `gpt-5-preview:GPT-5 Preview` + * `qwen3-72b:Qwen3 72B:open-source` + * + * Labels may contain spaces but not colons; the third segment is an explicit + * open-source marker rather than a substring guess at the model name. + */ +export const parseModelOverride = (raw: string): MatrixModelConfig => { + const segments = raw.split(':').map((segment) => segment.trim()); + const [id, label, openSourceFlag] = segments; + + if (!id) { + throw new Error( + `Invalid --model value "${raw}": model id is required (format: id[:label][:open-source]).` + ); + } + if (segments.length > 3) { + throw new Error( + `Invalid --model value "${raw}": expected at most 3 colon-separated segments (id[:label][:open-source]).` + ); + } + if (openSourceFlag !== undefined && openSourceFlag !== 'open-source') { + throw new Error( + `Invalid --model value "${raw}": third segment must be the literal "open-source", got "${openSourceFlag}".` + ); + } + + return { id, label: label || id, openSource: openSourceFlag === 'open-source' }; +}; + +/** + * Replaces the config's model set with an ad-hoc one for on-demand runs. + * + * The weekly matrix is a fixed, reviewed model set that must stay stable + * across runs, so this deliberately does not mutate the config file — an + * on-demand run with `--model` is a throwaway view over the same score data. + */ +export const applyModelOverrides = ( + config: MatrixConfig, + rawModels: readonly string[] +): MatrixConfig => { + if (rawModels.length === 0) { + return config; + } + + const models = rawModels.map(parseModelOverride); + const seen = new Set(); + for (const model of models) { + if (seen.has(model.id)) { + throw new Error(`Duplicate --model id "${model.id}".`); + } + seen.add(model.id); + } + + return { ...config, models }; +}; + +export const loadMatrixConfig = (configPath: string): MatrixConfig => { + if (!Fs.existsSync(configPath)) { + throw new Error(`Matrix config not found at: ${configPath}`); + } + + let parsed: unknown; + try { + parsed = JSON.parse(Fs.readFileSync(configPath, 'utf-8')); + } catch (error) { + throw new Error( + `Failed to parse matrix config at ${configPath}: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + + return parseMatrixConfig(parsed); +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.test.ts new file mode 100644 index 0000000000000..81683a160eb08 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.test.ts @@ -0,0 +1,229 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SomeDevLog } from '@kbn/some-dev-log'; +import { ToolingLog } from '@kbn/tooling-log'; +import type { EvaluationExperimentSummary } from '@kbn/evals-common'; +import type { EvalsClient, ExperimentStats } from '@kbn/evals'; +import { + pickLatestExperimentPerModel, + experimentStatsToDatasets, + queryMatrixScores, +} from './query_matrix_scores'; + +const experiment = ( + overrides: Partial & { modelId?: string } +): EvaluationExperimentSummary => { + const { modelId, ...rest } = overrides; + return { + experiment_id: 'exp', + timestamp: '2026-06-10T00:00:00.000Z', + task_model: modelId ? { id: modelId, family: 'fam', provider: 'prov' } : undefined, + ...rest, + } as EvaluationExperimentSummary; +}; + +describe('pickLatestExperimentPerModel', () => { + it('keeps the most recent experiment per model', () => { + const result = pickLatestExperimentPerModel([ + experiment({ experiment_id: 'old', modelId: 'm1', timestamp: '2026-06-01T00:00:00.000Z' }), + experiment({ experiment_id: 'new', modelId: 'm1', timestamp: '2026-06-09T00:00:00.000Z' }), + experiment({ experiment_id: 'other', modelId: 'm2', timestamp: '2026-06-05T00:00:00.000Z' }), + ]); + + expect(result.get('m1')?.experiment_id).toBe('new'); + expect(result.get('m2')?.experiment_id).toBe('other'); + }); + + it('ignores experiments without a task model id', () => { + const result = pickLatestExperimentPerModel([ + experiment({ experiment_id: 'no-model', modelId: undefined }), + ]); + expect(result.size).toBe(0); + }); + + it('drops experiments older than the lookback window', () => { + const now = Date.parse('2026-06-15T00:00:00.000Z'); + const result = pickLatestExperimentPerModel( + [ + experiment({ + experiment_id: 'stale', + modelId: 'm1', + timestamp: '2026-05-01T00:00:00.000Z', + }), + experiment({ + experiment_id: 'fresh', + modelId: 'm2', + timestamp: '2026-06-14T00:00:00.000Z', + }), + ], + { lookbackDays: 14, now } + ); + + expect(result.has('m1')).toBe(false); + expect(result.get('m2')?.experiment_id).toBe('fresh'); + }); + + it('drops experiments with unparseable timestamps instead of treating them as epoch 0', () => { + const now = Date.parse('2026-06-15T00:00:00.000Z'); + const result = pickLatestExperimentPerModel( + [ + experiment({ experiment_id: 'broken', modelId: 'm1', timestamp: 'not-a-date' }), + experiment({ + experiment_id: 'fresh', + modelId: 'm1', + timestamp: '2026-06-14T00:00:00.000Z', + }), + ], + { lookbackDays: 14, now } + ); + + expect(result.get('m1')?.experiment_id).toBe('fresh'); + }); +}); + +describe('experimentStatsToDatasets', () => { + it('groups evaluator stats by dataset with mean + count', () => { + const stats: ExperimentStats = { + taskModel: { id: 'm1' }, + evaluatorModel: { id: 'judge' }, + totalRepetitions: 1, + stats: [ + { + datasetId: 'd1', + datasetName: 'D1', + evaluatorName: 'correctness', + stats: { mean: 0.9, median: 0.9, stdDev: 0, min: 0.9, max: 0.9, count: 10 }, + }, + { + datasetId: 'd1', + datasetName: 'D1', + evaluatorName: 'groundedness', + stats: { mean: 0.8, median: 0.8, stdDev: 0, min: 0.8, max: 0.8, count: 10 }, + }, + { + datasetId: 'd2', + datasetName: 'D2', + evaluatorName: 'correctness', + stats: { mean: 0.7, median: 0.7, stdDev: 0, min: 0.7, max: 0.7, count: 5 }, + }, + ], + }; + + expect(experimentStatsToDatasets(stats)).toEqual([ + { + datasetId: 'd1', + datasetName: 'D1', + evaluators: [ + { evaluatorName: 'correctness', mean: 0.9, count: 10, min: 0.9, max: 0.9 }, + { evaluatorName: 'groundedness', mean: 0.8, count: 10, min: 0.8, max: 0.8 }, + ], + }, + { + datasetId: 'd2', + datasetName: 'D2', + evaluators: [{ evaluatorName: 'correctness', mean: 0.7, count: 5, min: 0.7, max: 0.7 }], + }, + ]); + }); +}); + +describe('queryMatrixScores', () => { + const log = new ToolingLog() as unknown as SomeDevLog; + + const stats: ExperimentStats = { + taskModel: { id: 'm1' }, + evaluatorModel: { id: 'judge' }, + totalRepetitions: 1, + stats: [ + { + datasetId: 'd1', + datasetName: 'D1', + evaluatorName: 'correctness', + stats: { mean: 0.9, median: 0.9, stdDev: 0, min: 0.9, max: 0.9, count: 10 }, + }, + ], + }; + + const createClient = ( + experimentsByModel: Record + ): { client: EvalsClient; listExperiments: jest.Mock; getExperimentStats: jest.Mock } => { + const listExperiments = jest + .fn() + .mockImplementation(async ({ taskModelId }: { taskModelId?: string }) => + taskModelId ? experimentsByModel[taskModelId] ?? [] : [] + ); + const getExperimentStats = jest.fn().mockResolvedValue(stats); + const client = { listExperiments, getExperimentStats } as unknown as EvalsClient; + return { client, listExperiments, getExperimentStats }; + }; + + it('queries each (suite, model) pair through the route model_id filter', async () => { + const { client, listExperiments, getExperimentStats } = createClient({ + m1: [experiment({ experiment_id: 'exp-m1', modelId: 'm1' })], + m2: [experiment({ experiment_id: 'exp-m2', modelId: 'm2' })], + }); + + const result = await queryMatrixScores(client, log, { + suiteIds: ['suite-a'], + modelIds: ['m1', 'm2'], + branch: 'main', + }); + + expect(listExperiments).toHaveBeenCalledTimes(2); + expect(listExperiments).toHaveBeenCalledWith( + expect.objectContaining({ suiteId: 'suite-a', taskModelId: 'm1', branch: 'main' }) + ); + expect(listExperiments).toHaveBeenCalledWith( + expect.objectContaining({ suiteId: 'suite-a', taskModelId: 'm2', branch: 'main' }) + ); + expect(getExperimentStats).toHaveBeenCalledTimes(2); + expect(result.map((model) => model.modelId).sort()).toEqual(['m1', 'm2']); + }); + + it('picks the newest experiment within the lookback window per model', async () => { + const now = Date.now(); + const recent = new Date(now - 2 * 24 * 60 * 60 * 1000).toISOString(); + const stale = new Date(now - 60 * 24 * 60 * 60 * 1000).toISOString(); + const { client, getExperimentStats } = createClient({ + // Route returns newest first; the stale one would be picked by a naive per_page: 1 + // request if the newest ever fell outside the window. + m1: [ + experiment({ experiment_id: 'recent', modelId: 'm1', timestamp: recent }), + experiment({ experiment_id: 'stale', modelId: 'm1', timestamp: stale }), + ], + }); + + const result = await queryMatrixScores(client, log, { + suiteIds: ['suite-a'], + modelIds: ['m1'], + lookbackDays: 7, + }); + + expect(getExperimentStats).toHaveBeenCalledWith( + 'recent', + expect.objectContaining({ suiteId: 'suite-a', taskModelId: 'm1' }) + ); + expect(result[0].suites[0].experimentId).toBe('recent'); + }); + + it('omits models with no experiment inside the lookback window', async () => { + const stale = new Date(Date.now() - 60 * 24 * 60 * 60 * 1000).toISOString(); + const { client, getExperimentStats } = createClient({ + m1: [experiment({ experiment_id: 'stale', modelId: 'm1', timestamp: stale })], + }); + + const result = await queryMatrixScores(client, log, { + suiteIds: ['suite-a'], + modelIds: ['m1'], + lookbackDays: 7, + }); + + expect(getExperimentStats).not.toHaveBeenCalled(); + expect(result).toEqual([]); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.ts new file mode 100644 index 0000000000000..3c408331ac99a --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.ts @@ -0,0 +1,189 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SomeDevLog } from '@kbn/some-dev-log'; +import type { EvaluationExperimentSummary } from '@kbn/evals-common'; +import { MAX_LIST_EXPERIMENTS, type EvalsClient, type ExperimentStats } from '@kbn/evals'; + +/** Aggregated evaluator score for a single dataset within a suite. */ +export interface AggregatedEvaluatorScore { + evaluatorName: string; + mean: number; + count: number; + /** Observed spread across the experiment's examples (used by the token axis). */ + min?: number; + max?: number; +} + +export interface AggregatedDatasetScores { + datasetId: string; + datasetName: string; + evaluators: AggregatedEvaluatorScore[]; +} + +export interface AggregatedSuiteScores { + suiteId: string; + experimentId: string; + timestamp?: string; + datasets: AggregatedDatasetScores[]; +} + +export interface AggregatedModelScores { + modelId: string; + family?: string; + provider?: string; + suites: AggregatedSuiteScores[]; +} + +export interface QueryMatrixScoresOptions { + suiteIds: string[]; + /** + * Task model ids to include (the config's `id` + `matchIds` per model). + * Each (suite, model) pair is queried separately via the route's `model_id` + * term filter, so a bounded single-page listing covers the full history for + * that pair instead of paging an ever-growing cross-model aggregation. + */ + modelIds: string[]; + branch?: string; + lookbackDays?: number; +} + +/** + * Selects, per task model, the most recent experiment from a list (typically + * the newest experiments for one suite + model). Experiments without a model + * id or timestamp older than the lookback window are ignored. Pure for unit + * testing. + */ +export const pickLatestExperimentPerModel = ( + experiments: EvaluationExperimentSummary[], + { lookbackDays, now = Date.now() }: { lookbackDays?: number; now?: number } = {} +): Map => { + const cutoff = lookbackDays ? now - lookbackDays * 24 * 60 * 60 * 1000 : undefined; + const latestByModel = new Map(); + + for (const experiment of experiments) { + const modelId = experiment.task_model?.id; + if (!modelId) { + continue; + } + + const at = Date.parse(experiment.timestamp); + // An unparseable timestamp must not be treated as epoch 0, or stale + // experiments would silently survive the lookback cutoff. + if (!Number.isFinite(at) || (cutoff !== undefined && at < cutoff)) { + continue; + } + + const existing = latestByModel.get(modelId); + if (!existing || at > existing.at) { + latestByModel.set(modelId, { experiment, at }); + } + } + + return new Map([...latestByModel].map(([modelId, { experiment }]) => [modelId, experiment])); +}; + +/** + * Converts the per-experiment stats returned by the evals plugin into the + * dataset-grouped structure consumed by the matrix builder. Pure for testing. + */ +export const experimentStatsToDatasets = (stats: ExperimentStats): AggregatedDatasetScores[] => { + const byDataset = new Map(); + + for (const stat of stats.stats) { + let dataset = byDataset.get(stat.datasetId); + if (!dataset) { + dataset = { datasetId: stat.datasetId, datasetName: stat.datasetName, evaluators: [] }; + byDataset.set(stat.datasetId, dataset); + } + dataset.evaluators.push({ + evaluatorName: stat.evaluatorName, + mean: stat.stats.mean, + count: stat.stats.count, + min: stat.stats.min, + max: stat.stats.max, + }); + } + + return [...byDataset.values()]; +}; + +/** + * Queries the evals plugin for the latest experiment per (model, suite) and + * returns mean evaluator scores grouped by dataset, ready for `buildMatrix`. + * + * Each (suite, model) pair is listed separately through the route's `model_id` + * term filter: the route answers with a terms aggregation whose bucket size + * grows with `page * per_page`, so a bounded single page per pair is the only + * query shape that scales. The newest experiment within the lookback window is + * then picked client-side (a bare `per_page: 1` request could not express the + * lookback fallback). + */ +export const queryMatrixScores = async ( + evalsClient: EvalsClient, + log: SomeDevLog, + { suiteIds, modelIds, branch, lookbackDays }: QueryMatrixScoresOptions +): Promise => { + const byModel = new Map(); + + for (const suiteId of suiteIds) { + for (const modelId of modelIds) { + const experiments = await evalsClient.listExperiments({ + suiteId, + taskModelId: modelId, + branch, + limit: MAX_LIST_EXPERIMENTS, + }); + const [latest] = [...pickLatestExperimentPerModel(experiments, { lookbackDays }).values()]; + + log.debug( + `Suite ${suiteId}, model ${modelId}: ${experiments.length} experiment(s)` + + (latest ? '' : ', none within the lookback window') + ); + + if (!latest) { + continue; + } + + // The experiments listing returns `execution_id` as its grouping key; the + // detail/stats route must be filtered by execution_id (+ suite + model), + // since a bare experiment_id path lookup targets a different field and 404s. + const stats = await evalsClient.getExperimentStats(latest.experiment_id, { + suiteId, + taskModelId: modelId, + executionId: latest.execution_id ?? latest.experiment_id, + }); + if (!stats) { + log.warning( + `No stats for experiment ${latest.experiment_id} (suite ${suiteId}, model ${modelId})` + ); + continue; + } + + let model = byModel.get(modelId); + if (!model) { + model = { + modelId, + family: latest.task_model?.family, + provider: latest.task_model?.provider, + suites: [], + }; + byModel.set(modelId, model); + } + + model.suites.push({ + suiteId, + experimentId: latest.experiment_id, + timestamp: latest.timestamp, + datasets: experimentStatsToDatasets(stats), + }); + } + } + + log.debug(`Matrix query resolved ${byModel.size} model(s) across ${suiteIds.length} suite(s)`); + return [...byModel.values()]; +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.ts new file mode 100644 index 0000000000000..2b9199d7dc71a --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.ts @@ -0,0 +1,228 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SomeDevLog } from '@kbn/some-dev-log'; +import type { EvalsClient } from '@kbn/evals'; +import type { EvaluationScoreDocument } from '@kbn/evals-common'; +import type { AggregatedModelScores } from './query_matrix_scores'; +import type { MatrixTraceData, MatrixTraceEntry, TraceStep } from './trace_types'; +import { traceKey } from './trace_types'; + +/** Maximum number of experiments to scan for complete scores per (suite, model). */ +const MAX_EXPERIMENTS_TO_SCAN = 5; + +/** + * Extracts trace data (initial question, tool trail, agent answer, step trace) + * from a single evaluation score document. + * + * Score documents store the full `task.output` which contains: + * - `steps`: array of `{ type: "reasoning"|"tool_call"|"relevant_skills", ... }` + * - `messages`: array of `{ message: string }` (the agent's final answer) + * And `example.input.question` holds the initial user question. + */ +const extractTraceFromScore = (score: EvaluationScoreDocument): MatrixTraceEntry => { + const question = (score.example?.input as { question?: string } | null)?.question; + const taskOutput = score.task?.output as + | { + steps?: Array>; + messages?: Array<{ message?: string }>; + } + | null + | undefined; + + const steps: TraceStep[] = []; + const toolTrail: string[] = []; + + for (const step of taskOutput?.steps ?? []) { + const stepType = step.type as string | undefined; + if (stepType === 'tool_call') { + const toolId = step.tool_id as string | undefined; + if (toolId) { + toolTrail.push(toolId); + } + steps.push({ + type: 'tool', + toolId, + toolParams: step.args ? JSON.stringify(step.args).slice(0, 300) : undefined, + }); + } else if (stepType === 'reasoning') { + steps.push({ + type: 'reasoning', + text: (step.reasoning as string | undefined)?.slice(0, 500), + }); + } else if (stepType === 'relevant_skills') { + const skills = Array.isArray(step.skills) + ? (step.skills as Array<{ id?: string }>) + .map((s) => s.id) + .filter((id): id is string => Boolean(id)) + : undefined; + steps.push({ type: 'skill', skills }); + } + } + + // The final answer is the last non-empty message + let answer: string | undefined; + for (const msg of taskOutput?.messages ?? []) { + const content = msg.message; + if (content && content.length > 50) { + answer = content; + } + } + + return { + question, + toolTrail: toolTrail.length > 0 ? toolTrail : undefined, + answer: answer || undefined, + steps: steps.length > 0 ? steps : undefined, + stepCount: steps.length, + toolCount: toolTrail.length, + }; +}; + +/** + * A score document is considered "complete" when: + * - `evaluator.score` is non-null (the evaluator finished and produced a verdict) + * - `task.output` is non-null (the agent produced output — steps and/or messages) + * + * Incomplete runs (e.g. worker SIGKILL, OOM, timeout) may still write partial + * score documents with null scores or empty output. These should not pollute + * the matrix because they don't represent a real evaluation. + */ +const isCompleteScore = (score: EvaluationScoreDocument): boolean => { + if (score.evaluator?.score == null) return false; + return score.task?.output != null; +}; + +/** + * Processes a batch of score documents (all from the same experiment) and + * populates `traces` with per-example and per-suite entries. Only the latest + * complete score per suite is used. + * + * @returns The number of complete scores found in this batch. + */ +const processScoreBatch = ( + scores: EvaluationScoreDocument[], + modelId: string, + suiteId: string, + traces: MatrixTraceData +): number => { + // Sort by @timestamp descending so the latest complete run wins + const sorted = [...scores].sort((a, b) => { + const ta = Date.parse(a['@timestamp'] ?? ''); + const tb = Date.parse(b['@timestamp'] ?? ''); + return tb - ta; + }); + + let suiteTraceAssigned = false; + let completeCount = 0; + + for (const score of sorted) { + const entry = extractTraceFromScore(score); + const datasetId = score.example?.dataset?.id; + const exampleId = score.example?.id; + + // Key by model:exampleId for per-prompt detail + if (exampleId) { + traces[traceKey(modelId, exampleId)] = entry; + } + if (datasetId) { + traces[traceKey(modelId, datasetId)] = entry; + } + + // For the suite-level key, pick the latest **complete** run only. + if (!suiteTraceAssigned && isCompleteScore(score)) { + traces[traceKey(modelId, suiteId)] = entry; + suiteTraceAssigned = true; + completeCount++; + } + } + + return completeCount; +}; + +/** + * Queries evaluation score documents from the golden cluster via the evals + * plugin and extracts trace data (initial question, tool trail, agent answer, + * step trace) for each (model, column) pair. + * + * Uses the same experiment IDs resolved by `queryMatrixScores` to fetch the + * full score documents — which include `task.output.steps` and + * `example.input.question` — and maps them into `MatrixTraceData`. + * + * If the latest experiment has no complete scores (e.g. worker died mid-run), + * it falls back to scanning up to `MAX_EXPERIMENTS_TO_SCAN` earlier + * experiments for the same (suite, model) to find the latest complete run. + */ +export const queryMatrixTraces = async ( + evalsClient: EvalsClient, + log: SomeDevLog, + aggregated: AggregatedModelScores[] +): Promise => { + const traces: MatrixTraceData = {}; + + for (const modelScores of aggregated) { + for (const suite of modelScores.suites) { + const { suiteId, experimentId } = suite; + log.debug( + `Fetching score documents for experiment ${experimentId} (model ${modelScores.modelId}, suite ${suiteId})` + ); + + // First try the latest experiment + let scores = await evalsClient.getExperimentScores(experimentId, { + suiteId, + taskModelId: modelScores.modelId, + executionId: experimentId, + }); + + let completeCount = processScoreBatch(scores, modelScores.modelId, suiteId, traces); + + // If no complete scores found, scan earlier experiments + if (completeCount === 0) { + log.debug( + `No complete scores in latest experiment ${experimentId}, scanning earlier runs...` + ); + + const experiments = await evalsClient.listExperiments({ + suiteId, + taskModelId: modelScores.modelId, + limit: MAX_EXPERIMENTS_TO_SCAN, + }); + + // Skip the first one (already tried), try the rest in order + for (const exp of experiments.slice(1)) { + if (exp.experiment_id === experimentId) continue; + + log.debug(`Trying earlier experiment ${exp.experiment_id}...`); + scores = await evalsClient.getExperimentScores(exp.experiment_id, { + suiteId, + taskModelId: modelScores.modelId, + executionId: exp.execution_id ?? exp.experiment_id, + }); + + completeCount = processScoreBatch(scores, modelScores.modelId, suiteId, traces); + + if (completeCount > 0) { + log.debug( + `Found ${completeCount} complete score(s) in earlier experiment ${exp.experiment_id}` + ); + break; + } + } + } + + if (completeCount === 0) { + log.warning( + `No complete score documents found for suite ${suiteId} (model ${modelScores.modelId}) ` + + `across ${MAX_EXPERIMENTS_TO_SCAN} experiments — trace will be unavailable` + ); + } + } + } + + log.debug(`Matrix traces resolved ${Object.keys(traces).length} trace entries`); + return traces; +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix.test.ts new file mode 100644 index 0000000000000..11aa0fd5e91b9 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix.test.ts @@ -0,0 +1,299 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { renderMatrix } from './render_matrix'; +import { parseMatrixConfig } from './load_matrix_config'; +import type { Matrix } from './build_matrix'; +import { buildMatrix, OVERALL_COLUMN_ID } from './build_matrix'; +import type { AggregatedModelScores } from './query_matrix_scores'; + +const config = parseMatrixConfig({ + title: 'Test Matrix', + columns: [ + { id: 'triage', label: 'Alert Triage', suites: ['a'] }, + { id: 'detect', label: 'Detection Engineering', suites: ['b'] }, + ], + models: [{ id: 'm', label: 'M' }], +}); + +const matrix: Matrix = { + columns: [ + { id: 'triage', label: 'Alert Triage' }, + { id: 'detect', label: 'Detection Engineering' }, + ], + composites: [], + displayColumns: [ + { id: 'triage', label: 'Alert Triage', kind: 'base' }, + { id: 'detect', label: 'Detection Engineering', kind: 'base' }, + { id: OVERALL_COLUMN_ID, label: 'Overall', kind: 'overall' }, + ], + overallLabel: 'Overall', + proprietary: [ + { + modelId: 'claude', + modelLabel: 'Claude Sonnet 4', + openSource: false, + cells: { + triage: { kind: 'score', value: 9.2 }, + detect: { kind: 'not-recommended' }, + }, + overall: { kind: 'score', value: 4.6 }, + }, + ], + openSource: [ + { + modelId: 'oss', + modelLabel: 'GPT OSS 120B', + openSource: true, + cells: { + triage: { kind: 'score', value: 7.6 }, + detect: { kind: 'missing' }, + }, + overall: { kind: 'score', value: 3.8 }, + }, + ], +}; + +describe('renderMatrix', () => { + it('renders CSV with a header row and one row per model', () => { + const { proprietaryCsv, openSourceCsv } = renderMatrix(matrix, config); + + expect(proprietaryCsv.split('\n')[0]).toBe('Model,Alert Triage,Detection Engineering,Overall'); + expect(proprietaryCsv).toContain('Claude Sonnet 4,9.2,Not recommended,4.6'); + // Missing cells render as empty fields. + expect(openSourceCsv).toContain('GPT OSS 120B,7.6,,3.8'); + }); + + it('renders markdown with proprietary and open-source sections', () => { + const { markdown } = renderMatrix(matrix, config); + + expect(markdown).toContain('# Test Matrix'); + expect(markdown).toContain('## Proprietary models'); + expect(markdown).toContain('## Open-source models'); + expect(markdown).toContain('| Claude Sonnet 4 | 9.2 | Not recommended | 4.6 |'); + }); + + it('produces valid JSON with the matrix structure', () => { + const { json } = renderMatrix(matrix, config); + const parsed = JSON.parse(json); + + expect(parsed.title).toBe('Test Matrix'); + expect(parsed.proprietary).toHaveLength(1); + expect(parsed.openSource[0].modelLabel).toBe('GPT OSS 120B'); + }); + + it('renders composite columns in displayColumns order (no trailing legacy overall)', () => { + const compositeMatrix: Matrix = { + columns: [ + { id: 'a', label: 'Alert Triage', group: 'Agent Builder' }, + { id: 'b', label: 'Investigation', group: 'Agent Builder' }, + ], + composites: [ + { id: 'ab', label: 'Agent Builder Score' }, + { id: 'overall_score', label: 'Overall Score' }, + ], + displayColumns: [ + { id: 'a', label: 'Alert Triage', group: 'Agent Builder', kind: 'base' }, + { id: 'b', label: 'Investigation', group: 'Agent Builder', kind: 'base' }, + { id: 'ab', label: 'Agent Builder Score', kind: 'composite' }, + { id: 'overall_score', label: 'Overall Score', kind: 'composite' }, + ], + overallLabel: 'Overall', + proprietary: [ + { + modelId: 'm', + modelLabel: 'Claude', + openSource: false, + cells: { + a: { kind: 'score', value: 8.6 }, + b: { kind: 'score', value: 7.4 }, + ab: { kind: 'score', value: 8 }, + overall_score: { kind: 'score', value: 8 }, + }, + overall: { kind: 'score', value: 8 }, + }, + ], + openSource: [], + }; + + const { proprietaryCsv } = renderMatrix(compositeMatrix, config); + // No trailing "Overall" column; composites appear in declared layout order. + expect(proprietaryCsv.split('\n')[0]).toBe( + 'Model,Alert Triage,Investigation,Agent Builder Score,Overall Score' + ); + expect(proprietaryCsv).toContain('Claude,8.6,7.4,8,8'); + }); + + it('escapes CSV fields that contain commas or quotes', () => { + const cfgWithComma = parseMatrixConfig({ + title: 'X', + columns: [{ id: 'c', label: 'Col, with comma', suites: ['a'] }], + models: [{ id: 'm', label: 'M' }], + }); + const m: Matrix = { + columns: [{ id: 'c', label: 'Col, with comma' }], + composites: [], + displayColumns: [ + { id: 'c', label: 'Col, with comma', kind: 'base' }, + { id: OVERALL_COLUMN_ID, label: 'Overall', kind: 'overall' }, + ], + overallLabel: 'Overall', + proprietary: [ + { + modelId: 'm', + modelLabel: 'M', + openSource: false, + cells: { c: { kind: 'score', value: 1 } }, + overall: { kind: 'score', value: 1 }, + }, + ], + openSource: [], + }; + + const { proprietaryCsv } = renderMatrix(m, cfgWithComma); + expect(proprietaryCsv.split('\n')[0]).toBe('Model,"Col, with comma",Overall'); + }); +}); + +describe('renderMatrix token axis', () => { + const tokenConfig = parseMatrixConfig({ + title: 'Token Matrix', + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'], weight: 1 }], + models: [{ id: 'm1', label: 'M1' }], + tokenCost: {}, + }); + + const withTokens: AggregatedModelScores[] = [ + { + modelId: 'm1', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'r1', + datasets: [ + { + datasetId: 'd1', + datasetName: 'D1', + evaluators: [ + { evaluatorName: 'correctness', mean: 0.9, count: 2 }, + { + evaluatorName: 'Input Tokens', + mean: 120_000, + count: 2, + min: 90_000, + max: 150_000, + }, + { evaluatorName: 'Output Tokens', mean: 3_000, count: 2, min: 2_000, max: 4_000 }, + ], + }, + ], + }, + ], + }, + ]; + + it('serializes tokenCost into matrix.json', () => { + const { json } = renderMatrix(buildMatrix(withTokens, tokenConfig), tokenConfig); + const parsed = JSON.parse(json); + + expect(parsed.tokenCost.models).toHaveLength(1); + expect(parsed.tokenCost.models[0].modelId).toBe('m1'); + expect(parsed.tokenCost.models[0].cells[0]).toEqual({ + columnId: 'triage', + inputTokens: { mean: 120_000, min: 90_000, max: 150_000, count: 2 }, + outputTokens: { mean: 3_000, min: 2_000, max: 4_000, count: 2 }, + totalMean: 123_000, + }); + }); + + it('omits the tokenCost key entirely when not configured', () => { + const plain = parseMatrixConfig({ + title: 'Plain', + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'], weight: 1 }], + models: [{ id: 'm1', label: 'M1' }], + }); + const { json } = renderMatrix(buildMatrix(withTokens, plain), plain); + expect(JSON.parse(json)).not.toHaveProperty('tokenCost'); + }); +}); + +describe('renderMatrix provenance', () => { + const provConfig = parseMatrixConfig({ + title: 'Prov Matrix', + columns: [{ id: 'triage', label: 'Triage', suites: ['suite-a'], weight: 1 }], + models: [{ id: 'm1', label: 'M1' }], + }); + + const scores: AggregatedModelScores[] = [ + { + modelId: 'm1', + suites: [ + { + suiteId: 'suite-a', + experimentId: 'r1', + datasets: [ + { + datasetId: 'd1', + datasetName: 'D1', + evaluators: [{ evaluatorName: 'correctness', mean: 0.9, count: 2 }], + }, + ], + }, + ], + }, + ]; + + const render = (provenance?: Parameters[2]) => + renderMatrix(buildMatrix(scores, provConfig), provConfig, provenance); + + it('stamps the filters that produced the numbers into markdown and json', () => { + const { markdown, json } = render({ + branch: 'main', + lookbackDays: 14, + suiteIds: ['suite-a'], + commitSha: 'abc123', + buildUrl: 'https://buildkite.com/b/1', + }); + + expect(markdown).toContain('branch `main`'); + expect(markdown).toContain('14-day lookback'); + expect(markdown).toContain('commit `abc123`'); + expect(markdown).toContain('[build](https://buildkite.com/b/1)'); + + const parsed = JSON.parse(json); + expect(parsed.provenance).toEqual({ + branch: 'main', + lookbackDays: 14, + suiteIds: ['suite-a'], + commitSha: 'abc123', + buildUrl: 'https://buildkite.com/b/1', + }); + expect(parsed.generatedAt).toEqual(expect.any(String)); + }); + + it('omits unknown fields rather than stamping placeholders', () => { + const { markdown, json } = render({ branch: 'main', lookbackDays: 7 }); + + expect(markdown).toContain('branch `main`'); + expect(markdown).not.toContain('commit'); + expect(markdown).not.toContain('undefined'); + expect(JSON.parse(json).provenance).toEqual({ branch: 'main', lookbackDays: 7 }); + }); + + it('still renders a dated line when no provenance is supplied', () => { + const { markdown, json } = render(); + + expect(markdown).toContain('Generated '); + expect(markdown).not.toContain('undefined'); + expect(JSON.parse(json).provenance).toEqual({}); + }); + + it('uses one timestamp for both markdown and json', () => { + const { markdown, json } = render(); + expect(markdown).toContain(`Generated ${JSON.parse(json).generatedAt}`); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix.ts new file mode 100644 index 0000000000000..2b1b586d39745 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix.ts @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MatrixConfig } from './load_matrix_config'; +import type { Matrix, MatrixCell, MatrixDisplayColumn, MatrixRow } from './build_matrix'; + +/** + * Where the numbers came from. Without this, a published matrix is an + * undated table of scores with no way to tell which eval run, branch, or + * lookback window produced it — so a stale artifact is indistinguishable + * from a fresh one. + */ +export interface MatrixProvenance { + /** Branch filter applied to the query (undefined = any branch). */ + branch?: string; + /** Lookback window in days used to select experiments. */ + lookbackDays?: number; + /** Suite ids the scores were drawn from. */ + suiteIds?: string[]; + /** Commit the generator ran against, when known. */ + commitSha?: string; + /** CI build URL that produced the artifact, when known. */ + buildUrl?: string; +} + +export interface RenderedMatrix { + /** CSV for the proprietary-models table (first row = header). */ + proprietaryCsv: string; + /** CSV for the open-source-models table (first row = header). */ + openSourceCsv: string; + /** Combined human-readable markdown document. */ + markdown: string; + /** Structured JSON artifact (machine-readable). */ + json: string; +} + +const cellToString = (cell: MatrixCell, notRecommendedLabel: string): string => { + switch (cell.kind) { + case 'score': + return String(cell.value); + case 'not-recommended': + return notRecommendedLabel; + case 'missing': + default: + return ''; + } +}; + +/** Escapes a value for inclusion in a CSV field per RFC 4180. */ +const csvEscape = (value: string): string => { + if (/[",\n]/.test(value)) { + return `"${value.replace(/"/g, '""')}"`; + } + return value; +}; + +const cellForColumn = (row: MatrixRow, column: MatrixDisplayColumn): MatrixCell => + column.kind === 'overall' ? row.overall : row.cells[column.id] ?? { kind: 'missing' }; + +const buildHeader = (displayColumns: MatrixDisplayColumn[]): string[] => [ + 'Model', + ...displayColumns.map((column) => column.label), +]; + +const rowToValues = ( + displayColumns: MatrixDisplayColumn[], + row: MatrixRow, + notRecommendedLabel: string +): string[] => [ + row.modelLabel, + ...displayColumns.map((column) => cellToString(cellForColumn(row, column), notRecommendedLabel)), +]; + +const renderCsv = ( + displayColumns: MatrixDisplayColumn[], + rows: MatrixRow[], + notRecommendedLabel: string +): string => { + const lines = [ + buildHeader(displayColumns), + ...rows.map((row) => rowToValues(displayColumns, row, notRecommendedLabel)), + ]; + return lines.map((cells) => cells.map(csvEscape).join(',')).join('\n') + '\n'; +}; + +const renderMarkdownTable = ( + displayColumns: MatrixDisplayColumn[], + rows: MatrixRow[], + notRecommendedLabel: string +): string => { + const header = buildHeader(displayColumns); + const separator = header.map(() => ':---'); + const body = rows.map((row) => rowToValues(displayColumns, row, notRecommendedLabel)); + + const toRow = (cells: string[]): string => `| ${cells.join(' | ')} |`; + + return [toRow(header), toRow(separator), ...body.map(toRow)].join('\n'); +}; + +export const renderMatrix = ( + matrix: Matrix, + config: MatrixConfig, + provenance: MatrixProvenance = {} +): RenderedMatrix => { + const { notRecommendedLabel } = config; + const displayColumns = matrix.displayColumns; + const generatedAt = new Date().toISOString(); + + const proprietaryCsv = renderCsv(displayColumns, matrix.proprietary, notRecommendedLabel); + const openSourceCsv = renderCsv(displayColumns, matrix.openSource, notRecommendedLabel); + + // Rendered as a plain line rather than a comment so it survives into the + // published docs — a provenance footer nobody can see defeats the purpose. + const provenanceLine = [ + `Generated ${generatedAt}`, + provenance.branch ? `branch \`${provenance.branch}\`` : undefined, + provenance.lookbackDays !== undefined ? `${provenance.lookbackDays}-day lookback` : undefined, + provenance.commitSha ? `commit \`${provenance.commitSha}\`` : undefined, + provenance.buildUrl ? `[build](${provenance.buildUrl})` : undefined, + ] + .filter(Boolean) + .join(' · '); + + const markdown = [ + `# ${config.title}`, + '', + provenanceLine, + '', + 'Higher scores indicate better performance. A score of 10 on a task means the model met or exceeded all task-specific benchmarks. ' + + `Models with a score of "${notRecommendedLabel}" failed testing.`, + '', + '## Proprietary models', + '', + matrix.proprietary.length > 0 + ? renderMarkdownTable(displayColumns, matrix.proprietary, notRecommendedLabel) + : '_No proprietary models with results._', + '', + '## Open-source models', + '', + matrix.openSource.length > 0 + ? renderMarkdownTable(displayColumns, matrix.openSource, notRecommendedLabel) + : '_No open-source models with results._', + '', + ].join('\n'); + + const json = JSON.stringify( + { + title: config.title, + generatedAt, + provenance, + columns: matrix.columns, + composites: matrix.composites ?? [], + displayColumns, + overallLabel: matrix.overallLabel, + proprietary: matrix.proprietary, + openSource: matrix.openSource, + ...(matrix.tokenCost ? { tokenCost: matrix.tokenCost } : {}), + }, + null, + 2 + ); + + return { proprietaryCsv, openSourceCsv, markdown, json }; +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.test.ts new file mode 100644 index 0000000000000..ece0ec2b0ac8d --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.test.ts @@ -0,0 +1,143 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { renderMatrixHtml } from './render_matrix_html'; +import type { Matrix } from './build_matrix'; +import type { MatrixConfig } from './load_matrix_config'; +import type { MatrixTraceData } from './trace_types'; + +const mockConfig: MatrixConfig = { + title: 'Test Matrix', + branch: 'main', + lookbackDays: 30, + defaultScale: 10, + decimals: 2, + notRecommendedBelow: 0, + notRecommendedLabel: 'Not recommended', + notRecommendedCountsAsZeroInOverall: true, + excludeEvaluators: [], + overall: { label: 'Overall', mode: 'weighted' }, + showOverall: true, + columns: [{ id: 'alert', label: 'Alert Analysis', suites: ['suite-1'], weight: 1 }], + composites: [], + models: [{ id: 'test-model', label: 'Test Model', openSource: false }], +}; + +const mockMatrix: Matrix = { + columns: [{ id: 'alert', label: 'Alert Analysis' }], + composites: [], + displayColumns: [ + { id: 'alert', label: 'Alert Analysis', kind: 'base' }, + { id: '__overall__', label: 'Overall', kind: 'overall' }, + ], + overallLabel: 'Overall', + proprietary: [ + { + modelId: 'test-model', + modelLabel: 'Test Model', + openSource: false, + cells: { alert: { kind: 'score', value: 8.5 } }, + overall: { kind: 'score', value: 8.5 }, + }, + ], + openSource: [], +}; + +describe('renderMatrixHtml', () => { + it('renders a self-contained HTML document', () => { + const html = renderMatrixHtml(mockMatrix, mockConfig); + expect(html).toContain(''); + expect(html).toContain(' + + +
+

${esc(config.title)}

+

${provenanceLine}

+${summaryTable} +

Each cell shows the model's score (0–10). Expand a prompt below to read the agent's full answer, tool trail, and reasoning trace.

+${modelCards} +
+ +`; +}; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/trace_types.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/trace_types.ts new file mode 100644 index 0000000000000..bf4afbb20415d --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/trace_types.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** A single step in the agent's reasoning + tool-call trace. */ +export interface TraceStep { + type: 'reasoning' | 'tool' | 'skill'; + /** Reasoning text (for `type: 'reasoning'`). */ + text?: string; + /** Tool call ID (for `type: 'tool'`). */ + toolId?: string; + /** Tool call parameters (for `type: 'tool'`). */ + toolParams?: string; + /** Skill IDs selected (for `type: 'skill'`). */ + skills?: string[]; +} + +/** Trace data for a single (model, column) pair. */ +export interface MatrixTraceEntry { + /** The initial user question from the eval dataset. */ + question?: string; + /** Ordered list of tool IDs the agent called. */ + toolTrail?: string[]; + /** The agent's final answer (markdown). */ + answer?: string; + /** Full reasoning + tool-call step trace. */ + steps?: TraceStep[]; + /** Number of steps (cached for summary table). */ + stepCount?: number; + /** Number of tool calls (cached for summary table). */ + toolCount?: number; +} + +/** + * Map of trace entries keyed by `${modelId}:${columnId}`. + * Lookups use the same model/column IDs as the matrix config. + */ +export type MatrixTraceData = Record; + +/** Build the trace-data lookup key. */ +export const traceKey = (modelId: string, columnId: string): string => `${modelId}:${columnId}`; diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/tsconfig.json b/x-pack/platform/packages/shared/kbn-evals-extensions/tsconfig.json index ca398649905c9..7336860d4f82d 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/tsconfig.json +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/tsconfig.json @@ -11,6 +11,9 @@ "@kbn/evals-common", "@kbn/dev-cli-runner", "@kbn/dev-cli-errors", - "@kbn/tooling-log" + "@kbn/tooling-log", + "@kbn/config-schema", + "@kbn/some-dev-log", + "@kbn/kbn-client" ] } diff --git a/x-pack/platform/packages/shared/kbn-evals/index.ts b/x-pack/platform/packages/shared/kbn-evals/index.ts index d1487add82cea..4a437a91d13b5 100644 --- a/x-pack/platform/packages/shared/kbn-evals/index.ts +++ b/x-pack/platform/packages/shared/kbn-evals/index.ts @@ -123,11 +123,20 @@ export type { export { createTable } from './src/utils/reporting/report_table'; export { EvalsClient, + MAX_LIST_EXPERIMENTS, type EvaluatorStats, type ExperimentStats, type UpsertDatasetInput, type DatasetWithId, + type ListExperimentsFilters, } from './src/utils/evals_client'; +export { + createEvaluationsEvalsClient, + getEvaluationsKbnClient, + DEFAULT_EVALUATIONS_KBN_URL, + type CreateEvaluationsEvalsClientParams, +} from './src/utils/evaluations_kbn_client'; +export { envFromDatasetsProfile } from './src/cli/profiles'; export { EvaluatorApiClient, type MapContextFn } from './src/utils/evaluator_api_client'; export { getBuildkiteCiMetadataFromEnv, type BuildkiteCiMetadata } from './src/utils/ci_metadata'; export { buildIngestRequest } from './src/utils/build_ingest_request'; diff --git a/x-pack/platform/packages/shared/kbn-evals/src/utils/evals_client.ts b/x-pack/platform/packages/shared/kbn-evals/src/utils/evals_client.ts index bc78eeb5188d3..e3b56eac65b64 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/utils/evals_client.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/utils/evals_client.ts @@ -31,6 +31,7 @@ import { UpsertEvaluationDatasetResponse, getDatasetId, type DatasetMaturity, + type EvaluationExperimentSummary, type EvaluationScoreDocument, type IngestScoresRequestBodyInput, type Model as EvalsModel, @@ -183,6 +184,18 @@ const buildExperimentQuery = (options?: GetExperimentFilters) => ({ const VERSIONED_HEADERS = { 'elastic-api-version': API_VERSIONS.internal.v1 }; +export interface ListExperimentsFilters { + suiteId?: string; + taskModelId?: string; + branch?: string; + datasetId?: string; + buildId?: string; + /** Maximum number of experiments to return (newest first). Defaults to and capped at 100 (the route's per_page maximum). */ + limit?: number; +} + +export const MAX_LIST_EXPERIMENTS = 100; + export class EvalsClient { /** The spaces this run writes to, in the order they were listed. */ private readonly spaceIds: string[]; @@ -572,4 +585,28 @@ export class EvalsClient { return undefined; } } + + async listExperiments(filters?: ListExperimentsFilters): Promise { + const limit = Math.min(filters?.limit ?? MAX_LIST_EXPERIMENTS, MAX_LIST_EXPERIMENTS); + const response = await this.kbnClient.request({ + path: EVALS_EXPERIMENTS_URL, + method: 'GET', + query: { + suite_id: filters?.suiteId, + model_id: filters?.taskModelId, + branch: filters?.branch, + dataset_id: filters?.datasetId, + build_id: filters?.buildId, + page: 1, + per_page: limit, + }, + headers: VERSIONED_HEADERS, + }); + + const parsed = GetEvaluationExperimentsResponse.parse(getResponseData(response)); + if (!filters?.branch) { + return parsed.experiments; + } + return parsed.experiments.filter((experiment) => experiment.git_branch === filters.branch); + } } diff --git a/x-pack/platform/packages/shared/kbn-evals/src/utils/evaluations_kbn_client.ts b/x-pack/platform/packages/shared/kbn-evals/src/utils/evaluations_kbn_client.ts index 01688e7100050..d552d08e260f4 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/utils/evaluations_kbn_client.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/utils/evaluations_kbn_client.ts @@ -10,6 +10,42 @@ import type { KbnClient } from '@kbn/kbn-client'; import { KbnClient as TestKbnClient } from '@kbn/kbn-client'; import { wrapKbnClientWithRetries } from './kbn_client_with_retries'; +import { EvalsClient } from './evals_client'; + +/** + * Default target used when no evaluations Kibana URL is provided. Keeps + * `@kbn/kbn-client` an implementation detail of `@kbn/evals` so callers (e.g. + * `@kbn/evals-extensions`) can build a client without depending on it directly. + */ +export const DEFAULT_EVALUATIONS_KBN_URL = 'http://elastic:changeme@localhost:5601'; + +export interface CreateEvaluationsEvalsClientParams { + log: ToolingLog; + /** Evaluations Kibana URL (falls back to {@link DEFAULT_EVALUATIONS_KBN_URL}). */ + url?: string; + /** API key used to authenticate against a non-local target. */ + apiKey?: string; +} + +/** + * Thin factory wiring the default {@link TestKbnClient} through + * {@link getEvaluationsKbnClient} (URL/API-key/version/retry handling). + */ +export function createEvaluationsEvalsClient({ + log, + url, + apiKey, +}: CreateEvaluationsEvalsClientParams): EvalsClient { + const defaultKbnClient = new TestKbnClient({ log, url: DEFAULT_EVALUATIONS_KBN_URL }); + const kbnClient = getEvaluationsKbnClient({ + kbnClient: defaultKbnClient, + log, + evaluationsKbnUrl: url, + evaluationsKbnApiKey: apiKey, + }); + return new EvalsClient(kbnClient, log); +} + export interface GetEvaluationsKbnClientParams { kbnClient: KbnClient; log: ToolingLog; From 545ddd5737f0284adea8cda8ae47e15634771861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Kopycin=CC=81ski?= Date: Tue, 18 Aug 2026 23:49:06 +0200 Subject: [PATCH 002/258] [AO] address review feedback on #285833 --- .../src/matrix/render_matrix_html.test.ts | 30 ++++++++++++++++--- .../src/matrix/render_matrix_html.ts | 6 +++- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.test.ts index ece0ec2b0ac8d..9aac9cac5fe56 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.test.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.test.ts @@ -133,11 +133,33 @@ describe('renderMatrixHtml', () => { }; const html = renderMatrixHtml(matrixWithSuite, configWithSuite, {}, traces); + // The 'triage' column resolves its trace via the suite ID fallback. expect(html).toContain('Triage this alert'); expect(html).toContain('alert.load'); - // The 'triage' column should have a trace; the 'Overall' column won't. - // Check that at least the triage section shows the question, not 'Trace unavailable'. - expect(html).toContain('Triage this alert'); - expect(html).toContain('alert.load'); + }); + + it('strips non-http(s) link targets from markdown answers', () => { + const traces: MatrixTraceData = { + 'test-model:alert': { + question: 'q', + toolTrail: [], + answer: + 'Safe: [elastic](https://elastic.co) and [guide](http://example.com). ' + + 'Bad: [click](javascript:alert(1)) and [x](data:text/html;base64,AAAA).', + stepCount: 0, + toolCount: 0, + }, + }; + + const html = renderMatrixHtml(mockMatrix, mockConfig, {}, traces); + // Safe links render as anchors. + expect(html).toContain('elastic'); + expect(html).toContain('guide'); + // Dangerous schemes are not emitted into href attributes. + expect(html).not.toContain('href="javascript:'); + expect(html).not.toContain('href="data:'); + // The link text is preserved (dropped back to plain text). + expect(html).toContain('click'); + expect(html).toContain('x'); }); }); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.ts index 247b140734cd6..81ca66ad58455 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.ts @@ -105,7 +105,11 @@ const inlineMd = (s: string): string => s .replace(/\*\*(.+?)\*\*/g, '$1') .replace(/`([^`]+)`/g, '$1') - .replace(/\[([^\]]+)\]\(([^)]+)\)/g, '$1'); + .replace(/\[([^\]]+)\]\(([^)]+)\)/g, (_m, text: string, url: string) => + // Only allow http(s) links — the markdown source is untrusted model + // output, and escaped schemes like `javascript:` still execute. + /^https?:\/\//i.test(url) ? `${text}` : text + ); /** Minimal markdown → HTML (headings, lists, bold, code, hr). */ const mdToHtml = (md: string): string => { From 86e0051e2b31fd5d57366f3906776a0e2b0a295f Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Wed, 19 Aug 2026 12:39:02 +0200 Subject: [PATCH 003/258] [kbn-evals-extensions] Fix trace detail lookup: per-example scores route The per-experiment scores route strips unbounded fields (task.output, example.input) from responses, so the HTML matrix always rendered 'Trace unavailable'. Switch queryMatrixTraces to enumerate example IDs from the stripped response, fetch full documents once per example via the per-example scores route (which does not strip), and filter client-side by task.model.id + metadata.execution_id. Adds EvalsClient.getExampleScores() wrapping EVALS_EXAMPLE_SCORES_URL. Verified against the golden cluster: 5/5 models now render question, tool trail, full step trace, and final answer in matrix.html. --- .../src/matrix/query_matrix_traces.ts | 191 ++++++++++-------- .../kbn-evals/src/utils/evals_client.ts | 36 ++++ 2 files changed, 143 insertions(+), 84 deletions(-) diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.ts index 2b9199d7dc71a..6a7ddb0e2b9d0 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.ts @@ -12,9 +12,6 @@ import type { AggregatedModelScores } from './query_matrix_scores'; import type { MatrixTraceData, MatrixTraceEntry, TraceStep } from './trace_types'; import { traceKey } from './trace_types'; -/** Maximum number of experiments to scan for complete scores per (suite, model). */ -const MAX_EXPERIMENTS_TO_SCAN = 5; - /** * Extracts trace data (initial question, tool trail, agent answer, step trace) * from a single evaluation score document. @@ -98,50 +95,60 @@ const isCompleteScore = (score: EvaluationScoreDocument): boolean => { }; /** - * Processes a batch of score documents (all from the same experiment) and - * populates `traces` with per-example and per-suite entries. Only the latest - * complete score per suite is used. - * - * @returns The number of complete scores found in this batch. + * Merges a batch of full (unstripped) score documents for one example into + * `traces`. Documents arrive in unknown order and span every experiment and + * repetition that ever scored this example, so this filters to the requested + * model + execution and keeps the newest complete document. */ -const processScoreBatch = ( +const processExampleBatch = ( scores: EvaluationScoreDocument[], modelId: string, suiteId: string, + executionId: string, traces: MatrixTraceData -): number => { - // Sort by @timestamp descending so the latest complete run wins - const sorted = [...scores].sort((a, b) => { - const ta = Date.parse(a['@timestamp'] ?? ''); - const tb = Date.parse(b['@timestamp'] ?? ''); - return tb - ta; - }); - - let suiteTraceAssigned = false; - let completeCount = 0; - - for (const score of sorted) { - const entry = extractTraceFromScore(score); - const datasetId = score.example?.dataset?.id; - const exampleId = score.example?.id; - - // Key by model:exampleId for per-prompt detail - if (exampleId) { - traces[traceKey(modelId, exampleId)] = entry; - } - if (datasetId) { - traces[traceKey(modelId, datasetId)] = entry; - } +): boolean => { + // Filter to this model + this execution, newest first + const relevant = scores + .filter((score) => score.task?.model?.id === modelId) + .filter((score) => score.metadata?.execution_id === executionId) + .sort((a, b) => { + const ta = Date.parse(a['@timestamp'] ?? ''); + const tb = Date.parse(b['@timestamp'] ?? ''); + return tb - ta; + }); + + if (relevant.length === 0) return false; + + const entry = extractTraceFromScore(relevant[0]); + const exampleId = relevant[0].example?.id; + const datasetId = relevant[0].example?.dataset?.id; + + const complete = isCompleteScore(relevant[0]); + + // Key by model:exampleId for per-prompt detail + if (exampleId) { + traces[traceKey(modelId, exampleId)] = entry; + } + if (datasetId) { + traces[traceKey(modelId, datasetId)] = entry; + } + // For the suite-level key, only complete runs qualify. + if (complete) { + traces[traceKey(modelId, suiteId)] = entry; + } - // For the suite-level key, pick the latest **complete** run only. - if (!suiteTraceAssigned && isCompleteScore(score)) { - traces[traceKey(modelId, suiteId)] = entry; - suiteTraceAssigned = true; - completeCount++; + // If the newest doc is incomplete, fall back to the newest complete one + if (!complete) { + const firstComplete = relevant.find(isCompleteScore); + if (firstComplete) { + const fallbackEntry = extractTraceFromScore(firstComplete); + const fid = firstComplete.example?.id; + if (fid) traces[traceKey(modelId, fid)] = fallbackEntry; + traces[traceKey(modelId, suiteId)] = fallbackEntry; } } - return completeCount; + return complete; }; /** @@ -149,13 +156,15 @@ const processScoreBatch = ( * plugin and extracts trace data (initial question, tool trail, agent answer, * step trace) for each (model, column) pair. * - * Uses the same experiment IDs resolved by `queryMatrixScores` to fetch the - * full score documents — which include `task.output.steps` and - * `example.input.question` — and maps them into `MatrixTraceData`. + * The per-experiment scores route (`getExperimentScores`) strips unbounded + * fields (`task.output`, `example.input`, `example.metadata`) from responses, + * which are exactly the fields the trace detail needs. This function instead + * uses the per-example scores route (`getExampleScores`), which returns full + * documents, and filters client-side by model and execution. * - * If the latest experiment has no complete scores (e.g. worker died mid-run), - * it falls back to scanning up to `MAX_EXPERIMENTS_TO_SCAN` earlier - * experiments for the same (suite, model) to find the latest complete run. + * Example IDs are enumerated from the stripped per-experiment response (which + * still carries `example.id` and `example.dataset.id`), then each example is + * fetched once and reused across all models that ran it. */ export const queryMatrixTraces = async ( evalsClient: EvalsClient, @@ -164,62 +173,76 @@ export const queryMatrixTraces = async ( ): Promise => { const traces: MatrixTraceData = {}; + // 1. Collect (suite, model, execution) tuples with the example IDs they ran, + // from the stripped experiment-scores responses (cheap, no heavy fields). + interface RunRef { + suiteId: string; + modelId: string; + executionId: string; + exampleIds: Set; + } + const runRefs: RunRef[] = []; + for (const modelScores of aggregated) { for (const suite of modelScores.suites) { const { suiteId, experimentId } = suite; log.debug( - `Fetching score documents for experiment ${experimentId} (model ${modelScores.modelId}, suite ${suiteId})` + `Enumerating examples for experiment ${experimentId} (model ${modelScores.modelId}, suite ${suiteId})` ); - // First try the latest experiment - let scores = await evalsClient.getExperimentScores(experimentId, { + const stripped = await evalsClient.getExperimentScores(experimentId, { suiteId, taskModelId: modelScores.modelId, executionId: experimentId, }); - let completeCount = processScoreBatch(scores, modelScores.modelId, suiteId, traces); - - // If no complete scores found, scan earlier experiments - if (completeCount === 0) { - log.debug( - `No complete scores in latest experiment ${experimentId}, scanning earlier runs...` - ); - - const experiments = await evalsClient.listExperiments({ - suiteId, - taskModelId: modelScores.modelId, - limit: MAX_EXPERIMENTS_TO_SCAN, - }); - - // Skip the first one (already tried), try the rest in order - for (const exp of experiments.slice(1)) { - if (exp.experiment_id === experimentId) continue; - - log.debug(`Trying earlier experiment ${exp.experiment_id}...`); - scores = await evalsClient.getExperimentScores(exp.experiment_id, { - suiteId, - taskModelId: modelScores.modelId, - executionId: exp.execution_id ?? exp.experiment_id, - }); - - completeCount = processScoreBatch(scores, modelScores.modelId, suiteId, traces); - - if (completeCount > 0) { - log.debug( - `Found ${completeCount} complete score(s) in earlier experiment ${exp.experiment_id}` - ); - break; - } - } + const exampleIds = new Set(); + for (const score of stripped) { + if (score.example?.id) exampleIds.add(score.example.id); } - if (completeCount === 0) { + if (exampleIds.size === 0) { log.warning( - `No complete score documents found for suite ${suiteId} (model ${modelScores.modelId}) ` + - `across ${MAX_EXPERIMENTS_TO_SCAN} experiments — trace will be unavailable` + `No example IDs found for suite ${suiteId} (model ${modelScores.modelId}) — trace will be unavailable` ); + continue; } + + runRefs.push({ + suiteId, + modelId: modelScores.modelId, + executionId: experimentId, + exampleIds, + }); + } + } + + // 2. Fetch full score documents once per example (the route returns every + // document for that example; we share the result across runs). + const exampleScores = new Map(); + const allExampleIds = new Set(); + for (const ref of runRefs) { + for (const id of ref.exampleIds) allExampleIds.add(id); + } + + for (const exampleId of allExampleIds) { + const scores = await evalsClient.getExampleScores(exampleId); + exampleScores.set(exampleId, scores); + } + + // 3. For each run, pick the newest complete document per example and merge + // into the traces map. + for (const ref of runRefs) { + let completeCount = 0; + for (const exampleId of ref.exampleIds) { + const scores = exampleScores.get(exampleId) ?? []; + const ok = processExampleBatch(scores, ref.modelId, ref.suiteId, ref.executionId, traces); + if (ok) completeCount++; + } + if (completeCount === 0) { + log.warning( + `No complete score documents found for suite ${ref.suiteId} (model ${ref.modelId}, execution ${ref.executionId}) — trace will be unavailable` + ); } } diff --git a/x-pack/platform/packages/shared/kbn-evals/src/utils/evals_client.ts b/x-pack/platform/packages/shared/kbn-evals/src/utils/evals_client.ts index e3b56eac65b64..5fccf54ad91ae 100644 --- a/x-pack/platform/packages/shared/kbn-evals/src/utils/evals_client.ts +++ b/x-pack/platform/packages/shared/kbn-evals/src/utils/evals_client.ts @@ -18,11 +18,13 @@ import { EVALS_EXPERIMENT_SCORES_URL, EVALS_EXPERIMENT_URL, EVALS_EXPERIMENTS_URL, + EVALS_EXAMPLE_SCORES_URL, EVALS_SCORES_URL, GetEvaluationDatasetResponse, GetEvaluationExperimentResponse, GetEvaluationExperimentScoresResponse, GetEvaluationExperimentsResponse, + GetExampleScoresResponse, IngestScoresRequestBody, IngestScoresResponse, MAX_SCORES_PER_QUERY, @@ -308,6 +310,40 @@ export class EvalsClient { } } + /** + * Retrieves scores for a single example across all experiments that include + * it. Unlike {@link getExperimentScores}, the response is NOT stripped of + * unbounded fields (`task.output`, `example.input`, `example.metadata`), + * because this route does not apply `_source_excludes`. + */ + async getExampleScores(exampleId: string): Promise { + try { + const response = await this.kbnClient.request({ + path: this.path( + EVALS_EXAMPLE_SCORES_URL.replace('{exampleId}', encodeURIComponent(exampleId)) + ), + method: 'GET', + headers: VERSIONED_HEADERS, + }); + const parsed = GetExampleScoresResponse.parse(getResponseData(response)); + + if (parsed.total > MAX_SCORES_PER_QUERY) { + throw new Error( + `Example ${exampleId} returned ${parsed.total} scores, which exceeds MAX_SCORES_PER_QUERY (${MAX_SCORES_PER_QUERY})` + ); + } + + return parsed.scores; + } catch (error: unknown) { + this.log.error( + `Failed to retrieve scores for example ID ${exampleId}: ${ + error instanceof Error ? error.message : String(error) + }` + ); + return []; + } + } + /** * Creates or updates a dataset and returns the id the server assigned it. Ids * derive from the owning space, so the caller can't compute one. From cd21d14d55d46cc84b034b3dc6c6dcb255e067aa Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Wed, 19 Aug 2026 14:00:57 +0200 Subject: [PATCH 004/258] [kbn-evals-extensions] Matrix: per-category columns via examplePrefixes Columns can now declare examplePrefixes to slice a single dataset into per-category columns (e.g. persona-matrix's 7 Agent Builder categories). queryMatrixScores fetches stripped per-example scores and synthesizes prefix: datasets; columnEvaluators maps examplePrefixes onto them. Dataset-id semantics unchanged for existing configs. --- .../src/cli/commands/matrix.ts | 5 + .../src/matrix/build_matrix.ts | 9 +- .../src/matrix/load_matrix_config.ts | 13 ++ .../src/matrix/query_matrix_scores.test.ts | 139 +++++++++++++++++- .../src/matrix/query_matrix_scores.ts | 85 ++++++++++- 5 files changed, 246 insertions(+), 5 deletions(-) diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.ts index 462a2a0d4f682..7e293a3945756 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.ts @@ -159,6 +159,11 @@ export const matrixCmd: Command = { modelIds, branch, lookbackDays, + examplePrefixes: [ + ...new Set( + config.columns.flatMap((column) => column.examplePrefixes ?? []) + ), + ], }); if (aggregated.length === 0) { diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.ts index 45f961ccd7251..08d5b2951f838 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.ts @@ -100,7 +100,14 @@ function* columnEvaluators( column: MatrixColumnConfig ): Generator { const suiteSet = new Set(column.suites); - const datasetSet = column.datasetIds ? new Set(column.datasetIds) : undefined; + // `examplePrefixes` columns consume the synthetic per-prefix datasets + // (datasetId `prefix:`) produced by queryMatrixScores; a `datasetIds` + // column keeps its raw dataset-id semantics unchanged. + const datasetSet = column.examplePrefixes + ? new Set(column.examplePrefixes.map((prefix) => `prefix:${prefix}`)) + : column.datasetIds + ? new Set(column.datasetIds) + : undefined; for (const suite of modelScores.suites) { if (!suiteSet.has(suite.suiteId)) { diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/load_matrix_config.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/load_matrix_config.ts index 2edb830ff80ec..5466c680de023 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/load_matrix_config.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/load_matrix_config.ts @@ -48,6 +48,19 @@ const columnSchema = schema.object({ maxSize: MAX_ARRAY_SIZE, }) ), + /** + * Optional restriction to `example.id` prefixes. Splits a single dataset + * (all examples share one `example.dataset.id`) into per-category columns: + * e.g. ['alert-analysis'] matches examples `alert-analysis-a/b/c`. When set, + * the matrix query fetches per-example scores (stripped experiment-scores + * route) and buckets them by prefix instead of the dataset-level stats. + */ + examplePrefixes: schema.maybe( + schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { + minSize: 1, + maxSize: MAX_ARRAY_SIZE, + }) + ), /** Optional restriction to specific `evaluator.name` values. */ evaluators: schema.maybe( schema.arrayOf(schema.string({ minLength: 1, maxLength: MAX_STRING_LENGTH }), { diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.test.ts index 81683a160eb08..0a6719bfa0c7c 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.test.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.test.ts @@ -7,12 +7,13 @@ import type { SomeDevLog } from '@kbn/some-dev-log'; import { ToolingLog } from '@kbn/tooling-log'; -import type { EvaluationExperimentSummary } from '@kbn/evals-common'; +import type { EvaluationExperimentSummary, EvaluationScoreDocument } from '@kbn/evals-common'; import type { EvalsClient, ExperimentStats } from '@kbn/evals'; import { pickLatestExperimentPerModel, experimentStatsToDatasets, queryMatrixScores, + scoresByPrefixToDatasets, } from './query_matrix_scores'; const experiment = ( @@ -227,3 +228,139 @@ describe('queryMatrixScores', () => { expect(result).toEqual([]); }); }); + +describe('scoresByPrefixToDatasets', () => { + const score = (exampleId: string, evaluatorName: string, s: number) => + ({ + example: { id: exampleId, index: 0, dataset: { id: 'ds', name: 'DS' } }, + task: { model: { id: 'm1' }, trace_id: 't' }, + evaluator: { name: evaluatorName, score: s }, + metadata: {}, + }) as unknown as EvaluationScoreDocument; + + it('buckets docs by example.id prefix and computes per-evaluator means', () => { + const datasets = scoresByPrefixToDatasets( + [ + score('alert-analysis-a', 'correctness', 1), + score('alert-analysis-b', 'correctness', 0), + score('threat-hunting-a', 'correctness', 0.5), + score('threat-hunting-b', 'groundedness', 0.8), + ], + ['alert-analysis', 'threat-hunting'] + ); + + const byId = new Map(datasets.map((d) => [d.datasetId, d])); + expect(byId.get('prefix:alert-analysis')?.evaluators).toEqual([ + { evaluatorName: 'correctness', mean: 0.5, count: 2 }, + ]); + expect(byId.get('prefix:threat-hunting')?.evaluators).toEqual( + expect.arrayContaining([ + { evaluatorName: 'correctness', mean: 0.5, count: 1 }, + { evaluatorName: 'groundedness', mean: 0.8, count: 1 }, + ]) + ); + }); + + it('matches exact example ids and prefix-dash boundaries only', () => { + const datasets = scoresByPrefixToDatasets( + [score('alert-analysis', 'correctness', 1), score('alert-analysisx', 'correctness', 0)], + ['alert-analysis'] + ); + // 'alert-analysisx' must NOT match prefix 'alert-analysis' + expect(datasets).toHaveLength(1); + expect(datasets[0].evaluators[0].count).toBe(1); + }); + + it('skips docs without evaluator score', () => { + const datasets = scoresByPrefixToDatasets( + [{ ...score('alert-analysis-a', 'correctness', 1), evaluator: { name: 'x' } } as EvaluationScoreDocument], + ['alert-analysis'] + ); + expect(datasets).toEqual([]); + }); +}); + +describe('queryMatrixScores with examplePrefixes', () => { + const log = new ToolingLog() as unknown as SomeDevLog; + + const stats: ExperimentStats = { + taskModel: { id: 'm1' }, + evaluatorModel: { id: 'judge' }, + totalRepetitions: 1, + stats: [ + { + datasetId: 'd1', + datasetName: 'D1', + evaluatorName: 'correctness', + stats: { mean: 0.9, median: 0.9, stdDev: 0, min: 0.9, max: 0.9, count: 10 }, + }, + ], + }; + + const createClient = (): { client: EvalsClient; getExperimentScores: jest.Mock } => { + const listExperiments = jest.fn().mockResolvedValue([ + { + experiment_id: 'e1', + execution_id: 'x1', + timestamp: new Date().toISOString(), + task_model: { id: 'm1' }, + }, + ]); + const getExperimentStats = jest.fn().mockResolvedValue(stats); + const getExperimentScores = jest.fn().mockResolvedValue([ + { + example: { id: 'alert-analysis-a', index: 0, dataset: { id: 'd1', name: 'D1' } }, + task: { model: { id: 'm1' }, trace_id: 't' }, + evaluator: { name: 'correctness', score: 0.6 }, + metadata: {}, + }, + ]); + const client = { listExperiments, getExperimentStats, getExperimentScores } as unknown as EvalsClient; + return { client, getExperimentScores }; + }; + + it('fetches per-example scores and appends synthetic prefix datasets when prefixes requested', async () => { + const { client, getExperimentScores } = createClient(); + + const result = await queryMatrixScores(client, log, { + suiteIds: ['suite-a'], + modelIds: ['m1'], + examplePrefixes: ['alert-analysis'], + }); + + expect(getExperimentScores).toHaveBeenCalledWith('e1', expect.anything()); + const datasetIds = result[0].suites[0].datasets.map((d) => d.datasetId); + expect(datasetIds).toContain('d1'); + expect(datasetIds).toContain('prefix:alert-analysis'); + }); + + it('does not fetch per-example scores when no prefixes requested', async () => { + const { client, getExperimentScores } = createClient(); + + await queryMatrixScores(client, log, { suiteIds: ['suite-a'], modelIds: ['m1'] }); + + expect(getExperimentScores).not.toHaveBeenCalled(); + }); + + it('degrades gracefully when the scores route fails', async () => { + const listExperiments = jest.fn().mockResolvedValue([ + { + experiment_id: 'e1', + execution_id: 'x1', + timestamp: new Date().toISOString(), + task_model: { id: 'm1' }, + }, + ]); + const getExperimentStats = jest.fn().mockResolvedValue(stats); + const getExperimentScores = jest.fn().mockRejectedValue(new Error('route down')); + const client = { listExperiments, getExperimentStats, getExperimentScores } as unknown as EvalsClient; + + const result = await queryMatrixScores(client, log, { + suiteIds: ['suite-a'], + modelIds: ['m1'], + examplePrefixes: ['alert-analysis'], + }); + + expect(result[0].suites[0].datasets.map((d) => d.datasetId)).toEqual(['d1']); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.ts index 3c408331ac99a..478c8b4e8e60f 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.ts @@ -6,7 +6,7 @@ */ import type { SomeDevLog } from '@kbn/some-dev-log'; -import type { EvaluationExperimentSummary } from '@kbn/evals-common'; +import type { EvaluationExperimentSummary, EvaluationScoreDocument } from '@kbn/evals-common'; import { MAX_LIST_EXPERIMENTS, type EvalsClient, type ExperimentStats } from '@kbn/evals'; /** Aggregated evaluator score for a single dataset within a suite. */ @@ -50,8 +50,61 @@ export interface QueryMatrixScoresOptions { modelIds: string[]; branch?: string; lookbackDays?: number; + /** + * When any config column sets `examplePrefixes`, per-example score documents + * are fetched (stripped experiment-scores route — unbounded fields excluded) + * and bucketed into synthetic per-prefix datasets alongside the dataset-level + * stats, so columns can slice a single dataset by example category. + */ + examplePrefixes?: string[]; } +/** + * Buckets stripped per-example score documents into synthetic per-prefix + * datasets. Each doc carries `example.id` (e.g. `alert-analysis-b`) and + * `evaluator.{name,score}`; docs are grouped by prefix, then per-evaluator + * means are computed the same way the server-side stats route would. + * Pure for unit testing. + */ +export const scoresByPrefixToDatasets = ( + scores: EvaluationScoreDocument[], + prefixes: string[] +): AggregatedDatasetScores[] => { + const byPrefix = new Map>(); + + for (const doc of scores) { + const exampleId = doc.example?.id ?? ''; + const prefix = prefixes.find((p) => exampleId === p || exampleId.startsWith(`${p}-`)); + if (!prefix) { + continue; + } + const evaluatorName = doc.evaluator?.name; + const score = doc.evaluator?.score; + if (!evaluatorName || typeof score !== 'number') { + continue; + } + let evaluators = byPrefix.get(prefix); + if (!evaluators) { + evaluators = new Map(); + byPrefix.set(prefix, evaluators); + } + const agg = evaluators.get(evaluatorName) ?? { sum: 0, count: 0 }; + agg.sum += score; + agg.count += 1; + evaluators.set(evaluatorName, agg); + } + + return [...byPrefix.entries()].map(([prefix, evaluators]) => ({ + datasetId: `prefix:${prefix}`, + datasetName: prefix, + evaluators: [...evaluators.entries()].map(([evaluatorName, agg]) => ({ + evaluatorName, + mean: agg.sum / agg.count, + count: agg.count, + })), + })); +}; + /** * Selects, per task model, the most recent experiment from a list (typically * the newest experiments for one suite + model). Experiments without a model @@ -126,7 +179,13 @@ export const experimentStatsToDatasets = (stats: ExperimentStats): AggregatedDat export const queryMatrixScores = async ( evalsClient: EvalsClient, log: SomeDevLog, - { suiteIds, modelIds, branch, lookbackDays }: QueryMatrixScoresOptions + { + suiteIds, + modelIds, + branch, + lookbackDays, + examplePrefixes = [], + }: QueryMatrixScoresOptions ): Promise => { const byModel = new Map(); @@ -175,11 +234,31 @@ export const queryMatrixScores = async ( byModel.set(modelId, model); } + const datasets = experimentStatsToDatasets(stats); + // Per-prefix synthetic datasets: one extra stripped-scores fetch per + // (suite, model). Cheap — unbounded fields are excluded server-side. + if (examplePrefixes.length > 0) { + try { + const scores = await evalsClient.getExperimentScores(latest.experiment_id, { + suiteId, + taskModelId: modelId, + executionId: latest.execution_id ?? latest.experiment_id, + }); + datasets.push(...scoresByPrefixToDatasets(scores, examplePrefixes)); + } catch (error) { + log.warning( + `Per-prefix scores unavailable for experiment ${latest.experiment_id} (suite ${suiteId}, model ${modelId}): ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } + model.suites.push({ suiteId, experimentId: latest.experiment_id, timestamp: latest.timestamp, - datasets: experimentStatsToDatasets(stats), + datasets, }); } } From cd21c793c6bcc5ade5aedae34931841bf5188401 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:16:11 +0000 Subject: [PATCH 005/258] Changes from node scripts/check --- .../src/cli/commands/matrix.ts | 4 +--- .../src/matrix/build_matrix.ts | 4 ++-- .../src/matrix/query_matrix_scores.test.ts | 21 +++++++++++++++---- .../src/matrix/query_matrix_scores.ts | 12 ++++------- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.ts index 7e293a3945756..8ad3980a449ef 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/cli/commands/matrix.ts @@ -160,9 +160,7 @@ export const matrixCmd: Command = { branch, lookbackDays, examplePrefixes: [ - ...new Set( - config.columns.flatMap((column) => column.examplePrefixes ?? []) - ), + ...new Set(config.columns.flatMap((column) => column.examplePrefixes ?? [])), ], }); diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.ts index 08d5b2951f838..46e71bcd6ad82 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/build_matrix.ts @@ -106,8 +106,8 @@ function* columnEvaluators( const datasetSet = column.examplePrefixes ? new Set(column.examplePrefixes.map((prefix) => `prefix:${prefix}`)) : column.datasetIds - ? new Set(column.datasetIds) - : undefined; + ? new Set(column.datasetIds) + : undefined; for (const suite of modelScores.suites) { if (!suiteSet.has(suite.suiteId)) { diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.test.ts index 0a6719bfa0c7c..f0af590e1062a 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.test.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.test.ts @@ -236,7 +236,7 @@ describe('scoresByPrefixToDatasets', () => { task: { model: { id: 'm1' }, trace_id: 't' }, evaluator: { name: evaluatorName, score: s }, metadata: {}, - }) as unknown as EvaluationScoreDocument; + } as unknown as EvaluationScoreDocument); it('buckets docs by example.id prefix and computes per-evaluator means', () => { const datasets = scoresByPrefixToDatasets( @@ -273,7 +273,12 @@ describe('scoresByPrefixToDatasets', () => { it('skips docs without evaluator score', () => { const datasets = scoresByPrefixToDatasets( - [{ ...score('alert-analysis-a', 'correctness', 1), evaluator: { name: 'x' } } as EvaluationScoreDocument], + [ + { + ...score('alert-analysis-a', 'correctness', 1), + evaluator: { name: 'x' }, + } as EvaluationScoreDocument, + ], ['alert-analysis'] ); expect(datasets).toEqual([]); @@ -315,7 +320,11 @@ describe('queryMatrixScores with examplePrefixes', () => { metadata: {}, }, ]); - const client = { listExperiments, getExperimentStats, getExperimentScores } as unknown as EvalsClient; + const client = { + listExperiments, + getExperimentStats, + getExperimentScores, + } as unknown as EvalsClient; return { client, getExperimentScores }; }; @@ -353,7 +362,11 @@ describe('queryMatrixScores with examplePrefixes', () => { ]); const getExperimentStats = jest.fn().mockResolvedValue(stats); const getExperimentScores = jest.fn().mockRejectedValue(new Error('route down')); - const client = { listExperiments, getExperimentStats, getExperimentScores } as unknown as EvalsClient; + const client = { + listExperiments, + getExperimentStats, + getExperimentScores, + } as unknown as EvalsClient; const result = await queryMatrixScores(client, log, { suiteIds: ['suite-a'], diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.ts index 478c8b4e8e60f..1d66fbf6d02b7 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_scores.ts @@ -179,13 +179,7 @@ export const experimentStatsToDatasets = (stats: ExperimentStats): AggregatedDat export const queryMatrixScores = async ( evalsClient: EvalsClient, log: SomeDevLog, - { - suiteIds, - modelIds, - branch, - lookbackDays, - examplePrefixes = [], - }: QueryMatrixScoresOptions + { suiteIds, modelIds, branch, lookbackDays, examplePrefixes = [] }: QueryMatrixScoresOptions ): Promise => { const byModel = new Map(); @@ -247,7 +241,9 @@ export const queryMatrixScores = async ( datasets.push(...scoresByPrefixToDatasets(scores, examplePrefixes)); } catch (error) { log.warning( - `Per-prefix scores unavailable for experiment ${latest.experiment_id} (suite ${suiteId}, model ${modelId}): ${ + `Per-prefix scores unavailable for experiment ${ + latest.experiment_id + } (suite ${suiteId}, model ${modelId}): ${ error instanceof Error ? error.message : String(error) }` ); From 0f57aa0ebf46b2301f288025413c02b56c1f1ad0 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Wed, 19 Aug 2026 15:03:36 +0200 Subject: [PATCH 006/258] [kbn-evals-extensions] Matrix HTML: group category columns under shared header When matrix columns share a config group (e.g. Agent Builder), the HTML summary table now renders a grouped thead row with colspan so the 7 persona matrix categories display under one group header. --- .../src/matrix/render_matrix_html.test.ts | 54 +++++++++++++++++-- .../src/matrix/render_matrix_html.ts | 31 +++++++++-- 2 files changed, 78 insertions(+), 7 deletions(-) diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.test.ts index 9aac9cac5fe56..f4fc068a7f694 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.test.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.test.ts @@ -22,16 +22,35 @@ const mockConfig: MatrixConfig = { excludeEvaluators: [], overall: { label: 'Overall', mode: 'weighted' }, showOverall: true, - columns: [{ id: 'alert', label: 'Alert Analysis', suites: ['suite-1'], weight: 1 }], + columns: [ + { + id: 'alert', + label: 'Alert Analysis', + group: 'Agent Builder', + suites: ['suite-1'], + weight: 1, + }, + { + id: 'threat', + label: 'Threat Hunting', + group: 'Agent Builder', + suites: ['suite-2'], + weight: 1, + }, + ], composites: [], models: [{ id: 'test-model', label: 'Test Model', openSource: false }], }; const mockMatrix: Matrix = { - columns: [{ id: 'alert', label: 'Alert Analysis' }], + columns: [ + { id: 'alert', label: 'Alert Analysis', group: 'Agent Builder' }, + { id: 'threat', label: 'Threat Hunting', group: 'Agent Builder' }, + ], composites: [], displayColumns: [ { id: 'alert', label: 'Alert Analysis', kind: 'base' }, + { id: 'threat', label: 'Threat Hunting', kind: 'base' }, { id: '__overall__', label: 'Overall', kind: 'overall' }, ], overallLabel: 'Overall', @@ -40,8 +59,11 @@ const mockMatrix: Matrix = { modelId: 'test-model', modelLabel: 'Test Model', openSource: false, - cells: { alert: { kind: 'score', value: 8.5 } }, - overall: { kind: 'score', value: 8.5 }, + cells: { + alert: { kind: 'score', value: 8.5 }, + threat: { kind: 'score', value: 7.4 }, + }, + overall: { kind: 'score', value: 7.95 }, }, ], openSource: [], @@ -138,6 +160,30 @@ describe('renderMatrixHtml', () => { expect(html).toContain('alert.load'); }); + it('renders grouped column headers when groups are present', () => { + const html = renderMatrixHtml(mockMatrix, mockConfig); + expect(html).toContain('colspan="2"'); + expect(html).toContain('Agent Builder'); + }); + + it('does not render a grouped header when no groups are present', () => { + const ungroupedConfig: MatrixConfig = { + ...mockConfig, + columns: [{ id: 'alert', label: 'Alert Analysis', suites: ['suite-1'], weight: 1 }], + }; + const ungroupedMatrix: Matrix = { + ...mockMatrix, + columns: [{ id: 'alert', label: 'Alert Analysis' }], + displayColumns: [ + { id: 'alert', label: 'Alert Analysis', kind: 'base' }, + { id: '__overall__', label: 'Overall', kind: 'overall' }, + ], + }; + const html = renderMatrixHtml(ungroupedMatrix, ungroupedConfig); + expect(html).not.toContain('colspan="2"'); + expect(html).not.toContain('Agent Builder'); + }); + it('strips non-http(s) link targets from markdown answers', () => { const traces: MatrixTraceData = { 'test-model:alert': { diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.ts index 81ca66ad58455..4ae222c4aa307 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.ts @@ -205,8 +205,33 @@ const stepHtml = (step: TraceStep, index: number): string => { )}`; }; -const renderSummaryTable = (matrix: Matrix): string => { - const header = `Model${matrix.displayColumns +const renderSummaryTable = (matrix: Matrix, config: MatrixConfig): string => { + const groupedColumns = matrix.displayColumns.map((col) => { + const source = col.kind === 'overall' ? undefined : config.columns.find((c) => c.id === col.id); + return { ...col, group: source?.group }; + }); + const hasGroups = groupedColumns.some((col) => col.group); + + const groupHeader = hasGroups + ? `${(() => { + let html = ''; + let i = 0; + while (i < groupedColumns.length) { + const group = groupedColumns[i].group; + let span = 1; + while (i + span < groupedColumns.length && groupedColumns[i + span].group === group) { + span += 1; + } + html += group + ? `${esc(group)}` + : ``; + i += span; + } + return html; + })()}` + : ''; + + const header = `${groupHeader}Model${matrix.displayColumns .map((c) => `${esc(c.label)}`) .join('')}`; const rows = [...matrix.proprietary, ...matrix.openSource] @@ -325,7 +350,7 @@ export const renderMatrixHtml = ( .filter(Boolean) .join(' · '); - const summaryTable = renderSummaryTable(matrix); + const summaryTable = renderSummaryTable(matrix, config); const modelCards = (matrix.proprietary.length > 0 ? renderModelCard(matrix.proprietary, matrix, config, traces) From 69c41cfa06291a4747ac61c59e23832c95519a20 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Wed, 19 Aug 2026 22:22:51 +0200 Subject: [PATCH 007/258] [Security Solution][kbn-evals] Match load_skill IDs in persona matrix --- .../src/evaluate_dataset.test.ts | 17 +++++++---------- .../src/evaluate_dataset.ts | 5 ++++- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.test.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.test.ts index 73ebdc31fd1a1..b63577c07a24b 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.test.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.test.ts @@ -236,14 +236,9 @@ describe('createPersonaMatrixSkillInvokedEvaluator', () => { expect(result.label).toBe('unavailable'); }); - it('matches the load_skill tool, not just the retired filestore.read', async () => { - // Regression guard. The agent loads skills via the `load_skill` tool: - // {"skill":"/skills///SKILL.md"} - // A predicate pinned to `filestore.read` can never match, so `skill_invoked` - // stays 0 while `total_tool_spans` is non-zero -- meaning the "unavailable" - // guard does NOT trip and the evaluator reports a confident false 0 for - // every model. Verified against the golden cluster: over 7 days, - // filestore.read = 0 spans, load_skill = 7,991 spans. + it('matches load_skill by skill ID while retaining the legacy SKILL.md path', async () => { + // `load_skill` accepts an ID or path. Current traces store {"skill":""}; + // older traces may contain a SKILL.md path. const query = jest.fn().mockResolvedValue({ columns: [{ name: 'total_tool_spans' }, { name: 'skill_invoked' }], values: [[2, 1]], @@ -253,13 +248,15 @@ describe('createPersonaMatrixSkillInvokedEvaluator', () => { log: buildLog(), }); - const result = await evaluator.evaluate( + await evaluator.evaluate( buildEvaluatorArgs(baseExample.metadata, '0af7651916cd43dd8448eb211c80319c') ); const sent = query.mock.calls[0][0].query as string; expect(sent).toContain('load_skill'); - expect(result.score).toBe(1); + expect(sent).toContain('filestore.read'); + expect(sent).toContain('*\\"skill\\":\\"alert-analysis\\"*'); + expect(sent).toContain('*/alert-analysis/SKILL.md*'); }); }); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.ts index 2ed5f0f7b35b3..4691f6c50c80f 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.ts @@ -194,7 +194,10 @@ export const createPersonaMatrixSkillInvokedEvaluator = ({ } const skillPredicate = acceptedSkills - .map((skillName) => `attributes.gen_ai.tool.call.arguments LIKE "*/${skillName}/SKILL.md*"`) + .flatMap((skillName) => [ + `attributes.gen_ai.tool.call.arguments LIKE "*\\\"skill\\\":\\\"${skillName}\\\"*"`, + `attributes.gen_ai.tool.call.arguments LIKE "*/${skillName}/SKILL.md*"`, + ]) .join(' OR '); const query = `FROM traces-* From 200e9b063b395f16496b82efddda9f180843d361 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Thu, 20 Aug 2026 13:20:35 +0200 Subject: [PATCH 008/258] [kbn-evals-extensions] Matrix: degrade gracefully when an example's scores exceed transport size limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-example scores route returns every document for that example across all runs. On multi-step examples with long trajectories across 15+ models the combined response can exceed the transport's maximum string size (observed 104,864,396 > 104,857,600 on multi-step-c), which aborted the whole matrix generation with 'Failed to retrieve scores for example ID'. Scores are already aggregated at that point — only this example's trace detail is at stake. Catch the fetch failure per example, log a warning, and continue with empty trace details. --- .../src/matrix/query_matrix_traces.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.ts index 6a7ddb0e2b9d0..cb1afdcb09960 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.ts @@ -226,8 +226,19 @@ export const queryMatrixTraces = async ( } for (const exampleId of allExampleIds) { - const scores = await evalsClient.getExampleScores(exampleId); - exampleScores.set(exampleId, scores); + try { + exampleScores.set(exampleId, await evalsClient.getExampleScores(exampleId)); + } catch (error) { + // A single example whose combined documents exceed the transport's size + // limit must not abort the whole report: scores are already aggregated, + // only this example's trace detail is lost. + log.warning( + `Skipping trace details for example ${exampleId}: ${ + error instanceof Error ? error.message : String(error) + }` + ); + exampleScores.set(exampleId, []); + } } // 3. For each run, pick the newest complete document per example and merge From 4a0e61ca5b6eb7de8ea4e9ea3b1dab09f0955ea4 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Thu, 20 Aug 2026 13:21:21 +0200 Subject: [PATCH 009/258] Revert "[Security Solution][kbn-evals] Match load_skill IDs in persona matrix" This reverts commit 7e02e5d1311492c6be5de439108b28ffa6c5b971. --- .../src/evaluate_dataset.test.ts | 17 ++++++++++------- .../src/evaluate_dataset.ts | 5 +---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.test.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.test.ts index b63577c07a24b..73ebdc31fd1a1 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.test.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.test.ts @@ -236,9 +236,14 @@ describe('createPersonaMatrixSkillInvokedEvaluator', () => { expect(result.label).toBe('unavailable'); }); - it('matches load_skill by skill ID while retaining the legacy SKILL.md path', async () => { - // `load_skill` accepts an ID or path. Current traces store {"skill":""}; - // older traces may contain a SKILL.md path. + it('matches the load_skill tool, not just the retired filestore.read', async () => { + // Regression guard. The agent loads skills via the `load_skill` tool: + // {"skill":"/skills///SKILL.md"} + // A predicate pinned to `filestore.read` can never match, so `skill_invoked` + // stays 0 while `total_tool_spans` is non-zero -- meaning the "unavailable" + // guard does NOT trip and the evaluator reports a confident false 0 for + // every model. Verified against the golden cluster: over 7 days, + // filestore.read = 0 spans, load_skill = 7,991 spans. const query = jest.fn().mockResolvedValue({ columns: [{ name: 'total_tool_spans' }, { name: 'skill_invoked' }], values: [[2, 1]], @@ -248,15 +253,13 @@ describe('createPersonaMatrixSkillInvokedEvaluator', () => { log: buildLog(), }); - await evaluator.evaluate( + const result = await evaluator.evaluate( buildEvaluatorArgs(baseExample.metadata, '0af7651916cd43dd8448eb211c80319c') ); const sent = query.mock.calls[0][0].query as string; expect(sent).toContain('load_skill'); - expect(sent).toContain('filestore.read'); - expect(sent).toContain('*\\"skill\\":\\"alert-analysis\\"*'); - expect(sent).toContain('*/alert-analysis/SKILL.md*'); + expect(result.score).toBe(1); }); }); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.ts index 4691f6c50c80f..2ed5f0f7b35b3 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix/src/evaluate_dataset.ts @@ -194,10 +194,7 @@ export const createPersonaMatrixSkillInvokedEvaluator = ({ } const skillPredicate = acceptedSkills - .flatMap((skillName) => [ - `attributes.gen_ai.tool.call.arguments LIKE "*\\\"skill\\\":\\\"${skillName}\\\"*"`, - `attributes.gen_ai.tool.call.arguments LIKE "*/${skillName}/SKILL.md*"`, - ]) + .map((skillName) => `attributes.gen_ai.tool.call.arguments LIKE "*/${skillName}/SKILL.md*"`) .join(' OR '); const query = `FROM traces-* From 94a9632c995681647dbe28264e9bab8de7a059d8 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Thu, 20 Aug 2026 13:58:36 +0200 Subject: [PATCH 010/258] [kbn-evals-extensions] Matrix: per-category trace cards show their own example's conversation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Category columns (examplePrefixes) resolved their detail trace through the suite-level key, which processExampleBatch overwrote on every example — last example processed won. Every category card for a model therefore rendered the SAME conversation (observed: the workflow-execution prompt shown under Alert Analysis and Entity Analytics alike). - processExampleBatch now emits a trace per matching prefix using the exact scoresByPrefixToDatasets semantics (equality or boundary dash), first variant wins for determinism. - The suite-level key keeps the FIRST complete trace instead of the last. - render_matrix_html resolves prefix:

keys ahead of the suite fallback. - Prefixes are harvested from the aggregated synthetic prefix:* dataset ids, so trace bucketing can never drift from score bucketing. Verified live against golden: 15 models x 7 categories now render 7 distinct category questions per model (previously 1). --- .../src/matrix/query_matrix_traces.ts | 54 ++++++++++++++++--- .../src/matrix/render_matrix_html.ts | 11 ++-- 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.ts index cb1afdcb09960..c41038df9a0e1 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/query_matrix_traces.ts @@ -105,7 +105,8 @@ const processExampleBatch = ( modelId: string, suiteId: string, executionId: string, - traces: MatrixTraceData + traces: MatrixTraceData, + examplePrefixes: ReadonlySet = new Set() ): boolean => { // Filter to this model + this execution, newest first const relevant = scores @@ -129,12 +130,31 @@ const processExampleBatch = ( if (exampleId) { traces[traceKey(modelId, exampleId)] = entry; } + // Category columns slice the dataset by example.id prefix (examplePrefixes), + // synthesizing `prefix:` dataset ids. Emit one trace per matching + // prefix — mirroring scoresByPrefixToDatasets' semantics exactly (equality + // or boundary dash) — so every category card shows ITS OWN example's + // conversation instead of falling through to the suite key (which + // previously made all cards render the same, last-processed example). + if (exampleId) { + for (const prefix of examplePrefixes) { + if (exampleId === prefix || exampleId.startsWith(`${prefix}-`)) { + const key = traceKey(modelId, `prefix:${prefix}`); + // First variant wins: all matching variants are valid examples of the + // category, so keep the lookup deterministic across Set iteration order. + if (traces[key] === undefined) traces[key] = entry; + } + } + } if (datasetId) { traces[traceKey(modelId, datasetId)] = entry; } - // For the suite-level key, only complete runs qualify. - if (complete) { - traces[traceKey(modelId, suiteId)] = entry; + // For the suite-level key, only complete runs qualify — and the FIRST + // complete one wins. Overwriting on every example made the suite key + // "last example processed", which every non-matching card then inherited. + const suiteTraceKey = traceKey(modelId, suiteId); + if (complete && traces[suiteTraceKey] === undefined) { + traces[suiteTraceKey] = entry; } // If the newest doc is incomplete, fall back to the newest complete one @@ -144,7 +164,9 @@ const processExampleBatch = ( const fallbackEntry = extractTraceFromScore(firstComplete); const fid = firstComplete.example?.id; if (fid) traces[traceKey(modelId, fid)] = fallbackEntry; - traces[traceKey(modelId, suiteId)] = fallbackEntry; + if (traces[suiteTraceKey] === undefined) { + traces[suiteTraceKey] = fallbackEntry; + } } } @@ -241,13 +263,33 @@ export const queryMatrixTraces = async ( } } + // Category prefixes come from the same synthetic `prefix:*` dataset ids the + // score aggregation created (scoresByPrefixToDatasets), so trace bucketing + // always matches score bucketing — no separate config source to keep in sync. + const examplePrefixes = new Set(); + for (const modelScores of aggregated) { + for (const suite of modelScores.suites) { + for (const dataset of suite.datasets) { + if (!dataset.datasetId?.startsWith('prefix:')) continue; + examplePrefixes.add(dataset.datasetId.slice('prefix:'.length)); + } + } + } + // 3. For each run, pick the newest complete document per example and merge // into the traces map. for (const ref of runRefs) { let completeCount = 0; for (const exampleId of ref.exampleIds) { const scores = exampleScores.get(exampleId) ?? []; - const ok = processExampleBatch(scores, ref.modelId, ref.suiteId, ref.executionId, traces); + const ok = processExampleBatch( + scores, + ref.modelId, + ref.suiteId, + ref.executionId, + traces, + examplePrefixes + ); if (ok) completeCount++; } if (completeCount === 0) { diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.ts index 4ae222c4aa307..e92b1417014d4 100644 --- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.ts +++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.ts @@ -272,15 +272,18 @@ const renderModelCard = ( ? 'failed' : 'missing'; + const column = config.columns.find((c) => c.id === col.id); const trace = traces?.[traceKey(row.modelId, col.id)] ?? + // examplePrefixes columns consume synthetic `prefix:

` datasets; + // resolve the trace for that prefix's own example. + column?.examplePrefixes + ?.map((p) => traces?.[traceKey(row.modelId, `prefix:${p}`)]) + .find((t) => t != null) ?? // Column IDs (e.g. 'alert_triage') don't match suite IDs (e.g. // 'security-alert-triage'). Fall back to the current column's // configured suite IDs to find the first matching trace. - config.columns - .find((c) => c.id === col.id) - ?.suites.map((s) => traces?.[traceKey(row.modelId, s)]) - .find((t) => t != null); + column?.suites.map((s) => traces?.[traceKey(row.modelId, s)]).find((t) => t != null); const scoreStr = cell.kind === 'score' ? `score ${cell.value}` : ''; const metaParts = [ scoreStr, From b4949eb8d2c401ebb00f5992bbbb78ad5ff033af Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Thu, 20 Aug 2026 14:30:07 +0200 Subject: [PATCH 011/258] [kbn-evals-extensions] Matrix HTML: render tables, code fences, blockquotes in answers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mdToHtml only handled headings, lists, bold, code, and hr. Model answers with pipe tables rendered as one paragraph per row (raw | visible), ES|QL code fences leaked their pipes as paragraph text, and > quotes were unstyled. - Pipe tables: header + |---| separator + body rows -> with zebra striping and borders; cell content escaped before inline markdown (untrusted model output). - Fenced code blocks: -> 
, verbatim — ES|QL pipes
  inside fences are query syntax, not tables.
- Blockquote lines -> 
.

Verified live against golden: 23 tables, 7 code blocks, 7 blockquotes
rendered; leaked pipe-paragraphs 12 -> 0. 76/76 jest, tsc 0, eslint clean.
---
 .../src/matrix/render_matrix_html.test.ts     | 74 +++++++++++++++++++
 .../src/matrix/render_matrix_html.ts          | 72 +++++++++++++++++-
 2 files changed, 144 insertions(+), 2 deletions(-)

diff --git a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.test.ts b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.test.ts
index f4fc068a7f694..5548e4fa7589a 100644
--- a/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.test.ts
+++ b/x-pack/platform/packages/shared/kbn-evals-extensions/src/matrix/render_matrix_html.test.ts
@@ -208,4 +208,78 @@ describe('renderMatrixHtml', () => {
     expect(html).toContain('click');
     expect(html).toContain('x');
   });
+
+  it('renders markdown pipe tables as HTML tables, not paragraph soup', () => {
+    const traces: MatrixTraceData = {
+      'test-model:alert': {
+        question: 'q',
+        toolTrail: [],
+        answer:
+          '| Field | Value |\n' +
+          '|-------|-------|\n' +
+          '| **Process** | `BluetoothService.exe` |\n' +
+          '| **User Context** | SYSTEM |\n\n' +
+          '### Context\n- item one\n- item two',
+        stepCount: 0,
+        toolCount: 0,
+      },
+    };
+
+    const html = renderMatrixHtml(mockMatrix, mockConfig, {}, traces);
+    // The table is recognized and rendered as a real table element.
+    expect(html).toContain('
'); + expect(html).toContain(''); + expect(html).toContain(''); + expect(html).toContain(''); + expect(html).toContain(''); + // Raw pipes must not leak as paragraph text once a table is parsed. + expect(html).not.toMatch(/

\|/); + // Content after the table still renders normally. + expect(html).toContain('

Context
'); + }); + + it('renders blockquote lines and escapes cell content in tables', () => { + const traces: MatrixTraceData = { + 'test-model:alert': { + question: 'q', + toolTrail: [], + answer: + '> quoted note \n' + + '| a | b |\n' + + '|---|---|\n' + + '| | plain |', + stepCount: 0, + toolCount: 0, + }, + }; + + const html = renderMatrixHtml(mockMatrix, mockConfig, {}, traces); + expect(html).toContain('
'); + // Untrusted cell/script content is escaped, never emitted as live HTML. + expect(html).not.toContain('
FieldValueProcessBluetoothService.exe