From 74fdadc2e3d01f3fc3db0d2c92fc2d6fe60161a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Lidue=C3=B1a?= Date: Fri, 11 Sep 2026 11:55:15 +0200 Subject: [PATCH 1/4] [kbn-evals] Report CI triage through a forced tool call --- .../kbn-evals/scripts/ci/ai_connectors.js | 2 + .../ci/build_suite_owner_slack_message.js | 3 + .../scripts/ci/failure_context_helpers.js | 211 +++++++++++-- .../ci/failure_context_helpers.test.js | 291 ++++++++++++++++++ 4 files changed, 481 insertions(+), 26 deletions(-) diff --git a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/ai_connectors.js b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/ai_connectors.js index be25738cca2a6..9d8080e4514d4 100644 --- a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/ai_connectors.js +++ b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/ai_connectors.js @@ -52,6 +52,8 @@ function parseVaultConfig() { } // CI notify triage uses this OpenRouter model. +// Model must support OpenAI-style tool calling with `tool_choice` on OpenRouter. +// Models without it fail every triage with "did not call the report_triage tool". const TRIAGE_OPENROUTER_MODEL = 'google/gemini-3.7-flash'; /** diff --git a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/build_suite_owner_slack_message.js b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/build_suite_owner_slack_message.js index 96ac1717d5939..56a6efe0f41da 100644 --- a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/build_suite_owner_slack_message.js +++ b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/build_suite_owner_slack_message.js @@ -146,6 +146,9 @@ async function main() { } catch (error) { const message = formatTriageError(error); console.error(`--- Triage summary failed: ${message}`); + if (error && typeof error === 'object' && typeof error.details === 'string') { + console.error(error.details); + } triage = { modelId: TRIAGE_OPENROUTER_CONNECTOR_ID, error: message }; } diff --git a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js index 4da2267c82d66..92395ae09d301 100644 --- a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js +++ b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js @@ -15,21 +15,80 @@ const MAX_CONTEXT_JSON_BYTES = 30 * 1024; // Triage/summary text is always generated with a small, low-cost OpenRouter model. const TRIAGE_OPENROUTER_CONNECTOR_ID = `openrouter-${slugifyId(TRIAGE_OPENROUTER_MODEL)}`; +// Output budget for the free-text weekly rollup (bullets, < 900 chars). +const DEFAULT_MAX_TOKENS = 800; + +// Output budget for the per-suite structured triage. +const TRIAGE_MAX_TOKENS = 4000; + +// Bounds enforced through the prompt so the structured output stays well inside +// TRIAGE_MAX_TOKENS regardless of how many models failed. +const TRIAGE_MAX_GROUPS = 6; +const TRIAGE_MAX_ERROR_CHARS = 200; + +// How much of a malformed model reply to surface in the Buildkite step log. +const TRIAGE_RAW_PREVIEW_CHARS = 300; + const TRIAGE_SYSTEM_PROMPT = 'You are an SRE assistant triaging failed LLM evaluation CI runs. Be concise and factual, and base every statement on the provided context.'; +const TRIAGE_TOOL_NAME = 'report_triage'; + +const TRIAGE_TOOL = { + type: 'function', + function: { + name: TRIAGE_TOOL_NAME, + description: + 'Report the triage of a failed LLM evaluation CI suite as a list of distinct error groups.', + parameters: { + type: 'object', + properties: { + groups: { + type: 'array', + description: 'One entry per distinct error. Empty when the excerpts show no clear error.', + items: { + type: 'object', + properties: { + error: { + type: 'string', + description: `Most relevant error line, verbatim from the excerpts, one line, at most ${TRIAGE_MAX_ERROR_CHARS} characters.`, + }, + location: { + type: 'string', + description: 'file:line, test, or scenario if shown in the excerpts; else empty.', + }, + models: { + type: 'array', + items: { type: 'string' }, + description: 'Ids of every failing model that hit this error.', + }, + rootCause: { + type: 'string', + description: 'One short sentence with the likely cause plus one short action.', + }, + }, + required: ['error', 'location', 'models', 'rootCause'], + }, + }, + }, + required: ['groups'], + }, + }, +}; + +const TRIAGE_TOOL_CHOICE = { type: 'function', function: { name: TRIAGE_TOOL_NAME } }; + // Static output contract for the per-suite structured triage prompt. -const TRIAGE_OUTPUT_INSTRUCTIONS = `Return ONLY a JSON object of this shape (no prose, no markdown, no code fences): -{"groups":[{"error":"","location":"","models":[""],"rootCause":""}]} +const TRIAGE_OUTPUT_INSTRUCTIONS = `Report the result by calling the \`${TRIAGE_TOOL_NAME}\` tool with {"groups":[{"error":"","location":"","models":[""],"rootCause":""}]}. Do not answer in prose. Rules: - One group per distinct error. Merge the same failure (same message/location) into one group and list all its affected models. -- Quote "error" verbatim from the excerpts; do not invent errors, file names, line numbers, or causes. +- Report at most ${TRIAGE_MAX_GROUPS} groups; if there are more distinct errors, keep the ones affecting the most models. +- Quote "error" verbatim from the excerpts, trimmed to a single line of at most ${TRIAGE_MAX_ERROR_CHARS} characters; do not invent errors, file names, line numbers, or causes. - Set "location" only if the excerpts show it; otherwise use an empty string. - Keep "rootCause" to one short sentence plus one short action. - Never reproduce secrets: if a line contains an API key, token, password, or other credential, replace that value with [REDACTED]. -- If the excerpts show no clear error, return {"groups": []}. -- Output valid JSON only: no prose before or after, no comments, no trailing commas, no code fences.`; +- If the excerpts show no clear error, call the tool with {"groups": []}.`; // Static output contract for the weekly cross-suite rollup prompt. const WEEKLY_ROLLUP_OUTPUT_INSTRUCTIONS = `Output exactly these bullets, in this order, and nothing else: @@ -233,7 +292,7 @@ function buildWeeklyRollupUserPrompt(suites, meta = {}) { return lines.join('\n'); } -function buildOpenrouterChatRequest(connector, messages) { +function buildOpenrouterChatRequest(connector, messages, options = {}) { const config = connector.config && typeof connector.config === 'object' ? connector.config : {}; const secrets = connector.secrets && typeof connector.secrets === 'object' ? connector.secrets : {}; @@ -246,32 +305,49 @@ function buildOpenrouterChatRequest(connector, messages) { throw new Error('OpenRouter connector is missing apiUrl, defaultModel, or apiKey'); } + const { maxTokens = DEFAULT_MAX_TOKENS, tools, toolChoice } = options; + + const body = { + model: defaultModel, + messages, + temperature: 0.2, + max_tokens: maxTokens, + }; + if (Array.isArray(tools) && tools.length > 0) { + body.tools = tools; + if (toolChoice) { + body.tool_choice = toolChoice; + } + } + return { url: apiUrl, headers: { 'content-type': 'application/json', authorization: `Bearer ${apiKey}`, }, - body: { - model: defaultModel, - messages, - temperature: 0.2, - max_tokens: 800, - }, + body, }; } -function parseOpenrouterChatContent(responseJson) { +function parseOpenrouterChatChoice(responseJson) { if (!responseJson || typeof responseJson !== 'object') { throw new Error('OpenRouter response was not JSON'); } - const choices = /** @type {{ choices?: Array<{ message?: { content?: string } }> }} */ ( - responseJson - ).choices; + const choice = Array.isArray(responseJson.choices) ? responseJson.choices[0] : undefined; + const message = choice?.message && typeof choice.message === 'object' ? choice.message : {}; - const content = choices?.[0]?.message?.content; - if (typeof content !== 'string' || !content.trim()) { + return { + content: typeof message.content === 'string' ? message.content : '', + toolCalls: Array.isArray(message.tool_calls) ? message.tool_calls : [], + finishReason: typeof choice?.finish_reason === 'string' ? choice.finish_reason : '', + }; +} + +function parseOpenrouterChatContent(responseJson) { + const { content } = parseOpenrouterChatChoice(responseJson); + if (!content.trim()) { throw new Error('OpenRouter response did not include message content'); } @@ -279,9 +355,9 @@ function parseOpenrouterChatContent(responseJson) { } /** - * POST a built OpenRouter chat request and return the message content. + * POST a built OpenRouter chat request and return the parsed response body. */ -async function postOpenrouterChatRequest({ url, headers, body }) { +async function postOpenrouterChat({ url, headers, body }) { const response = await fetch(url, { method: 'POST', headers, @@ -302,14 +378,18 @@ async function postOpenrouterChatRequest({ url, headers, body }) { throw new Error(`Inference request failed (${response.status}): ${detail}`); } - let json = null; try { - json = text ? JSON.parse(text) : null; + return text ? JSON.parse(text) : null; } catch { throw new Error('Inference response was not JSON'); } +} - return parseOpenrouterChatContent(json); +/** + * POST a built OpenRouter chat request and return the message content. + */ +async function postOpenrouterChatRequest(request) { + return parseOpenrouterChatContent(await postOpenrouterChat(request)); } /** @@ -324,6 +404,21 @@ function resolveTriageConnector() { }; } +function triageError(message, raw) { + const error = /** @type {Error & { details?: string }} */ (new Error(message)); + const preview = redactSecrets(String(raw ?? '')) + .replace(/\s+/g, ' ') + .trim(); + if (preview) { + const clipped = + preview.length > TRIAGE_RAW_PREVIEW_CHARS + ? `${preview.slice(0, TRIAGE_RAW_PREVIEW_CHARS)}…` + : preview; + error.details = `Raw model output (${preview.length} chars): ${clipped}`; + } + return error; +} + /** * Parse the structured triage groups returned by the model. */ @@ -338,7 +433,19 @@ function parseTriageGroups(rawText) { try { parsed = JSON.parse(unfenced); } catch { - throw new Error('Triage model did not return valid JSON'); + // Prose around the object: fall back to the outermost `{ ... }`. + const start = unfenced.indexOf('{'); + const end = unfenced.lastIndexOf('}'); + if (start >= 0 && end > start) { + try { + parsed = JSON.parse(unfenced.slice(start, end + 1)); + } catch { + // fall through + } + } + if (parsed === undefined) { + throw triageError('Triage model did not return valid JSON', text); + } } const groups = Array.isArray(parsed?.groups) ? parsed.groups : []; @@ -375,6 +482,42 @@ async function runTriageModel(userPrompt, { maxChars = 1500 } = {}) { return { summary: summary.trim(), modelId }; } +/** + * Extract the triage groups from a forced `report_triage` tool call. + */ +function parseTriageToolCall(responseJson) { + const { content, toolCalls, finishReason } = parseOpenrouterChatChoice(responseJson); + + const call = + toolCalls.find((candidate) => candidate?.function?.name === TRIAGE_TOOL_NAME) ?? toolCalls[0]; + const rawArguments = call?.function?.arguments; + // The OpenAI-compatible contract is a JSON string, but some providers inline the object. + let args = ''; + if (typeof rawArguments === 'string') { + args = rawArguments; + } else if (rawArguments && typeof rawArguments === 'object') { + args = JSON.stringify(rawArguments); + } + + if (finishReason === 'length') { + throw triageError( + `Triage model output was truncated at the ${TRIAGE_MAX_TOKENS}-token limit`, + args || content + ); + } + + if (!args.trim()) { + throw triageError( + `Triage model did not call the ${TRIAGE_TOOL_NAME} tool (finish_reason: ${ + finishReason || 'unknown' + })`, + content + ); + } + + return parseTriageGroups(args); +} + /** * Resolve the OpenRouter connector, send the shared system prompt + the given user * prompt, and return the parsed structured triage groups and the model id used. @@ -388,16 +531,29 @@ async function runTriageModelStructured(userPrompt) { { role: 'user', content: userPrompt }, ]; - const raw = await postOpenrouterChatRequest(buildOpenrouterChatRequest(connector, messages)); + const responseJson = await postOpenrouterChat( + buildOpenrouterChatRequest(connector, messages, { + maxTokens: TRIAGE_MAX_TOKENS, + tools: [TRIAGE_TOOL], + toolChoice: TRIAGE_TOOL_CHOICE, + }) + ); - return { groups: parseTriageGroups(raw), modelId }; + return { groups: parseTriageToolCall(responseJson), modelId }; } module.exports = { MAX_LOG_EXCERPT_CHARS, MAX_CONTEXT_JSON_BYTES, + DEFAULT_MAX_TOKENS, + TRIAGE_MAX_TOKENS, + TRIAGE_MAX_GROUPS, + TRIAGE_MAX_ERROR_CHARS, TRIAGE_SYSTEM_PROMPT, TRIAGE_OPENROUTER_CONNECTOR_ID, + TRIAGE_TOOL_NAME, + TRIAGE_TOOL, + TRIAGE_TOOL_CHOICE, failureLogMetadataKey, failureLogMetadataKeysForProject, truncateText, @@ -407,10 +563,13 @@ module.exports = { buildWeeklyRollupUserPrompt, extractSuiteRootCauseLine, buildOpenrouterChatRequest, + parseOpenrouterChatChoice, parseOpenrouterChatContent, + postOpenrouterChat, postOpenrouterChatRequest, resolveTriageConnector, parseTriageGroups, + parseTriageToolCall, runTriageModel, runTriageModelStructured, }; diff --git a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js index a09c55fbc5a04..0a51cc6411afd 100644 --- a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js +++ b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js @@ -9,11 +9,66 @@ const { failureLogMetadataKey, failureLogMetadataKeysForProject, TRIAGE_OPENROUTER_CONNECTOR_ID, + TRIAGE_TOOL_NAME, + TRIAGE_TOOL, + TRIAGE_TOOL_CHOICE, + TRIAGE_MAX_TOKENS, + DEFAULT_MAX_TOKENS, resolveTriageConnector, + buildOpenrouterChatRequest, + buildTriageUserPrompt, + parseTriageGroups, + parseTriageToolCall, + runTriageModelStructured, + runTriageModel, } = require('./failure_context_helpers'); const SUITE = 'significant-events'; +const CONNECTOR = { + config: { apiUrl: 'https://openrouter.test/api/v1/chat/completions', defaultModel: 'test/model' }, + secrets: { apiKey: 'sk-test' }, +}; + +const MESSAGES = [ + { role: 'system', content: 'system' }, + { role: 'user', content: 'user' }, +]; + +const GROUP = { + error: 'Error: expect(received).toBe(expected)', + location: 'find_rules.spec.ts:484', + models: ['eis-openai-gpt-5-4'], + rootCause: 'Assertion failed; update the expectation.', +}; + +function toolCallResponse(args, { finishReason = 'tool_calls', name = TRIAGE_TOOL_NAME } = {}) { + return { + choices: [ + { + finish_reason: finishReason, + message: { + role: 'assistant', + content: null, + tool_calls: [{ id: 'call_1', type: 'function', function: { name, arguments: args } }], + }, + }, + ], + }; +} + +function textResponse(content, finishReason = 'stop') { + return { choices: [{ finish_reason: finishReason, message: { role: 'assistant', content } }] }; +} + +function mockFetchJson(json) { + return jest.fn().mockResolvedValue({ + ok: true, + status: 200, + text: () => Promise.resolve(JSON.stringify(json)), + }); +} + describe('failureLogMetadataKey', () => { it('slugifies the suite and project into a stable key', () => { expect(failureLogMetadataKey(SUITE, 'eis/openai-gpt-5.4')).toBe( @@ -59,6 +114,242 @@ describe('failureLogMetadataKeysForProject', () => { }); }); +describe('buildOpenrouterChatRequest', () => { + it('builds a plain text request by default (weekly rollup path)', () => { + const { url, headers, body } = buildOpenrouterChatRequest(CONNECTOR, MESSAGES); + + expect(url).toBe(CONNECTOR.config.apiUrl); + expect(headers.authorization).toBe('Bearer sk-test'); + expect(body).toEqual({ + model: 'test/model', + messages: MESSAGES, + temperature: 0.2, + max_tokens: DEFAULT_MAX_TOKENS, + }); + expect(body).not.toHaveProperty('tools'); + expect(body).not.toHaveProperty('tool_choice'); + }); + + it('adds the tool, forced tool choice, and a larger token budget when asked', () => { + const { body } = buildOpenrouterChatRequest(CONNECTOR, MESSAGES, { + maxTokens: TRIAGE_MAX_TOKENS, + tools: [TRIAGE_TOOL], + toolChoice: TRIAGE_TOOL_CHOICE, + }); + + expect(body.max_tokens).toBe(TRIAGE_MAX_TOKENS); + expect(body.tools).toEqual([TRIAGE_TOOL]); + expect(body.tool_choice).toEqual({ + type: 'function', + function: { name: TRIAGE_TOOL_NAME }, + }); + }); + + it('throws when the connector is incomplete', () => { + expect(() => buildOpenrouterChatRequest({ config: {}, secrets: {} }, MESSAGES)).toThrow( + 'OpenRouter connector is missing apiUrl, defaultModel, or apiKey' + ); + }); +}); + +describe('buildTriageUserPrompt', () => { + it('asks the model to report through the triage tool and bounds the output', () => { + const prompt = buildTriageUserPrompt( + { suiteId: SUITE, failingProjects: ['gpt-5'], models: {} }, + { suiteName: 'Significant Events', suiteId: SUITE, failingProjects: ['gpt-5'] } + ); + + expect(prompt).toContain(`calling the \`${TRIAGE_TOOL_NAME}\` tool`); + expect(prompt).toContain('Report at most 6 groups'); + expect(prompt).toContain('at most 200 characters'); + expect(prompt).not.toContain('Return ONLY a JSON object'); + }); +}); + +describe('parseTriageGroups', () => { + it('parses a clean JSON object', () => { + expect(parseTriageGroups(JSON.stringify({ groups: [GROUP] }))).toEqual([GROUP]); + }); + + it('strips a markdown code fence', () => { + expect(parseTriageGroups(`\`\`\`json\n${JSON.stringify({ groups: [GROUP] })}\n\`\`\``)).toEqual( + [GROUP] + ); + }); + + it('recovers the object when the model wraps it in prose', () => { + const raw = `Here is the triage:\n${JSON.stringify({ + groups: [GROUP], + })}\nLet me know if you need more.`; + expect(parseTriageGroups(raw)).toEqual([GROUP]); + }); + + it('coerces loose field types and drops empty groups', () => { + const raw = JSON.stringify({ + groups: [ + { error: ' boom ', models: 'not-an-array', rootCause: 42 }, + { error: '', location: 'x', models: [], rootCause: '' }, + ], + }); + expect(parseTriageGroups(raw)).toEqual([ + { error: 'boom', location: '', models: [], rootCause: '42' }, + ]); + }); + + it('returns no groups when "groups" is missing', () => { + expect(parseTriageGroups('{}')).toEqual([]); + }); + + it('throws a short message and keeps a redacted preview of the raw output in details', () => { + const raw = `{"groups":[{"error":"Authorization: Bearer abc.def.ghi failed","models":["gpt-5"`; + let caught; + try { + parseTriageGroups(raw); + } catch (error) { + caught = error; + } + + expect(caught).toBeInstanceOf(Error); + expect(caught.message).toBe('Triage model did not return valid JSON'); + expect(caught.details).toContain('Raw model output'); + expect(caught.details).toContain('[REDACTED]'); + expect(caught.details).not.toContain('abc.def.ghi'); + }); + + it('clips the raw preview to 300 characters', () => { + const raw = `{${'x'.repeat(1000)}`; + let caught; + try { + parseTriageGroups(raw); + } catch (error) { + caught = error; + } + + expect(caught.details).toContain('(1001 chars)'); + expect(caught.details.length).toBeLessThan(400); + expect(caught.details.endsWith('…')).toBe(true); + }); +}); + +describe('parseTriageToolCall', () => { + it('reads the groups from the forced tool call arguments', () => { + const response = toolCallResponse(JSON.stringify({ groups: [GROUP] })); + expect(parseTriageToolCall(response)).toEqual([GROUP]); + }); + + it('accepts providers that inline the arguments as an object', () => { + const response = toolCallResponse({ groups: [GROUP] }); + expect(parseTriageToolCall(response)).toEqual([GROUP]); + }); + + it('prefers the report_triage call when several tool calls are present', () => { + const response = toolCallResponse(JSON.stringify({ groups: [GROUP] })); + response.choices[0].message.tool_calls.unshift({ + id: 'call_0', + type: 'function', + function: { name: 'other_tool', arguments: '{"groups":[]}' }, + }); + expect(parseTriageToolCall(response)).toEqual([GROUP]); + }); + + it('reports truncation at the token limit instead of a generic JSON error', () => { + const response = toolCallResponse('{"groups":[{"error":"boom","mo', { finishReason: 'length' }); + let caught; + try { + parseTriageToolCall(response); + } catch (error) { + caught = error; + } + + expect(caught.message).toBe( + `Triage model output was truncated at the ${TRIAGE_MAX_TOKENS}-token limit` + ); + expect(caught.details).toContain('{"groups":[{"error":"boom","mo'); + }); + + it('reports a missing tool call when the model answered in text', () => { + let caught; + try { + parseTriageToolCall(textResponse('Sure! Here is my analysis of the failures...')); + } catch (error) { + caught = error; + } + + expect(caught.message).toBe( + `Triage model did not call the ${TRIAGE_TOOL_NAME} tool (finish_reason: stop)` + ); + expect(caught.details).toContain('Sure! Here is my analysis'); + }); + + it('reports malformed tool arguments as invalid JSON with a preview', () => { + let caught; + try { + parseTriageToolCall(toolCallResponse('not json at all')); + } catch (error) { + caught = error; + } + + expect(caught.message).toBe('Triage model did not return valid JSON'); + expect(caught.details).toContain('not json at all'); + }); + + it('rejects a non-object response', () => { + expect(() => parseTriageToolCall(null)).toThrow('OpenRouter response was not JSON'); + }); +}); + +describe('runTriageModelStructured / runTriageModel (mocked fetch)', () => { + const originalEnv = { ...process.env }; + const originalFetch = global.fetch; + + beforeEach(() => { + process.env.OPENROUTER_BASE_URL = 'https://openrouter.test/api/v1'; + process.env.OPENROUTER_API_KEY = 'sk-test'; + }); + + afterEach(() => { + process.env = { ...originalEnv }; + global.fetch = originalFetch; + }); + + it('sends the tool-forced request and returns the parsed groups', async () => { + global.fetch = mockFetchJson(toolCallResponse(JSON.stringify({ groups: [GROUP] }))); + + const result = await runTriageModelStructured('triage this'); + + expect(result).toEqual({ groups: [GROUP], modelId: TRIAGE_OPENROUTER_CONNECTOR_ID }); + expect(global.fetch).toHaveBeenCalledTimes(1); + + const [url, init] = global.fetch.mock.calls[0]; + expect(url).toBe('https://openrouter.test/api/v1/chat/completions'); + const body = JSON.parse(init.body); + expect(body.max_tokens).toBe(TRIAGE_MAX_TOKENS); + expect(body.tools).toEqual([TRIAGE_TOOL]); + expect(body.tool_choice).toEqual(TRIAGE_TOOL_CHOICE); + expect(body.messages[1]).toEqual({ role: 'user', content: 'triage this' }); + }); + + it('surfaces a truncated reply as a token-limit error', async () => { + global.fetch = mockFetchJson(toolCallResponse('{"groups":[', { finishReason: 'length' })); + + await expect(runTriageModelStructured('triage this')).rejects.toThrow( + 'truncated at the 4000-token limit' + ); + }); + + it('keeps the weekly rollup on the plain text request', async () => { + global.fetch = mockFetchJson(textResponse('- Overall: 1 suite likely retryable.')); + + const result = await runTriageModel('summarize'); + + expect(result.summary).toBe('- Overall: 1 suite likely retryable.'); + const body = JSON.parse(global.fetch.mock.calls[0][1].body); + expect(body.max_tokens).toBe(DEFAULT_MAX_TOKENS); + expect(body).not.toHaveProperty('tools'); + expect(body).not.toHaveProperty('tool_choice'); + }); +}); + describe('resolveTriageConnector', () => { const originalEnv = { ...process.env }; From 831e7b964925a76e3aff2b839bec3890642f2d33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Lidue=C3=B1a?= Date: Fri, 11 Sep 2026 12:28:56 +0200 Subject: [PATCH 2/4] Simplify CI triage to two paths: tool call parses, or generic failure --- .../ci/build_suite_owner_slack_message.js | 3 - .../scripts/ci/failure_context_helpers.js | 141 ++++----------- .../ci/failure_context_helpers.test.js | 170 ++++++------------ 3 files changed, 83 insertions(+), 231 deletions(-) diff --git a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/build_suite_owner_slack_message.js b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/build_suite_owner_slack_message.js index 56a6efe0f41da..96ac1717d5939 100644 --- a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/build_suite_owner_slack_message.js +++ b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/build_suite_owner_slack_message.js @@ -146,9 +146,6 @@ async function main() { } catch (error) { const message = formatTriageError(error); console.error(`--- Triage summary failed: ${message}`); - if (error && typeof error === 'object' && typeof error.details === 'string') { - console.error(error.details); - } triage = { modelId: TRIAGE_OPENROUTER_CONNECTOR_ID, error: message }; } diff --git a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js index 92395ae09d301..0740f451b8a86 100644 --- a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js +++ b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js @@ -15,20 +15,10 @@ const MAX_CONTEXT_JSON_BYTES = 30 * 1024; // Triage/summary text is always generated with a small, low-cost OpenRouter model. const TRIAGE_OPENROUTER_CONNECTOR_ID = `openrouter-${slugifyId(TRIAGE_OPENROUTER_MODEL)}`; -// Output budget for the free-text weekly rollup (bullets, < 900 chars). -const DEFAULT_MAX_TOKENS = 800; - // Output budget for the per-suite structured triage. const TRIAGE_MAX_TOKENS = 4000; - -// Bounds enforced through the prompt so the structured output stays well inside -// TRIAGE_MAX_TOKENS regardless of how many models failed. -const TRIAGE_MAX_GROUPS = 6; const TRIAGE_MAX_ERROR_CHARS = 200; -// How much of a malformed model reply to surface in the Buildkite step log. -const TRIAGE_RAW_PREVIEW_CHARS = 300; - const TRIAGE_SYSTEM_PROMPT = 'You are an SRE assistant triaging failed LLM evaluation CI runs. Be concise and factual, and base every statement on the provided context.'; @@ -83,7 +73,6 @@ const TRIAGE_OUTPUT_INSTRUCTIONS = `Report the result by calling the \`${TRIAGE_ Rules: - One group per distinct error. Merge the same failure (same message/location) into one group and list all its affected models. -- Report at most ${TRIAGE_MAX_GROUPS} groups; if there are more distinct errors, keep the ones affecting the most models. - Quote "error" verbatim from the excerpts, trimmed to a single line of at most ${TRIAGE_MAX_ERROR_CHARS} characters; do not invent errors, file names, line numbers, or causes. - Set "location" only if the excerpts show it; otherwise use an empty string. - Keep "rootCause" to one short sentence plus one short action. @@ -305,20 +294,7 @@ function buildOpenrouterChatRequest(connector, messages, options = {}) { throw new Error('OpenRouter connector is missing apiUrl, defaultModel, or apiKey'); } - const { maxTokens = DEFAULT_MAX_TOKENS, tools, toolChoice } = options; - - const body = { - model: defaultModel, - messages, - temperature: 0.2, - max_tokens: maxTokens, - }; - if (Array.isArray(tools) && tools.length > 0) { - body.tools = tools; - if (toolChoice) { - body.tool_choice = toolChoice; - } - } + const { maxTokens = 800, tools, toolChoice } = options; return { url: apiUrl, @@ -326,28 +302,25 @@ function buildOpenrouterChatRequest(connector, messages, options = {}) { 'content-type': 'application/json', authorization: `Bearer ${apiKey}`, }, - body, + body: { + model: defaultModel, + messages, + temperature: 0.2, + max_tokens: maxTokens, + ...(tools ? { tools, tool_choice: toolChoice } : {}), + }, }; } -function parseOpenrouterChatChoice(responseJson) { +function parseOpenrouterChatContent(responseJson) { if (!responseJson || typeof responseJson !== 'object') { throw new Error('OpenRouter response was not JSON'); } - const choice = Array.isArray(responseJson.choices) ? responseJson.choices[0] : undefined; - const message = choice?.message && typeof choice.message === 'object' ? choice.message : {}; + const choices = responseJson.choices; - return { - content: typeof message.content === 'string' ? message.content : '', - toolCalls: Array.isArray(message.tool_calls) ? message.tool_calls : [], - finishReason: typeof choice?.finish_reason === 'string' ? choice.finish_reason : '', - }; -} - -function parseOpenrouterChatContent(responseJson) { - const { content } = parseOpenrouterChatChoice(responseJson); - if (!content.trim()) { + const content = choices?.[0]?.message?.content; + if (typeof content !== 'string' || !content.trim()) { throw new Error('OpenRouter response did not include message content'); } @@ -404,21 +377,6 @@ function resolveTriageConnector() { }; } -function triageError(message, raw) { - const error = /** @type {Error & { details?: string }} */ (new Error(message)); - const preview = redactSecrets(String(raw ?? '')) - .replace(/\s+/g, ' ') - .trim(); - if (preview) { - const clipped = - preview.length > TRIAGE_RAW_PREVIEW_CHARS - ? `${preview.slice(0, TRIAGE_RAW_PREVIEW_CHARS)}…` - : preview; - error.details = `Raw model output (${preview.length} chars): ${clipped}`; - } - return error; -} - /** * Parse the structured triage groups returned by the model. */ @@ -433,19 +391,7 @@ function parseTriageGroups(rawText) { try { parsed = JSON.parse(unfenced); } catch { - // Prose around the object: fall back to the outermost `{ ... }`. - const start = unfenced.indexOf('{'); - const end = unfenced.lastIndexOf('}'); - if (start >= 0 && end > start) { - try { - parsed = JSON.parse(unfenced.slice(start, end + 1)); - } catch { - // fall through - } - } - if (parsed === undefined) { - throw triageError('Triage model did not return valid JSON', text); - } + throw new Error('Triage model did not return valid JSON'); } const groups = Array.isArray(parsed?.groups) ? parsed.groups : []; @@ -482,46 +428,11 @@ async function runTriageModel(userPrompt, { maxChars = 1500 } = {}) { return { summary: summary.trim(), modelId }; } -/** - * Extract the triage groups from a forced `report_triage` tool call. - */ -function parseTriageToolCall(responseJson) { - const { content, toolCalls, finishReason } = parseOpenrouterChatChoice(responseJson); - - const call = - toolCalls.find((candidate) => candidate?.function?.name === TRIAGE_TOOL_NAME) ?? toolCalls[0]; - const rawArguments = call?.function?.arguments; - // The OpenAI-compatible contract is a JSON string, but some providers inline the object. - let args = ''; - if (typeof rawArguments === 'string') { - args = rawArguments; - } else if (rawArguments && typeof rawArguments === 'object') { - args = JSON.stringify(rawArguments); - } - - if (finishReason === 'length') { - throw triageError( - `Triage model output was truncated at the ${TRIAGE_MAX_TOKENS}-token limit`, - args || content - ); - } - - if (!args.trim()) { - throw triageError( - `Triage model did not call the ${TRIAGE_TOOL_NAME} tool (finish_reason: ${ - finishReason || 'unknown' - })`, - content - ); - } - - return parseTriageGroups(args); -} - /** * Resolve the OpenRouter connector, send the shared system prompt + the given user - * prompt, and return the parsed structured triage groups and the model id used. - * Used by the per-suite triage, which renders the message deterministically. + * prompt as a forced `report_triage` tool call, and return the parsed groups and + * the model id used. Used by the per-suite triage, which renders the message + * deterministically. */ async function runTriageModelStructured(userPrompt) { const { connector, modelId } = resolveTriageConnector(); @@ -539,16 +450,26 @@ async function runTriageModelStructured(userPrompt) { }) ); - return { groups: parseTriageToolCall(responseJson), modelId }; + const message = responseJson?.choices?.[0]?.message; + try { + return { groups: parseTriageGroups(message?.tool_calls?.[0]?.function?.arguments), modelId }; + } catch (error) { + // The Buildkite step log is the only place the raw reply is visible; the + // Slack/GitHub fallback line stays generic. + console.error( + `Raw triage model reply: ${redactSecrets(JSON.stringify(message ?? responseJson)).slice( + 0, + 500 + )}` + ); + throw error; + } } module.exports = { MAX_LOG_EXCERPT_CHARS, MAX_CONTEXT_JSON_BYTES, - DEFAULT_MAX_TOKENS, TRIAGE_MAX_TOKENS, - TRIAGE_MAX_GROUPS, - TRIAGE_MAX_ERROR_CHARS, TRIAGE_SYSTEM_PROMPT, TRIAGE_OPENROUTER_CONNECTOR_ID, TRIAGE_TOOL_NAME, @@ -563,13 +484,11 @@ module.exports = { buildWeeklyRollupUserPrompt, extractSuiteRootCauseLine, buildOpenrouterChatRequest, - parseOpenrouterChatChoice, parseOpenrouterChatContent, postOpenrouterChat, postOpenrouterChatRequest, resolveTriageConnector, parseTriageGroups, - parseTriageToolCall, runTriageModel, runTriageModelStructured, }; diff --git a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js index 0a51cc6411afd..e18e144e7d22e 100644 --- a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js +++ b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js @@ -13,12 +13,10 @@ const { TRIAGE_TOOL, TRIAGE_TOOL_CHOICE, TRIAGE_MAX_TOKENS, - DEFAULT_MAX_TOKENS, resolveTriageConnector, buildOpenrouterChatRequest, buildTriageUserPrompt, parseTriageGroups, - parseTriageToolCall, runTriageModelStructured, runTriageModel, } = require('./failure_context_helpers'); @@ -42,23 +40,29 @@ const GROUP = { rootCause: 'Assertion failed; update the expectation.', }; -function toolCallResponse(args, { finishReason = 'tool_calls', name = TRIAGE_TOOL_NAME } = {}) { +function toolCallResponse(args) { return { choices: [ { - finish_reason: finishReason, + finish_reason: 'tool_calls', message: { role: 'assistant', content: null, - tool_calls: [{ id: 'call_1', type: 'function', function: { name, arguments: args } }], + tool_calls: [ + { + id: 'call_1', + type: 'function', + function: { name: TRIAGE_TOOL_NAME, arguments: args }, + }, + ], }, }, ], }; } -function textResponse(content, finishReason = 'stop') { - return { choices: [{ finish_reason: finishReason, message: { role: 'assistant', content } }] }; +function textResponse(content) { + return { choices: [{ finish_reason: 'stop', message: { role: 'assistant', content } }] }; } function mockFetchJson(json) { @@ -124,10 +128,8 @@ describe('buildOpenrouterChatRequest', () => { model: 'test/model', messages: MESSAGES, temperature: 0.2, - max_tokens: DEFAULT_MAX_TOKENS, + max_tokens: 800, }); - expect(body).not.toHaveProperty('tools'); - expect(body).not.toHaveProperty('tool_choice'); }); it('adds the tool, forced tool choice, and a larger token budget when asked', () => { @@ -153,14 +155,13 @@ describe('buildOpenrouterChatRequest', () => { }); describe('buildTriageUserPrompt', () => { - it('asks the model to report through the triage tool and bounds the output', () => { + it('asks the model to report through the triage tool and bounds the error line', () => { const prompt = buildTriageUserPrompt( { suiteId: SUITE, failingProjects: ['gpt-5'], models: {} }, { suiteName: 'Significant Events', suiteId: SUITE, failingProjects: ['gpt-5'] } ); expect(prompt).toContain(`calling the \`${TRIAGE_TOOL_NAME}\` tool`); - expect(prompt).toContain('Report at most 6 groups'); expect(prompt).toContain('at most 200 characters'); expect(prompt).not.toContain('Return ONLY a JSON object'); }); @@ -177,13 +178,6 @@ describe('parseTriageGroups', () => { ); }); - it('recovers the object when the model wraps it in prose', () => { - const raw = `Here is the triage:\n${JSON.stringify({ - groups: [GROUP], - })}\nLet me know if you need more.`; - expect(parseTriageGroups(raw)).toEqual([GROUP]); - }); - it('coerces loose field types and drops empty groups', () => { const raw = JSON.stringify({ groups: [ @@ -200,116 +194,30 @@ describe('parseTriageGroups', () => { expect(parseTriageGroups('{}')).toEqual([]); }); - it('throws a short message and keeps a redacted preview of the raw output in details', () => { - const raw = `{"groups":[{"error":"Authorization: Bearer abc.def.ghi failed","models":["gpt-5"`; - let caught; - try { - parseTriageGroups(raw); - } catch (error) { - caught = error; - } - - expect(caught).toBeInstanceOf(Error); - expect(caught.message).toBe('Triage model did not return valid JSON'); - expect(caught.details).toContain('Raw model output'); - expect(caught.details).toContain('[REDACTED]'); - expect(caught.details).not.toContain('abc.def.ghi'); - }); - - it('clips the raw preview to 300 characters', () => { - const raw = `{${'x'.repeat(1000)}`; - let caught; - try { - parseTriageGroups(raw); - } catch (error) { - caught = error; - } - - expect(caught.details).toContain('(1001 chars)'); - expect(caught.details.length).toBeLessThan(400); - expect(caught.details.endsWith('…')).toBe(true); - }); -}); - -describe('parseTriageToolCall', () => { - it('reads the groups from the forced tool call arguments', () => { - const response = toolCallResponse(JSON.stringify({ groups: [GROUP] })); - expect(parseTriageToolCall(response)).toEqual([GROUP]); - }); - - it('accepts providers that inline the arguments as an object', () => { - const response = toolCallResponse({ groups: [GROUP] }); - expect(parseTriageToolCall(response)).toEqual([GROUP]); - }); - - it('prefers the report_triage call when several tool calls are present', () => { - const response = toolCallResponse(JSON.stringify({ groups: [GROUP] })); - response.choices[0].message.tool_calls.unshift({ - id: 'call_0', - type: 'function', - function: { name: 'other_tool', arguments: '{"groups":[]}' }, - }); - expect(parseTriageToolCall(response)).toEqual([GROUP]); - }); - - it('reports truncation at the token limit instead of a generic JSON error', () => { - const response = toolCallResponse('{"groups":[{"error":"boom","mo', { finishReason: 'length' }); - let caught; - try { - parseTriageToolCall(response); - } catch (error) { - caught = error; - } - - expect(caught.message).toBe( - `Triage model output was truncated at the ${TRIAGE_MAX_TOKENS}-token limit` - ); - expect(caught.details).toContain('{"groups":[{"error":"boom","mo'); - }); - - it('reports a missing tool call when the model answered in text', () => { - let caught; - try { - parseTriageToolCall(textResponse('Sure! Here is my analysis of the failures...')); - } catch (error) { - caught = error; - } - - expect(caught.message).toBe( - `Triage model did not call the ${TRIAGE_TOOL_NAME} tool (finish_reason: stop)` + it('throws on anything that is not JSON, including a missing tool call', () => { + expect(() => parseTriageGroups('not json')).toThrow('Triage model did not return valid JSON'); + expect(() => parseTriageGroups('{"groups":[{"error":"cut')).toThrow( + 'Triage model did not return valid JSON' ); - expect(caught.details).toContain('Sure! Here is my analysis'); - }); - - it('reports malformed tool arguments as invalid JSON with a preview', () => { - let caught; - try { - parseTriageToolCall(toolCallResponse('not json at all')); - } catch (error) { - caught = error; - } - - expect(caught.message).toBe('Triage model did not return valid JSON'); - expect(caught.details).toContain('not json at all'); - }); - - it('rejects a non-object response', () => { - expect(() => parseTriageToolCall(null)).toThrow('OpenRouter response was not JSON'); + expect(() => parseTriageGroups(undefined)).toThrow('Triage model did not return valid JSON'); }); }); describe('runTriageModelStructured / runTriageModel (mocked fetch)', () => { const originalEnv = { ...process.env }; const originalFetch = global.fetch; + let consoleError; beforeEach(() => { process.env.OPENROUTER_BASE_URL = 'https://openrouter.test/api/v1'; process.env.OPENROUTER_API_KEY = 'sk-test'; + consoleError = jest.spyOn(console, 'error').mockImplementation(() => {}); }); afterEach(() => { process.env = { ...originalEnv }; global.fetch = originalFetch; + consoleError.mockRestore(); }); it('sends the tool-forced request and returns the parsed groups', async () => { @@ -327,14 +235,42 @@ describe('runTriageModelStructured / runTriageModel (mocked fetch)', () => { expect(body.tools).toEqual([TRIAGE_TOOL]); expect(body.tool_choice).toEqual(TRIAGE_TOOL_CHOICE); expect(body.messages[1]).toEqual({ role: 'user', content: 'triage this' }); + expect(consoleError).not.toHaveBeenCalled(); + }); + + it('returns no groups when the model reports none', async () => { + global.fetch = mockFetchJson(toolCallResponse('{"groups":[]}')); + + await expect(runTriageModelStructured('triage this')).resolves.toEqual({ + groups: [], + modelId: TRIAGE_OPENROUTER_CONNECTOR_ID, + }); + }); + + it('fails generically and logs the redacted raw reply when the model answered in text', async () => { + global.fetch = mockFetchJson( + textResponse('Sure! Here is my analysis. Authorization: Bearer abc.def.ghi') + ); + + await expect(runTriageModelStructured('triage this')).rejects.toThrow( + 'Triage model did not return valid JSON' + ); + expect(consoleError).toHaveBeenCalledTimes(1); + const logged = consoleError.mock.calls[0][0]; + expect(logged).toContain('Raw triage model reply:'); + expect(logged).toContain('Sure! Here is my analysis'); + expect(logged).toContain('[REDACTED]'); + expect(logged).not.toContain('abc.def.ghi'); }); - it('surfaces a truncated reply as a token-limit error', async () => { - global.fetch = mockFetchJson(toolCallResponse('{"groups":[', { finishReason: 'length' })); + it('fails generically and logs the raw reply when the tool arguments are cut off', async () => { + global.fetch = mockFetchJson(toolCallResponse('{"groups":[{"error":"boom","mo')); await expect(runTriageModelStructured('triage this')).rejects.toThrow( - 'truncated at the 4000-token limit' + 'Triage model did not return valid JSON' ); + expect(consoleError.mock.calls[0][0]).toContain('Raw triage model reply:'); + expect(consoleError.mock.calls[0][0]).toContain('boom'); }); it('keeps the weekly rollup on the plain text request', async () => { @@ -344,7 +280,7 @@ describe('runTriageModelStructured / runTriageModel (mocked fetch)', () => { expect(result.summary).toBe('- Overall: 1 suite likely retryable.'); const body = JSON.parse(global.fetch.mock.calls[0][1].body); - expect(body.max_tokens).toBe(DEFAULT_MAX_TOKENS); + expect(body.max_tokens).toBe(800); expect(body).not.toHaveProperty('tools'); expect(body).not.toHaveProperty('tool_choice'); }); From 7673a9f402530c0b461f4c870fb674a11d318906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Lidue=C3=B1a?= Date: Fri, 11 Sep 2026 12:48:51 +0200 Subject: [PATCH 3/4] clean-up dead code --- .../packages/shared/kbn-evals/scripts/ci/ai_connectors.js | 1 - .../shared/kbn-evals/scripts/ci/failure_context_helpers.js | 6 +----- .../kbn-evals/scripts/ci/failure_context_helpers.test.js | 6 ------ 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/ai_connectors.js b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/ai_connectors.js index 9d8080e4514d4..6c36045841b7e 100644 --- a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/ai_connectors.js +++ b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/ai_connectors.js @@ -53,7 +53,6 @@ function parseVaultConfig() { // CI notify triage uses this OpenRouter model. // Model must support OpenAI-style tool calling with `tool_choice` on OpenRouter. -// Models without it fail every triage with "did not call the report_triage tool". const TRIAGE_OPENROUTER_MODEL = 'google/gemini-3.7-flash'; /** diff --git a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js index 0740f451b8a86..46215164f6d34 100644 --- a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js +++ b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js @@ -382,14 +382,10 @@ function resolveTriageConnector() { */ function parseTriageGroups(rawText) { const text = String(rawText ?? '').trim(); - const unfenced = text - .replace(/^```(?:json)?\s*/i, '') - .replace(/\s*```$/i, '') - .trim(); let parsed; try { - parsed = JSON.parse(unfenced); + parsed = JSON.parse(text); } catch { throw new Error('Triage model did not return valid JSON'); } diff --git a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js index e18e144e7d22e..6a760b0492ee3 100644 --- a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js +++ b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js @@ -172,12 +172,6 @@ describe('parseTriageGroups', () => { expect(parseTriageGroups(JSON.stringify({ groups: [GROUP] }))).toEqual([GROUP]); }); - it('strips a markdown code fence', () => { - expect(parseTriageGroups(`\`\`\`json\n${JSON.stringify({ groups: [GROUP] })}\n\`\`\``)).toEqual( - [GROUP] - ); - }); - it('coerces loose field types and drops empty groups', () => { const raw = JSON.stringify({ groups: [ From 7fbf136c9bd16c7c0c51e8880d3566cd6ac729e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Lidue=C3=B1a?= Date: Sat, 12 Sep 2026 12:08:10 +0200 Subject: [PATCH 4/4] replaced the try/catch block with an explicit args === undefined guard. --- .../scripts/ci/failure_context_helpers.js | 17 ++++++++--------- .../scripts/ci/failure_context_helpers.test.js | 11 +++++------ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js index 46215164f6d34..0d30da7573575 100644 --- a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js +++ b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js @@ -69,7 +69,7 @@ const TRIAGE_TOOL = { const TRIAGE_TOOL_CHOICE = { type: 'function', function: { name: TRIAGE_TOOL_NAME } }; // Static output contract for the per-suite structured triage prompt. -const TRIAGE_OUTPUT_INSTRUCTIONS = `Report the result by calling the \`${TRIAGE_TOOL_NAME}\` tool with {"groups":[{"error":"","location":"","models":[""],"rootCause":""}]}. Do not answer in prose. +const TRIAGE_OUTPUT_INSTRUCTIONS = `Report the result by calling the \`${TRIAGE_TOOL_NAME}\` tool with {"groups":[{"error":"","location":"","models":[""],"rootCause":""}]}. Rules: - One group per distinct error. Merge the same failure (same message/location) into one group and list all its affected models. @@ -447,19 +447,18 @@ async function runTriageModelStructured(userPrompt) { ); const message = responseJson?.choices?.[0]?.message; - try { - return { groups: parseTriageGroups(message?.tool_calls?.[0]?.function?.arguments), modelId }; - } catch (error) { + const args = message?.tool_calls?.[0]?.function?.arguments; + if (args === undefined) { // The Buildkite step log is the only place the raw reply is visible; the // Slack/GitHub fallback line stays generic. console.error( - `Raw triage model reply: ${redactSecrets(JSON.stringify(message ?? responseJson)).slice( - 0, - 500 - )}` + `Triage model ignored the tool call. Raw reply: ${redactSecrets( + JSON.stringify(message ?? responseJson) + ).slice(0, 1000)}` ); - throw error; + throw new Error('Triage model did not call the report_triage tool'); } + return { groups: parseTriageGroups(args), modelId }; } module.exports = { diff --git a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js index 6a760b0492ee3..d1382f728c9d7 100644 --- a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js +++ b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js @@ -241,30 +241,29 @@ describe('runTriageModelStructured / runTriageModel (mocked fetch)', () => { }); }); - it('fails generically and logs the redacted raw reply when the model answered in text', async () => { + it('logs the redacted raw reply and throws when the model answered in text instead of calling the tool', async () => { global.fetch = mockFetchJson( textResponse('Sure! Here is my analysis. Authorization: Bearer abc.def.ghi') ); await expect(runTriageModelStructured('triage this')).rejects.toThrow( - 'Triage model did not return valid JSON' + 'Triage model did not call the report_triage tool' ); expect(consoleError).toHaveBeenCalledTimes(1); const logged = consoleError.mock.calls[0][0]; - expect(logged).toContain('Raw triage model reply:'); + expect(logged).toContain('Triage model ignored the tool call. Raw reply:'); expect(logged).toContain('Sure! Here is my analysis'); expect(logged).toContain('[REDACTED]'); expect(logged).not.toContain('abc.def.ghi'); }); - it('fails generically and logs the raw reply when the tool arguments are cut off', async () => { + it('throws when the tool arguments are cut off (malformed JSON bubbles directly, no extra log)', async () => { global.fetch = mockFetchJson(toolCallResponse('{"groups":[{"error":"boom","mo')); await expect(runTriageModelStructured('triage this')).rejects.toThrow( 'Triage model did not return valid JSON' ); - expect(consoleError.mock.calls[0][0]).toContain('Raw triage model reply:'); - expect(consoleError.mock.calls[0][0]).toContain('boom'); + expect(consoleError).not.toHaveBeenCalled(); }); it('keeps the weekly rollup on the plain text request', async () => {