diff --git a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/ai_connectors.js b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/ai_connectors.js index be25738cca2a6..6c36045841b7e 100644 --- a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/ai_connectors.js +++ b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/ai_connectors.js @@ -52,6 +52,7 @@ function parseVaultConfig() { } // CI notify triage uses this OpenRouter model. +// Model must support OpenAI-style tool calling with `tool_choice` on OpenRouter. const TRIAGE_OPENROUTER_MODEL = 'google/gemini-3.7-flash'; /** diff --git a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js index 4da2267c82d66..0d30da7573575 100644 --- a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js +++ b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.js @@ -15,21 +15,69 @@ const MAX_CONTEXT_JSON_BYTES = 30 * 1024; // Triage/summary text is always generated with a small, low-cost OpenRouter model. const TRIAGE_OPENROUTER_CONNECTOR_ID = `openrouter-${slugifyId(TRIAGE_OPENROUTER_MODEL)}`; +// Output budget for the per-suite structured triage. +const TRIAGE_MAX_TOKENS = 4000; +const TRIAGE_MAX_ERROR_CHARS = 200; + const TRIAGE_SYSTEM_PROMPT = 'You are an SRE assistant triaging failed LLM evaluation CI runs. Be concise and factual, and base every statement on the provided context.'; +const TRIAGE_TOOL_NAME = 'report_triage'; + +const TRIAGE_TOOL = { + type: 'function', + function: { + name: TRIAGE_TOOL_NAME, + description: + 'Report the triage of a failed LLM evaluation CI suite as a list of distinct error groups.', + parameters: { + type: 'object', + properties: { + groups: { + type: 'array', + description: 'One entry per distinct error. Empty when the excerpts show no clear error.', + items: { + type: 'object', + properties: { + error: { + type: 'string', + description: `Most relevant error line, verbatim from the excerpts, one line, at most ${TRIAGE_MAX_ERROR_CHARS} characters.`, + }, + location: { + type: 'string', + description: 'file:line, test, or scenario if shown in the excerpts; else empty.', + }, + models: { + type: 'array', + items: { type: 'string' }, + description: 'Ids of every failing model that hit this error.', + }, + rootCause: { + type: 'string', + description: 'One short sentence with the likely cause plus one short action.', + }, + }, + required: ['error', 'location', 'models', 'rootCause'], + }, + }, + }, + required: ['groups'], + }, + }, +}; + +const TRIAGE_TOOL_CHOICE = { type: 'function', function: { name: TRIAGE_TOOL_NAME } }; + // Static output contract for the per-suite structured triage prompt. -const TRIAGE_OUTPUT_INSTRUCTIONS = `Return ONLY a JSON object of this shape (no prose, no markdown, no code fences): -{"groups":[{"error":"","location":"","models":[""],"rootCause":""}]} +const TRIAGE_OUTPUT_INSTRUCTIONS = `Report the result by calling the \`${TRIAGE_TOOL_NAME}\` tool with {"groups":[{"error":"","location":"","models":[""],"rootCause":""}]}. Rules: - One group per distinct error. Merge the same failure (same message/location) into one group and list all its affected models. -- Quote "error" verbatim from the excerpts; do not invent errors, file names, line numbers, or causes. +- Quote "error" verbatim from the excerpts, trimmed to a single line of at most ${TRIAGE_MAX_ERROR_CHARS} characters; do not invent errors, file names, line numbers, or causes. - Set "location" only if the excerpts show it; otherwise use an empty string. - Keep "rootCause" to one short sentence plus one short action. - Never reproduce secrets: if a line contains an API key, token, password, or other credential, replace that value with [REDACTED]. -- If the excerpts show no clear error, return {"groups": []}. -- Output valid JSON only: no prose before or after, no comments, no trailing commas, no code fences.`; +- If the excerpts show no clear error, call the tool with {"groups": []}.`; // Static output contract for the weekly cross-suite rollup prompt. const WEEKLY_ROLLUP_OUTPUT_INSTRUCTIONS = `Output exactly these bullets, in this order, and nothing else: @@ -233,7 +281,7 @@ function buildWeeklyRollupUserPrompt(suites, meta = {}) { return lines.join('\n'); } -function buildOpenrouterChatRequest(connector, messages) { +function buildOpenrouterChatRequest(connector, messages, options = {}) { const config = connector.config && typeof connector.config === 'object' ? connector.config : {}; const secrets = connector.secrets && typeof connector.secrets === 'object' ? connector.secrets : {}; @@ -246,6 +294,8 @@ function buildOpenrouterChatRequest(connector, messages) { throw new Error('OpenRouter connector is missing apiUrl, defaultModel, or apiKey'); } + const { maxTokens = 800, tools, toolChoice } = options; + return { url: apiUrl, headers: { @@ -256,7 +306,8 @@ function buildOpenrouterChatRequest(connector, messages) { model: defaultModel, messages, temperature: 0.2, - max_tokens: 800, + max_tokens: maxTokens, + ...(tools ? { tools, tool_choice: toolChoice } : {}), }, }; } @@ -266,9 +317,7 @@ function parseOpenrouterChatContent(responseJson) { throw new Error('OpenRouter response was not JSON'); } - const choices = /** @type {{ choices?: Array<{ message?: { content?: string } }> }} */ ( - responseJson - ).choices; + const choices = responseJson.choices; const content = choices?.[0]?.message?.content; if (typeof content !== 'string' || !content.trim()) { @@ -279,9 +328,9 @@ function parseOpenrouterChatContent(responseJson) { } /** - * POST a built OpenRouter chat request and return the message content. + * POST a built OpenRouter chat request and return the parsed response body. */ -async function postOpenrouterChatRequest({ url, headers, body }) { +async function postOpenrouterChat({ url, headers, body }) { const response = await fetch(url, { method: 'POST', headers, @@ -302,14 +351,18 @@ async function postOpenrouterChatRequest({ url, headers, body }) { throw new Error(`Inference request failed (${response.status}): ${detail}`); } - let json = null; try { - json = text ? JSON.parse(text) : null; + return text ? JSON.parse(text) : null; } catch { throw new Error('Inference response was not JSON'); } +} - return parseOpenrouterChatContent(json); +/** + * POST a built OpenRouter chat request and return the message content. + */ +async function postOpenrouterChatRequest(request) { + return parseOpenrouterChatContent(await postOpenrouterChat(request)); } /** @@ -329,14 +382,10 @@ function resolveTriageConnector() { */ function parseTriageGroups(rawText) { const text = String(rawText ?? '').trim(); - const unfenced = text - .replace(/^```(?:json)?\s*/i, '') - .replace(/\s*```$/i, '') - .trim(); let parsed; try { - parsed = JSON.parse(unfenced); + parsed = JSON.parse(text); } catch { throw new Error('Triage model did not return valid JSON'); } @@ -377,8 +426,9 @@ async function runTriageModel(userPrompt, { maxChars = 1500 } = {}) { /** * Resolve the OpenRouter connector, send the shared system prompt + the given user - * prompt, and return the parsed structured triage groups and the model id used. - * Used by the per-suite triage, which renders the message deterministically. + * prompt as a forced `report_triage` tool call, and return the parsed groups and + * the model id used. Used by the per-suite triage, which renders the message + * deterministically. */ async function runTriageModelStructured(userPrompt) { const { connector, modelId } = resolveTriageConnector(); @@ -388,16 +438,38 @@ async function runTriageModelStructured(userPrompt) { { role: 'user', content: userPrompt }, ]; - const raw = await postOpenrouterChatRequest(buildOpenrouterChatRequest(connector, messages)); + const responseJson = await postOpenrouterChat( + buildOpenrouterChatRequest(connector, messages, { + maxTokens: TRIAGE_MAX_TOKENS, + tools: [TRIAGE_TOOL], + toolChoice: TRIAGE_TOOL_CHOICE, + }) + ); - return { groups: parseTriageGroups(raw), modelId }; + const message = responseJson?.choices?.[0]?.message; + const args = message?.tool_calls?.[0]?.function?.arguments; + if (args === undefined) { + // The Buildkite step log is the only place the raw reply is visible; the + // Slack/GitHub fallback line stays generic. + console.error( + `Triage model ignored the tool call. Raw reply: ${redactSecrets( + JSON.stringify(message ?? responseJson) + ).slice(0, 1000)}` + ); + throw new Error('Triage model did not call the report_triage tool'); + } + return { groups: parseTriageGroups(args), modelId }; } module.exports = { MAX_LOG_EXCERPT_CHARS, MAX_CONTEXT_JSON_BYTES, + TRIAGE_MAX_TOKENS, TRIAGE_SYSTEM_PROMPT, TRIAGE_OPENROUTER_CONNECTOR_ID, + TRIAGE_TOOL_NAME, + TRIAGE_TOOL, + TRIAGE_TOOL_CHOICE, failureLogMetadataKey, failureLogMetadataKeysForProject, truncateText, @@ -408,6 +480,7 @@ module.exports = { extractSuiteRootCauseLine, buildOpenrouterChatRequest, parseOpenrouterChatContent, + postOpenrouterChat, postOpenrouterChatRequest, resolveTriageConnector, parseTriageGroups, diff --git a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js index a09c55fbc5a04..d1382f728c9d7 100644 --- a/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js +++ b/x-pack/platform/packages/shared/kbn-evals/scripts/ci/failure_context_helpers.test.js @@ -9,11 +9,70 @@ const { failureLogMetadataKey, failureLogMetadataKeysForProject, TRIAGE_OPENROUTER_CONNECTOR_ID, + TRIAGE_TOOL_NAME, + TRIAGE_TOOL, + TRIAGE_TOOL_CHOICE, + TRIAGE_MAX_TOKENS, resolveTriageConnector, + buildOpenrouterChatRequest, + buildTriageUserPrompt, + parseTriageGroups, + runTriageModelStructured, + runTriageModel, } = require('./failure_context_helpers'); const SUITE = 'significant-events'; +const CONNECTOR = { + config: { apiUrl: 'https://openrouter.test/api/v1/chat/completions', defaultModel: 'test/model' }, + secrets: { apiKey: 'sk-test' }, +}; + +const MESSAGES = [ + { role: 'system', content: 'system' }, + { role: 'user', content: 'user' }, +]; + +const GROUP = { + error: 'Error: expect(received).toBe(expected)', + location: 'find_rules.spec.ts:484', + models: ['eis-openai-gpt-5-4'], + rootCause: 'Assertion failed; update the expectation.', +}; + +function toolCallResponse(args) { + return { + choices: [ + { + finish_reason: 'tool_calls', + message: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'call_1', + type: 'function', + function: { name: TRIAGE_TOOL_NAME, arguments: args }, + }, + ], + }, + }, + ], + }; +} + +function textResponse(content) { + return { choices: [{ finish_reason: 'stop', message: { role: 'assistant', content } }] }; +} + +function mockFetchJson(json) { + return jest.fn().mockResolvedValue({ + ok: true, + status: 200, + text: () => Promise.resolve(JSON.stringify(json)), + }); +} + describe('failureLogMetadataKey', () => { it('slugifies the suite and project into a stable key', () => { expect(failureLogMetadataKey(SUITE, 'eis/openai-gpt-5.4')).toBe( @@ -59,6 +118,167 @@ describe('failureLogMetadataKeysForProject', () => { }); }); +describe('buildOpenrouterChatRequest', () => { + it('builds a plain text request by default (weekly rollup path)', () => { + const { url, headers, body } = buildOpenrouterChatRequest(CONNECTOR, MESSAGES); + + expect(url).toBe(CONNECTOR.config.apiUrl); + expect(headers.authorization).toBe('Bearer sk-test'); + expect(body).toEqual({ + model: 'test/model', + messages: MESSAGES, + temperature: 0.2, + max_tokens: 800, + }); + }); + + it('adds the tool, forced tool choice, and a larger token budget when asked', () => { + const { body } = buildOpenrouterChatRequest(CONNECTOR, MESSAGES, { + maxTokens: TRIAGE_MAX_TOKENS, + tools: [TRIAGE_TOOL], + toolChoice: TRIAGE_TOOL_CHOICE, + }); + + expect(body.max_tokens).toBe(TRIAGE_MAX_TOKENS); + expect(body.tools).toEqual([TRIAGE_TOOL]); + expect(body.tool_choice).toEqual({ + type: 'function', + function: { name: TRIAGE_TOOL_NAME }, + }); + }); + + it('throws when the connector is incomplete', () => { + expect(() => buildOpenrouterChatRequest({ config: {}, secrets: {} }, MESSAGES)).toThrow( + 'OpenRouter connector is missing apiUrl, defaultModel, or apiKey' + ); + }); +}); + +describe('buildTriageUserPrompt', () => { + it('asks the model to report through the triage tool and bounds the error line', () => { + const prompt = buildTriageUserPrompt( + { suiteId: SUITE, failingProjects: ['gpt-5'], models: {} }, + { suiteName: 'Significant Events', suiteId: SUITE, failingProjects: ['gpt-5'] } + ); + + expect(prompt).toContain(`calling the \`${TRIAGE_TOOL_NAME}\` tool`); + expect(prompt).toContain('at most 200 characters'); + expect(prompt).not.toContain('Return ONLY a JSON object'); + }); +}); + +describe('parseTriageGroups', () => { + it('parses a clean JSON object', () => { + expect(parseTriageGroups(JSON.stringify({ groups: [GROUP] }))).toEqual([GROUP]); + }); + + it('coerces loose field types and drops empty groups', () => { + const raw = JSON.stringify({ + groups: [ + { error: ' boom ', models: 'not-an-array', rootCause: 42 }, + { error: '', location: 'x', models: [], rootCause: '' }, + ], + }); + expect(parseTriageGroups(raw)).toEqual([ + { error: 'boom', location: '', models: [], rootCause: '42' }, + ]); + }); + + it('returns no groups when "groups" is missing', () => { + expect(parseTriageGroups('{}')).toEqual([]); + }); + + it('throws on anything that is not JSON, including a missing tool call', () => { + expect(() => parseTriageGroups('not json')).toThrow('Triage model did not return valid JSON'); + expect(() => parseTriageGroups('{"groups":[{"error":"cut')).toThrow( + 'Triage model did not return valid JSON' + ); + expect(() => parseTriageGroups(undefined)).toThrow('Triage model did not return valid JSON'); + }); +}); + +describe('runTriageModelStructured / runTriageModel (mocked fetch)', () => { + const originalEnv = { ...process.env }; + const originalFetch = global.fetch; + let consoleError; + + beforeEach(() => { + process.env.OPENROUTER_BASE_URL = 'https://openrouter.test/api/v1'; + process.env.OPENROUTER_API_KEY = 'sk-test'; + consoleError = jest.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + process.env = { ...originalEnv }; + global.fetch = originalFetch; + consoleError.mockRestore(); + }); + + it('sends the tool-forced request and returns the parsed groups', async () => { + global.fetch = mockFetchJson(toolCallResponse(JSON.stringify({ groups: [GROUP] }))); + + const result = await runTriageModelStructured('triage this'); + + expect(result).toEqual({ groups: [GROUP], modelId: TRIAGE_OPENROUTER_CONNECTOR_ID }); + expect(global.fetch).toHaveBeenCalledTimes(1); + + const [url, init] = global.fetch.mock.calls[0]; + expect(url).toBe('https://openrouter.test/api/v1/chat/completions'); + const body = JSON.parse(init.body); + expect(body.max_tokens).toBe(TRIAGE_MAX_TOKENS); + expect(body.tools).toEqual([TRIAGE_TOOL]); + expect(body.tool_choice).toEqual(TRIAGE_TOOL_CHOICE); + expect(body.messages[1]).toEqual({ role: 'user', content: 'triage this' }); + expect(consoleError).not.toHaveBeenCalled(); + }); + + it('returns no groups when the model reports none', async () => { + global.fetch = mockFetchJson(toolCallResponse('{"groups":[]}')); + + await expect(runTriageModelStructured('triage this')).resolves.toEqual({ + groups: [], + modelId: TRIAGE_OPENROUTER_CONNECTOR_ID, + }); + }); + + it('logs the redacted raw reply and throws when the model answered in text instead of calling the tool', async () => { + global.fetch = mockFetchJson( + textResponse('Sure! Here is my analysis. Authorization: Bearer abc.def.ghi') + ); + + await expect(runTriageModelStructured('triage this')).rejects.toThrow( + 'Triage model did not call the report_triage tool' + ); + expect(consoleError).toHaveBeenCalledTimes(1); + const logged = consoleError.mock.calls[0][0]; + expect(logged).toContain('Triage model ignored the tool call. Raw reply:'); + expect(logged).toContain('Sure! Here is my analysis'); + expect(logged).toContain('[REDACTED]'); + expect(logged).not.toContain('abc.def.ghi'); + }); + + it('throws when the tool arguments are cut off (malformed JSON bubbles directly, no extra log)', async () => { + global.fetch = mockFetchJson(toolCallResponse('{"groups":[{"error":"boom","mo')); + + await expect(runTriageModelStructured('triage this')).rejects.toThrow( + 'Triage model did not return valid JSON' + ); + expect(consoleError).not.toHaveBeenCalled(); + }); + + it('keeps the weekly rollup on the plain text request', async () => { + global.fetch = mockFetchJson(textResponse('- Overall: 1 suite likely retryable.')); + + const result = await runTriageModel('summarize'); + + expect(result.summary).toBe('- Overall: 1 suite likely retryable.'); + const body = JSON.parse(global.fetch.mock.calls[0][1].body); + expect(body.max_tokens).toBe(800); + expect(body).not.toHaveProperty('tools'); + expect(body).not.toHaveProperty('tool_choice'); + }); +}); + describe('resolveTriageConnector', () => { const originalEnv = { ...process.env };