[Threat Intel] Enrichment eval suite for the LLM-backed routes (#289345)

Author: stephmilovic

.buildkite/pipelines/evals/evals.suites.json

@@ -304,6 +304,19 @@
       ],
       "serverConfigSet": "evals_lead_generation"
     },
+    {
+      "id": "threat-intel-enrichment",
+      "name": "Threat Intel Enrichment",
+      "configPath": "x-pack/solutions/security/packages/kbn-evals-suite-threat-intel-enrichment/playwright.config.ts",
+      "tags": [
+        "security",
+        "threat-intel-enrichment"
+      ],
+      "ciLabels": [
+        "evals:threat-intel-enrichment"
+      ],
+      "serverConfigSet": "evals_threat_intel"
+    },
     {
       "id": "workflows",
       "name": "Workflows Authoring",

.github/CODEOWNERS

@@ -1421,6 +1421,7 @@ x-pack/solutions/security/packages/kbn-evals-suite-security-automatic-migrations
 x-pack/solutions/security/packages/kbn-evals-suite-security-esql-generation-regression @elastic/security-detection-platform
 x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix @elastic/security-generative-ai
 x-pack/solutions/security/packages/kbn-evals-suite-siem-readiness @elastic/contextual-security-apps
+x-pack/solutions/security/packages/kbn-evals-suite-threat-intel-enrichment @elastic/security-threat-hunting
 x-pack/solutions/security/packages/kbn-scout-security @elastic/appex-qa @elastic/security-engineering-productivity
 x-pack/solutions/security/packages/kbn-security-evals-alerts-snapshot @elastic/security-generative-ai
 x-pack/solutions/security/packages/kbn-security-evals-workflow-traces @elastic/security-generative-ai

package.json

@@ -1855,6 +1855,7 @@
     "@kbn/evals-suite-significant-events": "workspace:*",
     "@kbn/evals-suite-smoke-tests": "workspace:*",
     "@kbn/evals-suite-streams": "workspace:*",
+    "@kbn/evals-suite-threat-intel-enrichment": "workspace:*",
     "@kbn/evals-suite-workflows": "workspace:*",
     "@kbn/expect": "workspace:*",
     "@kbn/extract-plugin-translations": "workspace:*",

pnpm-lock.yaml

@@ -1610,6 +1610,7 @@ packages:
   - 'x-pack/solutions/security/packages/kbn-evals-suite-security-esql-generation-regression'
   - 'x-pack/solutions/security/packages/kbn-evals-suite-security-persona-matrix'
   - 'x-pack/solutions/security/packages/kbn-evals-suite-siem-readiness'
+  - 'x-pack/solutions/security/packages/kbn-evals-suite-threat-intel-enrichment'
   - 'x-pack/solutions/security/packages/kbn-scout-security'
   - 'x-pack/solutions/security/packages/kbn-security-evals-alerts-snapshot'
   - 'x-pack/solutions/security/packages/kbn-security-evals-workflow-traces'

pnpm-workspace.yaml

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { servers as evalsTracingConfig } from '../../evals_tracing/stateful/classic.stateful.config';
+import type { ScoutServerConfig } from '../../../../../types';
+
+/**
+ * Custom Scout stateful server configuration for Threat Intel enrichment evals.
+ *
+ * Extends the tracing config (EIS connectors + eval tracing) and:
+ *   - Enables the `threatIntelSupplyEnabled` experimental feature flag so the
+ *     `/internal/threat_intel/*` enrichment routes are registered.
+ *   - Disables `searchInferenceEndpoints` so `resolveScopedModel` takes its
+ *     `genAiSettings:defaultAIConnector` fallback path. The suite points that
+ *     setting at the per-project connector, which is how each model in the
+ *     EIS/LiteLLM matrix actually gets exercised. With the registry present the
+ *     route would deliberately refuse the global-default fallback and return a
+ *     no_connector error.
+ *
+ * Usage:
+ *   node scripts/scout start-server --arch stateful --domain classic --serverConfigSet evals_threat_intel
+ */
+export const servers: ScoutServerConfig = {
+  ...evalsTracingConfig,
+  kbnTestServer: {
+    ...evalsTracingConfig.kbnTestServer,
+    serverArgs: [
+      ...evalsTracingConfig.kbnTestServer.serverArgs,
+      `--xpack.securitySolution.enableExperimental=${JSON.stringify(['threatIntelSupplyEnabled'])}`,
+      '--xpack.searchInferenceEndpoints.enabled=false',
+    ],
+  },
+};

src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_threat_intel/stateful/classic.stateful.config.ts

@@ -1375,6 +1375,8 @@
       "@kbn/evals-suite-smoke-tests/*": ["./x-pack/platform/packages/shared/kbn-evals-suite-smoke-tests/*"],
       "@kbn/evals-suite-streams": ["./x-pack/platform/packages/shared/kbn-evals-suite-streams"],
       "@kbn/evals-suite-streams/*": ["./x-pack/platform/packages/shared/kbn-evals-suite-streams/*"],
+      "@kbn/evals-suite-threat-intel-enrichment": ["./x-pack/solutions/security/packages/kbn-evals-suite-threat-intel-enrichment"],
+      "@kbn/evals-suite-threat-intel-enrichment/*": ["./x-pack/solutions/security/packages/kbn-evals-suite-threat-intel-enrichment/*"],
       "@kbn/evals-suite-workflows": ["./src/platform/packages/shared/kbn-evals-suite-workflows"],
       "@kbn/evals-suite-workflows/*": ["./src/platform/packages/shared/kbn-evals-suite-workflows/*"],
       "@kbn/event-annotation-common": ["./src/platform/packages/shared/kbn-event-annotation-common"],

tsconfig.base.json

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { tags } from '@kbn/evals';
+import { evaluate } from '../../src/evaluate';
+import { assessRelevanceDataset } from '../../src/datasets/assess_relevance_dataset';
+import {
+  createIsIntelligenceEvaluator,
+  createRelevanceShapeEvaluator,
+} from '../../src/evaluators/assess_relevance_evaluators';
+
+evaluate.describe(
+  'Threat Intel Enrichment: assess_relevance',
+  { tag: tags.stateful.classic },
+  () => {
+    evaluate('classifies intelligence vs noise', async ({ executorClient, threatIntelClient }) => {
+      await executorClient.runExperiment(
+        {
+          datasets: [
+            {
+              name: 'threat_intel: assess_relevance',
+              description:
+                'BlackHat demo packs (intelligence) plus authored marketing/opinion distractors (not intelligence).',
+              examples: assessRelevanceDataset,
+            },
+          ],
+          task: async ({ input }) => {
+            if (!input) throw new Error('Missing assess_relevance input');
+            return threatIntelClient.assessRelevance(input);
+          },
+        },
+        [createIsIntelligenceEvaluator(), createRelevanceShapeEvaluator()]
+      );
+    });
+  }
+);

x-pack/solutions/security/packages/kbn-evals-suite-threat-intel-enrichment/evals/threat_intel_enrichment/assess_relevance.spec.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { tags } from '@kbn/evals';
+import { evaluate } from '../../src/evaluate';
+import { classifySeverityDataset } from '../../src/datasets/classify_severity_dataset';
+import {
+  createSeverityAdjacentEvaluator,
+  createSeverityExactEvaluator,
+} from '../../src/evaluators/classify_severity_evaluators';
+
+evaluate.describe(
+  'Threat Intel Enrichment: classify_severity',
+  { tag: tags.stateful.classic },
+  () => {
+    evaluate(
+      'assigns severity across the ladder',
+      async ({ executorClient, threatIntelClient }) => {
+        await executorClient.runExperiment(
+          {
+            datasets: [
+              {
+                name: 'threat_intel: classify_severity',
+                description:
+                  'Authored low/medium/high/critical ladder plus two fixture-derived anchors. Scored exact and within-one-level.',
+                examples: classifySeverityDataset,
+              },
+            ],
+            task: async ({ input }) => {
+              if (!input) throw new Error('Missing classify_severity input');
+              return threatIntelClient.classifySeverity(input);
+            },
+          },
+          [createSeverityExactEvaluator(), createSeverityAdjacentEvaluator()]
+        );
+      }
+    );
+  }
+);

x-pack/solutions/security/packages/kbn-evals-suite-threat-intel-enrichment/evals/threat_intel_enrichment/classify_severity.spec.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { tags } from '@kbn/evals';
+import { evaluate } from '../../src/evaluate';
+import { enrichTaxonomyDataset } from '../../src/datasets/enrich_taxonomy_dataset';
+import {
+  createCategoryRecallEvaluator,
+  createRegionRecallEvaluator,
+} from '../../src/evaluators/enrich_taxonomy_evaluators';
+
+evaluate.describe(
+  'Threat Intel Enrichment: enrich_taxonomy',
+  { tag: tags.stateful.classic },
+  () => {
+    evaluate('extracts categories and regions', async ({ executorClient, threatIntelClient }) => {
+      await executorClient.runExperiment(
+        {
+          datasets: [
+            {
+              name: 'threat_intel: enrich_taxonomy',
+              description:
+                'BlackHat demo packs with labelled closed-set categories/regions plus one authored multi-category example. Scored by recall.',
+              examples: enrichTaxonomyDataset,
+            },
+          ],
+          task: async ({ input }) => {
+            if (!input) throw new Error('Missing enrich_taxonomy input');
+            return threatIntelClient.enrichTaxonomy(input);
+          },
+        },
+        [createCategoryRecallEvaluator(), createRegionRecallEvaluator()]
+      );
+    });
+  }
+);