diff --git a/x-pack/solutions/security/plugins/security_solution/common/threat_intel/workflows/step_types/fetch_source/fetch_source_common.ts b/x-pack/solutions/security/plugins/security_solution/common/threat_intel/workflows/step_types/fetch_source/fetch_source_common.ts index caeb0b15c630a..a51ad6cf14008 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/threat_intel/workflows/step_types/fetch_source/fetch_source_common.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/threat_intel/workflows/step_types/fetch_source/fetch_source_common.ts @@ -36,6 +36,21 @@ export const fetchSourceInputSchema = z.object({ source: z.union([z.string(), sourceHitSchema]), }); +/** Must match the `extracted.iocs` nested mapping in setup/index_templates.ts. */ +export const iocEntrySchema = z.object({ + type: z.string(), + value: z.string(), + defanged: z.string().optional(), + tier: z.string(), + tier_heuristic: z.string(), + tier_basis: z.string(), + port: z.number().optional(), + reference: z.string().optional(), + block_index: z.number().optional(), +}); + +export type IocEntry = z.infer; + /** Must match `.kibana-threat-reports` strict mapping in setup/index_templates.ts. */ export const normalizedReportSchema = z.object({ '@timestamp': z.string(), @@ -69,21 +84,7 @@ export const normalizedReportSchema = z.object({ }), extracted: z .object({ - iocs: z - .array( - z.object({ - type: z.string(), - value: z.string(), - defanged: z.string().optional(), - tier: z.string(), - tier_heuristic: z.string(), - tier_basis: z.string(), - port: z.number().optional(), - reference: z.string().optional(), - block_index: z.number().optional(), - }) - ) - .optional(), + iocs: z.array(iocEntrySchema).optional(), categories: z.array(z.string()).optional(), vulnerability: z .object({ diff --git a/x-pack/solutions/security/plugins/security_solution/public/threat_intel/jest.config.js b/x-pack/solutions/security/plugins/security_solution/public/threat_intel/jest.config.js new file mode 100644 index 0000000000000..161cb1e18bcad --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/threat_intel/jest.config.js @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../../../../../..', + roots: ['/x-pack/solutions/security/plugins/security_solution/public/threat_intel'], + coverageDirectory: + '/target/kibana-coverage/jest/x-pack/solutions/security/plugins/security_solution/public/threat_intel', + coverageReporters: ['text', 'html'], + collectCoverageFrom: [ + '/x-pack/solutions/security/plugins/security_solution/public/threat_intel/**/*.{ts,tsx}', + ], + moduleNameMapper: require('../../server/__mocks__/module_name_map'), +}; diff --git a/x-pack/solutions/security/plugins/security_solution/public/threat_intel/workflows/step_types/fetch_source/fetch_source_step.ts b/x-pack/solutions/security/plugins/security_solution/public/threat_intel/workflows/step_types/fetch_source/fetch_source_step.ts new file mode 100644 index 0000000000000..da1a077745166 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/threat_intel/workflows/step_types/fetch_source/fetch_source_step.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import type { PublicStepDefinition } from '@kbn/workflows-extensions/public'; +import { fetchSourceStepCommonDefinition } from '../../../../../common/threat_intel/workflows/step_types/fetch_source/fetch_source_common'; + +/** YAML editor schema for threat_intel.fetch_source (handler is server-side). */ +export const fetchSourceStepDefinition: PublicStepDefinition = { + ...fetchSourceStepCommonDefinition, + icon: React.lazy(() => + import('@elastic/eui/es/components/icon/assets/download').then(({ icon }) => ({ + default: icon, + })) + ), +}; diff --git a/x-pack/solutions/security/plugins/security_solution/public/threat_intel/workflows/step_types/fetch_source/index.ts b/x-pack/solutions/security/plugins/security_solution/public/threat_intel/workflows/step_types/fetch_source/index.ts new file mode 100644 index 0000000000000..5e70d81e9d6cf --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/threat_intel/workflows/step_types/fetch_source/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { fetchSourceStepDefinition } from './fetch_source_step'; diff --git a/x-pack/solutions/security/plugins/security_solution/public/threat_intel/workflows/step_types/index.ts b/x-pack/solutions/security/plugins/security_solution/public/threat_intel/workflows/step_types/index.ts new file mode 100644 index 0000000000000..6da37fb0c6bc6 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/threat_intel/workflows/step_types/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { registerThreatIntelWorkflowSteps } from './register_workflow_steps'; diff --git a/x-pack/solutions/security/plugins/security_solution/public/threat_intel/workflows/step_types/register_workflow_steps.test.ts b/x-pack/solutions/security/plugins/security_solution/public/threat_intel/workflows/step_types/register_workflow_steps.test.ts new file mode 100644 index 0000000000000..df7036a85f64d --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/threat_intel/workflows/step_types/register_workflow_steps.test.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PublicStepDefinition } from '@kbn/workflows-extensions/public'; +import { workflowsExtensionsMock } from '@kbn/workflows-extensions/public/mocks'; +import { registerThreatIntelWorkflowSteps } from './register_workflow_steps'; +import { fetchSourceStepDefinition } from './fetch_source'; +import { FETCH_SOURCE_STEP_TYPE } from '../../../../common/threat_intel/workflows/step_types/fetch_source/fetch_source_common'; + +type StepLoader = () => Promise; + +describe('registerThreatIntelWorkflowSteps (public)', () => { + it('registers exactly one step definition for threat_intel.fetch_source', () => { + const workflowsExtensions = workflowsExtensionsMock.createSetup(); + + registerThreatIntelWorkflowSteps(workflowsExtensions); + + expect(workflowsExtensions.registerStepDefinition).toHaveBeenCalledTimes(1); + }); + + it('async loader resolves to the fetch_source step definition', async () => { + const workflowsExtensions = workflowsExtensionsMock.createSetup(); + + registerThreatIntelWorkflowSteps(workflowsExtensions); + + const [loader] = workflowsExtensions.registerStepDefinition.mock.calls.map( + ([arg]) => arg as StepLoader + ); + + const definition = await loader(); + expect(definition).toBe(fetchSourceStepDefinition); + // Sanity-check the id matches the YAML-side step type so the editor + // can find the schema for `type: threat_intel.fetch_source`. + expect(definition?.id).toBe(FETCH_SOURCE_STEP_TYPE); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/public/threat_intel/workflows/step_types/register_workflow_steps.ts b/x-pack/solutions/security/plugins/security_solution/public/threat_intel/workflows/step_types/register_workflow_steps.ts new file mode 100644 index 0000000000000..d59cb105a3a81 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/threat_intel/workflows/step_types/register_workflow_steps.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { WorkflowsExtensionsPublicPluginSetup } from '@kbn/workflows-extensions/public'; + +/** + * Register threat-intel-owned workflow step types with the public + * `workflowsExtensions` setup contract so the YAML editor's strict-schema + * validator (see + * `workflows_management/public/features/validate_workflow_yaml/model/use_workflow_json_schema.ts`) + * picks them up. + * + * Mirrors the server-side `registerThreatIntelWorkflowSteps` in + * `server/threat_intel/workflows/step_types/index.ts`. Steps are + * loaded lazily via async loaders to keep the heavy + * `fetch_source_common` Zod schemas off the critical-path bundle until the + * editor actually requests them. + * + * The caller is expected to invoke this only when the optional + * `workflowsExtensions` plugin is present and the + * `threatIntelSupplyEnabled` experimental feature is on, matching + * the gating policy applied server-side. Without that gate, dark-flagged + * deployments would advertise a step type whose handler is never + * registered. + */ +export const registerThreatIntelWorkflowSteps = ( + workflowsExtensions: WorkflowsExtensionsPublicPluginSetup +): void => { + workflowsExtensions.registerStepDefinition(async () => + import('./fetch_source').then((m) => m.fetchSourceStepDefinition) + ); +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/index.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/index.ts new file mode 100644 index 0000000000000..722f6646b4e99 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { runAdapter, UnknownAdapterError } from './run_adapter'; +export type { AdapterRunContext, FetchAdapter, NormalizedReport, SourceHit } from './types'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/kev/kev_adapter.test.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/kev/kev_adapter.test.ts new file mode 100644 index 0000000000000..19d6c951ec892 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/kev/kev_adapter.test.ts @@ -0,0 +1,325 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { kevAdapter } from './kev_adapter'; +import { normalizedReportSchema } from '../../../../common/threat_intel/workflows/step_types/fetch_source/fetch_source_common'; +import type { AdapterRunContext, SourceHit } from '../types'; + +const FIXED_NOW = new Date('2024-03-01T10:00:00.000Z'); +const FEED_URL = + 'https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json'; + +const VULN_1 = { + cveID: 'CVE-2021-44228', + vendorProject: 'Apache', + product: 'Log4j', + vulnerabilityName: 'Apache Log4j2 Remote Code Execution Vulnerability', + dateAdded: '2021-12-10', + shortDescription: 'Apache Log4j2 contains a remote code execution vulnerability.', + requiredAction: 'Apply updates per vendor instructions.', + dueDate: '2021-12-24', + knownRansomwareCampaignUse: 'Known', + notes: + 'https://logging.apache.org/log4j/2.x/security.html ; https://nvd.nist.gov/vuln/detail/CVE-2021-44228', + cwes: ['CWE-917', 'CWE-20'], +}; + +const VULN_2 = { + cveID: 'CVE-2022-30190', + vendorProject: 'Microsoft', + product: 'Windows', + vulnerabilityName: + 'Microsoft Windows Support Diagnostic Tool (MSDT) Remote Code Execution Vulnerability', + dateAdded: '2022-06-01', + shortDescription: 'Microsoft MSDT contains a remote code execution vulnerability.', + requiredAction: 'Apply updates per vendor instructions.', + dueDate: '2022-06-14', + knownRansomwareCampaignUse: 'Unknown', + notes: '', +}; + +const makeEnvelope = (vulns: unknown[] = [VULN_1, VULN_2]) => + JSON.stringify({ + catalogVersion: '2024.03.01', + dateReleased: '2024-03-01T00:00:00Z', + count: vulns.length, + vulnerabilities: vulns, + }); + +const makeContext = (body: string, status = 200): AdapterRunContext => { + const fetchImpl = jest.fn().mockResolvedValue( + new Response(body, { + status, + statusText: status === 200 ? 'OK' : 'Error', + headers: { 'Content-Type': 'application/json' }, + }) + ); + return { + logger: loggingSystemMock.createLogger(), + abortSignal: new AbortController().signal, + now: () => FIXED_NOW, + fetchFn: fetchImpl as unknown as typeof fetch, + lookupFn: async () => [{ address: '93.184.216.34' }], + }; +}; + +const makeSource = (name = 'CISA Known Exploited Vulnerabilities'): SourceHit => ({ + _id: 'kev:cisa-known-exploited-vulnerabilities', + _source: { + adapter_type: 'kev', + name, + space_id: '*', + }, +}); + +describe('kevAdapter', () => { + it('adapterType is kev', () => { + expect(kevAdapter.adapterType).toBe('kev'); + }); + + it('produces one report per CVE entry', async () => { + const reports = await kevAdapter.run(makeSource(), makeContext(makeEnvelope())); + expect(reports).toHaveLength(2); + }); + + it('each report passes the normalizedReportSchema', async () => { + const reports = await kevAdapter.run(makeSource(), makeContext(makeEnvelope())); + for (const report of reports) { + expect(() => normalizedReportSchema.parse(report)).not.toThrow(); + } + }); + + it('fingerprint is stable across re-fetches of unchanged entries', async () => { + const reports = await kevAdapter.run(makeSource(), makeContext(makeEnvelope())); + const fps = reports.map((r) => r.content_fingerprint); + // Run again — same fingerprints + const reports2 = await kevAdapter.run(makeSource(), makeContext(makeEnvelope())); + expect(reports2.map((r) => r.content_fingerprint)).toEqual(fps); + }); + + // The fingerprint used to be the CVE id alone, so any later CISA revision of + // the same CVE deduped away and never reached the stored report. + it.each([ + ['requiredAction', { requiredAction: 'Discontinue use of the product.' }], + ['dueDate', { dueDate: '2022-01-15' }], + ['knownRansomwareCampaignUse', { knownRansomwareCampaignUse: 'Unknown' }], + ['shortDescription', { shortDescription: 'Revised description.' }], + ['notes', { notes: 'https://example.com/new-advisory' }], + ])('re-fingerprints the same CVE when %s changes', async (_field, patch) => { + const [original] = await kevAdapter.run(makeSource(), makeContext(makeEnvelope([VULN_1]))); + const [revised] = await kevAdapter.run( + makeSource(), + makeContext(makeEnvelope([{ ...VULN_1, ...patch }])) + ); + + expect(revised.extracted?.vulnerability?.cve_id).toBe( + original.extracted?.vulnerability?.cve_id + ); + expect(revised.content_fingerprint).not.toBe(original.content_fingerprint); + }); + + // adapter_id identifies the source, not the item: list_sources aggregates + // report activity on it, so a per-CVE value left the KEV catalog row with no + // stats. The CVE identity lives in lineage.source_doc_ref.id instead. + it('stamps a source-stable adapter_id shared by every entry', async () => { + const reports = await kevAdapter.run(makeSource(), makeContext(makeEnvelope())); + + for (const report of reports) { + expect(report.source.adapter_id).toBe('kev:kev:cisa-known-exploited-vulnerabilities'); + } + expect(reports.map((r) => r.lineage.source_doc_ref?.id)).toEqual([ + 'CVE-2021-44228', + 'CVE-2022-30190', + ]); + }); + + it('fingerprints are distinct per CVE', async () => { + const reports = await kevAdapter.run(makeSource(), makeContext(makeEnvelope())); + const fps = reports.map((r) => r.content_fingerprint); + expect(new Set(fps).size).toBe(fps.length); + }); + + it('populates extracted.vulnerability fields', async () => { + const reports = await kevAdapter.run(makeSource(), makeContext(makeEnvelope())); + const [r] = reports; + expect(r.extracted?.vulnerability).toMatchObject({ + cve_id: 'CVE-2021-44228', + vendor: 'Apache', + product: 'Log4j', + name: 'Apache Log4j2 Remote Code Execution Vulnerability', + cwes: ['CWE-917', 'CWE-20'], + date_added: '2021-12-10', + due_date: '2021-12-24', + ransomware_use: 'Known', + }); + }); + + it('extracted.categories is ["vulnerability"]', async () => { + const reports = await kevAdapter.run(makeSource(), makeContext(makeEnvelope())); + for (const r of reports) { + expect(r.extracted?.categories).toEqual(['vulnerability']); + } + }); + + it('severity is high/70', async () => { + const reports = await kevAdapter.run(makeSource(), makeContext(makeEnvelope())); + for (const r of reports) { + expect(r.severity).toEqual({ level: 'high', score: 70 }); + } + }); + + it('extraction_method is kev (not pending — skipped by enrich_threat_report)', async () => { + const reports = await kevAdapter.run(makeSource(), makeContext(makeEnvelope())); + for (const r of reports) { + expect(r.lineage.extraction_method).toBe('kev'); + // Explicitly not 'pending' — the enrich_threat_report term:pending filter excludes this + expect(r.lineage.extraction_method).not.toBe('pending'); + } + }); + + it('sends a browser User-Agent header (CISA blocks default Kibana UA)', async () => { + const fetchImpl = jest.fn().mockResolvedValue( + new Response(makeEnvelope(), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + ); + const ctx: AdapterRunContext = { + logger: loggingSystemMock.createLogger(), + abortSignal: new AbortController().signal, + now: () => FIXED_NOW, + fetchFn: fetchImpl as unknown as typeof fetch, + lookupFn: async () => [{ address: '93.184.216.34' }], + }; + await kevAdapter.run(makeSource(), ctx); + + expect(fetchImpl).toHaveBeenCalled(); + const callArgs = fetchImpl.mock.calls[0]; + const requestInit = callArgs[1] as RequestInit; + const headers = requestInit?.headers as Record; + expect(headers?.['User-Agent']).toMatch(/Mozilla/); + }); + + it('stores a credential-free catalog feed URL in provenance', async () => { + const [report] = await kevAdapter.run(makeSource(), makeContext(makeEnvelope([VULN_1]))); + + expect(report.source.url).toBe(FEED_URL); + expect(JSON.stringify(report)).not.toContain('feed-password'); + }); + + it('skips malformed entries missing required fields', async () => { + const envelope = JSON.stringify({ + catalogVersion: '2024.03.01', + dateReleased: '2024-03-01T00:00:00Z', + count: 3, + vulnerabilities: [ + VULN_1, + // Missing cveID + { + vendorProject: 'Acme', + product: 'Widget', + vulnerabilityName: 'Bad', + dateAdded: '2024-01-01', + shortDescription: 'x', + requiredAction: 'y', + dueDate: '2024-01-10', + }, + VULN_2, + ], + }); + const reports = await kevAdapter.run(makeSource(), makeContext(envelope)); + expect(reports).toHaveLength(2); + }); + + it('tolerates missing optional fields (cwes, notes, ransomware_use)', async () => { + const minimal = { + cveID: 'CVE-2024-99999', + vendorProject: 'TestVendor', + product: 'TestProduct', + vulnerabilityName: 'Test Vuln', + dateAdded: '2024-01-01', + shortDescription: 'A test.', + requiredAction: 'Do something.', + dueDate: '2024-01-15', + }; + const reports = await kevAdapter.run( + makeSource(), + makeContext( + JSON.stringify({ + catalogVersion: '2024.01.01', + dateReleased: '2024-01-01T00:00:00Z', + count: 1, + vulnerabilities: [minimal], + }) + ) + ); + expect(reports).toHaveLength(1); + expect(reports[0].extracted?.vulnerability?.cwes).toBeUndefined(); + expect(reports[0].extracted?.vulnerability?.ransomware_use).toBeUndefined(); + expect(() => normalizedReportSchema.parse(reports[0])).not.toThrow(); + }); + + it('throws on non-200 HTTP response', async () => { + await expect(kevAdapter.run(makeSource(), makeContext('Forbidden', 403))).rejects.toThrow( + /HTTP 403/ + ); + }); + + it('throws on invalid JSON body', async () => { + await expect(kevAdapter.run(makeSource(), makeContext('not json'))).rejects.toThrow( + /not valid JSON/ + ); + }); + + it('throws when vulnerabilities array is absent', async () => { + await expect( + kevAdapter.run(makeSource(), makeContext(JSON.stringify({ catalogVersion: '2024.01.01' }))) + ).rejects.toThrow(/vulnerabilities array/); + }); + + it('does not blame JSON parsing when the feed parsed but has no vulnerabilities array', async () => { + // Well-formed JSON that is missing the array must not be reported as a parse + // failure: that sends an operator debugging a feed-schema change into the + // parser instead. + await expect( + kevAdapter.run(makeSource(), makeContext(JSON.stringify({ catalogVersion: '2024.01.01' }))) + ).rejects.toThrow(/^(?!.*not valid JSON).*vulnerabilities array/s); + }); + + it('carries the configured source name, not a hardcoded one', async () => { + const [report] = await kevAdapter.run( + makeSource('CISA KEV (corrected)'), + makeContext(makeEnvelope([VULN_1])) + ); + expect(report.source.name).toBe('CISA KEV (corrected)'); + }); + + // `notes` and `knownRansomwareCampaignUse` are optional and reach + // `buildFingerprint`, whose `.trim()` throws on a non-string. That throw sat + // outside the per-entry guard, so one bad row failed the whole feed. They are + // now dropped to `undefined`, keeping the rest of the entry. + it.each([ + ['notes', { notes: 42 }], + ['knownRansomwareCampaignUse', { knownRansomwareCampaignUse: false }], + ])('drops a non-string %s instead of failing the feed', async (_, patch) => { + const reports = await kevAdapter.run( + makeSource(), + makeContext(makeEnvelope([{ ...VULN_1, ...patch }, VULN_2])) + ); + expect(reports).toHaveLength(2); + expect(reports[0].extracted?.vulnerability?.cve_id).toBe(VULN_1.cveID); + }); + + it('keeps an entry whose optional notes are null', async () => { + const reports = await kevAdapter.run( + makeSource(), + makeContext(makeEnvelope([{ ...VULN_1, notes: null }])) + ); + expect(reports).toHaveLength(1); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/kev/kev_adapter.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/kev/kev_adapter.ts new file mode 100644 index 0000000000000..340dacbfb0591 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/kev/kev_adapter.ts @@ -0,0 +1,251 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { fetchUrlForContext, redactUrl } from '../http_client'; +import { buildFingerprint } from '../fingerprint'; +import { severityScore } from '../../services/severity'; +import { buildReportContent } from '../../services/report_content'; +import { normalizeProvenanceUrl } from '../../services/provenance_url'; +import { GLOBAL_SPACE_ID, resolveCatalogSourceUrl } from '../../../../common/threat_intel'; +import type { FetchAdapter, NormalizedReport, SourceHit, AdapterRunContext } from '../types'; + +/** + * Every field `buildKevReport` reads, not just the identity ones. + * + * The gate used to check `cveID`, `vendorProject`, `product`, and `vulnerabilityName` + * only, while the report body interpolates `shortDescription` and `requiredAction` and + * `normalizedReportSchema` requires `date_added` and `due_date`. A custom or + * temporarily malformed KEV feed missing any of those produced a report with the string + * `undefined` in its body and a missing required field, which fails output validation + * for the whole step rather than skipping the one bad entry. + */ +const isCompleteKevEntry = (vuln: KevVulnerability): boolean => + Boolean( + vuln.cveID && + vuln.vendorProject && + vuln.product && + vuln.vulnerabilityName && + vuln.shortDescription && + vuln.requiredAction && + vuln.dateAdded && + vuln.dueDate + ); + +const KEV_FEED_URL = + 'https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json'; + +// CISA returns 403 to the default Kibana UA. A browser-style UA is required. +const BROWSER_USER_AGENT = + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'; + +interface KevVulnerability { + cveID: string; + vendorProject: string; + product: string; + vulnerabilityName: string; + dateAdded: string; + shortDescription: string; + requiredAction: string; + dueDate: string; + knownRansomwareCampaignUse?: string; + notes?: string; + cwes?: string[]; +} + +interface KevEnvelope { + catalogVersion: string; + dateReleased: string; + count: number; + vulnerabilities: KevVulnerability[]; +} + +const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +const readOptionalStringArray = (value: unknown): string[] | undefined => { + if (value === undefined) return undefined; + if (!Array.isArray(value)) return undefined; + return value.every((entry) => typeof entry === 'string') ? value : undefined; +}; + +const parseKevVulnerability = (value: unknown): KevVulnerability | undefined => { + if (!isRecord(value)) return undefined; + const cwes = readOptionalStringArray(value.cwes); + if (value.cwes !== undefined && cwes === undefined) return undefined; + const vuln = { + cveID: value.cveID, + vendorProject: value.vendorProject, + product: value.product, + vulnerabilityName: value.vulnerabilityName, + dateAdded: value.dateAdded, + shortDescription: value.shortDescription, + requiredAction: value.requiredAction, + dueDate: value.dueDate, + // Optional, and read by `buildKevReport`. A non-string here (a custom or + // temporarily malformed feed sending `"notes": 42`) reaches + // `buildFingerprint`, whose `.trim()` throws outside the per-entry guard and + // fails the whole feed instead of skipping one row. + knownRansomwareCampaignUse: + typeof value.knownRansomwareCampaignUse === 'string' + ? value.knownRansomwareCampaignUse + : undefined, + notes: typeof value.notes === 'string' ? value.notes : undefined, + cwes, + }; + if ( + typeof vuln.cveID !== 'string' || + typeof vuln.vendorProject !== 'string' || + typeof vuln.product !== 'string' || + typeof vuln.vulnerabilityName !== 'string' || + typeof vuln.dateAdded !== 'string' || + typeof vuln.shortDescription !== 'string' || + typeof vuln.requiredAction !== 'string' || + typeof vuln.dueDate !== 'string' + ) { + return undefined; + } + return vuln as KevVulnerability; +}; + +const readFeedUrl = (source: SourceHit): string => { + const url = resolveCatalogSourceUrl(source._id); + return typeof url === 'string' && url.length > 0 ? url : KEV_FEED_URL; +}; + +const buildKevReport = ( + vuln: KevVulnerability, + provenanceUrl: string | undefined, + ingestedAt: string, + spaceId: string, + sourceId: string, + sourceName: string +): NormalizedReport => { + const bodyText = `${vuln.shortDescription}\n\nRequired Action: ${vuln.requiredAction}`; + + return { + '@timestamp': ingestedAt, + // The CVE id alone is stable for the life of the entry, so fingerprinting on + // it would dedup away every later CISA revision. Include the mutable fields + // so a changed due date, required action, ransomware status, or description + // produces a new fingerprint and is re-ingested. + content_fingerprint: buildFingerprint([ + vuln.cveID, + vuln.vulnerabilityName, + vuln.shortDescription, + vuln.requiredAction, + vuln.dueDate, + vuln.knownRansomwareCampaignUse, + vuln.notes, + (vuln.cwes ?? []).join(','), + ]), + space_id: spaceId, + source: { + type: 'kev', + // The configured source name from the approved catalog entry, matching the + // other adapters. Hardcoding it misattributes reports if the catalog name + // is corrected or a second KEV-format feed is seeded. + name: sourceName, + ...(provenanceUrl ? { url: provenanceUrl } : {}), + // Identifies the *source*, not the item, matching every other adapter + // (`:`). list_sources aggregates report activity on + // this field, so a per-CVE value would leave the KEV catalog row with no + // stats. The CVE identity lives in lineage.source_doc_ref.id below and in + // extracted.vulnerability.cve_id. + adapter_id: `kev:${sourceId}`, + }, + content: buildReportContent({ title: vuln.vulnerabilityName, bodyText, language: 'en' }), + severity: { + level: 'high', + score: severityScore('high'), + }, + lineage: { + ingested_at: ingestedAt, + extraction_method: 'kev', + extracted_at: ingestedAt, + source_doc_ref: { index: 'cisa:kev', id: vuln.cveID }, + }, + extracted: { + categories: ['vulnerability'], + vulnerability: { + cve_id: vuln.cveID, + vendor: vuln.vendorProject, + product: vuln.product, + name: vuln.vulnerabilityName, + cwes: vuln.cwes, + date_added: vuln.dateAdded, + due_date: vuln.dueDate, + ransomware_use: vuln.knownRansomwareCampaignUse, + }, + }, + }; +}; + +export const kevAdapter: FetchAdapter = { + adapterType: 'kev', + + async run(source: SourceHit, context: AdapterRunContext): Promise { + const { logger, abortSignal, now } = context; + const log = logger.get('kev-adapter'); + const fetchUrl = fetchUrlForContext(context); + + const feedUrl = readFeedUrl(source); + const provenanceUrl = normalizeProvenanceUrl(feedUrl); + const ingestedAt = now().toISOString(); + const spaceId = source._source.space_id ?? GLOBAL_SPACE_ID; + + const response = await fetchUrl(feedUrl, { + abortSignal, + headers: { 'User-Agent': BROWSER_USER_AGENT }, + }); + + if (response.status >= 400) { + throw new Error( + `KEV feed returned HTTP ${response.status} ${response.statusText} from ${redactUrl( + feedUrl + )}` + ); + } + + let parsed: unknown; + try { + parsed = JSON.parse(response.body) as unknown; + } catch (err) { + throw new Error(`KEV feed response is not valid JSON: ${(err as Error).message}`); + } + + if (!isRecord(parsed) || !Array.isArray(parsed.vulnerabilities)) { + throw new Error('KEV feed missing vulnerabilities array'); + } + + const envelope = parsed as unknown as KevEnvelope; + const vulnerabilities = envelope.vulnerabilities; + + const reports: NormalizedReport[] = []; + for (const rawEntry of vulnerabilities) { + const vuln = parseKevVulnerability(rawEntry); + if (vuln && isCompleteKevEntry(vuln)) { + reports.push( + buildKevReport(vuln, provenanceUrl, ingestedAt, spaceId, source._id, source._source.name) + ); + } else { + log.warn( + `kev-adapter: skipping malformed entry (missing required fields): ${JSON.stringify( + rawEntry + ).slice(0, 200)}` + ); + } + } + + log.info( + `kev-adapter: ${redactUrl(feedUrl)} → ${reports.length} reports (catalogVersion=${ + envelope.catalogVersion + })` + ); + + return reports; + }, +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/run_adapter.test.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/run_adapter.test.ts new file mode 100644 index 0000000000000..434cebfcfa354 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/run_adapter.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { runAdapter, UnknownAdapterError } from './run_adapter'; +import type { AdapterRunContext, SourceHit } from './types'; + +const buildContext = (): AdapterRunContext => ({ + logger: loggingSystemMock.createLogger(), + abortSignal: new AbortController().signal, + now: () => new Date('2026-05-16T12:00:00.000Z'), + fetchFn: jest.fn() as unknown as typeof fetch, +}); + +describe('runAdapter', () => { + it('throws UnknownAdapterError for unsupported adapter_type values', async () => { + const source = { + _id: 'manual:1', + _source: { + adapter_type: 'manual', + name: 'Analyst paste', + }, + } as unknown as SourceHit; + + await expect(runAdapter(source, buildContext())).rejects.toMatchObject({ + name: UnknownAdapterError.name, + message: expect.stringContaining('Known adapter types: rss, text_indicator_list, kev'), + }); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/run_adapter.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/run_adapter.ts new file mode 100644 index 0000000000000..f26e3163e570d --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/run_adapter.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FetchAdapterType } from '../../../common/threat_intel'; +import { rssAdapter } from './rss/rss_adapter'; +import { textIndicatorListAdapter } from './text_indicator_list/text_indicator_list_adapter'; +import { kevAdapter } from './kev/kev_adapter'; +import type { AdapterRunContext, FetchAdapter, NormalizedReport, SourceHit } from './types'; + +const ADAPTERS: Record = { + rss: rssAdapter, + text_indicator_list: textIndicatorListAdapter, + kev: kevAdapter, +}; + +export class UnknownAdapterError extends Error { + constructor(public readonly adapterType: string, public readonly sourceId: string) { + super( + `No adapter registered for source ${sourceId} (adapter_type=${adapterType}). ` + + `Known adapter types: ${Object.keys(ADAPTERS).join(', ')}.` + ); + this.name = 'UnknownAdapterError'; + } +} + +/** + * Resolve and run the adapter for a given source. + * + * Errors thrown by an adapter propagate up — the step handler converts + * them to `StepHandlerResult.error`, which the workflow engine then + * routes through the per-step `on-failure: continue: true` so a single + * misbehaving source can't break the rest of the run. Adapters that + * succeed but produce no reports return `[]`; the step output's + * `total_fetched: 0` is the signal for "ran cleanly, nothing new". + */ +export const runAdapter = async ( + source: SourceHit, + context: AdapterRunContext +): Promise => { + const adapter = ADAPTERS[source._source.adapter_type]; + if (!adapter) { + throw new UnknownAdapterError(source._source.adapter_type, source._id); + } + return adapter.run(source, context); +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/text_indicator_list/parse_indicator_list.test.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/text_indicator_list/parse_indicator_list.test.ts new file mode 100644 index 0000000000000..9e487bd40c062 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/text_indicator_list/parse_indicator_list.test.ts @@ -0,0 +1,328 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { parseIndicatorList } from './parse_indicator_list'; +import { extractIocs } from '../../services/extract_iocs'; +import type { ExtractIocsResult, ExtractedIoc } from '../../services/extract_iocs'; +import type { IocType } from '../../../../common/threat_intel'; + +jest.mock('../../services/extract_iocs'); + +const extractIocsMock = extractIocs as jest.MockedFunction; + +const empty = (): ExtractIocsResult => ({ count: 0, iocs: [], ioc_set_hash: null }); + +const makeIoc = ( + type: IocType, + value: string, + extra: Partial = {} +): ExtractedIoc => ({ + type, + value, + tier: 'contextual', + tier_heuristic: 'contextual', + tier_basis: 'original_basis', + ...extra, +}); + +const makeResult = (...iocs: ExtractedIoc[]): ExtractIocsResult => ({ + count: iocs.length, + iocs, + ioc_set_hash: null, +}); + +beforeEach(() => { + extractIocsMock.mockReset(); + extractIocsMock.mockReturnValue(empty()); +}); + +describe('parseIndicatorList', () => { + describe('empty / trivial inputs', () => { + it('returns [] for empty string', () => { + expect(parseIndicatorList('')).toEqual([]); + }); + + it('returns [] for whitespace-only body', () => { + expect(parseIndicatorList(' \n \n ')).toEqual([]); + }); + + it('returns [] for body with only non-reference comment lines', () => { + const body = [ + '# Copyright (c) 2014-2026 Maltrail developers', + '# See the file LICENSE for copying permission', + '', + '# another comment', + ].join('\n'); + expect(parseIndicatorList(body)).toEqual([]); + }); + }); + + describe('multi-block interleaved fixture', () => { + const TWITTER_REF = 'https://twitter.com/ozuma5119/status/112'; + const BLOG_REF = 'https://blog.somevendor.com/emotet-writeup'; + + const DOMAIN_IOC = makeIoc('domain', 'tamsuamy.com', { defanged: 'tamsuamy.com' }); + const IP_PORT_IOC = makeIoc('ip', '66.84.11.168', { port: 8080, defanged: '66.84.11.168' }); + const IP_7080_IOC = makeIoc('ip', '142.4.198.249', { port: 7080, defanged: '142.4.198.249' }); + const IP_8080_IOC = makeIoc('ip', '170.150.11.245', { port: 8080, defanged: '170.150.11.245' }); + + beforeEach(() => { + extractIocsMock.mockImplementation(({ text }) => { + if (text === 'tamsuamy.com') return makeResult(DOMAIN_IOC); + if (text === '66.84.11.168:8080') return makeResult(IP_PORT_IOC); + if (text === '142.4.198.249:7080') return makeResult(IP_7080_IOC); + if (text === '170.150.11.245:8080') return makeResult(IP_8080_IOC); + return empty(); + }); + }); + + const body = [ + '# Copyright (c) 2014-2026 Maltrail developers (https://github.com/stamparm/maltrail)', + "# See the file 'LICENSE' for copying permission", + `# Reference: ${TWITTER_REF}`, + 'tamsuamy.com', + '66.84.11.168:8080', + `# Reference: ${BLOG_REF}`, + '142.4.198.249:7080', + '170.150.11.245:8080', + ].join('\n'); + + it('produces two blocks with correct block_index', () => { + const blocks = parseIndicatorList(body); + expect(blocks).toHaveLength(2); + expect(blocks[0].block_index).toBe(0); + expect(blocks[1].block_index).toBe(1); + }); + + it('attributes each IOC to the correct nearest reference', () => { + const blocks = parseIndicatorList(body); + expect(blocks[0].reference).toBe(TWITTER_REF); + expect(blocks[1].reference).toBe(BLOG_REF); + }); + + it('block 0 contains the two IOCs under the twitter reference', () => { + const blocks = parseIndicatorList(body); + expect(blocks[0].iocs).toHaveLength(2); + expect(blocks[0].iocs[0].value).toBe('tamsuamy.com'); + expect(blocks[0].iocs[1].value).toBe('66.84.11.168'); + }); + + it('block 1 contains the two IOCs under the blog reference', () => { + const blocks = parseIndicatorList(body); + expect(blocks[1].iocs).toHaveLength(2); + expect(blocks[1].iocs[0].value).toBe('142.4.198.249'); + expect(blocks[1].iocs[1].value).toBe('170.150.11.245'); + }); + + it('skips copyright / license header comments — they do not become blocks', () => { + const blocks = parseIndicatorList(body); + // If copyright lines were treated as references they'd inflate the block count + expect(blocks).toHaveLength(2); + }); + }); + + describe('tier stamping', () => { + it('overrides extractIocs tier to discriminating and sets tier_basis to maltrail_indicator_list', () => { + const contextualIoc = makeIoc('ip', '1.2.3.4', { + tier: 'contextual', + tier_heuristic: 'contextual', + tier_basis: 'original', + }); + extractIocsMock.mockReturnValue(makeResult(contextualIoc)); + + const blocks = parseIndicatorList('# Reference: https://example.com/report\n1.2.3.4'); + expect(blocks[0].iocs[0].tier).toBe('discriminating'); + expect(blocks[0].iocs[0].tier_heuristic).toBe('discriminating'); + expect(blocks[0].iocs[0].tier_basis).toBe('maltrail_indicator_list'); + }); + + it('stamps tier_basis from the tierBasis argument when the caller overrides it', () => { + const ioc = makeIoc('ip', '1.2.3.4', { tier: 'contextual', tier_basis: 'original' }); + extractIocsMock.mockReturnValue(makeResult(ioc)); + + const blocks = parseIndicatorList( + '# Reference: https://example.com/report\n1.2.3.4', + 'curated_ip_list' + ); + expect(blocks[0].iocs[0].tier_basis).toBe('curated_ip_list'); + }); + + it('preserves type and value from extractIocs', () => { + const ioc = makeIoc('domain', 'evil.com'); + extractIocsMock.mockReturnValue(makeResult(ioc)); + + const blocks = parseIndicatorList('# Reference: https://example.com/report\nevil.com'); + expect(blocks[0].iocs[0].type).toBe('domain'); + expect(blocks[0].iocs[0].value).toBe('evil.com'); + }); + }); + + describe('port preservation', () => { + it('preserves port field from extractIocs for ip:port lines', () => { + const ipPortIoc = makeIoc('ip', '66.84.11.168', { port: 8080 }); + extractIocsMock.mockReturnValue(makeResult(ipPortIoc)); + + const blocks = parseIndicatorList('# Reference: https://example.com\n66.84.11.168:8080'); + expect(blocks[0].iocs[0].port).toBe(8080); + }); + + it('emits no port field for plain IP lines', () => { + const ipIoc = makeIoc('ip', '1.2.3.4'); + extractIocsMock.mockReturnValue(makeResult(ipIoc)); + + const blocks = parseIndicatorList('# Reference: https://example.com\n1.2.3.4'); + expect(blocks[0].iocs[0].port).toBeUndefined(); + }); + }); + + describe('IOC type variety', () => { + it('handles url-type IOCs from extractIocs', () => { + const urlIoc = makeIoc('url', 'http://malicious.example.com/payload'); + extractIocsMock.mockReturnValue(makeResult(urlIoc)); + + const blocks = parseIndicatorList( + '# Reference: https://blog.example.com\nhttp://malicious.example.com/payload' + ); + expect(blocks[0].iocs[0].type).toBe('url'); + }); + }); + + describe('indicators before first reference', () => { + it('puts pre-reference indicators in block_index 0 with reference: undefined', () => { + const orphanIoc = makeIoc('domain', 'orphan.com'); + const refIoc = makeIoc('domain', 'malicious.com'); + extractIocsMock.mockImplementation(({ text }) => { + if (text === 'orphan.com') return makeResult(orphanIoc); + if (text === 'malicious.com') return makeResult(refIoc); + return empty(); + }); + + const body = [ + 'orphan.com', + '# Reference: https://blog.example.com/post', + 'malicious.com', + ].join('\n'); + const blocks = parseIndicatorList(body); + + expect(blocks).toHaveLength(2); + expect(blocks[0].block_index).toBe(0); + expect(blocks[0].reference).toBeUndefined(); + expect(blocks[0].iocs[0].value).toBe('orphan.com'); + }); + + it('subsequent block after orphan block gets block_index 1', () => { + const orphanIoc = makeIoc('domain', 'orphan.com'); + const refIoc = makeIoc('domain', 'malicious.com'); + extractIocsMock.mockImplementation(({ text }) => { + if (text === 'orphan.com') return makeResult(orphanIoc); + if (text === 'malicious.com') return makeResult(refIoc); + return empty(); + }); + + const body = [ + 'orphan.com', + '# Reference: https://blog.example.com/post', + 'malicious.com', + ].join('\n'); + const blocks = parseIndicatorList(body); + + expect(blocks[1].block_index).toBe(1); + expect(blocks[1].reference).toBe('https://blog.example.com/post'); + }); + }); + + describe('tolerance of blank and unrecognised lines', () => { + it('skips blank lines between indicators without creating phantom blocks', () => { + const ioc = makeIoc('ip', '1.2.3.4'); + extractIocsMock.mockImplementation(({ text }) => + text === '1.2.3.4' ? makeResult(ioc) : empty() + ); + + const body = ['# Reference: https://example.com', '', ' ', '1.2.3.4', ''].join('\n'); + const blocks = parseIndicatorList(body); + + expect(blocks).toHaveLength(1); + expect(blocks[0].iocs).toHaveLength(1); + }); + + it('skips lines where extractIocs returns no IOCs', () => { + extractIocsMock.mockReturnValue(empty()); + + const body = ['# Reference: https://example.com', 'not-an-ioc-line'].join('\n'); + const blocks = parseIndicatorList(body); + + expect(blocks).toHaveLength(1); + expect(blocks[0].iocs).toHaveLength(0); + }); + + it('calls extractIocs with defang: false', () => { + const ioc = makeIoc('domain', 'evil.com'); + extractIocsMock.mockReturnValue(makeResult(ioc)); + + parseIndicatorList('# Reference: https://example.com\nevil.com'); + + expect(extractIocs).toHaveBeenCalledWith({ text: 'evil.com', defang: false }); + }); + }); + + describe('reference block with empty body → empty reference block', () => { + it('a reference with no subsequent IOCs still emits a block', () => { + extractIocsMock.mockReturnValue(empty()); + + const body = '# Reference: https://example.com/page'; + const blocks = parseIndicatorList(body); + + expect(blocks).toHaveLength(1); + expect(blocks[0].reference).toBe('https://example.com/page'); + expect(blocks[0].iocs).toHaveLength(0); + }); + }); +}); + +describe('parseIndicatorList — tier elevation', () => { + // Appearing in a curated trail file is a strong signal, so an uncertain or + // contextual value is elevated. It is not strong enough to override a verdict + // the extractor already reached, and the promote task admits everything that is + // not `reference` or `denied`, so blanket-elevating turned private addresses and + // vendor domains into live Indicator Match rows. + it.each([ + ['reference', 'ip', '10.0.0.1'], + ['reference', 'domain', 'virustotal.com'], + ['denied', 'domain', 'google.com'], + ])('preserves a %s verdict from the extractor', (tier, type, value) => { + extractIocsMock.mockReturnValue( + makeResult( + makeIoc(type as IocType, value, { + tier: tier as ExtractedIoc['tier'], + tier_heuristic: tier as ExtractedIoc['tier'], + tier_basis: 'private_ip', + }) + ) + ); + + const [block] = parseIndicatorList(`# https://ref.example/trail\n${value}`); + + expect(block.iocs[0].tier).toBe(tier); + expect(block.iocs[0].tier_basis).toBe('private_ip'); + }); + + it.each([['contextual'], ['uncertain']])('elevates a %s verdict to discriminating', (tier) => { + extractIocsMock.mockReturnValue( + makeResult( + makeIoc('ip', '185.220.101.45', { + tier: tier as ExtractedIoc['tier'], + tier_heuristic: tier as ExtractedIoc['tier'], + }) + ) + ); + + const [block] = parseIndicatorList('# https://ref.example/trail\n185.220.101.45'); + + expect(block.iocs[0].tier).toBe('discriminating'); + expect(block.iocs[0].tier_basis).toBe('maltrail_indicator_list'); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/text_indicator_list/parse_indicator_list.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/text_indicator_list/parse_indicator_list.ts new file mode 100644 index 0000000000000..c0ee593472f27 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/text_indicator_list/parse_indicator_list.ts @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ExtractedIoc } from '../../services/extract_iocs'; +import { extractIocs } from '../../services/extract_iocs'; + +/** + * Tiers the extractor has already judged not to be indicators. These are never + * elevated, whatever list they appear on. + */ +const NON_ELEVATABLE_TIERS: ReadonlySet = new Set(['reference', 'denied']); + +export interface IndicatorBlock { + reference?: string; + block_index: number; + iocs: ExtractedIoc[]; +} + +const REFERENCE_PREFIX = '# Reference:'; + +/** Default provenance stamp, kept for the Maltrail trail sources in the catalog. */ +export const DEFAULT_INDICATOR_LIST_TIER_BASIS = 'maltrail_indicator_list'; + +/** + * Parses a Maltrail-format indicator list body into blocks, each block grouping + * IOCs under the nearest preceding # Reference: line. + * + * - Skips all other # comment lines (copyright, license, blank comments). + * - Uses extractIocs for value classification; never reinvents IOC parsing. + * - Stamps every elevated IOC with `tierBasis` and tier 'discriminating'. + * - Tolerant: blank lines and unrecognised lines are silently skipped. + * + * `tierBasis` is a parameter because this parser is the generic + * `text_indicator_list` reader, and `tier_basis` flows through to indicator + * documents and detection rules. A second, non-Maltrail text list would + * otherwise be stamped with the wrong provenance. + */ +export const parseIndicatorList = ( + body: string, + tierBasis: string = DEFAULT_INDICATOR_LIST_TIER_BASIS +): IndicatorBlock[] => { + if (!body || !body.trim()) { + return []; + } + + const blocks: IndicatorBlock[] = []; + let currentBlock: IndicatorBlock = { block_index: 0, iocs: [] }; + let seenFirstReference = false; + + for (const rawLine of body.split('\n')) { + const line = rawLine.trim(); + + if (line.startsWith(REFERENCE_PREFIX)) { + const url = line.slice(REFERENCE_PREFIX.length).trim(); + + if (currentBlock.iocs.length > 0 || seenFirstReference) { + blocks.push(currentBlock); + currentBlock = { + block_index: currentBlock.block_index + 1, + iocs: [], + }; + } + + seenFirstReference = true; + currentBlock.reference = url; + } else if (line && !line.startsWith('#')) { + const { iocs } = extractIocs({ text: line, defang: false }); + for (const ioc of iocs) { + // Appearing in a curated trail file is a strong signal, so an uncertain or + // contextual value is elevated. It is not strong enough to override a + // verdict the extractor already reached: `reference` covers private and + // reserved addresses and security-vendor and research domains, `denied` is + // the benign denylist, and the promote task admits everything that is not + // one of those two. Blanket-elevating turned 10.0.0.1 and virustotal.com + // into live Indicator Match rows. + currentBlock.iocs.push( + NON_ELEVATABLE_TIERS.has(ioc.tier) + ? ioc + : { + ...ioc, + tier: 'discriminating', + tier_heuristic: 'discriminating', + tier_basis: tierBasis, + } + ); + } + } + } + + if (currentBlock.iocs.length > 0 || seenFirstReference) { + blocks.push(currentBlock); + } + + return blocks; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/text_indicator_list/text_indicator_list_adapter.test.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/text_indicator_list/text_indicator_list_adapter.test.ts new file mode 100644 index 0000000000000..877400e7233d4 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/text_indicator_list/text_indicator_list_adapter.test.ts @@ -0,0 +1,462 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { textIndicatorListAdapter } from './text_indicator_list_adapter'; +import { parseIndicatorList } from './parse_indicator_list'; +import { normalizedReportSchema } from '../../../../common/threat_intel/workflows/step_types/fetch_source/fetch_source_common'; +import type { AdapterRunContext, SourceHit } from '../types'; +import type { IndicatorBlock } from './parse_indicator_list'; +import type { IocType } from '../../../../common/threat_intel'; +import type { ExtractedIoc } from '../../services/extract_iocs'; + +jest.mock('./parse_indicator_list'); +const parseIndicatorListMock = parseIndicatorList as jest.MockedFunction; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +const FIXED_NOW = new Date('2024-01-15T12:00:00.000Z'); +const TRAIL_URL = + 'https://raw.githubusercontent.com/stamparm/maltrail/master/trails/static/malware/cobaltstrike.txt'; + +// Exposed for chunking tests — must match adapter constant. +const MAX_NESTED_PER_DOC = 5000; + +const makeContext = ( + fetchImpl: jest.Mock, [string | URL | Request, RequestInit?]> +): AdapterRunContext => ({ + logger: loggingSystemMock.createLogger(), + abortSignal: new AbortController().signal, + now: () => FIXED_NOW, + fetchFn: fetchImpl as unknown as typeof fetch, + lookupFn: async () => [{ address: '93.184.216.34' }], +}); + +const okResponse = () => + new Response('body-content', { + status: 200, + statusText: 'OK', + headers: { 'Content-Type': 'text/plain' }, + }); + +const SOURCE_ID = 'text_indicator_list:maltrail-cobaltstrike'; + +const makeSource = (id = SOURCE_ID, name = 'Maltrail — CobaltStrike C2 indicators'): SourceHit => ({ + _id: id, + _source: { + adapter_type: 'text_indicator_list', + name, + enabled: true, + }, +}); + +const makeIoc = (value: string, type: IocType = 'ip'): ExtractedIoc => ({ + type, + value, + tier: 'discriminating' as const, + tier_heuristic: 'discriminating', + tier_basis: 'maltrail_indicator_list', +}); + +/** Two interleaved blocks. 1.2.3.4 appears in both, so first-block attribution wins. */ +const BLOCKS_FIXTURE: IndicatorBlock[] = [ + { + block_index: 0, + reference: 'https://twitter.com/malware_traffic/status/12345', + iocs: [makeIoc('1.2.3.4'), makeIoc('5.6.7.8')], + }, + { + block_index: 1, + reference: 'https://blog.malwareanalysis.io/cobaltstrike-2024', + iocs: [makeIoc('9.10.11.12'), makeIoc('1.2.3.4')], + }, +]; + +/** Build N unique IOC entries for a block. */ +const makeIocBlock = (blockIndex: number, count: number, reference?: string): IndicatorBlock => ({ + block_index: blockIndex, + reference, + iocs: Array.from({ length: count }, (_, i) => makeIoc(`10.0.${blockIndex}.${i}`)), +}); + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe('textIndicatorListAdapter', () => { + beforeEach(() => { + parseIndicatorListMock.mockReset(); + }); + + // ------------------------------------------------------------------------- + // Existing behaviour (small fixture — all fits in one chunk) + // ------------------------------------------------------------------------- + + it('produces one report from a valid Maltrail body', async () => { + parseIndicatorListMock.mockReturnValue(BLOCKS_FIXTURE); + const fetchMock = jest.fn().mockResolvedValue(okResponse()); + + const reports = await textIndicatorListAdapter.run(makeSource(), makeContext(fetchMock)); + + expect(reports).toHaveLength(1); + expect(() => normalizedReportSchema.parse(reports[0])).not.toThrow(); + }); + + it('sets extraction_method to text_indicator_list and stamps extracted_at', async () => { + parseIndicatorListMock.mockReturnValue(BLOCKS_FIXTURE); + const fetchMock = jest.fn().mockResolvedValue(okResponse()); + + const [report] = await textIndicatorListAdapter.run(makeSource(), makeContext(fetchMock)); + + expect(report.lineage.extraction_method).toBe('text_indicator_list'); + expect(report.lineage.extracted_at).toBe('2024-01-15T12:00:00.000Z'); + expect(report.lineage.ingested_at).toBe('2024-01-15T12:00:00.000Z'); + }); + + it('derives the trail label from the URL filename stem', async () => { + parseIndicatorListMock.mockReturnValue(BLOCKS_FIXTURE); + const fetchMock = jest.fn().mockResolvedValue(okResponse()); + + const [report] = await textIndicatorListAdapter.run(makeSource(), makeContext(fetchMock)); + + expect(report.content.title).toBe('cobaltstrike'); + expect(report.lineage.source_doc_ref?.id).toBe('cobaltstrike'); + expect(report.content.body_text).toContain('cobaltstrike'); + }); + + it('populates extracted.iocs with reference and block_index from the block', async () => { + parseIndicatorListMock.mockReturnValue(BLOCKS_FIXTURE); + const fetchMock = jest.fn().mockResolvedValue(okResponse()); + + const [report] = await textIndicatorListAdapter.run(makeSource(), makeContext(fetchMock)); + + const iocs = report.extracted?.iocs ?? []; + + // 1.2.3.4 appears in block 0 and block 1 — dedup keeps block 0 entry + const ioc124 = iocs.find((i) => i.value === '1.2.3.4'); + expect(ioc124).toBeDefined(); + expect(ioc124?.block_index).toBe(0); + expect(ioc124?.reference).toContain('twitter.com'); + + // 9.10.11.12 belongs to block 1 + const ioc9 = iocs.find((i) => i.value === '9.10.11.12'); + expect(ioc9).toBeDefined(); + expect(ioc9?.block_index).toBe(1); + expect(ioc9?.reference).toContain('malwareanalysis'); + }); + + it('sanitizes IOC references before reports are written', async () => { + parseIndicatorListMock.mockReturnValue([ + { + block_index: 0, + reference: 'https://user:secret@example.com/report', + iocs: [makeIoc('1.2.3.4')], + }, + { + block_index: 1, + reference: 'file:///etc/passwd', + iocs: [makeIoc('5.6.7.8')], + }, + ]); + + const reports = await textIndicatorListAdapter.run( + makeSource(), + makeContext(jest.fn().mockResolvedValue(okResponse())) + ); + const iocs = reports.flatMap((report) => report.extracted?.iocs ?? []); + + expect(iocs.find(({ value }) => value === '1.2.3.4')?.reference).toBe( + 'https://example.com/report' + ); + expect(iocs.find(({ value }) => value === '5.6.7.8')?.reference).toBeUndefined(); + }); + + it('deduplicates IOCs by (type, value) — first block attribution wins', async () => { + parseIndicatorListMock.mockReturnValue(BLOCKS_FIXTURE); + const fetchMock = jest.fn().mockResolvedValue(okResponse()); + + const [report] = await textIndicatorListAdapter.run(makeSource(), makeContext(fetchMock)); + + const values = (report.extracted?.iocs ?? []).map((i) => i.value); + expect(values.filter((v) => v === '1.2.3.4')).toHaveLength(1); + expect(values).toHaveLength(3); + }); + + it('returns [] when the parser produces 0 blocks', async () => { + parseIndicatorListMock.mockReturnValue([]); + const fetchMock = jest.fn().mockResolvedValue(okResponse()); + + const reports = await textIndicatorListAdapter.run(makeSource(), makeContext(fetchMock)); + + expect(reports).toHaveLength(0); + }); + + it('returns [] when blocks have 0 parseable IOCs', async () => { + const emptyIocBlocks: IndicatorBlock[] = [ + { + block_index: 0, + reference: 'https://example.com', + iocs: [], + }, + ]; + parseIndicatorListMock.mockReturnValue(emptyIocBlocks); + const fetchMock = jest.fn().mockResolvedValue(okResponse()); + + const reports = await textIndicatorListAdapter.run(makeSource(), makeContext(fetchMock)); + + expect(reports).toHaveLength(0); + }); + + it('throws on HTTP 4xx', async () => { + const fetchMock = jest + .fn() + .mockResolvedValue(new Response('', { status: 404, statusText: 'Not Found' })); + + await expect( + textIndicatorListAdapter.run(makeSource(), makeContext(fetchMock)) + ).rejects.toThrow(/HTTP 404/); + }); + + it('returns [] when the source id has no catalog URL', async () => { + const source: SourceHit = { + _id: 'text_indicator_list:unknown', + _source: { adapter_type: 'text_indicator_list', name: 'maltrail' }, + }; + const fetchMock = jest.fn(); + + const reports = await textIndicatorListAdapter.run(source, makeContext(fetchMock)); + expect(reports).toHaveLength(0); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('back-compat: normalizedReportSchema still parses a pending (pre-existing) report', () => { + const oldReport = { + '@timestamp': '2024-01-01T00:00:00.000Z', + content_fingerprint: 'abc123', + space_id: '*', + source: { type: 'rss', name: 'Test', url: 'https://example.com', adapter_id: 'rss:1' }, + content: { title: 'Test', body_text: 'body', language: 'en' }, + severity: { level: 'medium', score: 40 }, + lineage: { + ingested_at: '2024-01-01T00:00:00.000Z', + extraction_method: 'pending', + }, + }; + expect(() => normalizedReportSchema.parse(oldReport)).not.toThrow(); + }); + + // ------------------------------------------------------------------------- + // Chunking tests + // ------------------------------------------------------------------------- + + it('chunking: trail with multiple blocks splits into N reports at block boundaries', async () => { + // Three blocks. Each has (MAX_NESTED_PER_DOC / 2) IOCs + 1 ref = just over half capacity. + // So block 0 fills one chunk; block 1 can't join it; block 2 fills another chunk; etc. + const halfCap = Math.floor(MAX_NESTED_PER_DOC / 2) + 1; + const blocks: IndicatorBlock[] = [ + makeIocBlock(0, halfCap, 'https://ref0.example.com/post'), + makeIocBlock(1, halfCap, 'https://ref1.example.com/post'), + makeIocBlock(2, halfCap, 'https://ref2.example.com/post'), + ]; + parseIndicatorListMock.mockReturnValue(blocks); + const fetchMock = jest.fn().mockResolvedValue(okResponse()); + + const reports = await textIndicatorListAdapter.run(makeSource(), makeContext(fetchMock)); + + // Two blocks exceed the per-document IOC limit, so at least two reports are required. + expect(reports.length).toBeGreaterThanOrEqual(2); + + // Block integrity: every IOC from block 0 appears in exactly one report. + const allIocValues = reports.flatMap((r) => (r.extracted?.iocs ?? []).map((i) => i.value)); + const block0Values = blocks[0].iocs.map((i) => i.value); + for (const v of block0Values) { + expect(allIocValues.filter((x) => x === v)).toHaveLength(1); + } + + // Total nested IOCs per document stays bounded. + for (const report of reports) { + const iocCount = report.extracted?.iocs?.length ?? 0; + expect(iocCount).toBeLessThanOrEqual(MAX_NESTED_PER_DOC); + } + }); + + it('chunking: each chunk gets a unique content_fingerprint', async () => { + const halfCap = Math.floor(MAX_NESTED_PER_DOC / 2) + 1; + const blocks: IndicatorBlock[] = [ + makeIocBlock(0, halfCap, 'https://ref0.example.com/'), + makeIocBlock(1, halfCap, 'https://ref1.example.com/'), + makeIocBlock(2, halfCap, 'https://ref2.example.com/'), + ]; + parseIndicatorListMock.mockReturnValue(blocks); + const fetchMock = jest.fn().mockResolvedValue(okResponse()); + + const reports = await textIndicatorListAdapter.run(makeSource(), makeContext(fetchMock)); + + expect(reports.length).toBeGreaterThan(1); + const fps = reports.map((r) => r.content_fingerprint); + expect(new Set(fps).size).toBe(fps.length); + }); + + it('chunking: a single oversized block splits without losing IOC references', async () => { + const bigCount = MAX_NESTED_PER_DOC * 2 + 1; + const ref = 'https://big-ref.example.com/post'; + const blocks: IndicatorBlock[] = [ + { + block_index: 0, + reference: ref, + iocs: Array.from({ length: bigCount }, (_, i) => + makeIoc(`192.168.${Math.floor(i / 256)}.${i % 256}`) + ), + }, + ]; + parseIndicatorListMock.mockReturnValue(blocks); + const fetchMock = jest.fn().mockResolvedValue(okResponse()); + + const reports = await textIndicatorListAdapter.run(makeSource(), makeContext(fetchMock)); + + expect(reports.length).toBeGreaterThanOrEqual(3); + + const emittedIocs = reports.flatMap((report) => report.extracted?.iocs ?? []); + expect(emittedIocs).toHaveLength(bigCount); + expect(emittedIocs.every(({ reference }) => reference === ref)).toBe(true); + for (const report of reports) { + expect(report.extracted?.iocs?.length ?? 0).toBeLessThanOrEqual(MAX_NESTED_PER_DOC); + } + }); + + it('chunking: total nested per emitted doc never exceeds MAX_NESTED_PER_DOC', async () => { + // Mix: a big block (forces split) + several small blocks to test boundary arithmetic. + const bigBlock = makeIocBlock(0, MAX_NESTED_PER_DOC * 3, 'https://big.example.com/'); + const smallBlocks = Array.from({ length: 5 }, (_, i) => + makeIocBlock(i + 1, 100, `https://small${i}.example.com/`) + ); + parseIndicatorListMock.mockReturnValue([bigBlock, ...smallBlocks]); + const fetchMock = jest.fn().mockResolvedValue(okResponse()); + + const reports = await textIndicatorListAdapter.run(makeSource(), makeContext(fetchMock)); + + for (const report of reports) { + expect(report.extracted?.iocs?.length ?? 0).toBeLessThanOrEqual(MAX_NESTED_PER_DOC); + } + }); + + it('chunking: trail-wide dedup — IOC in 2 blocks appears in at most one chunk', async () => { + const sharedIoc = makeIoc('1.1.1.1'); + const bigCount = Math.floor(MAX_NESTED_PER_DOC / 2); + + // Block 0 and block 1 each have bigCount unique IOCs plus the shared one. + // They will be in separate chunks. The shared IOC must appear in only one of them. + const block0Iocs = [ + ...Array.from({ length: bigCount }, (_, i) => makeIoc(`10.0.0.${i}`)), + sharedIoc, + ]; + const block1Iocs = [ + ...Array.from({ length: bigCount }, (_, i) => makeIoc(`10.1.0.${i}`)), + { ...sharedIoc }, // same type:value, different object + ]; + + parseIndicatorListMock.mockReturnValue([ + { + block_index: 0, + reference: 'https://ref0.example.com/', + iocs: block0Iocs, + }, + { + block_index: 1, + reference: 'https://ref1.example.com/', + iocs: block1Iocs, + }, + ]); + const fetchMock = jest.fn().mockResolvedValue(okResponse()); + + const reports = await textIndicatorListAdapter.run(makeSource(), makeContext(fetchMock)); + + const allValues = reports.flatMap((r) => (r.extracted?.iocs ?? []).map((i) => i.value)); + expect(allValues.filter((v) => v === '1.1.1.1')).toHaveLength(1); + }); + + // The change signal used to be body length + first IOC + last IOC, so two + // lists that held those three values but differed in the middle produced + // identical fingerprints and the dedup gate skipped the update. + describe('change signal', () => { + const fingerprintsFor = async (iocValues: string[]) => { + parseIndicatorListMock.mockReturnValue([ + { + block_index: 0, + reference: 'https://example.com/ref', + iocs: iocValues.map((v) => makeIoc(v)), + }, + ]); + const reports = await textIndicatorListAdapter.run( + makeSource(), + makeContext(jest.fn().mockResolvedValue(okResponse())) + ); + return reports.map((r) => r.content_fingerprint); + }; + + it('is stable when the list is unchanged', async () => { + expect(await fingerprintsFor(['1.1.1.1', '2.2.2.2', '9.9.9.9'])).toEqual( + await fingerprintsFor(['1.1.1.1', '2.2.2.2', '9.9.9.9']) + ); + }); + + it('changes when an interior indicator is replaced', async () => { + // Same response body, same first and last IOC — only the middle differs. + expect(await fingerprintsFor(['1.1.1.1', '2.2.2.2', '9.9.9.9'])).not.toEqual( + await fingerprintsFor(['1.1.1.1', '3.3.3.3', '9.9.9.9']) + ); + }); + + it('changes when interior indicators are reordered', async () => { + expect(await fingerprintsFor(['1.1.1.1', '2.2.2.2', '3.3.3.3', '9.9.9.9'])).not.toEqual( + await fingerprintsFor(['1.1.1.1', '3.3.3.3', '2.2.2.2', '9.9.9.9']) + ); + }); + }); +}); + +describe('textIndicatorListAdapter — attribution and credentials', () => { + const runWith = async (source: SourceHit) => { + parseIndicatorListMock.mockReturnValue(BLOCKS_FIXTURE); + return textIndicatorListAdapter.run( + source, + makeContext(jest.fn().mockResolvedValue(okResponse())) + ); + }; + + // The create API accepts arbitrary text-list sources, so hard-coding 'maltrail' + // attributed every custom feed's reports, and every indicator promoted from + // them, to maltrail. + it('attributes reports to the configured source name', async () => { + const reports = await runWith(makeSource(SOURCE_ID, 'Acme C2 list')); + + expect(reports.length).toBeGreaterThan(0); + expect(reports[0].source.name).toBe('Acme C2 list'); + }); + + // The credential reached the stored source.url, which the promote task copies + // onto the indicator document, so this leaked well past the logs. + it('stores a credential-free catalog source URL', async () => { + const reports = await runWith(makeSource()); + + expect(reports[0].source.url).toBe(TRAIL_URL); + expect(JSON.stringify(reports[0])).not.toContain('s3cret'); + }); + + it('keeps fetch failures readable without catalog secrets', async () => { + parseIndicatorListMock.mockReturnValue(BLOCKS_FIXTURE); + const failing = makeContext( + jest + .fn() + .mockResolvedValue(new Response('nope', { status: 503, statusText: 'Service Unavailable' })) + ); + + await expect(textIndicatorListAdapter.run(makeSource(), failing)).rejects.toThrow(/HTTP 503/); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/text_indicator_list/text_indicator_list_adapter.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/text_indicator_list/text_indicator_list_adapter.ts new file mode 100644 index 0000000000000..2cf9371739e4c --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/text_indicator_list/text_indicator_list_adapter.ts @@ -0,0 +1,234 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { GLOBAL_SPACE_ID, resolveCatalogSourceUrl } from '../../../../common/threat_intel'; +import { fetchUrlForContext, redactUrl } from '../http_client'; +import { buildFingerprint } from '../fingerprint'; +import { DEFAULT_SEVERITY_LEVEL, DEFAULT_SEVERITY_SCORE } from '../../services/severity'; +import { buildReportContent } from '../../services/report_content'; +import { normalizeProvenanceUrl } from '../../services/provenance_url'; +import type { + AdapterRunContext, + FetchAdapter, + IocEntry, + NormalizedReport, + SourceHit, +} from '../types'; +import { parseIndicatorList } from './parse_indicator_list'; +import type { IndicatorBlock } from './parse_indicator_list'; + +const SOURCE_DOC_REF_INDEX = 'maltrail:trail'; + +// ES defaults to 10k nested objects per document. Keep a wide margin for future fields. +const MAX_NESTED_PER_DOC = 5000; + +const readTrailUrl = (source: SourceHit): string | undefined => { + const url = resolveCatalogSourceUrl(source._id); + return typeof url === 'string' && url.length > 0 ? url : undefined; +}; + +/** Extract the filename stem from a URL path (e.g. `.../malware/cobaltstrike.txt` → `cobaltstrike`). */ +const trailLabelFromUrl = (url: string): string => { + try { + const { pathname } = new URL(url); + const filename = pathname.split('/').pop() ?? ''; + return filename.replace(/\.[^.]+$/, '') || 'unknown'; + } catch { + return 'unknown'; + } +}; + +/** + * Keep reference blocks together when possible while bounding nested IOCs per report. + * Trail-wide dedup happens before this function, and each IOC already carries its + * nearest reference, so no parallel reference metadata needs to be reconstructed. + */ +const chunkBlocks = ( + blocks: IndicatorBlock[], + dedupedIocs: Map +): IocEntry[][] => { + const iocsByBlock = new Map(); + for (const ioc of dedupedIocs.values()) { + const bi = ioc.block_index ?? 0; + const arr = iocsByBlock.get(bi); + if (arr) { + arr.push(ioc); + } else { + iocsByBlock.set(bi, [ioc]); + } + } + + const chunks: IocEntry[][] = []; + let currentIocs: IocEntry[] = []; + + const flush = () => { + if (currentIocs.length > 0) { + chunks.push(currentIocs); + currentIocs = []; + } + }; + + for (const block of blocks) { + const blockIocs = iocsByBlock.get(block.block_index) ?? []; + if (blockIocs.length > 0) { + if (blockIocs.length <= MAX_NESTED_PER_DOC) { + if (currentIocs.length + blockIocs.length > MAX_NESTED_PER_DOC) flush(); + currentIocs.push(...blockIocs); + } else { + flush(); + for (let offset = 0; offset < blockIocs.length; offset += MAX_NESTED_PER_DOC) { + chunks.push(blockIocs.slice(offset, offset + MAX_NESTED_PER_DOC)); + } + } + } + } + + flush(); + return chunks; +}; + +export const textIndicatorListAdapter: FetchAdapter = { + adapterType: 'text_indicator_list', + async run(source: SourceHit, context: AdapterRunContext): Promise { + const fetchUrl = fetchUrlForContext(context); + const log = context.logger.get('text-indicator-list-adapter'); + const url = readTrailUrl(source); + if (!url) { + log.warn(`Source ${source._id} has no catalog URL — skipping`); + return []; + } + + // Source URLs may embed `user:password@`, and this is the only adapter that was + // interpolating the raw URL into its errors and log lines. The credential also + // reached the stored `source.url`, which the promote task copies onto the + // indicator document, so it was leaking well past the logs. Keep the raw URL + // for the request only. + const redactedUrl = redactUrl(url); + const provenanceUrl = normalizeProvenanceUrl(url); + + const response = await fetchUrl(url, { + abortSignal: context.abortSignal, + headers: { Accept: 'text/plain, */*' }, + }); + if (response.status >= 400) { + throw new Error( + `text_indicator_list fetch ${redactedUrl} failed: HTTP ${response.status} ${response.statusText}` + ); + } + + const blocks = parseIndicatorList(response.body); + if (blocks.length === 0) { + log.warn(`text_indicator_list at ${redactedUrl} produced 0 blocks for source ${source._id}`); + return []; + } + + const trailLabel = trailLabelFromUrl(url); + + // Trail-wide IOC dedup by (type, value) before chunking — a value repeated across blocks + // is emitted once total; first-block attribution wins. Dedup happens here so that an IOC + // present in block 0 and block 5 doesn't appear in two separate chunks. + const dedupedIocs = new Map(); + for (const block of blocks) { + const reference = normalizeProvenanceUrl(block.reference); + for (const ioc of block.iocs) { + const key = `${ioc.type}:${ioc.value}`; + if (!dedupedIocs.has(key)) { + dedupedIocs.set(key, { + ...ioc, + ...(reference ? { reference } : {}), + block_index: block.block_index, + }); + } + } + } + + if (dedupedIocs.size === 0) { + log.warn( + `text_indicator_list at ${redactedUrl} had blocks but 0 parseable IOCs for source ${source._id}` + ); + return []; + } + + // Chunk into docs, each ≤ MAX_NESTED_PER_DOC total nested objects. + const chunks = chunkBlocks(blocks, dedupedIocs); + + const ingestedAt = context.now().toISOString(); + const spaceId = source._source.space_id ?? GLOBAL_SPACE_ID; + + // Change-signal for the content fingerprint: a hash over the whole canonical + // IOC set. This used to be body length plus the first and last IOC, which + // collided whenever interior indicators were added, removed, replaced, or + // reordered while those three values held — the dedup gate then skipped a + // genuinely updated list and the indicator index went stale. Hashing the set + // (rather than the raw body) also avoids re-ingesting on cosmetic feed + // changes such as an updated header comment. + const allIocs = [...dedupedIocs.values()]; + const changeSignal = buildFingerprint( + allIocs.map((ioc) => `${ioc.type}=${ioc.value}@${ioc.reference ?? ''}`) + ); + + const reports: NormalizedReport[] = []; + + for (let chunkIdx = 0; chunkIdx < chunks.length; chunkIdx++) { + const iocs = chunks[chunkIdx]; + + // Per-chunk fingerprint: fold chunk index so N chunks of one trail don't collide on dedup. + // Seeded with the redacted URL so rotating the feed credential does not + // change the identity of every report from that feed. + const contentFingerprint = buildFingerprint([ + provenanceUrl ?? redactedUrl, + trailLabel, + changeSignal, + String(chunkIdx), + ]); + + const bodyText = `Maltrail indicator list: ${trailLabel} (chunk ${chunkIdx + 1}/${ + chunks.length + }, ${iocs.length} indicators)`; + const baseContent = buildReportContent({ + title: trailLabel, + bodyText, + language: 'en', + }); + + const report: NormalizedReport = { + '@timestamp': ingestedAt, + content_fingerprint: contentFingerprint, + space_id: spaceId, + source: { + type: 'text_indicator_list', + // The configured source name from the approved catalog entry, not the literal + // 'maltrail'. The catalog can seed more than one text-list source, so hard-coding + // this would misattribute every text-list feed's reports and indicators to maltrail. + name: source._source.name, + ...(provenanceUrl ? { url: provenanceUrl } : {}), + adapter_id: `text_indicator_list:${source._id}`, + }, + content: baseContent, + severity: { + level: DEFAULT_SEVERITY_LEVEL, + score: DEFAULT_SEVERITY_SCORE, + }, + lineage: { + ingested_at: ingestedAt, + extraction_method: 'text_indicator_list', + extracted_at: ingestedAt, + source_doc_ref: { index: SOURCE_DOC_REF_INDEX, id: trailLabel }, + }, + extracted: { iocs }, + }; + + reports.push(report); + } + + log.info( + `text_indicator_list: ${redactedUrl} → ${dedupedIocs.size} deduped IOCs across ${reports.length} chunk(s) for source ${source._id}` + ); + + return reports; + }, +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/types.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/types.ts index 03f2dca63df7d..7994a5066502d 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/types.ts @@ -6,7 +6,10 @@ */ import type { Logger } from '@kbn/core/server'; -import type { NormalizedReport } from '../../../common/threat_intel/workflows/step_types/fetch_source/fetch_source_common'; +import type { + IocEntry, + NormalizedReport, +} from '../../../common/threat_intel/workflows/step_types/fetch_source/fetch_source_common'; import type { FetchAdapterType } from '../../../common/threat_intel'; import type { DnsLookupFn } from './http_client'; @@ -70,4 +73,4 @@ export interface FetchAdapter { run(source: SourceHit, context: AdapterRunContext): Promise; } -export type { NormalizedReport }; +export type { IocEntry, NormalizedReport }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/workflows/step_types/fetch_source/fetch_source_step.test.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/workflows/step_types/fetch_source/fetch_source_step.test.ts new file mode 100644 index 0000000000000..3bb6e6419bb00 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/workflows/step_types/fetch_source/fetch_source_step.test.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import type { StepHandlerContext } from '@kbn/workflows-extensions/server'; +import { buildFetchSourceStepDefinition } from './fetch_source_step'; +import { runAdapter, UnknownAdapterError } from '../../../adapters'; +import type { SourceHit } from '../../../adapters'; + +jest.mock('../../../adapters', () => { + const actual = jest.requireActual('../../../adapters'); + return { ...actual, runAdapter: jest.fn() }; +}); + +const runAdapterMock = runAdapter as jest.Mock; + +const SOURCE: SourceHit = { + _id: 'rss:mandiant-research', + _source: { adapter_type: 'rss', name: 'Mandiant Research' }, +}; + +const buildContext = (input: unknown): StepHandlerContext => + ({ + input, + abortSignal: new AbortController().signal, + } as unknown as StepHandlerContext); + +const buildStep = () => { + const logger = loggingSystemMock.createLogger(); + const step = buildFetchSourceStepDefinition({ logger }); + return { logger, handler: step.handler }; +}; + +describe('buildFetchSourceStepDefinition handler', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('returns a descriptive error when source arrives as a string', async () => { + const { handler } = buildStep(); + const result = await handler(buildContext({ source: '[object Object]' })); + + expect(result.error).toBeInstanceOf(Error); + expect(result.error?.message).toMatch(/received "source" as a string/); + expect(result.error?.message).toMatch(/foreach\.item/); + expect(runAdapterMock).not.toHaveBeenCalled(); + }); + + it('returns adapter output on the happy path', async () => { + const reports = [{ '@timestamp': '2026-09-08T00:00:00.000Z' }]; + runAdapterMock.mockResolvedValue(reports); + const { handler } = buildStep(); + + const result = await handler(buildContext({ source: SOURCE })); + + expect(result.error).toBeUndefined(); + expect(result.output).toEqual({ + adapter_type: 'rss', + source_id: 'rss:mandiant-research', + total_fetched: 1, + reports, + }); + expect(runAdapterMock).toHaveBeenCalledWith( + SOURCE, + expect.objectContaining({ abortSignal: expect.any(AbortSignal) }) + ); + }); + + it('propagates UnknownAdapterError as-is', async () => { + const unknown = new UnknownAdapterError('bogus', 'src-1'); + runAdapterMock.mockRejectedValue(unknown); + const { handler } = buildStep(); + + const result = await handler(buildContext({ source: SOURCE })); + + expect(result.error).toBe(unknown); + }); + + it('wraps generic adapter failures with the source id', async () => { + runAdapterMock.mockRejectedValue(new Error('HTTP 503')); + const { handler } = buildStep(); + + const result = await handler(buildContext({ source: SOURCE })); + + expect(result.error).toBeInstanceOf(Error); + expect(result.error?.message).toBe( + 'Failed to fetch threat intelligence source rss:mandiant-research: HTTP 503' + ); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/workflows/step_types/fetch_source/fetch_source_step.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/workflows/step_types/fetch_source/fetch_source_step.ts new file mode 100644 index 0000000000000..5ca39ad51e5af --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/workflows/step_types/fetch_source/fetch_source_step.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { createServerStepDefinition } from '@kbn/workflows-extensions/server'; +import { fetchSourceStepCommonDefinition } from '../../../../../common/threat_intel/workflows/step_types/fetch_source/fetch_source_common'; +import { runAdapter, UnknownAdapterError } from '../../../adapters'; +import type { AdapterRunContext, SourceHit } from '../../../adapters'; + +export interface BuildFetchSourceStepDeps { + logger: Logger; +} + +/** + * Build the `threat_intel.fetch_source` step definition. + * + * Wraps a step-scoped logger from the host plugin so the adapter run + * context's `logger.get()` chains end up under + * `securitySolution.threatIntel.fetch_source.` in the + * Kibana logs — that path is what operators grep when a feed misbehaves. + * + */ +export const buildFetchSourceStepDefinition = (deps: BuildFetchSourceStepDeps) => + createServerStepDefinition({ + ...fetchSourceStepCommonDefinition, + handler: async (context) => { + // The handler is called with the rendered `with` payload as `input`. + // Cast through the inferred input shape from the common schema — + // the Zod runtime parsing is the engine's responsibility (see + // `CustomStepImpl.createHandlerContext`); we re-narrow on access + // here so the adapter contract sees a `SourceHit`. + const { source } = context.input as { source: SourceHit | string }; + + // `fetchSourceInputSchema` accepts a string because `{{ foreach.item }}` + // (without the `$`) stringifies the hit instead of passing the object. + // That is schema-valid, so catch it here and return the fix rather than + // letting `source._source` throw a bare TypeError outside the try block. + if (typeof source === 'string') { + return { + error: new Error( + `threat_intel.fetch_source received "source" as a string ("${source.slice( + 0, + 80 + )}"). Pass the hit as \${{ foreach.item }} so it stays an object; {{ foreach.item }} stringifies it.` + ), + }; + } + + const stepLogger = deps.logger.get( + 'threatIntel', + 'fetch_source', + source._source.adapter_type + ); + + const runContext: AdapterRunContext = { + logger: stepLogger, + abortSignal: context.abortSignal, + now: () => new Date(), + }; + + try { + const reports = await runAdapter(source, runContext); + return { + output: { + adapter_type: source._source.adapter_type, + source_id: source._id, + total_fetched: reports.length, + reports, + }, + }; + } catch (err) { + // Engine convention: a returned `error` is still a step + // failure (see `BaseAtomicNodeImplementation`), but it lets us + // attach a structured message instead of the raw stack. The + // workflow's per-step `on-failure: continue: true` catches + // these so a single misbehaving feed doesn't abort the run. + const message = err instanceof Error ? err.message : String(err); + stepLogger.warn(`Adapter failed for source ${source._id}: ${message}`); + return { + error: + err instanceof UnknownAdapterError + ? err + : new Error(`Failed to fetch threat intelligence source ${source._id}: ${message}`), + }; + } + }, + }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/workflows/step_types/index.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/workflows/step_types/index.ts new file mode 100644 index 0000000000000..ec7630b1c2ec2 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/workflows/step_types/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import type { WorkflowsExtensionsServerPluginSetup } from '@kbn/workflows-extensions/server'; +import { buildFetchSourceStepDefinition } from './fetch_source/fetch_source_step'; + +/** Registers threat_intel.fetch_source on the workflowsExtensions contract. */ +export const registerThreatIntelWorkflowSteps = ({ + workflowsExtensions, + logger, +}: { + workflowsExtensions: WorkflowsExtensionsServerPluginSetup; + logger: Logger; +}): void => { + workflowsExtensions.registerStepDefinition(buildFetchSourceStepDefinition({ logger })); +};