From 3c14ba6fbba19ae9d042c787c22ab701a2f1bd6d Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Wed, 26 Aug 2026 14:36:26 -0600 Subject: [PATCH 01/27] feat(threat-intel): RSS, STIX, and TAXII source adapters --- .../threat_intel/adapters/fingerprint.test.ts | 73 ++++ .../threat_intel/adapters/fingerprint.ts | 30 ++ .../adapters/rss/parse_rss.test.ts | 154 +++++++ .../threat_intel/adapters/rss/parse_rss.ts | 238 +++++++++++ .../adapters/rss/rss_adapter.test.ts | 187 +++++++++ .../threat_intel/adapters/rss/rss_adapter.ts | 131 ++++++ .../adapters/stix/parse_pattern.test.ts | 383 ++++++++++++++++++ .../adapters/stix/parse_pattern.ts | 213 ++++++++++ .../adapters/stix/split_bundle.test.ts | 103 +++++ .../adapters/stix/split_bundle.ts | 104 +++++ .../adapters/stix/stix_adapter.test.ts | 329 +++++++++++++++ .../adapters/stix/stix_adapter.ts | 162 ++++++++ .../adapters/taxii/taxii_adapter.test.ts | 243 +++++++++++ .../adapters/taxii/taxii_adapter.ts | 214 ++++++++++ .../server/threat_intel/adapters/types.ts | 113 ++++++ 15 files changed, 2677 insertions(+) create mode 100644 x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/fingerprint.test.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/fingerprint.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/parse_rss.test.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/parse_rss.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/rss_adapter.test.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/rss_adapter.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/parse_pattern.test.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/parse_pattern.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/split_bundle.test.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/split_bundle.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/stix_adapter.test.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/stix_adapter.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/taxii/taxii_adapter.test.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/taxii/taxii_adapter.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/types.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/fingerprint.test.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/fingerprint.test.ts new file mode 100644 index 0000000000000..c19701426d28e --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/fingerprint.test.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createHash } from 'crypto'; +import { buildFingerprint } from './fingerprint'; + +describe('buildFingerprint', () => { + it('length-prefixes each part so the seed is unambiguous', () => { + // The seed is `:` per part, concatenated. There is no workflow-side + // computation to match: the definitions only pass `content_fingerprint` + // through, and the engine has no sha256 Liquid filter, so this encoding is + // ours alone and only has to be self-consistent. + const expected = createHash('sha256') + .update('28:https://example.com/feed.xml7:item-425:Title') + .digest('hex'); + expect(buildFingerprint(['https://example.com/feed.xml', 'item-42', 'Title'])).toBe(expected); + }); + + // A plain join on `:` collided across part boundaries, so two different feed + // items produced one fingerprint and the second was deduplicated away as + // already-ingested. Titles, URLs, and ids routinely contain colons. + it('does not collide when a colon moves across a part boundary', () => { + expect(buildFingerprint(['a:b', 'c'])).not.toBe(buildFingerprint(['a', 'b:c'])); + }); + + it('does not collide when a part boundary shifts', () => { + expect(buildFingerprint(['https://evil.test', 'a'])).not.toBe( + buildFingerprint(['https://evil.test:a', '']) + ); + }); + + it('returns a 64-char hex digest', () => { + expect(buildFingerprint(['a', 'b'])).toMatch(/^[0-9a-f]{64}$/); + }); + + it('NFKC-normalizes parts so unicode equivalents collapse', () => { + // U+FB01 (fi ligature) vs ASCII "fi". + expect(buildFingerprint(['url', 'id', '\ufb01nal'])).toBe( + buildFingerprint(['url', 'id', 'final']) + ); + }); + + it('trims leading/trailing whitespace per part', () => { + expect(buildFingerprint([' url ', ' id '])).toBe(buildFingerprint(['url', 'id'])); + }); + + it('treats undefined/null parts as empty strings (still positional)', () => { + // Two leading missing parts are still part of the seed shape — a + // re-fetch with the same shape produces the same digest. + const a = buildFingerprint([undefined, undefined, 'id']); + const b = buildFingerprint([undefined, undefined, 'id']); + expect(a).toBe(b); + // …but a missing part is *positionally distinct* from the part + // moving up by one — `:::id` and `id` produce different digests. + expect(a).not.toBe(buildFingerprint(['id'])); + }); + + it('produces a stable digest for the same logical input', () => { + const fp1 = buildFingerprint(['https://example.com', 'id-1', 'modified-2026-05-01']); + const fp2 = buildFingerprint(['https://example.com', 'id-1', 'modified-2026-05-01']); + expect(fp1).toBe(fp2); + }); + + it('produces different digests when the version stamp changes', () => { + const a = buildFingerprint(['https://example.com', 'id-1', '2026-05-01']); + const b = buildFingerprint(['https://example.com', 'id-1', '2026-05-02']); + expect(a).not.toBe(b); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/fingerprint.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/fingerprint.ts new file mode 100644 index 0000000000000..bb7a7e3f31564 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/fingerprint.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createHash } from 'crypto'; + +/** + * SHA-256 over NFKC-normalized parts, each length-prefixed. + * + * A plain `join(':')` is ambiguous: `['a:b', 'c']` and `['a', 'b:c']` produce the + * same seed, so two different items collide on one fingerprint and the second is + * deduplicated away as though it had already been ingested. The parts here are + * feed-controlled titles, URLs, and ids, which routinely contain colons, so this + * is reachable rather than theoretical. + * + * Length-prefixing makes the seed unambiguous. Nothing outside this module + * recomputes these values (the workflows only pass `content_fingerprint` through), + * so the change is safe; already-stored reports will be re-ingested once under + * their new fingerprint. + */ +export const buildFingerprint = (parts: ReadonlyArray): string => { + const seed = parts + .map((part) => (part ?? '').trim().normalize('NFKC')) + .map((part) => `${part.length}:${part}`) + .join(''); + return createHash('sha256').update(seed).digest('hex'); +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/parse_rss.test.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/parse_rss.test.ts new file mode 100644 index 0000000000000..da2751823c2c1 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/parse_rss.test.ts @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { parseRssFeed } from './parse_rss'; + +const RSS2 = ` + + + Acme Threat Research + en-US + + APT-1 campaign + https://acme.example/posts/apt1 + acme:apt1 + Mon, 12 May 2025 09:30:00 GMT + Brief summary of APT-1.

]]> + + + Ransomware uptick + https://acme.example/posts/ransom + Tue, 13 May 2025 10:00:00 GMT + Plain summary. + + + + Orphan item with no id, link, or guid. + + +`; + +const ATOM = ` + + Vendor Labs + + tag:vendor.example,2025:post-1 + Post one + + 2025-05-12T09:30:00Z + Short summary. + <p>Long body</p> + + + tag:vendor.example,2025:post-2 + Post two + + 2025-05-11T08:00:00Z + Only a summary. + +`; + +const RDF = ` + + + RDF Feed + en + + + RDF Item + https://example.com/posts/1 + RDF body. + 2025-05-12T09:30:00Z + +`; + +describe('parseRssFeed', () => { + it('returns an empty result for an empty input', async () => { + const parsed = await parseRssFeed(''); + expect(parsed).toEqual({ feedTitle: '', entries: [] }); + }); + + it('parses an RSS 2.0 feed and drops items without an identifier', async () => { + const parsed = await parseRssFeed(RSS2); + expect(parsed.feedTitle).toBe('Acme Threat Research'); + expect(parsed.language).toBe('en'); + expect(parsed.entries).toHaveLength(2); + expect(parsed.entries[0]).toMatchObject({ + id: 'acme:apt1', + title: 'APT-1 campaign', + link: 'https://acme.example/posts/apt1', + publishedAt: new Date('Mon, 12 May 2025 09:30:00 GMT').toISOString(), + }); + // CDATA contents preserved as-is on bodyHtml. + expect(parsed.entries[0].bodyHtml).toContain('APT-1'); + }); + + it('parses an Atom feed and prefers updated over published', async () => { + const parsed = await parseRssFeed(ATOM); + expect(parsed.feedTitle).toBe('Vendor Labs'); + expect(parsed.language).toBe('en'); + expect(parsed.entries).toHaveLength(2); + expect(parsed.entries[0].publishedAt).toBe('2025-05-12T09:30:00.000Z'); + expect(parsed.entries[0].link).toBe('https://vendor.example/post-1'); + expect(parsed.entries[0].bodyHtml).toBe('

Long body

'); + expect(parsed.entries[1].publishedAt).toBe('2025-05-11T08:00:00.000Z'); + }); + + it('parses an RDF / RSS 1.0 feed using rdf:about as the id', async () => { + const parsed = await parseRssFeed(RDF); + expect(parsed.feedTitle).toBe('RDF Feed'); + expect(parsed.language).toBe('en'); + expect(parsed.entries).toHaveLength(1); + expect(parsed.entries[0]).toMatchObject({ + id: 'https://example.com/posts/1', + title: 'RDF Item', + link: 'https://example.com/posts/1', + publishedAt: '2025-05-12T09:30:00.000Z', + }); + }); + + it('returns an empty result for an unrecognized root element', async () => { + const parsed = await parseRssFeed(''); + expect(parsed).toEqual({ feedTitle: '', entries: [] }); + }); +}); + +// RSS 2.0 permits an item with a description and no title, and real advisory feeds +// publish them. Every parser branch leaves `body` empty and puts the description in +// `bodyHtml`, so a `title || body` check dropped all of them. +describe('parseRssFeed — description-only items', () => { + it('keeps an RSS 2.0 item that has a description but no title', async () => { + const feed = ` + + + Advisories + + adv-1 + Threat actor deployed ransomware via 185.220.101.45. + + +`; + + const parsed = await parseRssFeed(feed); + + expect(parsed.entries).toHaveLength(1); + expect(parsed.entries[0].id).toBe('adv-1'); + expect(parsed.entries[0].bodyHtml).toContain('ransomware'); + }); + + it('still drops an item with no identifier', async () => { + const feed = ` + + + Advisories + No guid and no link. + +`; + + expect((await parseRssFeed(feed)).entries).toHaveLength(0); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/parse_rss.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/parse_rss.ts new file mode 100644 index 0000000000000..189074634d420 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/parse_rss.ts @@ -0,0 +1,238 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import xml2js from 'xml2js'; + +/** + * Format-agnostic representation of a single feed entry. The RSS + * adapter emits one `NormalizedReport` per `RssEntry` regardless of + * whether the upstream feed is RSS 2.0, Atom, or RDF — keeping the + * normalization step format-agnostic means the adapter doesn't have + * to branch on the feed family at every step. + * + * `id` is the most stable identifier the feed exposes (Atom ``, + * RSS ``, falling back to `` when neither is present). + * It's used by the adapter to seed the per-item `content_fingerprint` + * and to populate `lineage.source_doc_ref.id`. If we end up with + * two items that share the same `id` *and* the same title, the + * fingerprint will collapse them — which is what we want for + * RSS-syndicated copies of the same advisory. + */ +export interface RssEntry { + id: string; + title: string; + link?: string; + /** ISO-8601 string when the feed exposes a publish/update timestamp. */ + publishedAt?: string; + /** + * Best-effort plaintext body. The adapter strips HTML before writing + * to `content.body_text`; the original markup is preserved on + * `content.body_html` (mapped `index: false`) so extraction can be re-run + * without re-fetching. It is unsanitized feed markup and must not be rendered. + */ + body: string; + /** Original markup preserved verbatim. */ + bodyHtml?: string; +} + +export interface ParsedFeed { + /** Normalized feed-level title. Empty string when the feed declares no title. */ + feedTitle: string; + /** Lowercased ISO-639-1 if the feed declares one (`` / `xml:lang`). */ + language?: string; + entries: RssEntry[]; +} + +/** + * Parse an RSS 2.0, Atom, or RDF feed. + * + * Tolerant by design — feeds in the wild violate every spec at least + * sometimes, so we treat the structure as advisory and collect what + * we can. Items missing every identifying field (no `id`, `guid`, or + * `link`) are dropped because there's no stable seed for the + * fingerprint and including them would just create one new row per + * run forever. + */ +export const parseRssFeed = async (xml: string): Promise => { + const trimmed = xml.trim(); + if (!trimmed) return { feedTitle: '', entries: [] }; + + // `explicitArray: true` matches the existing siem_migrations XmlParser + // convention so all child accessors are arrays — which simplifies the + // walk below (no `Array.isArray` branches per field). + const parsed = await xml2js.parseStringPromise(trimmed, { + explicitArray: true, + explicitCharkey: true, + charkey: '_', + attrkey: '$', + trim: false, + normalizeTags: false, + }); + + // Atom: + if (parsed.feed) return parseAtom(parsed.feed); + // RSS 2.0: ... + if (parsed.rss?.channel?.[0]) return parseRss2(parsed.rss.channel[0]); + // RDF / RSS 1.0: ... + const rdfRoot = parsed['rdf:RDF'] ?? parsed.RDF ?? parsed['rss:RDF'] ?? parsed.feed ?? undefined; + if (rdfRoot) return parseRdf(rdfRoot); + + return { feedTitle: '', entries: [] }; +}; + +interface XmlNode { + $?: Record; + _?: string; + [key: string]: unknown; +} + +/** Best-effort string extraction from xml2js's nested array+object shape. */ +const text = (node: unknown): string => { + if (node == null) return ''; + if (typeof node === 'string') return node; + if (Array.isArray(node)) return text(node[0]); + if (typeof node === 'object') { + const obj = node as XmlNode; + if (typeof obj._ === 'string') return obj._; + // Atom uses an attribute-only shape. + if (obj.$ && typeof obj.$.href === 'string') return obj.$.href; + } + return ''; +}; + +const firstAttr = (node: unknown, attr: string): string | undefined => { + if (Array.isArray(node)) return firstAttr(node[0], attr); + if (node && typeof node === 'object') { + const obj = node as XmlNode; + return obj.$?.[attr]; + } + return undefined; +}; + +const toIsoDate = (raw: string | undefined): string | undefined => { + if (!raw) return undefined; + const trimmed = raw.trim(); + if (!trimmed) return undefined; + const ms = Date.parse(trimmed); + return Number.isFinite(ms) ? new Date(ms).toISOString() : undefined; +}; + +const pickLanguage = (raw: string | undefined): string | undefined => { + if (!raw) return undefined; + // `en-US` / `en_us` → `en`. Lowercasing matches ECS' `language` + // field convention used by downstream mappers. + const head = raw.split(/[-_]/)[0]; + return head ? head.toLowerCase() : undefined; +}; + +const dropEmpty = (entries: RssEntry[]): RssEntry[] => + // Items without any identifier would generate a fresh fingerprint on + // every run, defeating the dedup gate. Better to drop them than to + // pollute the reports index. + // + // `bodyHtml` has to be in the content check. Every parser branch below leaves + // `body` as an empty string and puts the actual description in `bodyHtml`, so + // testing `title || body` dropped every RSS 2.0 item that had a description but + // no title, which the spec permits and real advisory feeds do publish. + entries.filter((entry) => entry.id && (entry.title || entry.body || entry.bodyHtml)); + +const parseRss2 = (channel: XmlNode): ParsedFeed => { + const items = (channel.item as XmlNode[] | undefined) ?? []; + const entries: RssEntry[] = items.map((item) => { + const guid = text(item.guid); + const link = text(item.link); + const id = guid || link || text(item.id); + const title = text(item.title); + const description = text(item.description); + // ships the full HTML article body when the + // feed wants to provide more than the summary. Some feeds use it, + // some don't — fall back to the description. + const contentEncoded = text((item as XmlNode)['content:encoded']); + const bodyHtml = contentEncoded || description || undefined; + const publishedAt = + toIsoDate(text(item.pubDate)) ?? toIsoDate(text((item as XmlNode)['dc:date'])); + return { + id, + title, + link: link || undefined, + publishedAt, + body: '', + bodyHtml, + }; + }); + return { + feedTitle: text(channel.title), + language: pickLanguage(text(channel.language)), + entries: dropEmpty(entries), + }; +}; + +const parseAtom = (feed: XmlNode): ParsedFeed => { + const entriesRaw = (feed.entry as XmlNode[] | undefined) ?? []; + const entries: RssEntry[] = entriesRaw.map((entry) => { + const id = text(entry.id); + const title = text(entry.title); + // Atom links can be one or many; we want the first `rel="alternate"` + // (or the one without a rel attribute, which is how most feeds ship + // a single canonical link). + const linkArr = (entry.link as XmlNode[] | undefined) ?? []; + const link = + linkArr.find((l) => { + const rel = firstAttr(l, 'rel'); + return rel === undefined || rel === 'alternate'; + }) ?? linkArr[0]; + const linkHref = firstAttr(link, 'href') ?? text(link); + const summary = text(entry.summary); + const content = text(entry.content); + const bodyHtml = content || summary || undefined; + const publishedAt = toIsoDate(text(entry.updated)) ?? toIsoDate(text(entry.published)); + return { + id: id || linkHref, + title, + link: linkHref || undefined, + publishedAt, + body: '', + bodyHtml, + }; + }); + return { + feedTitle: text(feed.title), + language: pickLanguage(firstAttr(feed, 'xml:lang')), + entries: dropEmpty(entries), + }; +}; + +const parseRdf = (rdf: XmlNode): ParsedFeed => { + // RSS 1.0 puts items as siblings of at the RDF root rather + // than nested under it; the channel's `` only references + // them by `rdf:about`. We don't need the order — we just walk the + // siblings. + const channel = ((rdf.channel as XmlNode[] | undefined) ?? [])[0] ?? {}; + const items = (rdf.item as XmlNode[] | undefined) ?? []; + const entries: RssEntry[] = items.map((item) => { + const about = firstAttr(item, 'rdf:about') ?? firstAttr(item, 'about'); + const link = text(item.link); + const id = about || link || text(item.guid); + const title = text(item.title); + const description = text(item.description); + const publishedAt = + toIsoDate(text((item as XmlNode)['dc:date'])) ?? toIsoDate(text(item.pubDate)); + return { + id, + title, + link: link || undefined, + publishedAt, + body: '', + bodyHtml: description || undefined, + }; + }); + return { + feedTitle: text(channel.title), + language: pickLanguage(text((channel as XmlNode)['dc:language']) || text(channel.language)), + entries: dropEmpty(entries), + }; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/rss_adapter.test.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/rss_adapter.test.ts new file mode 100644 index 0000000000000..002fd0ff43b92 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/rss_adapter.test.ts @@ -0,0 +1,187 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { rssAdapter } from './rss_adapter'; +import type { AdapterRunContext, SourceHit } from '../types'; + +const FEED_URL = 'https://acme.example/feed.xml'; +const FEED_BODY = ` + + Acme + + Item one + acme:1 + https://acme.example/1 + Mon, 12 May 2025 09:30:00 GMT + Body one

]]> + + + Item two + acme:2 + https://acme.example/2 + Tue, 13 May 2025 09:30:00 GMT + Body two + +`; + +const buildSource = (overrides: Partial = {}): SourceHit => ({ + _id: 'rss:acme', + _source: { + adapter_type: 'rss', + name: 'Acme', + config: { url: FEED_URL }, + ...overrides, + }, +}); + +const NOW = new Date('2026-05-16T12:00:00.000Z'); + +const buildContext = ( + fetchImpl: jest.Mock, [string | URL | Request, RequestInit?]> +): AdapterRunContext => ({ + esClient: elasticsearchServiceMock.createElasticsearchClient(), + logger: loggingSystemMock.createLogger(), + abortSignal: new AbortController().signal, + now: () => NOW, + fetchFn: fetchImpl as unknown as typeof fetch, + lookupFn: async () => [{ address: '93.184.216.34' }], +}); + +const okResponse = (body: string): Response => + new Response(body, { + status: 200, + statusText: 'OK', + headers: { 'Content-Type': 'application/rss+xml' }, + }); + +describe('rssAdapter', () => { + it('emits one normalized report per RSS item', async () => { + const fetchMock = jest.fn().mockResolvedValue(okResponse(FEED_BODY)); + const reports = await rssAdapter.run(buildSource(), buildContext(fetchMock)); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock.mock.calls[0][0]).toBe(FEED_URL); + + expect(reports).toHaveLength(2); + expect(reports[0]).toMatchObject({ + '@timestamp': NOW.toISOString(), + space_id: '*', + source: { + type: 'rss', + name: 'Acme', + url: 'https://acme.example/1', + adapter_id: 'rss:rss:acme', + }, + content: { + title: 'Item one', + body_text: 'Body one', + body_html: '

Body one

', + language: 'en', + }, + severity: { level: 'medium', score: 40 }, + lineage: { + ingested_at: NOW.toISOString(), + extraction_method: 'pending', + source_doc_ref: { index: 'rss:feed', id: 'acme:1' }, + }, + }); + // Fingerprint must be a stable 64-hex digest (matches buildFingerprint). + expect(reports[0].content_fingerprint).toMatch(/^[0-9a-f]{64}$/); + expect(reports[0].content_fingerprint).not.toBe(reports[1].content_fingerprint); + }); + + // The fingerprint used to be feed URL + guid + title only, so an advisory + // that kept its guid and title while revising the text deduped away forever. + describe('revision detection', () => { + const feedWith = ({ body = 'Body one', pubDate = 'Mon, 12 May 2025 09:30:00 GMT' } = {}) => + ` + + Acme + + Item one + acme:1 + https://acme.example/1 + ${pubDate} + ${body} + +`; + + const fingerprintFor = async (feed: string) => { + const reports = await rssAdapter.run( + buildSource(), + buildContext(jest.fn().mockResolvedValue(okResponse(feed))) + ); + return reports[0].content_fingerprint; + }; + + it('is stable when the item is unchanged', async () => { + expect(await fingerprintFor(feedWith())).toBe(await fingerprintFor(feedWith())); + }); + + it('changes when the body is revised under the same guid and title', async () => { + expect(await fingerprintFor(feedWith({ body: 'Revised body with new IOCs' }))).not.toBe( + await fingerprintFor(feedWith()) + ); + }); + + it('changes when only the publish timestamp moves', async () => { + expect(await fingerprintFor(feedWith({ pubDate: 'Wed, 14 May 2025 09:30:00 GMT' }))).not.toBe( + await fingerprintFor(feedWith()) + ); + }); + }); + + it('stamps space_id from the source when set', async () => { + const fetchMock = jest.fn().mockResolvedValue(okResponse(FEED_BODY)); + const reports = await rssAdapter.run( + buildSource({ space_id: 'team-a' }), + buildContext(fetchMock) + ); + expect(reports[0].space_id).toBe('team-a'); + }); + + it('returns [] when the source has no config.url', async () => { + const fetchMock = jest.fn(); + const reports = await rssAdapter.run( + buildSource({ config: {} as Record }), + buildContext(fetchMock) + ); + expect(reports).toEqual([]); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('throws on a non-2xx response so the workflow surfaces the failure', async () => { + const fetchMock = jest + .fn() + .mockResolvedValue(new Response('boom', { status: 503, statusText: 'Service Unavailable' })); + await expect(rssAdapter.run(buildSource(), buildContext(fetchMock))).rejects.toThrow( + /HTTP 503/ + ); + }); + + it('returns [] when the feed contains no parseable items', async () => { + const empty = `Empty`; + const fetchMock = jest.fn().mockResolvedValue(okResponse(empty)); + const reports = await rssAdapter.run(buildSource(), buildContext(fetchMock)); + expect(reports).toEqual([]); + }); + + it('ingests percent-encoded data: fixture URLs without calling fetch', async () => { + const dataUrl = `data:application/rss+xml;charset=utf-8,${encodeURIComponent(FEED_BODY)}`; + const fetchMock = jest.fn(); + const reports = await rssAdapter.run( + buildSource({ config: { url: dataUrl } }), + buildContext(fetchMock) + ); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(reports).toHaveLength(2); + expect(reports[0].content.title).toBe('Item one'); + expect(reports[0].lineage.extraction_method).toBe('pending'); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/rss_adapter.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/rss_adapter.ts new file mode 100644 index 0000000000000..ee44acdbe2b92 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/rss_adapter.ts @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { GLOBAL_SPACE_ID } from '../../../../common/threat_intel'; +import { fetchUrlForContext, redactUrl } from '../http_client'; +import { buildFingerprint } from '../fingerprint'; +import { DEFAULT_SEVERITY_LEVEL, DEFAULT_SEVERITY_SCORE } from '../../content/severity'; +import { buildReportContent, collapseWhitespace, stripHtml, truncate } from '../../content/text'; +import type { AdapterRunContext, FetchAdapter, NormalizedReport, SourceHit } from '../types'; +import { decodeDataUrl, isDataUrl } from './decode_data_url'; +import { parseRssFeed } from './parse_rss'; + +const TITLE_MAX_LENGTH = 280; +const BODY_TEXT_MAX_LENGTH = 32_000; +const SOURCE_DOC_REF_INDEX = 'rss:feed'; + +const readFeedUrl = (source: SourceHit): string | undefined => { + const url = source._source.config.url; + return typeof url === 'string' && url.length > 0 ? url : undefined; +}; + +/** + * Resolve the RSS/Atom body for a source URL. + * + * `data:` URLs are decoded in-process (used by the security_solution data + * generator fixtures). Network URLs go through `fetchUrl`, which enforces + * the http/https SSRF guard. + */ +const readFeedBody = async (feedUrl: string, context: AdapterRunContext): Promise => { + if (isDataUrl(feedUrl)) { + return decodeDataUrl(feedUrl); + } + + const response = await fetchUrlForContext(context)(feedUrl, { + abortSignal: context.abortSignal, + headers: { Accept: 'application/rss+xml, application/atom+xml, application/xml, text/xml' }, + }); + + if (response.status >= 400) { + // Surface as a thrown error so the step's `on-failure: continue: true` + // still records the failure on the step result. Returning `[]` + // would silently mask broken feeds. + throw new Error( + `RSS fetch ${redactUrl(feedUrl)} failed: HTTP ${response.status} ${response.statusText}` + ); + } + + return response.body; +}; + +export const rssAdapter: FetchAdapter = { + adapterType: 'rss', + async run(source, context: AdapterRunContext) { + const log = context.logger.get('rss-adapter'); + const feedUrl = readFeedUrl(source); + if (!feedUrl) { + log.warn(`Source ${source._id} has no config.url — skipping`); + return []; + } + + const feedBody = await readFeedBody(feedUrl, context); + const parsed = await parseRssFeed(feedBody); + if (parsed.entries.length === 0) { + log.debug(`RSS feed ${feedUrl} returned 0 items for source ${source._id}`); + return []; + } + + const ingestedAt = context.now().toISOString(); + const spaceId = source._source.space_id ?? GLOBAL_SPACE_ID; + const language = parsed.language ?? 'en'; + const adapterId = `rss:${source._id}`; + + const reports: NormalizedReport[] = []; + for (const entry of parsed.entries) { + const title = collapseWhitespace(entry.title || parsed.feedTitle || source._source.name); + // Keep the untruncated text for the fingerprint so a revision that only + // differs past the stored-body cap is still detected as a change. + const fullBodyText = stripHtml(entry.bodyHtml ?? ''); + const bodyText = truncate(fullBodyText, BODY_TEXT_MAX_LENGTH); + // Per-item fingerprint seed: feed URL + stable item id + canonical title, + // plus the publish timestamp and a hash of the body. Advisories commonly + // keep their `` and title while revising the text and IOCs, so + // identity alone would dedup the revision away forever. Including the + // body means a re-fetch of the unchanged item still collapses to one + // fingerprint, while a revised item produces a fresh row for + // `enrich_threat_report` to re-extract over. + const fingerprint = buildFingerprint([ + feedUrl, + entry.id, + title, + entry.publishedAt, + fullBodyText, + ]); + reports.push({ + '@timestamp': ingestedAt, + content_fingerprint: fingerprint, + space_id: spaceId, + source: { + type: 'rss', + name: source._source.name, + url: entry.link ?? feedUrl, + adapter_id: adapterId, + }, + content: buildReportContent({ + title: truncate(title, TITLE_MAX_LENGTH), + bodyText, + bodyHtml: entry.bodyHtml, + language, + }), + severity: { + level: DEFAULT_SEVERITY_LEVEL, + score: DEFAULT_SEVERITY_SCORE, + }, + lineage: { + ingested_at: ingestedAt, + extraction_method: 'pending', + source_doc_ref: { + index: SOURCE_DOC_REF_INDEX, + id: entry.id, + }, + }, + }); + } + + return reports; + }, +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/parse_pattern.test.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/parse_pattern.test.ts new file mode 100644 index 0000000000000..a1b0d207106bf --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/parse_pattern.test.ts @@ -0,0 +1,383 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { parseStixPattern } from './parse_pattern'; +import type { ExtractedIoc } from '../../services/extract_iocs'; + +// Minimal structural check — all required ExtractedIoc fields present. +const hasRequiredFields = (ioc: ExtractedIoc): boolean => + typeof ioc.type === 'string' && + typeof ioc.value === 'string' && + typeof ioc.tier === 'string' && + typeof ioc.tier_heuristic === 'string' && + typeof ioc.tier_basis === 'string'; + +// ── Single-comparison patterns ─────────────────────────────────────────────── + +describe('parseStixPattern — single-comparison patterns', () => { + it('parses ipv4-addr:value', () => { + const result = parseStixPattern("[ipv4-addr:value = '1.2.3.4']"); + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ type: 'ip', value: '1.2.3.4' }); + }); + + it('parses ipv6-addr:value', () => { + const result = parseStixPattern("[ipv6-addr:value = '2001:db8::1']"); + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ type: 'ip', value: '2001:db8::1' }); + }); + + it('parses domain-name:value', () => { + const result = parseStixPattern("[domain-name:value = 'evil.example.com']"); + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ type: 'domain', value: 'evil.example.com' }); + }); + + it('parses url:value', () => { + const result = parseStixPattern("[url:value = 'https://evil.example.com/payload']"); + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ type: 'url', value: 'https://evil.example.com/payload' }); + }); + + it('parses email-addr:value', () => { + const result = parseStixPattern("[email-addr:value = 'attacker@evil.example.com']"); + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ type: 'email', value: 'attacker@evil.example.com' }); + }); +}); + +// ── Hash variants ──────────────────────────────────────────────────────────── + +describe('parseStixPattern — hash variants', () => { + it("parses file:hashes.'MD5' (quoted)", () => { + const result = parseStixPattern("[file:hashes.'MD5' = 'aabbccddeeff00112233445566778899']"); + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ type: 'hash', value: 'aabbccddeeff00112233445566778899' }); + }); + + it('parses file:hashes.MD5 (unquoted)', () => { + const result = parseStixPattern("[file:hashes.MD5 = 'AABBCCDDEEFF00112233445566778899']"); + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ type: 'hash', value: 'aabbccddeeff00112233445566778899' }); + }); + + it("parses file:hashes.'SHA-1' (quoted, hyphenated)", () => { + const result = parseStixPattern( + "[file:hashes.'SHA-1' = 'da39a3ee5e6b4b0d3255bfef95601890afd80709']" + ); + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ + type: 'hash', + value: 'da39a3ee5e6b4b0d3255bfef95601890afd80709', + }); + }); + + it("parses file:hashes.'SHA-256' (quoted, hyphenated)", () => { + const result = parseStixPattern( + "[file:hashes.'SHA-256' = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855']" + ); + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ + type: 'hash', + value: 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + }); + }); + + it('parses file:hashes.SHA256 (no hyphen, no quotes) and lowercases', () => { + const result = parseStixPattern( + "[file:hashes.SHA256 = 'E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855']" + ); + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ + type: 'hash', + value: 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + }); + }); + + it('parses file:hashes.SHA1 (no hyphen)', () => { + const result = parseStixPattern( + "[file:hashes.SHA1 = 'da39a3ee5e6b4b0d3255bfef95601890afd80709']" + ); + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ type: 'hash' }); + }); +}); + +// ── Tiering ────────────────────────────────────────────────────────────────── + +describe('parseStixPattern — tiering', () => { + it('assigns discriminating tier to hashes', () => { + const [ioc] = parseStixPattern("[file:hashes.MD5 = 'aabbccddeeff00112233445566778899']"); + expect(ioc.tier).toBe('discriminating'); + expect(ioc.tier_heuristic).toBe('discriminating'); + expect(ioc.tier_basis).toBe('stix_pattern'); + }); + + it('assigns contextual tier to ip/domain/url/email', () => { + const patterns = [ + "[ipv4-addr:value = '1.2.3.4']", + "[domain-name:value = 'evil.example.com']", + "[url:value = 'https://evil.example.com/']", + "[email-addr:value = 'x@evil.example.com']", + ]; + for (const p of patterns) { + const [ioc] = parseStixPattern(p); + expect(ioc.tier).toBe('contextual'); + expect(ioc.tier_heuristic).toBe('contextual'); + expect(ioc.tier_basis).toBe('stix_pattern'); + } + }); + + it('sets tier === tier_heuristic (no reassignment)', () => { + const [ioc] = parseStixPattern("[ipv4-addr:value = '8.8.8.8']"); + expect(ioc.tier).toBe(ioc.tier_heuristic); + }); +}); + +// ── Multi-comparison patterns ───────────────────────────────────────────────── + +describe('parseStixPattern — multi-comparison patterns', () => { + it('extracts all IOCs from an OR pattern', () => { + const result = parseStixPattern( + "[ipv4-addr:value = '1.2.3.4'] OR [domain-name:value = 'evil.example.com']" + ); + expect(result).toHaveLength(2); + expect(result.map((r) => r.type).sort()).toEqual(['domain', 'ip']); + }); + + it('extracts all IOCs from a bracketed AND pattern', () => { + const result = parseStixPattern( + "[ipv4-addr:value = '1.2.3.4' AND domain-name:value = 'evil.example.com']" + ); + expect(result).toHaveLength(2); + }); + + it('extracts all supported IOCs from a mixed-type pattern', () => { + const result = parseStixPattern( + "[ipv4-addr:value = '1.2.3.4'] OR [file:hashes.MD5 = 'aabbccddeeff00112233445566778899'] OR [domain-name:value = 'c2.example.com']" + ); + expect(result).toHaveLength(3); + expect(result.map((r) => r.type).sort()).toEqual(['domain', 'hash', 'ip']); + }); + + it('deduplicates identical (type, value) pairs within one pattern', () => { + const result = parseStixPattern( + "[ipv4-addr:value = '1.2.3.4'] OR [ipv4-addr:value = '1.2.3.4']" + ); + expect(result).toHaveLength(1); + }); + + it('dedup is case-insensitive for hash values', () => { + const result = parseStixPattern( + "[file:hashes.MD5 = 'AABB1122CCDD3344EEFF5566778899AA'] OR [file:hashes.MD5 = 'aabb1122ccdd3344eeff5566778899aa']" + ); + // Both canonicalize to the same lowercase value → one record + expect(result).toHaveLength(1); + }); +}); + +// ── Rejection cases ─────────────────────────────────────────────────────────── + +describe('parseStixPattern — rejection cases', () => { + it('returns [] for yara pattern_type', () => { + expect(parseStixPattern("[file:hashes.MD5 = 'aabb1122']", 'yara')).toEqual([]); + }); + + it('returns [] for snort pattern_type', () => { + expect(parseStixPattern('alert tcp ...', 'snort')).toEqual([]); + }); + + it('returns [] for sigma pattern_type', () => { + expect(parseStixPattern('title: Suspicious ...', 'sigma')).toEqual([]); + }); + + it('returns [] for pcre pattern_type', () => { + expect(parseStixPattern('/evil.*/i', 'pcre')).toEqual([]); + }); + + it('skips unsupported object paths (file:name)', () => { + const result = parseStixPattern("[file:name = 'payload.exe']"); + expect(result).toEqual([]); + }); + + it('skips unsupported object paths (process:pid)', () => { + const result = parseStixPattern("[process:pid = '1234']"); + expect(result).toEqual([]); + }); + + it('skips unsupported object paths (windows-registry-key:key)', () => { + const result = parseStixPattern("[windows-registry-key:key = 'HKLM\\\\Software\\\\Evil']"); + expect(result).toEqual([]); + }); + + it('skips LIKE comparisons (not a literal IOC)', () => { + // LIKE uses a different syntax — no `= '...'` form, so it's simply not matched + const result = parseStixPattern("[domain-name:value LIKE '%evil%']"); + expect(result).toEqual([]); + }); + + it('skips MATCHES comparisons', () => { + const result = parseStixPattern("[domain-name:value MATCHES '^evil.*\\.com$']"); + expect(result).toEqual([]); + }); + + it('skips != comparisons', () => { + const result = parseStixPattern("[ipv4-addr:value != '1.2.3.4']"); + expect(result).toEqual([]); + }); + + it('returns [] for empty string', () => { + expect(parseStixPattern('')).toEqual([]); + }); + + it('returns [] for malformed/unclosed bracket', () => { + expect(parseStixPattern('[ipv4-addr:value = ')).toEqual([]); + }); + + it('tolerates null cast through (returns [])', () => { + // Cast through — tolerant API + expect(parseStixPattern(null as unknown as string)).toEqual([]); + }); + + it('tolerates undefined cast through (returns [])', () => { + expect(parseStixPattern(undefined as unknown as string)).toEqual([]); + }); +}); + +// ── Canonicalization ────────────────────────────────────────────────────────── + +describe('parseStixPattern — value canonicalization', () => { + it('lowercases ip values', () => { + // IPv6 hex digits + const [ioc] = parseStixPattern("[ipv6-addr:value = '2001:DB8::1']"); + expect(ioc.value).toBe('2001:db8::1'); + }); + + it('lowercases domain values', () => { + const [ioc] = parseStixPattern("[domain-name:value = 'Evil.EXAMPLE.COM']"); + expect(ioc.value).toBe('evil.example.com'); + }); + + it('lowercases email values', () => { + const [ioc] = parseStixPattern("[email-addr:value = 'ATTACKER@EVIL.EXAMPLE.COM']"); + expect(ioc.value).toBe('attacker@evil.example.com'); + }); + + it('lowercases hash values', () => { + const [ioc] = parseStixPattern("[file:hashes.MD5 = 'AABB1122CCDD3344EEFF5566778899AA']"); + expect(ioc.value).toBe('aabb1122ccdd3344eeff5566778899aa'); + }); + + it('preserves URL case (path is case-sensitive)', () => { + const url = 'https://evil.example.com/PAYLOAD/Stage2.exe'; + const [ioc] = parseStixPattern(`[url:value = '${url}']`); + expect(ioc.value).toBe(url); + }); + + it('unescapes STIX backslash escape (\\\\)', () => { + const [ioc] = parseStixPattern("[url:value = 'https://evil.test/a\\\\b']"); + // \\\\ in STIX source → \\ in JS string → unescape → single \ + expect(ioc.value).toBe('https://evil.test/a\\b'); + }); + + it("unescapes STIX single-quote escape (\\')", () => { + const [ioc] = parseStixPattern("[url:value = 'https://evil.test/it\\'s']"); + expect(ioc.value).toBe("https://evil.test/it's"); + }); +}); + +// ── Structural integrity ────────────────────────────────────────────────────── + +describe('parseStixPattern — structural integrity', () => { + it('all records satisfy the ExtractedIoc shape', () => { + const result = parseStixPattern( + "[ipv4-addr:value = '1.2.3.4'] OR [file:hashes.MD5 = 'aabbccddeeff00112233445566778899']" + ); + for (const ioc of result) { + expect(hasRequiredFields(ioc)).toBe(true); + } + }); + + it('does not set defanged field (STIX values are fanged/live)', () => { + const result = parseStixPattern("[ipv4-addr:value = '1.2.3.4']"); + expect(result[0]).not.toHaveProperty('defanged'); + }); + + it('does not set port field (STIX indicator patterns rarely carry socket form)', () => { + const result = parseStixPattern("[ipv4-addr:value = '1.2.3.4']"); + expect(result[0]).not.toHaveProperty('port'); + }); +}); + +// ── Value validation and address tiering ───────────────────────────────────── + +describe('parseStixPattern — value validation', () => { + // The object path is the only thing establishing the type, so a feed can put + // anything on the right-hand side and it used to be taken at face value. + it('rejects a hash whose length does not match the declared algorithm', () => { + // 8 chars declared as MD5. This used to be emitted as a discriminating hash + // and then filed under sha256 by the promote task's length fallback. + expect(parseStixPattern("[file:hashes.MD5 = 'AABB1122']")).toEqual([]); + }); + + it('rejects a hash containing non-hex characters', () => { + expect(parseStixPattern("[file:hashes.MD5 = 'zzbb1122ccdd3344eeff5566778899aa']")).toEqual([]); + }); + + it('accepts a correctly sized sha-256', () => { + const [ioc] = parseStixPattern( + "[file:hashes.'SHA-256' = '44a2ab4206fc5d5d33974adbc3fd2a80966e7a88167914794f524fa29a3d8e8e']" + ); + expect(ioc.type).toBe('hash'); + expect(ioc.tier).toBe('discriminating'); + }); + + // An ipv4-addr value that is not an address reaches an ES `ip` field, which is a + // permanent item-level rejection rather than merely a bad row. + it('rejects an ipv4-addr value that is not an address', () => { + expect(parseStixPattern("[ipv4-addr:value = 'not-an-ip']")).toEqual([]); + }); + + it.each([ + ['no dot at all', 'notadomain'], + ['empty label', 'evil..example.com'], + ['numeric TLD', 'evil.example.123'], + ['trailing hyphen label', 'evil-.example.com'], + ])('rejects a domain that cannot resolve (%s)', (_label, value) => { + expect(parseStixPattern(`[domain-name:value = '${value}']`)).toEqual([]); + }); + + it('rejects a url with a non-http scheme', () => { + expect(parseStixPattern("[url:value = 'file:///etc/passwd']")).toEqual([]); + }); + + it('rejects an email that is not an address', () => { + expect(parseStixPattern("[email-addr:value = 'nope']")).toEqual([]); + }); +}); + +describe('parseStixPattern — private and reserved addresses', () => { + // `contextual` is promotable, so these used to become live Indicator Match rows + // matching essentially all internal traffic. + it.each([ + ['RFC1918', '10.0.0.1'], + ['loopback', '127.0.0.1'], + ['link-local', '169.254.169.254'], + ['IPv6 loopback', '::1'], + ['IPv6 unique-local', 'fc00::1'], + ])('tiers %s as reference rather than contextual', (_label, value) => { + const [ioc] = parseStixPattern(`[ipv4-addr:value = '${value}']`); + expect(ioc.tier).toBe('reference'); + expect(ioc.tier_basis).toBe('private_ip'); + }); + + it('leaves a public address contextual', () => { + const [ioc] = parseStixPattern("[ipv4-addr:value = '185.220.101.45']"); + expect(ioc.tier).toBe('contextual'); + expect(ioc.tier_basis).toBe('stix_pattern'); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/parse_pattern.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/parse_pattern.ts new file mode 100644 index 0000000000000..9bced3c3609b0 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/parse_pattern.ts @@ -0,0 +1,213 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import net from 'node:net'; +import type { IocType } from '../../../../common/threat_intel'; +import type { ExtractedIoc, IocTier } from '../../services/extract_iocs'; +import { isNonRoutableIPv4, isNonRoutableIPv6 } from '../../lib/ip_ranges'; + +/** + * Pattern dialects that are not observable patterns — skip them entirely. + * STIX (or unspecified → assumed stix) is the only parseable dialect here. + */ +const NON_STIX_DIALECTS = new Set([ + 'yara', + 'snort', + 'sigma', + 'pcre', + 'tanium-signal', + 'spl', + 'kql', + 'eql', +]); + +/** + * Matches a single STIX `=` comparison: : = '' + * + * Strategy: scan the raw pattern string for `=`-comparison forms without + * parsing brackets or logical operators. We only care about comparisons whose + * LHS maps to a supported IocType — AND/OR/FOLLOWEDBY and grouping are + * irrelevant. A new instance is created per call so lastIndex never leaks. + * + * Does NOT match `!=` (requires object:property before `=`; `!` breaks the + * property-path capture). LIKE / MATCHES / IN use different syntax and contain + * no `= ''` form, so they are skipped automatically. + * + * Not supported: nested precedence, network-traffic SCO references, IN lists. + */ +const makeComparisonRe = () => /([\w-]+):([\w.'"-]+)\s*=\s*'((?:[^'\\]|\\[\s\S])*)'/g; + +/** + * Maps a STIX object-type + property-path pair to an IocType. + * Returns null for anything not in the closed mapping set. + * + * Quoting and casing in the property path are normalised before comparison + * (e.g. `hashes.'SHA-256'` → `hashes.sha-256`, `hashes.SHA256` → `hashes.sha256`). + */ +const resolveIocType = (objectType: string, propertyPath: string): IocType | null => { + const obj = objectType.toLowerCase(); + // Strip all quote chars and lowercase for uniform comparison + const prop = propertyPath.replace(/['"]/g, '').toLowerCase(); + + if ((obj === 'ipv4-addr' || obj === 'ipv6-addr') && prop === 'value') return 'ip'; + if (obj === 'domain-name' && prop === 'value') return 'domain'; + if (obj === 'url' && prop === 'value') return 'url'; + if (obj === 'email-addr' && prop === 'value') return 'email'; + if (obj === 'file') { + if (prop === 'hashes.sha-256' || prop === 'hashes.sha256') return 'hash'; + if (prop === 'hashes.sha-1' || prop === 'hashes.sha1') return 'hash'; + if (prop === 'hashes.md5') return 'hash'; + } + return null; +}; + +/** + * Hash length the declared property implies, so a value can be checked against the + * algorithm the feed actually named rather than against "any hash". + */ +const resolveExpectedHashLength = (propertyPath: string): number | null => { + const prop = propertyPath.replace(/['"]/g, '').toLowerCase(); + if (prop === 'hashes.md5') return 32; + if (prop === 'hashes.sha-1' || prop === 'hashes.sha1') return 40; + if (prop === 'hashes.sha-256' || prop === 'hashes.sha256') return 64; + return null; +}; + +const DOMAIN_SYNTAX = /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,24}$/i; +const EMAIL_SYNTAX = /^[^\s@]+@(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,24}$/i; + +/** + * Checks a value against the type its object path claims. + * + * The object path is the *only* thing establishing the type here, so a feed can + * put anything on the right-hand side and it was taken at face value. Two + * consequences: `file:hashes.MD5 = 'AABB1122'` was accepted as a discriminating + * hash and then filed under `sha256` by the promote task's length fallback, and an + * `ipv4-addr` value that is not an address reached `threat.indicator.ip`, which is + * an ES `ip` field, so it was a permanent item-level rejection rather than merely a + * bad row. + */ +const isValidForType = ( + type: IocType, + value: string, + expectedHashLength: number | null +): boolean => { + if (value.length === 0) return false; + if (type === 'ip') return net.isIP(value) !== 0; + if (type === 'hash') { + if (!/^[a-f0-9]+$/i.test(value)) return false; + return expectedHashLength === null + ? [32, 40, 64, 128].includes(value.length) + : value.length === expectedHashLength; + } + if (type === 'domain') return DOMAIN_SYNTAX.test(value); + if (type === 'email') return EMAIL_SYNTAX.test(value); + if (type === 'url') { + try { + const { protocol } = new URL(value); + return protocol === 'http:' || protocol === 'https:'; + } catch { + return false; + } + } + return true; +}; + +/** + * True for an address in private, loopback, link-local, or otherwise reserved + * space. Those are not indicators: promoting `10.0.0.1` or `::1` puts a row in the + * live Indicator Match index that matches essentially all internal traffic. + * `extract_iocs` already tiers them `reference`; this is the structured path + * catching up. + */ +const isNonRoutableAddress = (value: string): boolean => { + const family = net.isIP(value); + if (family === 4) return isNonRoutableIPv4(value); + if (family === 6) return isNonRoutableIPv6(value); + return false; +}; + +/** Unescape STIX string escapes: \\ → \ and \' → ' (single pass, left-to-right). */ +const unescapeStixString = (s: string): string => s.replace(/\\([\s\S])/g, (_, ch: string) => ch); + +/** + * Canonicalize extracted value. + * + * URLs are case-sensitive in their path/query, so preserved as-is. + * All other types are lowercased to match extract_iocs canonicalization: + * IPs → lowercase (IPv6 hex digits), domains → lowercase, + * email → lowercase, hash → lowercase hex. + */ +const canonicalize = (type: IocType, value: string): string => + type === 'url' ? value : value.toLowerCase(); + +/** + * Parse a STIX 2.x indicator pattern string into structured ExtractedIoc records. + * + * @param pattern - The raw STIX pattern string, e.g. `[ipv4-addr:value = '1.2.3.4']` + * @param patternType - The pattern dialect (default: 'stix'). Non-stix dialects return []. + * @returns - One record per unique (type, value) pair found in the pattern. + * + * Never throws — returns [] for empty, malformed, or non-stix input. + * Does not emit `defanged` or `port` fields (STIX values are fanged/live; + * socket-form addresses are rare in indicator patterns). + * tier_basis is 'stix_pattern' for structured lineage, or 'private_ip' for an + * address in reserved space. Hashes are 'discriminating', private and reserved + * addresses are 'reference', everything else is 'contextual'. + * + * Values are validated against the type their object path claims, and anything + * that does not match is skipped rather than emitted. Domain tiering (CDN base vs + * purpose-registered) is still left to a later pass. + */ +export const parseStixPattern = (pattern: string, patternType?: string): ExtractedIoc[] => { + if (!pattern) return []; + if (patternType !== undefined && NON_STIX_DIALECTS.has(patternType.toLowerCase())) return []; + + const results: ExtractedIoc[] = []; + const seen = new Set(); + + // `forEach` rather than `for..of` so each comparison can bail with `return` + // instead of `continue` (see `no-continue`). + [...pattern.matchAll(makeComparisonRe())].forEach((match) => { + const [, objectType, propertyPath, rawValue] = match; + // A null type means a comparison against an object path we do not map. + const iocType = resolveIocType(objectType, propertyPath); + if (iocType === null) { + return; + } + + const value = canonicalize(iocType, unescapeStixString(rawValue)); + if (!isValidForType(iocType, value, resolveExpectedHashLength(propertyPath))) { + return; + } + + const dedupKey = `${iocType}:${value}`; + if (seen.has(dedupKey)) { + return; + } + seen.add(dedupKey); + + // Private and reserved addresses are `reference`, which the promote task + // refuses, so they never become live Indicator Match rows. + const nonRoutable = iocType === 'ip' && isNonRoutableAddress(value); + const tier: IocTier = nonRoutable + ? 'reference' + : iocType === 'hash' + ? 'discriminating' + : 'contextual'; + + results.push({ + type: iocType, + value, + tier, + tier_heuristic: tier, + tier_basis: nonRoutable ? 'private_ip' : 'stix_pattern', + }); + }); + + return results; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/split_bundle.test.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/split_bundle.test.ts new file mode 100644 index 0000000000000..b7c5d01b07d49 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/split_bundle.test.ts @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + composeStixBody, + composeStixTitle, + splitStixBundle, + STIX_REPORTABLE_TYPES, +} from './split_bundle'; + +const bundle = (objects: unknown[]) => ({ type: 'bundle', id: 'bundle--1', objects }); + +describe('splitStixBundle', () => { + it('keeps only reportable SDO types', () => { + const result = splitStixBundle( + bundle([ + { type: 'indicator', id: 'indicator--1', name: 'indicator-1' }, + { type: 'malware', id: 'malware--1', name: 'malware-1' }, + // structural — must be dropped + { type: 'marking-definition', id: 'marking-definition--1' }, + { type: 'identity', id: 'identity--1' }, + { type: 'relationship', id: 'relationship--1' }, + // unknown — must be dropped (closed-set policy) + { type: 'pet-rock', id: 'pet-rock--1' }, + ]) + ); + expect(result.map((r) => r.object.type)).toEqual(['indicator', 'malware']); + }); + + it('tolerates a bare objects array (no bundle envelope)', () => { + const objects = [{ type: 'indicator', id: 'indicator--1', name: 'i1' }]; + expect(splitStixBundle(objects)).toHaveLength(1); + }); + + it('tolerates a TAXII envelope with `{ objects: [] }` and no `type`', () => { + expect( + splitStixBundle({ objects: [{ type: 'malware', id: 'malware--1', name: 'm1' }] }) + ).toHaveLength(1); + }); + + it('skips objects missing `id` or `type`', () => { + expect( + splitStixBundle( + bundle([ + { id: 'no-type--1', name: 'no type' }, + { type: 'indicator', name: 'no id' }, + { type: 'indicator', id: 'indicator--1', name: 'good' }, + ]) + ) + ).toHaveLength(1); + }); + + it('returns [] for non-object inputs', () => { + expect(splitStixBundle(null)).toEqual([]); + expect(splitStixBundle('not a bundle')).toEqual([]); + expect(splitStixBundle(undefined)).toEqual([]); + }); + + it('exposes a closed reportable type set', () => { + expect(STIX_REPORTABLE_TYPES).toContain('indicator'); + expect(STIX_REPORTABLE_TYPES).toContain('threat-actor'); + expect(STIX_REPORTABLE_TYPES).not.toContain('marking-definition'); + }); +}); + +describe('composeStixTitle', () => { + it('uses `name` when present', () => { + expect(composeStixTitle({ type: 'malware', id: 'malware--1', name: 'PetRock' })).toBe( + 'PetRock' + ); + }); + it('falls back to ` `', () => { + expect(composeStixTitle({ type: 'indicator', id: 'indicator--1' })).toBe( + 'indicator indicator--1' + ); + }); +}); + +describe('composeStixBody', () => { + it('joins description, abstract, summary, and indicator pattern', () => { + const body = composeStixBody({ + type: 'indicator', + id: 'indicator--1', + description: 'Detects bad thing.', + abstract: 'A summary.', + pattern: "[file:hashes.MD5 = 'abc']", + pattern_type: 'stix', + labels: ['malicious-activity'], + }); + expect(body).toContain('Detects bad thing.'); + expect(body).toContain('A summary.'); + expect(body).toContain("Pattern (stix): [file:hashes.MD5 = 'abc']"); + expect(body).toContain('Labels: malicious-activity'); + }); + + it('returns a deterministic placeholder when nothing useful is set', () => { + expect(composeStixBody({ type: 'malware', id: 'malware--1' })).toBe('STIX malware malware--1'); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/split_bundle.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/split_bundle.ts new file mode 100644 index 0000000000000..c91b4d33bf6e5 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/split_bundle.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** STIX SDO types that become threat reports (unknown types are skipped). */ +export const STIX_REPORTABLE_TYPES = [ + 'indicator', + 'malware', + 'malware-analysis', + 'threat-actor', + 'intrusion-set', + 'campaign', + 'attack-pattern', + 'course-of-action', + 'tool', + 'vulnerability', + 'report', + 'note', + 'opinion', +] as const; + +export type StixReportableType = (typeof STIX_REPORTABLE_TYPES)[number]; + +const REPORTABLE_SET: ReadonlySet = new Set(STIX_REPORTABLE_TYPES); + +export interface StixObject { + type: string; + id: string; + name?: string; + description?: string; + pattern?: string; + pattern_type?: string; + labels?: string[]; + created?: string; + modified?: string; + /** STIX 2.x `report` SDO. Carries the human-readable narrative. */ + abstract?: string; + /** Some vendor STIX dialects ship `summary`. */ + summary?: string; + // Allow arbitrary extra fields without forcing every consumer to cast. + [key: string]: unknown; +} + +export interface ExtractedStixObject { + object: StixObject; +} + +const isReportableSdo = (obj: unknown): obj is StixObject => { + if (!obj || typeof obj !== 'object') return false; + const candidate = obj as Partial; + if (typeof candidate.type !== 'string' || typeof candidate.id !== 'string') return false; + return REPORTABLE_SET.has(candidate.type); +}; + +export const splitStixBundle = (raw: unknown): ExtractedStixObject[] => + extractObjects(raw) + .filter(isReportableSdo) + .map((object) => ({ object })); + +const extractObjects = (raw: unknown): unknown[] => { + if (!raw) return []; + if (Array.isArray(raw)) return raw; + if (typeof raw !== 'object') return []; + const bundle = raw as Record; + if (Array.isArray(bundle.objects)) return bundle.objects; + // Some TAXII servers return a bare object array without an `objects` envelope. + if (bundle.type === 'bundle' && Array.isArray(bundle.objects)) return bundle.objects; + return []; +}; + +export const composeStixBody = (object: StixObject): string => { + const parts: string[] = []; + if (typeof object.description === 'string' && object.description.length > 0) { + parts.push(object.description); + } + if (typeof object.abstract === 'string' && object.abstract.length > 0) { + parts.push(object.abstract); + } + if (typeof object.summary === 'string' && object.summary.length > 0) { + parts.push(object.summary); + } + if (object.type === 'indicator' && typeof object.pattern === 'string') { + const patternType = typeof object.pattern_type === 'string' ? object.pattern_type : 'stix'; + parts.push(`Pattern (${patternType}): ${object.pattern}`); + } + if (Array.isArray(object.labels) && object.labels.length > 0) { + parts.push(`Labels: ${object.labels.join(', ')}`); + } + if (parts.length === 0) { + return `STIX ${object.type} ${object.id}`; + } + return parts.join('\n\n'); +}; + +/** Display title — `name` when present, otherwise ` `. */ +export const composeStixTitle = (object: StixObject): string => { + if (typeof object.name === 'string' && object.name.trim().length > 0) { + return object.name; + } + return `${object.type} ${object.id}`; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/stix_adapter.test.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/stix_adapter.test.ts new file mode 100644 index 0000000000000..2e0719f5add1b --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/stix_adapter.test.ts @@ -0,0 +1,329 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { stixAdapter } from './stix_adapter'; +import { normalizedReportSchema } from '../../../../common/threat_intel/workflows/step_types/fetch_source/fetch_source_common'; +import type { AdapterRunContext, SourceHit } from '../types'; + +const URL = 'https://stix.example/bundle.json'; +const NOW = new Date('2026-05-16T12:00:00.000Z'); + +const buildSource = (): SourceHit => ({ + _id: 'stix:vendor', + _source: { + adapter_type: 'stix', + name: 'Vendor STIX', + config: { url: URL }, + }, +}); + +const buildContext = ( + fetchImpl: jest.Mock, [string | URL | Request, RequestInit?]> +): AdapterRunContext => ({ + esClient: elasticsearchServiceMock.createElasticsearchClient(), + logger: loggingSystemMock.createLogger(), + abortSignal: new AbortController().signal, + now: () => NOW, + fetchFn: fetchImpl as unknown as typeof fetch, + lookupFn: async () => [{ address: '93.184.216.34' }], +}); + +const okJson = (value: unknown): Response => + new Response(JSON.stringify(value), { + status: 200, + statusText: 'OK', + headers: { 'Content-Type': 'application/stix+json' }, + }); + +describe('stixAdapter', () => { + it('emits one normalized report per reportable SDO', async () => { + const fetchMock = jest.fn().mockResolvedValue( + okJson({ + type: 'bundle', + id: 'bundle--1', + objects: [ + { + type: 'indicator', + id: 'indicator--1', + name: 'IOC: bad domain', + description: 'Detects connections to bad.example.', + modified: '2026-05-15T00:00:00Z', + pattern: "[domain-name:value = 'bad.example']", + pattern_type: 'stix', + }, + { type: 'marking-definition', id: 'marking-definition--1' }, + { + type: 'threat-actor', + id: 'threat-actor--1', + name: 'APT-Test', + description: 'A test actor.', + created: '2026-05-10T00:00:00Z', + }, + ], + }) + ); + const reports = await stixAdapter.run(buildSource(), buildContext(fetchMock)); + expect(reports).toHaveLength(2); + expect(reports[0]).toMatchObject({ + source: { type: 'stix', adapter_id: 'stix:stix:vendor', url: URL }, + content: { title: 'IOC: bad domain' }, + lineage: { + // parseable stix pattern → structured at ingest; enrichment skips re-extraction + extraction_method: 'stix', + extracted_at: NOW.toISOString(), + source_doc_ref: { index: 'stix:bundle', id: 'indicator--1' }, + }, + }); + expect(reports[0].extracted?.iocs).toHaveLength(1); + expect(reports[0].extracted?.iocs![0]).toMatchObject({ type: 'domain', value: 'bad.example' }); + expect(reports[0].content.body_text).toContain( + "Pattern (stix): [domain-name:value = 'bad.example']" + ); + expect(reports[1].source.adapter_id).toBe('stix:stix:vendor'); + expect(reports[1].content.title).toBe('APT-Test'); + }); + + it('throws on non-2xx', async () => { + const fetchMock = jest + .fn() + .mockResolvedValue(new Response('nope', { status: 401, statusText: 'Unauthorized' })); + await expect(stixAdapter.run(buildSource(), buildContext(fetchMock))).rejects.toThrow( + /HTTP 401/ + ); + }); + + it('throws on a non-JSON response', async () => { + const fetchMock = jest + .fn() + .mockResolvedValue( + new Response('', { status: 200, headers: { 'Content-Type': 'text/plain' } }) + ); + await expect(stixAdapter.run(buildSource(), buildContext(fetchMock))).rejects.toThrow( + /not valid JSON/ + ); + }); + + it('returns [] when the bundle has no reportable SDOs', async () => { + const fetchMock = jest.fn().mockResolvedValue( + okJson({ + type: 'bundle', + id: 'bundle--empty', + objects: [{ type: 'marking-definition', id: 'marking-definition--1' }], + }) + ); + const reports = await stixAdapter.run(buildSource(), buildContext(fetchMock)); + expect(reports).toEqual([]); + }); + + it('indicator SDO with parseable pattern → extraction_method:stix, extracted_at set, extracted.iocs populated, body_text still present', async () => { + const fetchMock = jest.fn().mockResolvedValue( + okJson({ + type: 'bundle', + id: 'bundle--2', + objects: [ + { + type: 'indicator', + id: 'indicator--2', + name: 'Malicious IP', + description: 'Known C2 server.', + modified: '2026-05-15T00:00:00Z', + pattern: "[ipv4-addr:value = '1.2.3.4']", + pattern_type: 'stix', + }, + ], + }) + ); + const reports = await stixAdapter.run(buildSource(), buildContext(fetchMock)); + expect(reports).toHaveLength(1); + const [report] = reports; + expect(report.lineage.extraction_method).toBe('stix'); + expect(report.lineage.extracted_at).toBe(NOW.toISOString()); + expect(report.extracted?.iocs).toHaveLength(1); + expect(report.extracted?.iocs![0]).toMatchObject({ + type: 'ip', + value: '1.2.3.4', + tier: 'contextual', + }); + expect(report.content.body_text).toContain('Known C2 server'); + }); + + it('indicator SDO with unparseable pattern (yara dialect) → falls back to extraction_method:pending, no extracted', async () => { + const fetchMock = jest.fn().mockResolvedValue( + okJson({ + type: 'bundle', + id: 'bundle--3', + objects: [ + { + type: 'indicator', + id: 'indicator--3', + name: 'YARA rule', + description: 'Detects malware via YARA.', + modified: '2026-05-15T00:00:00Z', + pattern: 'rule malware { strings: $a = "bad" condition: $a }', + pattern_type: 'yara', + }, + ], + }) + ); + const reports = await stixAdapter.run(buildSource(), buildContext(fetchMock)); + expect(reports).toHaveLength(1); + const [report] = reports; + expect(report.lineage.extraction_method).toBe('pending'); + expect(report.lineage.extracted_at).toBeUndefined(); + expect(report.extracted).toBeUndefined(); + }); + + it('indicator SDO with IN-list pattern (no = literal) → falls back to pending', async () => { + const fetchMock = jest.fn().mockResolvedValue( + okJson({ + type: 'bundle', + id: 'bundle--4', + objects: [ + { + type: 'indicator', + id: 'indicator--4', + name: 'IN list', + description: 'Multiple IPs.', + modified: '2026-05-15T00:00:00Z', + pattern: "[ipv4-addr:value IN ('1.2.3.4', '5.6.7.8')]", + pattern_type: 'stix', + }, + ], + }) + ); + const reports = await stixAdapter.run(buildSource(), buildContext(fetchMock)); + expect(reports).toHaveLength(1); + const [report] = reports; + expect(report.lineage.extraction_method).toBe('pending'); + expect(report.extracted).toBeUndefined(); + }); + + it('indicator SDO with external_references → content.external_references structured and correct', async () => { + const fetchMock = jest.fn().mockResolvedValue( + okJson({ + type: 'bundle', + id: 'bundle--5', + objects: [ + { + type: 'indicator', + id: 'indicator--5', + name: 'Ref indicator', + description: 'Has references.', + modified: '2026-05-15T00:00:00Z', + pattern: "[domain-name:value = 'evil.example']", + pattern_type: 'stix', + external_references: [ + { source_name: 'mitre', external_id: 'T1234', url: 'https://attack.mitre.org/T1234' }, + { source_name: 'nvd', description: 'CVE notes' }, + { not_a_source_name: 'should be dropped' }, + ], + }, + ], + }) + ); + const reports = await stixAdapter.run(buildSource(), buildContext(fetchMock)); + expect(reports).toHaveLength(1); + const refs = reports[0].content.external_references; + expect(refs).toHaveLength(2); + expect(refs![0]).toEqual({ + source_name: 'mitre', + external_id: 'T1234', + url: 'https://attack.mitre.org/T1234', + }); + expect(refs![1]).toEqual({ source_name: 'nvd', description: 'CVE notes' }); + }); + + it('non-indicator SDO (malware) → unchanged shape: pending, no extracted, no external_references', async () => { + const fetchMock = jest.fn().mockResolvedValue( + okJson({ + type: 'bundle', + id: 'bundle--6', + objects: [ + { + type: 'malware', + id: 'malware--1', + name: 'BadBot', + description: 'A RAT.', + modified: '2026-05-15T00:00:00Z', + }, + ], + }) + ); + const reports = await stixAdapter.run(buildSource(), buildContext(fetchMock)); + expect(reports).toHaveLength(1); + const [report] = reports; + expect(report.lineage.extraction_method).toBe('pending'); + expect(report.lineage.extracted_at).toBeUndefined(); + expect(report.extracted).toBeUndefined(); + expect(report.content.external_references).toBeUndefined(); + }); + + it('report without external_references → field absent on non-indicator SDO', async () => { + const fetchMock = jest.fn().mockResolvedValue( + okJson({ + type: 'bundle', + id: 'bundle--7', + objects: [ + { + type: 'threat-actor', + id: 'threat-actor--2', + name: 'NoRefs', + description: 'No external refs.', + created: '2026-05-10T00:00:00Z', + }, + ], + }) + ); + const reports = await stixAdapter.run(buildSource(), buildContext(fetchMock)); + expect(reports[0].content.external_references).toBeUndefined(); + }); + + it('normalizedReportSchema parses both a legacy pending report and a new stix report (back-compat)', () => { + const pendingReport = { + '@timestamp': '2026-05-16T12:00:00.000Z', + content_fingerprint: 'abc123', + space_id: '*', + source: { + type: 'stix' as const, + name: 'Test', + url: 'https://example.com', + adapter_id: 'stix:test', + }, + content: { title: 'Test', body_text: 'body', language: 'en' }, + severity: { level: 'low' as const, score: 1 }, + lineage: { + ingested_at: '2026-05-16T12:00:00.000Z', + extraction_method: 'pending' as const, + source_doc_ref: { index: 'stix:bundle', id: 'indicator--1' }, + }, + }; + expect(() => normalizedReportSchema.parse(pendingReport)).not.toThrow(); + + const stixReport = { + ...pendingReport, + lineage: { + ingested_at: '2026-05-16T12:00:00.000Z', + extraction_method: 'stix' as const, + extracted_at: '2026-05-16T12:00:00.000Z', + source_doc_ref: { index: 'stix:bundle', id: 'indicator--1' }, + }, + extracted: { + iocs: [ + { + type: 'ip', + value: '1.2.3.4', + tier: 'contextual', + tier_heuristic: 'contextual', + tier_basis: 'stix_pattern', + }, + ], + }, + }; + expect(() => normalizedReportSchema.parse(stixReport)).not.toThrow(); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/stix_adapter.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/stix_adapter.ts new file mode 100644 index 0000000000000..d707165633fd8 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/stix_adapter.ts @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { GLOBAL_SPACE_ID } from '../../../../common/threat_intel'; +import { fetchUrlForContext, redactUrl } from '../http_client'; +import { buildFingerprint } from '../fingerprint'; +import { DEFAULT_SEVERITY_LEVEL, DEFAULT_SEVERITY_SCORE } from '../../content/severity'; +import { buildReportContent, collapseWhitespace, truncate } from '../../content/text'; +import type { AdapterRunContext, FetchAdapter, NormalizedReport, SourceHit } from '../types'; +import { composeStixBody, composeStixTitle, splitStixBundle } from './split_bundle'; +import { parseStixPattern } from './parse_pattern'; + +const TITLE_MAX_LENGTH = 280; +const BODY_TEXT_MAX_LENGTH = 32_000; +const SOURCE_DOC_REF_INDEX = 'stix:bundle'; + +const readBundleUrl = (source: SourceHit): string | undefined => { + const url = source._source.config.url; + return typeof url === 'string' && url.length > 0 ? url : undefined; +}; + +const safeParseJson = (body: string): unknown => { + try { + return JSON.parse(body); + } catch { + return undefined; + } +}; + +export const stixAdapter: FetchAdapter = { + adapterType: 'stix', + async run(source, context: AdapterRunContext) { + const fetchUrl = fetchUrlForContext(context); + const log = context.logger.get('stix-adapter'); + const url = readBundleUrl(source); + if (!url) { + log.warn(`Source ${source._id} has no config.url — skipping`); + return []; + } + + const response = await fetchUrl(url, { + abortSignal: context.abortSignal, + headers: { Accept: 'application/stix+json;version=2.1, application/json' }, + }); + if (response.status >= 400) { + throw new Error( + `STIX fetch ${redactUrl(url)} failed: HTTP ${response.status} ${response.statusText}` + ); + } + + const bundle = safeParseJson(response.body); + if (bundle == null) { + throw new Error(`STIX response at ${redactUrl(url)} was not valid JSON`); + } + + const sdos = splitStixBundle(bundle); + if (sdos.length === 0) { + log.debug(`STIX bundle at ${url} contained 0 reportable objects for source ${source._id}`); + return []; + } + + const ingestedAt = context.now().toISOString(); + const spaceId = source._source.space_id ?? GLOBAL_SPACE_ID; + const adapterId = `stix:${source._id}`; + const reports: NormalizedReport[] = []; + for (const { object } of sdos) { + const title = collapseWhitespace(composeStixTitle(object)); + const bodyText = truncate(composeStixBody(object), BODY_TEXT_MAX_LENGTH); + // SDO `modified || created` is the canonical "version" timestamp + // in STIX. Including it in the fingerprint seed means an updated + // SDO produces a new row (so `enrich_threat_report` re-runs) + // while a re-fetch of the unchanged SDO collapses. + const versionStamp = object.modified ?? object.created ?? ''; + const fingerprint = buildFingerprint([url, object.id, versionStamp]); + + // Capture external_references defensively — each entry must have a string source_name. + const rawRefs = Array.isArray(object.external_references) ? object.external_references : []; + const externalReferences = rawRefs + .filter((r): r is Record => r !== null && typeof r === 'object') + .flatMap((r) => { + if (typeof r.source_name !== 'string') return []; + return [ + { + source_name: r.source_name, + ...(typeof r.url === 'string' ? { url: r.url } : {}), + ...(typeof r.external_id === 'string' ? { external_id: r.external_id } : {}), + ...(typeof r.description === 'string' ? { description: r.description } : {}), + }, + ]; + }); + + // Branch: indicator SDOs get parsed IOCs seeded at ingest; all others fall through + // to the standard pending-extraction path. + let lineage: NormalizedReport['lineage']; + let extracted: NormalizedReport['extracted']; + + if (object.type === 'indicator') { + const iocs = parseStixPattern( + typeof object.pattern === 'string' ? object.pattern : '', + typeof object.pattern_type === 'string' ? object.pattern_type : undefined + ); + if (iocs.length > 0) { + // Parseable indicator pattern → structured at ingest; enrichment skips re-extraction. + lineage = { + ingested_at: ingestedAt, + extraction_method: 'stix', + extracted_at: ingestedAt, + source_doc_ref: { index: SOURCE_DOC_REF_INDEX, id: object.id }, + }; + extracted = { iocs }; + } else { + // unparseable indicator pattern → fall back to article path rather than emit + // an empty structured doc. + lineage = { + ingested_at: ingestedAt, + extraction_method: 'pending', + source_doc_ref: { index: SOURCE_DOC_REF_INDEX, id: object.id }, + }; + } + } else { + lineage = { + ingested_at: ingestedAt, + extraction_method: 'pending', + source_doc_ref: { index: SOURCE_DOC_REF_INDEX, id: object.id }, + }; + } + + const baseContent = buildReportContent({ + title: truncate(title, TITLE_MAX_LENGTH), + bodyText, + language: 'en', + }); + + reports.push({ + '@timestamp': ingestedAt, + content_fingerprint: fingerprint, + space_id: spaceId, + source: { + type: 'stix', + name: source._source.name, + url, + adapter_id: adapterId, + }, + content: + externalReferences.length > 0 + ? { ...baseContent, external_references: externalReferences } + : baseContent, + severity: { + level: DEFAULT_SEVERITY_LEVEL, + score: DEFAULT_SEVERITY_SCORE, + }, + lineage, + ...(extracted !== undefined ? { extracted } : {}), + }); + } + return reports; + }, +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/taxii/taxii_adapter.test.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/taxii/taxii_adapter.test.ts new file mode 100644 index 0000000000000..e96e0c7500ed4 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/taxii/taxii_adapter.test.ts @@ -0,0 +1,243 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { taxiiAdapter } from './taxii_adapter'; +import type { AdapterRunContext, ScopedActionsClient, SourceHit } from '../types'; + +const COLLECTION_URL = + 'https://taxii.example/api1/collections/aabbccdd-1234-4abc-9def-000000000001/objects/'; +const NOW = new Date('2026-05-16T12:00:00.000Z'); + +const buildSource = (overrides?: Partial): SourceHit => ({ + _id: 'taxii:vendor', + _source: { + adapter_type: 'taxii', + name: 'Vendor TAXII', + config: { url: COLLECTION_URL, ...overrides }, + }, +}); + +const buildContext = (overrides?: { + fetchImpl?: jest.Mock, [string | URL | Request, RequestInit?]>; + getActionsClient?: () => Promise; +}): AdapterRunContext => ({ + esClient: elasticsearchServiceMock.createElasticsearchClient(), + logger: loggingSystemMock.createLogger(), + abortSignal: new AbortController().signal, + now: () => NOW, + fetchFn: overrides?.fetchImpl as unknown as typeof fetch, + lookupFn: async () => [{ address: '93.184.216.34' }], + getActionsClient: overrides?.getActionsClient, +}); + +describe('taxiiAdapter', () => { + describe('anonymous transport (no connector_id)', () => { + it('derives the collection id from the URL and emits per-SDO reports', async () => { + const fetchMock = jest.fn().mockResolvedValue( + new Response( + JSON.stringify({ + objects: [ + { + type: 'indicator', + id: 'indicator--1', + name: 'Bad IP', + modified: '2026-05-15T00:00:00Z', + }, + ], + }), + { + status: 200, + headers: { 'Content-Type': 'application/taxii+json;version=2.1' }, + } + ) + ); + const reports = await taxiiAdapter.run(buildSource(), buildContext({ fetchImpl: fetchMock })); + expect(reports).toHaveLength(1); + expect(reports[0].lineage.source_doc_ref).toEqual({ + index: 'taxii:collection:aabbccdd-1234-4abc-9def-000000000001', + id: 'indicator--1', + }); + expect(reports[0].source).toMatchObject({ + type: 'taxii', + adapter_id: 'taxii:taxii:vendor', + url: COLLECTION_URL, + }); + }); + + it('falls back to "unknown" when the URL has no /collections/ segment', async () => { + const fetchMock = jest + .fn() + .mockResolvedValue( + new Response( + JSON.stringify({ objects: [{ type: 'indicator', id: 'indicator--1', name: 'x' }] }), + { status: 200 } + ) + ); + const odd: SourceHit = { + _id: 'taxii:weird', + _source: { + adapter_type: 'taxii', + name: 'Weird', + config: { url: 'https://taxii.example/feed' }, + }, + }; + const reports = await taxiiAdapter.run(odd, buildContext({ fetchImpl: fetchMock })); + expect(reports[0].lineage.source_doc_ref?.index).toBe('taxii:collection:unknown'); + }); + }); + + describe('credentialed transport (config.connector_id set)', () => { + it('invokes the .taxii connector pollCollection sub-action and reuses splitStixBundle', async () => { + const execute = jest.fn().mockResolvedValue({ + actionId: 'connector-1', + status: 'ok', + data: { + objects: [ + { + type: 'indicator', + id: 'indicator--cred-1', + name: 'Credentialed IOC', + modified: '2026-05-15T00:00:00Z', + }, + ], + more: false, + }, + }); + const fakeActionsClient = { execute } as unknown as ScopedActionsClient; + + const reports = await taxiiAdapter.run( + buildSource({ connector_id: 'connector-1' }), + buildContext({ getActionsClient: async () => fakeActionsClient }) + ); + + expect(execute).toHaveBeenCalledWith({ + actionId: 'connector-1', + params: { + subAction: 'pollCollection', + subActionParams: { collectionUrl: COLLECTION_URL }, + }, + }); + expect(reports).toHaveLength(1); + expect(reports[0].source).toMatchObject({ + type: 'taxii', + url: COLLECTION_URL, + }); + expect(reports[0].lineage.source_doc_ref?.id).toBe('indicator--cred-1'); + }); + + it('throws when the connector returns status: error', async () => { + const execute = jest.fn().mockResolvedValue({ + actionId: 'connector-1', + status: 'error', + message: 'Unauthorized', + }); + const fakeActionsClient = { execute } as unknown as ScopedActionsClient; + + await expect( + taxiiAdapter.run( + buildSource({ connector_id: 'connector-1' }), + buildContext({ getActionsClient: async () => fakeActionsClient }) + ) + ).rejects.toThrow(/connector "connector-1" pollCollection failed: Unauthorized/); + }); + + it('throws when connector_id is set but no actions factory is available in the context', async () => { + await expect( + taxiiAdapter.run(buildSource({ connector_id: 'connector-1' }), buildContext({})) + ).rejects.toThrow(/actions plugin is not available/); + }); + + it('throws when the actions factory resolves to undefined', async () => { + await expect( + taxiiAdapter.run( + buildSource({ connector_id: 'connector-1' }), + buildContext({ getActionsClient: async () => undefined }) + ) + ).rejects.toThrow(/no ActionsClient could be resolved/); + }); + }); + + // Only the first envelope used to be read, so every collection larger than + // the server page size silently lost all its later pages on every run. + describe('TAXII 2.1 pagination', () => { + const envelope = (id: string, paging: { more?: boolean; next?: string } = {}) => ({ + objects: [{ type: 'indicator', id, name: id, modified: '2026-05-15T00:00:00Z' }], + ...paging, + }); + + const jsonResponse = (body: unknown) => + new Response(JSON.stringify(body), { + status: 200, + headers: { 'Content-Type': 'application/taxii+json;version=2.1' }, + }); + + it('follows next until more is false and returns every page', async () => { + const fetchMock = jest + .fn() + .mockResolvedValueOnce(jsonResponse(envelope('indicator--1', { more: true, next: 'p2' }))) + .mockResolvedValueOnce(jsonResponse(envelope('indicator--2', { more: true, next: 'p3' }))) + .mockResolvedValueOnce(jsonResponse(envelope('indicator--3', { more: false }))); + + const reports = await taxiiAdapter.run(buildSource(), buildContext({ fetchImpl: fetchMock })); + + expect(reports.map((r) => r.lineage.source_doc_ref?.id)).toEqual([ + 'indicator--1', + 'indicator--2', + 'indicator--3', + ]); + // The continuation token travels as the `next` query parameter. + expect(fetchMock.mock.calls[1][0]).toContain('next=p2'); + expect(fetchMock.mock.calls[2][0]).toContain('next=p3'); + }); + + it('stops when more is true but the server sends no next token', async () => { + const fetchMock = jest + .fn() + .mockResolvedValue(jsonResponse(envelope('indicator--1', { more: true }))); + + const reports = await taxiiAdapter.run(buildSource(), buildContext({ fetchImpl: fetchMock })); + + expect(reports).toHaveLength(1); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('bounds the number of pages when a server always reports more', async () => { + const fetchMock = jest + .fn() + .mockImplementation(async () => + jsonResponse(envelope('indicator--loop', { more: true, next: 'always' })) + ); + + await taxiiAdapter.run(buildSource(), buildContext({ fetchImpl: fetchMock })); + + expect(fetchMock).toHaveBeenCalledTimes(50); + }); + + it('paginates the connector path through subActionParams.next', async () => { + const execute = jest + .fn() + .mockResolvedValueOnce({ + status: 'ok', + data: envelope('indicator--c1', { more: true, next: 'c2' }), + }) + .mockResolvedValueOnce({ status: 'ok', data: envelope('indicator--c2', { more: false }) }); + const fakeActionsClient = { execute } as unknown as ScopedActionsClient; + + const reports = await taxiiAdapter.run( + buildSource({ connector_id: 'connector-1' }), + buildContext({ getActionsClient: async () => fakeActionsClient }) + ); + + expect(reports).toHaveLength(2); + expect(execute.mock.calls[1][0].params.subActionParams).toEqual({ + collectionUrl: COLLECTION_URL, + next: 'c2', + }); + }); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/taxii/taxii_adapter.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/taxii/taxii_adapter.ts new file mode 100644 index 0000000000000..37bf67e403472 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/taxii/taxii_adapter.ts @@ -0,0 +1,214 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { GLOBAL_SPACE_ID } from '../../../../common/threat_intel'; +import { fetchUrlForContext, redactUrl } from '../http_client'; +import { buildFingerprint } from '../fingerprint'; +import { DEFAULT_SEVERITY_LEVEL, DEFAULT_SEVERITY_SCORE } from '../../content/severity'; +import { buildReportContent, collapseWhitespace, truncate } from '../../content/text'; +import type { AdapterRunContext, FetchAdapter, NormalizedReport, SourceHit } from '../types'; +import { composeStixBody, composeStixTitle, splitStixBundle } from '../stix/split_bundle'; + +const TITLE_MAX_LENGTH = 280; +const BODY_TEXT_MAX_LENGTH = 32_000; +const TAXII_ACCEPT = 'application/taxii+json;version=2.1, application/json'; +const TAXII_CONNECTOR_POLL_SUB_ACTION = 'pollCollection'; + +/** + * Bound on TAXII 2.1 continuation pages per run. A server that always answers + * `more: true` would otherwise spin forever; the next scheduled run resumes. + */ +const MAX_TAXII_PAGES = 50; + +/** TAXII 2.1 envelope paging fields. */ +const readEnvelopePaging = (envelope: unknown): { more: boolean; next?: string } => { + const env = envelope as { more?: unknown; next?: unknown } | null | undefined; + const next = typeof env?.next === 'string' && env.next.length > 0 ? env.next : undefined; + return { more: env?.more === true, next }; +}; + +/** TAXII 2.1 passes the continuation token as the `next` query parameter. */ +const withNextParam = (url: string, next: string): string => { + const parsed = new URL(url); + parsed.searchParams.set('next', next); + return parsed.toString(); +}; + +const deriveCollectionId = (url: string): string => { + const match = /\/collections\/([^/]+)/.exec(url); + return match ? match[1] : 'unknown'; +}; + +const readCollectionUrl = (source: SourceHit): string | undefined => { + const url = source._source.config.url; + return typeof url === 'string' && url.length > 0 ? url : undefined; +}; + +const readConnectorId = (source: SourceHit): string | undefined => { + const id = source._source.config.connector_id; + return typeof id === 'string' && id.length > 0 ? id : undefined; +}; + +const safeParseJson = (body: string): unknown => { + try { + return JSON.parse(body); + } catch { + return undefined; + } +}; + +const fetchViaConnector = async ( + connectorId: string, + collectionUrl: string, + context: AdapterRunContext, + next?: string +): Promise => { + if (!context.getActionsClient) { + throw new Error( + `Source has connector_id "${connectorId}" but the actions plugin is not available in the workflow execution context` + ); + } + const actionsClient = await context.getActionsClient(); + if (!actionsClient) { + throw new Error( + `Source has connector_id "${connectorId}" but no ActionsClient could be resolved (actions plugin not started?)` + ); + } + const result = await actionsClient.execute({ + actionId: connectorId, + params: { + subAction: TAXII_CONNECTOR_POLL_SUB_ACTION, + subActionParams: { collectionUrl, ...(next ? { next } : {}) }, + }, + }); + if (result.status !== 'ok') { + const reason = result.message ?? result.serviceMessage ?? 'unknown'; + throw new Error(`TAXII connector "${connectorId}" pollCollection failed: ${reason}`); + } + return result.data; +}; + +/** TAXII 2.1 collection poll; credentialed via connector_id or anonymous fetch. */ +export const taxiiAdapter: FetchAdapter = { + adapterType: 'taxii', + async run(source, context: AdapterRunContext) { + const fetchUrl = fetchUrlForContext(context); + const log = context.logger.get('taxii-adapter'); + const url = readCollectionUrl(source); + if (!url) { + log.warn(`Source ${source._id} has no config.url — skipping`); + return []; + } + + const connectorId = readConnectorId(source); + if (connectorId) { + log.debug( + `Polling TAXII collection ${url} via connector ${connectorId} for source ${source._id}` + ); + } + + const fetchEnvelope = async (next?: string): Promise => { + if (connectorId) { + return fetchViaConnector(connectorId, url, context, next); + } + const pageUrl = next ? withNextParam(url, next) : url; + const response = await fetchUrl(pageUrl, { + abortSignal: context.abortSignal, + headers: { Accept: TAXII_ACCEPT }, + }); + if (response.status >= 400) { + throw new Error( + `TAXII poll ${redactUrl(pageUrl)} failed: HTTP ${response.status} ${response.statusText}` + ); + } + const parsed = safeParseJson(response.body); + if (parsed == null) { + throw new Error(`TAXII response at ${redactUrl(pageUrl)} was not valid JSON`); + } + return parsed; + }; + + // TAXII 2.1 pages collections with `more` + `next`. Reading only the first + // envelope silently dropped every later page on every run, so a collection + // larger than the server page size was never fully ingested. + const sdos: ReturnType = []; + let nextToken: string | undefined; + let pages = 0; + do { + const envelope = await fetchEnvelope(nextToken); + sdos.push(...splitStixBundle(envelope)); + pages += 1; + + const paging = readEnvelopePaging(envelope); + nextToken = paging.more ? paging.next : undefined; + + if (nextToken && pages >= MAX_TAXII_PAGES) { + log.warn( + `TAXII collection at ${url} still reported more pages after ${MAX_TAXII_PAGES} for ` + + `source ${source._id}; stopping and resuming on the next run` + ); + break; + } + } while (nextToken && !context.abortSignal.aborted); + + if (pages > 1) { + log.debug( + `TAXII collection at ${url} returned ${sdos.length} objects across ${pages} pages for source ${source._id}` + ); + } + + if (sdos.length === 0) { + log.debug( + `TAXII collection at ${url} returned 0 reportable objects for source ${source._id}` + ); + return []; + } + + const ingestedAt = context.now().toISOString(); + const spaceId = source._source.space_id ?? GLOBAL_SPACE_ID; + const adapterId = `taxii:${source._id}`; + const collectionId = deriveCollectionId(url); + const sourceDocRefIndex = `taxii:collection:${collectionId}`; + + const reports: NormalizedReport[] = []; + for (const { object } of sdos) { + const title = collapseWhitespace(composeStixTitle(object)); + const bodyText = truncate(composeStixBody(object), BODY_TEXT_MAX_LENGTH); + const versionStamp = object.modified ?? object.created ?? ''; + const fingerprint = buildFingerprint([url, object.id, versionStamp]); + reports.push({ + '@timestamp': ingestedAt, + content_fingerprint: fingerprint, + space_id: spaceId, + source: { + type: 'taxii', + name: source._source.name, + url, + adapter_id: adapterId, + }, + content: buildReportContent({ + title: truncate(title, TITLE_MAX_LENGTH), + bodyText, + language: 'en', + }), + severity: { + level: DEFAULT_SEVERITY_LEVEL, + score: DEFAULT_SEVERITY_SCORE, + }, + lineage: { + ingested_at: ingestedAt, + extraction_method: 'pending', + source_doc_ref: { + index: sourceDocRefIndex, + id: object.id, + }, + }, + }); + } + return reports; + }, +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/types.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/types.ts new file mode 100644 index 0000000000000..3cb8e01b8eb0c --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/types.ts @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import type { ActionsClient } from '@kbn/actions-plugin/server'; +import type { PublicMethodsOf } from '@kbn/utility-types'; +import type { NormalizedReport } from '../../../common/threat_intel/workflows/step_types/fetch_source/fetch_source_common'; +import type { SourceType } from '../../../common/threat_intel'; +import type { DnsLookupFn } from './http_client'; + +/** + * The runtime shape of an `ActionsClient` returned by + * `getActionsClientWithRequest()` — public methods only. The actions + * plugin exposes the public-method-only flavor at every call site + * (see `request_context_factory.ts`); aliasing it here keeps adapter + * code agnostic of whether the caller hands us the full class or the + * narrowed handle. + */ +export type ScopedActionsClient = PublicMethodsOf; + +/** + * The `.kibana-threat-intel-sources` hit shape an adapter sees. + * + * Re-declared here (instead of importing the Zod-inferred type from + * `common/`) so adapter implementations don't have to depend on + * `@kbn/zod/v4`. Tracks the schema in + * `common/.../fetch_source_common.ts:sourceHitSchema` — keep them in + * lock-step. + */ +export interface SourceHit { + _id: string; + _index?: string; + _source: { + adapter_type: SourceType; + name: string; + enabled?: boolean; + config: Record; + tags?: string[]; + space_id?: string; + }; +} + +/** + * Runtime context passed to every adapter. Mirrors what the workflow + * step's `StepHandlerContext` exposes, narrowed to just what an adapter + * actually needs. + */ +export interface AdapterRunContext { + /** + * Scoped Elasticsearch client, for adapters that need to read or write source + * state such as TAXII cursors. No adapter uses it yet. + * + * It runs as the requesting user, so it CANNOT touch the plugin-owned + * `.kibana-threat-*` indices: those are system indices, and a Kibana feature + * privilege is not an Elasticsearch privilege, so every non-superuser gets a + * security_exception. Anything reading or writing those has to go through the + * internal user, as the routes and tasks do. + */ + esClient: ElasticsearchClient; + /** Step-scoped logger. Per-adapter messages are tagged with the adapter type. */ + logger: Logger; + /** Cancellation signal from the workflow engine. Adapters MUST honor it on outbound HTTP. */ + abortSignal: AbortSignal; + /** Wall-clock for `@timestamp` and `lineage.ingested_at`. Injected for tests. */ + now: () => Date; + /** Optional fetch override for tests. Defaults to `globalThis.fetch`. */ + fetchFn?: typeof fetch; + /** + * Optional DNS override for the SSRF pre-flight. Defaults to real + * resolution. Tests that stub `fetchFn` must stub this too, otherwise the + * guard tries to resolve the fixture hostname for real. + */ + lookupFn?: DnsLookupFn; + /** + * Lazy resolver for an `ActionsClient` scoped to the workflow's fake + * request. Adapters that need to invoke a configured Connectors v2 + * connector (e.g. `.taxii` for credentialed TAXII feeds) call this + * on demand. Resolves to `undefined` when the actions plugin is not + * available — callers MUST treat that as a hard error if the source + * configuration requires a connector (anonymous paths can ignore). + * + * Returning a factory rather than a pre-resolved client keeps the + * step handler's startup cost flat for adapters that don't need + * actions (RSS, anonymous STIX, anonymous TAXII). + */ + getActionsClient?: () => Promise; +} + +/** + * The single contract every adapter implements. Adapters are pure (no + * Elasticsearch writes) — they normalize upstream content into report + * documents and let the workflow handle dedup-and-write. + */ +export interface FetchAdapter { + /** Discriminator on the source's `adapter_type`. */ + readonly adapterType: SourceType; + /** + * Fetch the source and return zero or more normalized reports. + * + * Adapters that don't apply to a specific source (e.g. an unknown + * vendor under `vendor_api`) should return `[]` and log a `warn` + * rather than throw — the workflow distinguishes "ran successfully, + * nothing new" from "errored" and we want the no-content path to + * keep the per-source `on-failure: continue: true` honest. + */ + run(source: SourceHit, context: AdapterRunContext): Promise; +} + +export type { NormalizedReport }; From d5891e7c8f69861b46b960172a80b79825310572 Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Wed, 26 Aug 2026 16:07:22 -0600 Subject: [PATCH 02/27] fix(threat-intel): redact feed URLs in the remaining adapter log lines `rss`, `stix`, and `taxii` all import `redactUrl` and use it in their thrown errors, but each still interpolated the raw URL into at least one log line. Source URLs may embed `user:password@`, so those lines put feed credentials into Kibana's logs. Six sites across the three adapters. The raw URL is still what gets fetched; only the log text changes. Co-Authored-By: Claude Opus 5 --- .../server/threat_intel/adapters/rss/rss_adapter.ts | 2 +- .../server/threat_intel/adapters/stix/stix_adapter.ts | 4 +++- .../server/threat_intel/adapters/taxii/taxii_adapter.ts | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/rss_adapter.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/rss_adapter.ts index ee44acdbe2b92..5e2ebc06fdd6b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/rss_adapter.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/rss_adapter.ts @@ -65,7 +65,7 @@ export const rssAdapter: FetchAdapter = { const feedBody = await readFeedBody(feedUrl, context); const parsed = await parseRssFeed(feedBody); if (parsed.entries.length === 0) { - log.debug(`RSS feed ${feedUrl} returned 0 items for source ${source._id}`); + log.debug(`RSS feed ${redactUrl(feedUrl)} returned 0 items for source ${source._id}`); return []; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/stix_adapter.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/stix_adapter.ts index d707165633fd8..0a53fb4e5f020 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/stix_adapter.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/stix/stix_adapter.ts @@ -59,7 +59,9 @@ export const stixAdapter: FetchAdapter = { const sdos = splitStixBundle(bundle); if (sdos.length === 0) { - log.debug(`STIX bundle at ${url} contained 0 reportable objects for source ${source._id}`); + log.debug( + `STIX bundle at ${redactUrl(url)} contained 0 reportable objects for source ${source._id}` + ); return []; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/taxii/taxii_adapter.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/taxii/taxii_adapter.ts index 37bf67e403472..9294b72567c42 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/taxii/taxii_adapter.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/taxii/taxii_adapter.ts @@ -107,7 +107,7 @@ export const taxiiAdapter: FetchAdapter = { const connectorId = readConnectorId(source); if (connectorId) { log.debug( - `Polling TAXII collection ${url} via connector ${connectorId} for source ${source._id}` + `Polling TAXII collection ${redactUrl(url)} via connector ${connectorId} for source ${source._id}` ); } @@ -148,7 +148,7 @@ export const taxiiAdapter: FetchAdapter = { if (nextToken && pages >= MAX_TAXII_PAGES) { log.warn( - `TAXII collection at ${url} still reported more pages after ${MAX_TAXII_PAGES} for ` + + `TAXII collection at ${redactUrl(url)} still reported more pages after ${MAX_TAXII_PAGES} for ` + `source ${source._id}; stopping and resuming on the next run` ); break; @@ -157,7 +157,7 @@ export const taxiiAdapter: FetchAdapter = { if (pages > 1) { log.debug( - `TAXII collection at ${url} returned ${sdos.length} objects across ${pages} pages for source ${source._id}` + `TAXII collection at ${redactUrl(url)} returned ${sdos.length} objects across ${pages} pages for source ${source._id}` ); } From 6b621391d81d2ba768b650dd7d40052178d61ce3 Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Wed, 26 Aug 2026 16:55:34 -0600 Subject: [PATCH 03/27] fix(threat-intel): redact the last raw feed URL in the taxii adapter Missed in the previous commit: the zero-reportable-objects log line. Co-Authored-By: Claude Opus 5 --- .../threat_intel/adapters/taxii/taxii_adapter.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/taxii/taxii_adapter.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/taxii/taxii_adapter.ts index 9294b72567c42..a06878c28dca3 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/taxii/taxii_adapter.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/taxii/taxii_adapter.ts @@ -107,7 +107,9 @@ export const taxiiAdapter: FetchAdapter = { const connectorId = readConnectorId(source); if (connectorId) { log.debug( - `Polling TAXII collection ${redactUrl(url)} via connector ${connectorId} for source ${source._id}` + `Polling TAXII collection ${redactUrl(url)} via connector ${connectorId} for source ${ + source._id + }` ); } @@ -148,7 +150,9 @@ export const taxiiAdapter: FetchAdapter = { if (nextToken && pages >= MAX_TAXII_PAGES) { log.warn( - `TAXII collection at ${redactUrl(url)} still reported more pages after ${MAX_TAXII_PAGES} for ` + + `TAXII collection at ${redactUrl( + url + )} still reported more pages after ${MAX_TAXII_PAGES} for ` + `source ${source._id}; stopping and resuming on the next run` ); break; @@ -157,13 +161,17 @@ export const taxiiAdapter: FetchAdapter = { if (pages > 1) { log.debug( - `TAXII collection at ${redactUrl(url)} returned ${sdos.length} objects across ${pages} pages for source ${source._id}` + `TAXII collection at ${redactUrl(url)} returned ${ + sdos.length + } objects across ${pages} pages for source ${source._id}` ); } if (sdos.length === 0) { log.debug( - `TAXII collection at ${url} returned 0 reportable objects for source ${source._id}` + `TAXII collection at ${redactUrl(url)} returned 0 reportable objects for source ${ + source._id + }` ); return []; } From 5e10320eaf79f98977980ba1b50a712484e06c08 Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Wed, 26 Aug 2026 18:23:16 -0600 Subject: [PATCH 04/27] docs(threat-intel): correct why the scoped client cannot reach the plugin indices The conclusion was right and the mechanism was wrong. These are not system or restricted indices: `.kibana-*` with a hyphen is not in Elasticsearch's restricted set, which covers `.kibana` and `.kibana_*` with an underscore, and an ordinary role granted `read` on one of them reads it fine without `allow_restricted_indices`. Verified against a live cluster rather than reasoned about. The real reason a non-superuser gets a security_exception is simpler: a Kibana feature privilege is not an Elasticsearch privilege, and nothing in this plugin grants an index privilege on these indices. Worth correcting rather than leaving as a harmless inaccuracy, because the two explanations imply opposite fixes. "Restricted index" reads as "no role can be given access, stop trying"; the truth is "no role has been given access yet", which is something an operator can act on. Co-Authored-By: Claude Opus 5 --- .../server/threat_intel/adapters/types.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/types.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/types.ts index 3cb8e01b8eb0c..259fdbe2bd965 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/types.ts @@ -54,11 +54,18 @@ export interface AdapterRunContext { * Scoped Elasticsearch client, for adapters that need to read or write source * state such as TAXII cursors. No adapter uses it yet. * - * It runs as the requesting user, so it CANNOT touch the plugin-owned - * `.kibana-threat-*` indices: those are system indices, and a Kibana feature - * privilege is not an Elasticsearch privilege, so every non-superuser gets a - * security_exception. Anything reading or writing those has to go through the - * internal user, as the routes and tasks do. + * It runs as the requesting user, so in practice it CANNOT touch the plugin-owned + * `.kibana-threat-*` indices: a Kibana feature privilege is not an Elasticsearch + * privilege, and nothing in this plugin grants an index privilege on them, so every + * non-superuser gets a security_exception. Anything reading or writing those has to + * go through the internal user, as the routes and tasks do. + * + * Not because they are system or restricted indices. They are ordinary hidden + * indices: `.kibana-*` (hyphen) is not in Elasticsearch's restricted set, which + * covers `.kibana` and `.kibana_*` (underscore), and an ordinary role granted `read` + * on one of these reads it fine without `allow_restricted_indices`. Verified against + * a live cluster. The distinction matters because it is the difference between "no + * role can be given access" and "no role has been given access yet". */ esClient: ElasticsearchClient; /** Step-scoped logger. Per-adapter messages are tagged with the adapter type. */ From 0137dfba1ca6e4d46671f4272b1c076b8fa41869 Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Thu, 27 Aug 2026 08:47:26 -0600 Subject: [PATCH 05/27] feat(threat-intel): resolve Atom/RSS namespace and type semantics in parse_rss Turns on xml2js's namespace and child-order tracking so this file decides, once, what a feed entry's body actually is, instead of leaving that to be re-derived by hand downstream in content/text.ts (companion change on threat-intel-3-content). parseAtom/parseRss2/parseRdf now return a tagged EntryBody (`{ kind: 'markup', html }` or `{ kind: 'text', text }`) instead of an ambiguous bodyHtml string. rss_adapter.ts branches on body.kind instead of always calling stripHtml and always writing body_html. Fixes two real gaps along the way: RDF feeds had no content:encoded support at all, and Atom type="xhtml" content was silently dropped instead of walked as markup. A type="text" (or untyped, which RFC 4287 defaults to text) construct is also no longer mislabeled into body_html. Co-Authored-By: Claude Sonnet 5 --- .../adapters/rss/parse_rss.test.ts | 190 ++++++++++++- .../threat_intel/adapters/rss/parse_rss.ts | 256 ++++++++++++++++-- .../threat_intel/adapters/rss/rss_adapter.ts | 12 +- 3 files changed, 423 insertions(+), 35 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/parse_rss.test.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/parse_rss.test.ts index da2751823c2c1..3f7e44afb0087 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/parse_rss.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/parse_rss.test.ts @@ -83,8 +83,9 @@ describe('parseRssFeed', () => { link: 'https://acme.example/posts/apt1', publishedAt: new Date('Mon, 12 May 2025 09:30:00 GMT').toISOString(), }); - // CDATA contents preserved as-is on bodyHtml. - expect(parsed.entries[0].bodyHtml).toContain('APT-1'); + // CDATA contents preserved as-is on the markup body. + expect(parsed.entries[0].body).toMatchObject({ kind: 'markup' }); + expect((parsed.entries[0].body as { html: string }).html).toContain('APT-1'); }); it('parses an Atom feed and prefers updated over published', async () => { @@ -94,8 +95,11 @@ describe('parseRssFeed', () => { expect(parsed.entries).toHaveLength(2); expect(parsed.entries[0].publishedAt).toBe('2025-05-12T09:30:00.000Z'); expect(parsed.entries[0].link).toBe('https://vendor.example/post-1'); - expect(parsed.entries[0].bodyHtml).toBe('

Long body

'); + expect(parsed.entries[0].body).toEqual({ kind: 'markup', html: '

Long body

' }); expect(parsed.entries[1].publishedAt).toBe('2025-05-11T08:00:00.000Z'); + // Second entry has no `type` on its , which defaults to `text` per RFC 4287 — + // literal text, not markup, so it must not land in body_html. + expect(parsed.entries[1].body).toEqual({ kind: 'text', text: 'Only a summary.' }); }); it('parses an RDF / RSS 1.0 feed using rdf:about as the id', async () => { @@ -137,7 +141,8 @@ describe('parseRssFeed — description-only items', () => { expect(parsed.entries).toHaveLength(1); expect(parsed.entries[0].id).toBe('adv-1'); - expect(parsed.entries[0].bodyHtml).toContain('ransomware'); + expect(parsed.entries[0].body).toMatchObject({ kind: 'markup' }); + expect((parsed.entries[0].body as { html: string }).html).toContain('ransomware'); }); it('still drops an item with no identifier', async () => { @@ -152,3 +157,180 @@ describe('parseRssFeed — description-only items', () => { expect((await parseRssFeed(feed)).entries).toHaveLength(0); }); }); + +// Namespace resolution for the RSS Content Module: the conventional `content:` prefix is +// accepted unconditionally (real feeds routinely omit the declaration and always mean the +// module), but any other prefix has to actually resolve, via a real `xmlns:` declaration, to +// the module's namespace URI. An aliased prefix that resolves to something else is not the +// Content Module and must not be treated as one. +describe('parseRssFeed — RSS Content Module namespace resolution', () => { + it('resolves content:encoded under an aliased prefix declared on the root', async () => { + const feed = ` + + + F + 1T<p>full</p> + +`; + + const parsed = await parseRssFeed(feed); + + expect(parsed.entries[0].body).toEqual({ kind: 'markup', html: '

full

' }); + }); + + it('resolves content:encoded under an alias declared on the element itself', async () => { + const feed = ` + + + F + 1T + <p>full</p> + + +`; + + const parsed = await parseRssFeed(feed); + + expect(parsed.entries[0].body).toEqual({ kind: 'markup', html: '

full

' }); + }); + + it('accepts the conventional content: prefix even when undeclared', async () => { + const feed = ` + + + F + 1T<p>full</p> + +`; + + const parsed = await parseRssFeed(feed); + + expect(parsed.entries[0].body).toEqual({ kind: 'markup', html: '

full

' }); + }); + + it('does not treat an aliased prefix bound to an unrelated namespace as the Content Module', async () => { + const feed = ` + + + F + 1Tfallbacknot the module + +`; + + const parsed = await parseRssFeed(feed); + + // Falls back to , since the aliased did not resolve. + expect(parsed.entries[0].body).toEqual({ kind: 'markup', html: 'fallback' }); + }); + + it('resolves content:encoded on the RDF / RSS 1.0 branch too', async () => { + const feed = ` + + F + + T + summary only + <p>full RDF body</p> + +`; + + const parsed = await parseRssFeed(feed); + + expect(parsed.entries[0].body).toEqual({ kind: 'markup', html: '

full RDF body

' }); + }); +}); + +// Atom's `type=` attribute decides whether / is literal text, entity-encoded +// HTML, or inline XML markup. Ignoring it either mislabels plain text as HTML or — for +// type="xhtml" — silently drops the real child markup, since a plain-text extraction of that +// node only sees the (mostly empty) text between the child elements. +describe('parseRssFeed — Atom content-type resolution', () => { + const entryWith = (contentOrSummary: string) => ` + + F + 1T${contentOrSummary} +`; + + it('keeps a text-typed summary literal, not markup', async () => { + const feed = entryWith( + 'Exploit uses <script> and c2.evil.test' + ); + + const parsed = await parseRssFeed(feed); + + expect(parsed.entries[0].body).toEqual({ + kind: 'text', + text: 'Exploit uses

after

' + ); + expect(text).not.toContain('alert'); + expect(text).not.toContain('evil.com'); + expect(text).toContain('before'); + expect(text).toContain('after'); + }); + + it('drops

visible

'); + expect(text).toBe('visible'); + }); + + it.each(['template', 'iframe', 'noembed', 'noframes', 'title', 'textarea'])( + 'drops <%s> content', + (tag) => { + const text = htmlFragmentToText(`

keep

<${tag}>secret-token`); + expect(text).not.toContain('secret-token'); + expect(text).toContain('keep'); + } + ); + + it('replaces a stripped subtree with a space so tokens cannot reassemble', () => { + // Without a separating space, `1.1.1.` + `1` would glue into the IP `1.1.1.1`. + const text = htmlFragmentToText('1.1.1.1'); + expect(text).not.toContain('1.1.1.1'); + expect(text).toBe('1.1.1. 1'); + }); + }); + + describe('token boundaries', () => { + it('separates adjacent block elements', () => { + expect(htmlFragmentToText('

alpha

beta

')).toBe('alpha beta'); + }); + + it('separates list items', () => { + expect(htmlFragmentToText('
  • one
  • two
')).toBe('one two'); + }); + + it('separates table cells', () => { + expect(htmlFragmentToText('
ab
')).toBe('a b'); + }); + + it('turns
into a separator', () => { + expect(htmlFragmentToText('line one
line two')).toBe('line one line two'); + }); + + it('collapses runs of whitespace introduced by boundaries', () => { + expect(htmlFragmentToText('

x

\n\n

y

')).toBe('x y'); + }); + }); + + describe('malformed and adversarial input', () => { + it('tolerates unclosed tags without throwing', () => { + expect(htmlFragmentToText('

unclosed bold')).toBe('unclosed bold'); + }); + + it('tolerates a bare < that is not markup', () => { + const text = htmlFragmentToText('price < 5 and value > 3'); + expect(text).toContain('price'); + expect(text).toContain('value'); + }); + + it('returns plain text unchanged when there is no markup', () => { + expect(htmlFragmentToText('just plain text with 8.8.8.8')).toBe( + 'just plain text with 8.8.8.8' + ); + }); + }); + + describe('bounds', () => { + it('bounds the output length', () => { + const huge = `

${'a'.repeat(5_000_000)}

`; + const text = htmlFragmentToText(huge); + expect(text.length).toBeLessThanOrEqual(200_000); + }); + + it('does not throw on very large input', () => { + expect(() => htmlFragmentToText('

x

'.repeat(500_000))).not.toThrow(); + }); + }); + + it('does not fetch or expand external references', () => { + // An / is inert text-wise: only the visible text survives, no URL. + const text = htmlFragmentToText('click'); + expect(text).toBe('click'); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/html_fragment_to_text.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/html_fragment_to_text.ts new file mode 100644 index 0000000000000..4aabda0588924 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/html_fragment_to_text.ts @@ -0,0 +1,185 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { load } from 'cheerio'; + +/** + * RSS/Atom is the one narrow markup boundary this feature accepts: feed entries may embed + * an HTML fragment in `content:encoded`, an RSS ``, or an Atom `html`/`xhtml` + * construct. This turns such a fragment into bounded plain text so nothing downstream ever + * sees, stores, or interprets markup. + * + * It is deliberately NOT a general HTML utility: it does not identify an article container, + * inspect inline CSS, or fetch anything. Keep it RSS-local — do not export it for reuse. + */ + +/** + * Cap the fragment fed to the parser. Feed bodies are summaries or article text, not whole + * sites, so anything past this is almost certainly abuse or a malformed feed; truncating + * up front bounds parse work regardless of what the feed sends. + */ +const MAX_INPUT_CHARS = 500_000; + +/** Cap the returned text so a pathological fragment cannot expand into an unbounded string. */ +const MAX_OUTPUT_CHARS = 200_000; + +/** + * Subtrees whose text is never article content. Skipped entirely — and replaced with a + * space, not deleted outright — so tokens on either side of a stripped `

after

' - ); - expect(text).not.toContain('alert'); - expect(text).not.toContain('evil.com'); - expect(text).toContain('before'); - expect(text).toContain('after'); - }); - - it('drops

visible

'); - expect(text).toBe('visible'); - }); - - it.each(['template', 'iframe', 'noembed', 'noframes', 'title', 'textarea'])( - 'drops <%s> content', - (tag) => { - const text = htmlFragmentToText(`

keep

<${tag}>secret-token`); - expect(text).not.toContain('secret-token'); - expect(text).toContain('keep'); - } - ); - - it('replaces a stripped subtree with a space so tokens cannot reassemble', () => { - // Without a separating space, `1.1.1.` + `1` would glue into the IP `1.1.1.1`. - const text = htmlFragmentToText('1.1.1.1'); - expect(text).not.toContain('1.1.1.1'); - expect(text).toBe('1.1.1. 1'); - }); - }); - - describe('token boundaries', () => { - it('separates adjacent block elements', () => { - expect(htmlFragmentToText('

alpha

beta

')).toBe('alpha beta'); - }); - - it('separates list items', () => { - expect(htmlFragmentToText('
  • one
  • two
')).toBe('one two'); - }); - - it('separates table cells', () => { - expect(htmlFragmentToText('
ab
')).toBe('a b'); - }); - - it('turns
into a separator', () => { - expect(htmlFragmentToText('line one
line two')).toBe('line one line two'); - }); - - it('collapses runs of whitespace introduced by boundaries', () => { - expect(htmlFragmentToText('

x

\n\n

y

')).toBe('x y'); - }); - }); - - describe('malformed and adversarial input', () => { - it('tolerates unclosed tags without throwing', () => { - expect(htmlFragmentToText('

unclosed bold')).toBe('unclosed bold'); - }); - - it('tolerates a bare < that is not markup', () => { - const text = htmlFragmentToText('price < 5 and value > 3'); - expect(text).toContain('price'); - expect(text).toContain('value'); - }); - - it('returns plain text unchanged when there is no markup', () => { - expect(htmlFragmentToText('just plain text with 8.8.8.8')).toBe( - 'just plain text with 8.8.8.8' - ); - }); - }); - - describe('bounds', () => { - it('bounds the output length', () => { - const huge = `

${'a'.repeat(5_000_000)}

`; - const text = htmlFragmentToText(huge); - expect(text.length).toBeLessThanOrEqual(200_000); - }); - - it('does not throw on very large input', () => { - expect(() => htmlFragmentToText('

x

'.repeat(500_000))).not.toThrow(); - }); - }); - - it('does not fetch or expand external references', () => { - // An / is inert text-wise: only the visible text survives, no URL. - const text = htmlFragmentToText('click'); - expect(text).toBe('click'); - }); -}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/html_fragment_to_text.ts b/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/html_fragment_to_text.ts deleted file mode 100644 index 4aabda0588924..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/html_fragment_to_text.ts +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { load } from 'cheerio'; - -/** - * RSS/Atom is the one narrow markup boundary this feature accepts: feed entries may embed - * an HTML fragment in `content:encoded`, an RSS ``, or an Atom `html`/`xhtml` - * construct. This turns such a fragment into bounded plain text so nothing downstream ever - * sees, stores, or interprets markup. - * - * It is deliberately NOT a general HTML utility: it does not identify an article container, - * inspect inline CSS, or fetch anything. Keep it RSS-local — do not export it for reuse. - */ - -/** - * Cap the fragment fed to the parser. Feed bodies are summaries or article text, not whole - * sites, so anything past this is almost certainly abuse or a malformed feed; truncating - * up front bounds parse work regardless of what the feed sends. - */ -const MAX_INPUT_CHARS = 500_000; - -/** Cap the returned text so a pathological fragment cannot expand into an unbounded string. */ -const MAX_OUTPUT_CHARS = 200_000; - -/** - * Subtrees whose text is never article content. Skipped entirely — and replaced with a - * space, not deleted outright — so tokens on either side of a stripped `