feat(threat-intel): RSS, STIX, and TAXII source adapters

Author: stephmilovic

x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/fingerprint.test.ts

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createHash } from 'crypto';
+import { buildFingerprint } from './fingerprint';
+
+describe('buildFingerprint', () => {
+  it('length-prefixes each part so the seed is unambiguous', () => {
+    // The seed is `<len>:<part>` per part, concatenated. There is no workflow-side
+    // computation to match: the definitions only pass `content_fingerprint`
+    // through, and the engine has no sha256 Liquid filter, so this encoding is
+    // ours alone and only has to be self-consistent.
+    const expected = createHash('sha256')
+      .update('28:https://example.com/feed.xml7:item-425:Title')
+      .digest('hex');
+    expect(buildFingerprint(['https://example.com/feed.xml', 'item-42', 'Title'])).toBe(expected);
+  });
+
+  // A plain join on `:` collided across part boundaries, so two different feed
+  // items produced one fingerprint and the second was deduplicated away as
+  // already-ingested. Titles, URLs, and ids routinely contain colons.
+  it('does not collide when a colon moves across a part boundary', () => {
+    expect(buildFingerprint(['a:b', 'c'])).not.toBe(buildFingerprint(['a', 'b:c']));
+  });
+
+  it('does not collide when a part boundary shifts', () => {
+    expect(buildFingerprint(['https://evil.test', 'a'])).not.toBe(
+      buildFingerprint(['https://evil.test:a', ''])
+    );
+  });
+
+  it('returns a 64-char hex digest', () => {
+    expect(buildFingerprint(['a', 'b'])).toMatch(/^[0-9a-f]{64}$/);
+  });
+
+  it('NFKC-normalizes parts so unicode equivalents collapse', () => {
+    // U+FB01 (fi ligature) vs ASCII "fi".
+    expect(buildFingerprint(['url', 'id', '\ufb01nal'])).toBe(
+      buildFingerprint(['url', 'id', 'final'])
+    );
+  });
+
+  it('trims leading/trailing whitespace per part', () => {
+    expect(buildFingerprint(['  url  ', '  id  '])).toBe(buildFingerprint(['url', 'id']));
+  });
+
+  it('treats undefined/null parts as empty strings (still positional)', () => {
+    // Two leading missing parts are still part of the seed shape — a
+    // re-fetch with the same shape produces the same digest.
+    const a = buildFingerprint([undefined, undefined, 'id']);
+    const b = buildFingerprint([undefined, undefined, 'id']);
+    expect(a).toBe(b);
+    // …but a missing part is *positionally distinct* from the part
+    // moving up by one — `:::id` and `id` produce different digests.
+    expect(a).not.toBe(buildFingerprint(['id']));
+  });
+
+  it('produces a stable digest for the same logical input', () => {
+    const fp1 = buildFingerprint(['https://example.com', 'id-1', 'modified-2026-05-01']);
+    const fp2 = buildFingerprint(['https://example.com', 'id-1', 'modified-2026-05-01']);
+    expect(fp1).toBe(fp2);
+  });
+
+  it('produces different digests when the version stamp changes', () => {
+    const a = buildFingerprint(['https://example.com', 'id-1', '2026-05-01']);
+    const b = buildFingerprint(['https://example.com', 'id-1', '2026-05-02']);
+    expect(a).not.toBe(b);
+  });
+});

x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/fingerprint.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createHash } from 'crypto';
+
+/**
+ * SHA-256 over NFKC-normalized parts, each length-prefixed.
+ *
+ * A plain `join(':')` is ambiguous: `['a:b', 'c']` and `['a', 'b:c']` produce the
+ * same seed, so two different items collide on one fingerprint and the second is
+ * deduplicated away as though it had already been ingested. The parts here are
+ * feed-controlled titles, URLs, and ids, which routinely contain colons, so this
+ * is reachable rather than theoretical.
+ *
+ * Length-prefixing makes the seed unambiguous. Nothing outside this module
+ * recomputes these values (the workflows only pass `content_fingerprint` through),
+ * so the change is safe; already-stored reports will be re-ingested once under
+ * their new fingerprint.
+ */
+export const buildFingerprint = (parts: ReadonlyArray<string | undefined | null>): string => {
+  const seed = parts
+    .map((part) => (part ?? '').trim().normalize('NFKC'))
+    .map((part) => `${part.length}:${part}`)
+    .join('');
+  return createHash('sha256').update(seed).digest('hex');
+};

x-pack/solutions/security/plugins/security_solution/server/threat_intel/adapters/rss/parse_rss.test.ts

@@ -0,0 +1,154 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { parseRssFeed } from './parse_rss';
+
+const RSS2 = `<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0">
+  <channel>
+    <title>Acme Threat Research</title>
+    <language>en-US</language>
+    <item>
+      <title>APT-1 campaign</title>
+      <link>https://acme.example/posts/apt1</link>
+      <guid isPermaLink="false">acme:apt1</guid>
+      <pubDate>Mon, 12 May 2025 09:30:00 GMT</pubDate>
+      <description><![CDATA[<p>Brief summary of <b>APT-1</b>.</p>]]></description>
+    </item>
+    <item>
+      <title>Ransomware uptick</title>
+      <link>https://acme.example/posts/ransom</link>
+      <pubDate>Tue, 13 May 2025 10:00:00 GMT</pubDate>
+      <description>Plain summary.</description>
+    </item>
+    <item>
+      <!-- intentionally missing every identifier — should be dropped -->
+      <description>Orphan item with no id, link, or guid.</description>
+    </item>
+  </channel>
+</rss>`;
+
+const ATOM = `<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom" xml:lang="en">
+  <title>Vendor Labs</title>
+  <entry>
+    <id>tag:vendor.example,2025:post-1</id>
+    <title>Post one</title>
+    <link rel="alternate" href="https://vendor.example/post-1"/>
+    <updated>2025-05-12T09:30:00Z</updated>
+    <summary>Short summary.</summary>
+    <content type="html">&lt;p&gt;Long body&lt;/p&gt;</content>
+  </entry>
+  <entry>
+    <id>tag:vendor.example,2025:post-2</id>
+    <title>Post two</title>
+    <link href="https://vendor.example/post-2"/>
+    <published>2025-05-11T08:00:00Z</published>
+    <summary>Only a summary.</summary>
+  </entry>
+</feed>`;
+
+const RDF = `<?xml version="1.0"?>
+<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:dc="http://purl.org/dc/elements/1.1/">
+  <channel rdf:about="https://example.com">
+    <title>RDF Feed</title>
+    <dc:language>en</dc:language>
+  </channel>
+  <item rdf:about="https://example.com/posts/1">
+    <title>RDF Item</title>
+    <link>https://example.com/posts/1</link>
+    <description>RDF body.</description>
+    <dc:date>2025-05-12T09:30:00Z</dc:date>
+  </item>
+</rdf:RDF>`;
+
+describe('parseRssFeed', () => {
+  it('returns an empty result for an empty input', async () => {
+    const parsed = await parseRssFeed('');
+    expect(parsed).toEqual({ feedTitle: '', entries: [] });
+  });
+
+  it('parses an RSS 2.0 feed and drops items without an identifier', async () => {
+    const parsed = await parseRssFeed(RSS2);
+    expect(parsed.feedTitle).toBe('Acme Threat Research');
+    expect(parsed.language).toBe('en');
+    expect(parsed.entries).toHaveLength(2);
+    expect(parsed.entries[0]).toMatchObject({
+      id: 'acme:apt1',
+      title: 'APT-1 campaign',
+      link: 'https://acme.example/posts/apt1',
+      publishedAt: new Date('Mon, 12 May 2025 09:30:00 GMT').toISOString(),
+    });
+    // CDATA contents preserved as-is on bodyHtml.
+    expect(parsed.entries[0].bodyHtml).toContain('<b>APT-1</b>');
+  });
+
+  it('parses an Atom feed and prefers updated over published', async () => {
+    const parsed = await parseRssFeed(ATOM);
+    expect(parsed.feedTitle).toBe('Vendor Labs');
+    expect(parsed.language).toBe('en');
+    expect(parsed.entries).toHaveLength(2);
+    expect(parsed.entries[0].publishedAt).toBe('2025-05-12T09:30:00.000Z');
+    expect(parsed.entries[0].link).toBe('https://vendor.example/post-1');
+    expect(parsed.entries[0].bodyHtml).toBe('<p>Long body</p>');
+    expect(parsed.entries[1].publishedAt).toBe('2025-05-11T08:00:00.000Z');
+  });
+
+  it('parses an RDF / RSS 1.0 feed using rdf:about as the id', async () => {
+    const parsed = await parseRssFeed(RDF);
+    expect(parsed.feedTitle).toBe('RDF Feed');
+    expect(parsed.language).toBe('en');
+    expect(parsed.entries).toHaveLength(1);
+    expect(parsed.entries[0]).toMatchObject({
+      id: 'https://example.com/posts/1',
+      title: 'RDF Item',
+      link: 'https://example.com/posts/1',
+      publishedAt: '2025-05-12T09:30:00.000Z',
+    });
+  });
+
+  it('returns an empty result for an unrecognized root element', async () => {
+    const parsed = await parseRssFeed('<?xml version="1.0"?><unknown/>');
+    expect(parsed).toEqual({ feedTitle: '', entries: [] });
+  });
+});
+
+// RSS 2.0 permits an item with a description and no title, and real advisory feeds
+// publish them. Every parser branch leaves `body` empty and puts the description in
+// `bodyHtml`, so a `title || body` check dropped all of them.
+describe('parseRssFeed — description-only items', () => {
+  it('keeps an RSS 2.0 item that has a description but no title', async () => {
+    const feed = `<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0">
+  <channel>
+    <title>Advisories</title>
+    <item>
+      <guid>adv-1</guid>
+      <description>Threat actor deployed ransomware via 185.220.101.45.</description>
+    </item>
+  </channel>
+</rss>`;
+
+    const parsed = await parseRssFeed(feed);
+
+    expect(parsed.entries).toHaveLength(1);
+    expect(parsed.entries[0].id).toBe('adv-1');
+    expect(parsed.entries[0].bodyHtml).toContain('ransomware');
+  });
+
+  it('still drops an item with no identifier', async () => {
+    const feed = `<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0">
+  <channel>
+    <title>Advisories</title>
+    <item><description>No guid and no link.</description></item>
+  </channel>
+</rss>`;
+
+    expect((await parseRssFeed(feed)).entries).toHaveLength(0);
+  });
+});