[Security Solution] Report zero-shard rule searches as actionable warnings instead of "expected to find aggregations"

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts

@@ -452,6 +452,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
                     eventsTelemetry,
                     licensing,
                     scheduleNotificationResponseActionsService: responseActionsService,
+                    cpsData: options.cpsData,
                   },
                 });
 

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts

@@ -16,6 +16,7 @@ import { NewTermsRuleParams } from '../../rule_schema';
 import type { SecurityAlertType } from '../types';
 import { createNewTermsFieldCardinalityTracker } from '../utils/telemetry/new_terms_field_cardinality_tracker';
 import { singleSearchAfter } from '../utils/single_search_after';
+import { reportMissingAggregations } from '../utils/no_readable_shards';
 import { buildEventsSearchQuery } from '../utils/build_events_query';
 import { getFilter } from '../utils/get_filter';
 import { wrapNewTermsAlerts } from './wrap_new_terms_alerts';
@@ -201,6 +202,7 @@ export const createNewTermsAlertType = (): SecurityAlertType<
           searchResult,
           searchDuration,
           searchErrors,
+          searchWarnings,
           loggedRequests: firstPhaseLoggedRequests = [],
         } = await singleSearchAfter({
           searchRequest,
@@ -217,13 +219,24 @@ export const createNewTermsAlertType = (): SecurityAlertType<
             : undefined,
         });
         loggedRequests.push(...firstPhaseLoggedRequests);
-        if (!searchResult.aggregations) {
-          throw new Error('Aggregations were missing on recent terms search result');
-        }
         logger.debug(`Time spent on composite agg: ${searchDuration}`);
 
         result.searchAfterTimes.push(searchDuration);
         result.errors.push(...searchErrors);
+        result.warningMessages.push(...searchWarnings);
+
+        if (!searchResult.aggregations) {
+          reportMissingAggregations({
+            searchResult,
+            searchErrors,
+            searchWarnings,
+            result,
+            inputIndex,
+            cpsLinkedProjects: sharedParams.cpsData?.linkedProjects,
+            unexpectedErrorMessage: 'Aggregations were missing on recent terms search result',
+          });
+          break;
+        }
 
         // If the aggregation returns no after_key it signals that we've paged through all results
         // and the current page is empty so we can immediately break.
@@ -349,6 +362,7 @@ export const createNewTermsAlertType = (): SecurityAlertType<
             searchResult: pageSearchResult,
             searchDuration: pageSearchDuration,
             searchErrors: pageSearchErrors,
+            searchWarnings: pageSearchWarnings,
             loggedRequests: pageSearchLoggedRequests = [],
           } = await singleSearchAfter({
             searchRequest: pageSearchRequest,
@@ -364,12 +378,22 @@ export const createNewTermsAlertType = (): SecurityAlertType<
           });
           result.searchAfterTimes.push(pageSearchDuration);
           result.errors.push(...pageSearchErrors);
+          result.warningMessages.push(...pageSearchWarnings);
           loggedRequests.push(...pageSearchLoggedRequests);
 
           logger.debug(`Time spent on phase 2 terms agg: ${pageSearchDuration}`);
 
           if (!pageSearchResult.aggregations) {
-            throw new Error('Aggregations were missing on new terms search result');
+            reportMissingAggregations({
+              searchResult: pageSearchResult,
+              searchErrors: pageSearchErrors,
+              searchWarnings: pageSearchWarnings,
+              result,
+              inputIndex,
+              cpsLinkedProjects: sharedParams.cpsData?.linkedProjects,
+              unexpectedErrorMessage: 'Aggregations were missing on new terms search result',
+            });
+            break;
           }
 
           // PHASE 3: For each term that is not in the history window, fetch the oldest document in
@@ -402,6 +426,7 @@ export const createNewTermsAlertType = (): SecurityAlertType<
               searchResult: docFetchSearchResult,
               searchDuration: docFetchSearchDuration,
               searchErrors: docFetchSearchErrors,
+              searchWarnings: docFetchSearchWarnings,
               loggedRequests: docFetchLoggedRequests = [],
             } = await singleSearchAfter({
               searchRequest: docFetchSearchRequest,
@@ -419,10 +444,20 @@ export const createNewTermsAlertType = (): SecurityAlertType<
             });
             result.searchAfterTimes.push(docFetchSearchDuration);
             result.errors.push(...docFetchSearchErrors);
+            result.warningMessages.push(...docFetchSearchWarnings);
             loggedRequests.push(...docFetchLoggedRequests);
 
             if (!docFetchSearchResult.aggregations) {
-              throw new Error('Aggregations were missing on document fetch search result');
+              reportMissingAggregations({
+                searchResult: docFetchSearchResult,
+                searchErrors: docFetchSearchErrors,
+                searchWarnings: docFetchSearchWarnings,
+                result,
+                inputIndex,
+                cpsLinkedProjects: sharedParams.cpsData?.linkedProjects,
+                unexpectedErrorMessage: 'Aggregations were missing on document fetch search result',
+              });
+              break;
             }
 
             // Collect rule execution metrics

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts

@@ -11,6 +11,7 @@ import type { NewTermsRuleParams } from '../../rule_schema';
 import type { GetFilterArgs } from '../utils/get_filter';
 import { getFilter } from '../utils/get_filter';
 import { singleSearchAfter } from '../utils/single_search_after';
+import { reportMissingAggregations } from '../utils/no_readable_shards';
 import {
   buildCompositeNewTermsAgg,
   buildCompositeDocFetchAgg,
@@ -160,6 +161,7 @@ const multiTermsCompositeNonRetryable = async ({
       searchResult: pageSearchResult,
       searchDuration: pageSearchDuration,
       searchErrors: pageSearchErrors,
+      searchWarnings: pageSearchWarnings,
       loggedRequests: pageSearchLoggedRequests = [],
     } = await singleSearchAfter({
       searchRequest,
@@ -178,12 +180,23 @@ const multiTermsCompositeNonRetryable = async ({
 
     result.searchAfterTimes.push(pageSearchDuration);
     result.errors.push(...pageSearchErrors);
+    result.warningMessages.push(...pageSearchWarnings);
     loggedRequests.push(...pageSearchLoggedRequests);
     logger.debug(`Time spent on phase 2 terms agg: ${pageSearchDuration}`);
 
     const pageSearchResultWithAggs = pageSearchResult as CompositeNewTermsAggResult;
     if (!pageSearchResultWithAggs.aggregations) {
-      throw new Error('Aggregations were missing on new terms search result');
+      reportMissingAggregations({
+        searchResult: pageSearchResult,
+        searchErrors: pageSearchErrors,
+        searchWarnings: pageSearchWarnings,
+        result,
+        inputIndex,
+        cpsLinkedProjects: sharedParams.cpsData?.linkedProjects,
+        unexpectedErrorMessage: 'Aggregations were missing on new terms search result',
+      });
+
+      return { loggedRequests };
     }
 
     // PHASE 3: For each term that is not in the history window, fetch the oldest document in
@@ -213,6 +226,7 @@ const multiTermsCompositeNonRetryable = async ({
         searchResult: docFetchSearchResult,
         searchDuration: docFetchSearchDuration,
         searchErrors: docFetchSearchErrors,
+        searchWarnings: docFetchSearchWarnings,
         loggedRequests: docFetchLoggedRequests = [],
       } = await singleSearchAfter({
         searchRequest: searchRequestPhase3,
@@ -230,12 +244,23 @@ const multiTermsCompositeNonRetryable = async ({
       });
       result.searchAfterTimes.push(docFetchSearchDuration);
       result.errors.push(...docFetchSearchErrors);
+      result.warningMessages.push(...docFetchSearchWarnings);
       loggedRequests.push(...docFetchLoggedRequests);
 
       const docFetchResultWithAggs = docFetchSearchResult as CompositeDocFetchAggResult;
 
       if (!docFetchResultWithAggs.aggregations) {
-        throw new Error('Aggregations were missing on document fetch search result');
+        reportMissingAggregations({
+          searchResult: docFetchSearchResult,
+          searchErrors: docFetchSearchErrors,
+          searchWarnings: docFetchSearchWarnings,
+          result,
+          inputIndex,
+          cpsLinkedProjects: sharedParams.cpsData?.linkedProjects,
+          unexpectedErrorMessage: 'Aggregations were missing on document fetch search result',
+        });
+
+        return { loggedRequests };
       }
 
       const bulkCreateResult = await createAlertsHook(docFetchResultWithAggs);

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.test.ts

@@ -6,8 +6,17 @@
  */
 
 import moment from 'moment';
-import { buildBucketHistoryFilter, filterBucketHistory } from './group_and_bulk_create';
+import type { PersistenceExecutorOptionsMock } from '@kbn/rule-registry-plugin/server/utils/create_persistence_rule_type_wrapper.mock';
+import { createPersistenceExecutorOptionsMock } from '@kbn/rule-registry-plugin/server/utils/create_persistence_rule_type_wrapper.mock';
+import {
+  buildBucketHistoryFilter,
+  filterBucketHistory,
+  groupAndBulkCreate,
+} from './group_and_bulk_create';
 import type { BucketHistory } from './group_and_bulk_create';
+import { getQueryRuleParams } from '../../../rule_schema/mocks';
+import { getSharedParamsMock } from '../../__mocks__/shared_params';
+import { getNoReadableShardsWarning } from '../../utils/no_readable_shards';
 
 describe('groupAndBulkCreate utils', () => {
   const bucketHistory: BucketHistory[] = [
@@ -56,3 +65,111 @@ describe('groupAndBulkCreate utils', () => {
     ]);
   });
 });
+
+describe('groupAndBulkCreate', () => {
+  const inputIndex = ['logs-m365_defender.incident-*'];
+  const linkedProjects = [
+    { id: 'project-1', alias: 'kayak', type: 'security', organization: 'org-1' },
+  ];
+  const sharedParams = getSharedParamsMock({
+    ruleParams: getQueryRuleParams({
+      alertSuppression: { groupBy: ['host.name'] },
+    }),
+    rewrites: { inputIndex },
+  });
+  let ruleServices: PersistenceExecutorOptionsMock;
+
+  const emptyResponse = ({ shardsTotal }: { shardsTotal: number }) => ({
+    took: 1,
+    timed_out: false,
+    _shards: { total: shardsTotal, successful: shardsTotal, failed: 0, skipped: 0 },
+    hits: { total: { value: 0, relation: 'eq' as const }, max_score: null, hits: [] },
+  });
+
+  const run = (params: Partial<Parameters<typeof groupAndBulkCreate>[0]> = {}) =>
+    groupAndBulkCreate({
+      sharedParams,
+      services: ruleServices,
+      filter: { match_all: {} },
+      buildReasonMessage: jest.fn().mockReturnValue('reason'),
+      groupByFields: ['host.name'],
+      eventsTelemetry: undefined,
+      isLoggedRequestsEnabled: false,
+      ...params,
+    });
+
+  beforeEach(() => {
+    ruleServices = createPersistenceExecutorOptionsMock();
+  });
+
+  it('reports a warning instead of failing when the search resolved to no shards', async () => {
+    ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
+      emptyResponse({ shardsTotal: 0 })
+    );
+
+    const result = await run();
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toEqual([]);
+    expect(result.createdSignalsCount).toBe(0);
+    expect(result.warningMessages).toEqual([getNoReadableShardsWarning({ inputIndex })]);
+  });
+
+  it('names the linked projects in the warning when the rule runs with CPS', async () => {
+    ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
+      emptyResponse({ shardsTotal: 0 })
+    );
+
+    const result = await run({
+      sharedParams: {
+        ...sharedParams,
+        cpsData: { resolvedExpression: '_alias:*', linkedProjects },
+      },
+    });
+
+    expect(result.warningMessages).toEqual([
+      getNoReadableShardsWarning({ inputIndex, cpsLinkedProjects: linkedProjects }),
+    ]);
+  });
+
+  it('fails with the shard failures as user errors instead of the generic aggregations error', async () => {
+    ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce({
+      ...emptyResponse({ shardsTotal: 1 }),
+      _shards: {
+        total: 1,
+        successful: 0,
+        failed: 1,
+        skipped: 0,
+        failures: [
+          {
+            shard: 0,
+            index: 'logs-m365_defender.incident-default',
+            node: 'node-1',
+            reason: { type: 'index_not_found_exception', reason: 'no such index' },
+          },
+        ],
+      },
+    });
+
+    const result = await run();
+
+    expect(result.success).toBe(false);
+    expect(result.userError).toBe(true);
+    expect(result.errors).toEqual([
+      'index: "logs-m365_defender.incident-default" reason: "no such index" type: "index_not_found_exception"',
+    ]);
+    expect(result.warningMessages).toEqual([]);
+  });
+
+  it('still fails with the generic error when shards were searched but aggregations are missing', async () => {
+    ruleServices.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
+      emptyResponse({ shardsTotal: 3 })
+    );
+
+    const result = await run();
+
+    expect(result.success).toBe(false);
+    expect(result.errors).toEqual(['expected to find aggregations on search result']);
+    expect(result.warningMessages).toEqual([]);
+  });
+});

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts

@@ -28,6 +28,7 @@ import { wrapSuppressedAlerts } from './wrap_suppressed_alerts';
 import { buildGroupByFieldAggregation } from './build_group_by_field_aggregation';
 import type { EventGroupingMultiBucketAggregationResult } from './build_group_by_field_aggregation';
 import { singleSearchAfter } from '../../utils/single_search_after';
+import { reportMissingAggregations } from '../../utils/no_readable_shards';
 import { bulkCreateWithSuppression } from '../../utils/bulk_create_with_suppression';
 import type { UnifiedQueryRuleParams } from '../../../rule_schema';
 import type { BuildReasonMessage } from '../../utils/reason_formatters';
@@ -200,7 +201,7 @@ export const groupAndBulkCreate = async ({
         runtimeMappings: sharedParams.runtimeMappings,
         additionalFilters: bucketHistoryFilter,
       });
-      const { searchResult, searchDuration, searchErrors, loggedRequests } =
+      const { searchResult, searchDuration, searchErrors, searchWarnings, loggedRequests } =
         await singleSearchAfter({
           searchRequest,
           services,
@@ -218,12 +219,23 @@ export const groupAndBulkCreate = async ({
       }
       toReturn.searchAfterTimes.push(searchDuration);
       toReturn.errors.push(...searchErrors);
+      toReturn.warningMessages.push(...searchWarnings);
       toReturn.totalEventsFound = getTotalHitsValue(searchResult.hits.total);
 
       const eventsByGroupResponseWithAggs =
         searchResult as EventGroupingMultiBucketAggregationResult;
       if (!eventsByGroupResponseWithAggs.aggregations) {
-        throw new Error('expected to find aggregations on search result');
+        reportMissingAggregations({
+          searchResult,
+          searchErrors,
+          searchWarnings,
+          result: toReturn,
+          inputIndex: sharedParams.inputIndex,
+          cpsLinkedProjects: sharedParams.cpsData?.linkedProjects,
+          unexpectedErrorMessage: 'expected to find aggregations on search result',
+        });
+
+        return toReturn;
       }
 
       const buckets = eventsByGroupResponseWithAggs.aggregations.eventGroups.buckets;