[Security Solution][CTI] Update legacy CTI signals to latest ECS threat fields

packages/kbn-securitysolution-t-grid/src/mock/mock_event_details.ts

@@ -39,12 +39,12 @@ export const eventHit = {
       '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go',
     ],
     'source.geo.location': [{ coordinates: [118.7778, 32.0617], type: 'Point' }],
-    'threat.indicator': [
+    'threat.enrichments': [
       {
         'matched.field': ['matched_field', 'other_matched_field'],
-        first_seen: ['2021-02-22T17:29:25.195Z'],
-        provider: ['yourself'],
-        type: ['custom'],
+        'indicator.first_seen': ['2021-02-22T17:29:25.195Z'],
+        'indicator.provider': ['yourself'],
+        'indicator.type': ['custom'],
         'matched.atomic': ['matched_atomic'],
         lazer: [
           {
@@ -57,9 +57,9 @@ export const eventHit = {
       },
       {
         'matched.field': ['matched_field_2'],
-        first_seen: ['2021-02-22T17:29:25.195Z'],
-        provider: ['other_you'],
-        type: ['custom'],
+        'indicator.first_seen': ['2021-02-22T17:29:25.195Z'],
+        'indicator.provider': ['other_you'],
+        'indicator.type': ['custom'],
         'matched.atomic': ['matched_atomic_2'],
         lazer: [
           {
@@ -259,70 +259,70 @@ export const eventDetailsFormattedFields = [
   },
   {
     category: 'threat',
-    field: 'threat.indicator.matched.field',
+    field: 'threat.enrichments.matched.field',
     values: ['matched_field', 'other_matched_field', 'matched_field_2'],
     originalValue: ['matched_field', 'other_matched_field', 'matched_field_2'],
     isObjectArray: false,
   },
   {
     category: 'threat',
-    field: 'threat.indicator.first_seen',
+    field: 'threat.enrichments.indicator.first_seen',
     values: ['2021-02-22T17:29:25.195Z'],
     originalValue: ['2021-02-22T17:29:25.195Z'],
     isObjectArray: false,
   },
   {
     category: 'threat',
-    field: 'threat.indicator.provider',
+    field: 'threat.enrichments.indicator.provider',
     values: ['yourself', 'other_you'],
     originalValue: ['yourself', 'other_you'],
     isObjectArray: false,
   },
   {
     category: 'threat',
-    field: 'threat.indicator.type',
+    field: 'threat.enrichments.indicator.type',
     values: ['custom'],
     originalValue: ['custom'],
     isObjectArray: false,
   },
   {
     category: 'threat',
-    field: 'threat.indicator.matched.atomic',
+    field: 'threat.enrichments.matched.atomic',
     values: ['matched_atomic', 'matched_atomic_2'],
     originalValue: ['matched_atomic', 'matched_atomic_2'],
     isObjectArray: false,
   },
   {
     category: 'threat',
-    field: 'threat.indicator.lazer.great.field',
+    field: 'threat.enrichments.lazer.great.field',
     values: ['grrrrr', 'grrrrr_2'],
     originalValue: ['grrrrr', 'grrrrr_2'],
     isObjectArray: false,
   },
   {
     category: 'threat',
-    field: 'threat.indicator.lazer.great.field.wowoe.fooooo',
+    field: 'threat.enrichments.lazer.great.field.wowoe.fooooo',
     values: ['grrrrr'],
     originalValue: ['grrrrr'],
     isObjectArray: false,
   },
   {
     category: 'threat',
-    field: 'threat.indicator.lazer.great.field.astring',
+    field: 'threat.enrichments.lazer.great.field.astring',
     values: ['cool'],
     originalValue: ['cool'],
     isObjectArray: false,
   },
   {
     category: 'threat',
-    field: 'threat.indicator.lazer.great.field.aNumber',
+    field: 'threat.enrichments.lazer.great.field.aNumber',
     values: ['1'],
     originalValue: ['1'],
     isObjectArray: false,
   },
   {
     category: 'threat',
-    field: 'threat.indicator.lazer.great.field.neat',
+    field: 'threat.enrichments.lazer.great.field.neat',
     values: ['true'],
     originalValue: ['true'],
     isObjectArray: false,

x-pack/plugins/security_solution/common/constants.ts

@@ -59,9 +59,9 @@ export const SAVED_OBJECTS_MANAGEMENT_FEATURE_ID = 'Saved Objects Management';
 export const DEFAULT_SPACE_ID = 'default';
 
 // Document path where threat indicator fields are expected. Fields are used
-// to enrich signals, and are copied to threat.indicator.
+// to enrich signals, and are copied to threat.enrichments.
 export const DEFAULT_INDICATOR_SOURCE_PATH = 'threatintel.indicator';
-export const INDICATOR_DESTINATION_PATH = 'threat.indicator';
+export const ENRICHMENT_DESTINATION_PATH = 'threat.enrichments';
 export const DEFAULT_THREAT_INDEX_KEY = 'securitySolution:defaultThreatIndex';
 export const DEFAULT_THREAT_INDEX_VALUE = ['filebeat-*'];
 

x-pack/plugins/security_solution/common/cti/constants.ts

@@ -5,36 +5,34 @@
  * 2.0.
  */
 
-import { INDICATOR_DESTINATION_PATH } from '../constants';
+import { ENRICHMENT_DESTINATION_PATH } from '../constants';
 
 export const MATCHED_ATOMIC = 'matched.atomic';
 export const MATCHED_FIELD = 'matched.field';
 export const MATCHED_ID = 'matched.id';
 export const MATCHED_TYPE = 'matched.type';
 export const INDICATOR_MATCH_SUBFIELDS = [MATCHED_ATOMIC, MATCHED_FIELD, MATCHED_TYPE];
 
-export const INDICATOR_MATCHED_ATOMIC = `${INDICATOR_DESTINATION_PATH}.${MATCHED_ATOMIC}`;
-export const INDICATOR_MATCHED_FIELD = `${INDICATOR_DESTINATION_PATH}.${MATCHED_FIELD}`;
-export const INDICATOR_MATCHED_TYPE = `${INDICATOR_DESTINATION_PATH}.${MATCHED_TYPE}`;
+export const INDICATOR_MATCHED_ATOMIC = `${ENRICHMENT_DESTINATION_PATH}.${MATCHED_ATOMIC}`;
+export const INDICATOR_MATCHED_FIELD = `${ENRICHMENT_DESTINATION_PATH}.${MATCHED_FIELD}`;
+export const INDICATOR_MATCHED_TYPE = `${ENRICHMENT_DESTINATION_PATH}.${MATCHED_TYPE}`;
 
 export const EVENT_DATASET = 'event.dataset';
-export const EVENT_REFERENCE = 'event.reference';
-export const EVENT_URL = 'event.url';
-export const PROVIDER = 'provider';
-export const FIRSTSEEN = 'first_seen';
 
-export const INDICATOR_DATASET = `${INDICATOR_DESTINATION_PATH}.${EVENT_DATASET}`;
-export const INDICATOR_EVENT_URL = `${INDICATOR_DESTINATION_PATH}.${EVENT_URL}`;
-export const INDICATOR_FIRSTSEEN = `${INDICATOR_DESTINATION_PATH}.${FIRSTSEEN}`;
-export const INDICATOR_LASTSEEN = `${INDICATOR_DESTINATION_PATH}.last_seen`;
-export const INDICATOR_PROVIDER = `${INDICATOR_DESTINATION_PATH}.${PROVIDER}`;
-export const INDICATOR_REFERENCE = `${INDICATOR_DESTINATION_PATH}.${EVENT_REFERENCE}`;
+export const FIRST_SEEN = 'indicator.first_seen';
+export const LAST_SEEN = 'indicator.last_seen';
+export const PROVIDER = 'indicator.provider';
+export const REFERENCE = 'indicator.reference';
+
+export const INDICATOR_FIRSTSEEN = `${ENRICHMENT_DESTINATION_PATH}.${FIRST_SEEN}`;
+export const INDICATOR_LASTSEEN = `${ENRICHMENT_DESTINATION_PATH}.${LAST_SEEN}`;
+export const INDICATOR_PROVIDER = `${ENRICHMENT_DESTINATION_PATH}.${PROVIDER}`;
+export const INDICATOR_REFERENCE = `${ENRICHMENT_DESTINATION_PATH}.${REFERENCE}`;
 
 export const CTI_ROW_RENDERER_FIELDS = [
   INDICATOR_MATCHED_ATOMIC,
   INDICATOR_MATCHED_FIELD,
   INDICATOR_MATCHED_TYPE,
-  INDICATOR_DATASET,
   INDICATOR_REFERENCE,
   INDICATOR_PROVIDER,
 ];

x-pack/plugins/security_solution/common/ecs/threat/index.ts

@@ -6,6 +6,7 @@
  */
 
 import { EventEcs } from '../event';
+import { UrlEcs } from '../url';
 
 interface ThreatMatchEcs {
   atomic?: string[];
@@ -15,13 +16,27 @@ interface ThreatMatchEcs {
   type?: string[];
 }
 
-export interface ThreatIndicatorEcs {
+export interface LegacyThreatIndicatorEcs {
+  domain?: string[];
   matched?: ThreatMatchEcs;
   event?: EventEcs & { reference?: string[] };
   provider?: string[];
   type?: string[];
 }
 
+export interface ThreatIndicatorEcs {
+  url?: UrlEcs;
+  provider?: string[];
+  reference?: string[];
+  type?: string[];
+}
+
+export interface ThreatEnrichmentEcs {
+  indicator?: ThreatIndicatorEcs;
+  matched?: ThreatMatchEcs;
+}
+
 export interface ThreatEcs {
-  indicator: ThreatIndicatorEcs[];
+  indicator?: LegacyThreatIndicatorEcs[];
+  enrichments?: ThreatEnrichmentEcs[];
 }

x-pack/plugins/security_solution/cypress/integration/detection_alerts/cti_enrichments.spec.ts

@@ -56,13 +56,13 @@ describe('CTI Enrichment', () => {
 
   it('Displays enrichment matched.* fields on the timeline', () => {
     const expectedFields = {
-      'threat.indicator.matched.atomic': getNewThreatIndicatorRule().atomic,
-      'threat.indicator.matched.type': 'indicator_match_rule',
-      'threat.indicator.matched.field': getNewThreatIndicatorRule().indicatorMappingField,
+      'threat.enrichments.matched.atomic': getNewThreatIndicatorRule().atomic,
+      'threat.enrichments.matched.type': 'indicator_match_rule',
+      'threat.enrichments.matched.field': getNewThreatIndicatorRule().indicatorMappingField,
     };
     const fields = Object.keys(expectedFields) as Array<keyof typeof expectedFields>;
 
-    addsFieldsToTimeline('threat.indicator.matched', fields);
+    addsFieldsToTimeline('threat.enrichments.matched', fields);
 
     fields.forEach((field) => {
       cy.get(TIMELINE_FIELD(field)).should('have.text', expectedFields[field]);
@@ -75,7 +75,7 @@ describe('CTI Enrichment', () => {
       {
         line: 3,
         text:
-          '    "indicator": "{\\"first_seen\\":\\"2021-03-10T08:02:14.000Z\\",\\"file\\":{\\"size\\":80280,\\"pe\\":{},\\"type\\":\\"elf\\",\\"hash\\":{\\"sha256\\":\\"a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3\\",\\"tlsh\\":\\"6D7312E017B517CC1371A8353BED205E9128223972AE35302E97528DF957703BAB2DBE\\",\\"ssdeep\\":\\"1536:87vbq1lGAXSEYQjbChaAU2yU23M51DjZgSQAvcYkFtZTjzBht5:8D+CAXFYQChaAUk5ljnQssL\\",\\"md5\\":\\"9b6c3518a91d23ed77504b5416bfb5b3\\"}},\\"type\\":\\"file\\",\\"event\\":{\\"reference\\":\\"https://urlhaus-api.abuse.ch/v1/download/a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3/\\",\\"ingested\\":\\"2021-03-10T14:51:09.809069Z\\",\\"created\\":\\"2021-03-10T14:51:07.663Z\\",\\"kind\\":\\"enrichment\\",\\"module\\":\\"threatintel\\",\\"category\\":\\"threat\\",\\"type\\":\\"indicator\\",\\"dataset\\":\\"threatintel.abusemalware\\"},\\"matched\\":{\\"atomic\\":\\"a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3\\",\\"field\\":\\"myhash.mysha256\\",\\"id\\":\\"84cf452c1e0375c3d4412cb550bd1783358468a3b3b777da4829d72c7d6fb74f\\",\\"index\\":\\"filebeat-7.12.0-2021.03.10-000001\\",\\"type\\":\\"indicator_match_rule\\"}}"',
+          '    "enrichments": "{\\"indicator\\":{\\"first_seen\\":\\"2021-03-10T08:02:14.000Z\\",\\"file\\":{\\"size\\":80280,\\"pe\\":{},\\"type\\":\\"elf\\",\\"hash\\":{\\"sha256\\":\\"a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3\\",\\"tlsh\\":\\"6D7312E017B517CC1371A8353BED205E9128223972AE35302E97528DF957703BAB2DBE\\",\\"ssdeep\\":\\"1536:87vbq1lGAXSEYQjbChaAU2yU23M51DjZgSQAvcYkFtZTjzBht5:8D+CAXFYQChaAUk5ljnQssL\\",\\"md5\\":\\"9b6c3518a91d23ed77504b5416bfb5b3\\"}},\\"type\\":\\"file\\"},\\"matched\\":{\\"atomic\\":\\"a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3\\",\\"field\\":\\"myhash.mysha256\\",\\"id\\":\\"84cf452c1e0375c3d4412cb550bd1783358468a3b3b777da4829d72c7d6fb74f\\",\\"index\\":\\"filebeat-7.12.0-2021.03.10-000001\\",\\"type\\":\\"indicator_match_rule\\"}}"',
       },
       { line: 2, text: '  }' },
     ];
@@ -97,34 +97,23 @@ describe('CTI Enrichment', () => {
 
   it('Displays threat indicator details on the threat intel tab', () => {
     const expectedThreatIndicatorData = [
-      { field: 'event.category', value: 'threat' },
-      { field: 'event.created', value: '2021-03-10T14:51:07.663Z' },
-      { field: 'event.dataset', value: 'threatintel.abusemalware' },
-      { field: 'event.ingested', value: '2021-03-10T14:51:09.809069Z' },
-      { field: 'event.kind', value: 'enrichment' },
-      { field: 'event.module', value: 'threatintel' },
+      { field: 'indicator.file.hash.md5', value: '9b6c3518a91d23ed77504b5416bfb5b3' },
       {
-        field: 'event.reference',
-        value:
-          'https://urlhaus-api.abuse.ch/v1/download/a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3/(opens in a new tab or window)',
-      },
-      { field: 'event.type', value: 'indicator' },
-      { field: 'file.hash.md5', value: '9b6c3518a91d23ed77504b5416bfb5b3' },
-      {
-        field: 'file.hash.sha256',
+        field: 'indicator.file.hash.sha256',
         value: 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3',
       },
       {
-        field: 'file.hash.ssdeep',
+        field: 'indicator.file.hash.ssdeep',
         value: '1536:87vbq1lGAXSEYQjbChaAU2yU23M51DjZgSQAvcYkFtZTjzBht5:8D+CAXFYQChaAUk5ljnQssL',
       },
       {
-        field: 'file.hash.tlsh',
+        field: 'indicator.file.hash.tlsh',
         value: '6D7312E017B517CC1371A8353BED205E9128223972AE35302E97528DF957703BAB2DBE',
       },
-      { field: 'file.size', value: '80280' },
-      { field: 'file.type', value: 'elf' },
-      { field: 'first_seen', value: '2021-03-10T08:02:14.000Z' },
+      { field: 'indicator.file.size', value: '80280' },
+      { field: 'indicator.file.type', value: 'elf' },
+      { field: 'indicator.first_seen', value: '2021-03-10T08:02:14.000Z' },
+      { field: 'indicator.type', value: 'file' },
       {
         field: 'matched.atomic',
         value: 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3',
@@ -136,7 +125,6 @@ describe('CTI Enrichment', () => {
       },
       { field: 'matched.index', value: 'filebeat-7.12.0-2021.03.10-000001' },
       { field: 'matched.type', value: 'indicator_match_rule' },
-      { field: 'type', value: 'file' },
     ];
 
     expandFirstAlert();

x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts

@@ -490,8 +490,6 @@ describe('indicator match', () => {
 
       it('Investigate alert in timeline', () => {
         const accessibilityText = `Press enter for options, or press space to begin dragging.`;
-        const threatIndicatorPath =
-          '../../../x-pack/test/security_solution_cypress/es_archives/threat_indicator/data.json';
 
         loadPrepackagedTimelineTemplates();
 
@@ -506,27 +504,21 @@ describe('indicator match', () => {
         cy.get(PROVIDER_BADGE).should('have.length', 3);
         cy.get(PROVIDER_BADGE).should(
           'have.text',
-          `threat.indicator.matched.atomic: "${
+          `threat.enrichments.matched.atomic: "${
             getNewThreatIndicatorRule().atomic
-          }"threat.indicator.matched.type: "indicator_match_rule"threat.indicator.matched.field: "${
+          }"threat.enrichments.matched.type: "indicator_match_rule"threat.enrichments.matched.field: "${
             getNewThreatIndicatorRule().indicatorMappingField
           }"`
         );
 
-        cy.readFile(threatIndicatorPath).then((threatIndicator) => {
-          cy.get(INDICATOR_MATCH_ROW_RENDER).should(
-            'have.text',
-            `threat.indicator.matched.field${
-              getNewThreatIndicatorRule().indicatorMappingField
-            }${accessibilityText}matched${getNewThreatIndicatorRule().indicatorMappingField}${
-              getNewThreatIndicatorRule().atomic
-            }${accessibilityText}threat.indicator.matched.typeindicator_match_rule${accessibilityText}fromthreat.indicator.event.dataset${
-              threatIndicator.value.source.event.dataset
-            }${accessibilityText}:threat.indicator.event.reference${
-              threatIndicator.value.source.event.reference
-            }(opens in a new tab or window)${accessibilityText}`
-          );
-        });
+        cy.get(INDICATOR_MATCH_ROW_RENDER).should(
+          'have.text',
+          `threat.enrichments.matched.field${
+            getNewThreatIndicatorRule().indicatorMappingField
+          }${accessibilityText}matched${getNewThreatIndicatorRule().indicatorMappingField}${
+            getNewThreatIndicatorRule().atomic
+          }${accessibilityText}threat.enrichments.matched.typeindicator_match_rule${accessibilityText}`
+        );
       });
     });
 

x-pack/plugins/security_solution/public/common/components/event_details/__mocks__/index.ts

@@ -659,14 +659,14 @@ export const mockAlertDetailsData = [
   },
   {
     category: 'threat',
-    field: 'threat.indicator',
-    values: [`{"first_seen":"2021-03-25T18:17:00.000Z"}`],
-    originalValue: [`{"first_seen":"2021-03-25T18:17:00.000Z"}`],
+    field: 'threat.enrichments',
+    values: [`{"indicator":{"first_seen":"2021-03-25T18:17:00.000Z"}}`],
+    originalValue: [`{"indicator":{"first_seen":"2021-03-25T18:17:00.000Z"}}`],
   },
   {
     category: 'threat',
-    field: 'threat.indicator.matched',
-    values: `["file", "url"]`,
-    originalValue: ['file', 'url'],
+    field: 'threat.enrichments.matched.field',
+    values: ['host.name'],
+    originalValue: ['host.name'],
   },
 ];