[TI AbuseCH] Fix field mappings

packages/ti_abusech/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.2.1"
+  changes:
+    - description: Fix field mapping conflicts in `threat.indicator.file.x509.not_before/not_after`
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2893
 - version: "1.2.0"
   changes:
     - description: Update to ECS 8.0

packages/ti_abusech/data_stream/malware/_dev/test/pipeline/test-common-config.yml

@@ -1,5 +1,3 @@
-dynamic_fields:
-  event.ingested: ".*"
 fields:
   tags:
     - preserve_original_event

packages/ti_abusech/data_stream/malware/elasticsearch/ingest_pipeline/default.yml

@@ -4,9 +4,6 @@ processors:
   ####################
   # Event ECS fields #
   ####################
-  - set:
-      field: event.ingested
-      value: "{{_ingest.timestamp}}"
   - set:
       field: ecs.version
       value: "8.0.0"

packages/ti_abusech/data_stream/malwarebazaar/_dev/test/pipeline/test-common-config.yml

@@ -1,5 +1,3 @@
-dynamic_fields:
-  event.ingested: ".*"
 fields:
   tags:
     - preserve_original_event

packages/ti_abusech/data_stream/malwarebazaar/_dev/test/pipeline/test-malwarebazaar-ndjson.log-expected.json

@@ -19,7 +19,6 @@
             },
             "event": {
                 "category": "threat",
-                "ingested": "2022-01-24T02:54:30.644972581Z",
                 "kind": "enrichment",
                 "original": "{\"sha256_hash\":\"5bce7d528c1363104a93fbb5a7fa9bdd991ce929cc09cc7fb29052a68d4fd24b\",\"sha3_384_hash\":\"3b454eb6421d17d093f19292b64d30bf918cb91e9322d0e2d2512857997f574ea2ca5b005133c16f6c33c7cee9c1bd0e\",\"sha1_hash\":\"a71fd0504821092e003f350080a6bcc5fa6a972e\",\"md5_hash\":\"0af07660056a692b7cb82fa329221ddd\",\"first_seen\":\"2021-04-06 20:34:58\",\"last_seen\":null,\"file_name\":\"SALM0BRU.exe\",\"file_size\":399872,\"file_type_mime\":\"application/x-dosexec\",\"file_type\":\"exe\",\"reporter\":\"James_inthe_box\",\"origin_country\":\"US\",\"anonymous\":0,\"signature\":null,\"imphash\":\"f34d5f2d4577ed6d9ceec516c1f5a744\",\"tlsh\":\"F9848B24AF932F9BC6CCC1FE50C2D165C9A9F85DD2B1251A73B6CB89FE00544ED2C686\",\"telfhash\":null,\"ssdeep\":\"3072:DsPPK3p+8r5igrL1Tq50cVBDmDJhE9yV4veedHrP6FXK7:D+PL8bronBDmDJ69JeedHriFG\",\"tags\":[\"exe\"],\"code_sign\":[],\"intelligence\":{\"clamav\":null,\"downloads\":\"15\",\"uploads\":\"1\",\"mail\":null}}",
                 "type": "indicator"
@@ -82,7 +81,6 @@
             },
             "event": {
                 "category": "threat",
-                "ingested": "2022-01-24T02:54:30.644975757Z",
                 "kind": "enrichment",
                 "original": "{\"sha256_hash\":\"83d0429a2c5f1b611ebc30391eeeb75bebb51212ee1af51dbcf2624b48f9d27f\",\"sha3_384_hash\":\"0a1536add280715320040d5ac5340d3b205d90045ff5c90993b8e909edb9b3e9338b3ffbb3febcaf82584d00d516e8c7\",\"sha1_hash\":\"c454be4eb0892d61a4ad6bac16f97724e73cd795\",\"md5_hash\":\"296aad7075596d21516b30bfbc17fcac\",\"first_seen\":\"2021-04-06 20:32:25\",\"last_seen\":null,\"file_name\":\"PO_NO.ENQUIRY-210604.zip\",\"file_size\":476768,\"file_type_mime\":\"application/zip\",\"file_type\":\"zip\",\"reporter\":\"GovCERT_CH\",\"origin_country\":\"US\",\"anonymous\":0,\"signature\":null,\"imphash\":null,\"tlsh\":\"74A4233B9A6D5CA02B224AA69F37537D13A8406300944EAEFD375CA431583056B9F6FF\",\"telfhash\":null,\"ssdeep\":\"12288:j++y4mulTPaYJSaHwvJblQpLGwYeHU9vPpNGd+Zr:j3HPaMtQxblje01pNHZr\",\"tags\":null,\"code_sign\":[],\"intelligence\":{\"clamav\":null,\"downloads\":\"11\",\"uploads\":\"1\",\"mail\":null}}",
                 "type": "indicator"
@@ -145,7 +143,6 @@
             },
             "event": {
                 "category": "threat",
-                "ingested": "2022-01-24T02:54:30.644976999Z",
                 "kind": "enrichment",
                 "original": "{\"sha256_hash\":\"f4910ea08d14eeb634084de47cf590d4dc5e554552f111da20d22ae71d7b425b\",\"sha3_384_hash\":\"ee7586cb085fde3c14c9c1bea4635ccb30b1af2020f64e87a9983e61b05026ec9b35255670a3d9ecaab436c4ba302dcc\",\"sha1_hash\":\"bf103996196df8255881127dee103c22fc12bef3\",\"md5_hash\":\"a4838dd31c672122441bebcbf7e9d277\",\"first_seen\":\"2021-04-06 20:12:29\",\"last_seen\":null,\"file_name\":\"DropDll.dat\",\"file_size\":435926,\"file_type_mime\":\"application/x-dosexec\",\"file_type\":\"dll\",\"reporter\":\"DmitriyMelikov\",\"origin_country\":\"DE\",\"anonymous\":0,\"signature\":\"Hancitor\",\"imphash\":\"0b5a952a025c2783c3126cdb9bef2844\",\"tlsh\":\"0C947D11BA96C473E572163008399F6A17BE7A900B704BDBE3CC097E4E755C24B36BA7\",\"telfhash\":null,\"ssdeep\":\"12288:L2X/txpFDEVkUNglTovKfoLy+hqK/cEUMMlGOG:RzglgLm/9lGOG\",\"tags\":[\"Hancitor\"],\"code_sign\":[],\"intelligence\":{\"clamav\":null,\"downloads\":\"30\",\"uploads\":\"1\",\"mail\":null}}",
                 "type": "indicator"
@@ -214,7 +211,6 @@
             },
             "event": {
                 "category": "threat",
-                "ingested": "2022-01-24T02:54:30.644978112Z",
                 "kind": "enrichment",
                 "original": "{\"sha256_hash\":\"e45ffc61a85c2f5c0cbe9376ff215cad324bf14f925bf52ec0d2949f7d235a00\",\"sha3_384_hash\":\"788f61cf45bbc8cad5775de18d0d5f42c4e028af0aaa34c570645efc96af8ebc3d7fe330aaf22ef34d35360bbd4a708c\",\"sha1_hash\":\"a68ca1b41cb93fe2879bb3baeb8e19990758f099\",\"md5_hash\":\"8d7c8b55ac49d241fb7f75a27a5ef8d5\",\"first_seen\":\"2021-04-06 20:07:59\",\"last_seen\":null,\"file_name\":\"vabsheche.py\",\"file_size\":11717,\"file_type_mime\":\"text/x-script.python\",\"file_type\":\"unknown\",\"reporter\":\"ArkbirdDevil\",\"origin_country\":\"FR\",\"anonymous\":0,\"signature\":null,\"imphash\":null,\"tlsh\":\"AE3222515C6A881A03B3C66F7992B844FB588303C7116607F6FC86782F79568CAF1BBD\",\"telfhash\":null,\"ssdeep\":\"192:z7X/yHo/yz/yBKiSOINLyhQMYd+LiTfq6LTf3ZoTta3Grj6rg2:z7CIKnNNLwufPfAPq7\",\"tags\":[\"backdoor\",\"python\"],\"code_sign\":[],\"intelligence\":{\"clamav\":null,\"downloads\":\"27\",\"uploads\":\"1\",\"mail\":null}}",
                 "type": "indicator"
@@ -277,7 +273,6 @@
             },
             "event": {
                 "category": "threat",
-                "ingested": "2022-01-24T02:54:30.644979152Z",
                 "kind": "enrichment",
                 "original": "{\"sha256_hash\":\"42f5f5474431738f91f612d9765b3fc9b85a547274ea64aa034298ad97ad28f4\",\"sha3_384_hash\":\"752e5d56a166227d06f8cbd40cd3f693f543f9c3f798c673c1430957bb7e149a12d9158138fa449479105f472e70f68f\",\"sha1_hash\":\"e8378aede9f26f09b7d503d79a05d67612be15f6\",\"md5_hash\":\"fe185f106730583156f39233f77f8019\",\"first_seen\":\"2021-04-06 20:00:48\",\"last_seen\":null,\"file_name\":\"42f5f5474431738f91f612d9765b3fc9b85a547274ea64aa034298ad97ad28f4.bin\",\"file_size\":7929856,\"file_type_mime\":\"application/msword\",\"file_type\":\"docx\",\"reporter\":\"ArkbirdDevil\",\"origin_country\":\"FR\",\"anonymous\":0,\"signature\":null,\"imphash\":null,\"tlsh\":\"13863341B085EE2EE2CA41BA0DA9C2BD43B63D131E054F677269B72D3EB76E0E7D4144\",\"telfhash\":null,\"ssdeep\":\"196608:KQaeKLOiBEp+uc+iuYmbMdHmN1Rwyd2jecXeaH1pHE+2:oeIOTp+p+iNJC1ChjhXZ1pHz2\",\"tags\":[\"maldoc\"],\"code_sign\":[],\"intelligence\":{\"clamav\":null,\"downloads\":\"21\",\"uploads\":\"1\",\"mail\":null}}",
                 "type": "indicator"
@@ -341,7 +336,6 @@
             },
             "event": {
                 "category": "threat",
-                "ingested": "2022-01-24T02:54:30.644980242Z",
                 "kind": "enrichment",
                 "original": "{\"sha256_hash\":\"2d705f0b76f24a18e08163db2f187140ee9f03e43697a9ea0d840c829692d43c\",\"sha3_384_hash\":\"c82132559381b7b3b184b4ce8c7a58c301a46001621f346b637139f5987dee968ae2ef009a17b2388852b2db15a45b58\",\"sha1_hash\":\"b2da45913353bfc66d189455f9ad80ef26968143\",\"md5_hash\":\"70da6872b6b2da9ddc94d14b02302917\",\"first_seen\":\"2021-04-06 19:58:50\",\"last_seen\":null,\"file_name\":\"winlog.wll\",\"file_size\":131584,\"file_type_mime\":\"application/x-dosexec\",\"file_type\":\"dll\",\"reporter\":\"ArkbirdDevil\",\"origin_country\":\"FR\",\"anonymous\":0,\"signature\":null,\"imphash\":\"6476b7c4dd55eafbdf922a7ba1e2d5f9\",\"tlsh\":\"A2D38C067790C071DAAF013908799E624B7F7D70DDB49D8B77841A8E69342D0AF3AB27\",\"telfhash\":null,\"ssdeep\":\"1536:2NVi7z0r0lJRn6I8+YDgr1fnWG5Ff0+adgBYlCtMiQMX1c0E4JsWjcdonPv870E1:YM7zh8+Cofnp5eRm6riQ6OZoPv870E\",\"tags\":[\"apt\",\"tonto\"],\"code_sign\":[],\"intelligence\":{\"clamav\":null,\"downloads\":\"30\",\"uploads\":\"1\",\"mail\":null}}",
                 "type": "indicator"
@@ -404,7 +398,6 @@
             },
             "event": {
                 "category": "threat",
-                "ingested": "2022-01-24T02:54:30.644981284Z",
                 "kind": "enrichment",
                 "original": "{\"sha256_hash\":\"30787f32adc487311d764b19d4504fdeab08c0d385e2fa065bd8d5836c031606\",\"sha3_384_hash\":\"a3ec981ed158fe08cc2cd97303807cfbed147e59ccfd92fcaa9395c5718b4d9b892d6e9fa6337f5976dc1bd042562fe4\",\"sha1_hash\":\"3d613d5678e43faeea1c636185a0b4c3ec80e742\",\"md5_hash\":\"de80e1d7d9f5b1c64ec9f8d4f5063989\",\"first_seen\":\"2021-04-06 19:58:44\",\"last_seen\":null,\"file_name\":\"30787f32adc487311d764b19d4504fdeab08c0d385e2fa065bd8d5836c031606.bin.sample\",\"file_size\":1088000,\"file_type_mime\":\"application/msword\",\"file_type\":\"docx\",\"reporter\":\"DmitriyMelikov\",\"origin_country\":\"DE\",\"anonymous\":0,\"signature\":null,\"imphash\":null,\"tlsh\":\"8635D001BA82C573D5621A35083ADBAA177E7D604F704ADBB3C83B2E5D355C14B32BA7\",\"telfhash\":null,\"ssdeep\":\"24576:WKEiZxl3A4yJJG2dPQQCthXzglgLm/9lGO:WKEGByvGOQQC/XElga/9lGO\",\"tags\":null,\"code_sign\":[],\"intelligence\":{\"clamav\":null,\"downloads\":\"32\",\"uploads\":\"1\",\"mail\":null}}",
                 "type": "indicator"
@@ -471,7 +464,6 @@
             },
             "event": {
                 "category": "threat",
-                "ingested": "2022-01-24T02:54:30.644982321Z",
                 "kind": "enrichment",
                 "original": "{\"sha256_hash\":\"84f983067868de50e5b1553782c056c1f5b5118bb2084473ca4b6908f221cd3b\",\"sha3_384_hash\":\"138dc28a74d15c1f9797ce732e99097c8c6db4549cb17cb7b20c1c6738a170328e45aea2d4c3b593912f14a97f521c1d\",\"sha1_hash\":\"00b52e8ca1785d5086703ad8cff1d28fc3354934\",\"md5_hash\":\"2759c73c986c6a757bf9d25621c5595a\",\"first_seen\":\"2021-04-06 19:52:32\",\"last_seen\":null,\"file_name\":\"Purchase Order.8000.scan.pdf...exe\",\"file_size\":752128,\"file_type_mime\":\"application/x-dosexec\",\"file_type\":\"exe\",\"reporter\":\"James_inthe_box\",\"origin_country\":\"FR\",\"anonymous\":0,\"signature\":\"SnakeKeylogger\",\"imphash\":\"f34d5f2d4577ed6d9ceec516c1f5a744\",\"tlsh\":\"23F4AE212684C9C0D93E67B4D43584F003BABD16D631F69F6E887C693EB32D2D63B646\",\"telfhash\":null,\"ssdeep\":\"12288:8t11ulRZRLZNh4YeX6f6XmwNShqE73YXy7moh:S11gZpZNmBX06WmAcy7m0\",\"tags\":[\"exe\",\"SnakeKeylogger\"],\"code_sign\":[],\"intelligence\":{\"clamav\":null,\"downloads\":\"38\",\"uploads\":\"1\",\"mail\":{\"Generic\":\"low\"}}}",
                 "type": "indicator"
@@ -543,7 +535,6 @@
             },
             "event": {
                 "category": "threat",
-                "ingested": "2022-01-24T02:54:30.644983396Z",
                 "kind": "enrichment",
                 "original": "{\"sha256_hash\":\"0661d87116f44cbd5b5c6bec7fb06c4e5cd5b6ecbc5455d959e65f1ee46c54c8\",\"sha3_384_hash\":\"ed5d03454121d81adf65a01ba90af81b1a7cea052709c22bb9170508069d17242861f85e5546b2cc3efb07c10926368c\",\"sha1_hash\":\"a34fd5e57d75d17bc2d84055ca4752e5ee2e92f5\",\"md5_hash\":\"596b3dbf07a287dcf76860b5e54762c3\",\"first_seen\":\"2021-04-06 19:47:13\",\"last_seen\":null,\"file_name\":\"New Order PO#121012020_____PDF_______.exe\",\"file_size\":836096,\"file_type_mime\":\"application/x-dosexec\",\"file_type\":\"exe\",\"reporter\":\"James_inthe_box\",\"origin_country\":\"FR\",\"anonymous\":0,\"signature\":\"AgentTesla\",\"imphash\":\"f34d5f2d4577ed6d9ceec516c1f5a744\",\"tlsh\":\"A505CF712694C9A4FABD53B80434403007F5FE42E232FA9A6FD17C993E72782DA3B655\",\"telfhash\":null,\"ssdeep\":\"12288:qRedcNeqimzAEmN03VgdZfBOMx+RVBM7pdWje9ppB5nAZGNY2:ZaNeqikqN0udZfBFUYp55nFN\",\"tags\":[\"AgentTesla\",\"exe\"],\"code_sign\":[],\"intelligence\":{\"clamav\":null,\"downloads\":\"40\",\"uploads\":\"1\",\"mail\":{\"Generic\":\"low\"}}}",
                 "type": "indicator"

packages/ti_abusech/data_stream/malwarebazaar/elasticsearch/ingest_pipeline/default.yml

@@ -4,9 +4,6 @@ processors:
   ####################
   # Event ECS fields #
   ####################
-  - set:
-      field: event.ingested
-      value: "{{_ingest.timestamp}}"
   - set:
       field: ecs.version
       value: "8.0.0"

packages/ti_abusech/data_stream/malwarebazaar/fields/ecs.yml

@@ -55,23 +55,17 @@
 - external: ecs
   name: threat.indicator.file.elf.telfhash
 - name: threat.indicator.file.x509.subject.common_name
-  type: keyword
-  description: "List of common names (CN) of subject."
+  external: ecs
 - name: threat.indicator.file.x509.issuer.common_name
-  type: keyword
-  description: "List of common name (CN) of issuing certificate authority."
+  external: ecs
 - name: threat.indicator.file.x509.public_key_algorithm
-  type: keyword
-  description: "Algorithm used to generate the public key."
+  external: ecs
 - name: threat.indicator.file.x509.not_before
-  type: keyword
-  description: "Time at which the certificate is first considered valid."
+  external: ecs
 - name: threat.indicator.file.x509.not_after
-  type: keyword
-  description: "Time at which the certificate is no longer considered valid."
+  external: ecs
 - name: threat.indicator.file.x509.serial_number
-  type: keyword
-  description: "Unique serial number issued by the certificate authority."
+  external: ecs
 - external: ecs
   name: threat.indicator.provider
 - external: ecs