[TI_Cybersixgill] Change from API to TAXII for the integration

packages/ti_cybersixgill/_dev/build/docs/README.md

@@ -1,14 +1,12 @@
-# Cybersixgill Webhook Integration
+# Cybersixgill Darkfeed TAXII Integration
 
-This integration creates an HTTP listener that accepts incoming HTTP requests from Cybersixgill integration script which retrieves indicators from [Cybersixgill Darkfeed](https://www.cybersixgill.com/products/darkfeed/).
+This integration connects with the commercial [Cybersixgill Darkfeed](https://www.cybersixgill.com/products/darkfeed/) TAXII server.
 
 ## Logs
 
 ### Threat
 
-The Cybersixgill integration works together with a python script provided by Cybersixgill which usually runs on the same host as the Elastic Agent, polling the Cybersixgill API using a scheduler like systemd, cron, or Windows Task Scheduler; then it forwards the results to Elastic Agent over HTTP(s) on the same host.
-
-All relevant documentation on how to install and configure the Python script is provided in its README.(https://github.com/elastic/filebeat-cybersixgill-integration#readme).
+The Cybersixgill Darkfeed integration collects threat intelligence from the Darkfeed TAXII service available using the credentials provided from Cybersixgill.
 
 {{fields "threat"}}
 

packages/ti_cybersixgill/_dev/deploy/docker/docker-compose.yml

@@ -1,25 +1,14 @@
 version: "2.3"
 services:
-  cybersixgill-webhook-http:
+  cybersixgill-http:
     image: docker.elastic.co/observability/stream:v0.6.1
+    ports:
+      - 8080
     volumes:
-      - ./sample_logs:/sample_logs:ro
+      - ./files:/files:ro
     environment:
-      - STREAM_PROTOCOL=webhook
-      - STREAM_ADDR=http://elastic-agent:9080/cybersixgill
-      - STREAM_WEBHOOK_HEADER=Content-Type=application/json
-      - STREAM_USERNAME=abc123
-      - STREAM_PASSWORD=abc123
-    command: log --start-signal=SIGHUP --webhook-username=abc123 --webhook-password=abc123 --delay=5s /sample_logs/test-cybersixgill-threat.ndjson.log
-  cybersixgill-webhook-https:
-    image: docker.elastic.co/observability/stream:v0.6.1
-    volumes:
-      - ./sample_logs:/sample_logs:ro
-    environment:
-      - STREAM_PROTOCOL=webhook
-      - STREAM_INSECURE=true
-      - STREAM_ADDR=https://elastic-agent:7443/cybersixgill
-      - STREAM_WEBHOOK_HEADER=Content-Type=application/json
-      - STREAM_USERNAME=abc123
-      - STREAM_PASSWORD=abc123
-    command: log --start-signal=SIGHUP --webhook-username=abc123 --webhook-password=abc123 --delay=5s /sample_logs/test-cybersixgill-threat.ndjson.log
+      PORT: 8080
+    command:
+      - http-server
+      - --addr=:8080
+      - --config=/files/config.yml

packages/ti_cybersixgill/_dev/deploy/docker/files/config.yml

@@ -0,0 +1,150 @@
+rules:
+  - path: /taxii/sixgill-taxii/collections/102/objects
+    methods: ["GET"]
+    request_headers:
+      Authorization:
+        - "Basic dGVzdDp0ZXN0"
+    query_params:
+      match[type]: "indicator"
+    responses:
+      - status_code: 200
+        headers:
+          X-TAXII-Date-Added-Last: "2022-03-02T12:44:27.839Z"
+          Content-Type: "application/taxii+json;version=2.1"
+        body: |-
+          {
+            "id": "bundle--53bf8cb5-4ea7-439b-a27d-3f86bfcca0ea",
+            "objects": [
+                  {
+                    "confidence": 80,
+                    "created": "2021-12-07T18:04:26.451Z",
+                    "description": "Malware available for download from file-sharing sites",
+                    "extensions": {
+                        "extension-definition--3de9ff00-174d-4d41-87c9-05a27a7e117c": {
+                            "extension_type": "toplevel-property-extension"
+                        }
+                    },
+                    "external_references": [
+                        {
+                            "description": "Mitre attack tactics and technique reference",
+                            "mitre_attack_tactic": "Build Capabilities",
+                            "mitre_attack_tactic_id": "TA0024",
+                            "mitre_attack_tactic_url": "https://attack.mitre.org/tactics/TA0024/",
+                            "mitre_attack_technique": "Obtain/re-use payloads",
+                            "mitre_attack_technique_id": "T1346",
+                            "mitre_attack_technique_url": "https://attack.mitre.org/techniques/T1346/",
+                            "source_name": "mitre-attack"
+                        }
+                    ],
+                    "id": "indicator--ade54b36-752d-4107-a2ed-dd666fa1cb85",
+                    "indicator_types": [
+                        "malicious-activity"
+                    ],
+                    "lang": "ru",
+                    "modified": "2021-12-07T18:04:26.451Z",
+                    "name": "https://ru.scribd.com/user/456422024/ForkLog#from_embed",
+                    "pattern": "[url:value = 'https://ru.scribd.com/user/456422024/ForkLog#from_embed']",
+                    "pattern_type": "stix",
+                    "sixgill_actor": "CoinProject.info",
+                    "sixgill_confidence": 80,
+                    "sixgill_feedid": "darkfeed_010",
+                    "sixgill_feedname": "malware_download_urls",
+                    "sixgill_postid": "3f8c56e4cf6407ee7608e0f605503cb1e3fcedb9",
+                    "sixgill_posttitle": "Банковский регулятор США напомнил о рисках внедрения криптовалют",
+                    "sixgill_severity": 70,
+                    "sixgill_source": "forum_hyipinvest",
+                    "spec_version": "2.1",
+                    "type": "indicator",
+                    "valid_from": "2021-12-07T14:52:00Z"
+                },
+                {
+                    "confidence": 90,
+                    "created": "2021-12-07T22:48:59.141Z",
+                    "description": "Shell access to this domain is being sold on dark web markets",
+                    "extensions": {
+                        "extension-definition--3de9ff00-174d-4d41-87c9-05a27a7e117c": {
+                            "extension_type": "toplevel-property-extension"
+                        }
+                    },
+                    "external_references": [
+                        {
+                            "description": "Mitre attack tactics and technique reference",
+                            "mitre_attack_tactic": "Establish & Maintain Infrastructure",
+                            "mitre_attack_tactic_id": "TA0022",
+                            "mitre_attack_tactic_url": "https://attack.mitre.org/tactics/TA0022/",
+                            "mitre_attack_technique": "Compromise 3rd party infrastructure to support delivery",
+                            "mitre_attack_technique_id": "T1334",
+                            "mitre_attack_technique_url": "https://attack.mitre.org/techniques/T1334/",
+                            "source_name": "mitre-attack"
+                        }
+                    ],
+                    "id": "indicator--557a6021-b0c0-441a-8fba-e8a734f19ada",
+                    "indicator_types": [
+                        "compromised"
+                    ],
+                    "lang": "en",
+                    "modified": "2021-12-07T22:48:59.141Z",
+                    "name": "sdbpibandung.sch.id",
+                    "pattern": "[domain-name:value = 'sdbpibandung.sch.id']",
+                    "pattern_type": "stix",
+                    "sixgill_actor": "enginewo",
+                    "sixgill_confidence": 90,
+                    "sixgill_feedid": "darkfeed_001",
+                    "sixgill_feedname": "compromised_sites",
+                    "sixgill_postid": "955f5379c2828ce483b74a671e498a5f69f9ea36",
+                    "sixgill_posttitle": "Beranda       http://sdbpibandung.sch.id",
+                    "sixgill_severity": 70,
+                    "sixgill_source": "market_magbo",
+                    "spec_version": "2.1",
+                    "type": "indicator",
+                    "valid_from": "2021-12-07T22:43:29Z"
+                },
+                {
+                    "confidence": 70,
+                    "created": "2021-12-07T13:58:01.596Z",
+                    "description": "Hash attributed to malware that was discovered in the dark and deep web",
+                    "extensions": {
+                        "extension-definition--3de9ff00-174d-4d41-87c9-05a27a7e117c": {
+                            "extension_type": "toplevel-property-extension"
+                        }
+                    },
+                    "external_references": [
+                        {
+                            "positive_rate": "medium",
+                            "source_name": "VirusTotal",
+                            "url": "https://virustotal.com/#/file/7bdf8b8594ec269da864ee662334f4da53d4820a3f0f8aa665a0fa096ca8f22d"
+                        },
+                        {
+                            "description": "Mitre attack tactics and technique reference",
+                            "mitre_attack_tactic": "Build Capabilities",
+                            "mitre_attack_tactic_id": "TA0024",
+                            "mitre_attack_tactic_url": "https://attack.mitre.org/tactics/TA0024/",
+                            "source_name": "mitre-attack"
+                        }
+                    ],
+                    "id": "indicator--302dab0f-64dc-42f5-b99e-702b28c1aaa9",
+                    "indicator_types": [
+                        "malicious-activity"
+                    ],
+                    "lang": "en",
+                    "modified": "2021-12-07T13:58:01.596Z",
+                    "name": "4d0f21919d623bd1631ee15ca7429f28;5ce39ef0700b64bd0c71b55caf64ae45d8400965;7bdf8b8594ec269da864ee662334f4da53d4820a3f0f8aa665a0fa096ca8f22d",
+                    "pattern": "[file:hashes.MD5 = '4d0f21919d623bd1631ee15ca7429f28' OR file:hashes.'SHA-1' = '5ce39ef0700b64bd0c71b55caf64ae45d8400965' OR file:hashes.'SHA-256' = '7bdf8b8594ec269da864ee662334f4da53d4820a3f0f8aa665a0fa096ca8f22d']",
+                    "pattern_type": "stix",
+                    "sixgill_actor": "vaedzy",
+                    "sixgill_confidence": 70,
+                    "sixgill_feedid": "darkfeed_012",
+                    "sixgill_feedname": "dark_web_hashes",
+                    "sixgill_post_virustotallink": "https://virustotal.com/#/file/7bdf8b8594ec269da864ee662334f4da53d4820a3f0f8aa665a0fa096ca8f22d",
+                    "sixgill_postid": "c0c9a0085fb5281cfb40a0ddb62e1d2c6a53eb7a",
+                    "sixgill_posttitle": "[病毒样本] #Trickbot (2021-12-07)",
+                    "sixgill_severity": 70,
+                    "sixgill_source": "forum_kafan",
+                    "spec_version": "2.1",
+                    "type": "indicator",
+                    "valid_from": "2021-12-07T02:55:17Z"
+                }
+            ],
+            "spec_version": "2.0",
+            "type": "bundle"
+          }

packages/ti_cybersixgill/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.3.0"
+  changes:
+    - description: Moving integration to use the TAXII service rather than python scripts
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2771
 - version: "1.2.0"
   changes:
     - description: Update to ECS 8.0

packages/ti_cybersixgill/data_stream/threat/_dev/test/pipeline/test-common-config.yml

@@ -1,5 +1,3 @@
-dynamic_fields:
-  event.ingested: ".*"
 fields:
   tags:
     - preserve_original_event

packages/ti_cybersixgill/data_stream/threat/_dev/test/pipeline/test-cybersixgill-ndjson.log

@@ -0,0 +1,4 @@
+{"confidence":80,"created":"2021-12-07T09:22:41.485Z","description":"Virustotal link that appeared on a dark web site, generally to show malware that is undetected","extensions":{"extension-definition--4582f9eb-bad5-46ac-bd26-1b4201d52537":{"extension_type":"toplevel-property-extension"}},"external_references":[{"positive_rate":"none","source_name":"VirusTotal","url":"https://virustotal.com/#/file/2e7e43be1fc3cbefef8d686ce63ceb30456a4a67d555407fb6797e969972945c"},{"description":"Mitre attack tactics and technique reference","mitre_attack_tactic":"Test capabilities","mitre_attack_tactic_id":"TA0025","mitre_attack_tactic_url":"https://attack.mitre.org/tactics/TA0025/","mitre_attack_technique":"Test signature detection for file upload/email filters","mitre_attack_technique_id":"T1361","mitre_attack_technique_url":"https://attack.mitre.org/techniques/T1361/","source_name":"mitre-attack"}],"id":"indicator--b34d3caa-e4e2-49bd-9b57-f585728320e8","indicator_types":["malicious-activity"],"lang":"en","modified":"2021-12-07T09:22:41.485Z","name":"31aef6bddfeeb3f519dfe3d5ebe9c2ae;e54ef45c82899dd2b20372cf47958cea94dd80a7;2e7e43be1fc3cbefef8d686ce63ceb30456a4a67d555407fb6797e969972945c","pattern":"[file:hashes.MD5 = '31aef6bddfeeb3f519dfe3d5ebe9c2ae' OR file:hashes.'SHA-1' = 'e54ef45c82899dd2b20372cf47958cea94dd80a7' OR file:hashes.'SHA-256' = '2e7e43be1fc3cbefef8d686ce63ceb30456a4a67d555407fb6797e969972945c']","pattern_type":"stix","sixgill_actor":"layna61524","sixgill_confidence":80,"sixgill_feedid":"darkfeed_002","sixgill_feedname":"darkweb_vt_links","sixgill_post_virustotallink":"https://virustotal.com/#/file/2e7e43be1fc3cbefef8d686ce63ceb30456a4a67d555407fb6797e969972945c","sixgill_postid":"a452593da2f6314c2f2d6c98c6473608e11914e3","sixgill_posttitle":"[GET] LAYNA'S LAGNIAPPE - DECEMBER 6, 2021","sixgill_severity":70,"sixgill_source":"forum_bestblackhat","spec_version":"2.1","type":"indicator","valid_from":"2021-12-07T00:03:00Z"}
+{"confidence":80,"created":"2021-12-07T18:04:26.451Z","description":"Malware available for download from file-sharing sites","extensions":{"extension-definition--4582f9eb-bad5-46ac-bd26-1b4201d52537":{"extension_type":"toplevel-property-extension"}},"external_references":[{"description":"Mitre attack tactics and technique reference","mitre_attack_tactic":"Build Capabilities","mitre_attack_tactic_id":"TA0024","mitre_attack_tactic_url":"https://attack.mitre.org/tactics/TA0024/","mitre_attack_technique":"Obtain/re-use payloads","mitre_attack_technique_id":"T1346","mitre_attack_technique_url":"https://attack.mitre.org/techniques/T1346/","source_name":"mitre-attack"}],"id":"indicator--ade54b36-752d-4107-a2ed-dd666fa1cb85","indicator_types":["malicious-activity"],"lang":"ru","modified":"2021-12-07T18:04:26.451Z","name":"https://ru.scribd.com/user/456422024/ForkLog#from_embed","pattern":"[url:value = 'https://ru.scribd.com/user/456422024/ForkLog#from_embed']","pattern_type":"stix","sixgill_actor":"CoinProject.info","sixgill_confidence":80,"sixgill_feedid":"darkfeed_010","sixgill_feedname":"malware_download_urls","sixgill_postid":"3f8c56e4cf6407ee7608e0f605503cb1e3fcedb9","sixgill_posttitle":"Банковский регулятор США напомнил о рисках внедрения криптовалют","sixgill_severity":70,"sixgill_source":"forum_hyipinvest","spec_version":"2.1","type":"indicator","valid_from":"2021-12-07T14:52:00Z"}
+{"confidence":70,"created":"2021-12-07T21:24:50.350Z","description":"Hash attributed to malware that was discovered in the dark and deep web","extensions":{"extension-definition--4582f9eb-bad5-46ac-bd26-1b4201d52537":{"extension_type":"toplevel-property-extension"}},"external_references":[{"description":"Mitre attack tactics and technique reference","mitre_attack_tactic":"Build Capabilities","mitre_attack_tactic_id":"TA0024","mitre_attack_tactic_url":"https://attack.mitre.org/tactics/TA0024/","source_name":"mitre-attack"}],"id":"indicator--18f0351d-b61f-4961-ab41-0b10566ee602","indicator_types":["malicious-activity"],"lang":"zh","modified":"2021-12-07T21:24:50.350Z","name":"1dce6f3ba4a8d355df21a17584c514697ee0c37b51ab5657bc5b3a297b65955f","pattern":"[file:hashes.'SHA-256' = '1dce6f3ba4a8d355df21a17584c514697ee0c37b51ab5657bc5b3a297b65955f']","pattern_type":"stix","sixgill_actor":"Admin","sixgill_confidence":70,"sixgill_feedid":"darkfeed_012","sixgill_feedname":"dark_web_hashes","sixgill_post_virustotallink":"https://virustotal.com/#/file/1dce6f3ba4a8d355df21a17584c514697ee0c37b51ab5657bc5b3a297b65955f","sixgill_postid":"c550f74ba76c0b2c9c46b0577f551ba5ef855813","sixgill_posttitle":"海康威视因自身漏洞被黑客利用而遭受攻击","sixgill_severity":70,"sixgill_source":"blog_hackdig","spec_version":"2.1","type":"indicator","valid_from":"2021-12-07T21:23:33Z"}
+{"confidence":90,"created":"2021-12-07T22:48:59.141Z","description":"Shell access to this domain is being sold on dark web markets","extensions":{"extension-definition--4582f9eb-bad5-46ac-bd26-1b4201d52537":{"extension_type":"toplevel-property-extension"}},"external_references":[{"description":"Mitre attack tactics and technique reference","mitre_attack_tactic":"Establish & Maintain Infrastructure","mitre_attack_tactic_id":"TA0022","mitre_attack_tactic_url":"https://attack.mitre.org/tactics/TA0022/","mitre_attack_technique":"Compromise 3rd party infrastructure to support delivery","mitre_attack_technique_id":"T1334","mitre_attack_technique_url":"https://attack.mitre.org/techniques/T1334/","source_name":"mitre-attack"}],"id":"indicator--557a6021-b0c0-441a-8fba-e8a734f19ada","indicator_types":["compromised"],"lang":"en","modified":"2021-12-07T22:48:59.141Z","name":"sdbpibandung.sch.id","pattern":"[domain-name:value = 'sdbpibandung.sch.id']","pattern_type":"stix","sixgill_actor":"enginewo","sixgill_confidence":90,"sixgill_feedid":"darkfeed_001","sixgill_feedname":"compromised_sites","sixgill_postid":"955f5379c2828ce483b74a671e498a5f69f9ea36","sixgill_posttitle":"Beranda       http://sdbpibandung.sch.id","sixgill_severity":70,"sixgill_source":"market_magbo","spec_version":"2.1","type":"indicator","valid_from":"2021-12-07T22:43:29Z"}