chore: add elastic SIEM integration (#197)

* chore: use update script for logstash alarm output

This prevents out-of-order updates from filebeat to siem_alarms index
[skip ci]

* chore: add example on elastic SIEM integration

* docs: add the doc for elastic SIEM integration

* docs: also update the list and install doc

[skip ci]

* remove fields.go from deploments dir

deployments/docker/conf/filebeat-es/filebeat.yml

@@ -0,0 +1,14 @@
+filebeat.config.modules:
+  path: /usr/share/filebeat/modules.d/*.yml
+  reload.enabled: false
+
+output.elasticsearch:
+  hosts: ["elasticsearch:9200"]
+
+setup.template.settings:
+  index.number_of_shards: 1
+  index.number_of_replicas: 0
+
+setup.template.overwrite: true
+setup.template.enabled: true
+setup.template.fields: /usr/share/filebeat/fields.yml

deployments/docker/conf/filebeat-es/module/dsiem/_meta/config.yml

@@ -0,0 +1,3 @@
+- module: dsiem
+  alarm:
+    enabled: true

deployments/docker/conf/filebeat-es/module/dsiem/_meta/docs.asciidoc

@@ -0,0 +1,47 @@
+:modulename: dsiem
+:has-dashboards: true
+
+== dsiem module
+
+This is the dsiem module.
+
+include::../include/what-happens.asciidoc[]
+
+[float]
+=== Compatibility
+
+TODO: document with what versions of the software is this tested
+
+
+include::../include/running-modules.asciidoc[]
+
+[float]
+=== Example dashboard
+
+This module comes with a sample dashboard. For example:
+
+TODO: include an image of a sample dashboard. If you do not include a dashboard,
+remove this section and set `:has-dashboards: false` at the top of this file.
+
+include::../include/configuring-intro.asciidoc[]
+
+TODO: provide an example configuration
+
+:fileset_ex: {fileset}
+
+include::../include/config-option-intro.asciidoc[]
+
+TODO: document the variables from each fileset. If you're describing a variable
+that's common to other modules, you can reuse shared descriptions by including
+the relevant file. For example:
+
+[float]
+==== `{fileset}` log fileset settings
+
+include::../include/var-paths.asciidoc[]
+
+:has-dashboards!:
+
+:fileset_ex!:
+
+:modulename!:

deployments/docker/conf/filebeat-es/module/dsiem/_meta/fields.yml

@@ -0,0 +1,9 @@
+- key: dsiem
+  title: "dsiem"
+  description: >
+    dsiem Module
+  fields:
+    - name: dsiem
+      type: group
+      description: >
+      fields:

deployments/docker/conf/filebeat-es/module/dsiem/alarm/_meta/fields.yml

@@ -0,0 +1,6 @@
+- name: alarm
+  description: dsiem alarm
+  type: group
+  fields:
+    - name: updated_time
+      type: date

deployments/docker/conf/filebeat-es/module/dsiem/alarm/config/alarm.yml

@@ -0,0 +1,72 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+exclude_files: [".gz$"]
+
+processors:
+  - rename:
+      fields:
+        - {from: message, to: event.original}
+  - decode_json_fields:
+      fields: [event.original]
+      target: dsiem.alarm
+      process_array: true
+      max_depth: 2
+  - convert:
+      mode: copy
+      fields:
+        - {from: '@timestamp', to: event.created}
+  - timestamp:
+      field: dsiem.alarm.created_time
+      layouts: 
+        - UNIX
+      test:
+        - '1571045906'
+  - timestamp:
+      field: dsiem.alarm.update_time
+      target_field: dsiem.alarm.update_time
+      layouts:
+        - UNIX
+      test: 
+        - '1571045906'        
+  - add_fields:
+      target: event
+      fields:
+        kind: 'alarm'
+  - add_fields:
+      target: threat
+      fields:
+        framework: 'OSSIM Cyber Kill Chain'
+  - convert:
+      mode: rename
+      ignore_missing: true
+      fields: 
+        - {from: dsiem.alarm.src_ips, to: source.address}
+        - {from: dsiem.alarm.dst_ips, to: destination.address}
+        - {from: dsiem.alarm.proto, to: network.transport}
+        - {from: dsiem.alarm.risk, to: event.risk_score}
+        - {from: dsiem.alarm.title, to: message}
+        - {from: dsiem.alarm.kingdom, to: threat.tactic.name}
+        - {from: dsiem.alarm.category, to: threat.technique.name}
+        - {from: dsiem.alarm.update_time, to: dsiem.alarm.updated_time}
+  - convert:
+      mode: copy
+      fields:
+        - {from: dsiem.alarm.alarm_id, to: event.id}
+        - {from: source.address, to: source.ip}
+        - {from: destination.address, to: destination.ip}
+  - drop_fields:
+      fields:
+        - dsiem.alarm.created_time
+        - dsiem.alarm.status
+        - dsiem.alarm.tag
+        - dsiem.alarm.src_ips
+        - dsiem.alarm.dst_ips
+        - dsiem.alarm.proto
+        - dsiem.alarm.category
+        - dsiem.alarm.kingdom
+        - dsiem.alarm.rules
+        - dsiem.alarm.title
+      ignore_missing: true

deployments/docker/conf/filebeat-es/module/dsiem/alarm/ingest/pipeline.json

@@ -0,0 +1,11 @@
+{
+  "description": "use dsiem alarm_id as _id",
+  "processors": [
+    {
+      "set": {
+        "field": "_id",
+        "value": "{{event.id}}"
+      }
+    }
+  ]
+}

deployments/docker/conf/filebeat-es/module/dsiem/alarm/manifest.yml

@@ -0,0 +1,13 @@
+module_version: 1.0
+
+var:
+  - name: paths
+    default:
+      - /tmp/siem_alarms.json
+    os.darwin:
+      - /var/log/dsiem/siem_alarms.json
+    os.windows:
+      - c:/programdata/dsiem/log/siem_alarms.json
+
+ingest_pipeline: ingest/pipeline.json
+input: config/alarm.yml

deployments/docker/conf/filebeat-es/module/dsiem/module.yml

@@ -0,0 +1 @@
+dashboards:

deployments/docker/conf/filebeat-es/modules.d/dsiem.yml

@@ -0,0 +1,5 @@
+- module: dsiem
+  alarm:
+    enabled: true
+    var.paths: 
+    - /var/log/dsiem/siem_alarms.json

deployments/docker/docker-compose.yml

@@ -3,7 +3,7 @@ services:
 
   elasticsearch:
     container_name: elasticsearch
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.1.1
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.4.0
     environment:
       - discovery.type=single-node
       - "ES_JAVA_OPTS=-Xms256m -Xmx256m"
@@ -24,7 +24,7 @@ services:
 
   logstash:
     container_name: logstash
-    image: defenxor/docker-logstash:7.1.1
+    image: defenxor/docker-logstash:7.4.0
     command:
       - -f/etc/logstash/conf.d
     environment:
@@ -37,7 +37,7 @@ services:
 
   kibana:
     container_name: kibana
-    image: docker.elastic.co/kibana/kibana:7.1.1
+    image: docker.elastic.co/kibana/kibana:7.4.0
     environment:
       - XPACK_MONITORING_ENABLED=false
     ports:
@@ -60,7 +60,7 @@ services:
 
   filebeat:
     container_name: filebeat
-    image: docker.elastic.co/beats/filebeat:7.1.1
+    image: docker.elastic.co/beats/filebeat:7.4.0
     user: root
     networks:
       - siemnet
@@ -70,6 +70,20 @@ services:
       - dsiem-log:/var/log/dsiem
       - suricata-log:/var/log/suricata
 
+  filebeat-es:
+    container_name: filebeat-es
+    image: docker.elastic.co/beats/filebeat:7.4.0
+    user: root
+    networks:
+      - siemnet
+    volumes:
+      - ./conf/filebeat-es/filebeat.yml:/usr/share/filebeat/filebeat.yml
+      - ./conf/filebeat-es/fields.yml:/usr/share/filebeat/fields.yml
+      - ./conf/filebeat-es/module:/usr/share/filebeat/module
+      - ./conf/filebeat-es/modules.d:/usr/share/filebeat/modules.d
+      - filebeat-es-data:/usr/share/filebeat/data
+      - dsiem-log:/var/log/dsiem
+
   suricata:
     container_name: suricata
     image: defenxor/suricata:1710
@@ -83,6 +97,7 @@ services:
      - suricata-log:/var/log/suricata
 
 volumes:
+  filebeat-es-data:
   filebeat-data:
   es-data:
   dsiem-log:

docs/README.md

@@ -14,3 +14,4 @@ Available topics are:
 * [Advanced Deployment](./adv_deployment.md)
 * [Dsiem Elasticsearch Indices](./es_indices.md)
 * [Notes on Security](./security.md)
+* [Elastic SIEM integration](./elastic_siem_integration.md)

docs/elastic_siem_integration.md

@@ -0,0 +1,19 @@
+# Elastic SIEM Integration
+
+Elastic stack 7.2 and newer versions include a [SIEM app in Kibana](https://www.elastic.co/guide/en/kibana/current/xpack-siem.html) that provides an interactive workspace for security teams to triage events and perform initial investigations.
+
+Integrating Dsiem into the app allows us to analyze Dsiem alarms data in it, and to further correlate those alarms and other compatible log entries such as those coming from Filebeat, Winlogbeat, Auditbeat, and others.
+
+The following picture shows an example of an Elastic SIEM timeline analyzing potential connection between Dsiem alarms and several auditbeat logs.
+
+![Elastic SIEM UI integration](images/es-siem-integration.png)
+
+## Dsiem Filebeat Module
+
+Elastic SIEM app reads data from [Elastic Common Schema - ECS](https://www.elastic.co/guide/en/ecs/current/index.html) compliant documents stored in Elasticsearch indices.
+
+From a practical stand point, we can transform Dsiem alarms data into ECS format using either a Logstash filter, a Filebeat module, or a dedicated custom Beat. For now we opt to use the Filebeat module approach for its builtin support for ECS fields compared to a Logstash filter, and for its lower maintenance compared to a custom Beat.
+
+The Docker Compose environment created in the [installation guide](https://github.com/defenxor/dsiem/blob/master/docs/installation.md#using-docker-compose) already has Filebeat module for Dsiem enabled, so you can directly play with it by visiting http://localhost:5601/app/siem#/hosts/events. In that environment, all generated alarms should be visible in both Dsiem web UI and Kibana SIEM app.
+
+For reference, the Filebeat module for Dsiem files are located [here](https://github.com/defenxor/dsiem/tree/master/deployments/docker/conf/filebeat-es/).

docs/installation.md

@@ -24,10 +24,10 @@ Then after you get a feel on how everything fits together, you can start integra
 
     Replace `eth0` above with the actual interface name given by `ifconfig` or similar commands. For testing purpose, it's not necessary to configure the interface to really operate in promiscuous mode.
 
-* Set the owner of filebeat config file to root ([here's why](https://www.elastic.co/guide/en/beats/libbeat/6.4/config-file-permissions.html)):
+* Set the owner of filebeat config files to root ([here's why](https://www.elastic.co/guide/en/beats/libbeat/6.4/config-file-permissions.html)):
     ```shell
     $ cd deployments/docker && \
-    sudo chown root conf/filebeat/filebeat.yml
+    sudo chown root $(find conf/filebeat/ conf/filebeat-es/ -name "*.yml") 
     ```
 
 * Run ELK, Suricata, and Dsiem in standalone mode:
@@ -41,7 +41,7 @@ Then after you get a feel on how everything fits together, you can start integra
   
     * Dsiem web UI should be accessible from http://localhost:8080/ui, Elasticsearch from http://localhost:9200, and Kibana from http://localhost:5601.
     * Suricata comes with [Emerging Threats ICMP Info Ruleset](https://rules.emergingthreats.net/open/suricata/rules/emerging-icmp_info.rules) enabled and `EXTERNAL_NET: "any"`, so you can easily trigger a test alarm just by continuously pinging a host in the same subnet. Dsiem comes with an [example directive configuration](https://github.com/defenxor/dsiem/blob/master/configs/directives_dsiem-backend-0_testing1.json) that will intercept this "attack".
-    * Recorded events will be stored in Elasticsearch index pattern `siem_events-*`, and alarms will be in `siem_alarms`. You can view their content from Kibana or the builtin SIEM web UI.
+    * Recorded events will be stored in Elasticsearch index pattern `siem_events-*`, and alarms will be in `siem_alarms`. You can view their content from Kibana or Dsiem web UI.
 
 #### Importing Kibana Dashboard