add docker-compose example

README.md

@@ -1,3 +1,3 @@
-# DSiem 
+# Dsiem 
 
 

deployments/docker/conf/filebeat/filebeat.yml

@@ -0,0 +1,40 @@
+filebeat.inputs:
+- type: log
+  paths:
+    - /var/log/dsiem/siem_alarm_events.json
+  fields_under_root: true
+  fields: { application: siem, siem_data_type: alarm_events }
+  tail_files: true
+  json.keys_under_root: true
+  json.overwrite_keys: true
+
+- type: log
+  paths:
+    - /var/log/dsiem/siem_alarms.json
+  fields_under_root: true
+  fields: { application: siem, siem_data_type: alarms }
+  tail_files: true
+  json.keys_under_root: true
+  json.overwrite_keys: true
+
+- type: log
+  paths:
+    - /var/log/dsiem/dtester.json
+  fields_under_root: true
+  fields: { application: dtester }
+  tail_files: true
+  json.keys_under_root: true
+  json.overwrite_keys: true
+
+- type: log
+  paths:
+    - "/var/log/suricata/eve.json"
+  fields_under_root: true
+  fields: { application: suricata }
+  json.message_key: log
+  json.keys_under_root: true
+  json.overwrite_keys: true
+
+output:
+ logstash:
+   hosts: ["logstash:5001"]

deployments/logstash/conf.d/00_input-beats.conf → deployments/docker/conf/logstash/conf.d/00_input.conf

@@ -1,12 +1,3 @@
-# this is for vanilla beats input
-
-input {
-  beats {
-    port => 5044
-    tags => [ "beats" ]
-  }
-}
-
 # this is json codec
 
 input {

deployments/docker/conf/logstash/conf.d/50_suricata.conf

@@ -0,0 +1,16 @@
+filter {
+  if [application] == "suricata" {
+    date {
+      match => [ "timestamp", "ISO8601" ]
+      target => "timestamp"
+    }
+    ruby {
+      code => "
+      if event.get('[event_type]') == 'fileinfo'
+         event.set('[fileinfo][type]', event.get('[fileinfo][magic]').to_s.split(',')[0])
+      end
+      "
+    }
+  }
+}
+

deployments/logstash/conf.d/70_siem-plugin-dtester.conf → deployments/docker/conf/logstash/conf.d/70_siem-plugin-dtester.conf

@@ -1,11 +1,10 @@
 #########################################
-# SIEM dtester plugin:                  #
+# Dsiem special config for dtester tool #
 #########################################
 
-filter {
-
-# 1st step: identify the source log and clone it to another event with type => siem_events
+# dtester output is directly considered as normalizedEvent
 
+filter {
   if [application] == "dtester" {
     mutate {
       remove_field => [ "application", "beat", "host.name", "source"  ]

deployments/docker/conf/logstash/conf.d/70_siem-plugin-suricata.conf

@@ -0,0 +1,108 @@
+
+###############################################################################
+# Dsiem suricata Plugin
+# Type: SID
+#
+# Auto-generated by dpluger on 2018-10-30T19:35:27+07:00
+###############################################################################
+
+filter {
+
+# 1st step: identify the source log and clone it to another event with type => siem_events
+
+  if [application] == "suricata" and [alert] {
+    clone {
+      clones => [ "siem_events" ]
+    }
+
+# 2nd step: remove the source log identifier from the clone, so that the clone will not
+# go through the same pipeline as the source log. Also remove the temporary type field,
+# replacing it with metadata field that will be read by the rest of siem pipeline.
+
+    if [type] == "siem_events" {
+      mutate {
+        remove_field => [ "[application]" , "type" ]
+        add_field => {
+          "[@metadata][siem_plugin_type]" => "suricata"
+          "[@metadata][siem_data_type]" => "normalizedEvent"
+        }
+      }
+    }
+  }
+}
+
+# 3rd step: the actual event normalization so that it matches the format that siem expect.
+#
+# Required fields:
+#   timestamp (date), title (string), sensor (string), product (string), dst_ip (string), src_ip (string)
+#
+# For PluginRule type plugin, the following are also required:
+#   plugin_id (integer), plugin_sid (integer)
+#
+# For TaxonomyRule type plugin, the following is also required:
+#   category (string)
+#
+# Optional fields:
+# These fields are optional but should be included whenever possible since they can be used in directive rules:
+#   dst_port (integer), src_port (integer), protocol (string), subcategory (string)
+#
+# These fields are also optional and can be used in directive rules. They should be used for custom data that
+# are not defined in standard SIEM fields.
+#   custom_label1 (string), custom_data1 (string), custom_label2 (string), custom_data2 (string)
+#   custom_label3 (string), custom_data3 (string)
+#
+# And this field is optional, and should be included if the original logs are also stored in elasticsearch.
+# This will allow direct pivoting from alarm view in the web UI to the source index.
+#   src_index_pattern (string)
+#
+# As for other fields from source log, they will be removed by logstash plugin prune below
+
+filter {
+  if [@metadata][siem_plugin_type] == "suricata" {
+    date {
+      match => [ "[timestamp]", "ISO8601" ]
+      target => [timestamp]
+    }
+
+
+  #  if ![src_ip] {
+  #    mutate {
+  #      add_field => {}
+  #    }
+  #  }
+    mutate {
+      replace => {
+        "title" => "%{[alert][signature]}"
+        "src_index_pattern" => "suricata-*"
+        "sensor" => "%{[host][name]}"
+        "product" => "Intrusion Detection System"
+        "src_ip" => "%{[src_ip]}"
+        "dst_ip" => "%{[dest_ip]}"
+        "protocol" => "%{[proto]}"
+        "category" => "%{[alert][category]}"
+
+        "plugin_id" => "1001"
+        "plugin_sid" => "%{[alert][signature_id]}"
+        "src_port" => "%{[src_port]}"
+        "dst_port" => "%{[dest_port]}"
+
+      }
+    }
+
+    mutate {
+      convert => {
+        "plugin_id" => "integer"
+        "plugin_sid" => "integer"
+        "src_port" => "integer"
+        "dst_port" => "integer"
+      }
+    }
+
+    prune {
+      whitelist_names => [ "timestamp", "@metadata", "src_index_pattern", "title", "sensor", "product",
+        "src_ip", "dst_ip", "plugin_id", "plugin_sid", "category", "subcategory",
+        "src_port", "dst_port", "protocol", "custom_label1", "custom_label2", "custom_label3",
+        "custom_data1", "custom_data2", "custom_data3" ]
+    }
+  }
+}

deployments/logstash/conf.d/80_siem.conf → deployments/docker/conf/logstash/conf.d/80_siem.conf

@@ -33,7 +33,7 @@ output{
 }
 
 #########################################
-# From SIEM                             #
+# From Dsiem                            #
 #########################################
 
 filter {
@@ -56,8 +56,8 @@ filter {
     }
     mutate {
       add_field => {
-	"[@metadata][alarm_id]" => "%{[alarm_id]}"
-	"[@metadata][siem_data_type]" => "alarms"
+	      "[@metadata][alarm_id]" => "%{[alarm_id]}"
+	      "[@metadata][siem_data_type]" => "alarms"
       }
       remove_field => [ "created_time", "update_time", "siem_data_type", "alarm_id", "host", "path", "beat", "source", "tags", "application", "offset" ]
     }

deployments/docker/conf/logstash/conf.d/99_output.conf

@@ -0,0 +1,8 @@
+output {
+  if [application] == "suricata" {
+    elasticsearch {
+      hosts => ["elasticsearch:9200"]
+      index => "suricata-%{+YYYY.MM.dd}"
+    }
+  }
+}

deployments/docker/conf/logstash/index-template.d/siem_alarm_events-template.json

@@ -0,0 +1,41 @@
+{
+  "template" : "siem_alarm_events-*",
+  "version" : 1,
+  "settings" : {
+    "number_of_replicas": 1,
+    "number_of_shards": 2,
+    "index.refresh_interval" : "5s"
+  },
+  "mappings" : {
+    "_default_" : {
+      "dynamic_templates" : [ {
+        "message_field" : {
+          "path_match" : "message",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text",
+            "norms" : false
+          }
+        }
+      }, {
+        "string_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text", "norms" : false,
+            "fields" : {
+              "keyword" : { "type": "keyword", "index": "true", "ignore_above": 256 },
+              "raw" : { "type": "keyword", "index": "true", "ignore_above": 256 }
+            }
+          }
+        }
+      } ],
+      "properties" : {
+        "timestamp": { "type": "date"  },
+        "@version": { "type": "keyword" },
+        "stage": { "type": "keyword" },
+        "event_id": { "type": "keyword" }
+      }
+    }
+  }
+}

deployments/docker/conf/logstash/index-template.d/siem_alarms-template.json

@@ -0,0 +1,54 @@
+{
+  "template" : "siem_alarms",
+  "version" : 1,
+  "settings" : {
+    "number_of_replicas": 1,
+    "number_of_shards": 2,
+    "index.refresh_interval" : "5s"
+  },
+  "mappings" : {
+    "_default_" : {
+      "dynamic_templates" : [ {
+        "message_field" : {
+          "path_match" : "message",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text",
+            "norms" : false
+          }
+        }
+      }, {
+        "string_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text", "norms" : false,
+            "fields" : {
+              "keyword" : { "type": "keyword", "index": "true", "ignore_above": 256 },
+              "raw" : { "type": "keyword", "index": "true", "ignore_above": 256 }
+            }
+          }
+        }
+      } ],      
+      "properties" : {
+        "timestamp": { "type": "date"  },
+        "@version": { "type": "keyword" },
+        "updated_time": { "type": "date" },
+        "dst_ips": {
+          "type": "ip",
+          "fields": {
+            "raw": {"index": "true", "type": "keyword"},
+            "keyword": {"index": "true", "type": "keyword"}
+          }
+        },
+        "src_ips": {
+          "type": "ip",
+          "fields": {
+            "raw": {"index": "true", "type": "keyword"},
+            "keyword": {"index": "true", "type": "keyword"}
+          }
+        }
+      }
+    }
+  }
+}

deployments/docker/conf/logstash/index-template.d/siem_events-template.json

@@ -0,0 +1,71 @@
+{
+  "template" : "siem_events-*",
+  "version" : 1,
+  "settings" : {
+    "number_of_replicas": 1,
+    "number_of_shards": 2,
+    "index.refresh_interval" : "5s"
+  },
+  "mappings" : {
+    "_default_" : {
+      "dynamic_templates" : [ {
+        "message_field" : {
+          "path_match" : "message",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text",
+            "norms" : false
+          }
+        }
+      }, {
+        "string_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text", "norms" : false,
+            "fields" : {
+              "keyword" : { "type": "keyword", "index": "true", "ignore_above": 256 },
+              "raw" : { "type": "keyword", "index": "true", "ignore_above": 256 }
+            }
+          }
+        }
+      } ],
+      "properties" : {
+        "@timestamp": { "type": "date"  },
+        "@version": { "type": "keyword" },
+        "src_geoip"  : {
+          "dynamic": true,
+          "properties" : {
+            "ip": { "type": "ip" },
+            "location" : { "type" : "geo_point" },
+            "latitude" : { "type" : "half_float" },
+            "longitude" : { "type" : "half_float" }
+          }
+        },
+        "dst_geoip"  : {
+          "dynamic": true,
+          "properties" : {
+            "ip": { "type": "ip" },
+            "location" : { "type" : "geo_point" },
+            "latitude" : { "type" : "half_float" },
+            "longitude" : { "type" : "half_float" }
+          }
+        },
+        "dst_ip": {
+            "type": "ip",
+            "fields": {
+                "raw": {"index": "true", "type": "keyword"},
+                "keyword": {"index": "true", "type": "keyword"}
+             }
+        },
+        "src_ip": {
+            "type": "ip",
+            "fields": {
+                "raw": {"index": "true", "type": "keyword"},
+                "keyword": {"index": "true", "type": "keyword"}
+             }
+        }
+      }
+    }
+  }
+}