Skip to content

Commit

Permalink
chore: switch siem_alarms to daily index (#114)
Browse files Browse the repository at this point in the history 
* use elasticsearch filter plugin and perm_index

* follow ES7.x defaults and remove deprecated items

* remove unnecessary fields from index pattern

also reset dashboard items version to 1

* add info on how ES indices are implemented
  • Loading branch information
@mmta
mmta authored Jun 2, 2019
1 parent e64523f commit dfd1b3e
Show file tree
Hide file tree
Showing 10 changed files with 177 additions and 290 deletions.
36 changes: 23 additions & 13 deletions deployments/docker/conf/logstash/conf.d/80_siem.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,31 @@ filter {
"[@metadata][siem_data_type]" => "alarms"
}
}
# set target_index with the actual index for an existing ID
elasticsearch {
hosts => ["elasticsearch:9200"]
index => "siem_alarms"
query => "_id:%{[alarm_id]}"
fields => { "perm_index" => "[@metadata][target_index]" }
}
# if previous step failed or couldn't find a match in the case of new ID, then use today's date
if ![@metadata][target_index] {
mutate {
add_field => {
"[@metadata][target_index]" => "siem_alarms-%{+YYYY.MM.dd}"
}
}
}
# elasticsearch filter plugin only search within _source, so the following extra perm_index field is necessary
mutate {
add_field => {
"perm_index" => "%{[@metadata][target_index]}"
}
}
prune {
whitelist_names => [ "timestamp", "@metadata", "title", "status", "kingdom", "category",
"updated_time", "risk", "risk_class", "tag$", "src_ips", "dst_ips", "intel_hits", "vulnerabilities",
"networks", "rules", "custom_data" ]
"networks", "rules", "custom_data", "^perm_index$" ]
}
}
}
Expand All @@ -83,22 +104,11 @@ output {
if [@metadata][siem_data_type] == "alarms" {
elasticsearch {
hosts => "elasticsearch:9200"
index => "siem_alarms"
index => "%{[@metadata][target_index]}"
document_id => "%{[@metadata][alarm_id]}"
template => "/etc/logstash/index-template.d/siem_alarms-template.json"
template_name => "siem_alarms"
template_overwrite => true
}
}
# unused output, a workaround to upload extra template
if [@metadata][siem_data_type] == "doesntexist" {
elasticsearch {
hosts => "elasticsearch:9200"
index => "siem_alarms"
document_id => "%{[@metadata][alarm_id]}"
template => "/etc/logstash/index-template.d/siem_alarms-multi-template.json"
template_name => "siem_alarms-multi"
template_overwrite => true
}
}
}
44 changes: 17 additions & 27 deletions deployments/docker/conf/logstash/index-template.d/siem_alarm_events-template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,38 +3,28 @@
"version" : 1,
"settings" : {
"number_of_replicas": 0,
"number_of_shards": 2,
"number_of_shards": 1,
"index.refresh_interval" : "1s"
},
"mappings" : {
"_default_" : {
"dynamic_templates" : [ {
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text", "norms" : false,
"fields" : {
"keyword" : { "type": "keyword", "index": "true", "ignore_above": 256 },
"raw" : { "type": "keyword", "index": "true", "ignore_above": 256 }
"mappings": {
"doc": {
"dynamic_templates": [
{
"strings_as_keywords": {
"match_mapping_type": "string",
"mapping": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
} ],
"properties" : {
"@version": { "type": "keyword" },
"stage": { "type": "keyword" },
"event_id": { "type": "keyword" }
}
]
}
}
}
57 changes: 0 additions & 57 deletions deployments/docker/conf/logstash/index-template.d/siem_alarms-multi-template.json
View file

This file was deleted.

65 changes: 24 additions & 41 deletions deployments/docker/conf/logstash/index-template.d/siem_alarms-template.json
View file
Original file line number Diff line number Diff line change
@@ -1,53 +1,36 @@
{
"index_patterns" : [ "siem_alarms" ],
"index_patterns" : [ "siem_alarms-*" ],
"version" : 1,
"settings" : {
"number_of_replicas": 0,
"number_of_shards": 2,
"number_of_shards": 1,
"index.refresh_interval" : "1s"
},
"mappings" : {
"_default_" : {
"dynamic_templates" : [ {
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text", "norms" : false,
"fields" : {
"keyword" : { "type": "keyword", "index": "true", "ignore_above": 256 },
"raw" : { "type": "keyword", "index": "true", "ignore_above": 256 }
"aliases" : {
"siem_alarms" : {}
},
"mappings": {
"doc": {
"dynamic_templates": [
{
"strings_as_keywords": {
"match_mapping_type": "string",
"mapping": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
} ],
"properties" : {
"timestamp": { "type": "date" },
"@version": { "type": "keyword" },
"updated_time": { "type": "date" },
"dst_ips": {
"type": "ip",
"fields": {
"raw": {"index": "true", "type": "keyword"},
"keyword": {"index": "true", "type": "keyword"}
}
},
"src_ips": {
"type": "ip",
"fields": {
"raw": {"index": "true", "type": "keyword"},
"keyword": {"index": "true", "type": "keyword"}
}
}
],
"properties": {
"src_ips": { "type": "ip" },
"dst_ips": { "type": "ip" }
}
}
}
Expand Down
77 changes: 20 additions & 57 deletions deployments/docker/conf/logstash/index-template.d/siem_events-template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,68 +3,31 @@
"version" : 1,
"settings" : {
"number_of_replicas": 0,
"number_of_shards": 2,
"number_of_shards": 1,
"index.refresh_interval" : "1s"
},
"mappings" : {
"_default_" : {
"dynamic_templates" : [ {
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text", "norms" : false,
"fields" : {
"keyword" : { "type": "keyword", "index": "true", "ignore_above": 256 },
"raw" : { "type": "keyword", "index": "true", "ignore_above": 256 }
"mappings": {
"doc": {
"dynamic_templates": [
{
"strings_as_keywords": {
"match_mapping_type": "string",
"mapping": {
"type": "text",
"norms": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
} ],
"properties" : {
"timestamp": { "type": "date" },
"@version": { "type": "keyword" },
"src_geoip" : {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "half_float" },
"longitude" : { "type" : "half_float" }
}
},
"dst_geoip" : {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "half_float" },
"longitude" : { "type" : "half_float" }
}
},
"dst_ip": {
"type": "ip",
"fields": {
"raw": {"index": "true", "type": "keyword"},
"keyword": {"index": "true", "type": "keyword"}
}
},
"src_ip": {
"type": "ip",
"fields": {
"raw": {"index": "true", "type": "keyword"},
"keyword": {"index": "true", "type": "keyword"}
}
}
],
"properties": {
"src_ip": { "type": "ip" },
"dst_ip": { "type": "ip" }
}
}
}
Expand Down
Loading

0 comments on commit dfd1b3e

Please sign in to comment.