[Zeek] Add additional data sets

packages/zeek/_dev/build/docs/README.md

@@ -110,6 +110,24 @@ contains kerberos data.
 
 {{fields "kerberos"}}
 
+### known_certs
+
+The `known_certs` dataset captures information about SSL/TLS certificates seen on the local network. See the [documentation](https://docs.zeek.org/en/master/logs/known-and-software.html#known-certs-log) for more details.
+
+{{fields "known_certs"}}
+
+### known_hosts
+
+The `known_hosts` dataset simply records a timestamp and an IP address when Zeek observes a new system on the local network.. See the [documentation](https://docs.zeek.org/en/master/logs/known-and-software.html#known-hosts-log) for more details.
+
+{{fields "known_hosts"}}
+
+### known_services
+
+The `known_services` dataset records a timestamp, IP, port number, protocol, and service (if available) when Zeek observes a system offering a new service on the local network. See the [documentation](https://docs.zeek.org/en/master/logs/known-and-software.html#known-services-log) for more details.
+
+{{fields "known_services"}}
+
 ### modbus
 
 The `modbus` dataset collects the Zeek modbus.log file, which contains
@@ -236,6 +254,12 @@ SOCKS proxy requests.
 
 {{fields "socks"}}
 
+### software
+
+The `software` dataset collects details on applications operated by the hosts it sees on the local network. See the [documentation](https://docs.zeek.org/en/master/logs/known-and-software.html#software-log) for more details.
+
+{{fields "software"}}
+
 ### ssh
 
 The `ssh` dataset collects the Zeek ssh.log file, which contains SSH

packages/zeek/_dev/deploy/docker/sample_logs/known_certs.log

@@ -0,0 +1 @@
+{"ts":"2020-12-31T15:15:53.690221Z","host":"192.168.4.1","port_num":443,"subject":"L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US","issuer_subject":"L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US","serial":"98D0AD47D748CDD6"}

packages/zeek/_dev/deploy/docker/sample_logs/known_hosts.log

@@ -0,0 +1,5 @@
+{"ts":"2021-01-03T01:19:26.260073Z","host":"192.168.4.25"}
+{"ts":"2021-01-03T01:19:27.353353Z","host":"192.168.4.29"}
+{"ts":"2021-01-03T01:19:32.488179Z","host":"192.168.4.43"}
+{"ts":"2021-01-03T01:19:58.792683Z","host":"192.168.4.142"}
+{"ts":"2021-01-03T12:17:22.496004Z","host":"192.168.4.115"}

packages/zeek/_dev/deploy/docker/sample_logs/known_services.log

@@ -0,0 +1 @@
+{"ts":"2021-01-03T01:19:36.242774Z","host":"192.168.4.1","port_num":53,"port_proto":"udp","service":["DNS"]}

packages/zeek/_dev/deploy/docker/sample_logs/software.log

@@ -0,0 +1 @@
+{"ts":"2021-01-03T00:16:22.694616Z","host":"192.168.4.25","software_type":"HTTP::BROWSER","name":"Windows-Update-Agent","version.major":10,"version.minor":0,"version.minor2":10011,"version.minor3":16384,"version.addl":"Client","unparsed_version":"Windows-Update-Agent/10.0.10011.16384 Client-Protocol/2.0"}

packages/zeek/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.1.0"
+  changes:
+    - description: Add new data sets for known_hosts, known_certs, known_services, & software logs files.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/3340
 - version: "2.0.0"
   changes:
     - description: Migrate map visualisation from tile_map to map object

packages/zeek/data_stream/known_certs/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,4 @@
+fields:
+  "@timestamp": "2020-04-28T11:07:58.223Z"
+  tags:
+    - preserve_original_event

packages/zeek/data_stream/known_certs/_dev/test/pipeline/test-known_certs.log

@@ -0,0 +1 @@
+{"ts":"2020-12-31T15:15:53.690221Z","host":"192.168.4.1","port_num":443,"subject":"L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US","issuer_subject":"L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US","serial":"98D0AD47D748CDD6"}

packages/zeek/data_stream/known_certs/_dev/test/pipeline/test-known_certs.log-expected.json

@@ -0,0 +1,47 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2020-12-31T15:15:53.690Z",
+            "ecs": {
+                "version": "8.2.0"
+            },
+            "event": {
+                "category": "network",
+                "created": "2020-04-28T11:07:58.223Z",
+                "kind": "info",
+                "original": "{\"ts\":\"2020-12-31T15:15:53.690221Z\",\"host\":\"192.168.4.1\",\"port_num\":443,\"subject\":\"L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US\",\"issuer_subject\":\"L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US\",\"serial\":\"98D0AD47D748CDD6\"}"
+            },
+            "host": {
+                "ip": "192.168.4.1"
+            },
+            "network": {
+                "type": "ipv4"
+            },
+            "related": {
+                "ip": [
+                    "192.168.4.1"
+                ]
+            },
+            "server": {
+                "ip": "192.168.4.1",
+                "port": 443
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "tls": {
+                "server": {
+                    "x509": {
+                        "issuer": {
+                            "distinguished_name": "L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US"
+                        },
+                        "serial_number": "98D0AD47D748CDD6",
+                        "subject": {
+                            "distinguished_name": "L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US"
+                        }
+                    }
+                }
+            }
+        }
+    ]
+}

packages/zeek/data_stream/known_certs/_dev/test/system/test-logs-config.yml

@@ -0,0 +1,6 @@
+vars:
+  base_paths:
+    - "{{SERVICE_LOGS_DIR}}"
+input: logfile
+data_stream:
+  vars: ~

packages/zeek/data_stream/known_certs/agent/stream/log.yml.hbs

@@ -0,0 +1,21 @@
+paths:
+{{#each base_paths}}
+  {{#each ../filenames}}
+ - {{../this}}/{{this}}
+  {{/each}}
+{{/each}}
+exclude_files: [".gz$"]
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/zeek/data_stream/known_certs/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,101 @@
+---
+description: Pipeline for normalizing Zeek known_certs.log
+processors:
+  - rename:
+      field: message
+      target_field: event.original
+  - json:
+      field: event.original
+      target_field: json
+  - drop:
+      description: Drop if no timestamp (invalid json)
+      if: 'ctx?.json?.ts == null'
+
+# Sets event.created from the @timestamp field generated by filebeat before being overwritten further down
+  - set:
+      field: event.created
+      copy_from: "@timestamp"
+  - set:
+      field: ecs.version
+      value: '8.2.0'
+  - set:
+      field: event.kind
+      value: info
+  - set:
+      field: event.category
+      value: network
+  - date:
+      field: json.ts
+      formats:
+        - UNIX
+        - ISO8601
+  - rename:
+      field: json.host
+      target_field: host.ip
+      ignore_missing: true
+  - set:
+      field: network.type
+      value: ipv4
+      if: ctx.host?.ip.contains('.')
+  - set:
+      field: network.type
+      value: ipv6
+      if: ctx.host?.ip.contains(':')
+  - append:
+      field: related.ip
+      value: "{{host.ip}}"
+      if: ctx?.host?.ip != null
+      allow_duplicates: false
+  - geoip:
+      field: host.ip
+      target_field: host.geo
+      ignore_missing: true
+  - set:
+      field: server
+      copy_from: host
+      ignore_empty_value: true
+  - rename:
+      field: json.port_num
+      target_field: server.port
+      ignore_missing: true
+  - geoip:
+      database_file: GeoLite2-ASN.mmdb
+      field: server.ip
+      target_field: server.as
+      properties:
+        - asn
+        - organization_name
+      ignore_missing: true
+  - rename:
+      field: server.as.asn
+      target_field: server.as.number
+      ignore_missing: true
+  - rename:
+      field: server.as.organization_name
+      target_field: server.as.organization.name
+      ignore_missing: true
+  - rename:
+      field: json.subject
+      target_field: tls.server.x509.subject.distinguished_name
+      ignore_missing: true
+  - rename:
+      field: json.issuer_subject
+      target_field: tls.server.x509.issuer.distinguished_name
+      ignore_missing: true
+  - rename:
+      field: json.serial
+      target_field: tls.server.x509.serial_number
+      ignore_missing: true
+  - remove:
+      field: 
+        - json
+      ignore_missing: true
+  - remove:
+      field: event.original
+      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      ignore_failure: true
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: "{{ _ingest.on_failure_message }}"