tychon arp demonstration

.github/CODEOWNERS

@@ -249,6 +249,7 @@
 /packages/trellix_epo_cloud @elastic/security-external-integrations
 /packages/trend_micro_vision_one @elastic/security-external-integrations
 /packages/trendmicro @elastic/security-external-integrations
+/packages/tychon @elastic/security-external-integrations
 /packages/udp @elastic/security-external-integrations
 /packages/universal_profiling_agent @elastic/profiling
 /packages/universal_profiling_collector @elastic/profiling

packages/tychon/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: git@v8.10.0

packages/tychon/_dev/build/docs/README.md

@@ -0,0 +1,21 @@
+# TYCHON Agentless
+
+[TYCHON Agentless](https://tychon.io/products/tychon-agentless/) is an integration that lets you collect TYCHON's gold source Master Endpoint Record  data from endpoints, including vulnerability and STIG results, without heavy resource use or software installation. You can then investigate the TYCHON data using Elastic's analytics, visualizations, and dashboards. [Contact us to learn more.](https://tychon.io/start-a-free-trial/)
+
+## Compatibility
+
+* This integration supports Windows and RedHat/CENTOS Endpoint Operating Systems.
+* This integration requires a TYCHON Agentless license.
+* This integration requires [TYCHON Vulnerability Definition](https://support.tychon.io/) files.
+* The Linux Endpoint requires RedHat's [OpenScap](https://www.open-scap.org/tools/openscap-base/) to be installed for STIG and CVE to report data.
+* This integration supports Elastic 8.8+.
+
+## Returned Data Fields
+
+### ARP Table Information
+
+TYCHON scans Endpoint ARP Tables and returns the results.
+
+{{fields "tychon_arp"}}
+
+{{event "tychon_arp"}}

packages/tychon/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,8 @@
+version: '2.3'
+services:
+  tychon-filestream:
+    image: alpine
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+      - ${SERVICE_LOGS_DIR}:/var/log
+    command: /bin/sh -c "cp /sample_logs/* /var/log/"

packages/tychon/_dev/deploy/docker/sample_logs/tychon-arp.log

@@ -0,0 +1 @@
+{"script.type":"powershell","host.os.build":"22621","host.ip":["10.154.5.200"],"host.hostname":"DESKTOP-AF7CIQM","host.os.name":"Microsoft Windows 11 Pro","host.hardware.manufacturer":"Dell Inc.","@timestamp":"2023-08-16T05:22:36Z","script.start":"2023-08-16T05:22:36Z","destination.mac":"00-09-0F-AA-00-02","host.hardware.owner":"james_sudbury@msn.com","host.hardware.cpu.caption":"Intel64 Family 6 Model 141 Stepping 1","destination.hostname":"Request timed out (700 ms)","host.os.organization":"","host.workgroup":"WORKGROUP","host.hardware.serial_number":"HYLCKG3","host.ipv4":"10.154.5.200","host.os.version":"2009","network.direction":"external","host.hardware.bios.name":"Dell Inc.","host.type":"Workstation","network.type":"IPv4","destination.name":"Request timed out (700 ms)","host.id":"47b5d5906f7d4b288a1366b2f6483148_4C4C4544-0059-4C10-8043-C8C04F4B4733_DESKTOP-AF7CIQM_WORKGROUP","host.biossn":"4C4C4544-0059-4C10-8043-C8C04F4B4733","host.mac":["60:E3:2B:4B:40:E2"],"network.interface":"Ethernet 3","host.oem.model":"XPS 17 9710","host.uptime":"594263.4592614","id":"DESKTOP-AF7CIQM#6#10.70.4.15#10.70.4.16","script.current_time":"2023-08-16T05:22:37Z","script.name":"Get-TychonArpInfo.ps1","network.state":"dynamic","script.version":"2.3.53.0","host.oem.manufacturer":"Dell","host.os.description":"","script.current_duration":"1809.94","host.ipv6":"fe80::c2c9:f4e0:eb65:2c33","destination.ip":"10.70.4.16","host.hardware.bios.version":"1.20.1","host.domain":"","host.os.family":"Windows"}

packages/tychon/changelog.yml

@@ -0,0 +1,5 @@
+- version: 0.0.1
+  changes:
+    - description: Initial release of package.
+      type: enhancement
+      link: https://github.com/joeperuzzi/integrations/pull/1 # FIXME Replace with the real PR link

packages/tychon/data_stream/tychon_arp/_dev/test/pipeline/test-arp.log

@@ -0,0 +1 @@
+{"script.type":"powershell","host.os.build":"22621","host.ip":["10.154.5.200"],"host.hostname":"DESKTOP-AF7CIQM","host.os.name":"Microsoft Windows 11 Pro","host.hardware.manufacturer":"Dell Inc.","@timestamp":"2023-08-16T05:22:36Z","script.start":"2023-08-16T05:22:36Z","destination.mac":"00-09-0F-AA-00-02","host.hardware.owner":"james_sudbury@msn.com","host.hardware.cpu.caption":"Intel64 Family 6 Model 141 Stepping 1","destination.hostname":"Request timed out (700 ms)","host.os.organization":"","host.workgroup":"WORKGROUP","host.hardware.serial_number":"HYLCKG3","host.ipv4":"10.154.5.200","host.os.version":"2009","network.direction":"external","host.hardware.bios.name":"Dell Inc.","host.type":"Workstation","network.type":"IPv4","destination.name":"Request timed out (700 ms)","host.id":"47b5d5906f7d4b288a1366b2f6483148_4C4C4544-0059-4C10-8043-C8C04F4B4733_DESKTOP-AF7CIQM_WORKGROUP","host.biossn":"4C4C4544-0059-4C10-8043-C8C04F4B4733","host.mac":["60:E3:2B:4B:40:E2"],"network.interface":"Ethernet 3","host.oem.model":"XPS 17 9710","host.uptime":"594263.4592614","id":"DESKTOP-AF7CIQM#6#10.70.4.15#10.70.4.16","script.current_time":"2023-08-16T05:22:37Z","script.name":"Get-TychonArpInfo.ps1","network.state":"dynamic","script.version":"2.3.53.0","host.oem.manufacturer":"Dell","host.os.description":"","script.current_duration":"1809.94","host.ipv6":"fe80::c2c9:f4e0:eb65:2c33","destination.ip":"10.70.4.16","host.hardware.bios.version":"1.20.1","host.domain":"","host.os.family":"Windows"}

packages/tychon/data_stream/tychon_arp/_dev/test/pipeline/test-arp.log-expected.json

@@ -0,0 +1,87 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2023-08-16T05:22:36.000Z",
+            "ecs": {
+                "version": "8.10.0"
+            },
+            "event": {
+                "category": [
+                    "network"
+                ],
+                "kind": "state",
+                "type": [
+                    "info"
+                ]
+            },
+            "tychon": {
+                "arp": {
+                    "destination": {
+                        "hostname": "Request timed out (700 ms)",
+                        "ip": "10.70.4.16",
+                        "mac": "00-09-0F-AA-00-02",
+                        "name": "Request timed out (700 ms)"
+                    },
+                    "host": {
+                        "biossn": "4C4C4544-0059-4C10-8043-C8C04F4B4733",
+                        "domain": "",
+                        "hardware": {
+                            "bios": {
+                                "name": "Dell Inc.",
+                                "version": "1.20.1"
+                            },
+                            "cpu": {
+                                "caption": "Intel64 Family 6 Model 141 Stepping 1"
+                            },
+                            "manufacturer": "Dell Inc.",
+                            "owner": "james_sudbury@msn.com",
+                            "serial_number": "HYLCKG3"
+                        },
+                        "hostname": "DESKTOP-AF7CIQM",
+                        "id": "47b5d5906f7d4b288a1366b2f6483148_4C4C4544-0059-4C10-8043-C8C04F4B4733_DESKTOP-AF7CIQM_WORKGROUP",
+                        "ip": [
+                            "10.154.5.200"
+                        ],
+                        "ipv4": [
+                            "10.154.5.200"
+                        ],
+                        "ipv6": "fe80::c2c9:f4e0:eb65:2c33",
+                        "mac": [
+                            "60-E3-2B-4B-40-E2"
+                        ],
+                        "oem": {
+                            "manufacturer": "Dell",
+                            "model": "XPS 17 9710"
+                        },
+                        "os": {
+                            "build": "22621",
+                            "description": "",
+                            "family": "Windows",
+                            "name": "Microsoft Windows 11 Pro",
+                            "organization": "",
+                            "version": "2009"
+                        },
+                        "type": "Workstation",
+                        "uptime": 594263,
+                        "workgroup": "WORKGROUP"
+                    },
+                    "network": {
+                        "direction": "external",
+                        "interface": "Ethernet 3",
+                        "state": "dynamic",
+                        "type": "IPv4"
+                    },
+                    "script": {
+                        "current_duration": "1809.94",
+                        "current_time": "2023-08-16T05:22:37Z",
+                        "name": "Get-TychonArpInfo.ps1",
+                        "start": "2023-08-16T05:22:36Z",
+                        "type": "powershell",
+                        "version": "2.3.53.0"
+                    }
+                },
+                "id": "DESKTOP-AF7CIQM#6#10.70.4.15#10.70.4.16"
+            }
+        }
+    ]
+}

packages/tychon/data_stream/tychon_arp/_dev/test/system/test-logfile-config.yml

@@ -0,0 +1,8 @@
+service: tychon-filestream
+input: filestream
+data_stream:
+  vars:
+    preserve_original_event: true
+    preserve_duplicate_custom_fields: true
+    paths:
+      - '{{SERVICE_LOGS_DIR}}/*.log'

packages/tychon/data_stream/tychon_arp/agent/stream/stream.yml.hbs

@@ -0,0 +1,22 @@
+paths:
+{{#each paths as |path|}}
+  - {{path}}
+{{/each}}
+prospector.scanner.exclude_files: [".gz$"]
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#if preserve_duplicate_custom_fields}}
+  - preserve_duplicate_custom_fields
+{{/if}}
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/tychon/data_stream/tychon_arp/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,95 @@
+---
+description: Pipeline for TYCHON ARP Tables
+processors:
+  - rename:
+      tag: rename_message
+      field: message
+      target_field: event.original
+  - json:
+      field: event.original
+      target_field: tychon.arp
+  - dot_expander:
+      tag: expand_dots
+      path: tychon.arp
+      field: '*'
+  - rename:
+      tag: rename_tychon_timestamp
+      field: 'tychon.arp.@timestamp'
+      target_field: 'tychon.timestamp'
+  - rename:
+      tag: rename_tychon_id
+      field: 'tychon.arp.id'
+      target_field: 'tychon.id'
+  - date:
+      tag: date_timestamp
+      field: 'tychon.timestamp'
+      formats:
+        - ISO8601
+  - set:
+      field: ecs.version
+      value: 8.10.0
+  - set:
+      field: event.kind
+      value: state
+  - gsub:
+      field: tychon.arp.host.mac
+      pattern: ':'
+      replacement: '-'
+      ignore_missing: true
+  - split:
+      field: tychon.arp.host.ipv4
+      separator: ','
+      ignore_missing: true
+  - convert:
+      field: tychon.arp.host.uptime
+      type: string
+      ignore_missing: true
+  - split:
+      field: tychon.arp.host.uptime
+      separator: '\.+'
+      target_field: tempuptime
+      ignore_failure: true
+  - set:
+      field: tychon.arp.host.uptime
+      value: '{{{tempuptime.0}}}'
+      ignore_failure: true
+  - remove:
+      field: tempuptime
+      ignore_failure: true
+      ignore_missing: true
+  - convert:
+      tag: convert_host_uptime
+      field: tychon.arp.host.uptime
+      type: long
+      ignore_missing: true
+  - set:
+      field: event.category
+      value: [network]
+  - set:
+      field: event.type
+      value: [info]
+  - convert:
+      tag: convert_script_current_duration
+      field: tychon.script.current_duration
+      type: float
+      ignore_missing: true
+  - remove:
+      tag: remove_preserve_original_event
+      field: event.original
+      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
+      ignore_failure: true
+      ignore_missing: true
+  - remove:
+      tag: remove_preserve_duplicate_custom_fields
+      # add fields here that have been copied into ECS fields.
+      field:
+        - tychon.timestamp
+      if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))
+      ignore_missing: true
+on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
+      field: error.message
+      value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'

packages/tychon/data_stream/tychon_arp/fields/agent.yml

@@ -0,0 +1,101 @@
+- name: elastic_agent
+  type: group
+  fields:
+    - name: id
+      description: Elastic Agent Id.
+      type: keyword
+    - name: snapshot
+      description: Elastic Agent snapshot.
+      type: boolean
+    - name: version
+      description: Elastic Agent Version.
+      type: keyword
+- name: script
+  type: group
+  fields:
+    - name: current_duration
+      description: Scanner Script Duration.
+      type: long
+    - name: current_time
+      description: Current datetime.
+      type: date
+    - name: name
+      description: Scanner Script Name.
+      type: keyword
+    - name: start
+      description: Scanner Start datetime.
+      type: date
+    - name: type
+      description: Scanner Script Type.
+      type: keyword
+    - name: version
+      description: Scanner Script Version.
+      type: version
+- name: host
+  title: Host
+  group: 2
+  description: 'A host is defined as a general computing instance.
+
+    ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.'
+  type: group
+  fields:
+    - name: biossn
+      description: Host BIOS Serial Number.
+      type: keyword
+    - name: ipv4
+      description: Host IPv4 addresses.
+      type: ip
+    - name: ipv6
+      description: Host IPv6 addresses.
+      type: keyword
+    - name: workgroup
+      description: Host Workgroup Network Name.
+      type: keyword
+    - name: oem
+      type: group
+      fields:
+        - name: manufacturer
+          description: Host OEM Manufacturer.
+          type: keyword
+        - name: model
+          description: Host OEM Model.
+          type: keyword
+    - name: os
+      type: group
+      fields:
+        - name: build
+          description: Host OS Build.
+          type: keyword
+        - name: description
+          description: Host OS Description.
+          type: text
+        - name: organization
+          description: Host OS Organization.
+          type: keyword
+    - name: hardware
+      type: group
+      fields:
+        - name: bios
+          type: group
+          fields:
+            - name: name
+              description: Host BIOS Name.
+              type: keyword
+            - name: version
+              description: Host BIOS Version.
+              type: keyword
+        - name: cpu
+          type: group
+          fields:
+            - name: caption
+              description: Host CPU Caption.
+              type: keyword
+        - name: manufacturer
+          description: Host BIOS Manufacturer.
+          type: keyword
+        - name: owner
+          description: Host BIOS Owner.
+          type: keyword
+        - name: serial_number
+          description: Host BIOS Serial Number.
+          type: keyword

packages/tychon/data_stream/tychon_arp/fields/base-fields.yml

@@ -0,0 +1,17 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset name.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: event.module
+  type: constant_keyword
+  description: Event module
+  value: tychon
+- name: event.dataset
+  type: constant_keyword
+  description: Event dataset
+  value: tychon.tychon_arp