[panw] Replace syslog input with UDP/TCP input and syslog processor

packages/panw/_dev/deploy/docker/docker-compose.yml

@@ -6,8 +6,13 @@ services:
       - ./sample_logs:/sample_logs:ro
       - ${SERVICE_LOGS_DIR}:/var/log
     command: /bin/sh -c "cp /sample_logs/* /var/log/"
-  panw-panos-syslog:
-    image: akroh/stream:v0.0.1
+  panw-panos-tcp:
+    image: docker.elastic.co/observability/stream:v0.6.2
     volumes:
-      - ./sample_logs:/sample_logs:ro
+      - ./syslog_logs:/sample_logs:ro
+    command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9514 -p=tcp /sample_logs/panw-panos-traffic.log
+  panw-panos-udp:
+    image: docker.elastic.co/observability/stream:v0.6.2
+    volumes:
+      - ./syslog_logs:/sample_logs:ro
     command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9514 -p=udp /sample_logs/panw-panos-traffic.log

packages/panw/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.2.0"
+  changes:
+    - description: Replace syslog input with UDP/TCP input and syslog processor.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3323
 - version: "2.1.0"
   changes:
     - description: Add GeoIP/ASN data for `*.nat.ip` fields

packages/panw/data_stream/panos/_dev/test/system/test-tcp-config.yml

@@ -0,0 +1,7 @@
+service: panw-panos-tcp
+service_notify_signal: SIGHUP
+input: tcp
+data_stream:
+  vars:
+    syslog_host: 0.0.0.0
+    syslog_port: 9514

packages/panw/data_stream/panos/_dev/test/system/test-udp-config.yml

@@ -0,0 +1,7 @@
+service: panw-panos-udp
+service_notify_signal: SIGHUP
+input: udp
+data_stream:
+  vars:
+    syslog_host: 0.0.0.0
+    syslog_port: 9514

packages/panw/data_stream/panos/agent/stream/tcp.yml.hbs

@@ -0,0 +1,44 @@
+host: "{{syslog_host}}:{{syslog_port}}"
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if ssl}}
+ssl: {{ssl}}
+{{/if}}
+processors:
+  - add_locale: ~
+  - syslog:
+      field: message
+      format: auto
+      timezone: {{tz_offset}}
+{{#if processors}}
+  {{processors}}
+{{/if}}
+{{#if internal_zones.length}}
+  - add_fields:
+      target: _conf
+      fields:
+        internal_zones:
+        {{#each internal_zones as |zone i|}}
+          - {{zone}}
+        {{/each}}
+{{/if}}
+{{#if external_zones.length}}
+  - add_fields:
+      target: _conf
+      fields:
+        external_zones:
+        {{#each external_zones as |zone i|}}
+          - {{zone}}
+        {{/each}}
+{{/if}}
+{{#if tcp_options}}
+{{tcp_options}}
+{{/if}}

packages/panw/data_stream/panos/agent/stream/udp.yml.hbs

@@ -0,0 +1,38 @@
+host: "{{syslog_host}}:{{syslog_port}}"
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+processors:
+  - add_locale: ~
+  - syslog:
+      field: message
+      format: auto
+      timezone: {{tz_offset}}
+{{#if processors}}
+    {{processors}}
+{{/if}}
+{{#if internal_zones.length}}
+  - add_fields:
+      target: _conf
+      fields:
+        internal_zones:
+        {{#each internal_zones as |zone i|}}
+          - {{zone}}
+        {{/each}}
+{{/if}}
+{{#if external_zones.length}}
+  - add_fields:
+      target: _conf
+      fields:
+        external_zones:
+        {{#each external_zones as |zone i|}}
+          - {{zone}}
+        {{/each}}
+{{/if}}

packages/panw/data_stream/panos/fields/ecs.yml

@@ -88,6 +88,20 @@
   name: labels
 - external: ecs
   name: log.level
+- external: ecs
+  name: log.syslog.facility.code
+- external: ecs
+  name: log.syslog.facility.name
+- external: ecs
+  name: log.syslog.hostname
+- external: ecs
+  name: log.syslog.priority
+- external: ecs
+  name: log.syslog.severity.code
+- external: ecs
+  name: log.syslog.severity.name
+- external: ecs
+  name: log.syslog.version
 - external: ecs
   name: message
 - external: ecs

packages/panw/data_stream/panos/manifest.yml

@@ -1,17 +1,135 @@
 type: logs
 title: Palo Alto Networks PAN-OS firewall logs
 streams:
-  - input: syslog
+  - input: tcp
+    title: "Collect logs via syslog over TCP"
+    description: "Collecting firewall logs from PAN-OS instances (input: tcp)"
+    template_path: tcp.yml.hbs
     vars:
+      - name: syslog_host
+        type: text
+        title: Syslog Host
+        multi: false
+        required: true
+        show_user: true
+        default: localhost
+      - name: syslog_port
+        type: integer
+        title: Syslog Port
+        multi: false
+        required: true
+        show_user: true
+        default: 9001
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        required: true
+        show_user: false
+        default:
+          - panw-panos
+          - forwarded
+      - name: internal_zones
+        type: text
+        title: Internal Zones
+        multi: true
+        required: false
+        show_user: false
+        default:
+          - trust
+      - name: external_zones
+        type: text
+        title: External Zones
+        multi: true
+        required: false
+        show_user: false
+        default:
+          - untrust
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false
       - name: tz_offset
         type: text
-        title: Timezone Offset
+        title: Time Zone Offset
         multi: false
         required: true
         show_user: true
-        default: local
+        default: Local
         description: >-
-          By default, datetimes in the logs will be interpreted as relative to the timezone configured in the host where the agent is running. If ingesting logs from a host on a different timezone, use this field to set the timezone offset so that datetimes are correctly parsed. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00") from UTC.
+          By default, datetimes in the logs without a time zone will be interpreted as relative to the time zone configured in the host where the agent is running. If ingesting logs from a different time zone, use this field to set the time zone offset so that datetimes are correctly parsed. Acceptable time zone formats are: local time (Local), a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00") from UTC.
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
+      - name: ssl
+        type: yaml
+        title: SSL Configuration
+        description: i.e. certificate_authorities, supported_protocols, verification_mode etc.
+        multi: false
+        required: false
+        show_user: false
+        default: |
+          #certificate_authorities:
+          #  - |
+          #    -----BEGIN CERTIFICATE-----
+          #    MIIDCjCCAfKgAwIBAgITJ706Mu2wJlKckpIvkWxEHvEyijANBgkqhkiG9w0BAQsF
+          #    ADAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwIBcNMTkwNzIyMTkyOTA0WhgPMjExOTA2
+          #    MjgxOTI5MDRaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB
+          #    BQADggEPADCCAQoCggEBANce58Y/JykI58iyOXpxGfw0/gMvF0hUQAcUrSMxEO6n
+          #    fZRA49b4OV4SwWmA3395uL2eB2NB8y8qdQ9muXUdPBWE4l9rMZ6gmfu90N5B5uEl
+          #    94NcfBfYOKi1fJQ9i7WKhTjlRkMCgBkWPkUokvBZFRt8RtF7zI77BSEorHGQCk9t
+          #    /D7BS0GJyfVEhftbWcFEAG3VRcoMhF7kUzYwp+qESoriFRYLeDWv68ZOvG7eoWnP
+          #    PsvZStEVEimjvK5NSESEQa9xWyJOmlOKXhkdymtcUd/nXnx6UTCFgnkgzSdTWV41
+          #    CI6B6aJ9svCTI2QuoIq2HxX/ix7OvW1huVmcyHVxyUECAwEAAaNTMFEwHQYDVR0O
+          #    BBYEFPwN1OceFGm9v6ux8G+DZ3TUDYxqMB8GA1UdIwQYMBaAFPwN1OceFGm9v6ux
+          #    8G+DZ3TUDYxqMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAG5D
+          #    874A4YI7YUwOVsVAdbWtgp1d0zKcPRR+r2OdSbTAV5/gcS3jgBJ3i1BN34JuDVFw
+          #    3DeJSYT3nxy2Y56lLnxDeF8CUTUtVQx3CuGkRg1ouGAHpO/6OqOhwLLorEmxi7tA
+          #    H2O8mtT0poX5AnOAhzVy7QW0D/k4WaoLyckM5hUa6RtvgvLxOwA0U+VGurCDoctu
+          #    8F4QOgTAWyh8EZIwaKCliFRSynDpv3JTUwtfZkxo6K6nce1RhCWFAsMvDZL8Dgc0
+          #    yvgJ38BRsFOtkRuAGSf6ZUwTO8JJRRIFnpUzXflAnGivK9M13D5GEQMmIl6U9Pvk
+          #    sxSmbIUfc2SGJGCJD4I=
+          #    -----END CERTIFICATE-----
+      - name: tcp_options
+        type: yaml
+        title: Custom TCP Options
+        multi: false
+        required: false
+        show_user: false
+        default: |
+          #max_connections: 1
+          #framing: delimitier
+          #line_delimiter: "\n"
+        description: Specify custom configuration options for the TCP input.
+  - input: udp
+    title: "Collect logs via syslog over UDP"
+    description: "Collecting firewall logs from PAN-OS instances (input: udp)"
+    template_path: udp.yml.hbs
+    vars:
+      - name: syslog_host
+        type: text
+        title: Syslog Host
+        multi: false
+        required: true
+        show_user: true
+        default: localhost
+      - name: syslog_port
+        type: integer
+        title: Syslog Port
+        multi: false
+        required: true
+        show_user: true
+        default: 9001
       - name: tags
         type: text
         title: Tags
@@ -45,6 +163,15 @@ streams:
         type: bool
         multi: false
         default: false
+      - name: tz_offset
+        type: text
+        title: Time Zone Offset
+        multi: false
+        required: true
+        show_user: true
+        default: Local
+        description: >-
+          By default, datetimes in the logs without a time zone will be interpreted as relative to the time zone configured in the host where the agent is running. If ingesting logs from a different time zone, use this field to set the time zone offset so that datetimes are correctly parsed. Acceptable time zone formats are: local time (Local), a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00") from UTC.
       - name: processors
         type: yaml
         title: Processors
@@ -54,20 +181,17 @@ streams:
         description: >
           Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
 
-    template_path: syslog.yml.hbs
-    title: Syslog logs
-    description: Collect logs via syslog
   - input: logfile
     vars:
       - name: tz_offset
         type: text
-        title: Timezone Offset
+        title: Time Zone Offset
         multi: false
         required: true
         show_user: true
         default: local
         description: >-
-          By default, datetimes in the logs will be interpreted as relative to the timezone configured in the host where the agent is running. If ingesting logs from a host on a different timezone, use this field to set the timezone offset so that datetimes are correctly parsed. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00") from UTC.
+          By default, datetimes in the logs without a time zone will be interpreted as relative to the time zone configured in the host where the agent is running. If ingesting logs from a different time zone, use this field to set the time zone offset so that datetimes are correctly parsed. Acceptable time zone formats are: local time (Local), a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00") from UTC.
       - name: tags
         type: text
         title: Tags

packages/panw/docs/README.md

@@ -115,6 +115,13 @@ The ingest-geoip Elasticsearch plugin is required to run this module.
 | log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword |
 | log.offset | Offset of the entry in the log file. | long |
 | log.source.address | Source address from which the log event was read / sent from. | keyword |
+| log.syslog.facility.code | The Syslog numeric facility of the log event, if available. According to RFCs 5424 and 3164, this value should be an integer between 0 and 23. | long |
+| log.syslog.facility.name | The Syslog text-based facility of the log event, if available. | keyword |
+| log.syslog.hostname | The hostname, FQDN, or IP of the machine that originally sent the Syslog message. This is sourced from the hostname field of the syslog header. Depending on the environment, this value may be different from the host that handled the event, especially if the host handling the events is acting as a collector. | keyword |
+| log.syslog.priority | Syslog numeric priority of the event, if available. According to RFCs 5424 and 3164, the priority is 8 \* facility + severity. This number is therefore expected to contain a value between 0 and 191. | long |
+| log.syslog.severity.code | The Syslog numeric severity of the log event, if available. If the event source publishing via Syslog provides a different numeric severity value (e.g. firewall, IDS), your source's numeric severity should go to `event.severity`. If the event source does not specify a distinct severity, you can optionally copy the Syslog severity to `event.severity`. | long |
+| log.syslog.severity.name | The Syslog numeric severity of the log event, if available. If the event source publishing via Syslog provides a different severity value (e.g. firewall, IDS), your source's text severity should go to `log.level`. If the event source does not specify a distinct severity, you can optionally copy the Syslog severity to `log.level`. | keyword |
+| log.syslog.version | The version of the Syslog protocol specification. Only applicable for RFC 5424 messages. | keyword |
 | message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text |
 | network.application | When a specific application or service is identified from network connection details (source/dest IPs, ports, certificates, or wire format), this field captures the application's or service's name. For example, the original event identifies the network connection being from a specific web service in a `https` network connection, like `facebook` or `twitter`. The field value must be normalized to lowercase for querying. | keyword |
 | network.bytes | Total bytes transferred in both directions. If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. | long |