[tcp/udp] Add option to parse syslog

packages/tcp/_dev/deploy/docker/docker-compose.yml

@@ -5,3 +5,13 @@ services:
     volumes:
       - ./sample_logs:/sample_logs:ro
     command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9515 -p=tcp /sample_logs/test-tcp.log
+  test-tls:
+    image: docker.elastic.co/observability/stream:v0.6.1
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9516 -p=tls --insecure /sample_logs/test-tcp.log
+  test-syslog:
+    image: docker.elastic.co/observability/stream:v0.6.1
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9517 -p=tcp /sample_logs/test-tcp.log

packages/tcp/_dev/deploy/docker/sample_logs/test-tcp.log

@@ -1 +1 @@
-<134>1 2020-03-29T13:19:20Z testhostname testproductname 1930 - some longer testmessage. - a {2:2}.
+<134>1 2020-03-29T13:19:20Z testhostname testproductname 1930 - - some longer testmessage. - a {2:2}.

packages/tcp/changelog.yml

@@ -1,3 +1,8 @@
+- version: "1.3.0"
+  changes:
+    - description: Add syslog parsing option
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3587
 - version: "1.2.0"
   changes:
     - description: Update package to ECS 8.3.0.

packages/tcp/data_stream/generic/_dev/test/system/test-syslog-config.yml

@@ -0,0 +1,8 @@
+service: test-syslog
+service_notify_signal: SIGHUP
+input: tcp
+data_stream:
+  vars:
+    listen_address: 0.0.0.0
+    listen_port: 9517
+    syslog: true

packages/tcp/data_stream/generic/_dev/test/system/test-tcp-config.yml

@@ -5,54 +5,3 @@ data_stream:
   vars:
     listen_address: 0.0.0.0
     listen_port: 9515
-    ssl: |-
-      certificate:
-        -----BEGIN CERTIFICATE-----
-        MIIDJjCCAg6gAwIBAgIRAO76bP2QhJVqbLjcsWD6gkUwDQYJKoZIhvcNAQELBQAw
-        JjEkMCIGA1UEChMbVEVTVCAtIEVsYXN0aWMgSW50ZWdyYXRpb25zMB4XDTIxMDIw
-        MjE2NTUzOVoXDTQxMDEyODE2NTUzOVowJjEkMCIGA1UEChMbVEVTVCAtIEVsYXN0
-        aWMgSW50ZWdyYXRpb25zMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
-        vUs3YgX4RnEKzSyqH599ffHMDidw3JUTNzp/alRByiGN2gnC2YLLeB8gbZHn2Xkl
-        YCET1oUrVmPAijwV2RzPYwIn0kIh4zVOKO7+RDCCrgq8CgIG1xZyUhMF3uwn868r
-        SmX5FZ3T3/max51EsAJmzawef1TqrQRdEKxuPBQs/4qWaYeQCeYeZBVcg2b8CUmg
-        3w1lB072Xzt7cJUp8FU1s7U9Hfgg2Dslh9+DSVX0yoqwN8Ynw4FXMSyqAu/OdBbG
-        aidOR6YjlKx3OSUUYsuB7q3XDyigb6Va7W737QTLIhtEb56l4E0iO4jDT41LaYyw
-        vRpWegfvHoFBHXbXT6AxtQIDAQABo08wTTAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0l
-        BAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAYBgNVHREEETAPgg1lbGFzdGlj
-        LWFnZW50MA0GCSqGSIb3DQEBCwUAA4IBAQAcFeQS0QtPFpGMQ55NO+ycsXAsYZsJ
-        XvdUMoGygkkbrUQXmQbMMSMPGAGdMfc9V6BMA8x6JgGyKZBcIN/RTkBKjpXFwL03
-        su+9liQnIMbYFvBfc1HDjAN5u2HpMdH0sCOe0W4XF5r6n8Q+6WuCl51HND6ObsyR
-        nU/7PySQ6Bv2PftPI1LMFeLsmgQsCJ/z8jcP4oW4PtgyK7vb+NWGLzRnkgaHYqh3
-        oT7VnxPZQtWBJQa2LJhcp+u5k2Y6PipAyh4mCm/IRr1UHpGT/qBGnaUC+DRWd/pk
-        T4UnmUgq6eJL4IY+v/wpUPS+uHVtFhPSvRp+5hhicuK1YN4Ug/qKirVs
-        -----END CERTIFICATE-----
-      key:
-        -----BEGIN PRIVATE KEY-----
-        MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC9SzdiBfhGcQrN
-        LKofn3198cwOJ3DclRM3On9qVEHKIY3aCcLZgst4HyBtkefZeSVgIRPWhStWY8CK
-        PBXZHM9jAifSQiHjNU4o7v5EMIKuCrwKAgbXFnJSEwXe7CfzrytKZfkVndPf+ZrH
-        nUSwAmbNrB5/VOqtBF0QrG48FCz/ipZph5AJ5h5kFVyDZvwJSaDfDWUHTvZfO3tw
-        lSnwVTWztT0d+CDYOyWH34NJVfTKirA3xifDgVcxLKoC7850FsZqJ05HpiOUrHc5
-        JRRiy4HurdcPKKBvpVrtbvftBMsiG0RvnqXgTSI7iMNPjUtpjLC9GlZ6B+8egUEd
-        dtdPoDG1AgMBAAECggEAP2ks+ldJnj9MAQNPUhyZa1FOrAcmVZ5Su5OLD1F+YHnx
-        DPNsJHUeN/UlZc8UvdNJY/RwstIVfHEaFLSgFQUDrAUS1ep1c6ltr2SwJKOjgy3x
-        Y+Dd7buFPF1HADBYCdfKRrf2QvmF+mehI/FZCyUizw8zgDAwFRl7G5THsLSJhmiQ
-        wDc9WbPFLyswtmeKoAqMiHHqV63PtJunqvGbrDTHh9f4P5JVtreMoPWzE9czQ2ZI
-        5nBHOFP/EA6twyRalqOsm3XoFmyrWMmJtm/JJsDlGr/LZcVbtghxybEYo8p/VLpo
-        JmBSJgM17rwGhniDWXWXXOfx2fkNZEhVIeGvZYJRgQKBgQDOHnepihIu650pTfRD
-        fcUyPN9oYLzI2mwv70H3FzJQftt3pqmWhlX2adaXYJ65/8xwr6SmkHmYjTvfuCoT
-        SFApzv9fnYcD6vCsk5AhLpbarWR3MEU1SCvaiFuRNrdTcR8MGSglWPLLVXCI6f/g
-        F9kZ/Ngz7MkvD2bNT/WjNj3LMQKBgQDrGmPo0gvfk+QoFtL05+dDDrB2IxUokdqa
-        RzdecC8wV01l8lIj4TDqo7W1wwxdEUvCbUYriE2BoXi1v3jF+wfluqJOL30Ex5kb
-        UO5At+DWakxzgy3v0F32AOZRISAGMdbrNFaLpjD9t9NGbL8kiestfs2QuTISHJwU
-        fD47jFDlxQKBgHrczGVh6O7RAVByqCxm1tnYUS8torpzAFQeYQrBZ/t1cqrCzInu
-        L2V/tytqq5KheKKfAB1NNz4IyezUITh3PVl+itja1HUwYR/todc1pzRYcO9e9ZIK
-        ICHWcAaCQArb/i6+/CAvAiLUHg1utlhEvuNvxQxGk7Gak6PEit4r4e+xAoGBAIOR
-        rT/p7IMefJyCyWQNM7qvScmTMJAXr8KPAEl1drMS6FmZFqbFq15kZ5hko1KiD0er
-        Z42NJfLZrnfnw2roZS8HFzWyFcDLAr/qtqq5PLZBnq82RkrizPKS5lGYvBc7ZQ8T
-        pytXwir66N2MlhuYo2g+gkPvoDnKkP5V2W3xxIQRAoGBAIDayGKqE1iZwF72R0xQ
-        Vg8y2x9JoxY1lDGA8oLzYKcp7OslI6sPhv/NGnkQBwV964dcffnn6dezFyKKBGir
-        DSiM9duWTttlzzUhUQMHCua2z/LXjz1XMb0LoSEOVdk00TDgRMSFhBLhr3ZXmoLb
-        Iqi7is4z2mP8pbcIIlmloogE
-        -----END PRIVATE KEY-----
-      verification_mode: none

packages/tcp/data_stream/generic/_dev/test/system/test-tls-config.yml

@@ -0,0 +1,59 @@
+service: test-tls
+service_notify_signal: SIGHUP
+input: tcp
+data_stream:
+  vars:
+    listen_address: 0.0.0.0
+    listen_port: 9516
+    ssl: |
+      key: |
+        -----BEGIN PRIVATE KEY-----
+        MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDhCLvLsQAHufsN
+        U+u1x/CequAUphfXZqLhDo2Eo/holfBS0+ey4bnzPL6lS9NFL5JkLQA2gYESqsXU
+        /Ru8E76Az1egzMwT3TVAPLVU8NbrxBqeNiQa2m9wC37HQy4qC9OxL28LUoKtFjxS
+        cD1sa0oikXCJN1a3BSoAf9iiZ/dxz4WVfrNhrzq2JFXjravY84n5ujkZOg45Pg70
+        4vHOeg0rBbIoSNfjDUVZWjwC95K1BMN3msOTL9juv/EDa6BujqCxl+G1nY7JPFDL
+        SHWis65p+1AAa5xieYDb47vyJ0SSR7lEURTXZOkkM6k5JWfgkATEmGzRxPkOloIT
+        Xg9ag1OlAgMBAAECggEAEHfPJmzhj68wjB0kFr13AmWG2Hv/Kqg8KzQhbx+AwkaW
+        u7j+L70NGpvLZ9VQtLNyhxoz9cksZO1SZO/Q48aeHlcOFppmJN3/U6AdtQWa9M35
+        FLLpmX16wjxVHsfvzOvopgLOoYl8PqZt66qDFDgVyMnT7na6RdJ+7GJuvBPXq+Bc
+        vgThvAZitHSAOhnBFYmTMlBi6AzOMMsaFlgE3Xf9v3M0pAKItPRKMhXlC3MyvA/v
+        jgbra4Ib+0ryohggHheHB3bn3Jgv7iFKoW9OQSePVxacJ+kfr9H+No5g495URzqR
+        mx/96WCiv3rAh3ct8Sk/C4/3zMC8fUueDJIVjhgw0QKBgQD8NufLINNkIpBrLoCS
+        972oFEjZB2u6EusQ7X9raROqpaw26ZSu+zSHeIKCGQ93M3aRb3FpdGeOxgZ095MV
+        8a+nlh4stOvHj2Mm5YhTBDUavTC7o9aVR3Od5eTXUpHnaJpNI/uyIcKupeK1UJnV
+        UlBLeIwo/vJ1gsVrKMMAJkuKbwKBgQDkaWRRd0w2gUIbCTGf203BqXft0VdIiOW7
+        +gnkeaNHAf09XljzxMcQzrB8kG63aKVGbJffphEfzxtiJ+HRQVH+7QpKRhU/GHmu
+        +6OKkxTcxJm5zhoRFxcSi2wG4PWmUGJvc7ss1OJGcaOUxwocCepO7N/jfdDz9Uke
+        KnA+YWOdKwKBgQDteZkYlojT0QOgF8HyH5gQyUCqMKWLJ0LzxltiPCbLV4Dml1pq
+        w5Z7M8nWS1hXiTpLx93GSFc1hFkSCwYP9GfK6Lryp0sVtHnMZvTMDbseuSJImwRx
+        vDwtYQfugg1lEQWwOoBEAiu3m/PxernNtNprpU57T0nlwUK3GkM5QdWAuwKBgQCZ
+        ZF3GiANapzupxGbbH//8Cr9LqsafI7CEqMpz8WxBh4h16iJ6sq+tDeFgBe8UpOY5
+        gTwNKg1d+0w8guQYD3HtbWr3rlEeamVtqfiOW3ArQqyqJ0tCJuuLvK3zgKf35Qv2
+        JRaSaPT8sdxVUcXsRoxgLJu+vwPQke1koMN4YRbwuQKBgQDJiZ/WSeqa5oIqkXbn
+        hjm7RXKaf2oE1U/bNjdSFtdEP7T4vUvvr7Hq2f/jiBLtCE7w16PJjKx9iIq2+jMl
+        qIY43Sk9bdi5FxtYTHda0hwrbH274P+QVcVs5PXCT0TGktOleHGBlXaaPrxl9iCh
+        8tmmxZZYa5aQxEO/lxB9xQKaiQ==
+        -----END PRIVATE KEY-----
+      certificate: |
+        -----BEGIN CERTIFICATE-----
+        MIIDazCCAlOgAwIBAgIUW5TDu1tJMY2Oa7PsL+BQSmeWqz0wDQYJKoZIhvcNAQEL
+        BQAwRTELMAkGA1UEBhMCVVMxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+        GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMTEwMDEwNTAwMjNaFw0yMTEw
+        MDIwNTAwMjNaMEUxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
+        HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB
+        AQUAA4IBDwAwggEKAoIBAQDhCLvLsQAHufsNU+u1x/CequAUphfXZqLhDo2Eo/ho
+        lfBS0+ey4bnzPL6lS9NFL5JkLQA2gYESqsXU/Ru8E76Az1egzMwT3TVAPLVU8Nbr
+        xBqeNiQa2m9wC37HQy4qC9OxL28LUoKtFjxScD1sa0oikXCJN1a3BSoAf9iiZ/dx
+        z4WVfrNhrzq2JFXjravY84n5ujkZOg45Pg704vHOeg0rBbIoSNfjDUVZWjwC95K1
+        BMN3msOTL9juv/EDa6BujqCxl+G1nY7JPFDLSHWis65p+1AAa5xieYDb47vyJ0SS
+        R7lEURTXZOkkM6k5JWfgkATEmGzRxPkOloITXg9ag1OlAgMBAAGjUzBRMB0GA1Ud
+        DgQWBBRYUSKDHBBE9Q6fTeTqogicCxcXwDAfBgNVHSMEGDAWgBRYUSKDHBBE9Q6f
+        TeTqogicCxcXwDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBc
+        T8B+GpvPy9NQ700LsywRPY0L9IJCKiu6j3TP1tqqSPjAC/cg9ac+bFXuWOu7V+KJ
+        s09Q/pItq9SLX6UvnfRzTxu5lCBwwGX9cL131mTIu5SmFo7Eks+sorbiIarWDMoC
+        e+9An3GFpagW+YhOt4BdIM5lTqoeodzganDBsOUZI9aDAj2Yo5h2O7r6Wd12cb6T
+        mz8vMfB2eG8BxU20ZMfkdERWjiyXHOSBQqeqfkV8d9370gMu5RcJNcIgnbmTRdho
+        X3HJFiimZVaNjXATqmC/y2A1KXvJdamPLy3mGXkW2cFLoPCdK2OZFUHqiuc1bigA
+        qEf55SihFqErRMeURPPF
+        -----END CERTIFICATE-----

packages/tcp/data_stream/generic/agent/stream/tcp.yml.hbs

@@ -37,5 +37,13 @@ publisher_pipeline.disable_host: true
 {{/contains}}
 {{#if processors}}
 processors:
+{{#if syslog}}
+  - syslog:
+      {{syslog_options}}
+{{/if}}
 {{processors}}
-{{/if}}
+{{else if syslog}}
+processors:
+  - syslog:
+      {{syslog_options}}
+{{/if}}

packages/tcp/data_stream/generic/fields/ecs.yml

@@ -7,3 +7,21 @@
   description: The IP or DNS name of the source sending the UDP packet.
 - name: message
   external: ecs
+- name: log.syslog.appname
+  external: ecs
+- name: log.syslog.facility.code
+  external: ecs
+- name: log.syslog.facility.name
+  external: ecs
+- name: log.syslog.hostname
+  external: ecs
+- name: log.syslog.priority
+  external: ecs
+- name: log.syslog.procid
+  external: ecs
+- name: log.syslog.severity.code
+  external: ecs
+- name: log.syslog.severity.name
+  external: ecs
+- name: log.syslog.version
+  external: ecs

packages/tcp/data_stream/generic/manifest.yml

@@ -88,3 +88,36 @@ streams:
         required: false
         multi: true
         show_user: true
+      - name: syslog
+        type: bool
+        title: Syslog Parsing
+        description: Enable the syslog parser to automatically parse syslog data. The syslog parser can be configured under Advanced Options.
+        required: false
+        show_user: true
+      - name: syslog_options
+        type: yaml
+        title: Syslog Configuration
+        description: i.e. field, format, time zone, etc. See [Syslog](https://www.elastic.co/guide/en/beats/filebeat/current/syslog.html) for details.
+        multi: false
+        required: false
+        show_user: false
+        default: |
+          field: message
+          #format: auto
+          #timezone: Local
+      - name: ssl
+        type: yaml
+        title: SSL Configuration
+        description: i.e. certificate, keys, supported_protocols, verification_mode etc. See [SSL](https://www.elastic.co/guide/en/beats/filebeat/current/configuration-ssl.html#ssl-server-config) for details.
+        multi: false
+        required: false
+        show_user: false
+        default: |
+          #certificate: |
+          #    -----BEGIN CERTIFICATE-----
+          #    ...
+          #    -----END CERTIFICATE-----
+          #key: |
+          #    -----BEGIN PRIVATE KEY-----
+          #    ...
+          #    -----END PRIVATE KEY-----

packages/tcp/manifest.yml

@@ -3,10 +3,10 @@ name: tcp
 title: Custom TCP Logs
 description: Collect raw TCP data from listening TCP port with Elastic Agent.
 type: integration
-version: "1.2.0"
+version: "1.3.0"
 release: ga
 conditions:
-  kibana.version: "^7.16.0 || ^8.0.0"
+  kibana.version: "^8.2.1"
 license: basic
 categories:
   - custom

packages/udp/_dev/deploy/docker/docker-compose.yml

@@ -5,3 +5,8 @@ services:
     volumes:
       - ./sample_logs:/sample_logs:ro
     command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9515 -p=udp /sample_logs/test-udp.log
+  test-syslog:
+    image: docker.elastic.co/observability/stream:v0.6.1
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9516 -p=udp /sample_logs/test-udp.log

packages/udp/_dev/deploy/docker/sample_logs/test-udp.log

@@ -1 +1 @@
-<134>1 2020-03-29T13:19:20Z testhostname testproductname 1930 - some longer testmessage. - a {2:2}.
+<134>1 2020-03-29T13:19:20Z testhostname testproductname 1930 - - some longer testmessage. - a {2:2}.

packages/udp/changelog.yml

@@ -1,3 +1,8 @@
+- version: "1.3.0"
+  changes:
+    - description: Add syslog parsing option, expose SSL config
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3587
 - version: "1.2.0"
   changes:
     - description: Update package to ECS 8.3.0.

packages/udp/data_stream/generic/_dev/test/system/test-syslog-config.yml

@@ -0,0 +1,8 @@
+service: test-syslog
+service_notify_signal: SIGHUP
+input: udp
+data_stream:
+  vars:
+    listen_address: 0.0.0.0
+    listen_port: 9516
+    syslog: true

packages/udp/data_stream/generic/agent/stream/udp.yml.hbs

@@ -27,5 +27,13 @@ publisher_pipeline.disable_host: true
 {{/contains}}
 {{#if processors}}
 processors:
+{{#if syslog}}
+  - syslog:
+      {{syslog_options}}
+{{/if}}
 {{processors}}
-{{/if}}
+{{else if syslog}}
+processors:
+  - syslog:
+      {{syslog_options}}
+{{/if}}

packages/udp/data_stream/generic/fields/ecs.yml

@@ -7,3 +7,21 @@
   description: The IP or DNS name of the source sending the UDP packet.
 - name: message
   external: ecs
+- name: log.syslog.appname
+  external: ecs
+- name: log.syslog.facility.code
+  external: ecs
+- name: log.syslog.facility.name
+  external: ecs
+- name: log.syslog.hostname
+  external: ecs
+- name: log.syslog.priority
+  external: ecs
+- name: log.syslog.procid
+  external: ecs
+- name: log.syslog.severity.code
+  external: ecs
+- name: log.syslog.severity.name
+  external: ecs
+- name: log.syslog.version
+  external: ecs

packages/udp/data_stream/generic/manifest.yml

@@ -79,3 +79,20 @@ streams:
         required: false
         multi: true
         show_user: true
+      - name: syslog
+        type: bool
+        title: Syslog Parsing
+        description: Enable the syslog parser to automatically parse syslog data. The syslog parser can be configured under Advanced Options.
+        required: false
+        show_user: true
+      - name: syslog_options
+        type: yaml
+        title: Syslog Options
+        description: i.e. format, time zone, etc.
+        multi: false
+        required: false
+        show_user: false
+        default: |
+          field: message
+          #format: auto
+          #timezone: Local

packages/udp/manifest.yml

@@ -3,10 +3,10 @@ name: udp
 title: Custom UDP Logs
 description: Collect raw UDP data from listening UDP port with Elastic Agent.
 type: integration
-version: "1.2.0"
+version: "1.3.0"
 release: ga
 conditions:
-  kibana.version: "^7.16.0 || ^8.0.0"
+  kibana.version: "^8.2.1"
 license: basic
 categories:
   - custom