diff --git a/packages/network_traffic/changelog.yml b/packages/network_traffic/changelog.yml index 11c806a675c..f392764ebf5 100644 --- a/packages/network_traffic/changelog.yml +++ b/packages/network_traffic/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.10.0" + changes: + - description: Add configuration options for each protocol. + type: enhancement + link: https://github.com/elastic/integrations/pull/3157 - version: "0.9.0" changes: - description: Update to ECS 8.2 diff --git a/packages/network_traffic/data_stream/amqp/agent/stream/amqp.yml.hbs b/packages/network_traffic/data_stream/amqp/agent/stream/amqp.yml.hbs index 8f815e180d8..22fb1883a07 100644 --- a/packages/network_traffic/data_stream/amqp/agent/stream/amqp.yml.hbs +++ b/packages/network_traffic/data_stream/amqp/agent/stream/amqp.yml.hbs @@ -1,12 +1,43 @@ type: amqp -ports: [{{port}}] -{{#if processes}} -procs: - enabled: true - monitored: - {{#each processes}} - - cmdline_grep: {{this}} - {{/each}} +{{#if port}} +ports: +{{#each port as |p|}} + - {{p}} +{{/each}} +{{/if}} +{{#if max_body_length}} +max_body_length: {{max_body_length}} +{{/if}} +{{#if parse_headers}} +parse_headers: {{parse_headers}} +{{/if}} +{{#if parse_arguments}} +parse_arguments: {{parse_arguments}} +{{/if}} +{{#if hide_connection_information}} +hide_connection_information: {{hide_connection_information}} +{{/if}} +{{#if send_request}} +send_request: {{send_request}} +{{/if}} +{{#if send_response}} +send_response: {{send_response}} +{{/if}} +{{#if keep_null}} +keep_null: {{keep_null}} +{{/if}} +{{#if transaction_timeout}} +transaction_timeout: {{transaction_timeout}} +{{/if}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +{{#if tags}} +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} {{/if}} {{#if interface}} interface: diff --git a/packages/network_traffic/data_stream/amqp/manifest.yml b/packages/network_traffic/data_stream/amqp/manifest.yml index ce5ab31ad73..392448511ad 100644 --- a/packages/network_traffic/data_stream/amqp/manifest.yml +++ b/packages/network_traffic/data_stream/amqp/manifest.yml @@ -5,15 +5,101 @@ streams: - input: packet vars: - name: port - type: integer # currently the Kibana UI doesn't support multi inputs # that are numeric, you get "Error: r.toLowerCase is not a function" - # multi: true - title: Port + # so map this as text + type: text + multi: true + title: Ports required: true show_user: true - default: 5672 - # default: [5672] + default: [5672] + - name: max_body_length + type: integer + title: Max Body Length + description: |- + Truncate messages that are published and avoid huge messages being + indexed. + Default: 1000 + show_user: false + multi: false + required: false + - name: parse_headers + type: bool + title: Parse Headers + description: |- + Hide the header fields in header frames. + Default: false + show_user: false + multi: false + required: false + - name: parse_arguments + type: bool + title: Parse Arguments + description: |- + Hide the additional arguments of method frames. + Default: false + show_user: false + multi: false + required: false + - name: hide_connection_information + type: bool + title: Hide Connection Information + description: |- + Hide all methods relative to connection negotiation between server and + client. + Default: true + show_user: false + multi: false + required: false + - name: send_request + type: bool + title: Send Request + description: |- + If this option is enabled, the raw message of the request (`request` field) + is sent to Elasticsearch. The default is false. + show_user: false + multi: false + required: false + - name: send_response + type: bool + title: Send Response + description: |- + If this option is enabled, the raw message of the response (`response` + field) is sent to Elasticsearch. The default is false. + show_user: false + multi: false + required: false + - name: keep_null + type: bool + title: Keep Null + description: Set to true to publish fields with null values in events. + show_user: false + multi: false + required: false + - name: transaction_timeout + type: text + title: Transaction Timeout + description: |- + Transaction timeout. Expired transactions will no longer be correlated to + incoming responses, but sent to Elasticsearch immediately. + show_user: false + multi: false + required: false + - name: processors + type: yaml + title: Processors + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + show_user: false + multi: false + required: false + - name: tags + type: text + title: Tags + description: Tags to include in the published event. + show_user: false + multi: true + required: false title: AMQP description: Capture AMQP Traffic template_path: amqp.yml.hbs diff --git a/packages/network_traffic/data_stream/cassandra/agent/stream/cassandra.yml.hbs b/packages/network_traffic/data_stream/cassandra/agent/stream/cassandra.yml.hbs index c0c76718827..9c4ec167d10 100644 --- a/packages/network_traffic/data_stream/cassandra/agent/stream/cassandra.yml.hbs +++ b/packages/network_traffic/data_stream/cassandra/agent/stream/cassandra.yml.hbs @@ -1,12 +1,43 @@ type: cassandra -ports: [{{port}}] -{{#if processes}} -procs: - enabled: true - monitored: - {{#each processes}} - - cmdline_grep: {{this}} - {{/each}} +{{#if port}} +ports: +{{#each port as |p|}} + - {{p}} +{{/each}} +{{/if}} +{{#if send_request}} +send_request: {{send_request}} +{{/if}} +{{#if send_request_header}} +send_request_header: {{send_request_header}} +{{/if}} +{{#if send_response}} +send_response: {{send_response}} +{{/if}} +{{#if send_response_header}} +send_response_header: {{send_response_header}} +{{/if}} +{{#if keep_null}} +keep_null: {{keep_null}} +{{/if}} +{{#if compressor}} +compressor: {{compressor}} +{{/if}} +{{#if ignored_ops}} +ignored_ops: +{{#each ignored_ops as |ignored_op|}} + - {{ignored_op}} +{{/each}} +{{/if}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +{{#if tags}} +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} {{/if}} {{#if interface}} interface: diff --git a/packages/network_traffic/data_stream/cassandra/manifest.yml b/packages/network_traffic/data_stream/cassandra/manifest.yml index 20ea417599e..b05f2d1e4ed 100644 --- a/packages/network_traffic/data_stream/cassandra/manifest.yml +++ b/packages/network_traffic/data_stream/cassandra/manifest.yml @@ -5,15 +5,88 @@ streams: - input: packet vars: - name: port - type: integer - title: Port - required: true # currently the Kibana UI doesn't support multi inputs # that are numeric, you get "Error: r.toLowerCase is not a function" - # multi: true + # so map this as text + type: text + multi: true + title: Ports + required: true show_user: true - default: 9042 - # default: [9042] + default: [9042] + - name: send_request + type: bool + title: Send Request + description: |- + If this option is enabled, the raw message of the request (`cassandra_request` field) + is included in published events. The default is true. + show_user: false + multi: false + required: false + - name: send_request_header + type: bool + title: Send Request Header + description: |- + If this option is enabled, the raw message of the response (`cassandra_request.request_headers` field) + is included in published events. The default is true. enable `send_request` first before enable this option. + show_user: false + multi: false + required: false + - name: send_response + type: bool + title: Send Response + description: |- + If this option is enabled, the raw message of the response (`cassandra_response` field) + is included in published events. The default is true. + show_user: false + multi: false + required: false + - name: send_response_header + type: bool + title: Send Response Header + description: |- + If this option is enabled, the raw message of the response (`cassandra_response.response_headers` field) + is included in published events. The default is true. enable `send_response` first before enable this option. + show_user: false + multi: false + required: false + - name: keep_null + type: bool + title: Keep Null + description: Set to true to publish fields with null values in events. + show_user: false + multi: false + required: false + - name: compressor + type: text + title: Compressor + description: |- + Configures the default compression algorithm being used to uncompress compressed frames by name. Currently only `snappy` is can be configured. + By default no compressor is configured. + show_user: false + multi: false + required: false + - name: ignored_ops + type: text + title: Ignored Ops + description: This option indicates which Operator/Operators will be ignored. + show_user: false + multi: true + required: false + - name: processors + type: yaml + title: Processors + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + show_user: false + multi: false + required: false + - name: tags + type: text + title: Tags + description: Tags to include in the published event. + show_user: false + multi: true + required: false title: Cassandra description: Capture Cassandra Traffic template_path: cassandra.yml.hbs diff --git a/packages/network_traffic/data_stream/dhcpv4/agent/stream/dhcp.yml.hbs b/packages/network_traffic/data_stream/dhcpv4/agent/stream/dhcp.yml.hbs deleted file mode 100644 index 78c0b643203..00000000000 --- a/packages/network_traffic/data_stream/dhcpv4/agent/stream/dhcp.yml.hbs +++ /dev/null @@ -1,18 +0,0 @@ -type: dhcpv4 -ports: [{{port}}] -{{#if processes}} -procs: - enabled: true - monitored: - {{#each processes}} - - cmdline_grep: {{this}} - {{/each}} -{{/if}} -{{#if interface}} -interface: -{{#if (contains ".pcap" interface)}} - file: {{interface}} -{{else}} - device: {{interface}} -{{/if}} -{{/if}} diff --git a/packages/network_traffic/data_stream/dhcpv4/agent/stream/dhcpv4.yml.hbs b/packages/network_traffic/data_stream/dhcpv4/agent/stream/dhcpv4.yml.hbs new file mode 100644 index 00000000000..2c566382552 --- /dev/null +++ b/packages/network_traffic/data_stream/dhcpv4/agent/stream/dhcpv4.yml.hbs @@ -0,0 +1,28 @@ +type: dhcpv4 +{{#if port}} +ports: +{{#each port as |p|}} + - {{p}} +{{/each}} +{{/if}} +{{#if keep_null}} +keep_null: {{keep_null}} +{{/if}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +{{#if tags}} +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{/if}} +{{#if interface}} +interface: +{{#if (contains ".pcap" interface)}} + file: {{interface}} +{{else}} + device: {{interface}} +{{/if}} +{{/if}} diff --git a/packages/network_traffic/data_stream/dhcpv4/manifest.yml b/packages/network_traffic/data_stream/dhcpv4/manifest.yml index 9bc2cfa4b5d..fc09a927815 100644 --- a/packages/network_traffic/data_stream/dhcpv4/manifest.yml +++ b/packages/network_traffic/data_stream/dhcpv4/manifest.yml @@ -5,15 +5,36 @@ streams: - input: packet vars: - name: port - type: integer # currently the Kibana UI doesn't support multi inputs # that are numeric, you get "Error: r.toLowerCase is not a function" - # multi: true - title: Port + # so map this as text + type: text + multi: true + title: Ports required: true show_user: true - default: 67 - # default: [67, 68] + default: [67, 68] + - name: keep_null + type: bool + title: Keep Null + description: Set to true to publish fields with null values in events. + show_user: false + multi: false + required: false + - name: processors + type: yaml + title: Processors + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + show_user: false + multi: false + required: false + - name: tags + type: text + title: Tags + description: Tags to include in the published event. + show_user: false + multi: true + required: false title: DHCP description: Capture DHCP Traffic - template_path: dhcp.yml.hbs + template_path: dhcpv4.yml.hbs diff --git a/packages/network_traffic/data_stream/dns/agent/stream/dns.yml.hbs b/packages/network_traffic/data_stream/dns/agent/stream/dns.yml.hbs index 43fba04bf53..e68885b2f8f 100644 --- a/packages/network_traffic/data_stream/dns/agent/stream/dns.yml.hbs +++ b/packages/network_traffic/data_stream/dns/agent/stream/dns.yml.hbs @@ -1,12 +1,37 @@ type: dns -ports: [{{port}}] -{{#if processes}} -procs: - enabled: true - monitored: - {{#each processes}} - - cmdline_grep: {{this}} - {{/each}} +{{#if port}} +ports: +{{#each port as |p|}} + - {{p}} +{{/each}} +{{/if}} +{{#if include_authorities}} +include_authorities: {{include_authorities}} +{{/if}} +{{#if include_additionals}} +include_additionals: {{include_additionals}} +{{/if}} +{{#if send_request}} +send_request: {{send_request}} +{{/if}} +{{#if send_response}} +send_response: {{send_response}} +{{/if}} +{{#if keep_null}} +keep_null: {{keep_null}} +{{/if}} +{{#if transaction_timeout}} +transaction_timeout: {{transaction_timeout}} +{{/if}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +{{#if tags}} +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} {{/if}} {{#if interface}} interface: diff --git a/packages/network_traffic/data_stream/dns/manifest.yml b/packages/network_traffic/data_stream/dns/manifest.yml index 032be07aee9..cc5476bfad0 100644 --- a/packages/network_traffic/data_stream/dns/manifest.yml +++ b/packages/network_traffic/data_stream/dns/manifest.yml @@ -5,15 +5,91 @@ streams: - input: packet vars: - name: port - type: integer # currently the Kibana UI doesn't support multi inputs # that are numeric, you get "Error: r.toLowerCase is not a function" - # multi: true - title: Port + # so map this as text + type: text + multi: true + title: Ports required: true show_user: true - default: 53 - # default: [53] + default: [53] + - name: include_authorities + type: bool + title: Include Authorities + description: |- + include_authorities controls whether or not the dns.authorities field + (authority resource records) is added to messages. + Default: false + show_user: false + multi: false + required: false + - name: include_additionals + type: bool + title: Include Additionals + description: |- + include_additionals controls whether or not the dns.additionals field + (additional resource records) is added to messages. + Default: false + show_user: false + multi: false + required: false + - name: send_request + type: bool + title: Send Request + description: |- + send_request controls whether or not the stringified DNS + request messages are added to the result. + Nearly all data about the request/response is available in the dns.* + fields, but this can be useful if you need visibility specifically + into the request or the response. + Default: false + show_user: false + multi: false + required: false + - name: send_response + type: bool + title: Send Response + description: |- + send_response controls whether or not the stringified DNS + response messages are added to the result. + Nearly all data about the request/response is available in the dns.* + fields, but this can be useful if you need visibility specifically + into the request or the response. + Default: false + show_user: false + multi: false + required: false + - name: keep_null + type: bool + title: Keep Null + description: Set to true to publish fields with null values in events. + show_user: false + multi: false + required: false + - name: transaction_timeout + type: text + title: Transaction Timeout + description: |- + Transaction timeout. Expired transactions will no longer be correlated to + incoming responses, but sent to Elasticsearch immediately. + show_user: false + multi: false + required: false + - name: processors + type: yaml + title: Processors + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + show_user: false + multi: false + required: false + - name: tags + type: text + title: Tags + description: Tags to include in the published event. + show_user: false + multi: true + required: false title: DNS description: Capture DNS Traffic template_path: dns.yml.hbs diff --git a/packages/network_traffic/data_stream/http/_dev/test/system/test-http-10-connection-close-config.yml b/packages/network_traffic/data_stream/http/_dev/test/system/test-http-10-connection-close-config.yml index 927668d9708..89f4b901abe 100644 --- a/packages/network_traffic/data_stream/http/_dev/test/system/test-http-10-connection-close-config.yml +++ b/packages/network_traffic/data_stream/http/_dev/test/system/test-http-10-connection-close-config.yml @@ -3,4 +3,4 @@ vars: input: packet data_stream: vars: - port: 8000 + port: [8000] diff --git a/packages/network_traffic/data_stream/http/_dev/test/system/test-http-get-2k-file-config.yml b/packages/network_traffic/data_stream/http/_dev/test/system/test-http-get-2k-file-config.yml index 816ffed836b..eb6eb3c8b49 100644 --- a/packages/network_traffic/data_stream/http/_dev/test/system/test-http-get-2k-file-config.yml +++ b/packages/network_traffic/data_stream/http/_dev/test/system/test-http-get-2k-file-config.yml @@ -3,4 +3,4 @@ vars: input: packet data_stream: vars: - port: 8000 + port: [8000] diff --git a/packages/network_traffic/data_stream/http/_dev/test/system/test-http-minitwit-config.yml b/packages/network_traffic/data_stream/http/_dev/test/system/test-http-minitwit-config.yml index f4dedae8329..6ec7b604fb9 100644 --- a/packages/network_traffic/data_stream/http/_dev/test/system/test-http-minitwit-config.yml +++ b/packages/network_traffic/data_stream/http/_dev/test/system/test-http-minitwit-config.yml @@ -3,4 +3,4 @@ vars: input: packet data_stream: vars: - port: 5000 + port: [5000] diff --git a/packages/network_traffic/data_stream/http/_dev/test/system/test-http-over-vlan-config.yml b/packages/network_traffic/data_stream/http/_dev/test/system/test-http-over-vlan-config.yml index df38ac3b76a..607a49b6dfa 100644 --- a/packages/network_traffic/data_stream/http/_dev/test/system/test-http-over-vlan-config.yml +++ b/packages/network_traffic/data_stream/http/_dev/test/system/test-http-over-vlan-config.yml @@ -3,4 +3,4 @@ vars: input: packet data_stream: vars: - port: 8080 + port: [8080] diff --git a/packages/network_traffic/data_stream/http/_dev/test/system/test-http-post-json-config.yml b/packages/network_traffic/data_stream/http/_dev/test/system/test-http-post-json-config.yml index 2809f53fa63..48a0211d7bd 100644 --- a/packages/network_traffic/data_stream/http/_dev/test/system/test-http-post-json-config.yml +++ b/packages/network_traffic/data_stream/http/_dev/test/system/test-http-post-json-config.yml @@ -3,4 +3,4 @@ vars: input: packet data_stream: vars: - port: 9200 + port: [9200] diff --git a/packages/network_traffic/data_stream/http/_dev/test/system/test-http-realip-config.yml b/packages/network_traffic/data_stream/http/_dev/test/system/test-http-realip-config.yml index 9a7348fd019..200332cd44c 100644 --- a/packages/network_traffic/data_stream/http/_dev/test/system/test-http-realip-config.yml +++ b/packages/network_traffic/data_stream/http/_dev/test/system/test-http-realip-config.yml @@ -3,4 +3,4 @@ vars: input: packet data_stream: vars: - port: 8002 + port: [8002] diff --git a/packages/network_traffic/data_stream/http/agent/stream/http.yml.hbs b/packages/network_traffic/data_stream/http/agent/stream/http.yml.hbs index eaf79676220..4c2aecad102 100644 --- a/packages/network_traffic/data_stream/http/agent/stream/http.yml.hbs +++ b/packages/network_traffic/data_stream/http/agent/stream/http.yml.hbs @@ -1,12 +1,79 @@ type: http -ports: [{{port}}] -{{#if processes}} -procs: - enabled: true - monitored: - {{#each processes}} - - cmdline_grep: {{this}} - {{/each}} +{{#if port}} +ports: +{{#each port as |p|}} + - {{p}} +{{/each}} +{{/if}} +{{#if hide_keywords}} +hide_keywords: +{{#each hide_keywords as |hide_keyword|}} + - {{hide_keyword}} +{{/each}} +{{/if}} +{{#if send_headers}} +send_headers: {{send_headers}} +{{/if}} +{{#if send_all_headers}} +send_all_headers: {{send_all_headers}} +{{/if}} +{{#if redact_headers}} +redact_headers: +{{#each redact_headers as |redact_header|}} + - {{redact_header}} +{{/each}} +{{/if}} +{{#if include_body_for}} +include_body_for: +{{#each include_body_for as |include_body_for_elem|}} + - {{include_body_for_elem}} +{{/each}} +{{/if}} +{{#if include_request_body_for}} +include_request_body_for: +{{#each include_request_body_for as |include_request_body_for_elem|}} + - {{include_request_body_for_elem}} +{{/each}} +{{/if}} +{{#if include_response_body_for}} +include_response_body_for: +{{#each include_response_body_for as |include_response_body_for_elem|}} + - {{include_response_body_for_elem}} +{{/each}} +{{/if}} +{{#if decode_body}} +decode_body: {{decode_body}} +{{/if}} +{{#if split_cookie}} +split_cookie: {{split_cookie}} +{{/if}} +{{#if real_ip_header}} +real_ip_header: {{real_ip_header}} +{{/if}} +{{#if send_request}} +send_request: {{send_request}} +{{/if}} +{{#if send_response}} +send_response: {{send_response}} +{{/if}} +{{#if keep_null}} +keep_null: {{keep_null}} +{{/if}} +{{#if transaction_timeout}} +transaction_timeout: {{transaction_timeout}} +{{/if}} +{{#if max_message_size}} +max_message_size: {{max_message_size}} +{{/if}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +{{#if tags}} +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} {{/if}} {{#if interface}} interface: diff --git a/packages/network_traffic/data_stream/http/manifest.yml b/packages/network_traffic/data_stream/http/manifest.yml index f08abadff2c..f16188331c1 100644 --- a/packages/network_traffic/data_stream/http/manifest.yml +++ b/packages/network_traffic/data_stream/http/manifest.yml @@ -5,15 +5,169 @@ streams: - input: packet vars: - name: port - type: integer # currently the Kibana UI doesn't support multi inputs # that are numeric, you get "Error: r.toLowerCase is not a function" - # multi: true - title: Port + # so map this as text + type: text + multi: true + title: Ports required: true show_user: true - default: 80 - # default: [80, 8080, 8000, 5000, 8002] + default: [80, 8080, 8000, 5000, 8002] + - name: hide_keywords + type: text + title: Hide Keywords + description: |- + Uncomment the following to hide certain parameters in URL or forms attached + to HTTP requests. The names of the parameters are case insensitive. + The value of the parameters will be replaced with the 'xxxxx' string. + This is generally useful for avoiding storing user passwords or other + sensitive information. + Only query parameters and top level form parameters are replaced. + show_user: false + multi: true + required: false + - name: send_headers + type: bool + title: Send Headers + description: |- + A list of header names to capture and send to Elasticsearch. These headers + are placed under the `headers` dictionary in the resulting JSON. + show_user: false + multi: false + required: false + - name: send_all_headers + type: bool + title: Send All Headers + description: |- + Instead of sending a white list of headers to Elasticsearch, you can send + all headers by setting this option to true. The default is false. + show_user: false + multi: false + required: false + - name: redact_headers + type: text + title: Redact Headers + description: |- + A list of headers to redact if present in the HTTP request. This will keep + the header field present, but will redact it's value to show the headers + presence. + show_user: false + multi: true + required: false + - name: include_body_for + type: text + title: Include Body For + description: |- + The list of content types for which Packetbeat includes the full HTTP + payload. If the request's or response's Content-Type matches any on this + list, the full body will be included under the request or response field. + show_user: false + multi: true + required: false + - name: include_request_body_for + type: text + title: Include Request Body For + description: |- + The list of content types for which Packetbeat includes the full HTTP + request payload. + show_user: false + multi: true + required: false + - name: include_response_body_for + type: text + title: Include Response Body For + description: |- + The list of content types for which Packetbeat includes the full HTTP + response payload. + show_user: false + multi: true + required: false + - name: decode_body + type: bool + title: Decode Body + description: |- + Whether the body of a request must be decoded when a content-encoding + or transfer-encoding has been applied. + show_user: false + multi: false + required: false + - name: split_cookie + type: bool + title: Split Cookie + description: |- + If the Cookie or Set-Cookie headers are sent, this option controls whether + they are split into individual values. + show_user: false + multi: false + required: false + - name: real_ip_header + type: bool + title: Real Ip Header + description: |- + The header field to extract the real IP from. This setting is useful when + you want to capture traffic behind a reverse proxy, but you want to get the + geo-location information. + show_user: false + multi: false + required: false + - name: send_request + type: bool + title: Send Request + description: |- + If this option is enabled, the raw message of the request (`request` field) + is sent to Elasticsearch. The default is false. + show_user: false + multi: false + required: false + - name: send_response + type: bool + title: Send Response + description: |- + If this option is enabled, the raw message of the response (`response` + field) is sent to Elasticsearch. The default is false. + show_user: false + multi: false + required: false + - name: keep_null + type: bool + title: Keep Null + description: Set to true to publish fields with null values in events. + show_user: false + multi: false + required: false + - name: transaction_timeout + type: text + title: Transaction Timeout + description: |- + Transaction timeout. Expired transactions will no longer be correlated to + incoming responses, but sent to Elasticsearch immediately. + show_user: false + multi: false + required: false + - name: max_message_size + type: integer + title: Max Message Size + description: |- + Maximum message size. If an HTTP message is larger than this, it will + be trimmed to this size. Default is 10 MB. + show_user: false + multi: false + required: false + - name: processors + type: yaml + title: Processors + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + show_user: false + multi: false + required: false + - name: tags + type: text + title: Tags + description: Tags to include in the published event. + show_user: false + multi: true + required: false title: HTTP description: Capture HTTP Traffic template_path: http.yml.hbs diff --git a/packages/network_traffic/data_stream/icmp/agent/stream/icmp.yml.hbs b/packages/network_traffic/data_stream/icmp/agent/stream/icmp.yml.hbs index 138d78682f7..f550ca79fa5 100644 --- a/packages/network_traffic/data_stream/icmp/agent/stream/icmp.yml.hbs +++ b/packages/network_traffic/data_stream/icmp/agent/stream/icmp.yml.hbs @@ -1,11 +1,16 @@ type: icmp -{{#if processes}} -procs: - enabled: true - monitored: - {{#each processes}} - - cmdline_grep: {{this}} - {{/each}} +{{#if keep_null}} +keep_null: {{keep_null}} +{{/if}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +{{#if tags}} +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} {{/if}} {{#if interface}} interface: diff --git a/packages/network_traffic/data_stream/icmp/manifest.yml b/packages/network_traffic/data_stream/icmp/manifest.yml index 244dc2a9c42..ca911dc8e07 100644 --- a/packages/network_traffic/data_stream/icmp/manifest.yml +++ b/packages/network_traffic/data_stream/icmp/manifest.yml @@ -6,3 +6,25 @@ streams: title: ICMP description: Capture ICMP Traffic template_path: icmp.yml.hbs + vars: + - name: keep_null + type: bool + title: Keep Null + description: Set to true to publish fields with null values in events. + show_user: false + multi: false + required: false + - name: processors + type: yaml + title: Processors + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + show_user: false + multi: false + required: false + - name: tags + type: text + title: Tags + description: Tags to include in the published event. + show_user: false + multi: true + required: false diff --git a/packages/network_traffic/data_stream/memcached/agent/stream/memcached.yml.hbs b/packages/network_traffic/data_stream/memcached/agent/stream/memcached.yml.hbs index 5050fed11ff..136c8ad8779 100644 --- a/packages/network_traffic/data_stream/memcached/agent/stream/memcached.yml.hbs +++ b/packages/network_traffic/data_stream/memcached/agent/stream/memcached.yml.hbs @@ -1,12 +1,43 @@ type: memcache -ports: [{{port}}] -{{#if processes}} -procs: - enabled: true - monitored: - {{#each processes}} - - cmdline_grep: {{this}} - {{/each}} +{{#if port}} +ports: +{{#each port as |p|}} + - {{p}} +{{/each}} +{{/if}} +{{#if parseunknown}} +parseunknown: {{parseunknown}} +{{/if}} +{{#if maxvalues}} +maxvalues: {{maxvalues}} +{{/if}} +{{#if maxbytespervalue}} +maxbytespervalue: {{maxbytespervalue}} +{{/if}} +{{#if udptransactiontimeout}} +udptransactiontimeout: {{udptransactiontimeout}} +{{/if}} +{{#if send_request}} +send_request: {{send_request}} +{{/if}} +{{#if send_response}} +send_response: {{send_response}} +{{/if}} +{{#if keep_null}} +keep_null: {{keep_null}} +{{/if}} +{{#if transaction_timeout}} +transaction_timeout: {{transaction_timeout}} +{{/if}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +{{#if tags}} +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} {{/if}} {{#if interface}} interface: diff --git a/packages/network_traffic/data_stream/memcached/manifest.yml b/packages/network_traffic/data_stream/memcached/manifest.yml index 9d02f60c4a8..9120331b9d9 100644 --- a/packages/network_traffic/data_stream/memcached/manifest.yml +++ b/packages/network_traffic/data_stream/memcached/manifest.yml @@ -5,15 +5,112 @@ streams: - input: packet vars: - name: port - type: integer # currently the Kibana UI doesn't support multi inputs # that are numeric, you get "Error: r.toLowerCase is not a function" - # multi: true - title: Port + # so map this as text + type: text + multi: true + title: Ports required: true show_user: true - default: 11211 - # default: [11211] + default: [11211] + - name: parseunknown + type: bool + title: Parseunknown + description: |- + Uncomment the parseunknown option to force the memcache text protocol parser + to accept unknown commands. + Note: All unknown commands MUST not contain any data parts! + Default: false + show_user: false + multi: false + required: false + - name: maxvalues + type: integer + title: Maxvalues + description: |- + Update the maxvalue option to store the values - base64 encoded - in the + json output. + possible values: + maxvalue: -1 store all values (text based protocol multi-get) + maxvalue: 0 store no values at all + maxvalue: N store up to N values + Default: 0 + show_user: false + multi: false + required: false + - name: maxbytespervalue + type: integer + title: Maxbytespervalue + description: |- + Use maxbytespervalue to limit the number of bytes to be copied per value element. + Note: Values will be base64 encoded, so actual size in json document + will be 4 times maxbytespervalue. + Default: unlimited + show_user: false + multi: false + required: false + - name: udptransactiontimeout + type: integer + title: Udptransactiontimeout + description: |- + UDP transaction timeout in milliseconds. + Note: Quiet messages in UDP binary protocol will get response only in error case. + The memcached analyzer will wait for udptransactiontimeout milliseconds + before publishing quiet messages. Non quiet messages or quiet requests with + error response will not have to wait for the timeout. + Default: 200 + show_user: false + multi: false + required: false + - name: send_request + type: bool + title: Send Request + description: |- + If this option is enabled, the raw message of the request (`request` field) + is sent to Elasticsearch. The default is false. + show_user: false + multi: false + required: false + - name: send_response + type: bool + title: Send Response + description: |- + If this option is enabled, the raw message of the response (`response` + field) is sent to Elasticsearch. The default is false. + show_user: false + multi: false + required: false + - name: keep_null + type: bool + title: Keep Null + description: Set to true to publish fields with null values in events. + show_user: false + multi: false + required: false + - name: transaction_timeout + type: text + title: Transaction Timeout + description: |- + Transaction timeout. Expired transactions will no longer be correlated to + incoming responses, but sent to Elasticsearch immediately. + show_user: false + multi: false + required: false + - name: processors + type: yaml + title: Processors + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + show_user: false + multi: false + required: false + - name: tags + type: text + title: Tags + description: Tags to include in the published event. + show_user: false + multi: true + required: false title: Memcached description: Capture Memcached Traffic template_path: memcached.yml.hbs diff --git a/packages/network_traffic/data_stream/mongodb/agent/stream/mongo.yml.hbs b/packages/network_traffic/data_stream/mongodb/agent/stream/mongo.yml.hbs deleted file mode 100644 index 8f5692b5d5f..00000000000 --- a/packages/network_traffic/data_stream/mongodb/agent/stream/mongo.yml.hbs +++ /dev/null @@ -1,18 +0,0 @@ -type: mongodb -ports: [{{port}}] -{{#if processes}} -procs: - enabled: true - monitored: - {{#each processes}} - - cmdline_grep: {{this}} - {{/each}} -{{/if}} -{{#if interface}} -interface: -{{#if (contains ".pcap" interface)}} - file: {{interface}} -{{else}} - device: {{interface}} -{{/if}} -{{/if}} diff --git a/packages/network_traffic/data_stream/mongodb/agent/stream/mongodb.yml.hbs b/packages/network_traffic/data_stream/mongodb/agent/stream/mongodb.yml.hbs new file mode 100644 index 00000000000..fe92042bcc6 --- /dev/null +++ b/packages/network_traffic/data_stream/mongodb/agent/stream/mongodb.yml.hbs @@ -0,0 +1,43 @@ +type: mongodb +{{#if port}} +ports: +{{#each port as |p|}} + - {{p}} +{{/each}} +{{/if}} +{{#if max_docs}} +max_docs: {{max_docs}} +{{/if}} +{{#if max_doc_length}} +max_doc_length: {{max_doc_length}} +{{/if}} +{{#if send_request}} +send_request: {{send_request}} +{{/if}} +{{#if send_response}} +send_response: {{send_response}} +{{/if}} +{{#if keep_null}} +keep_null: {{keep_null}} +{{/if}} +{{#if transaction_timeout}} +transaction_timeout: {{transaction_timeout}} +{{/if}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +{{#if tags}} +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{/if}} +{{#if interface}} +interface: +{{#if (contains ".pcap" interface)}} + file: {{interface}} +{{else}} + device: {{interface}} +{{/if}} +{{/if}} diff --git a/packages/network_traffic/data_stream/mongodb/manifest.yml b/packages/network_traffic/data_stream/mongodb/manifest.yml index 145edaa8bfc..0ff11578a24 100644 --- a/packages/network_traffic/data_stream/mongodb/manifest.yml +++ b/packages/network_traffic/data_stream/mongodb/manifest.yml @@ -5,15 +5,82 @@ streams: - input: packet vars: - name: port - type: integer # currently the Kibana UI doesn't support multi inputs # that are numeric, you get "Error: r.toLowerCase is not a function" - # multi: true - title: Port + # so map this as text + type: text + multi: true + title: Ports required: true show_user: true - default: 27017 - # default: [27017] + default: [27017] + - name: max_docs + type: integer + title: Max Docs + description: |- + The maximum number of documents from the response to index in the `response` + field. The default is 10. + show_user: false + multi: false + required: false + - name: max_doc_length + type: integer + title: Max Doc Length + description: |- + The maximum number of characters in a single document indexed in the + `response` field. The default is 5000. You can set this to 0 to index an + unlimited number of characters per document. + show_user: false + multi: false + required: false + - name: send_request + type: bool + title: Send Request + description: |- + If this option is enabled, the raw message of the request (`request` field) + is sent to Elasticsearch. The default is false. + show_user: false + multi: false + required: false + - name: send_response + type: bool + title: Send Response + description: |- + If this option is enabled, the raw message of the response (`response` + field) is sent to Elasticsearch. The default is false. + show_user: false + multi: false + required: false + - name: keep_null + type: bool + title: Keep Null + description: Set to true to publish fields with null values in events. + show_user: false + multi: false + required: false + - name: transaction_timeout + type: text + title: Transaction Timeout + description: |- + Transaction timeout. Expired transactions will no longer be correlated to + incoming responses, but sent to Elasticsearch immediately. + show_user: false + multi: false + required: false + - name: processors + type: yaml + title: Processors + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + show_user: false + multi: false + required: false + - name: tags + type: text + title: Tags + description: Tags to include in the published event. + show_user: false + multi: true + required: false title: MongoDB description: Capture MongoDB Traffic - template_path: mongo.yml.hbs + template_path: mongodb.yml.hbs diff --git a/packages/network_traffic/data_stream/mysql/agent/stream/mysql.yml.hbs b/packages/network_traffic/data_stream/mysql/agent/stream/mysql.yml.hbs index d9d4d0e89e2..85b82a47b39 100644 --- a/packages/network_traffic/data_stream/mysql/agent/stream/mysql.yml.hbs +++ b/packages/network_traffic/data_stream/mysql/agent/stream/mysql.yml.hbs @@ -1,12 +1,31 @@ type: mysql -ports: [{{port}}] -{{#if processes}} -procs: - enabled: true - monitored: - {{#each processes}} - - cmdline_grep: {{this}} - {{/each}} +{{#if port}} +ports: +{{#each port as |p|}} + - {{p}} +{{/each}} +{{/if}} +{{#if send_request}} +send_request: {{send_request}} +{{/if}} +{{#if send_response}} +send_response: {{send_response}} +{{/if}} +{{#if keep_null}} +keep_null: {{keep_null}} +{{/if}} +{{#if transaction_timeout}} +transaction_timeout: {{transaction_timeout}} +{{/if}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +{{#if tags}} +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} {{/if}} {{#if interface}} interface: diff --git a/packages/network_traffic/data_stream/mysql/manifest.yml b/packages/network_traffic/data_stream/mysql/manifest.yml index 91d36f45343..c4655854f09 100644 --- a/packages/network_traffic/data_stream/mysql/manifest.yml +++ b/packages/network_traffic/data_stream/mysql/manifest.yml @@ -5,15 +5,63 @@ streams: - input: packet vars: - name: port - type: integer # currently the Kibana UI doesn't support multi inputs # that are numeric, you get "Error: r.toLowerCase is not a function" - # multi: true - title: Port + # so map this as text + type: text + multi: true + title: Ports required: true show_user: true - default: 3306 - # default: [3306, 3307] + default: [3306, 3307] + - name: send_request + type: bool + title: Send Request + description: |- + If this option is enabled, the raw message of the request (`request` field) + is sent to Elasticsearch. The default is false. + show_user: false + multi: false + required: false + - name: send_response + type: bool + title: Send Response + description: |- + If this option is enabled, the raw message of the response (`response` + field) is sent to Elasticsearch. The default is false. + show_user: false + multi: false + required: false + - name: keep_null + type: bool + title: Keep Null + description: Set to true to publish fields with null values in events. + show_user: false + multi: false + required: false + - name: transaction_timeout + type: text + title: Transaction Timeout + description: |- + Transaction timeout. Expired transactions will no longer be correlated to + incoming responses, but sent to Elasticsearch immediately. + show_user: false + multi: false + required: false + - name: processors + type: yaml + title: Processors + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + show_user: false + multi: false + required: false + - name: tags + type: text + title: Tags + description: Tags to include in the published event. + show_user: false + multi: true + required: false title: MySQL description: Capture MySQL Traffic template_path: mysql.yml.hbs diff --git a/packages/network_traffic/data_stream/nfs/agent/stream/nfs.yml.hbs b/packages/network_traffic/data_stream/nfs/agent/stream/nfs.yml.hbs index 25e196a65e5..c8349a7bcb5 100644 --- a/packages/network_traffic/data_stream/nfs/agent/stream/nfs.yml.hbs +++ b/packages/network_traffic/data_stream/nfs/agent/stream/nfs.yml.hbs @@ -1,12 +1,31 @@ type: nfs -ports: [{{port}}] -{{#if processes}} -procs: - enabled: true - monitored: - {{#each processes}} - - cmdline_grep: {{this}} - {{/each}} +{{#if port}} +ports: +{{#each port as |p|}} + - {{p}} +{{/each}} +{{/if}} +{{#if send_request}} +send_request: {{send_request}} +{{/if}} +{{#if send_response}} +send_response: {{send_response}} +{{/if}} +{{#if keep_null}} +keep_null: {{keep_null}} +{{/if}} +{{#if transaction_timeout}} +transaction_timeout: {{transaction_timeout}} +{{/if}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +{{#if tags}} +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} {{/if}} {{#if interface}} interface: diff --git a/packages/network_traffic/data_stream/nfs/manifest.yml b/packages/network_traffic/data_stream/nfs/manifest.yml index 929c6070e68..4e5323fa1e1 100644 --- a/packages/network_traffic/data_stream/nfs/manifest.yml +++ b/packages/network_traffic/data_stream/nfs/manifest.yml @@ -5,15 +5,63 @@ streams: - input: packet vars: - name: port - type: integer # currently the Kibana UI doesn't support multi inputs # that are numeric, you get "Error: r.toLowerCase is not a function" - # multi: true - title: Port + # so map this as text + type: text + multi: true + title: Ports required: true show_user: true - default: 2049 - # default: [2049] + default: [2049] + - name: send_request + type: bool + title: Send Request + description: |- + If this option is enabled, the raw message of the request (`request` field) + is sent to Elasticsearch. The default is false. + show_user: false + multi: false + required: false + - name: send_response + type: bool + title: Send Response + description: |- + If this option is enabled, the raw message of the response (`response` + field) is sent to Elasticsearch. The default is false. + show_user: false + multi: false + required: false + - name: keep_null + type: bool + title: Keep Null + description: Set to true to publish fields with null values in events. + show_user: false + multi: false + required: false + - name: transaction_timeout + type: text + title: Transaction Timeout + description: |- + Transaction timeout. Expired transactions will no longer be correlated to + incoming responses, but sent to Elasticsearch immediately. + show_user: false + multi: false + required: false + - name: processors + type: yaml + title: Processors + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + show_user: false + multi: false + required: false + - name: tags + type: text + title: Tags + description: Tags to include in the published event. + show_user: false + multi: true + required: false title: NFS description: Capture NFS Traffic template_path: nfs.yml.hbs diff --git a/packages/network_traffic/data_stream/pgsql/agent/stream/pgsql.yml.hbs b/packages/network_traffic/data_stream/pgsql/agent/stream/pgsql.yml.hbs new file mode 100644 index 00000000000..8680c36b1a0 --- /dev/null +++ b/packages/network_traffic/data_stream/pgsql/agent/stream/pgsql.yml.hbs @@ -0,0 +1,37 @@ +type: pgsql +{{#if port}} +ports: +{{#each port as |p|}} + - {{p}} +{{/each}} +{{/if}} +{{#if send_request}} +send_request: {{send_request}} +{{/if}} +{{#if send_response}} +send_response: {{send_response}} +{{/if}} +{{#if keep_null}} +keep_null: {{keep_null}} +{{/if}} +{{#if transaction_timeout}} +transaction_timeout: {{transaction_timeout}} +{{/if}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +{{#if tags}} +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{/if}} +{{#if interface}} +interface: +{{#if (contains ".pcap" interface)}} + file: {{interface}} +{{else}} + device: {{interface}} +{{/if}} +{{/if}} diff --git a/packages/network_traffic/data_stream/pgsql/agent/stream/postgres.yml.hbs b/packages/network_traffic/data_stream/pgsql/agent/stream/postgres.yml.hbs deleted file mode 100644 index 9fb8064b900..00000000000 --- a/packages/network_traffic/data_stream/pgsql/agent/stream/postgres.yml.hbs +++ /dev/null @@ -1,18 +0,0 @@ -type: pgsql -ports: [{{port}}] -{{#if processes}} -procs: - enabled: true - monitored: - {{#each processes}} - - cmdline_grep: {{this}} - {{/each}} -{{/if}} -{{#if interface}} -interface: -{{#if (contains ".pcap" interface)}} - file: {{interface}} -{{else}} - device: {{interface}} -{{/if}} -{{/if}} diff --git a/packages/network_traffic/data_stream/pgsql/manifest.yml b/packages/network_traffic/data_stream/pgsql/manifest.yml index d83a022072a..eb205cd8376 100644 --- a/packages/network_traffic/data_stream/pgsql/manifest.yml +++ b/packages/network_traffic/data_stream/pgsql/manifest.yml @@ -5,15 +5,63 @@ streams: - input: packet vars: - name: port - type: integer # currently the Kibana UI doesn't support multi inputs # that are numeric, you get "Error: r.toLowerCase is not a function" - # multi: true - title: Port + # so map this as text + type: text + multi: true + title: Ports required: true show_user: true - default: 5432 - # default: [5432] + default: [5432] + - name: send_request + type: bool + title: Send Request + description: |- + If this option is enabled, the raw message of the request (`request` field) + is sent to Elasticsearch. The default is false. + show_user: false + multi: false + required: false + - name: send_response + type: bool + title: Send Response + description: |- + If this option is enabled, the raw message of the response (`response` + field) is sent to Elasticsearch. The default is false. + show_user: false + multi: false + required: false + - name: keep_null + type: bool + title: Keep Null + description: Set to true to publish fields with null values in events. + show_user: false + multi: false + required: false + - name: transaction_timeout + type: text + title: Transaction Timeout + description: |- + Transaction timeout. Expired transactions will no longer be correlated to + incoming responses, but sent to Elasticsearch immediately. + show_user: false + multi: false + required: false + - name: processors + type: yaml + title: Processors + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + show_user: false + multi: false + required: false + - name: tags + type: text + title: Tags + description: Tags to include in the published event. + show_user: false + multi: true + required: false title: PostgreSQL description: Capture PostgreSQL Traffic - template_path: postgres.yml.hbs + template_path: pgsql.yml.hbs diff --git a/packages/network_traffic/data_stream/redis/_dev/test/system/test-redis-one-transaction-config.yml b/packages/network_traffic/data_stream/redis/_dev/test/system/test-redis-one-transaction-config.yml index 920822ab0a5..212ff1a2e07 100644 --- a/packages/network_traffic/data_stream/redis/_dev/test/system/test-redis-one-transaction-config.yml +++ b/packages/network_traffic/data_stream/redis/_dev/test/system/test-redis-one-transaction-config.yml @@ -3,4 +3,4 @@ vars: input: packet data_stream: vars: - port: 6380 + port: [6380] diff --git a/packages/network_traffic/data_stream/redis/_dev/test/system/test-redis-session-config.yml b/packages/network_traffic/data_stream/redis/_dev/test/system/test-redis-session-config.yml index cd81b8fdf84..6f6e40948b4 100644 --- a/packages/network_traffic/data_stream/redis/_dev/test/system/test-redis-session-config.yml +++ b/packages/network_traffic/data_stream/redis/_dev/test/system/test-redis-session-config.yml @@ -3,4 +3,4 @@ vars: input: packet data_stream: vars: - port: 6380 + port: [6380] diff --git a/packages/network_traffic/data_stream/redis/agent/stream/redis.yml.hbs b/packages/network_traffic/data_stream/redis/agent/stream/redis.yml.hbs index d4f89e180c0..f357ca3a6d1 100644 --- a/packages/network_traffic/data_stream/redis/agent/stream/redis.yml.hbs +++ b/packages/network_traffic/data_stream/redis/agent/stream/redis.yml.hbs @@ -1,12 +1,37 @@ type: redis -ports: [{{port}}] -{{#if processes}} -procs: - enabled: true - monitored: - {{#each processes}} - - cmdline_grep: {{this}} - {{/each}} +{{#if port}} +ports: +{{#each port as |p|}} + - {{p}} +{{/each}} +{{/if}} +{{#if send_request}} +send_request: {{send_request}} +{{/if}} +{{#if send_response}} +send_response: {{send_response}} +{{/if}} +{{#if keep_null}} +keep_null: {{keep_null}} +{{/if}} +{{#if transaction_timeout}} +transaction_timeout: {{transaction_timeout}} +{{/if}} +{{#if queue_max_bytes}} +queue_max_bytes: {{queue_max_bytes}} +{{/if}} +{{#if queue_max_messages}} +queue_max_messages: {{queue_max_messages}} +{{/if}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +{{#if tags}} +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} {{/if}} {{#if interface}} interface: diff --git a/packages/network_traffic/data_stream/redis/manifest.yml b/packages/network_traffic/data_stream/redis/manifest.yml index 1e9896b997c..9fe0ce4e18e 100644 --- a/packages/network_traffic/data_stream/redis/manifest.yml +++ b/packages/network_traffic/data_stream/redis/manifest.yml @@ -5,15 +5,82 @@ streams: - input: packet vars: - name: port - type: integer # currently the Kibana UI doesn't support multi inputs # that are numeric, you get "Error: r.toLowerCase is not a function" - # multi: true - title: Port + # so map this as text + type: text + multi: true + title: Ports required: true show_user: true - default: 6379 - # default: [6379] + default: [6379] + - name: send_request + type: bool + title: Send Request + description: |- + If this option is enabled, the raw message of the request (`request` field) + is sent to Elasticsearch. The default is false. + show_user: false + multi: false + required: false + - name: send_response + type: bool + title: Send Response + description: |- + If this option is enabled, the raw message of the response (`response` + field) is sent to Elasticsearch. The default is false. + show_user: false + multi: false + required: false + - name: keep_null + type: bool + title: Keep Null + description: Set to true to publish fields with null values in events. + show_user: false + multi: false + required: false + - name: transaction_timeout + type: text + title: Transaction Timeout + description: |- + Transaction timeout. Expired transactions will no longer be correlated to + incoming responses, but sent to Elasticsearch immediately. + show_user: false + multi: false + required: false + - name: queue_max_bytes + type: integer + title: Queue Max Bytes + description: |- + Max size for per-session message queue. This places a limit on the memory + that can be used to buffer requests and responses for correlation. + show_user: false + multi: false + required: false + - name: queue_max_messages + type: integer + title: Queue Max Messages + description: |- + Max number of messages for per-session message queue. This limits the number + of requests or responses that can be buffered for correlation. Set a value + large enough to allow for pipelining. + show_user: false + multi: false + required: false + - name: processors + type: yaml + title: Processors + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + show_user: false + multi: false + required: false + - name: tags + type: text + title: Tags + description: Tags to include in the published event. + show_user: false + multi: true + required: false title: Redis description: Capture Redis Traffic template_path: redis.yml.hbs diff --git a/packages/network_traffic/data_stream/sip/agent/stream/sip.yml.hbs b/packages/network_traffic/data_stream/sip/agent/stream/sip.yml.hbs index 6e87ba42e1a..935ea011ee1 100644 --- a/packages/network_traffic/data_stream/sip/agent/stream/sip.yml.hbs +++ b/packages/network_traffic/data_stream/sip/agent/stream/sip.yml.hbs @@ -1,12 +1,28 @@ type: sip -ports: [{{port}}] -{{#if processes}} -procs: - enabled: true - monitored: - {{#each processes}} - - cmdline_grep: {{this}} - {{/each}} +{{#if port}} +ports: +{{#each port as |p|}} + - {{p}} +{{/each}} +{{/if}} +{{#if parse_authorization}} +parse_authorization: {{parse_authorization}} +{{/if}} +{{#if parse_body}} +parse_body: {{parse_body}} +{{/if}} +{{#if keep_original}} +keep_original: {{keep_original}} +{{/if}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +{{#if tags}} +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} {{/if}} {{#if interface}} interface: diff --git a/packages/network_traffic/data_stream/sip/manifest.yml b/packages/network_traffic/data_stream/sip/manifest.yml index 29213ce9a3f..79dd27ea524 100644 --- a/packages/network_traffic/data_stream/sip/manifest.yml +++ b/packages/network_traffic/data_stream/sip/manifest.yml @@ -5,15 +5,50 @@ streams: - input: packet vars: - name: port - type: integer # currently the Kibana UI doesn't support multi inputs # that are numeric, you get "Error: r.toLowerCase is not a function" - # multi: true - title: Port + # so map this as text + type: text + multi: true + title: Ports required: true show_user: true - default: 5060 - # default: [5060] + default: [5060] + - name: parse_authorization + type: bool + title: Parse Authorization + description: Parse the authorization headers + show_user: false + multi: false + required: false + - name: parse_body + type: bool + title: Parse Body + description: Parse body contents (only when body is SDP) + show_user: false + multi: false + required: false + - name: keep_original + type: bool + title: Keep Original + description: Preserve original contents in event.original + show_user: false + multi: false + required: false + - name: processors + type: yaml + title: Processors + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + show_user: false + multi: false + required: false + - name: tags + type: text + title: Tags + description: Tags to include in the published event. + show_user: false + multi: true + required: false title: SIP description: Capture SIP Traffic template_path: sip.yml.hbs diff --git a/packages/network_traffic/data_stream/thrift/agent/stream/thrift.yml.hbs b/packages/network_traffic/data_stream/thrift/agent/stream/thrift.yml.hbs index 3029157c2be..d6d96042535 100644 --- a/packages/network_traffic/data_stream/thrift/agent/stream/thrift.yml.hbs +++ b/packages/network_traffic/data_stream/thrift/agent/stream/thrift.yml.hbs @@ -1,12 +1,58 @@ type: thrift -ports: [{{port}}] -{{#if processes}} -procs: - enabled: true - monitored: - {{#each processes}} - - cmdline_grep: {{this}} - {{/each}} +{{#if port}} +ports: +{{#each port as |p|}} + - {{p}} +{{/each}} +{{/if}} +{{#if transport_type}} +transport_type: {{transport_type}} +{{/if}} +{{#if protocol_type}} +protocol_type: {{protocol_type}} +{{/if}} +{{#if idl_files}} +idl_files: +{{#each idl_files as |idl_file|}} + - {{idl_file}} +{{/each}} +{{/if}} +{{#if string_max_size}} +string_max_size: {{string_max_size}} +{{/if}} +{{#if collection_max_size}} +collection_max_size: {{collection_max_size}} +{{/if}} +{{#if capture_reply}} +capture_reply: {{capture_reply}} +{{/if}} +{{#if obfuscate_strings}} +obfuscate_strings: {{obfuscate_strings}} +{{/if}} +{{#if drop_after_n_struct_fields}} +drop_after_n_struct_fields: {{drop_after_n_struct_fields}} +{{/if}} +{{#if send_request}} +send_request: {{send_request}} +{{/if}} +{{#if send_response}} +send_response: {{send_response}} +{{/if}} +{{#if keep_null}} +keep_null: {{keep_null}} +{{/if}} +{{#if transaction_timeout}} +transaction_timeout: {{transaction_timeout}} +{{/if}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +{{#if tags}} +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} {{/if}} {{#if interface}} interface: diff --git a/packages/network_traffic/data_stream/thrift/manifest.yml b/packages/network_traffic/data_stream/thrift/manifest.yml index 779ac934958..29eabbeb193 100644 --- a/packages/network_traffic/data_stream/thrift/manifest.yml +++ b/packages/network_traffic/data_stream/thrift/manifest.yml @@ -5,15 +5,137 @@ streams: - input: packet vars: - name: port - type: integer # currently the Kibana UI doesn't support multi inputs # that are numeric, you get "Error: r.toLowerCase is not a function" - # multi: true - title: Port + # so map this as text + type: text + multi: true + title: Ports required: true show_user: true - default: 9090 - # default: [9090] + default: [9090] + - name: transport_type + type: text + title: Transport Type + description: |- + The Thrift transport type. Currently this option accepts the values socket + for TSocket, which is the default Thrift transport, and framed for the + TFramed Thrift transport. The default is socket. + show_user: false + multi: false + required: false + - name: protocol_type + type: text + title: Protocol Type + description: |- + The Thrift protocol type. Currently the only accepted value is binary for + the TBinary protocol, which is the default Thrift protocol. + show_user: false + multi: false + required: false + - name: idl_files + type: text + title: Idl Files + description: |- + The Thrift interface description language (IDL) files for the service that + Packetbeat is monitoring. Providing the IDL enables Packetbeat to include + parameter and exception names. + show_user: false + multi: true + required: false + - name: string_max_size + type: integer + title: String Max Size + description: |- + The maximum length for strings in parameters or return values. If a string + is longer than this value, the string is automatically truncated to this + length. + show_user: false + multi: false + required: false + - name: collection_max_size + type: integer + title: Collection Max Size + description: The maximum number of elements in a Thrift list, set, map, or structure. + show_user: false + multi: false + required: false + - name: capture_reply + type: bool + title: Capture Reply + description: |- + If this option is set to false, Packetbeat decodes the method name from the + reply and simply skips the rest of the response message. + show_user: false + multi: false + required: false + - name: obfuscate_strings + type: bool + title: Obfuscate Strings + description: |- + If this option is set to true, Packetbeat replaces all strings found in + method parameters, return codes, or exception structures with the "*" + string. + show_user: false + multi: false + required: false + - name: drop_after_n_struct_fields + type: integer + title: Drop After N Struct Fields + description: |- + The maximum number of fields that a structure can have before Packetbeat + ignores the whole transaction. + show_user: false + multi: false + required: false + - name: send_request + type: bool + title: Send Request + description: |- + If this option is enabled, the raw message of the request (`request` field) + is sent to Elasticsearch. The default is false. + show_user: false + multi: false + required: false + - name: send_response + type: bool + title: Send Response + description: |- + If this option is enabled, the raw message of the response (`response` + field) is sent to Elasticsearch. The default is false. + show_user: false + multi: false + required: false + - name: keep_null + type: bool + title: Keep Null + description: Set to true to publish fields with null values in events. + show_user: false + multi: false + required: false + - name: transaction_timeout + type: text + title: Transaction Timeout + description: |- + Transaction timeout. Expired transactions will no longer be correlated to + incoming responses, but sent to Elasticsearch immediately. + show_user: false + multi: false + required: false + - name: processors + type: yaml + title: Processors + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + show_user: false + multi: false + required: false + - name: tags + type: text + title: Tags + description: Tags to include in the published event. + show_user: false + multi: true + required: false title: Thrift description: Capture Thrift Traffic template_path: thrift.yml.hbs diff --git a/packages/network_traffic/data_stream/tls/agent/stream/tls.yml.hbs b/packages/network_traffic/data_stream/tls/agent/stream/tls.yml.hbs index e4c7f923641..877a553bfd0 100644 --- a/packages/network_traffic/data_stream/tls/agent/stream/tls.yml.hbs +++ b/packages/network_traffic/data_stream/tls/agent/stream/tls.yml.hbs @@ -1,12 +1,34 @@ type: tls -ports: [{{port}}] -{{#if processes}} -procs: - enabled: true - monitored: - {{#each processes}} - - cmdline_grep: {{this}} - {{/each}} +{{#if port}} +ports: +{{#each port as |p|}} + - {{p}} +{{/each}} +{{/if}} +{{#if fingerprints}} +fingerprints: +{{#each fingerprints as |fingerprint|}} + - {{fingerprint}} +{{/each}} +{{/if}} +{{#if send_certificates}} +send_certificates: {{send_certificates}} +{{/if}} +{{#if include_raw_certificates}} +include_raw_certificates: {{include_raw_certificates}} +{{/if}} +{{#if keep_null}} +keep_null: {{keep_null}} +{{/if}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +{{#if tags}} +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} {{/if}} {{#if interface}} interface: diff --git a/packages/network_traffic/data_stream/tls/manifest.yml b/packages/network_traffic/data_stream/tls/manifest.yml index 3aeb1834854..d2b8f403da6 100644 --- a/packages/network_traffic/data_stream/tls/manifest.yml +++ b/packages/network_traffic/data_stream/tls/manifest.yml @@ -5,15 +5,63 @@ streams: - input: packet vars: - name: port - type: integer # currently the Kibana UI doesn't support multi inputs # that are numeric, you get "Error: r.toLowerCase is not a function" - # multi: true - title: Port + # so map this as text + type: text + multi: true + title: Ports required: true show_user: true - default: 443 - # default: [443, 993, 995, 5223, 8443, 8883, 9243] + default: [443, 993, 995, 5223, 8443, 8883, 9243] + - name: fingerprints + type: text + title: Fingerprints + description: |- + List of hash algorithms to use to calculate certificates' fingerprints. + Valid values are `sha1`, `sha256` and `md5`. + show_user: false + multi: true + required: false + - name: send_certificates + type: bool + title: Send Certificates + description: |- + If this option is enabled, the client and server certificates and + certificate chains are sent to Elasticsearch. The default is true. + show_user: false + multi: false + required: false + - name: include_raw_certificates + type: bool + title: Include Raw Certificates + description: |- + If this option is enabled, the raw certificates will be stored + in PEM format under the `raw` key. The default is false. + show_user: false + multi: false + required: false + - name: keep_null + type: bool + title: Keep Null + description: Set to true to publish fields with null values in events. + show_user: false + multi: false + required: false + - name: processors + type: yaml + title: Processors + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + show_user: false + multi: false + required: false + - name: tags + type: text + title: Tags + description: Tags to include in the published event. + show_user: false + multi: true + required: false title: TLS description: Capture TLS Traffic template_path: tls.yml.hbs diff --git a/packages/network_traffic/manifest.yml b/packages/network_traffic/manifest.yml index 09d640389d8..3fa9e55cab3 100644 --- a/packages/network_traffic/manifest.yml +++ b/packages/network_traffic/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: network_traffic title: Network Packet Capture -version: 0.9.0 +version: 0.10.0 license: basic description: Capture and analyze network traffic from a host with Elastic Agent. type: integration