diff --git a/.backportrc.json b/.backportrc.json new file mode 100644 index 000000000..133b70f9d --- /dev/null +++ b/.backportrc.json @@ -0,0 +1,18 @@ +{ + "repoOwner": "elastic", + "repoName": "rally-tracks", + "targetBranchChoices": [ + "8.15", + "8.14", + "8.13" + ], + "targetPRLabels": [ + "backport" + ], + "branchLabelMapping": { + "^backport-to-(.+)$": "$1" + }, + "autoMerge": true, + "autoMergeMethod": "squash", + "prDescription": "{defaultPrDescription}\n\n" +} diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml new file mode 100644 index 000000000..4e139fea2 --- /dev/null +++ b/.github/workflows/backport.yml @@ -0,0 +1,27 @@ +name: Automatic backport action + +on: + pull_request_target: + branches: ['master'] + types: ["labeled", "closed"] + +jobs: + backport: + name: Backport PR + runs-on: ubuntu-latest + if: | + github.event.pull_request.merged == true + && !contains(github.event.pull_request.labels.*.name, 'backport') + steps: + - name: Backport Action + uses: sorenlouv/backport-github-action@v9.5.1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Info log + if: ${{ success() }} + run: cat ~/.backport/backport.info.log + + - name: Debug log + if: ${{ failure() }} + run: cat ~/.backport/backport.debug.log diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99e7e3e67..43683846f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,11 +79,11 @@ jobs: python-version: ${{ matrix.python-version }} cache: pip cache-dependency-path: pyproject.toml - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: "temurin" - java-version: "17" - - run: echo "JAVA17_HOME=$JAVA_HOME_17_X64" >> $GITHUB_ENV + java-version: "21" + - run: echo "JAVA21_HOME=$JAVA_HOME_21_X64" >> $GITHUB_ENV - run: echo "JAVA11_HOME=$JAVA_HOME_11_X64" >> $GITHUB_ENV - name: "Install dependencies" run: python -m pip install .[develop] diff --git a/.gitignore b/.gitignore index 3239aea4f..73633544c 100644 --- a/.gitignore +++ b/.gitignore @@ -106,4 +106,10 @@ target/ .venv # pytest -.pytest_cache \ No newline at end of file +.pytest_cache + +# Editors +*~ +/.project +/.pydevproject +/.vscode diff --git a/dense_vector/README.md b/dense_vector/README.md index 0417610af..e1c283b2a 100644 --- a/dense_vector/README.md +++ b/dense_vector/README.md @@ -33,4 +33,4 @@ This track accepts the following parameters with Rally 0.8.0+ using `--track-par * `number_of_replicas` (default: 0) * `post_ingest_sleep` (default: false): Whether to pause after ingest and prior to subsequent operations. * `post_ingest_sleep_duration` (default: 30): Sleep duration in seconds. -* `vector_index_type` (default: "hnsw"): The index kind for storing the vectors. +* `vector_index_type` (default: "int8_hnsw"): The index kind for storing the vectors. diff --git a/dense_vector/index.json b/dense_vector/index.json index 58ee28f04..ce0331ece 100644 --- a/dense_vector/index.json +++ b/dense_vector/index.json @@ -19,7 +19,7 @@ "index" : true, "similarity": "dot_product", "index_options": { - "type": {{ vector_index_type | default("hnsw") | tojson }}, + "type": {{ vector_index_type | default("int8_hnsw") | tojson }}, "m": 32, "ef_construction": 100 } diff --git a/elastic/logs/README.md b/elastic/logs/README.md index b7aac8c27..d68b52bf9 100644 --- a/elastic/logs/README.md +++ b/elastic/logs/README.md @@ -220,6 +220,7 @@ The following parameters are available: * `corpora_uri_base` (default: `https://rally-tracks.elastic.co`) - Specify the base location of the datasets used by this track. * `lifecycle` (default: unset to fall back on Serverless detection) - Specifies the lifecycle management feature to use for data streams. Use `ilm` for index lifecycle management or `dlm` for data lifecycle management. By default, `dlm` will be used for benchmarking Serverless Elasticsearch. * `workflow-request-cache` (default: `true`) - Explicit control of request cache query parameter in searches executed in a workflow. This can be further overriden at an operation level with `request-cache` parameter. +* `synthetic_source_keep` (default: unset) - Allows overriding the default synthetic source behaviour for all field types with the following values: `none` (equivalent to unset) - no source is stored, `arrays` - source stored as is only for multi-value (array) fields. ### Data Download Parameters diff --git a/elastic/logs/challenges/logging-querying.json b/elastic/logs/challenges/logging-querying.json index e0793fa11..4cd30a4aa 100644 --- a/elastic/logs/challenges/logging-querying.json +++ b/elastic/logs/challenges/logging-querying.json @@ -79,6 +79,25 @@ {%- endif -%}{# non-serverless-index-statistics-marker-end #} , {% endif %} + { + "operation": "discovery-search-request-size-100", + "clients": 1, + "warmup-iterations": 0, + "iterations": 50 + }, + { + "operation": "discovery-search-request-size-500", + "clients": 1, + "warmup-iterations": 0, + "iterations": 50 + }, + + { + "operation": "discovery-search-request-size-1000", + "clients": 1, + "warmup-iterations": 0, + "iterations": 50 + }, { "name": "logging-queries", "parallel": { diff --git a/elastic/logs/operations/search-with-source.json b/elastic/logs/operations/search-with-source.json new file mode 100644 index 000000000..31939793d --- /dev/null +++ b/elastic/logs/operations/search-with-source.json @@ -0,0 +1,300 @@ + { + "name": "discovery-search-request-size-100", + "operation-type": "search", + "index": "logs-*", + "request-params": { + "batched_reduce_size": "64", + "ignore_unavailable": "true", + "track_total_hits": "false", + "enable_fields_emulation": "true", + "preference": "1650039059666" + }, + "body": { + "sort": [ + { + "@timestamp": { + "order": "desc", + "unmapped_type": "boolean" + } + } + ], + "fields": [ + { + "field": "*", + "include_unmapped": "true" + }, + { + "field": "@timestamp", + "format": "strict_date_optional_time" + }, + { + "field": "eden.created_at", + "format": "strict_date_optional_time" + }, + { + "field": "event.created", + "format": "strict_date_optional_time" + }, + { + "field": "event.end", + "format": "strict_date_optional_time" + }, + { + "field": "event.ingested", + "format": "strict_date_optional_time" + }, + { + "field": "event.start", + "format": "strict_date_optional_time" + }, + { + "field": "file.accessed", + "format": "strict_date_optional_time" + }, + { + "field": "file.created", + "format": "strict_date_optional_time" + }, + { + "field": "file.ctime", + "format": "strict_date_optional_time" + }, + { + "field": "file.mtime", + "format": "strict_date_optional_time" + }, + { + "field": "postgresql.log.session_start_time", + "format": "strict_date_optional_time" + }, + { + "field": "process.parent.start", + "format": "strict_date_optional_time" + }, + { + "field": "process.start", + "format": "strict_date_optional_time" + } + ], + "size": 100, + "version": true, + "script_fields": {}, + "stored_fields": [ + "*" + ], + "runtime_mappings": {}, + "_source": false, + "highlight": { + "pre_tags": [ + "@kibana-highlighted-field@" + ], + "post_tags": [ + "@/kibana-highlighted-field@" + ], + "fields": { + "*": {} + }, + "fragment_size": 2147483647 + } + } + }, + { + "name": "discovery-search-request-size-500", + "operation-type": "search", + "index": "logs-*", + "request-params": { + "batched_reduce_size": "64", + "ignore_unavailable": "true", + "track_total_hits": "false", + "enable_fields_emulation": "true", + "preference": "1650039059666" + }, + "body": { + "sort": [ + { + "@timestamp": { + "order": "desc", + "unmapped_type": "boolean" + } + } + ], + "fields": [ + { + "field": "*", + "include_unmapped": "true" + }, + { + "field": "@timestamp", + "format": "strict_date_optional_time" + }, + { + "field": "eden.created_at", + "format": "strict_date_optional_time" + }, + { + "field": "event.created", + "format": "strict_date_optional_time" + }, + { + "field": "event.end", + "format": "strict_date_optional_time" + }, + { + "field": "event.ingested", + "format": "strict_date_optional_time" + }, + { + "field": "event.start", + "format": "strict_date_optional_time" + }, + { + "field": "file.accessed", + "format": "strict_date_optional_time" + }, + { + "field": "file.created", + "format": "strict_date_optional_time" + }, + { + "field": "file.ctime", + "format": "strict_date_optional_time" + }, + { + "field": "file.mtime", + "format": "strict_date_optional_time" + }, + { + "field": "postgresql.log.session_start_time", + "format": "strict_date_optional_time" + }, + { + "field": "process.parent.start", + "format": "strict_date_optional_time" + }, + { + "field": "process.start", + "format": "strict_date_optional_time" + } + ], + "size": 500, + "version": true, + "script_fields": {}, + "stored_fields": [ + "*" + ], + "runtime_mappings": {}, + "_source": false, + "highlight": { + "pre_tags": [ + "@kibana-highlighted-field@" + ], + "post_tags": [ + "@/kibana-highlighted-field@" + ], + "fields": { + "*": {} + }, + "fragment_size": 2147483647 + } + } + }, + { + "name": "discovery-search-request-size-1000", + "operation-type": "search", + "index": "logs-*", + "request-params": { + "batched_reduce_size": "64", + "ignore_unavailable": "true", + "track_total_hits": "false", + "enable_fields_emulation": "true", + "preference": "1650039059666" + }, + "body": { + "sort": [ + { + "@timestamp": { + "order": "desc", + "unmapped_type": "boolean" + } + } + ], + "fields": [ + { + "field": "*", + "include_unmapped": "true" + }, + { + "field": "@timestamp", + "format": "strict_date_optional_time" + }, + { + "field": "eden.created_at", + "format": "strict_date_optional_time" + }, + { + "field": "event.created", + "format": "strict_date_optional_time" + }, + { + "field": "event.end", + "format": "strict_date_optional_time" + }, + { + "field": "event.ingested", + "format": "strict_date_optional_time" + }, + { + "field": "event.start", + "format": "strict_date_optional_time" + }, + { + "field": "file.accessed", + "format": "strict_date_optional_time" + }, + { + "field": "file.created", + "format": "strict_date_optional_time" + }, + { + "field": "file.ctime", + "format": "strict_date_optional_time" + }, + { + "field": "file.mtime", + "format": "strict_date_optional_time" + }, + { + "field": "postgresql.log.session_start_time", + "format": "strict_date_optional_time" + }, + { + "field": "process.parent.start", + "format": "strict_date_optional_time" + }, + { + "field": "process.start", + "format": "strict_date_optional_time" + } + ], + "size": 1000, + "version": true, + "script_fields": {}, + "stored_fields": [ + "*" + ], + "runtime_mappings": {}, + "_source": false, + "highlight": { + "pre_tags": [ + "@kibana-highlighted-field@" + ], + "post_tags": [ + "@/kibana-highlighted-field@" + ], + "fields": { + "*": {} + }, + "fragment_size": 2147483647 + } + } + } \ No newline at end of file diff --git a/elastic/logs/pipelines/.fleet_final_pipeline-1.json b/elastic/logs/pipelines/.fleet_final_pipeline-1.json index e23682f8e..f77d0f90c 100644 --- a/elastic/logs/pipelines/.fleet_final_pipeline-1.json +++ b/elastic/logs/pipelines/.fleet_final_pipeline-1.json @@ -1,22 +1,19 @@ { - "version": 1, + "version": 4, + "_meta": { + "managed_by": "fleet", + "managed": true + }, "description": "Final pipeline for processing all incoming Fleet Agent documents.", "processors": [ - { - "set": { - "description": "Add time when event was ingested.", - "field": "event.ingested", - "copy_from": "_ingest.timestamp" - } - }, { "script": { - "description": "Remove sub-seconds from event.ingested to improve storage efficiency.", + "description": "Add time when event was ingested (and remove sub-seconds to improve storage efficiency)", "tag": "truncate-subseconds-event-ingested", - "source": "ctx.event.ingested = ctx.event.ingested.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);", - "ignore_failure": true - } - }, + "ignore_failure": true, + "source": "if (ctx?.event == null) { ctx.event = [:]; } ctx.event.ingested = metadata().now.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);" + } + }, { "remove": { "description": "Remove any pre-existing untrusted values.", @@ -27,6 +24,15 @@ "ignore_missing": true } }, + { + "remove": { + "description": "Remove event.original unless the preserve_original_event tag is set", + "field": "event.original", + "if": "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))", + "ignore_failure": true, + "ignore_missing": true + } + }, { "set_security_user": { "field": "_security", @@ -85,4 +91,4 @@ } } ] -} +} \ No newline at end of file diff --git a/elastic/logs/pipelines/logs-apache.access-1.18.0.json b/elastic/logs/pipelines/logs-apache.access-1.18.0.json new file mode 100644 index 000000000..e9a75c410 --- /dev/null +++ b/elastic/logs/pipelines/logs-apache.access-1.18.0.json @@ -0,0 +1,289 @@ +{ + "description": "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins.", + "processors": [ + { + "pipeline": { + "if": "ctx.message.startsWith('{')", + "name": "logs-apache.access-1.18.0-third-party" + } + }, + { + "set": { + "field": "event.ingested", + "value": "{{_ingest.timestamp}}" + } + }, + { + "set": { + "field": "ecs.version", + "value": "8.5.1" + } + }, + { + "rename": { + "field": "message", + "target_field": "event.original", + "ignore_missing": true, + "if": "ctx.event?.original == null" + } + }, + { + "remove": { + "field": "message", + "ignore_missing": true, + "if": "ctx.event?.original != null", + "description": "The `message` field is no longer required if the document has an `event.original` field." + } + }, + { + "grok": { + "field": "event.original", + "patterns": [ + "(%{IPORHOST:destination.domain} )?%{IPORHOST:source.address} %{DATA:apache.access.identity} %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?( X-Forwarded-For=\"%{ADDRESS_LIST:apache.access.remote_addresses}\")?", + "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -", + "\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}\" (-|%{NUMBER:http.response.body.bytes:long})" + ], + "ignore_missing": true, + "pattern_definitions": { + "ADDRESS_LIST": "(%{IP})(\"?,?\\s*(%{IP}))*" + } + } + }, + { + "split": { + "field": "apache.access.remote_addresses", + "separator": "\"?,\\s*", + "ignore_missing": true + } + }, + { + "set": { + "field": "network.forwarded_ip", + "value": "{{{apache.access.remote_addresses.0}}}", + "if": "ctx.apache?.access?.remote_addresses != null && ctx.apache.access.remote_addresses.length > 0" + } + }, + { + "script": { + "if": "ctx.apache?.access?.remote_addresses != null && ctx.apache.access.remote_addresses.length > 0", + "lang": "painless", + "tag": "Get source address", + "description": "Extract from remote_addresses, the first non-private IP to ctx.client.ip", + "source": "boolean isPrivateCIDR(def ip) {\n CIDR class_a_network = new CIDR('10.0.0.0/8');\n CIDR class_b_network = new CIDR('172.16.0.0/12');\n CIDR class_c_network = new CIDR('192.168.0.0/16');\n\n try {\n return class_a_network.contains(ip) || class_b_network.contains(ip) || class_c_network.contains(ip);\n } catch (IllegalArgumentException e) {\n return false;\n }\n}\ntry {\n if (ctx.client == null) {\n Map map = new HashMap();\n ctx.put(\"client\", map);\n }\n\n def found = false;\n for (def item : ctx.apache.access.remote_addresses) {\n if (!isPrivateCIDR(item)) {\n ctx.client.ip = item;\n found = true;\n break;\n }\n }\n if (!found) {\n ctx.client.ip = ctx.apache.access.remote_addresses[0];\n }\n} catch (Exception e) {\n ctx.client.ip = null;\n}" + } + }, + { + "append": { + "field": "apache.access.remote_addresses", + "value": [ + "{{source.address}}" + ], + "if": "ctx.source?.address != null" + } + }, + { + "uri_parts": { + "field": "_tmp.url_orig", + "ignore_failure": true + } + }, + { + "remove": { + "field": [ + "_tmp" + ], + "ignore_missing": true + } + }, + { + "set": { + "field": "url.domain", + "value": "{{destination.domain}}", + "if": "ctx.url?.domain == null && ctx.destination?.domain != null" + } + }, + { + "set": { + "field": "event.kind", + "value": "event" + } + }, + { + "append": { + "field": "event.category", + "value": "web" + } + }, + { + "set": { + "field": "event.outcome", + "value": "success", + "if": "ctx.http?.response?.status_code != null && ctx.http.response.status_code < 400" + } + }, + { + "set": { + "field": "event.outcome", + "value": "failure", + "if": "ctx.http?.response?.status_code != null && ctx.http.response.status_code > 399" + } + }, + { + "grok": { + "field": "source.address", + "ignore_missing": true, + "patterns": [ + "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$" + ] + } + }, + { + "remove": { + "field": "event.created", + "ignore_missing": true, + "ignore_failure": true + } + }, + { + "rename": { + "field": "@timestamp", + "target_field": "event.created" + } + }, + { + "date": { + "field": "apache.access.time", + "target_field": "@timestamp", + "formats": [ + "dd/MMM/yyyy:H:m:s Z" + ], + "ignore_failure": true + } + }, + { + "remove": { + "field": "apache.access.time", + "ignore_failure": true + } + }, + { + "user_agent": { + "field": "user_agent.original", + "ignore_failure": true + } + }, + { + "geoip": { + "field": "source.ip", + "target_field": "source.geo", + "ignore_missing": true + } + }, + { + "geoip": { + "database_file": "GeoLite2-ASN.mmdb", + "field": "source.ip", + "target_field": "source.as", + "properties": [ + "asn", + "organization_name" + ], + "ignore_missing": true + } + }, + { + "rename": { + "field": "source.as.asn", + "target_field": "source.as.number", + "ignore_missing": true + } + }, + { + "rename": { + "field": "source.as.organization_name", + "target_field": "source.as.organization.name", + "ignore_missing": true + } + }, + { + "set": { + "field": "tls.cipher", + "value": "{{apache.access.ssl.cipher}}", + "if": "ctx.apache?.access?.ssl?.cipher != null" + } + }, + { + "script": { + "lang": "painless", + "if": "ctx.apache?.access?.ssl?.protocol != null", + "source": "def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken(\"v\"); if (parts.length != 2) {\n return;\n} if (parts[1].contains(\".\")) {\n ctx.tls.version = parts[1];\n} else {\n ctx.tls.version = parts[1] + \".0\";\n} ctx.tls.version_protocol = parts[0];" + } + }, + { + "script": { + "lang": "painless", + "description": "This script processor iterates over the whole document to remove fields with null values.", + "source": "void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null);\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n" + } + }, + { + "remove": { + "field": "event.original", + "if": "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))", + "ignore_failure": true, + "ignore_missing": true + } + }, + { + "remove": { + "field": "apache.access.identity", + "if": "ctx.apache?.access?.identity == \"-\"", + "ignore_failure": true, + "ignore_missing": true + } + }, + { + "pipeline": { + "name": "global@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Global pipeline for all data streams" + } + }, + { + "pipeline": { + "name": "logs@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs`" + } + }, + { + "pipeline": { + "name": "logs-apache.integration@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `apache` integration" + } + }, + { + "pipeline": { + "name": "logs-apache.access@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for the `apache.access` dataset" + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ], + "_meta": { + "managed_by": "fleet", + "managed": true, + "package": { + "name": "apache" + } + } +} \ No newline at end of file diff --git a/elastic/logs/pipelines/logs-apache.access-1.3.0.json b/elastic/logs/pipelines/logs-apache.access-1.3.0.json deleted file mode 100644 index c33473b6c..000000000 --- a/elastic/logs/pipelines/logs-apache.access-1.3.0.json +++ /dev/null @@ -1,202 +0,0 @@ -{ - "description" : "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins.", - "processors" : [ - { - "pipeline" : { - "if" : "ctx.message.startsWith('{')", - "name" : "logs-apache.access-1.3.0-third-party" - } - }, - { - "set" : { - "field" : "event.ingested", - "value" : "{{_ingest.timestamp}}" - } - }, - { - "set" : { - "field" : "ecs.version", - "value" : "1.12.0" - } - }, - { - "rename" : { - "field" : "message", - "target_field" : "event.original" - } - }, - { - "grok" : { - "field" : "event.original", - "patterns" : [ - "%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?", - "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?", - "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -", - "\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}\" (-|%{NUMBER:http.response.body.bytes:long})" - ], - "ignore_missing" : true - } - }, - { - "uri_parts" : { - "field" : "_tmp.url_orig", - "ignore_failure" : true - } - }, - { - "remove" : { - "field" : [ - "_tmp" - ], - "ignore_missing" : true - } - }, - { - "set" : { - "field" : "url.domain", - "value" : "{{destination.domain}}", - "if" : "ctx.url?.domain == null && ctx.destination?.domain != null" - } - }, - { - "set" : { - "field" : "event.kind", - "value" : "event" - } - }, - { - "set" : { - "field" : "event.category", - "value" : "web" - } - }, - { - "set" : { - "field" : "event.outcome", - "value" : "success", - "if" : "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" - } - }, - { - "set" : { - "field" : "event.outcome", - "value" : "failure", - "if" : "ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399" - } - }, - { - "grok" : { - "field" : "source.address", - "ignore_missing" : true, - "patterns" : [ - "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$" - ] - } - }, - { - "remove" : { - "field" : "event.created", - "ignore_missing" : true, - "ignore_failure" : true - } - }, - { - "rename" : { - "field" : "@timestamp", - "target_field" : "event.created" - } - }, - { - "date" : { - "field" : "apache.access.time", - "target_field" : "@timestamp", - "formats" : [ - "dd/MMM/yyyy:H:m:s Z" - ], - "ignore_failure" : true - } - }, - { - "remove" : { - "field" : "apache.access.time", - "ignore_failure" : true - } - }, - { - "user_agent" : { - "field" : "user_agent.original", - "ignore_failure" : true - } - }, - { - "geoip" : { - "field" : "source.ip", - "target_field" : "source.geo", - "ignore_missing" : true - } - }, - { - "geoip" : { - "database_file" : "GeoLite2-ASN.mmdb", - "field" : "source.ip", - "target_field" : "source.as", - "properties" : [ - "asn", - "organization_name" - ], - "ignore_missing" : true - } - }, - { - "rename" : { - "field" : "source.as.asn", - "target_field" : "source.as.number", - "ignore_missing" : true - } - }, - { - "rename" : { - "field" : "source.as.organization_name", - "target_field" : "source.as.organization.name", - "ignore_missing" : true - } - }, - { - "set" : { - "field" : "tls.cipher", - "value" : "{{apache.access.ssl.cipher}}", - "if" : "ctx?.apache?.access?.ssl?.cipher != null" - } - }, - { - "script" : { - "lang" : "painless", - "if" : "ctx?.apache?.access?.ssl?.protocol != null", - "source" : "def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken(\"v\"); if (parts.length != 2) {return;} if (parts[1].contains(\".\")) {ctx.tls.version = parts[1];} else {ctx.tls.version = parts[1] + \".0\";} ctx.tls.version_protocol = parts[0];" - } - }, - { - "script" : { - "lang" : "painless", - "description" : "This script processor iterates over the whole document to remove fields with null values.", - "source" : "void handleMap(Map map) {for (def x : map.values()) {if (x instanceof Map) {handleMap(x);} else if (x instanceof List) {handleList(x);}}map.values().removeIf(v -> v == null);}void handleList(List list) {for (def x : list) {if (x instanceof Map) {handleMap(x);} else if (x instanceof List) {handleList(x);}}}handleMap(ctx);" - } - }, - { - "remove" : { - "field" : "event.original", - "if" : "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))", - "ignore_failure" : true, - "ignore_missing" : true - } - } - ], - "on_failure" : [ - { - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/elastic/logs/pipelines/logs-apache.error-1.18.0.json b/elastic/logs/pipelines/logs-apache.error-1.18.0.json new file mode 100644 index 000000000..8e6d4069e --- /dev/null +++ b/elastic/logs/pipelines/logs-apache.error-1.18.0.json @@ -0,0 +1,235 @@ +{ + "description": "Pipeline for parsing apache error logs", + "processors": [ + { + "pipeline": { + "if": "ctx.message.startsWith('{')", + "name": "logs-apache.error-1.18.0-third-party" + } + }, + { + "set": { + "field": "event.ingested", + "value": "{{_ingest.timestamp}}" + } + }, + { + "set": { + "field": "ecs.version", + "value": "8.5.1" + } + }, + { + "rename": { + "field": "message", + "target_field": "event.original", + "ignore_missing": true, + "if": "ctx.event?.original == null" + } + }, + { + "remove": { + "field": "message", + "ignore_missing": true, + "if": "ctx.event?.original != null", + "description": "The `message` field is no longer required if the document has an `event.original` field." + } + }, + { + "grok": { + "field": "event.original", + "patterns": [ + "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{LOGLEVEL:log.level}\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}", + "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{DATA:apache.error.module}:%{APACHE_LOGLEVEL:log.level}\\] \\[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}" + ], + "pattern_definitions": { + "APACHE_LOGLEVEL": "%{LOGLEVEL}[0-9]*", + "APACHE_TIME": "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}" + }, + "ignore_missing": true + } + }, + { + "grok": { + "field": "message", + "patterns": [ + "File does not exist: %{URIPATH:file.path}, referer: %{URI:http.request.referrer}", + "File does not exist: %{URIPATH:file.path}" + ], + "ignore_missing": true, + "ignore_failure": true + } + }, + { + "date": { + "if": "ctx.event.timezone == null", + "field": "apache.error.timestamp", + "target_field": "@timestamp", + "formats": [ + "EEE MMM dd H:m:s yyyy", + "EEE MMM dd H:m:s.SSSSSS yyyy" + ], + "on_failure": [ + { + "append": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] + } + }, + { + "date": { + "if": "ctx.event.timezone != null", + "field": "apache.error.timestamp", + "target_field": "@timestamp", + "formats": [ + "EEE MMM dd H:m:s yyyy", + "EEE MMM dd H:m:s.SSSSSS yyyy" + ], + "timezone": "{{ event.timezone }}", + "on_failure": [ + { + "append": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] + } + }, + { + "remove": { + "field": "apache.error.timestamp", + "ignore_failure": true + } + }, + { + "set": { + "field": "event.kind", + "value": "event" + } + }, + { + "append": { + "field": "event.category", + "value": "web" + } + }, + { + "script": { + "if": "ctx?.log?.level != null", + "lang": "painless", + "source": "def err_levels = [\"emerg\", \"alert\", \"crit\", \"error\", \"warn\"]; if (err_levels.contains(ctx.log.level)) {\n ctx.event.type = [\"error\"];\n} else {\n ctx.event.type = [\"info\"];\n}" + } + }, + { + "grok": { + "field": "source.address", + "ignore_missing": true, + "patterns": [ + "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$" + ] + } + }, + { + "geoip": { + "field": "source.ip", + "target_field": "source.geo", + "ignore_missing": true + } + }, + { + "geoip": { + "database_file": "GeoLite2-ASN.mmdb", + "field": "source.ip", + "target_field": "source.as", + "properties": [ + "asn", + "organization_name" + ], + "ignore_missing": true + } + }, + { + "rename": { + "field": "source.as.asn", + "target_field": "source.as.number", + "ignore_missing": true + } + }, + { + "rename": { + "field": "source.as.organization_name", + "target_field": "source.as.organization.name", + "ignore_missing": true + } + }, + { + "convert": { + "field": "source.port", + "type": "long", + "ignore_missing": true + } + }, + { + "script": { + "lang": "painless", + "description": "This script processor iterates over the whole document to remove fields with null values.", + "source": "void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null);\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n" + } + }, + { + "remove": { + "field": "event.original", + "if": "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))", + "ignore_failure": true, + "ignore_missing": true + } + }, + { + "pipeline": { + "name": "global@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Global pipeline for all data streams" + } + }, + { + "pipeline": { + "name": "logs@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs`" + } + }, + { + "pipeline": { + "name": "logs-apache.integration@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `apache` integration" + } + }, + { + "pipeline": { + "name": "logs-apache.error@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for the `apache.error` dataset" + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ], + "_meta": { + "managed_by": "fleet", + "managed": true, + "package": { + "name": "apache" + } + } +} \ No newline at end of file diff --git a/elastic/logs/pipelines/logs-apache.error-1.3.0.json b/elastic/logs/pipelines/logs-apache.error-1.3.0.json deleted file mode 100644 index 5f5495159..000000000 --- a/elastic/logs/pipelines/logs-apache.error-1.3.0.json +++ /dev/null @@ -1,189 +0,0 @@ -{ - "description": "Pipeline for parsing apache error logs", - "processors": [ - { - "pipeline": { - "if": "ctx.message.startsWith('{')", - "name": "logs-apache.error-1.3.0-third-party" - } - }, - { - "set": { - "field": "event.ingested", - "value": "{{_ingest.timestamp}}" - } - }, - { - "set": { - "field": "ecs.version", - "value": "1.12.0" - } - }, - { - "rename": { - "field": "message", - "target_field": "event.original" - } - }, - { - "grok": { - "field": "event.original", - "patterns": [ - "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{LOGLEVEL:log.level}\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}", - "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{DATA:apache.error.module}:%{LOGLEVEL:log.level}\\] \\[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}" - ], - "pattern_definitions": { - "APACHE_TIME": "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}" - }, - "ignore_missing": true - } - }, - { - "grok": { - "field": "message", - "patterns": [ - "File does not exist: %{URIPATH:file.path}, referer: %{URI:http.request.referrer}", - "File does not exist: %{URIPATH:file.path}" - ], - "ignore_missing": true, - "ignore_failure": true - } - }, - { - "date": { - "if": "ctx.event.timezone == null", - "field": "apache.error.timestamp", - "target_field": "@timestamp", - "formats": [ - "EEE MMM dd H:m:s yyyy", - "EEE MMM dd H:m:s.SSSSSS yyyy" - ], - "on_failure": [ - { - "append": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] - } - }, - { - "date": { - "if": "ctx.event.timezone != null", - "field": "apache.error.timestamp", - "target_field": "@timestamp", - "formats": [ - "EEE MMM dd H:m:s yyyy", - "EEE MMM dd H:m:s.SSSSSS yyyy" - ], - "timezone": "{{ event.timezone }}", - "on_failure": [ - { - "append": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] - } - }, - { - "remove": { - "field": "apache.error.timestamp", - "ignore_failure": true - } - }, - { - "set": { - "field": "event.kind", - "value": "event" - } - }, - { - "set": { - "field": "event.category", - "value": "web" - } - }, - { - "script": { - "if": "ctx?.log?.level != null", - "lang": "painless", - "source": "def err_levels = [\"emerg\", \"alert\", \"crit\", \"error\", \"warn\"]; if (err_levels.contains(ctx.log.level)) {ctx.event.type = \"error\";} else {ctx.event.type = \"info\";}" - } - }, - { - "grok": { - "field": "source.address", - "ignore_missing": true, - "patterns": [ - "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$" - ] - } - }, - { - "geoip": { - "field": "source.ip", - "target_field": "source.geo", - "ignore_missing": true - } - }, - { - "geoip": { - "database_file": "GeoLite2-ASN.mmdb", - "field": "source.ip", - "target_field": "source.as", - "properties": [ - "asn", - "organization_name" - ], - "ignore_missing": true - } - }, - { - "rename": { - "field": "source.as.asn", - "target_field": "source.as.number", - "ignore_missing": true - } - }, - { - "rename": { - "field": "source.as.organization_name", - "target_field": "source.as.organization.name", - "ignore_missing": true - } - }, - { - "convert": { - "field": "source.port", - "type": "long", - "ignore_missing": true - } - }, - { - "script": { - "lang": "painless", - "description": "This script processor iterates over the whole document to remove fields with null values.", - "source": "void handleMap(Map map) {for (def x : map.values()) {if (x instanceof Map) {handleMap(x);} else if (x instanceof List) {handleList(x);}}map.values().removeIf(v -> v == null);}void handleList(List list) {for (def x : list) {if (x instanceof Map) {handleMap(x);} else if (x instanceof List) {handleList(x);}}}handleMap(ctx);" - } - }, - { - "remove": { - "field": "event.original", - "if": "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))", - "ignore_failure": true, - "ignore_missing": true - } - } -], -"on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } -] -} diff --git a/elastic/logs/pipelines/logs-kafka.log-0.5.0.json b/elastic/logs/pipelines/logs-kafka.log-0.5.0.json deleted file mode 100644 index 1b9d1fc5c..000000000 --- a/elastic/logs/pipelines/logs-kafka.log-0.5.0.json +++ /dev/null @@ -1,151 +0,0 @@ -{ - "description" : "Pipeline for parsing Kafka log messages", - "processors" : [ - { - "set" : { - "field" : "event.ingested", - "value" : "{{_ingest.timestamp}}" - } - }, - { - "set" : { - "field" : "ecs.version", - "value" : "1.10.0" - } - }, - { - "rename" : { - "field" : "message", - "target_field" : "event.original", - "ignore_missing" : true - } - }, - { - "grok" : { - "field" : "event.original", - "trace_match" : true, - "patterns" : [ - "(?m)%{TIMESTAMP_ISO8601:kafka.log.timestamp}. %{LOGLEVEL:log.level} +%{JAVALOGMESSAGE:message} \\(%{JAVACLASS:kafka.log.class}\\)$[ \n]*(?'kafka.log.trace.full'.*)", - "(?m)\\[%{TIMESTAMP_ISO8601:kafka.log.timestamp}\\] \\[%{LOGLEVEL:log.level} ?\\] \\[%{NOTSPACE:kafka.log.thread}\\] \\[%{NOTSPACE:kafka.log.class}\\] \\- %{GREEDYDATA:message}" - ] - } - }, - { - "grok" : { - "field" : "message", - "pattern_definitions" : { - "KAFKA_COMPONENT" : "[^\\]]*" - }, - "patterns" : [ - "\\[%{KAFKA_COMPONENT:kafka.log.component}\\][,:.]? +%{JAVALOGMESSAGE:message}" - ], - "on_failure" : [ - { - "set" : { - "field" : "kafka.log.component", - "value" : "unknown" - } - } - ] - } - }, - { - "grok" : { - "field" : "kafka.log.trace.full", - "ignore_missing" : true, - "patterns" : [ - "%{JAVACLASS:kafka.log.trace.class}:\\s*%{JAVALOGMESSAGE:kafka.log.trace.message}" - ], - "on_failure" : [ - { - "remove" : { - "field" : "kafka.log.trace" - } - } - ] - } - }, - { - "remove" : { - "field" : "kafka.log.trace.full", - "ignore_missing" : true - } - }, - { - "rename" : { - "field" : "@timestamp", - "target_field" : "event.created" - } - }, - { - "date" : { - "if" : "ctx.event.timezone == null", - "field" : "kafka.log.timestamp", - "target_field" : "@timestamp", - "formats" : [ - "yyyy-MM-dd HH:mm:ss,SSS" - ], - "on_failure" : [ - { - "append" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - } - ] - } - }, - { - "date" : { - "if" : "ctx.event.timezone != null", - "field" : "kafka.log.timestamp", - "target_field" : "@timestamp", - "formats" : [ - "yyyy-MM-dd HH:mm:ss,SSS" - ], - "timezone" : "{{ event.timezone }}", - "on_failure" : [ - { - "append" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - } - ] - } - }, - { - "remove" : { - "field" : "kafka.log.timestamp" - } - }, - { - "set" : { - "field" : "event.kind", - "value" : "event" - } - }, - { - "script" : { - "lang" : "painless", - "source" : "def errorLevels = [\"ERROR\", \"FATAL\"]; if (ctx?.log?.level != null) {if (errorLevels.contains(ctx.log.level)) {ctx.event.type = \"error\";} else {ctx.event.type = \"info\";}}" - } - }, - { - "remove" : { - "field" : "event.original", - "if" : "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))", - "ignore_failure" : true, - "ignore_missing" : true - } - } - ], - "on_failure" : [ - { - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/elastic/logs/pipelines/logs-kafka.log-1.13.0.json b/elastic/logs/pipelines/logs-kafka.log-1.13.0.json new file mode 100644 index 000000000..26da1eaef --- /dev/null +++ b/elastic/logs/pipelines/logs-kafka.log-1.13.0.json @@ -0,0 +1,181 @@ +{ + "description": "Pipeline for parsing Kafka log messages", + "processors": [ + { + "set": { + "field": "ecs.version", + "value": "8.5.1" + } + }, + { + "rename": { + "field": "message", + "target_field": "event.original", + "ignore_missing": true, + "if": "ctx.event?.original == null" + } + }, + { + "grok": { + "field": "event.original", + "trace_match": true, + "patterns": [ + "(?m)%{TIMESTAMP_ISO8601:kafka.log.timestamp}. %{LOGLEVEL:log.level} +%{JAVALOGMESSAGE:message} \\(%{JAVACLASS:kafka.log.class}\\)$[ \\n]*(?'kafka.log.trace.full'.*)", + "(?m)\\[%{TIMESTAMP_ISO8601:kafka.log.timestamp}\\] \\[%{LOGLEVEL:log.level} ?\\] \\[%{NOTSPACE:kafka.log.thread}\\] \\[%{NOTSPACE:kafka.log.class}\\] \\- %{GREEDYDATA:message}" + ] + } + }, + { + "grok": { + "field": "message", + "pattern_definitions": { + "KAFKA_COMPONENT": "[^\\]]*" + }, + "patterns": [ + "\\[%{KAFKA_COMPONENT:kafka.log.component}\\][,:.]? +%{JAVALOGMESSAGE:message}" + ], + "on_failure": [ + { + "set": { + "field": "kafka.log.component", + "value": "unknown" + } + } + ] + } + }, + { + "grok": { + "field": "kafka.log.trace.full", + "ignore_missing": true, + "patterns": [ + "%{JAVACLASS:kafka.log.trace.class}:\\s*%{JAVALOGMESSAGE:kafka.log.trace.message}" + ], + "on_failure": [ + { + "remove": { + "field": "kafka.log.trace" + } + } + ] + } + }, + { + "remove": { + "field": "kafka.log.trace.full", + "ignore_missing": true + } + }, + { + "rename": { + "field": "@timestamp", + "target_field": "event.created" + } + }, + { + "date": { + "if": "ctx.event.timezone == null", + "field": "kafka.log.timestamp", + "target_field": "@timestamp", + "formats": [ + "yyyy-MM-dd HH:mm:ss,SSS" + ], + "on_failure": [ + { + "append": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] + } + }, + { + "date": { + "if": "ctx.event.timezone != null", + "field": "kafka.log.timestamp", + "target_field": "@timestamp", + "formats": [ + "yyyy-MM-dd HH:mm:ss,SSS" + ], + "timezone": "{{ event.timezone }}", + "on_failure": [ + { + "append": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] + } + }, + { + "remove": { + "field": "kafka.log.timestamp" + } + }, + { + "set": { + "field": "event.kind", + "value": "event" + } + }, + { + "script": { + "lang": "painless", + "source": "def errorLevels = [\"ERROR\", \"FATAL\"]; if (ctx?.log?.level != null) {\n if (errorLevels.contains(ctx.log.level)) {\n ctx.event.type = [\"error\"];\n } else {\n ctx.event.type = [\"info\"];\n }\n}" + } + }, + { + "remove": { + "field": "event.original", + "if": "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))", + "ignore_failure": true, + "ignore_missing": true + } + }, + { + "pipeline": { + "name": "global@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Global pipeline for all data streams" + } + }, + { + "pipeline": { + "name": "logs@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs`" + } + }, + { + "pipeline": { + "name": "logs-kafka.integration@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `kafka` integration" + } + }, + { + "pipeline": { + "name": "logs-kafka.log@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for the `kafka.log` dataset" + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ], + "_meta": { + "managed_by": "fleet", + "managed": true, + "package": { + "name": "kafka" + } + } +} \ No newline at end of file diff --git a/elastic/logs/pipelines/logs-mysql.error-1.1.0.json b/elastic/logs/pipelines/logs-mysql.error-1.19.0.json similarity index 65% rename from elastic/logs/pipelines/logs-mysql.error-1.1.0.json rename to elastic/logs/pipelines/logs-mysql.error-1.19.0.json index 70365e099..fcf024677 100644 --- a/elastic/logs/pipelines/logs-mysql.error-1.1.0.json +++ b/elastic/logs/pipelines/logs-mysql.error-1.19.0.json @@ -7,6 +7,20 @@ "value": "{{_ingest.timestamp}}" } }, + { + "set": { + "field": "ecs.version", + "value": "8.5.1" + } + }, + { + "set": { + "field": "event.original", + "value": "{{{message}}}", + "ignore_empty_value": true, + "if": "ctx.event?.original == null" + } + }, { "grok": { "field": "message", @@ -106,6 +120,42 @@ "value": "error", "if": "ctx?.log?.level != null && ctx.log.level.toLowerCase() == 'error'" } + }, + { + "remove": { + "field": "event.original", + "if": "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))", + "ignore_failure": true, + "ignore_missing": true + } + }, + { + "pipeline": { + "name": "global@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Global pipeline for all data streams" + } + }, + { + "pipeline": { + "name": "logs@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs`" + } + }, + { + "pipeline": { + "name": "logs-mysql.integration@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `mysql` integration" + } + }, + { + "pipeline": { + "name": "logs-mysql.error@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for the `mysql.error` dataset" + } } ], "on_failure": [ @@ -115,5 +165,12 @@ "value": "{{ _ingest.on_failure_message }}" } } - ] -} + ], + "_meta": { + "managed_by": "fleet", + "managed": true, + "package": { + "name": "mysql" + } + } +} \ No newline at end of file diff --git a/elastic/logs/pipelines/logs-mysql.slowlog-1.1.0.json b/elastic/logs/pipelines/logs-mysql.slowlog-1.1.0.json deleted file mode 100644 index c524ef04b..000000000 --- a/elastic/logs/pipelines/logs-mysql.slowlog-1.1.0.json +++ /dev/null @@ -1,112 +0,0 @@ -{ - "description" : "Pipeline for parsing MySQL slow logs.", - "processors" : [ - { - "drop" : { - "if": "ctx.message ==~ /^# Time:.*/ || ctx.message ==~ /^[\\/\\w\\.]+, Version: .*\\. started with:.*/ms" - } - }, - { - "set" : { - "field" : "event.ingested", - "value" : "{{_ingest.timestamp}}" - } - }, - { - "grok" : { - "field" : "message", - "patterns" : [ - "^# User@Host: %{USER:user.name}(\\[%{USER:mysql.slowlog.current_user}\\])? @ %{HOSTNAME:source.domain}? \\[%{IP:source.ip}?\\]%{METRICSPACE}(Id:%{SPACE}%{NUMBER:mysql.thread_id:long}%{METRICSPACE})?(Thread_id:%{SPACE}%{NUMBER:mysql.thread_id:long}%{METRICSPACE})?(Schema:%{SPACE}%{WORD:mysql.slowlog.schema}?%{METRICSPACE})?(Last_errno: %{NUMBER:mysql.slowlog.last_errno}%{METRICSPACE})?(Killed: %{NUMBER:mysql.slowlog.killed}%{METRICSPACE})?(QC_hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Query_time: %{NUMBER:temp.duration:float}%{METRICSPACE})?(Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec:float}%{METRICSPACE})?(Rows_sent: %{NUMBER:mysql.slowlog.rows_sent:long}%{METRICSPACE})?(Rows_examined: %{NUMBER:mysql.slowlog.rows_examined:long}%{METRICSPACE})?(Rows_affected: %{NUMBER:mysql.slowlog.rows_affected:long}%{METRICSPACE})?(Thread_id: %{NUMBER:mysql.thread_id:long}%{METRICSPACE})?(Errno: %{NUMBER:mysql.slowlog.last_errno}%{METRICSPACE})?(Killed: %{NUMBER:mysql.slowlog.killed}%{METRICSPACE})?(Bytes_received: %{NUMBER:mysql.slowlog.bytes_received:long}%{METRICSPACE})?(Bytes_sent: %{NUMBER:mysql.slowlog.bytes_sent:long}%{METRICSPACE})?(Read_first: %{NUMBER:mysql.slowlog.read_first:long}%{METRICSPACE})?(Read_last: %{NUMBER:mysql.slowlog.read_last:long}%{METRICSPACE})?(Read_key: %{NUMBER:mysql.slowlog.read_key:long}%{METRICSPACE})?(Read_next: %{NUMBER:mysql.slowlog.read_next:long}%{METRICSPACE})?(Read_prev: %{NUMBER:mysql.slowlog.read_prev:long}%{METRICSPACE})?(Read_rnd: %{NUMBER:mysql.slowlog.read_rnd:long}%{METRICSPACE})?(Read_rnd_next: %{NUMBER:mysql.slowlog.read_rnd_next:long}%{METRICSPACE})?(Sort_merge_passes: %{NUMBER:mysql.slowlog.sort_merge_passes:long}%{METRICSPACE})?(Sort_range_count: %{NUMBER:mysql.slowlog.sort_range_count:long}%{METRICSPACE})?(Sort_rows: %{NUMBER:mysql.slowlog.sort_rows:long}%{METRICSPACE})?(Sort_scan_count: %{NUMBER:mysql.slowlog.sort_scan_count:long}%{METRICSPACE})?(Created_tmp_disk_tables: %{NUMBER:mysql.slowlog.tmp_disk_tables:long}%{METRICSPACE})?(Created_tmp_tables: %{NUMBER:mysql.slowlog.tmp_tables:long}%{METRICSPACE})?(Tmp_tables: %{NUMBER:mysql.slowlog.tmp_tables:long}%{METRICSPACE})?(Tmp_disk_tables: %{NUMBER:mysql.slowlog.tmp_disk_tables:long}%{METRICSPACE})?(Tmp_table_sizes: %{NUMBER:mysql.slowlog.tmp_table_sizes:long}%{METRICSPACE})?(Start: %{TIMESTAMP_ISO8601:event.start}%{METRICSPACE})?(End: %{TIMESTAMP_ISO8601:event.end}%{METRICSPACE})?(InnoDB_trx_id: %{WORD:mysql.slowlog.innodb.trx_id}%{METRICSPACE})?(QC_Hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Full_scan: %{WORD:mysql.slowlog.full_scan}%{METRICSPACE})?(Full_join: %{WORD:mysql.slowlog.full_join}%{METRICSPACE})?(Tmp_table: %{WORD:mysql.slowlog.tmp_table}%{METRICSPACE})?(Tmp_table_on_disk: %{WORD:mysql.slowlog.tmp_table_on_disk}%{METRICSPACE})?(Filesort: %{WORD:mysql.slowlog.filesort}%{METRICSPACE})?(Filesort_on_disk: %{WORD:mysql.slowlog.filesort_on_disk}%{METRICSPACE})?(Merge_passes: %{NUMBER:mysql.slowlog.merge_passes:long}%{METRICSPACE})?(Priority_queue: %{WORD:mysql.slowlog.priority_queue}%{METRICSPACE})?(No InnoDB statistics available for this query%{METRICSPACE})?(InnoDB_IO_r_ops: %{NUMBER:mysql.slowlog.innodb.io_r_ops:long}%{METRICSPACE})?(InnoDB_IO_r_bytes: %{NUMBER:mysql.slowlog.innodb.io_r_bytes:long}%{METRICSPACE})?(InnoDB_IO_r_wait: %{NUMBER:mysql.slowlog.innodb.io_r_wait.sec:float}%{METRICSPACE})?(InnoDB_rec_lock_wait: %{NUMBER:mysql.slowlog.innodb.rec_lock_wait.sec:float}%{METRICSPACE})?(InnoDB_queue_wait: %{NUMBER:mysql.slowlog.innodb.queue_wait.sec:float}%{METRICSPACE})?(InnoDB_pages_distinct: %{NUMBER:mysql.slowlog.innodb.pages_distinct:long}%{METRICSPACE})?(Log_slow_rate_type: %{WORD:mysql.slowlog.log_slow_rate_type}%{METRICSPACE})?(Log_slow_rate_limit: %{NUMBER:mysql.slowlog.log_slow_rate_limit:long}%{METRICSPACE})?%{EXPLAIN}(use %{WORD:mysql.slowlog.schema};)?SET timestamp=%{NUMBER:mysql.slowlog.timestamp:long};%{GREEDYMULTILINE:mysql.slowlog.query}" - ], - "pattern_definitions" : { - "GREEDYMULTILINE" : "(.|\n)*", - "METRICSPACE" : "([ #\n]*)", - "EXPLAIN" : "(# explain:.*\n|#\\s*\n)*" - }, - "ignore_missing" : true - } - }, - { - "remove" : { - "field" : "message" - } - }, - { - "script" : { - "lang" : "painless", - "source" : "for (field in params.fields) { def v = ctx.mysql.slowlog.get(field); if (v != null) { ctx.mysql.slowlog.put(field, params.mapping.get(v)) } }", - "params" : { - "mapping" : { - "Yes" : true, - "No" : false - }, - "fields" : [ - "query_cache_hit", - "tmp_table", - "tmp_table_on_disk", - "filesort", - "filesort_on_disk", - "priority_queue", - "full_scan", - "full_join" - ] - } - } - }, - { - "script" : { - "lang" : "painless", - "source" : "ctx.event.duration = Math.round(ctx.temp.duration * 1000000) * 1000", - "if" : "ctx.temp?.duration != null" - } - }, - { - "remove" : { - "field" : "temp.duration", - "ignore_missing" : true - } - }, - { - "date" : { - "field" : "mysql.slowlog.timestamp", - "target_field" : "@timestamp", - "formats" : [ - "UNIX" - ], - "ignore_failure" : true - } - }, - { - "remove" : { - "field" : "mysql.slowlog.timestamp", - "ignore_missing" : true - } - }, - { - "set" : { - "field" : "event.kind", - "value" : "event" - } - }, - { - "append" : { - "field" : "event.category", - "value" : "database" - } - }, - { - "append" : { - "field" : "event.type", - "value" : "info" - } - } - ], - "on_failure" : [ - { - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/elastic/logs/pipelines/logs-mysql.slowlog-1.19.0.json b/elastic/logs/pipelines/logs-mysql.slowlog-1.19.0.json new file mode 100644 index 000000000..0448778af --- /dev/null +++ b/elastic/logs/pipelines/logs-mysql.slowlog-1.19.0.json @@ -0,0 +1,189 @@ +{ + "description": "Pipeline for parsing MySQL slow logs.", + "processors": [ + { + "drop": { + "if": "ctx.message ==~ /^# Time:.*/ || ctx.message ==~ /^[\\/\\w\\.]+, Version: .*\\. started with:.*/ms" + } + }, + { + "set": { + "field": "ecs.version", + "value": "8.5.1" + } + }, + { + "set": { + "field": "event.ingested", + "value": "{{_ingest.timestamp}}" + } + }, + { + "grok": { + "field": "message", + "patterns": [ + "^# User@Host: %{USER:user.name}(\\[%{USER:mysql.slowlog.current_user}\\])? @ %{HOSTNAME:source.domain}? \\[%{IP:source.ip}?\\]%{METRICSPACE}(Id:%{SPACE}%{NUMBER:mysql.thread_id:long}%{METRICSPACE})?(Thread_id:%{SPACE}%{NUMBER:mysql.thread_id:long}%{METRICSPACE})?(Schema:%{SPACE}%{NOTSPACE:mysql.slowlog.schema}?%{METRICSPACE})?(Last_errno: %{NUMBER:mysql.slowlog.last_errno}%{METRICSPACE})?(Killed: %{NUMBER:mysql.slowlog.killed}%{METRICSPACE})?(QC_hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Query_time: %{NUMBER:temp.duration:float}%{METRICSPACE})?(Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec:float}%{METRICSPACE})?(Rows_sent: %{NUMBER:mysql.slowlog.rows_sent:long}%{METRICSPACE})?(Rows_examined: %{NUMBER:mysql.slowlog.rows_examined:long}%{METRICSPACE})?(Rows_affected: %{NUMBER:mysql.slowlog.rows_affected:long}%{METRICSPACE})?(Thread_id: %{NUMBER:mysql.thread_id:long}%{METRICSPACE})?(Errno: %{NUMBER:mysql.slowlog.last_errno}%{METRICSPACE})?(Killed: %{NUMBER:mysql.slowlog.killed}%{METRICSPACE})?(Bytes_received: %{NUMBER:mysql.slowlog.bytes_received:long}%{METRICSPACE})?(Bytes_sent: %{NUMBER:mysql.slowlog.bytes_sent:long}%{METRICSPACE})?(Read_first: %{NUMBER:mysql.slowlog.read_first:long}%{METRICSPACE})?(Read_last: %{NUMBER:mysql.slowlog.read_last:long}%{METRICSPACE})?(Read_key: %{NUMBER:mysql.slowlog.read_key:long}%{METRICSPACE})?(Read_next: %{NUMBER:mysql.slowlog.read_next:long}%{METRICSPACE})?(Read_prev: %{NUMBER:mysql.slowlog.read_prev:long}%{METRICSPACE})?(Read_rnd: %{NUMBER:mysql.slowlog.read_rnd:long}%{METRICSPACE})?(Read_rnd_next: %{NUMBER:mysql.slowlog.read_rnd_next:long}%{METRICSPACE})?(Sort_merge_passes: %{NUMBER:mysql.slowlog.sort_merge_passes:long}%{METRICSPACE})?(Sort_range_count: %{NUMBER:mysql.slowlog.sort_range_count:long}%{METRICSPACE})?(Sort_rows: %{NUMBER:mysql.slowlog.sort_rows:long}%{METRICSPACE})?(Sort_scan_count: %{NUMBER:mysql.slowlog.sort_scan_count:long}%{METRICSPACE})?(Created_tmp_disk_tables: %{NUMBER:mysql.slowlog.tmp_disk_tables:long}%{METRICSPACE})?(Created_tmp_tables: %{NUMBER:mysql.slowlog.tmp_tables:long}%{METRICSPACE})?(Tmp_tables: %{NUMBER:mysql.slowlog.tmp_tables:long}%{METRICSPACE})?(Tmp_disk_tables: %{NUMBER:mysql.slowlog.tmp_disk_tables:long}%{METRICSPACE})?(Tmp_table_sizes: %{NUMBER:mysql.slowlog.tmp_table_sizes:long}%{METRICSPACE})?(Start: %{TIMESTAMP_ISO8601:event.start}%{METRICSPACE})?(End: %{TIMESTAMP_ISO8601:event.end}%{METRICSPACE})?(InnoDB_trx_id: %{WORD:mysql.slowlog.innodb.trx_id}%{METRICSPACE})?(QC_Hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Full_scan: %{WORD:mysql.slowlog.full_scan}%{METRICSPACE})?(Full_join: %{WORD:mysql.slowlog.full_join}%{METRICSPACE})?(Tmp_table: %{WORD:mysql.slowlog.tmp_table}%{METRICSPACE})?(Tmp_table_on_disk: %{WORD:mysql.slowlog.tmp_table_on_disk}%{METRICSPACE})?(Filesort: %{WORD:mysql.slowlog.filesort}%{METRICSPACE})?(Filesort_on_disk: %{WORD:mysql.slowlog.filesort_on_disk}%{METRICSPACE})?(Merge_passes: %{NUMBER:mysql.slowlog.merge_passes:long}%{METRICSPACE})?(Priority_queue: %{WORD:mysql.slowlog.priority_queue}%{METRICSPACE})?(No InnoDB statistics available for this query%{METRICSPACE})?(InnoDB_IO_r_ops: %{NUMBER:mysql.slowlog.innodb.io_r_ops:long}%{METRICSPACE})?(InnoDB_IO_r_bytes: %{NUMBER:mysql.slowlog.innodb.io_r_bytes:long}%{METRICSPACE})?(InnoDB_IO_r_wait: %{NUMBER:mysql.slowlog.innodb.io_r_wait.sec:float}%{METRICSPACE})?(InnoDB_rec_lock_wait: %{NUMBER:mysql.slowlog.innodb.rec_lock_wait.sec:float}%{METRICSPACE})?(InnoDB_queue_wait: %{NUMBER:mysql.slowlog.innodb.queue_wait.sec:float}%{METRICSPACE})?(InnoDB_pages_distinct: %{NUMBER:mysql.slowlog.innodb.pages_distinct:long}%{METRICSPACE})?(Log_slow_rate_type: %{WORD:mysql.slowlog.log_slow_rate_type}%{METRICSPACE})?(Log_slow_rate_limit: %{NUMBER:mysql.slowlog.log_slow_rate_limit:long}%{METRICSPACE})?%{EXPLAIN}(use %{NOTSPACE:mysql.slowlog.schema};\n)?SET timestamp=%{NUMBER:mysql.slowlog.timestamp:long};\n%{GREEDYMULTILINE:mysql.slowlog.query}" + ], + "pattern_definitions": { + "GREEDYMULTILINE": "(.|\n)*", + "METRICSPACE": "([ #\n]*)", + "EXPLAIN": "(# explain:.*\n|#\\s*\n)*" + }, + "ignore_missing": true + } + }, + { + "rename": { + "field": "message", + "target_field": "event.original", + "ignore_missing": true, + "if": "ctx.event?.original == null", + "description": "Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document." + } + }, + { + "script": { + "lang": "painless", + "if": "ctx?.mysql != null", + "source": "for (field in params.fields) { if (ctx.mysql[field] instanceof List) { def vals = ctx.mysql[field]; vals = vals.stream().distinct().collect(Collectors.toList()); if (vals.size() == 1) { ctx.mysql[field] = vals[0] } else { ctx.mysql[field] = vals } } }", + "params": { + "fields": [ + "thread_id" + ] + } + } + }, + { + "script": { + "lang": "painless", + "if": "ctx?.mysql?.slowlog != null", + "source": "for (field in params.fields) { if (ctx.mysql.slowlog[field] instanceof List) { def vals = ctx.mysql.slowlog[field]; vals = vals.stream().distinct().collect(Collectors.toList()); if (vals.size() == 1) { ctx.mysql.slowlog[field] = vals[0] } else { ctx.mysql.slowlog[field] = vals } } }", + "params": { + "fields": [ + "schema" + ] + } + } + }, + { + "script": { + "lang": "painless", + "source": "for (field in params.fields) { def v = ctx.mysql.slowlog.get(field); if (v != null) { ctx.mysql.slowlog.put(field, params.mapping.get(v)) } }", + "params": { + "mapping": { + "Yes": true, + "No": false + }, + "fields": [ + "query_cache_hit", + "tmp_table", + "tmp_table_on_disk", + "filesort", + "filesort_on_disk", + "priority_queue", + "full_scan", + "full_join" + ] + } + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.event.duration = Math.round(ctx.temp.duration * 1000000) * 1000", + "if": "ctx.temp?.duration != null" + } + }, + { + "remove": { + "field": "temp.duration", + "ignore_missing": true + } + }, + { + "date": { + "field": "mysql.slowlog.timestamp", + "target_field": "@timestamp", + "formats": [ + "UNIX" + ], + "ignore_failure": true + } + }, + { + "remove": { + "field": "mysql.slowlog.timestamp", + "ignore_missing": true + } + }, + { + "set": { + "field": "event.kind", + "value": "event" + } + }, + { + "append": { + "field": "event.category", + "value": "database" + } + }, + { + "append": { + "field": "event.type", + "value": "info" + } + }, + { + "remove": { + "field": "event.original", + "if": "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))", + "ignore_failure": true, + "ignore_missing": true + } + }, + { + "pipeline": { + "name": "global@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Global pipeline for all data streams" + } + }, + { + "pipeline": { + "name": "logs@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs`" + } + }, + { + "pipeline": { + "name": "logs-mysql.integration@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `mysql` integration" + } + }, + { + "pipeline": { + "name": "logs-mysql.slowlog@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for the `mysql.slowlog` dataset" + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ], + "_meta": { + "managed_by": "fleet", + "managed": true, + "package": { + "name": "mysql" + } + } +} \ No newline at end of file diff --git a/elastic/logs/pipelines/logs-nginx.access-1.2.0.json b/elastic/logs/pipelines/logs-nginx.access-1.2.0.json deleted file mode 100644 index 329c657c9..000000000 --- a/elastic/logs/pipelines/logs-nginx.access-1.2.0.json +++ /dev/null @@ -1,285 +0,0 @@ -{ - "description" : "Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins.", - "processors" : [ - { - "pipeline" : { - "if" : "ctx.message.startsWith('{')", - "name" : "logs-nginx.access-1.2.0-third-party" - } - }, - { - "set" : { - "field" : "event.ingested", - "value" : "{{_ingest.timestamp}}" - } - }, - { - "set" : { - "field" : "ecs.version", - "value" : "1.12.0" - } - }, - { - "rename" : { - "field" : "message", - "target_field" : "event.original", - "ignore_missing" : true - } - }, - { - "grok" : { - "field" : "event.original", - "patterns" : [ - "(%{NGINX_HOST} )?\"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address}) - (-|%{DATA:user.name}) \\[%{HTTPDATE:nginx.access.time}\\] \"%{DATA:nginx.access.info}\" %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} \"(-|%{DATA:http.request.referrer})\" \"(-|%{DATA:user_agent.original})\"" - ], - "pattern_definitions" : { - "NGINX_HOST" : "(?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})?", - "NGINX_NOTSEPARATOR" : "[^\t ,:]+", - "NGINX_ADDRESS_LIST" : "(?:%{IP}|%{WORD})(\"?,?\\s*(?:%{IP}|%{WORD}))*" - }, - "ignore_missing" : true - } - }, - { - "grok" : { - "field" : "nginx.access.info", - "patterns" : [ - "%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}", - "" - ], - "ignore_missing" : true - } - }, - { - "uri_parts" : { - "field" : "_tmp.url_orig", - "ignore_failure" : true - } - }, - { - "set" : { - "field" : "url.domain", - "value" : "{{destination.domain}}", - "if" : "ctx.url?.domain == null && ctx.destination?.domain != null" - } - }, - { - "remove" : { - "field" : [ - "nginx.access.info", - "_tmp.url_orig" - ], - "ignore_missing" : true - } - }, - { - "split" : { - "field" : "nginx.access.remote_ip_list", - "separator" : "\"?,?\\s+", - "ignore_missing" : true - } - }, - { - "split" : { - "field" : "nginx.access.origin", - "separator" : "\"?,?\\s+", - "ignore_missing" : true - } - }, - { - "set" : { - "field" : "source.address", - "if" : "ctx.source?.address == null", - "value" : "" - } - }, - { - "script" : { - "if" : "ctx.nginx?.access?.remote_ip_list != null && ctx.nginx.access.remote_ip_list.length > 0", - "lang" : "painless", - "source" : "boolean isPrivate(def dot, def ip) {try {StringTokenizer tok = new StringTokenizer(ip, dot);int firstByte = Integer.parseInt(tok.nextToken());int secondByte = Integer.parseInt(tok.nextToken());if (firstByte == 10) {return true;}if (firstByte == 192 && secondByte == 168) {return true;}if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) {return true;}if (firstByte == 127) {return true;}return false;}catch (Exception e) {return false;}} try {ctx.source.address = null;if (ctx.nginx.access.remote_ip_list == null) {return;}def found = false;for (def item : ctx.nginx.access.remote_ip_list) {if (!isPrivate(params.dot, item)) {ctx.source.address = item;found = true;break;}}if (!found) {ctx.source.address = ctx.nginx.access.remote_ip_list[0];}} catch (Exception e) {ctx.source.address = null;}", - "params" : { - "dot" : "." - } - } - }, - { - "remove" : { - "field" : "source.address", - "if" : "ctx.source.address == null" - } - }, - { - "grok" : { - "field" : "source.address", - "patterns" : [ - "^%{IP:source.ip}$" - ], - "ignore_failure" : true - } - }, - { - "remove" : { - "field" : "event.created", - "ignore_missing" : true, - "ignore_failure" : true - } - }, - { - "rename" : { - "field" : "@timestamp", - "target_field" : "event.created" - } - }, - { - "convert" : { - "field" : "destination.port", - "type" : "long", - "if" : "ctx.destination?.port != null" - } - }, - { - "date" : { - "field" : "nginx.access.time", - "target_field" : "@timestamp", - "formats" : [ - "dd/MMM/yyyy:H:m:s Z" - ], - "on_failure" : [ - { - "append" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - } - ] - } - }, - { - "remove" : { - "field" : "nginx.access.time" - } - }, - { - "user_agent" : { - "field" : "user_agent.original", - "ignore_missing" : true - } - }, - { - "geoip" : { - "field" : "source.ip", - "target_field" : "source.geo", - "ignore_missing" : true - } - }, - { - "geoip" : { - "database_file" : "GeoLite2-ASN.mmdb", - "field" : "source.ip", - "target_field" : "source.as", - "properties" : [ - "asn", - "organization_name" - ], - "ignore_missing" : true - } - }, - { - "rename" : { - "field" : "source.as.asn", - "target_field" : "source.as.number", - "ignore_missing" : true - } - }, - { - "rename" : { - "field" : "source.as.organization_name", - "target_field" : "source.as.organization.name", - "ignore_missing" : true - } - }, - { - "set" : { - "field" : "event.kind", - "value" : "event" - } - }, - { - "append" : { - "field" : "event.category", - "value" : "web" - } - }, - { - "append" : { - "field" : "event.type", - "value" : "access" - } - }, - { - "set" : { - "field" : "event.outcome", - "value" : "success", - "if" : "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" - } - }, - { - "set" : { - "field" : "event.outcome", - "value" : "failure", - "if" : "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" - } - }, - { - "lowercase" : { - "field" : "http.request.method", - "ignore_missing" : true - } - }, - { - "append" : { - "field" : "related.ip", - "value" : "{{source.ip}}", - "if" : "ctx?.source?.ip != null" - } - }, - { - "append" : { - "field" : "related.ip", - "value" : "{{destination.ip}}", - "if" : "ctx?.destination?.ip != null" - } - }, - { - "append" : { - "field" : "related.user", - "value" : "{{user.name}}", - "if" : "ctx?.user?.name != null" - } - }, - { - "script" : { - "lang" : "painless", - "description" : "This script processor iterates over the whole document to remove fields with null values.", - "source" : "void handleMap(Map map) {for (def x : map.values()) {if (x instanceof Map) {handleMap(x);} else if (x instanceof List) {handleList(x);}}map.values().removeIf(v -> v == null);}void handleList(List list) {for (def x : list) {if (x instanceof Map) {handleMap(x);} else if (x instanceof List) {handleList(x);}}}handleMap(ctx);" - } - }, - { - "remove" : { - "field" : "event.original", - "if" : "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))", - "ignore_failure" : true, - "ignore_missing" : true - } - } - ], - "on_failure" : [ - { - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/elastic/logs/pipelines/logs-nginx.access-1.20.0.json b/elastic/logs/pipelines/logs-nginx.access-1.20.0.json new file mode 100644 index 000000000..5388abf34 --- /dev/null +++ b/elastic/logs/pipelines/logs-nginx.access-1.20.0.json @@ -0,0 +1,315 @@ +{ + "description": "Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins.", + "processors": [ + { + "pipeline": { + "if": "ctx.message.startsWith('{')", + "name": "logs-nginx.access-1.20.0-third-party" + } + }, + { + "set": { + "field": "event.ingested", + "value": "{{_ingest.timestamp}}" + } + }, + { + "set": { + "field": "ecs.version", + "value": "8.5.1" + } + }, + { + "rename": { + "field": "message", + "target_field": "event.original", + "ignore_missing": true, + "if": "ctx.event?.original == null" + } + }, + { + "grok": { + "field": "event.original", + "patterns": [ + "(%{NGINX_HOST} )?\"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address}) - (-|%{DATA:user.name}) \\[%{HTTPDATE:nginx.access.time}\\] \"%{DATA:nginx.access.info}\" %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} \"(-|%{DATA:http.request.referrer})\" \"(-|%{DATA:user_agent.original})\"" + ], + "pattern_definitions": { + "NGINX_HOST": "(?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})?", + "NGINX_NOTSEPARATOR": "[^\t ,:]+", + "NGINX_ADDRESS_LIST": "(?:%{IP}|%{WORD})(\"?,?\\s*(?:%{IP}|%{WORD}))*" + }, + "ignore_missing": true + } + }, + { + "grok": { + "field": "nginx.access.info", + "patterns": [ + "%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}", + "" + ], + "ignore_missing": true + } + }, + { + "uri_parts": { + "field": "_tmp.url_orig", + "ignore_failure": true + } + }, + { + "set": { + "field": "url.domain", + "value": "{{destination.domain}}", + "if": "ctx.url?.domain == null && ctx.destination?.domain != null" + } + }, + { + "remove": { + "field": [ + "nginx.access.info", + "_tmp.url_orig" + ], + "ignore_missing": true + } + }, + { + "split": { + "field": "nginx.access.remote_ip_list", + "separator": "\"?,?\\s+", + "ignore_missing": true + } + }, + { + "split": { + "field": "nginx.access.origin", + "separator": "\"?,?\\s+", + "ignore_missing": true + } + }, + { + "set": { + "field": "source.address", + "if": "ctx.source?.address == null", + "value": "" + } + }, + { + "script": { + "if": "ctx.nginx?.access?.remote_ip_list != null && ctx.nginx.access.remote_ip_list.length > 0", + "lang": "painless", + "source": "boolean isPrivate(def dot, def ip) {\n try {\n StringTokenizer tok = new StringTokenizer(ip, dot);\n int firstByte = Integer.parseInt(tok.nextToken());\n int secondByte = Integer.parseInt(tok.nextToken());\n if (firstByte == 10) {\n return true;\n }\n if (firstByte == 192 && secondByte == 168) {\n return true;\n }\n if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) {\n return true;\n }\n if (firstByte == 127) {\n return true;\n }\n return false;\n }\n catch (Exception e) {\n return false;\n }\n} try {\n ctx.source.address = null;\n if (ctx.nginx.access.remote_ip_list == null) {\n return;\n }\n def found = false;\n for (def item : ctx.nginx.access.remote_ip_list) {\n if (!isPrivate(params.dot, item)) {\n ctx.source.address = item;\n found = true;\n break;\n }\n }\n if (!found) {\n ctx.source.address = ctx.nginx.access.remote_ip_list[0];\n }\n} catch (Exception e) {\n ctx.source.address = null;\n}", + "params": { + "dot": "." + } + } + }, + { + "remove": { + "field": "source.address", + "if": "ctx.source.address == null" + } + }, + { + "grok": { + "field": "source.address", + "patterns": [ + "^%{IP:source.ip}$" + ], + "ignore_failure": true + } + }, + { + "remove": { + "field": "event.created", + "ignore_missing": true, + "ignore_failure": true + } + }, + { + "rename": { + "field": "@timestamp", + "target_field": "event.created" + } + }, + { + "convert": { + "field": "destination.port", + "type": "long", + "if": "ctx.destination?.port != null" + } + }, + { + "date": { + "field": "nginx.access.time", + "target_field": "@timestamp", + "formats": [ + "dd/MMM/yyyy:H:m:s Z" + ], + "on_failure": [ + { + "append": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] + } + }, + { + "remove": { + "field": "nginx.access.time" + } + }, + { + "user_agent": { + "field": "user_agent.original", + "ignore_missing": true + } + }, + { + "geoip": { + "field": "source.ip", + "target_field": "source.geo", + "ignore_missing": true + } + }, + { + "geoip": { + "database_file": "GeoLite2-ASN.mmdb", + "field": "source.ip", + "target_field": "source.as", + "properties": [ + "asn", + "organization_name" + ], + "ignore_missing": true + } + }, + { + "rename": { + "field": "source.as.asn", + "target_field": "source.as.number", + "ignore_missing": true + } + }, + { + "rename": { + "field": "source.as.organization_name", + "target_field": "source.as.organization.name", + "ignore_missing": true + } + }, + { + "set": { + "field": "event.kind", + "value": "event" + } + }, + { + "append": { + "field": "event.category", + "value": "web" + } + }, + { + "append": { + "field": "event.type", + "value": "access" + } + }, + { + "set": { + "field": "event.outcome", + "value": "success", + "if": "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" + } + }, + { + "set": { + "field": "event.outcome", + "value": "failure", + "if": "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" + } + }, + { + "append": { + "field": "related.ip", + "value": "{{source.ip}}", + "if": "ctx?.source?.ip != null" + } + }, + { + "append": { + "field": "related.ip", + "value": "{{destination.ip}}", + "if": "ctx?.destination?.ip != null" + } + }, + { + "append": { + "field": "related.user", + "value": "{{user.name}}", + "if": "ctx?.user?.name != null" + } + }, + { + "script": { + "lang": "painless", + "description": "This script processor iterates over the whole document to remove fields with null values.", + "source": "void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null);\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n" + } + }, + { + "remove": { + "field": "event.original", + "if": "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))", + "ignore_failure": true, + "ignore_missing": true + } + }, + { + "pipeline": { + "name": "global@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Global pipeline for all data streams" + } + }, + { + "pipeline": { + "name": "logs@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs`" + } + }, + { + "pipeline": { + "name": "logs-nginx.integration@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `nginx` integration" + } + }, + { + "pipeline": { + "name": "logs-nginx.access@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for the `nginx.access` dataset" + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ], + "_meta": { + "managed_by": "fleet", + "managed": true, + "package": { + "name": "nginx" + } + } +} \ No newline at end of file diff --git a/elastic/logs/pipelines/logs-nginx.error-1.2.0.json b/elastic/logs/pipelines/logs-nginx.error-1.2.0.json deleted file mode 100644 index bc4fe5aad..000000000 --- a/elastic/logs/pipelines/logs-nginx.error-1.2.0.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "description" : "Pipeline for parsing the Nginx error logs", - "processors" : [ - { - "pipeline" : { - "if" : "ctx.message.startsWith('{')", - "name" : "logs-nginx.error-1.2.0-third-party" - } - }, - { - "set" : { - "field" : "event.ingested", - "value" : "{{_ingest.timestamp}}" - } - }, - { - "set" : { - "field" : "ecs.version", - "value" : "1.12.0" - } - }, - { - "rename" : { - "field" : "message", - "target_field" : "event.original", - "ignore_missing" : true - } - }, - { - "grok" : { - "field" : "event.original", - "patterns" : [ - "%{DATA:nginx.error.time} \\[%{DATA:log.level}\\] %{NUMBER:process.pid:long}#%{NUMBER:process.thread.id:long}: (\\*%{NUMBER:nginx.error.connection_id:long} )?%{GREEDYMULTILINE:message}" - ], - "pattern_definitions" : { - "GREEDYMULTILINE" : "(.|\n|\t)*" - }, - "ignore_missing" : true - } - }, - { - "remove" : { - "field" : "event.created", - "ignore_missing" : true, - "ignore_failure" : true - } - }, - { - "rename" : { - "field" : "@timestamp", - "target_field" : "event.created" - } - }, - { - "date" : { - "if" : "ctx.event.timezone == null", - "field" : "nginx.error.time", - "target_field" : "@timestamp", - "formats" : [ - "yyyy/MM/dd H:m:s" - ], - "on_failure" : [ - { - "append" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - } - ] - } - }, - { - "date" : { - "if" : "ctx.event.timezone != null", - "field" : "nginx.error.time", - "target_field" : "@timestamp", - "formats" : [ - "yyyy/MM/dd H:m:s" - ], - "timezone" : "{{ event.timezone }}", - "on_failure" : [ - { - "append" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - } - ] - } - }, - { - "remove" : { - "field" : "nginx.error.time" - } - }, - { - "set" : { - "field" : "event.kind", - "value" : "event" - } - }, - { - "append" : { - "field" : "event.category", - "value" : "web" - } - }, - { - "append" : { - "field" : "event.type", - "value" : "error" - } - }, - { - "remove" : { - "field" : "event.original", - "if" : "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))", - "ignore_failure" : true, - "ignore_missing" : true - } - } - ], - "on_failure" : [ - { - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/elastic/logs/pipelines/logs-nginx.error-1.20.0.json b/elastic/logs/pipelines/logs-nginx.error-1.20.0.json new file mode 100644 index 000000000..330cb1187 --- /dev/null +++ b/elastic/logs/pipelines/logs-nginx.error-1.20.0.json @@ -0,0 +1,167 @@ +{ + "description": "Pipeline for parsing the Nginx error logs", + "processors": [ + { + "pipeline": { + "if": "ctx.message.startsWith('{')", + "name": "logs-nginx.error-1.20.0-third-party" + } + }, + { + "set": { + "field": "event.ingested", + "value": "{{_ingest.timestamp}}" + } + }, + { + "set": { + "field": "ecs.version", + "value": "8.5.1" + } + }, + { + "rename": { + "field": "message", + "target_field": "event.original", + "ignore_missing": true, + "if": "ctx.event?.original == null" + } + }, + { + "grok": { + "field": "event.original", + "patterns": [ + "%{DATA:nginx.error.time} \\[%{DATA:log.level}\\] %{NUMBER:process.pid:long}#%{NUMBER:process.thread.id:long}: (\\*%{NUMBER:nginx.error.connection_id:long} )?%{GREEDYMULTILINE:message}" + ], + "pattern_definitions": { + "GREEDYMULTILINE": "(.|\n|\t)*" + }, + "ignore_missing": true + } + }, + { + "remove": { + "field": "event.created", + "ignore_missing": true, + "ignore_failure": true + } + }, + { + "rename": { + "field": "@timestamp", + "target_field": "event.created" + } + }, + { + "date": { + "if": "ctx.event.timezone == null", + "field": "nginx.error.time", + "target_field": "@timestamp", + "formats": [ + "yyyy/MM/dd H:m:s" + ], + "on_failure": [ + { + "append": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] + } + }, + { + "date": { + "if": "ctx.event.timezone != null", + "field": "nginx.error.time", + "target_field": "@timestamp", + "formats": [ + "yyyy/MM/dd H:m:s" + ], + "timezone": "{{ event.timezone }}", + "on_failure": [ + { + "append": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] + } + }, + { + "remove": { + "field": "nginx.error.time" + } + }, + { + "set": { + "field": "event.kind", + "value": "event" + } + }, + { + "append": { + "field": "event.category", + "value": "web" + } + }, + { + "append": { + "field": "event.type", + "value": "error" + } + }, + { + "remove": { + "field": "event.original", + "if": "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))", + "ignore_failure": true, + "ignore_missing": true + } + }, + { + "pipeline": { + "name": "global@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Global pipeline for all data streams" + } + }, + { + "pipeline": { + "name": "logs@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs`" + } + }, + { + "pipeline": { + "name": "logs-nginx.integration@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `nginx` integration" + } + }, + { + "pipeline": { + "name": "logs-nginx.error@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for the `nginx.error` dataset" + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ], + "_meta": { + "managed_by": "fleet", + "managed": true, + "package": { + "name": "nginx" + } + } +} \ No newline at end of file diff --git a/elastic/logs/pipelines/logs-postgresql.log-1.2.0-pipeline-log.json b/elastic/logs/pipelines/logs-postgresql.log-1.20.0-pipeline-log.json similarity index 85% rename from elastic/logs/pipelines/logs-postgresql.log-1.2.0-pipeline-log.json rename to elastic/logs/pipelines/logs-postgresql.log-1.20.0-pipeline-log.json index ee107f1ab..90da1c36c 100644 --- a/elastic/logs/pipelines/logs-postgresql.log-1.2.0-pipeline-log.json +++ b/elastic/logs/pipelines/logs-postgresql.log-1.20.0-pipeline-log.json @@ -12,9 +12,16 @@ "GREEDYDATA": "(.|\n|\t)*", "POSTGRESQL_DB_NAME": "[a-zA-Z0-9_]+[a-zA-Z0-9_\\$]*", "POSTGRESQL_QUERY_STEP": "%{WORD:postgresql.log.query_step}(?: | %{WORD:postgresql.log.query_name})?", - "POSTGRESQL_ERROR": "\b[A-Z0-9]{5}\b" + "POSTGRESQL_ERROR": "\\b[A-Z0-9]{5}\\b" } } } - ] -} + ], + "_meta": { + "managed_by": "fleet", + "managed": true, + "package": { + "name": "postgresql" + } + } +} \ No newline at end of file diff --git a/elastic/logs/pipelines/logs-postgresql.log-1.2.0.json b/elastic/logs/pipelines/logs-postgresql.log-1.20.0.json similarity index 67% rename from elastic/logs/pipelines/logs-postgresql.log-1.2.0.json rename to elastic/logs/pipelines/logs-postgresql.log-1.20.0.json index 50bc8b106..6bd3c8b5a 100644 --- a/elastic/logs/pipelines/logs-postgresql.log-1.2.0.json +++ b/elastic/logs/pipelines/logs-postgresql.log-1.20.0.json @@ -10,14 +10,15 @@ { "set": { "field": "ecs.version", - "value": "1.12.0" + "value": "8.5.1" } }, { "rename": { "field": "message", "target_field": "event.original", - "ignore_missing": true + "ignore_missing": true, + "if": "ctx.event?.original == null" } }, { @@ -35,16 +36,22 @@ }, { "pipeline": { - "name": "logs-postgresql.log-1.2.0-pipeline-log", - "if": "ctx.separator != ','" + "name": "logs-postgresql.log-1.20.0-pipeline-log", + "if": "ctx.separator != ',' && ctx.separator != ':'" } }, { "pipeline": { - "name": "logs-postgresql.log-1.2.0-pipeline-csv", + "name": "logs-postgresql.log-1.20.0-pipeline-csv", "if": "ctx.separator == ','" } }, + { + "pipeline": { + "name": "logs-postgresql.log-1.20.0-pipeline-aws-log", + "if": "ctx.separator == ':'" + } + }, { "date": { "field": "postgresql.log.timestamp", @@ -131,6 +138,34 @@ "ignore_failure": true, "ignore_missing": true } + }, + { + "pipeline": { + "name": "global@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Global pipeline for all data streams" + } + }, + { + "pipeline": { + "name": "logs@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs`" + } + }, + { + "pipeline": { + "name": "logs-postgresql.integration@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `postgresql` integration" + } + }, + { + "pipeline": { + "name": "logs-postgresql.log@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for the `postgresql.log` dataset" + } } ], "on_failure": [ @@ -140,5 +175,12 @@ "value": "{{ _ingest.on_failure_message }}" } } - ] -} + ], + "_meta": { + "managed_by": "fleet", + "managed": true, + "package": { + "name": "postgresql" + } + } +} \ No newline at end of file diff --git a/elastic/logs/pipelines/logs-redis.log-1.1.0.json b/elastic/logs/pipelines/logs-redis.log-1.1.0.json deleted file mode 100644 index be2f6784c..000000000 --- a/elastic/logs/pipelines/logs-redis.log-1.1.0.json +++ /dev/null @@ -1,114 +0,0 @@ -{ - "description" : "Pipeline for parsing redis logs", - "processors" : [ - { - "set" : { - "field" : "event.ingested", - "value" : "{{_ingest.timestamp}}" - } - }, - { - "set" : { - "field" : "ecs.version", - "value" : "1.12.0" - } - }, - { - "rename" : { - "field" : "message", - "target_field" : "event.original", - "ignore_missing" : true - } - }, - { - "grok" : { - "field" : "event.original", - "patterns" : [ - "(%{POSINT:process.pid:long}:%{CHAR:redis.log.role} )?(%{REDISTIMESTAMP1:redis.log.timestamp}||%{REDISTIMESTAMP2:redis.log.timestamp}) %{REDISLEVEL:log.level} %{GREEDYDATA:message}", - "%{POSINT:process.pid:long}:signal-handler \\(%{POSINT:redis.log.timestamp}\\) %{GREEDYDATA:message}" - ], - "pattern_definitions" : { - "CHAR" : "[a-zA-Z]", - "REDISLEVEL" : "[.\\-*#]", - "REDISTIMESTAMP1" : "%{MONTHDAY} %{MONTH} %{TIME}", - "REDISTIMESTAMP2" : "%{MONTHDAY} %{MONTH} %{YEAR} %{TIME}" - } - } - }, - { - "script" : { - "lang" : "painless", - "if" : "ctx.log?.level != null", - "source" : "if (ctx.log.level == params.dot) {ctx.log.level = params.debug;} else if (ctx.log.level == params.dash) {ctx.log.level = params.verbose;} else if (ctx.log.level == params.asterisk) {ctx.log.level = params.notice;} else if (ctx.log.level == params.hash) {ctx.log.level = params.warning;}", - "params" : { - "dot" : ".", - "debug" : "debug", - "dash" : "-", - "verbose" : "verbose", - "asterisk" : "*", - "notice" : "notice", - "hash" : "#", - "warning" : "warning" - } - } - }, - { - "script" : { - "lang" : "painless", - "if" : "ctx.redis?.log?.role != null", - "source" : "if (ctx.redis.log.role == params.master_abbrev) {ctx.redis.log.role = params.master;} else if (ctx.redis.log.role == params.slave_abbrev) {ctx.redis.log.role = params.slave;} else if (ctx.redis.log.role == params.child_abbrev) {ctx.redis.log.role = params.child;} else if (ctx.redis.log.role == params.sentinel_abbrev) {ctx.redis.log.role = params.sentinel;}", - "params" : { - "master_abbrev" : "M", - "master" : "master", - "slave_abbrev" : "S", - "slave" : "slave", - "child_abbrev" : "C", - "child" : "child", - "sentinel_abbrev" : "X", - "sentinel" : "sentinel" - } - } - }, - { - "rename" : { - "field" : "@timestamp", - "target_field" : "event.created" - } - }, - { - "date" : { - "field" : "redis.log.timestamp", - "target_field" : "@timestamp", - "formats" : [ - "dd MMM yyyy H:m:s.SSS", - "dd MMM H:m:s.SSS", - "dd MMM H:m:s", - "UNIX" - ], - "ignore_failure" : false - } - }, - { - "remove" : { - "field" : "redis.log.timestamp", - "ignore_failure" : true - } - }, - { - "remove" : { - "field" : "event.original", - "if" : "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))", - "ignore_failure" : true, - "ignore_missing" : true - } - } - ], - "on_failure" : [ - { - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/elastic/logs/pipelines/logs-redis.log-1.15.0.json b/elastic/logs/pipelines/logs-redis.log-1.15.0.json new file mode 100644 index 000000000..787d6577f --- /dev/null +++ b/elastic/logs/pipelines/logs-redis.log-1.15.0.json @@ -0,0 +1,150 @@ +{ + "description": "Pipeline for parsing redis logs", + "processors": [ + { + "set": { + "field": "event.ingested", + "value": "{{_ingest.timestamp}}" + } + }, + { + "set": { + "field": "ecs.version", + "value": "8.5.1" + } + }, + { + "rename": { + "field": "message", + "target_field": "event.original", + "ignore_missing": true, + "if": "ctx.event?.original == null" + } + }, + { + "grok": { + "field": "event.original", + "patterns": [ + "(%{POSINT:process.pid:long}:%{CHAR:redis.log.role} )?(%{REDISTIMESTAMP1:redis.log.timestamp}||%{REDISTIMESTAMP2:redis.log.timestamp}) %{REDISLEVEL:log.level} %{GREEDYDATA:message}", + "%{POSINT:process.pid:long}:signal-handler \\(%{POSINT:redis.log.timestamp}\\) %{GREEDYDATA:message}" + ], + "pattern_definitions": { + "CHAR": "[a-zA-Z]", + "REDISLEVEL": "[.\\-*#]", + "REDISTIMESTAMP1": "%{MONTHDAY} %{MONTH} %{TIME}", + "REDISTIMESTAMP2": "%{MONTHDAY} %{MONTH} %{YEAR} %{TIME}" + } + } + }, + { + "script": { + "lang": "painless", + "if": "ctx.log?.level != null", + "source": "if (ctx.log.level == params.dot) {\n ctx.log.level = params.debug;\n} else if (ctx.log.level == params.dash) {\n ctx.log.level = params.verbose;\n} else if (ctx.log.level == params.asterisk) {\n ctx.log.level = params.notice;\n} else if (ctx.log.level == params.hash) {\n ctx.log.level = params.warning;\n}", + "params": { + "dot": ".", + "debug": "debug", + "dash": "-", + "verbose": "verbose", + "asterisk": "*", + "notice": "notice", + "hash": "#", + "warning": "warning" + } + } + }, + { + "script": { + "lang": "painless", + "if": "ctx.redis?.log?.role != null", + "source": "if (ctx.redis.log.role == params.master_abbrev) {\n ctx.redis.log.role = params.master;\n} else if (ctx.redis.log.role == params.slave_abbrev) {\n ctx.redis.log.role = params.slave;\n} else if (ctx.redis.log.role == params.child_abbrev) {\n ctx.redis.log.role = params.child;\n} else if (ctx.redis.log.role == params.sentinel_abbrev) {\n ctx.redis.log.role = params.sentinel;\n}", + "params": { + "master_abbrev": "M", + "master": "master", + "slave_abbrev": "S", + "slave": "slave", + "child_abbrev": "C", + "child": "child", + "sentinel_abbrev": "X", + "sentinel": "sentinel" + } + } + }, + { + "rename": { + "field": "@timestamp", + "target_field": "event.created" + } + }, + { + "date": { + "field": "redis.log.timestamp", + "target_field": "@timestamp", + "formats": [ + "dd MMM yyyy H:m:s.SSS", + "dd MMM H:m:s.SSS", + "dd MMM H:m:s", + "UNIX" + ], + "ignore_failure": false + } + }, + { + "remove": { + "field": "redis.log.timestamp", + "ignore_failure": true + } + }, + { + "remove": { + "field": "event.original", + "if": "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))", + "ignore_failure": true, + "ignore_missing": true + } + }, + { + "pipeline": { + "name": "global@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Global pipeline for all data streams" + } + }, + { + "pipeline": { + "name": "logs@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs`" + } + }, + { + "pipeline": { + "name": "logs-redis.integration@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `redis` integration" + } + }, + { + "pipeline": { + "name": "logs-redis.log@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for the `redis.log` dataset" + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ], + "_meta": { + "managed_by": "fleet", + "managed": true, + "package": { + "name": "redis" + } + } +} \ No newline at end of file diff --git a/elastic/logs/pipelines/logs-redis.slowlog-1.1.0.json b/elastic/logs/pipelines/logs-redis.slowlog-1.1.0.json deleted file mode 100644 index eb4be0fd3..000000000 --- a/elastic/logs/pipelines/logs-redis.slowlog-1.1.0.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "description": "Pipeline for parsing redis slowlog logs", - "processors": [], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/elastic/logs/pipelines/logs-redis.slowlog-1.15.0.json b/elastic/logs/pipelines/logs-redis.slowlog-1.15.0.json new file mode 100644 index 000000000..474465890 --- /dev/null +++ b/elastic/logs/pipelines/logs-redis.slowlog-1.15.0.json @@ -0,0 +1,48 @@ +{ + "description": "Pipeline for parsing redis slowlog logs", + "processors": [ + { + "pipeline": { + "name": "global@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Global pipeline for all data streams" + } + }, + { + "pipeline": { + "name": "logs@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs`" + } + }, + { + "pipeline": { + "name": "logs-redis.integration@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `redis` integration" + } + }, + { + "pipeline": { + "name": "logs-redis.slowlog@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for the `redis.slowlog` dataset" + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ], + "_meta": { + "managed_by": "fleet", + "managed": true, + "package": { + "name": "redis" + } + } +} \ No newline at end of file diff --git a/elastic/logs/pipelines/logs-system.auth-1.58.1.json b/elastic/logs/pipelines/logs-system.auth-1.58.1.json new file mode 100644 index 000000000..04c723ab0 --- /dev/null +++ b/elastic/logs/pipelines/logs-system.auth-1.58.1.json @@ -0,0 +1,612 @@ +{ + "description": "Pipeline for parsing system authorization and secure logs.", + "processors": [ + { + "rename": { + "tag": "rename_message", + "if": "ctx.event?.original == null", + "field": "message", + "target_field": "event.original", + "ignore_missing": true + } + }, + { + "grok": { + "if": "ctx.log?.syslog == null", + "description": "Grok the message header.", + "tag": "grok-message-header", + "field": "event.original", + "pattern_definitions": { + "GREEDYMULTILINE": "(.|\\n)*", + "TIMESTAMP": "(?:%{TIMESTAMP_ISO8601}|%{SYSLOGTIMESTAMP})" + }, + "patterns": [ + "^<%{NONNEGINT:log.syslog.priority:int}>(?:%{NONNEGINT:system.auth.syslog.version} )?+(?:-|%{TIMESTAMP:system.auth.timestamp}) +(?:-|%{IPORHOST:host.hostname}) +(?:-|%{SYSLOG5424PRINTASCII:process.name}) +(?:-|%{POSINT:process.pid:long}) +(?:-|%{SYSLOG5424PRINTASCII:event.code}) +(?:-|%{SYSLOG5424SD:syslog5424_sd})? +%{GREEDYDATA:_temp.message}$", + "^%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname}? %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?:%{SPACE}%{GREEDYMULTILINE:_temp.message}$", + "^<%{NONNEGINT:log.syslog.priority:int}>(?:%{NONNEGINT:system.auth.syslog.version} )?%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname}? %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?:%{SPACE}%{GREEDYMULTILINE:_temp.message}$" + ] + } + }, + { + "rename": { + "tag": "rename_message_tmp_message", + "field": "message", + "target_field": "_temp.message", + "if": "ctx.log?.syslog != null && ctx.message != null" + } + }, + { + "grok": { + "description": "Grok specific auth messages.", + "tag": "grok-specific-messages", + "field": "_temp.message", + "ignore_missing": true, + "patterns": [ + "^%{DATA:system.auth.ssh.event} %{DATA:system.auth.ssh.method} for (invalid user)?%{DATA:user.name} from %{IPORHOST:source.address} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?", + "^%{DATA:system.auth.ssh.event} user %{DATA:user.name} from %{IPORHOST:source.address}", + "^Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}", + "^%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}", + "^new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id}", + "^new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id}, GID=%{NUMBER:group.id}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$" + ], + "on_failure": [ + { + "rename": { + "description": "Leave the unmatched content in message.", + "field": "_temp.message", + "target_field": "message" + } + } + ] + } + }, + { + "append": { + "tag": "append_category_process", + "field": "event.category", + "value": "process", + "allow_duplicates": false, + "if": "ctx.system?.auth?.sudo?.command != null" + } + }, + { + "remove": { + "tag": "remove_temp", + "field": "_temp", + "ignore_missing": true + } + }, + { + "grok": { + "description": "Grok usernames from PAM messages.", + "tag": "grok-pam-users", + "field": "message", + "ignore_missing": true, + "ignore_failure": true, + "patterns": [ + "for user %{QUOTE}?%{DATA:_temp.foruser}%{QUOTE}? by %{QUOTE}?%{DATA:_temp.byuser}%{QUOTE}?(?:\\(uid=%{NUMBER:_temp.byuid}\\))?$", + "for user %{QUOTE}?%{DATA:_temp.foruser}%{QUOTE}?$", + "by user %{QUOTE}?%{DATA:_temp.byuser}%{QUOTE}?$", + "%{BOUNDARY} user %{QUOTE}%{DATA:_temp.user}%{QUOTE}" + ], + "pattern_definitions": { + "QUOTE": "['\"]", + "BOUNDARY": "(?