diff --git a/elastic/logs/challenges/logging-chicken.json b/elastic/logs/challenges/logging-chicken.json new file mode 100644 index 000000000..5135c4d93 --- /dev/null +++ b/elastic/logs/challenges/logging-chicken.json @@ -0,0 +1,90 @@ +{% import "rally.helpers" as rally %} +{ + "name": "logging-insist-chicken", + "description": "INSIST_🐔 everything", + "schedule": [ + {% include "tasks/index-setup.json" %}, + { + "name": "bulk-index", + "operation": { + "operation-type": "raw-bulk", + "param-source": "processed-source", + "time-format": "milliseconds", + "profile": "fixed_interval", + "bulk-size": {{ p_bulk_size }}, + "detailed-results": true + }, + "clients": {{ p_bulk_indexing_clients }}{% if p_throttle_indexing %}, + "ignore-response-error-level": "{{error_level | default('non-fatal')}}", + "schedule": "timestamp-throttler", + "max-delay-secs": 1 + {% endif %} + }, + { + "name": "refresh-after-index", + "index": "logs-*", + "operation": "refresh" + }, + { + "name": "wait-until-index-merges-fininshes", + "operation": { + "operation-type": "index-stats", + "index": "logs-*", + "condition": { + "path": "_all.total.merges.current", + "expected-value": 0 + }, + "retry-until-success": true, + "include-in-reporting": false + }, + "tags": ["wait"] + }, + { + "operation": "disable_query_cache", + "tags": ["esql", "settings"] + }, + { + "operation": "limit_500", + "clients": {{ p_search_clients }}, + "warmup-iterations": {{ warmup_iterations | default(20) }}, + "iterations": {{ iterations | default(100) }}, + "tags": ["esql"] + }, + { + "operation": "chicken_1", + "clients": {{ p_search_clients }}, + {# TODO: restore iterations / warmup_iterations once insist_:chicken: has acceptable performance #} + "warmup-iterations": {{ warmup_iterations | default(3) }}, + "iterations": {{ iterations | default(5) }}, + "tags": ["esql", "insist"] + }, + { + "operation": "chicken_2", + "clients": {{ p_search_clients }}, + "warmup-iterations": {{ warmup_iterations | default(3) }}, + "iterations": {{ iterations | default(5) }}, + "tags": ["esql", "insist"] + }, + { + "operation": "chicken_3", + "clients": {{ p_search_clients }}, + "warmup-iterations": {{ warmup_iterations | default(3) }}, + "iterations": {{ iterations | default(5) }}, + "tags": ["esql", "insist"] + }, + { + "operation": "chicken_3_with_where", + "clients": {{ p_search_clients }}, + "warmup-iterations": {{ warmup_iterations | default(3) }}, + "iterations": {{ iterations | default(5) }}, + "tags": ["esql", "insist"] + }, + { + "operation": "chicken_4", + "clients": {{ p_search_clients }}, + "warmup-iterations": {{ warmup_iterations | default(3) }}, + "iterations": {{ iterations | default(5) }}, + "tags": ["esql", "insist"] + } + ] +} diff --git a/elastic/logs/operations/esql-chicken.json b/elastic/logs/operations/esql-chicken.json new file mode 100644 index 000000000..89017ae7e --- /dev/null +++ b/elastic/logs/operations/esql-chicken.json @@ -0,0 +1,31 @@ + { + "name": "limit_500", + "operation-type": "esql", + "query": "FROM logs-* | LIMIT 500" + }, + {# TODO: All COALESCE usages can be removed once https://github.com/elastic/elasticsearch/issues/130220 is fixed #} + { + "name": "chicken_1", + "operation-type": "esql", + "query": "FROM logs-* | INSIST_🐔 agent.id | EVAL col0 = COALESCE(agent.id, \"\") | WHERE MATCH(message, \"204.188.85.203\") and col0 == \"703ef3bd-dc06-4c6c-a621-13c89245e6b0\"" + }, + { + "name": "chicken_2", + "operation-type": "esql", + "query": "FROM logs-* | INSIST_🐔 kubernetes.container.image | EVAL col0 = COALESCE(kubernetes.container.image, \"\") | WHERE col0 != \"\" | STATS col1 = COUNT() BY col0,data_stream.dataset" + }, + { + "name": "chicken_3", + "operation-type": "esql", + "query": "FROM logs-* | INSIST_🐔 agent.id | STATS c=count() BY agent.id, data_stream.dataset | SORT c DESC" + }, + { + "name": "chicken_3_with_where", + "operation-type": "esql", + "query": "FROM logs-* | INSIST_🐔 agent.id | EVAL end_time = DATE_PARSE(\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\",\"{{ bulk_end_date }}\") | WHERE @timestamp >= end_time - 1 hour | STATS c=count() BY agent.id, data_stream.dataset | SORT c DESC" + }, + { + "name": "chicken_4", + "operation-type": "esql", + "query": "FROM logs-* | INSIST_🐔 agent.hostname | EVAL col0 = COALESCE(agent.hostname, \"\") | WHERE col0 == \"elasticsearch-ci-immutable-centos-7-1599241536066250344\"" + } \ No newline at end of file diff --git a/elastic/logs/tasks/index-setup.json b/elastic/logs/tasks/index-setup.json index 3e505938c..f18272964 100644 --- a/elastic/logs/tasks/index-setup.json +++ b/elastic/logs/tasks/index-setup.json @@ -60,6 +60,7 @@ "operation-type": "create-data-stream" } }, +{% if p_mapping == "mapped" %} { "name": "validate-package-template-installation", "operation": { @@ -68,6 +69,7 @@ "asset-types": ["index-templates"] } }, +{% endif %} { "name": "update-custom-package-templates", "operation": { diff --git a/elastic/logs/templates/composable/logs-dynamic-false.json b/elastic/logs/templates/composable/logs-dynamic-false.json new file mode 100644 index 000000000..79e268133 --- /dev/null +++ b/elastic/logs/templates/composable/logs-dynamic-false.json @@ -0,0 +1,34 @@ +{ + "name": "logs-dynamic-false", + "index_template": { + "index_patterns": [ + "logs-*-*" + ], + "template": { + "settings": {}, + "mappings": { + "dynamic": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "message": { + "type": "match_only_text" + } + } + } + }, + "composed_of": [ + "logs@settings", + "logs@mappings", + "track-custom-shared-settings", + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" + ], + "priority": 200, + "data_stream": { + "hidden": false, + "allow_custom_routing": false + } + } +} \ No newline at end of file diff --git a/elastic/logs/track.json b/elastic/logs/track.json index 8c6878810..2dd24f851 100644 --- a/elastic/logs/track.json +++ b/elastic/logs/track.json @@ -73,6 +73,8 @@ {% set es_version = "7.13.2" %} +{% set p_mapping = ( mapping | default('mapped') ) %} + {% set p_integration_ratios = (integration_ratios | default({ "kafka": { "corpora": { @@ -218,6 +220,7 @@ } ], "component-templates": [ + {% if p_mapping == "mapped" %} {% if build_flavor != "serverless" %} { "name": ".fleet_agent_id_verification-1", @@ -339,6 +342,7 @@ "name": "logs-system.syslog@custom", "template": "./templates/component/logs-system.syslog@custom.json" }, + {% endif %} { "name": "track-custom-mappings", "template": "./templates/component/track-custom-mappings.json" @@ -361,6 +365,7 @@ } ], "composable-templates": [ + {% if p_mapping == "mapped" %} { "name": "logs-kafka.log", "index-pattern": "logs-kafka.log-*", @@ -469,6 +474,16 @@ "delete-matching-indices": false, "template": "./templates/composable/auditbeat-quantitative.json" } + {% endif %} + {% if p_mapping == "unmapped" %} + { + "name": "logs-unmapped", + "index-pattern": "logs-*-*", + "delete-matching-indices": false, + "template": "./templates/composable/logs-dynamic-false.json", + "template-path": "index_template" + } + {% endif %} ], "corpora": [ {