diff --git a/big5/operations/ppl.json b/big5/operations/ppl.json index f0065414..5c899c13 100644 --- a/big5/operations/ppl.json +++ b/big5/operations/ppl.json @@ -31,7 +31,7 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} match(`process.name`, 'kernel') | sort + `@timestamp` | head 10" + "query": "source = {{index_name | default('big5')}} {% if distribution_version.split('.') | map('int') | list >= '3.3.0'.split('.') | map('int') | list %}process.name=kernel{% else %}match(`process.name`, 'kernel'){% endif %} | sort + `@timestamp` | head 10" } }, { @@ -40,7 +40,7 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} match(`process.name`, 'kernel') | sort + `@timestamp` | head 10" + "query": "source = {{index_name | default('big5')}} {% if distribution_version.split('.') | map('int') | list >= '3.3.0'.split('.') | map('int') | list %}process.name=kernel{% else %}match(`process.name`, 'kernel'){% endif %} | sort + `@timestamp` | head 10" } }, { @@ -76,7 +76,7 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} | where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-02 10:00:00' | stats count() by `process.name`, `cloud.region`, `aws.cloudwatch.log_stream` | sort - `process.name`, + `cloud.region`, + `aws.cloudwatch.log_stream` | head 10" + "query": "source = {{index_name | default('big5')}} | where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-02 10:00:00' | stats {% if distribution_version.split('.') | map('int') | list >= '3.3.0'.split('.') | map('int') | list %}bucket_nullable = false {% endif %}count() by `process.name`, `cloud.region`, `aws.cloudwatch.log_stream` | sort - `process.name`, + `cloud.region`, + `aws.cloudwatch.log_stream` | head 10" } }, { @@ -85,7 +85,7 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} | where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-02 10:00:00' | stats count() by `process.name`, `cloud.region` | sort - `process.name`, + `cloud.region` | head 10" + "query": "source = {{index_name | default('big5')}} | where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-02 10:00:00' | stats {% if distribution_version.split('.') | map('int') | list >= '3.3.0'.split('.') | map('int') | list %}bucket_nullable = false {% endif %}count() by `process.name`, `cloud.region` | sort - `process.name`, + `cloud.region` | head 10" } }, { @@ -112,7 +112,7 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} match(`process.name`, 'kernel') | sort - `@timestamp` | head 10" + "query": "source = {{index_name | default('big5')}} {% if distribution_version.split('.') | map('int') | list >= '3.3.0'.split('.') | map('int') | list %}process.name=kernel{% else %}match(`process.name`, 'kernel'){% endif %} | sort - `@timestamp` | head 10" } }, { @@ -121,7 +121,7 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} match(`process.name`, 'kernel') | sort - `@timestamp` | head 10" + "query": "source = {{index_name | default('big5')}} {% if distribution_version.split('.') | map('int') | list >= '3.3.0'.split('.') | map('int') | list %}process.name=kernel{% else %}match(`process.name`, 'kernel'){% endif %} | sort - `@timestamp` | head 10" } }, { @@ -148,7 +148,7 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} match(`process.name`, 'kernel') | where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' | head 10" + "query": "source = {{index_name | default('big5')}} {% if distribution_version.split('.') | map('int') | list >= '3.3.0'.split('.') | map('int') | list %}process.name=kernel{% else %}match(`process.name`, 'kernel'){% endif %} | where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' | head 10" } }, { @@ -157,7 +157,7 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} | stats count() as country by `aws.cloudwatch.log_stream` | sort - country | head 50" + "query": "source = {{index_name | default('big5')}} | stats {% if distribution_version.split('.') | map('int') | list >= '3.3.0'.split('.') | map('int') | list %}bucket_nullable = false {% endif %}count() as country by `aws.cloudwatch.log_stream` | sort - country | head 50" } }, { @@ -166,7 +166,7 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} | stats count() as station by `aws.cloudwatch.log_stream` | sort - station | head 500" + "query": "source = {{index_name | default('big5')}} | stats {% if distribution_version.split('.') | map('int') | list >= '3.3.0'.split('.') | map('int') | list %}bucket_nullable = false {% endif %}count() as station by `aws.cloudwatch.log_stream` | sort - station | head 500" } }, { @@ -175,7 +175,7 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} | where `@timestamp` >= '2023-01-05 00:00:00' and `@timestamp` < '2023-01-05 05:00:00' | stats count() by `process.name`, `cloud.region` | sort - `count()`" + "query": "source = {{index_name | default('big5')}} | where `@timestamp` >= '2023-01-05 00:00:00' and `@timestamp` < '2023-01-05 05:00:00' | stats {% if distribution_version.split('.') | map('int') | list >= '3.3.0'.split('.') | map('int') | list %}bucket_nullable = false {% endif %}count() by `process.name`, `cloud.region` | sort - `count()` | head 10" } }, { @@ -184,7 +184,7 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} query_string(['message'], 'monkey jackal bear') | where `@timestamp` >= '2023-01-03 00:00:00' and `@timestamp` < '2023-01-03 10:00:00' | sort + `@timestamp` | head 10" + "query": "source = {{index_name | default('big5')}} | where query_string(['message'], 'monkey jackal bear') and `@timestamp` >= '2023-01-03 00:00:00' and `@timestamp` < '2023-01-03 10:00:00' | sort + `@timestamp` | head 10" } }, { @@ -193,7 +193,7 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} query_string(['message'], 'monkey jackal bear') | where `@timestamp` >= '2023-01-03 00:00:00' and `@timestamp` < '2023-01-03 10:00:00' | head 10" + "query": "source = {{index_name | default('big5')}} | where query_string(['message'], 'monkey jackal bear') and `@timestamp` >= '2023-01-03 00:00:00' and `@timestamp` < '2023-01-03 10:00:00' | head 10" } }, { @@ -202,7 +202,7 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} query_string(['message'], 'monkey jackal bear') | head 10" + "query": "source = {{index_name | default('big5')}} {% if distribution_version.split('.') | map('int') | list >= '3.3.0'.split('.') | map('int') | list %}| where query_string(['message'], 'monkey jackal bear'){% else %}query_string(['message'], 'monkey jackal bear'){% endif %} | head 10" } }, { @@ -211,7 +211,7 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} | eval range_bucket = case(`metrics.size` < -10, 'range_1', `metrics.size` >= -10 and `metrics.size` < 10, 'range_2', `metrics.size` >= 10 and `metrics.size` < 100, 'range_3', `metrics.size` >= 100 and `metrics.size` < 1000, 'range_4', `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_5', `metrics.size` >= 2000, 'range_6') | stats min(`metrics.tmin`) as tmin, avg(`metrics.size`) as tavg, max(`metrics.size`) as tmax by range_bucket, span(`@timestamp`, 1h) as auto_span | sort + range_bucket, + auto_span" + "query": "source = {{index_name | default('big5')}} | eval range_bucket = case(`metrics.size` < -10, 'range_1', `metrics.size` >= -10 and `metrics.size` < 10, 'range_2', `metrics.size` >= 10 and `metrics.size` < 100, 'range_3', `metrics.size` >= 100 and `metrics.size` < 1000, 'range_4', `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_5', `metrics.size` >= 2000, 'range_6') | {% if distribution_version.split('.') | map('int') | list >= '3.4.0'.split('.') | map('int') | list %}bin `@timestamp` bins=10 | stats min(`metrics.tmin`) as tmin, avg(`metrics.size`) as tavg, max(`metrics.size`) as tmax by range_bucket, `@timestamp`{% else %}stats min(`metrics.tmin`) as tmin, avg(`metrics.size`) as tavg, max(`metrics.size`) as tmax by range_bucket, span(`@timestamp`, 1h) as auto_span | sort + range_bucket, + auto_span{% endif %}" } }, { @@ -220,7 +220,52 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} | eval range_bucket = case(`metrics.size` < -10, 'range_1', `metrics.size` >= -10 and `metrics.size` < 10, 'range_2', `metrics.size` >= 10 and `metrics.size` < 100, 'range_3', `metrics.size` >= 100 and `metrics.size` < 1000, 'range_4', `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_5', `metrics.size` >= 2000, 'range_6') | stats count() by range_bucket, span(`@timestamp`, 1h) as auto_span | sort + range_bucket, + auto_span" + "query": "source = {{index_name | default('big5')}} | eval range_bucket = case(`metrics.size` < -10, 'range_1', `metrics.size` >= -10 and `metrics.size` < 10, 'range_2', `metrics.size` >= 10 and `metrics.size` < 100, 'range_3', `metrics.size` >= 100 and `metrics.size` < 1000, 'range_4', `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_5', `metrics.size` >= 2000, 'range_6') | {% if distribution_version.split('.') | map('int') | list >= '3.4.0'.split('.') | map('int') | list %}bin `@timestamp` bins=20 | stats count() by range_bucket, `@timestamp`{% else %}stats count() by range_bucket, span(`@timestamp`, 1h) as auto_span | sort + range_bucket, + auto_span{% endif %}" + } + }, + { + "name": "ppl-cardinality-agg-high", + "operation-type": "raw-request", + "path": "/_plugins/_ppl", + "method": "POST", + "body": { + "query": "source = {{index_name | default('big5')}} | stats {% if distribution_version.split('.') | map('int') | list >= '3.3.0'.split('.') | map('int') | list %}bucket_nullable = false {% endif %}dc(`agent.name`)" + } + }, + { + "name": "ppl-cardinality-agg-high-2", + "operation-type": "raw-request", + "path": "/_plugins/_ppl", + "method": "POST", + "body": { + "query": "source = {{index_name | default('big5')}} | stats {% if distribution_version.split('.') | map('int') | list >= '3.3.0'.split('.') | map('int') | list %}bucket_nullable = false {% endif %}dc(`event.id`)" + } + }, + { + "name": "ppl-cardinality-agg-low", + "operation-type": "raw-request", + "path": "/_plugins/_ppl", + "method": "POST", + "body": { + "query": "source = {{index_name | default('big5')}} | stats {% if distribution_version.split('.') | map('int') | list >= '3.3.0'.split('.') | map('int') | list %}bucket_nullable = false {% endif %}dc(`cloud.region`)" + } + }, + { + "name": "ppl-range-agg-1", + "operation-type": "raw-request", + "path": "/_plugins/_ppl", + "method": "POST", + "body": { + "query": "source = {{index_name | default('big5')}} | eval range_bucket = case(`metrics.size` < -10, 'range_1', `metrics.size` >= -10 and `metrics.size` < 10, 'range_2', `metrics.size` >= 10 and `metrics.size` < 100, 'range_3', `metrics.size` >= 100 and `metrics.size` < 1000, 'range_4', `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_5', `metrics.size` >= 2000, 'range_6') | stats count() by range_bucket" + } + }, + { + "name": "ppl-range-agg-2", + "operation-type": "raw-request", + "path": "/_plugins/_ppl", + "method": "POST", + "body": { + "query": "source = {{index_name | default('big5')}} | eval range_bucket = case(`metrics.size` < 100, 'range_1', `metrics.size` >= 100 and `metrics.size` < 1000, 'range_2', `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_3', `metrics.size` >= 2000, 'range_4') | stats count() by range_bucket" } }, { @@ -301,7 +346,7 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} match(`process.name`, 'kernel') | sort + `@timestamp` | head 10" + "query": "source = {{index_name | default('big5')}} {% if distribution_version.split('.') | map('int') | list >= '3.3.0'.split('.') | map('int') | list %}process.name=kernel{% else %}match(`process.name`, 'kernel'){% endif %} | sort + `meta.file` | head 10" } }, { @@ -310,7 +355,7 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} match(`process.name`, 'kernel') | sort + `@timestamp` | head 10" + "query": "source = {{index_name | default('big5')}} {% if distribution_version.split('.') | map('int') | list >= '3.3.0'.split('.') | map('int') | list %}process.name=kernel{% else %}match(`process.name`, 'kernel'){% endif %} | sort + `meta.file` | head 10" } }, { @@ -319,7 +364,7 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} match(`log.file.path`, '/var/log/messages/solarshark') | sort + `metrics.size` | head 10" + "query": "source = {{index_name | default('big5')}} {% if distribution_version.split('.') | map('int') | list >= '3.3.0'.split('.') | map('int') | list %}log.file.path=\"/var/log/messages/solarshark\"{% else %}match(`log.file.path`, '/var/log/messages/solarshark'){% endif %} | sort + `metrics.size` | head 10" } }, { @@ -337,7 +382,7 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} match(`log.file.path`, '/var/log/messages/solarshark') | sort - `metrics.size` | head 10" + "query": "source = {{index_name | default('big5')}} {% if distribution_version.split('.') | map('int') | list >= '3.3.0'.split('.') | map('int') | list %}log.file.path=\"/var/log/messages/solarshark\"{% else %}match(`log.file.path`, '/var/log/messages/solarshark'){% endif %} | sort - `metrics.size` | head 10" } }, { @@ -355,7 +400,7 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} | where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' | stats count() by `aws.cloudwatch.log_stream` | head 10" + "query": "source = {{index_name | default('big5')}} | where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' | stats count() by `aws.cloudwatch.log_stream`, `process.name` | head 10" } }, { @@ -364,6 +409,6 @@ "path": "/_plugins/_ppl", "method": "POST", "body": { - "query": "source = {{index_name | default('big5')}} | where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' | stats count() by `process.name` | head 10" + "query": "source = {{index_name | default('big5')}} | where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' | stats count() by `process.name`, `aws.cloudwatch.log_stream` | head 10" } } \ No newline at end of file diff --git a/big5/queries/ppl/cardinality_agg_high.ppl b/big5/queries/ppl/cardinality_agg_high.ppl new file mode 100644 index 00000000..0e8c3dec --- /dev/null +++ b/big5/queries/ppl/cardinality_agg_high.ppl @@ -0,0 +1,22 @@ +/* +{ + "name": "cardinality-agg-high", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "agent": { + "cardinality": { + "field": "agent.name" + {% if distribution_version.split('.') | map('int') | list >= "2.19.1".split('.') | map('int') | list and distribution_version.split('.') | map('int') | list < "6.0.0".split('.') | map('int') | list %} + , "execution_hint": "ordinals" + {% endif %} + } + } + } + } +} +*/ +source = big5 +| stats dc(`agent.name`) \ No newline at end of file diff --git a/big5/queries/ppl/cardinality_agg_high_2.ppl b/big5/queries/ppl/cardinality_agg_high_2.ppl new file mode 100644 index 00000000..93130873 --- /dev/null +++ b/big5/queries/ppl/cardinality_agg_high_2.ppl @@ -0,0 +1,21 @@ +/* +{ + "name": "cardinality-agg-high-2", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-timeout": 1800, + "body": { + "size": 0, + "aggs": { + "agent": { + "cardinality": { + "field": "event.id", + "execution_hint":"ordinals" + } + } + } + } +} +*/ +source = big5 +| stats dc(`event.id`) \ No newline at end of file diff --git a/big5/queries/ppl/cardinality_agg_low.ppl b/big5/queries/ppl/cardinality_agg_low.ppl new file mode 100644 index 00000000..ca6c8b21 --- /dev/null +++ b/big5/queries/ppl/cardinality_agg_low.ppl @@ -0,0 +1,19 @@ +/* +{ + "name": "cardinality-agg-low", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "region": { + "cardinality": { + "field": "cloud.region" + } + } + } + } +} +*/ +source = big5 +| stats dc(`cloud.region`) \ No newline at end of file diff --git a/big5/queries/ppl/range_agg_1.ppl b/big5/queries/ppl/range_agg_1.ppl new file mode 100644 index 00000000..95280e69 --- /dev/null +++ b/big5/queries/ppl/range_agg_1.ppl @@ -0,0 +1,50 @@ +/* +{ + "name": "range-agg-1", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "tmax": { + "range": { + "field": "metrics.size", + "ranges": [ + { + "to": -10 + }, + { + "from": -10, + "to": 10 + }, + { + "from": 10, + "to": 100 + }, + { + "from": 100, + "to": 1000 + }, + { + "from": 1000, + "to": 2000 + }, + { + "from": 2000 + } + ] + } + } + } + } +} +*/ +source = big5 +| eval range_bucket = case( + `metrics.size` < -10, 'range_1', + `metrics.size` >= -10 and `metrics.size` < 10, 'range_2', + `metrics.size` >= 10 and `metrics.size` < 100, 'range_3', + `metrics.size` >= 100 and `metrics.size` < 1000, 'range_4', + `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_5', + `metrics.size` >= 2000, 'range_6') +| stats count() by range_bucket \ No newline at end of file diff --git a/big5/queries/ppl/range_agg_2.ppl b/big5/queries/ppl/range_agg_2.ppl new file mode 100644 index 00000000..0988d688 --- /dev/null +++ b/big5/queries/ppl/range_agg_2.ppl @@ -0,0 +1,40 @@ +/* +{ + "name": "range-agg-2", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "tmax": { + "range": { + "field": "metrics.size", + "ranges": [ + { + "to": 100 + }, + { + "from": 100, + "to": 1000 + }, + { + "from": 1000, + "to": 2000 + }, + { + "from": 2000 + } + ] + } + } + } + } +} +*/ +source = big5 +| eval range_bucket = case( + `metrics.size` < 100, 'range_1', + `metrics.size` >= 100 and `metrics.size` < 1000, 'range_2', + `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_3', + `metrics.size` >= 2000, 'range_4') +| stats count() by range_bucket \ No newline at end of file diff --git a/big5/test_procedures/ppl/ppl-schedule.json b/big5/test_procedures/ppl/ppl-schedule.json index 962d6914..ca56e8cb 100644 --- a/big5/test_procedures/ppl/ppl-schedule.json +++ b/big5/test_procedures/ppl/ppl-schedule.json @@ -173,6 +173,41 @@ "target-throughput": {{ ppl_range_auto_date_histo_target_throughput or target_throughput | default(2) | tojson }}, "clients": {{ ppl_range_auto_date_histo_clients or search_clients | default(1) }} }, +{ + "operation": "ppl-cardinality-agg-high", + "warmup-iterations": {{ ppl_cardinality_agg_high_warmup_iterations or warmup_iterations | default(200) | tojson }}, + "iterations": {{ ppl_cardinality_agg_high_iterations or test_iterations | default(100) | tojson }}, + "target-throughput": {{ ppl_cardinality_agg_high_target_throughput or target_throughput | default(2) | tojson }}, + "clients": {{ ppl_cardinality_agg_high_clients or search_clients | default(1) }} +}, +{ + "operation": "ppl-cardinality-agg-high-2", + "warmup-iterations": {{ ppl_cardinality_agg_high_2_warmup_iterations or warmup_iterations | default(200) | tojson }}, + "iterations": {{ ppl_cardinality_agg_high_2_iterations or test_iterations | default(100) | tojson }}, + "target-throughput": {{ ppl_cardinality_agg_high_2_target_throughput or target_throughput | default(2) | tojson }}, + "clients": {{ ppl_cardinality_agg_high_2_clients or search_clients | default(1) }} +}, +{ + "operation": "ppl-cardinality-agg-low", + "warmup-iterations": {{ ppl_cardinality_agg_low_warmup_iterations or warmup_iterations | default(200) | tojson }}, + "iterations": {{ ppl_cardinality_agg_low_iterations or test_iterations | default(100) | tojson }}, + "target-throughput": {{ ppl_cardinality_agg_low_target_throughput or target_throughput | default(2) | tojson }}, + "clients": {{ ppl_cardinality_agg_low_clients or search_clients | default(1) }} +}, +{ + "operation": "ppl-range-agg-1", + "warmup-iterations": {{ ppl_range_agg_1_warmup_iterations or warmup_iterations | default(200) | tojson }}, + "iterations": {{ ppl_range_agg_1_iterations or test_iterations | default(100) | tojson }}, + "target-throughput": {{ ppl_range_agg_1_target_throughput or target_throughput | default(2) | tojson }}, + "clients": {{ ppl_range_agg_1_clients or search_clients | default(1) }} +}, +{ + "operation": "ppl-range-agg-2", + "warmup-iterations": {{ ppl_range_agg_2_warmup_iterations or warmup_iterations | default(200) | tojson }}, + "iterations": {{ ppl_range_agg_2_iterations or test_iterations | default(100) | tojson }}, + "target-throughput": {{ ppl_range_agg_2_target_throughput or target_throughput | default(2) | tojson }}, + "clients": {{ ppl_range_agg_2_clients or search_clients | default(1) }} +}, { "operation": "ppl-range-field-conjunction-big-range-big-term-query", "warmup-iterations": {{ ppl_range_field_conjunction_big_range_big_term_query_warmup_iterations or warmup_iterations | default(200) | tojson }},