Add ES|QL specific tracks

esql/README.md

@@ -0,0 +1,24 @@
+## ES|QL rally tracks
+
+### Parameters
+
+This track allows to overwrite the following parameters using `--track-params`:
+
+* `bulk_size` (default: 10000)
+* `bulk_indexing_clients` (default: 8): Number of clients that issue bulk indexing requests.
+* `ingest_percentage` (default: 100): A number between 0 and 100 that defines how much of the document corpus should be ingested. It will be applied to the main index and to the large join indexes (ie. not to join indexes with up to 500K documents)
+* `max_concurrent_shards_per_node` (default 10): A number between 1 and 100 that defines how many concurrent threads will run per query in a node
+* `number_of_replicas` (default: 1): This only applies to the main index (not to lookup indexes)
+* `number_of_shards` (default: 5): This only applies to the main index (not to lookup indexes)
+* `source_mode` (default: stored): Should the `_source` be `stored` to disk exactly as sent (the default), thrown away (`disabled`), or reconstructed on the fly (`synthetic`)
+* `index_settings`: A list of index settings. Index settings defined elsewhere need to be overridden explicitly.
+* `cluster_health` (default: "green"): The minimum required cluster health.
+* `include_non_serverless_index_settings` (default: true for non-serverless clusters, false for serverless clusters): Whether to include non-serverless index settings.
+* `auto_expand_replicas` (default: "0-all"): Set the auto_expand_replicas behaviour for lookup indices.
+* `query_clients` (default 1): number of queries to be run in parallel
+* `n_indices` (default 1): number of queries to be run in parallel
+
+
+### License
+
+According to the [Open Data Law](https://opendata.cityofnewyork.us/open-data-law/) this data is available as public domain.

esql/challenges/default.json

@@ -0,0 +1,67 @@
+    {
+      "name": "esql_fanout",
+      "description": "Scale tests for ES|QL",
+      "default": true,
+      "schedule": [
+        {
+          "operation": "delete-index",
+          "tags": ["setup"]
+        },
+        {
+          "operation": {
+            "operation-type": "create-index",
+            "settings": {%- if index_settings is defined %} {{index_settings | tojson}} {%- else %} {
+              {# non-serverless-index-settings-marker-start #}{%- if build_flavor != "serverless" or serverless_operator == true -%}
+                {% if p_include_non_serverless_index_settings %}
+              "index.translog.flush_threshold_size": "4g",
+                {% endif %}
+              {%- endif -%}{# non-serverless-index-settings-marker-end #}
+              "index.codec": "best_compression",
+              "index.refresh_interval": "30s"
+            }{%- endif %}
+          },
+          "tags": ["setup"]
+        },
+
+        {
+          "name": "check-cluster-health",
+          "operation": {
+            "operation-type": "cluster-health",
+            "index": "idx_0",
+            "request-params": {
+              "wait_for_status": "{{cluster_health | default('green')}}",
+              "wait_for_no_relocating_shards": "true"
+            },
+            "retry-until-success": true
+          },
+          "tags": ["setup"]
+        },
+
+        {
+          "operation": "index-base",
+          "clients": {{bulk_indexing_clients | default(8)}},
+          "tags": ["setup"]
+        },
+        {
+          "operation": "from_idx_limit_1000",
+          "tags": ["esql_fanout"],
+          "clients":  {{query_clients | default(1)}},
+          "warmup-iterations": 10,
+          "iterations": 100
+        },
+        {
+          "operation": "from_idx_limit_10000",
+          "tags": ["esql_fanout"],
+          "clients":  {{query_clients | default(1)}},
+          "warmup-iterations": 10,
+          "iterations": 100
+        },
+        {
+          "operation": "from_idx_limit_1",
+          "tags": ["esql_fanout"],
+          "clients":  {{query_clients | default(1)}},
+          "warmup-iterations": 10,
+          "iterations": 100
+        }
+      ]
+    }

esql/idx_mapping0.json

@@ -0,0 +1,54 @@
+{% set p_include_non_serverless_index_settings = (include_non_serverless_index_settings | default(build_flavor != "serverless")) %}
+{% set n_fields = (n_fields | default(500)) %}
+{% set field_prefix = "field_0" %}
+
+{
+  "settings": {
+    {# non-serverless-index-settings-marker-start #}{%- if build_flavor != "serverless" or serverless_operator == true -%}
+      {% if p_include_non_serverless_index_settings %}
+        "index.number_of_shards": {{number_of_shards | default(5)}},
+        "index.number_of_replicas": {{number_of_replicas | default(0)}},
+      {% endif %}
+        "index.requests.cache.enable": false
+    {%- endif -%}{# non-serverless-index-settings-marker-end #}
+  },
+  "mappings": {
+    "_source": {
+      "mode": {{ source_mode | default("stored") | tojson }}
+    },
+    "properties": {
+      "id": {
+        "type": "long"
+      },
+      "@timestamp": {
+        "type": "date"
+      },
+{% for i in range(0, n_fields) %}
+    "{{field_prefix}}_{{i}}": {
+      "type": "keyword"
+    },
+{% endfor %}
+      "key_1000": {
+        "type": "keyword"
+      },
+      "key_100000": {
+        "type": "keyword"
+      },
+      "key_200000": {
+        "type": "keyword"
+      },
+      "key_500000": {
+        "type": "keyword"
+      },
+      "key_1000000": {
+        "type": "keyword"
+      },
+      "key_5000000": {
+        "type": "keyword"
+      },
+      "key_100000000": {
+        "type": "keyword"
+      }
+    }
+  }
+}

esql/idx_mapping1.json

@@ -0,0 +1,54 @@
+{% set p_include_non_serverless_index_settings = (include_non_serverless_index_settings | default(build_flavor != "serverless")) %}
+{% set n_fields = (n_fields | default(500)) %}
+{% set field_prefix = "field_1" %}
+
+{
+  "settings": {
+    {# non-serverless-index-settings-marker-start #}{%- if build_flavor != "serverless" or serverless_operator == true -%}
+      {% if p_include_non_serverless_index_settings %}
+        "index.number_of_shards": {{number_of_shards | default(5)}},
+        "index.number_of_replicas": {{number_of_replicas | default(0)}},
+      {% endif %}
+        "index.requests.cache.enable": false
+    {%- endif -%}{# non-serverless-index-settings-marker-end #}
+  },
+  "mappings": {
+    "_source": {
+      "mode": {{ source_mode | default("stored") | tojson }}
+    },
+    "properties": {
+      "id": {
+        "type": "long"
+      },
+      "@timestamp": {
+        "type": "date"
+      },
+{% for i in range(0, n_fields) %}
+    "{{field_prefix}}_{{i}}": {
+      "type": "keyword"
+    },
+{% endfor %}
+      "key_1000": {
+        "type": "keyword"
+      },
+      "key_100000": {
+        "type": "keyword"
+      },
+      "key_200000": {
+        "type": "keyword"
+      },
+      "key_500000": {
+        "type": "keyword"
+      },
+      "key_1000000": {
+        "type": "keyword"
+      },
+      "key_5000000": {
+        "type": "keyword"
+      },
+      "key_100000000": {
+        "type": "keyword"
+      }
+    }
+  }
+}

esql/idx_mapping10.json

@@ -0,0 +1,54 @@
+{% set p_include_non_serverless_index_settings = (include_non_serverless_index_settings | default(build_flavor != "serverless")) %}
+{% set n_fields = (n_fields | default(500)) %}
+{% set field_prefix = "field_10" %}
+
+{
+  "settings": {
+    {# non-serverless-index-settings-marker-start #}{%- if build_flavor != "serverless" or serverless_operator == true -%}
+      {% if p_include_non_serverless_index_settings %}
+        "index.number_of_shards": {{number_of_shards | default(5)}},
+        "index.number_of_replicas": {{number_of_replicas | default(0)}},
+      {% endif %}
+        "index.requests.cache.enable": false
+    {%- endif -%}{# non-serverless-index-settings-marker-end #}
+  },
+  "mappings": {
+    "_source": {
+      "mode": {{ source_mode | default("stored") | tojson }}
+    },
+    "properties": {
+      "id": {
+        "type": "long"
+      },
+      "@timestamp": {
+        "type": "date"
+      },
+{% for i in range(0, n_fields) %}
+    "{{field_prefix}}_{{i}}": {
+      "type": "keyword"
+    },
+{% endfor %}
+      "key_1000": {
+        "type": "keyword"
+      },
+      "key_100000": {
+        "type": "keyword"
+      },
+      "key_200000": {
+        "type": "keyword"
+      },
+      "key_500000": {
+        "type": "keyword"
+      },
+      "key_1000000": {
+        "type": "keyword"
+      },
+      "key_5000000": {
+        "type": "keyword"
+      },
+      "key_100000000": {
+        "type": "keyword"
+      }
+    }
+  }
+}

esql/idx_mapping11.json

@@ -0,0 +1,54 @@
+{% set p_include_non_serverless_index_settings = (include_non_serverless_index_settings | default(build_flavor != "serverless")) %}
+{% set n_fields = (n_fields | default(500)) %}
+{% set field_prefix = "field_11" %}
+
+{
+  "settings": {
+    {# non-serverless-index-settings-marker-start #}{%- if build_flavor != "serverless" or serverless_operator == true -%}
+      {% if p_include_non_serverless_index_settings %}
+        "index.number_of_shards": {{number_of_shards | default(5)}},
+        "index.number_of_replicas": {{number_of_replicas | default(0)}},
+      {% endif %}
+        "index.requests.cache.enable": false
+    {%- endif -%}{# non-serverless-index-settings-marker-end #}
+  },
+  "mappings": {
+    "_source": {
+      "mode": {{ source_mode | default("stored") | tojson }}
+    },
+    "properties": {
+      "id": {
+        "type": "long"
+      },
+      "@timestamp": {
+        "type": "date"
+      },
+{% for i in range(0, n_fields) %}
+    "{{field_prefix}}_{{i}}": {
+      "type": "keyword"
+    },
+{% endfor %}
+      "key_1000": {
+        "type": "keyword"
+      },
+      "key_100000": {
+        "type": "keyword"
+      },
+      "key_200000": {
+        "type": "keyword"
+      },
+      "key_500000": {
+        "type": "keyword"
+      },
+      "key_1000000": {
+        "type": "keyword"
+      },
+      "key_5000000": {
+        "type": "keyword"
+      },
+      "key_100000000": {
+        "type": "keyword"
+      }
+    }
+  }
+}

esql/idx_mapping12.json

@@ -0,0 +1,54 @@
+{% set p_include_non_serverless_index_settings = (include_non_serverless_index_settings | default(build_flavor != "serverless")) %}
+{% set n_fields = (n_fields | default(500)) %}
+{% set field_prefix = "field_12"%}
+
+{
+  "settings": {
+    {# non-serverless-index-settings-marker-start #}{%- if build_flavor != "serverless" or serverless_operator == true -%}
+      {% if p_include_non_serverless_index_settings %}
+        "index.number_of_shards": {{number_of_shards | default(5)}},
+        "index.number_of_replicas": {{number_of_replicas | default(0)}},
+      {% endif %}
+        "index.requests.cache.enable": false
+    {%- endif -%}{# non-serverless-index-settings-marker-end #}
+  },
+  "mappings": {
+    "_source": {
+      "mode": {{ source_mode | default("stored") | tojson }}
+    },
+    "properties": {
+      "id": {
+        "type": "long"
+      },
+      "@timestamp": {
+        "type": "date"
+      },
+{% for i in range(0, n_fields) %}
+    "{{field_prefix}}_{{i}}": {
+      "type": "keyword"
+    },
+{% endfor %}
+      "key_1000": {
+        "type": "keyword"
+      },
+      "key_100000": {
+        "type": "keyword"
+      },
+      "key_200000": {
+        "type": "keyword"
+      },
+      "key_500000": {
+        "type": "keyword"
+      },
+      "key_1000000": {
+        "type": "keyword"
+      },
+      "key_5000000": {
+        "type": "keyword"
+      },
+      "key_100000000": {
+        "type": "keyword"
+      }
+    }
+  }
+}