Introduce a new operation to benchmark the shard changes recovery api

elastic/logs/README.md

@@ -223,6 +223,11 @@ The following parameters are available:
 * `synthetic_source_keep` (default: unset): If specified, configures the `index.mapping.synthetic_source_keep` index setting.
 * `source_mode` (default: unset) - Specifies the source mode to be used.
 * `use_synthetic_source_recovery` (default: unset): Whether synthetic recovery source will be used.
+* `recovery_target` (required) - The target index or data stream for fetching shard changes via the recovery API.
+* `recovery_from_seq_no` (default: `0`) - The sequence number from which to start fetching translog operations.
+* `recovery_poll_timeout` (default: `1m`) - The maximum time to wait for additional translog operations before returning an empty result.
+* `recovery_max_batch_size` (default: `32MB`) - The maximum estimated size for the batch of translog operations to return.
+* `recovery_max_operations_count` (default: `1048576`) - The maximum number of translog operations to return in a single batch.
 
 ### Data Download Parameters
 

elastic/logs/challenges/synthetic-source-recovery.json

@@ -0,0 +1,54 @@
+{% import "rally.helpers" as rally %}
+{
+  "name": "synthetic-source-recovery",
+  "description": "Indexes logs for a specified time period and volume per day and then runs the recovery api fetching translog operations for a given target index.",
+  "default": false,
+  "schedule": [
+    {% include "tasks/index-setup.json" %},
+    {# serverless-shard-recovery-marker-start #}{%- if build_flavor != "serverless" -%}
+    {
+      "name": "add-retention-lease",
+      "operation": {
+        "operation-type": "raw-request",
+        "path": {{ "/%s/seq_no/add_retention_lease" | format(p_recovery_target) | tojson }},
+        "method": "PUT"
+      },
+      "warmup-iterations": 0,
+      "iterations": 1
+    },
+    {%- endif -%}{# serverless-shard-recovery-marker-end #}
+    {
+      "name": "bulk-index",
+      "operation": {
+        "operation-type": "raw-bulk",
+        "param-source": "processed-source",
+        "time-format": "milliseconds",
+        "profile": "fixed_interval",
+        "init-load": true,
+        "bulk-size": {{ p_bulk_size }},
+        "detailed-results": true
+      },
+      "clients": {{ p_bulk_indexing_clients }},
+      "ignore-response-error-level": "{{error_level | default('non-fatal')}}"
+    }
+    {# serverless-shard-recovery-marker-start #}{%- if build_flavor != "serverless" -%}
+    ,
+    {
+      "name": "shard-recovery",
+      "operation": {
+        "operation-type": "raw-request",
+        "path": {{ "/%s/ccr/shard_changes" | format(p_recovery_target) | tojson }},
+        "method": "GET",
+        "request-params": {
+          "from_seq_no": {{ p_recovery_from_seq_no | tojson }},
+          "max_batch_size": {{ p_recovery_max_batch_size | tojson }},
+          "poll_timeout": {{ p_recovery_poll_timeout | tojson }},
+          "max_operations_count": {{ p_recovery_max_operations_count | tojson }}
+        }
+      },
+      "warmup-iterations": 10,
+      "iterations": 20
+    }
+    {%- endif -%}{# serverless-shard-recovery-marker-end #}
+  ]
+}

elastic/logs/track.json

@@ -60,6 +60,12 @@
 {% set p_dsl_default_rollover = (dsl_default_rollover | default(false) ) %}
 {% set p_skip_fleet_globals = (skip_fleet_globals | default(false) ) %}
 
+{% set p_recovery_target = (recovery_target | default('logs-k8-application.log-default') ) %}
+{% set p_recovery_from_seq_no = (recovery_seq_no | default(0) ) %}
+{% set p_recovery_max_batch_size = (recovery_max_batch_size | default ("32MB") ) %}
+{% set p_recovery_poll_timeout = (recovery_poll_timeout | default("1m")) %}
+{% set p_recovery_max_operations_count = (recovery_max_operations_count | default(1048576)) %}
+
 {% set p_worker_threads_enabled = (worker_threads_enabled | default(true))  %}
 
 {% set es_version = "7.13.2" %}

elastic/security/README.md

@@ -87,6 +87,11 @@ The following parameters are available:
 * `synthetic_source_keep` (default: unset): If specified, configures the `index.mapping.synthetic_source_keep` index setting.
 * `source_mode` (default: unset) - Specifies the source mode to be used.
 * `use_synthetic_source_recovery` (default: unset): Whether synthetic recovery source will be used.
+* `recovery_target` (required) - The target index or data stream for fetching shard changes via the recovery API.
+* `recovery_from_seq_no` (default: `0`) - The sequence number from which to start fetching translog operations.
+* `recovery_poll_timeout` (default: `1m`) - The maximum time to wait for additional translog operations before returning an empty result.
+* `recovery_max_batch_size` (default: `32MB`) - The maximum estimated size for the batch of translog operations to return.
+* `recovery_max_operations_count` (default: `1048576`) - The maximum number of translog operations to return in a single batch.
 
 ### Data Generation Parameters
 

elastic/security/challenges/synthetic-source-recovery.json

@@ -0,0 +1,53 @@
+{% import "rally.helpers" as rally %}
+{
+  "name": "synthetic-source-recovery",
+  "description": "Indexes security data for a specified time period and volume per day and then runs the recovery api fetching translog operations for a given target index.",
+  "default": false,
+  "schedule": [
+    {% include "tasks/index-setup.json" %},
+    {# serverless-shard-recovery-marker-start #}{%- if build_flavor != "serverless" -%}
+    {
+      "name": "add-retention-lease",
+      "operation": {
+        "operation-type": "raw-request",
+        "path": {{ "/%s/seq_no/add_retention_lease" | format(p_recovery_target) | tojson }},
+        "method": "PUT"
+      },
+      "warmup-iterations": 0,
+      "iterations": 1
+    },
+    {%- endif -%}{# serverless-shard-recovery-marker-end #}
+    {
+      "name": "bulk-index",
+      "operation": {
+        "operation-type": "raw-bulk",
+        "param-source": "processed-source",
+        "time-format": "milliseconds",
+        "profile": "fixed_interval",
+        "detailed-results": true,
+        "bulk-size": {{ (bulk_size | default(50)) }}
+      },
+      "ingest-percentage": {{ ingest_percentage | default(100) }},
+      "clients": {{ p_bulk_indexing_clients }}
+    }
+    {# serverless-shard-recovery-marker-start #}{%- if build_flavor != "serverless" -%}
+    ,
+    {
+      "name": "shard-recovery",
+      "operation": {
+        "operation-type": "raw-request",
+        "path": {{ "/%s/ccr/shard_changes" | format(p_recovery_target) | tojson }},
+        "method": "GET",
+        "request-params": {
+          "from_seq_no": {{ p_recovery_from_seq_no | tojson }},
+          "max_batch_size": {{ p_recovery_max_batch_size | tojson }},
+          "poll_timeout": {{ p_recovery_poll_timeout | tojson }},
+          "max_operations_count": {{ p_recovery_max_operations_count | tojson }}
+        }
+      },
+      "warmup-iterations": 10,
+      "iterations": 20
+    }
+    {%- endif -%}{# serverless-shard-recovery-marker-end #}
+  ]
+}

elastic/security/track.json

@@ -10,6 +10,11 @@
 {% set p_number_of_replicas = (number_of_replicas | default(1)) %}
 {% set p_skip_delete_component_template = (skip_delete_component_template | default(false) ) %}
 {% set p_logs_endpoint_from_kibana = (logs_endpoint_from_kibana | default(false) ) %}
+{% set p_recovery_from_seq_no = (recovery_seq_no | default(0) ) %}
+{% set p_recovery_target = ( recovery_target | default('filebeat-default') ) %}
+{% set p_recovery_max_batch_size = (recovery_max_batch_size | default ("32MB") ) %}
+{% set p_recovery_poll_timeout = (recovery_poll_timeout | default("1m")) %}
+{% set p_recovery_max_operations_count = (recovery_max_operations_count | default(1048576)) %}
 {% set p_include_non_serverless_index_settings = (include_non_serverless_index_settings | default(build_flavor != "serverless")) %}
 {% set p_integration_ratios = (integration_ratios | default({
     "auditbeat": {

tsdb/README.md

@@ -205,6 +205,11 @@ This track allows to overwrite the following parameters using `--track-params`:
 * `document_ids`: documentd IDs to use for search, get and mget apis in the `low-latency` challenge. If empty, a default set of 4 values is used.
 * `synthetic_source_keep` (default: unset): If specified, configures the `index.mapping.synthetic_source_keep` index setting.
 * `use_synthetic_source_recovery` (default: unset): Whether synthetic recovery source will be used.
+* `recovery_target` (required) - The target index or data stream for fetching shard changes via the recovery API.
+* `recovery_from_seq_no` (default: `0`) - The sequence number from which to start fetching translog operations.
+* `recovery_poll_timeout` (default: `1m`) - The maximum time to wait for additional translog operations before returning an empty result.
+* `recovery_max_batch_size` (default: `32MB`) - The maximum estimated size for the batch of translog operations to return.
+* `recovery_max_operations_count` (default: `1048576`) - The maximum number of translog operations to return in a single batch.
 
 ### License
 

tsdb/challenges/synthetic-source-recovery.json

@@ -0,0 +1,65 @@
+{
+  "name": "synthetic-source-recovery",
+  "description": "Indexes the whole document corpus and then runs the recovery api fetching translog operations for a given target index.",
+  "default": false,
+  "schedule": [
+    {
+      "operation": "delete-index"
+    },
+    {
+      "operation": {
+        "operation-type": "create-index",
+        "settings": {{index_settings | default({}) | tojson}}
+      }
+    },
+    {
+      "name": "check-cluster-health",
+      "operation": {
+        "operation-type": "cluster-health",
+        "index": "tsdb",
+        "request-params": {
+          "wait_for_status": "{{cluster_health | default('green')}}",
+          "wait_for_no_relocating_shards": "true"
+        },
+        "retry-until-success": true
+      }
+    },
+    {# serverless-shard-recovery-marker-start #}{%- if build_flavor != "serverless" -%}
+    {
+      "name": "add-retention-lease",
+      "operation": {
+        "operation-type": "raw-request",
+        "path": {{ "/%s/seq_no/add_retention_lease" | format(p_recovery_target) | tojson }},
+        "method": "PUT"
+      },
+      "warmup-iterations": 0,
+      "iterations": 1
+    },
+    {%- endif -%}{# serverless-shard-recovery-marker-end #}
+    {
+      "operation": "index",
+      "warmup-time-period": 240,
+      "clients": {{bulk_indexing_clients | default(8)}},
+      "ignore-response-error-level": "{{error_level | default('non-fatal')}}"
+    }
+    {# serverless-shard-recovery-marker-start #}{%- if build_flavor != "serverless" -%}
+    ,
+    {
+      "name": "shard-recovery",
+      "operation": {
+        "operation-type": "raw-request",
+        "path": {{ "/%s/ccr/shard_changes" | format(p_recovery_target) | tojson }},
+        "method": "GET",
+        "request-params": {
+          "from_seq_no": {{ p_recovery_from_seq_no | tojson }},
+          "max_batch_size": {{ p_recovery_max_batch_size | tojson }},
+          "poll_timeout": {{ p_recovery_poll_timeout | tojson }},
+          "max_operations_count": {{ p_recovery_max_operations_count | tojson }}
+        }
+      },
+      "warmup-iterations": 10,
+      "iterations": 20
+    }
+    {%- endif -%}{# serverless-shard-recovery-marker-end #}
+  ]
+}

tsdb/track.json

@@ -1,30 +1,20 @@
 {% import "rally.helpers" as rally with context %}
 
+{% set p_recovery_target = ( recovery_target | default('tsdb') ) %}
+{% set p_recovery_from_seq_no = (recovery_seq_no | default(0) ) %}
+{% set p_recovery_max_batch_size = (recovery_max_batch_size | default ("32MB") ) %}
+{% set p_recovery_poll_timeout = (recovery_poll_timeout | default("1m")) %}
+{% set p_recovery_max_operations_count = (recovery_max_operations_count | default(1048576)) %}
+
 {
   "version": 2,
   "description": "metricbeat information for elastic-app k8s cluster",
-  {%- if ingest_mode is defined and ingest_mode == "data_stream" %}
-  "composable-templates": [
-    {
-      "name": "tsdb-template",
-      "index-pattern": "k8s*",
-      "delete-matching-indices": true,
-      "template": "index-template.json"
-    }
-  ],
-  "data-streams": [
-    {
-      "name": "k8s"
-    }
-  ],
-  {%- else %}
   "indices": [
     {
       "name": "tsdb",
       "body": "index.json"
     }
   ],
-  {%- endif %}
   "corpora": [
     {
       "name": "tsdb",