[add] Enabled the property on test suite spec (to avoid exploding total variants per build) (#67)

Author: filipecosta90

redis_benchmarks_specification/__common__/spec.py

@@ -7,6 +7,13 @@ def extract_client_cpu_limit(benchmark_config):
     return ceil_db_cpu_limit
 
 
+def extract_build_variant_variations(benchmark_config, keyname="build-variants"):
+    result = None
+    if keyname in benchmark_config:
+        result = benchmark_config[keyname]
+    return result
+
+
 def extract_client_container_image(benchmark_config, keyname="clientconfig"):
     client_container_image = None
     if keyname in benchmark_config:

redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -44,6 +44,7 @@
     extract_client_cpu_limit,
     extract_client_container_image,
     extract_client_tool,
+    extract_build_variant_variations,
 )
 from redis_benchmarks_specification.__self_contained_coordinator__.args import (
     create_self_contained_coordinator_args,
@@ -165,7 +166,7 @@ def main():
     if stream_id is None:
         stream_id = args.consumer_start_id
     while True:
-        _, stream_id, _ = self_contained_coordinator_blocking_read(
+        _, stream_id, _, _ = self_contained_coordinator_blocking_read(
             conn,
             datasink_push_results_redistimeseries,
             docker_client,
@@ -218,6 +219,7 @@ def self_contained_coordinator_blocking_read(
     platform_name,
 ):
     num_process_streams = 0
+    num_process_test_suites = 0
     overall_result = False
     consumer_name = "{}-self-contained-proc#{}".format(
         get_runners_consumer_group_name(platform_name), "1"
@@ -232,7 +234,11 @@ def self_contained_coordinator_blocking_read(
     if len(newTestInfo[0]) < 2 or len(newTestInfo[0][1]) < 1:
         stream_id = ">"
     else:
-        stream_id, overall_result = process_self_contained_coordinator_stream(
+        (
+            stream_id,
+            overall_result,
+            total_test_suite_runs,
+        ) = process_self_contained_coordinator_stream(
             conn,
             datasink_push_results_redistimeseries,
             docker_client,
@@ -244,6 +250,7 @@ def self_contained_coordinator_blocking_read(
             platform_name,
         )
         num_process_streams = num_process_streams + 1
+        num_process_test_suites = num_process_test_suites + total_test_suite_runs
         if overall_result is True:
             ack_reply = conn.xack(
                 STREAM_KEYNAME_NEW_BUILD_EVENTS,
@@ -264,7 +271,7 @@ def self_contained_coordinator_blocking_read(
                         stream_id, ack_reply
                     )
                 )
-    return overall_result, stream_id, num_process_streams
+    return overall_result, stream_id, num_process_streams, num_process_test_suites
 
 
 def prepare_memtier_benchmark_parameters(
@@ -308,6 +315,7 @@ def process_self_contained_coordinator_stream(
     stream_id = stream_id.decode()
     logging.info("Received work . Stream id {}.".format(stream_id))
     overall_result = False
+    total_test_suite_runs = 0
 
     if b"git_hash" in testDetails:
         (
@@ -338,6 +346,22 @@ def process_self_contained_coordinator_stream(
                     redis_configuration_parameters,
                     _,
                 ) = extract_redis_dbconfig_parameters(benchmark_config, "dbconfig")
+                build_variants = extract_build_variant_variations(benchmark_config)
+                if build_variants is not None:
+                    logging.info("Detected build variant filter")
+                    if build_variant_name not in build_variants:
+                        logging.error(
+                            "Skipping {} given it's not part of build-variants for this test-suite {}".format(
+                                build_variant_name, build_variants
+                            )
+                        )
+                        continue
+                    else:
+                        logging.error(
+                            "Running build variant {} given it's present on the build-variants spec {}".format(
+                                build_variant_name, build_variants
+                            )
+                        )
                 for topology_spec_name in benchmark_config["redis-topologies"]:
                     test_result = False
                     try:
@@ -557,6 +581,7 @@ def process_self_contained_coordinator_stream(
                             running_platform,
                         )
                         test_result = True
+                        total_test_suite_runs = total_test_suite_runs + 1
 
                     except:
                         logging.critical(
@@ -598,7 +623,7 @@ def process_self_contained_coordinator_stream(
 
     else:
         logging.error("Missing commit information within received message.")
-    return stream_id, overall_result
+    return stream_id, overall_result, total_test_suite_runs
 
 
 def data_prepopulation_step(

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-100B-expire-use-case.yml

@@ -8,18 +8,22 @@ dbconfig:
   preload_tool:
     run_image: redislabs/memtier_benchmark:1.3.0
     tool: memtier_benchmark
-    arguments: '"--command" "SETEX __key__ 360 __value__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram"'
+    arguments: '"--data-size" "100" "--command" "SET __key__ __value__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram"'
 tested-commands:
   - SET
   - SETX
   - GET
   - DEL
 redis-topologies:
   - oss-standalone
+
+build-variants:
+  - gcc:8.5.0-amd64-debian-buster-default
+
 clientconfig:
   run_image: redislabs/memtier_benchmark:1.3.0
   tool: memtier_benchmark
-  arguments: '--command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R"  -c 50 -t 2 --hide-histogram --test-time 300'
+  arguments: '"--data-size" "100" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R"  -c 50 -t 2 --hide-histogram --test-time 300'
   resources:
     requests:
       cpus: "3"

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-10B-expire-use-case.yml

@@ -0,0 +1,39 @@
+version: 0.4
+name: "memtier_benchmark-1Mkeys-10B-expire-use-case"
+description: "Runs memtier_benchmark, for a keyspace length of 1M keys
+              with a data size of 10 Bytes for each key, which 50% of the keys have expiration set during the benchmark."
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:1.3.0
+    tool: memtier_benchmark
+    arguments: '"--data-size" "10" "--command" "SET __key__ __value__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram"'
+tested-commands:
+  - SET
+  - SETX
+  - GET
+  - DEL
+redis-topologies:
+  - oss-standalone
+
+build-variants:
+  - gcc:8.5.0-amd64-debian-buster-default
+
+clientconfig:
+  run_image: redislabs/memtier_benchmark:1.3.0
+  tool: memtier_benchmark
+  arguments: '"--data-size" "10" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R"  -c 50 -t 2 --hide-histogram --test-time 300'
+  resources:
+    requests:
+      cpus: "3"
+      memory: "2g"
+exporter:
+  redistimeseries:
+    break_by:
+      - version
+      - commit
+    timemetric: '$."ALL STATS".Runtime."Start time"'
+    metrics:
+      - '$."ALL STATS".*."Ops/sec"'
+      - '$."ALL STATS".*."Latency"'

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-1KiB-expire-use-case.yml

@@ -0,0 +1,39 @@
+version: 0.4
+name: "memtier_benchmark-1Mkeys-1KiB-expire-use-case"
+description: "Runs memtier_benchmark, for a keyspace length of 1M keys
+              with a data size of 1KiB Bytes for each key, which 50% of the keys have expiration set during the benchmark."
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:1.3.0
+    tool: memtier_benchmark
+    arguments: '"--data-size" "1000" "--command" "SET __key__ __value__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram"'
+tested-commands:
+  - SET
+  - SETX
+  - GET
+  - DEL
+redis-topologies:
+  - oss-standalone
+
+build-variants:
+  - gcc:8.5.0-amd64-debian-buster-default
+
+clientconfig:
+  run_image: redislabs/memtier_benchmark:1.3.0
+  tool: memtier_benchmark
+  arguments: '"--data-size" "1000" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R"  -c 50 -t 2 --hide-histogram --test-time 300'
+  resources:
+    requests:
+      cpus: "3"
+      memory: "2g"
+exporter:
+  redistimeseries:
+    break_by:
+      - version
+      - commit
+    timemetric: '$."ALL STATS".Runtime."Start time"'
+    metrics:
+      - '$."ALL STATS".*."Ops/sec"'
+      - '$."ALL STATS".*."Latency"'

redis_benchmarks_specification/test-suites/redis-benchmark-full-suite-1Mkeys-100B.yml

@@ -23,6 +23,10 @@ tested-commands:
   - MSET
 redis-topologies:
   - oss-standalone
+
+build-variants:
+  - gcc:8.5.0-amd64-debian-buster-default
+
 clientconfig:
   run_image: redis:6.2.4
   tool: redis-benchmark

utils/tests/test_data/api_builder_common.py

@@ -12,7 +12,9 @@
 )
 
 
-def flow_1_and_2_api_builder_checks(conn):
+def flow_1_and_2_api_builder_checks(
+    conn, build_spec_name="gcc:8.5.0-amd64-debian-buster-default"
+):
     builder_consumer_group_create(conn)
     assert conn.xlen(STREAM_KEYNAME_GH_EVENTS_COMMIT) == 0
     result, reply_fields, error_msg = commit_schema_to_stream(
@@ -26,7 +28,7 @@ def flow_1_and_2_api_builder_checks(conn):
         "redis",
         GH_TOKEN,
     )
-    build_spec_name = "gcc:8.5.0-amd64-debian-buster-default"
+
     assert result == True
     assert error_msg == None
     assert STREAM_KEYNAME_GH_EVENTS_COMMIT.encode() in conn.keys()

utils/tests/test_data/dump.rdb

@@ -0,0 +1,37 @@
+version: 0.4
+name: "memtier_benchmark-1Mkeys-100B-expire-use-case"
+description: "Runs memtier_benchmark, for a keyspace length of 1M keys
+              with a data size of 100 Bytes for each key, which 50% of the keys have expiration set during the benchmark."
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:1.3.0
+    tool: memtier_benchmark
+    arguments: '"--command" "SETEX __key__ 360 __value__" "--command-key-pattern" "P" "-c" "5" "-t" "2" "--hide-histogram"'
+tested-commands:
+  - SET
+  - SETX
+  - GET
+  - DEL
+redis-topologies:
+  - oss-standalone
+build-variants:
+  - abc
+clientconfig:
+  run_image: redislabs/memtier_benchmark:1.3.0
+  tool: memtier_benchmark
+  arguments: '--command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R"  -c 5 -t 1 --hide-histogram --test-time 10'
+  resources:
+    requests:
+      cpus: "1"
+      memory: "2g"
+exporter:
+  redistimeseries:
+    break_by:
+      - version
+      - commit
+    timemetric: '$."ALL STATS".Runtime."Start time"'
+    metrics:
+      - '$."ALL STATS".*."Ops/sec"'
+      - '$."ALL STATS".*."Latency"'

utils/tests/test_data/test-suites/memtier_benchmark-1Mkeys-100B-expire-use-case-with-variant.yml

@@ -16,6 +16,8 @@ tested-commands:
   - DEL
 redis-topologies:
   - oss-standalone
+build-variants:
+  - gcc:8.5.0-amd64-debian-buster-default
 clientconfig:
   run_image: redislabs/memtier_benchmark:1.3.0
   tool: memtier_benchmark