Enforcing keyspace length check and checking for datasets within scc (#136)

* Enforcing keyspace length check and checking for datasets within scc

* Fixed flake8 related issues after additions

* Kickoff GEO commands benchmark tracking

* Added geodist geohash and geopos benchmarks

* reusing get_branch_version_from_test_details across builder and scc

* geodist geohash geopos geosearch pipeline 10 variants

Author: filipecosta90

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.59"
+version = "0.1.60"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "Readme.md"

redis_benchmarks_specification/__builder__/builder.py

@@ -13,6 +13,9 @@
     get_build_config,
     get_build_config_metadata,
 )
+from redis_benchmarks_specification.__common__.builder_schema import (
+    get_branch_version_from_test_details,
+)
 from redis_benchmarks_specification.__common__.env import (
     STREAM_KEYNAME_GH_EVENTS_COMMIT,
     GH_REDIS_SERVER_HOST,
@@ -201,16 +204,9 @@ def builder_process_stream(
                 )
             )
             buffer = conn.get(binary_zip_key)
-            git_branch = None
-            git_version = None
-            if b"git_branch" in testDetails:
-                git_branch = testDetails[b"git_branch"]
-            if b"ref_label" in testDetails:
-                git_branch = testDetails[b"ref_label"]
-            if b"git_version" in testDetails:
-                git_version = testDetails[b"git_version"]
             git_timestamp_ms = None
             use_git_timestamp = False
+            git_branch, git_version = get_branch_version_from_test_details(testDetails)
             if b"use_git_timestamp" in testDetails:
                 use_git_timestamp = bool(testDetails[b"use_git_timestamp"])
             if b"git_timestamp_ms" in testDetails:

redis_benchmarks_specification/__common__/builder_schema.py

@@ -146,3 +146,25 @@ def request_build_from_commit_info(
     id = conn.xadd(STREAM_KEYNAME_GH_EVENTS_COMMIT.encode(), fields)
     reply_fields["id"] = id
     return result, reply_fields, error_msg
+
+
+def get_branch_version_from_test_details(testDetails):
+    git_branch = None
+    git_version = None
+    if b"git_branch" in testDetails:
+        git_branch = testDetails[b"git_branch"]
+    if b"ref_label" in testDetails:
+        git_branch = testDetails[b"ref_label"]
+    if b"git_version" in testDetails:
+        git_version = testDetails[b"git_version"]
+    if git_branch is not None:
+        # remove event prefix
+        if type(git_branch) == bytes:
+            git_branch = git_branch.decode()
+        if git_branch.startswith("/refs/heads/"):
+            git_branch = git_branch.replace("/refs/heads/", "")
+    if git_version is not None:
+        if type(git_version) == bytes:
+            git_version = git_version.decode()
+
+    return git_branch, git_version

redis_benchmarks_specification/__self_contained_coordinator__/build_info.py

@@ -1,12 +1,14 @@
 import json
 import logging
 
+from redis_benchmarks_specification.__common__.builder_schema import (
+    get_branch_version_from_test_details,
+)
+
 
 def extract_build_info_from_streamdata(testDetails):
     use_git_timestamp = False
     git_timestamp_ms = None
-    git_version = None
-    git_branch = None
     metadata = None
     build_variant_name = None
     fields = [fieldname.decode() for fieldname in testDetails.keys()]
@@ -20,14 +22,7 @@ def extract_build_info_from_streamdata(testDetails):
         build_variant_name = testDetails[b"id"]
         if type(build_variant_name) == bytes:
             build_variant_name = build_variant_name.decode()
-    if b"git_branch" in testDetails:
-        git_branch = testDetails[b"git_branch"]
-        if type(git_branch) == bytes:
-            git_branch = git_branch.decode()
-    if b"git_version" in testDetails:
-        git_version = testDetails[b"git_version"]
-        if type(git_version) == bytes:
-            git_version = git_version.decode()
+    git_branch, git_version = get_branch_version_from_test_details(testDetails)
     if type(git_hash) == bytes:
         git_hash = git_hash.decode()
     logging.info("Received commit hash specifier {}.".format(git_hash))

redis_benchmarks_specification/__self_contained_coordinator__/runners.py

@@ -8,6 +8,13 @@
 
 import redis
 from redisbench_admin.environments.oss_cluster import generate_cluster_redis_server_args
+
+from redisbench_admin.utils.local import check_dataset_local_requirements
+
+from redisbench_admin.run.common import (
+    dbconfig_keyspacelen_check,
+)
+
 from redisbench_admin.profilers.profilers_local import (
     local_profilers_platform_checks,
     profilers_start_if_required,
@@ -211,6 +218,23 @@ def process_self_contained_coordinator_stream(
                             restore_build_artifacts_from_test_details(
                                 build_artifacts, conn, temporary_dir, testDetails
                             )
+
+                            logging.info("Checking if there is a dataset requirement")
+                            (
+                                dataset,
+                                dataset_name,
+                                _,
+                                _,
+                            ) = check_dataset_local_requirements(
+                                benchmark_config,
+                                temporary_dir,
+                                None,
+                                "./datasets",
+                                "dbconfig",
+                                1,
+                                False,
+                            )
+
                             dso = "redis-server"
                             profilers_artifacts_matrix = []
 
@@ -285,6 +309,9 @@ def process_self_contained_coordinator_stream(
                             client_mnt_point = "/mnt/client/"
                             benchmark_tool_workdir = client_mnt_point
 
+                            logging.info(
+                                "Checking if there is a data preload_tool requirement"
+                            )
                             if "preload_tool" in benchmark_config["dbconfig"]:
                                 data_prepopulation_step(
                                     benchmark_config,
@@ -297,6 +324,14 @@ def process_self_contained_coordinator_stream(
                                     test_name,
                                 )
 
+                            logging.info(
+                                "Checking if there is a keyspace check being enforced"
+                            )
+                            dbconfig_keyspacelen_check(
+                                benchmark_config,
+                                [r],
+                            )
+
                             benchmark_tool = extract_client_tool(benchmark_config)
                             # backwards compatible
                             if benchmark_tool is None:

redis_benchmarks_specification/setups/builders/icc-2021.3.0-amd64-ubuntu18.04-libc.yml

@@ -0,0 +1,39 @@
+version: 0.4
+name: "memtier_benchmark-1key-geo-60M-elements-geodist-pipeline-10"
+description: "Runs memtier_benchmark, for a keyspace length of 1 GEO key.
+              The GEO key contains 60841557 elements in it and we query it using GEODIST command between 2 elements.
+              "
+dbconfig:
+  - configuration-parameters:
+    - save: '""'
+  - dataset: "https://s3.us-east-2.amazonaws.com/redis.benchmarks.spec/datasets/geopoint/dump.rdb"
+
+tested-groups:
+  - geo
+tested-commands:
+  - GEODIST
+redis-topologies:
+  - oss-standalone
+
+build-variants:
+  - gcc:8.5.0-amd64-debian-buster-default
+
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '--pipeline 10 -c 2 -t 2 --command="GEODIST key 1 2"  --hide-histogram --test-time 180'
+  resources:
+    requests:
+      cpus: "4"
+      memory: "2g"
+exporter:
+  redistimeseries:
+    break_by:
+      - version
+      - commit
+    timemetric: '$."ALL STATS".Runtime."Start time"'
+    metrics:
+      - '$."ALL STATS".Totals."Ops/sec"'
+      - '$."ALL STATS".Totals."Latency"'
+      - '$."ALL STATS".Totals."Misses/sec"'
+      - '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-60M-elements-geodist-pipeline-10.yml

@@ -0,0 +1,39 @@
+version: 0.4
+name: "memtier_benchmark-1key-geo-60M-elements-geodist"
+description: "Runs memtier_benchmark, for a keyspace length of 1 GEO key.
+              The GEO key contains 60841557 elements in it and we query it using GEODIST command between 2 elements.
+              "
+dbconfig:
+  - configuration-parameters:
+    - save: '""'
+  - dataset: "https://s3.us-east-2.amazonaws.com/redis.benchmarks.spec/datasets/geopoint/dump.rdb"
+
+tested-groups:
+  - geo
+tested-commands:
+  - GEODIST
+redis-topologies:
+  - oss-standalone
+
+build-variants:
+  - gcc:8.5.0-amd64-debian-buster-default
+
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '-c 2 -t 2 --command="GEODIST key 1 2"  --hide-histogram --test-time 180'
+  resources:
+    requests:
+      cpus: "4"
+      memory: "2g"
+exporter:
+  redistimeseries:
+    break_by:
+      - version
+      - commit
+    timemetric: '$."ALL STATS".Runtime."Start time"'
+    metrics:
+      - '$."ALL STATS".Totals."Ops/sec"'
+      - '$."ALL STATS".Totals."Latency"'
+      - '$."ALL STATS".Totals."Misses/sec"'
+      - '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-60M-elements-geodist.yml

@@ -0,0 +1,39 @@
+version: 0.4
+name: "memtier_benchmark-1key-geo-60M-elements-geohash-pipeline-10"
+description: "Runs memtier_benchmark, for a keyspace length of 1 GEO key.
+              The GEO key contains 60841557 elements in it and we query it using GEOHASH command.
+              "
+dbconfig:
+  - configuration-parameters:
+    - save: '""'
+  - dataset: "https://s3.us-east-2.amazonaws.com/redis.benchmarks.spec/datasets/geopoint/dump.rdb"
+
+tested-groups:
+  - geo
+tested-commands:
+  - GEOHASH
+redis-topologies:
+  - oss-standalone
+
+build-variants:
+  - gcc:8.5.0-amd64-debian-buster-default
+
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '--pipeline 10 -c 2 -t 2 --command="GEOHASH key 1"  --hide-histogram --test-time 180'
+  resources:
+    requests:
+      cpus: "4"
+      memory: "2g"
+exporter:
+  redistimeseries:
+    break_by:
+      - version
+      - commit
+    timemetric: '$."ALL STATS".Runtime."Start time"'
+    metrics:
+      - '$."ALL STATS".Totals."Ops/sec"'
+      - '$."ALL STATS".Totals."Latency"'
+      - '$."ALL STATS".Totals."Misses/sec"'
+      - '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-60M-elements-geohash-pipeline-10.yml

@@ -0,0 +1,39 @@
+version: 0.4
+name: "memtier_benchmark-1key-geo-60M-elements-geohash"
+description: "Runs memtier_benchmark, for a keyspace length of 1 GEO key.
+              The GEO key contains 60841557 elements in it and we query it using GEOHASH command.
+              "
+dbconfig:
+  - configuration-parameters:
+    - save: '""'
+  - dataset: "https://s3.us-east-2.amazonaws.com/redis.benchmarks.spec/datasets/geopoint/dump.rdb"
+
+tested-groups:
+  - geo
+tested-commands:
+  - GEOHASH
+redis-topologies:
+  - oss-standalone
+
+build-variants:
+  - gcc:8.5.0-amd64-debian-buster-default
+
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '-c 2 -t 2 --command="GEOHASH key 1"  --hide-histogram --test-time 180'
+  resources:
+    requests:
+      cpus: "4"
+      memory: "2g"
+exporter:
+  redistimeseries:
+    break_by:
+      - version
+      - commit
+    timemetric: '$."ALL STATS".Runtime."Start time"'
+    metrics:
+      - '$."ALL STATS".Totals."Ops/sec"'
+      - '$."ALL STATS".Totals."Latency"'
+      - '$."ALL STATS".Totals."Misses/sec"'
+      - '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'