diff --git a/redis_benchmarks_specification/__runner__/runner.py b/redis_benchmarks_specification/__runner__/runner.py index 32369d5a..5a549de4 100644 --- a/redis_benchmarks_specification/__runner__/runner.py +++ b/redis_benchmarks_specification/__runner__/runner.py @@ -65,6 +65,10 @@ def main(): ) args = parser.parse_args() + run_client_runner_logic(args, project_name, project_name_suffix, project_version) + + +def run_client_runner_logic(args, project_name, project_name_suffix, project_version): if args.logname is not None: print("Writting log to {}".format(args.logname)) logging.basicConfig( @@ -83,7 +87,6 @@ def main(): ) logging.info(get_version_string(project_name, project_version)) testsuite_spec_files = extract_testsuites(args) - datasink_conn = None if args.datasink_push_results_redistimeseries: logging.info( @@ -113,7 +116,6 @@ def main(): ) logging.error("Error message {}".format(e.__str__())) exit(1) - running_platform = args.platform_name tls_enabled = args.tls tls_skip_verify = args.tls_skip_verify @@ -124,7 +126,6 @@ def main(): preserve_temporary_client_dirs = args.preserve_temporary_client_dirs docker_client = docker.from_env() home = str(Path.home()) - profilers_list = [] profilers_enabled = args.enable_profilers if profilers_enabled: @@ -137,9 +138,7 @@ def main(): ) ) exit(1) - logging.info("Running the benchmark specs.") - process_self_contained_coordinator_stream( args, args.datasink_push_results_redistimeseries, diff --git a/tox.ini b/tox.ini index f5d5d699..bf51406d 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,7 @@ commands = docker = datasink + db_server [docker:datasink] image = redis/redis-stack-server:7.0.2-RC4 @@ -27,3 +28,9 @@ ports = 16379:6379/tcp volumes = bind:rw:{toxinidir}/utils/tests/test_data/:/data + + +[docker:db_server] +image = redis/redis-stack-server:7.0.2-RC4 +ports = + 6380:6379/tcp diff --git a/utils/tests/test_data/dump.rdb b/utils/tests/test_data/dump.rdb index d9369b1f..0dbbec2d 100644 Binary files a/utils/tests/test_data/dump.rdb and b/utils/tests/test_data/dump.rdb differ diff --git a/utils/tests/test_data/test-suites/memtier_benchmark-10keys-100B-expire-use-case-with-variant.yml b/utils/tests/test_data/test-suites/memtier_benchmark-10keys-100B-expire-use-case-with-variant.yml new file mode 100644 index 00000000..fd29eaf7 --- /dev/null +++ b/utils/tests/test_data/test-suites/memtier_benchmark-10keys-100B-expire-use-case-with-variant.yml @@ -0,0 +1,38 @@ +version: 0.4 +name: "memtier_benchmark-10keys-100B-expire-use-case" +description: "Runs memtier_benchmark, for a keyspace length of 1M keys + with a data size of 100 Bytes for each key, which 50% of the keys have expiration set during the benchmark." +dbconfig: + configuration-parameters: + save: '""' + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--command" "SETEX __key__ 360 __value__" "--command-key-pattern" "P" "-c" "5" "-t" "2" "--hide-histogram" "--key-minimum" 1 "--key-maximum" "10"' +tested-commands: + - SET + - SETX + - GET +redis-topologies: + - oss-standalone +build-variants: + - abc +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--key-minimum" 1 "--key-maximum" "10" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" -c 5 -t 1 --hide-histogram --test-time 10' + resources: + requests: + cpus: "1" + memory: "2g" +exporter: + redistimeseries: + break_by: + - version + - commit + timemetric: '$."ALL STATS".Runtime."Start time"' + metrics: + - '$."ALL STATS".Totals."Ops/sec"' + - '$."ALL STATS".Totals."Latency"' + - '$."ALL STATS".Totals."Misses/sec"' + - '$."ALL STATS".Totals."Percentile Latencies"."p50.00"' diff --git a/utils/tests/test_runner.py b/utils/tests/test_runner.py index f6425f42..6f8e46a7 100644 --- a/utils/tests/test_runner.py +++ b/utils/tests/test_runner.py @@ -1,8 +1,14 @@ +import argparse + +import redis import yaml +from redis_benchmarks_specification.__common__.package import get_version_string from redis_benchmarks_specification.__common__.spec import extract_client_tool +from redis_benchmarks_specification.__runner__.args import create_client_runner_args from redis_benchmarks_specification.__runner__.runner import ( prepare_memtier_benchmark_parameters, + run_client_runner_logic, ) @@ -140,3 +146,65 @@ def test_prepare_memtier_benchmark_parameters(): benchmark_command_str == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --tls --cert cert.file --key key.file --cacert cacert.file "--data-size" "100" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' ) + + +def test_run_client_runner_logic(): + project_name = "tool" + project_version = "v0" + parser = argparse.ArgumentParser( + description="test", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser = create_client_runner_args( + get_version_string(project_name, project_version) + ) + db_host = "localhost" + db_port = "6380" + datasink_port = "16379" + db_port_int = int(db_port) + datasink_port_int = int(db_port) + args = parser.parse_args( + args=[ + "--test", + "../../utils/tests/test_data/test-suites/memtier_benchmark-10keys-100B-expire-use-case-with-variant.yml", + "--db_server_host", + "{}".format(db_host), + "--db_server_port", + "{}".format(db_port), + "--flushall_on_every_test_start", + ] + ) + try: + run_client_runner_logic(args, "tool", "", "v0") + except SystemExit as e: + assert e.code == 0 + + r = redis.Redis(host=db_host, port=db_port_int) + total_keys = r.info("keyspace")["db0"]["keys"] + assert total_keys == 10 + + # run while pushing to redistimeseries + args = parser.parse_args( + args=[ + "--test", + "../../utils/tests/test_data/test-suites/memtier_benchmark-10keys-100B-expire-use-case-with-variant.yml", + "--datasink_push_results_redistimeseries", + "--datasink_redistimeseries_host", + "{}".format(db_host), + "--datasink_redistimeseries_port", + "{}".format(datasink_port), + "--db_server_host", + "{}".format(db_host), + "--db_server_port", + "{}".format(db_port), + "--flushall_on_every_test_start", + ] + ) + try: + run_client_runner_logic(args, "tool", "", "v0") + except SystemExit as e: + assert e.code == 0 + + r = redis.Redis(host=db_host, port=db_port_int) + total_keys = r.info("keyspace")["db0"]["keys"] + assert total_keys == 10