Added 142 new benchmarks focused on latency tracking (rate-limited) (#…

…273)

* Added 142 new benchmarks focused on latency tracking (rate-limited)

* Bumping version from 0.1.240 to 0.1.241

* Added 142 new benchmarks focused on latency tracking (rate-limited)

* Using docker image redislabs/memtier_benchmark:2.1.0 on latency related benchmarks

* Removed benchmarks above 25gb

* Removed 2TB test

* Deleted wrongly designed test

* Added 3M keys 512 and 1000B GET/SET use-cases

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.240"
+version = "0.1.245"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "Readme.md"

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-hash-hgetall-50-fields-100B-values.yml

@@ -0,0 +1,51 @@
+version: 0.4
+name: 
+  latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-hash-hgetall-50-fields-100B-values
+description: Runs memtier_benchmark, for a keyspace length of 1M keys pre-loading
+  HASHes in which the value has a data size of 10 Bytes. After pre-loading the data
+  it issues HGETALL command. Rate limited to 10000 ops/sec.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 100000
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:2.1.0
+    tool: memtier_benchmark
+    arguments: '"--data-size" "100" --command "HSET __key__ field:1 __data__ field:2
+      __data__ field:3 __data__ field:4 __data__ field:5 __data__ field:6 __data__
+      field:7 __data__ field:8 __data__ field:9 __data__ field:10 __data__ field:11
+      __data__ field:12 __data__ field:13 __data__ field:14 __data__ field:15 __data__
+      field:16 __data__ field:17 __data__ field:18 __data__ field:19 __data__ field:20
+      __data__ field:21 __data__ field:22 __data__ field:23 __data__ field:24 __data__
+      field:25 __data__ field:26 __data__ field:27 __data__ field:28 __data__ field:29
+      __data__ field:30 __data__ field:31 __data__ field:32 __data__ field:33 __data__
+      field:34 __data__ field:35 __data__ field:36 __data__ field:37 __data__ field:38
+      __data__ field:39 __data__ field:40 __data__ field:41 __data__ field:42 __data__
+      field:43 __data__ field:44 __data__ field:45 __data__ field:46 __data__ field:47
+      __data__ field:48 __data__ field:49 __data__ field:50 __data__" --command-key-pattern="P"
+      --key-minimum=1 --key-maximum 1000000 -n 500 -c 50 -t 4 --hide-histogram'
+  resources:
+    requests:
+      memory: 2g
+tested-groups:
+- hash
+tested-commands:
+- hgetall
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:2.1.0
+  tool: memtier_benchmark
+  arguments: "--command \"HGETALL __key__\" --command-key-pattern=\"R\" --key-minimum=1
+    --key-maximum 100000 --test-time 120 -c 50 -t 4 --hide-histogram --rate-limiting
+    50"
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 96

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-load-hash-50-fields-with-1000B-values.yml

@@ -0,0 +1,46 @@
+version: 0.4
+name: 
+  latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-load-hash-50-fields-with-1000B-values
+description: Runs memtier_benchmark, for a keyspace length of 100K keys loading HASHES
+  with 50 fields each. Each field value has a data size of 1000 Bytes. Rate limited
+  to 10000 ops/sec.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 6g
+tested-groups:
+- hash
+tested-commands:
+- hset
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:2.1.0
+  tool: memtier_benchmark
+  arguments: "--test-time 120 --distinct-client-seed \"--data-size\" \"1000\" --command
+    \"HSET __key__ field:1 __data__ field:2 __data__ field:3 __data__ field:4 __data__
+    field:5 __data__ field:6 __data__ field:7 __data__ field:8 __data__ field:9 __data__
+    field:10 __data__ field:11 __data__ field:12 __data__ field:13 __data__ field:14
+    __data__ field:15 __data__ field:16 __data__ field:17 __data__ field:18 __data__
+    field:19 __data__ field:20 __data__ field:21 __data__ field:22 __data__ field:23
+    __data__ field:24 __data__ field:25 __data__ field:26 __data__ field:27 __data__
+    field:28 __data__ field:29 __data__ field:30 __data__ field:31 __data__ field:32
+    __data__ field:33 __data__ field:34 __data__ field:35 __data__ field:36 __data__
+    field:37 __data__ field:38 __data__ field:39 __data__ field:40 __data__ field:41
+    __data__ field:42 __data__ field:43 __data__ field:44 __data__ field:45 __data__
+    field:46 __data__ field:47 __data__ field:48 __data__ field:49 __data__ field:50
+    __data__\" --command-key-pattern=\"R\" --key-minimum=1 --key-maximum 100000 -c
+    50 -t 4 --hide-histogram --rate-limiting 50"
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 5

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-load-hash-50-fields-with-100B-values.yml

@@ -0,0 +1,46 @@
+version: 0.4
+name: 
+  latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-load-hash-50-fields-with-100B-values
+description: Runs memtier_benchmark, for a keyspace length of 100K keys loading HASHES
+  with 50 fields each. Each field value has a data size of 100 Bytes. Rate limited
+  to 10000 ops/sec.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 6g
+tested-groups:
+- hash
+tested-commands:
+- hset
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:2.1.0
+  tool: memtier_benchmark
+  arguments: "--test-time 120 --distinct-client-seed \"--data-size\" \"100\" --command
+    \"HSET __key__ field:1 __data__ field:2 __data__ field:3 __data__ field:4 __data__
+    field:5 __data__ field:6 __data__ field:7 __data__ field:8 __data__ field:9 __data__
+    field:10 __data__ field:11 __data__ field:12 __data__ field:13 __data__ field:14
+    __data__ field:15 __data__ field:16 __data__ field:17 __data__ field:18 __data__
+    field:19 __data__ field:20 __data__ field:21 __data__ field:22 __data__ field:23
+    __data__ field:24 __data__ field:25 __data__ field:26 __data__ field:27 __data__
+    field:28 __data__ field:29 __data__ field:30 __data__ field:31 __data__ field:32
+    __data__ field:33 __data__ field:34 __data__ field:35 __data__ field:36 __data__
+    field:37 __data__ field:38 __data__ field:39 __data__ field:40 __data__ field:41
+    __data__ field:42 __data__ field:43 __data__ field:44 __data__ field:45 __data__
+    field:46 __data__ field:47 __data__ field:48 __data__ field:49 __data__ field:50
+    __data__\" --command-key-pattern=\"R\" --key-minimum=1 --key-maximum 100000 -c
+    50 -t 4 --hide-histogram --rate-limiting 50"
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 5

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-load-hash-50-fields-with-10B-values.yml

@@ -0,0 +1,46 @@
+version: 0.4
+name: 
+  latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-load-hash-50-fields-with-10B-values
+description: Runs memtier_benchmark, for a keyspace length of 100K keys loading HASHES
+  with 50 fields each. Each field value has a data size of 10 Bytes. Rate limited
+  to 10000 ops/sec.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 6g
+tested-groups:
+- hash
+tested-commands:
+- hset
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:2.1.0
+  tool: memtier_benchmark
+  arguments: "--test-time 120 --distinct-client-seed \"--data-size\" \"10\" --command
+    \"HSET __key__ field:1 __data__ field:2 __data__ field:3 __data__ field:4 __data__
+    field:5 __data__ field:6 __data__ field:7 __data__ field:8 __data__ field:9 __data__
+    field:10 __data__ field:11 __data__ field:12 __data__ field:13 __data__ field:14
+    __data__ field:15 __data__ field:16 __data__ field:17 __data__ field:18 __data__
+    field:19 __data__ field:20 __data__ field:21 __data__ field:22 __data__ field:23
+    __data__ field:24 __data__ field:25 __data__ field:26 __data__ field:27 __data__
+    field:28 __data__ field:29 __data__ field:30 __data__ field:31 __data__ field:32
+    __data__ field:33 __data__ field:34 __data__ field:35 __data__ field:36 __data__
+    field:37 __data__ field:38 __data__ field:39 __data__ field:40 __data__ field:41
+    __data__ field:42 __data__ field:43 __data__ field:44 __data__ field:45 __data__
+    field:46 __data__ field:47 __data__ field:48 __data__ field:49 __data__ field:50
+    __data__\" --command-key-pattern=\"R\" --key-minimum=1 --key-maximum 100000 -c
+    50 -t 4 --hide-histogram --rate-limiting 50"
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 5

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-10Mkeys-load-hash-5-fields-with-100B-values-pipeline-10.yml

@@ -0,0 +1,36 @@
+version: 0.4
+name: 
+  latency-rate-limited-10000_qps-memtier_benchmark-10Mkeys-load-hash-5-fields-with-100B-values-pipeline-10
+description: Runs memtier_benchmark, for a keyspace length of 10M keys loading HASHES
+  with 5 fields each. Each field value has a data size of 100 Bytes. Rate limited
+  to 10000 ops/sec.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 9g
+tested-commands:
+- hset
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:2.1.0
+  tool: memtier_benchmark
+  arguments: "\"--pipeline\" \"10\" \"--data-size\" \"100\" --command \"HSET __key__
+    field1 __data__ field2 __data__ field3 __data__ field4 __data__ field5 __data__\"\
+    \ \"--command-key-pattern\" \"P\" --key-minimum=1 --key-maximum 10000000 -c 50
+    -t 4 --hide-histogram --test-time=180 --rate-limiting 50"
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+tested-groups:
+- hash
+priority: 5

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-10Mkeys-load-hash-5-fields-with-100B-values.yml

@@ -0,0 +1,36 @@
+version: 0.4
+name: 
+  latency-rate-limited-10000_qps-memtier_benchmark-10Mkeys-load-hash-5-fields-with-100B-values
+description: Runs memtier_benchmark, for a keyspace length of 10M keys loading HASHES
+  with 5 fields each. Each field value has a data size of 100 Bytes. Rate limited
+  to 10000 ops/sec.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 9g
+tested-commands:
+- hset
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:2.1.0
+  tool: memtier_benchmark
+  arguments: "--test-time 180 \"--data-size\" \"100\" --command \"HSET __key__ field1
+    __data__ field2 __data__ field3 __data__ field4 __data__ field5 __data__\" --command-key-pattern=\"\
+    P\" --key-minimum=1 --key-maximum 10000000 -c 50 -t 4 --hide-histogram --rate-limiting
+    50"
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+tested-groups:
+- hash
+priority: 5

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-10Mkeys-load-hash-5-fields-with-10B-values-pipeline-10.yml

@@ -0,0 +1,36 @@
+version: 0.4
+name: 
+  latency-rate-limited-10000_qps-memtier_benchmark-10Mkeys-load-hash-5-fields-with-10B-values-pipeline-10
+description: Runs memtier_benchmark, for a keyspace length of 10M keys loading HASHES
+  with 5 fields each. Each field value has a data size of 10 Bytes. Rate limited to
+  10000 ops/sec.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 2g
+tested-commands:
+- hset
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:2.1.0
+  tool: memtier_benchmark
+  arguments: "--test-time 180 \"--pipeline\" \"10\" \"--data-size\" \"10\" --command
+    \"HSET __key__ field1 __data__ field2 __data__ field3 __data__ field4 __data__
+    field5 __data__\" --command-key-pattern=\"P\" --key-minimum=1 --key-maximum 10000000
+    -c 50 -t 4 --hide-histogram --rate-limiting 50"
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+tested-groups:
+- hash
+priority: 5

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-10Mkeys-load-hash-5-fields-with-10B-values.yml

@@ -0,0 +1,36 @@
+version: 0.4
+name: 
+  latency-rate-limited-10000_qps-memtier_benchmark-10Mkeys-load-hash-5-fields-with-10B-values
+description: Runs memtier_benchmark, for a keyspace length of 10M keys loading HASHES
+  with 5 fields each. Each field value has a data size of 10 Bytes. Rate limited to
+  10000 ops/sec.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 2g
+tested-commands:
+- hset
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:2.1.0
+  tool: memtier_benchmark
+  arguments: "--test-time 180 \"--data-size\" \"10\" --command \"HSET __key__ field1
+    __data__ field2 __data__ field3 __data__ field4 __data__ field5 __data__\" --command-key-pattern=\"\
+    P\" --key-minimum=1 --key-maximum 10000000 -c 50 -t 4 --hide-histogram --rate-limiting
+    50"
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+tested-groups:
+- hash
+priority: 5

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-1Mkeys-100B-expire-use-case.yml

@@ -0,0 +1,45 @@
+version: 0.4
+name: latency-rate-limited-10000_qps-memtier_benchmark-1Mkeys-100B-expire-use-case
+description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data
+  size of 100 Bytes for each key, which 50% of the keys have expiration set during
+  the benchmark. Rate limited to 10000 ops/sec.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1000000
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:2.1.0
+    tool: memtier_benchmark
+    arguments: '"--data-size" "100" "--command" "SET __key__ __data__" "--command-key-pattern"
+      "P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"'
+  resources:
+    requests:
+      memory: 1g
+tested-groups:
+- string
+- generic
+tested-commands:
+- set
+- setx
+- get
+- del
+- setex
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:2.1.0
+  tool: memtier_benchmark
+  arguments: "\"--data-size\" \"100\" --command \"SETEX __key__ 10 __data__\" --command-key-pattern=\"\
+    R\" --command \"SET __key__ __data__\" --command-key-pattern=\"R\" --command \"\
+    GET __key__\" --command-key-pattern=\"R\" --command \"DEL __key__\" --command-key-pattern=\"\
+    R\"  -c 50 -t 2 --hide-histogram --test-time 300 --rate-limiting 100"
+  resources:
+    requests:
+      cpus: '3'
+      memory: 2g
+
+priority: 33