docker benchmark updates

docker/docker-cluster/caddy/Caddyfile

@@ -14,6 +14,15 @@
     tls off
 }
 
+:6063 {
+    proxy / metrictank0:6060 metrictank1:6060 metrictank2:6060 metrictank3:6060 {
+            header_upstream x-org-id 1
+        }
+    errors stderr
+    tls off
+}
+
+
 :8081 {
     proxy / graphite {
             header_upstream x-org-id 1

docker/docker-cluster/docker-compose.yml

@@ -173,6 +173,7 @@ services:
     ports:
       - "6061:6061"
       - "6062:6062"
+      - "6063:6063"
       - "8081:8081"
       - "8082:8082"
     volumes:

docker/docker-cluster/metrictank.ini

@@ -198,7 +198,7 @@ topics = mdm
 # offset to start consuming from. Can be one of newest, oldest,last or a time duration
 # When using a duration but the offset request fails (e.g. Kafka doesn't have data so far back), metrictank falls back to `oldest`.
 # the further back in time you go, the more old data you can load into metrictank, but the longer it takes to catch up to realtime data
-offset = last
+offset = oldest
 # kafka partitions to consume. use '*' or a comma separated list of id's
 partitions = *
 # save interval for offsets
@@ -295,7 +295,7 @@ partitions = *
 # offset to start consuming from. Can be one of newest, oldest,last or a time duration
 # When using a duration but the offset request fails (e.g. Kafka doesn't have data so far back), metrictank falls back to `oldest`.
 # Should match your kafka-mdm-in setting
-offset = last
+offset = oldest
 # save interval for offsets
 offset-commit-interval = 5s
 # Maximum time backlog processing can block during metrictank startup.

docker/docker-dev-custom-cfg-kafka/metrictank.ini

@@ -198,7 +198,7 @@ topics = mdm
 # offset to start consuming from. Can be one of newest, oldest,last or a time duration
 # When using a duration but the offset request fails (e.g. Kafka doesn't have data so far back), metrictank falls back to `oldest`.
 # the further back in time you go, the more old data you can load into metrictank, but the longer it takes to catch up to realtime data
-offset = last
+offset = oldest
 # kafka partitions to consume. use '*' or a comma separated list of id's
 partitions = *
 # save interval for offsets
@@ -295,7 +295,7 @@ partitions = *
 # offset to start consuming from. Can be one of newest, oldest,last or a time duration
 # When using a duration but the offset request fails (e.g. Kafka doesn't have data so far back), metrictank falls back to `oldest`.
 # Should match your kafka-mdm-in setting
-offset = last
+offset = oldest
 # save interval for offsets
 offset-commit-interval = 5s
 # Maximum time backlog processing can block during metrictank startup.

docker/scripts/bench-ingest-back-fill-speed.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# for use with environments that have kafka offset=auto such as docker-cluster and docker-dev-custom-cfg-kafka
+# first fill up kafka then restart the mt's with docker compose to observe backfill speed
+fakemetrics backfill --kafka-mdm-addr localhost:9092 --kafka-mdm-v2=true --offset $((2*366*24))h --period 1800s --speedup 360000 --mpo 500 

docker/scripts/bench-realistic-workload-single-tenant.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+trap 'kill $(jobs -p)' EXIT
+# load up some old data
+fakemetrics backfill --kafka-mdm-addr localhost:9092 --offset 5h --period 10s --speedup 100 --mpo 5000
+# then continue with a realtime feed
+fakemetrics feed --kafka-mdm-addr localhost:9092 --period 10s --mpo 5000 &
+sleep 30
+mt-index-cat -addr http://localhost:6063 -from 60min cass -hosts localhost:9042 -schema-file ./scripts/config/schema-idx-cassandra.toml vegeta-render-patterns | vegeta attack -duration 300s | vegeta report
+sleep 20