feat: Kafka thoughput (#402)

Signed-off-by: Alex Collins <[email protected]>
argoproj-labs · Oct 4, 2021 · d515a7e · d515a7e
1 parent 6b82f9d
commit d515a7e
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 5 deletions.
diff --git a/runner/sidecar/source/kafka/kafka.go b/runner/sidecar/source/kafka/kafka.go
@@ -57,8 +57,8 @@ func New(ctx context.Context, secretInterface corev1.SecretInterface, mntr monit
 	}
 	config["statistics.interval.ms"] = 5 * seconds
 	// https://docs.confluent.io/cloud/current/client-apps/optimizing/throughput.html
-	// config["fetch.min.bytes"] = 100000
-	// config["fetch.max.wait.ms"] = seconds / 2
+	config["fetch.min.bytes"] = 100000
+	config["fetch.wait.max.ms"] = seconds / 2
 	logger.Info("Kafka config", "config", sharedutil.MustJSON(sharedkafka.RedactConfigMap(config)))
 	// https://github.com/confluentinc/confluent-kafka-go/blob/master/examples/consumer_example/consumer_example.go
 	consumer, err := kafka.NewConsumer(&config)

diff --git a/test/kafka-stress/test-results.json b/test/kafka-stress/test-results.json
@@ -1,13 +1,13 @@
 {
-  "TestKafkaAsyncSinkStress/.tps": 400,
-  "TestKafkaSinkStress/.tps": 300,
+  "TestKafkaAsyncSinkStress/.tps": 350,
+  "TestKafkaSinkStress/.tps": 200,
   "TestKafkaSinkStress/N=10,messageSize=100.tps": 200,
   "TestKafkaSinkStress/N=10,messageSize=1000.tps": 150,
   "TestKafkaSinkStress/N=50000.tps": 750,
   "TestKafkaSinkStress/async=true.tps": 400,
   "TestKafkaSinkStress/messageSize=1000.tps": 300,
   "TestKafkaSinkStress/replicas=2.tps": 400,
-  "TestKafkaSourceStress/.tps": 400,
+  "TestKafkaSourceStress/.tps": 350,
   "TestKafkaSourceStress/N=10,messageSize=100.tps": 450,
   "TestKafkaSourceStress/N=10,messageSize=1000.tps": 650,
   "TestKafkaSourceStress/N=50000.tps": 3150,