From c9e93c8e1d62c92f9eee4d20d0ff3cb227cc0daf Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 14 Sep 2020 16:57:10 +0000 Subject: [PATCH 1/4] Refactor: extract function to set up ingester benchmark data Signed-off-by: Bryan Boreham --- pkg/ingester/ingester_test.go | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index b193094b5f4..013975d9fd2 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -843,19 +843,9 @@ func BenchmarkIngesterPushErrors(b *testing.B) { benchmarkIngesterPush(b, limits, true) } -func benchmarkIngesterPush(b *testing.B, limits validation.Limits, errorsExpected bool) { - cfg := defaultIngesterTestConfig() - clientCfg := defaultClientTestConfig() - - const ( - series = 100 - samples = 100 - ) - - // Construct a set of realistic-looking samples, all with slightly different label sets - var allLabels []labels.Labels - var allSamples []client.Sample - for j := 0; j < series; j++ { +// Construct a set of realistic-looking samples, all with slightly different label sets +func benchmarkData(nSeries int) (allLabels []labels.Labels, allSamples []client.Sample) { + for j := 0; j < nSeries; j++ { labels := chunk.BenchmarkLabels.Copy() for i := range labels { if labels[i].Name == "cpu" { @@ -865,6 +855,19 @@ func benchmarkIngesterPush(b *testing.B, limits validation.Limits, errorsExpecte allLabels = append(allLabels, labels) allSamples = append(allSamples, client.Sample{TimestampMs: 0, Value: float64(j)}) } + return +} + +func benchmarkIngesterPush(b *testing.B, limits validation.Limits, errorsExpected bool) { + cfg := defaultIngesterTestConfig() + clientCfg := defaultClientTestConfig() + + const ( + series = 100 + samples = 100 + ) + + allLabels, allSamples := benchmarkData(series) ctx := user.InjectOrgID(context.Background(), "1") encodings := []struct { From 98421d058c730bb3d2e6cfac89899fca3f2233e9 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 14 Sep 2020 16:57:44 +0000 Subject: [PATCH 2/4] Add benchmark for ingester QueryStream() Signed-off-by: Bryan Boreham --- pkg/ingester/ingester_test.go | 46 +++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 013975d9fd2..cb82865e8b1 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5,6 +5,7 @@ import ( "fmt" "io/ioutil" "math" + "math/rand" "net/http" "os" "path/filepath" @@ -900,3 +901,48 @@ func benchmarkIngesterPush(b *testing.B, limits validation.Limits, errorsExpecte } } + +func BenchmarkIngester_QueryStream(b *testing.B) { + cfg := defaultIngesterTestConfig() + clientCfg := defaultClientTestConfig() + limits := defaultLimitsTestConfig() + _, ing := newTestStore(b, cfg, clientCfg, limits, nil) + ctx := user.InjectOrgID(context.Background(), "1") + + const ( + series = 2000 + samples = 1000 + ) + + allLabels, allSamples := benchmarkData(series) + + // Bump the timestamp and set a random value on each of our test samples each time round the loop + for j := 0; j < samples; j++ { + for i := range allSamples { + allSamples[i].TimestampMs = int64(j + 1) + allSamples[i].Value = rand.Float64() + } + _, err := ing.Push(ctx, client.ToWriteRequest(allLabels, allSamples, nil, client.API)) + require.NoError(b, err) + } + + req := &client.QueryRequest{ + StartTimestampMs: 0, + EndTimestampMs: samples + 1, + + Matchers: []*client.LabelMatcher{{ + Type: client.EQUAL, + Name: model.MetricNameLabel, + Value: "container_cpu_usage_seconds_total", + }}, + } + + mockStream := &mockQueryStreamServer{ctx: ctx} + + b.ResetTimer() + + for ix := 0; ix < b.N; ix++ { + err := ing.QueryStream(req, mockStream) + require.NoError(b, err) + } +} From 04f676cacd30c09b71fb0a2b17f28ec38fb167fd Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 14 Sep 2020 15:25:16 +0000 Subject: [PATCH 3/4] Re-use memory for chunks in ingester QueryStream This improves performance of queries and reduces garbage-collection. Signed-off-by: Bryan Boreham --- pkg/ingester/ingester.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 19eade8d6d7..d541b2ce10b 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -710,6 +710,7 @@ func (i *Ingester) QueryStream(req *client.QueryRequest, stream client.Ingester_ } numSeries, numChunks := 0, 0 + reuseWireChunks := [queryStreamBatchSize][]client.Chunk{} batch := make([]client.TimeSeriesChunk, 0, queryStreamBatchSize) // We'd really like to have series in label order, not FP order, so we // can iteratively merge them with entries coming from the chunk store. But @@ -728,10 +729,12 @@ func (i *Ingester) QueryStream(req *client.QueryRequest, stream client.Ingester_ } numSeries++ - wireChunks, err := toWireChunks(chunks, nil) + reusePos := len(batch) + wireChunks, err := toWireChunks(chunks, reuseWireChunks[reusePos]) if err != nil { return err } + reuseWireChunks[reusePos] = wireChunks numChunks += len(wireChunks) batch = append(batch, client.TimeSeriesChunk{ From 95adc02c91a21fe978d25d4c94fb71a7afdff00f Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 14 Sep 2020 17:08:45 +0000 Subject: [PATCH 4/4] Add CHANGELOG entry Signed-off-by: Bryan Boreham --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8a0a9c04b8..764723a0090 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ * [ENHANCEMENT] When a tenant accesses the Alertmanager UI or its API, if we have valid `-alertmanager.configs.fallback` we'll use that to start the manager and avoid failing the request. #3073 * [ENHANCEMENT] Add `DELETE api/v1/rules/{namespace}` to the Ruler. It allows all the rule groups of a namespace to be deleted. #3120 * [ENHANCEMENT] Experimental Delete Series: Retry processing of Delete requests during failures. #2926 +* [ENHANCEMENT] Improve performance of QueryStream() in ingesters. #3177 * [BUGFIX] Query-frontend: Fixed rounding for incoming query timestamps, to be 100% Prometheus compatible. #2990 * [BUGFIX] Querier: Merge results from chunks and blocks ingesters when using streaming of results. #3013 * [BUGFIX] Querier: query /series from ingesters regardless the `-querier.query-ingesters-within` setting. #3035