From a04a7be7032d658369de89c5f69ec40e05fb8e71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Wed, 2 Sep 2020 17:42:23 +0200 Subject: [PATCH 1/3] Set compaction sources correctly, otherwise compactor deletes these blocks. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- tools/blocksconvert/builder/tsdb.go | 4 ++++ tools/blocksconvert/builder/tsdb_test.go | 1 + 2 files changed, 5 insertions(+) diff --git a/tools/blocksconvert/builder/tsdb.go b/tools/blocksconvert/builder/tsdb.go index c77a7a01588..30b89695e1a 100644 --- a/tools/blocksconvert/builder/tsdb.go +++ b/tools/blocksconvert/builder/tsdb.go @@ -191,6 +191,10 @@ func (d *tsdbBuilder) finishBlock(source string, labels map[string]string) (ulid Version: 1, MinTime: math.MaxInt64, MaxTime: math.MinInt64, + Compaction: tsdb.BlockMetaCompaction{ + Level: 1, + Sources: []ulid.ULID{d.ulid}, + }, }, Thanos: metadata.Thanos{ diff --git a/tools/blocksconvert/builder/tsdb_test.go b/tools/blocksconvert/builder/tsdb_test.go index db18afda53f..8224f06d773 100644 --- a/tools/blocksconvert/builder/tsdb_test.go +++ b/tools/blocksconvert/builder/tsdb_test.go @@ -78,6 +78,7 @@ func TestTsdbBuilder(t *testing.T) { blocks := db.Blocks() require.Equal(t, 1, len(blocks)) require.Equal(t, id, blocks[0].Meta().ULID) + require.Equal(t, id, blocks[0].Meta().Compaction.Sources[0]) require.Equal(t, uint64(seriesCount), blocks[0].Meta().Stats.NumSeries) require.Equal(t, uint64(totalSamples.Load()), blocks[0].Meta().Stats.NumSamples) From 706415056eb19e3f067911bfd703dd33fc97d8f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Wed, 2 Sep 2020 17:53:08 +0200 Subject: [PATCH 2/3] Added test showing that deduplication filter doesn't remove the block anymore. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- tools/blocksconvert/builder/tsdb_test.go | 32 ++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tools/blocksconvert/builder/tsdb_test.go b/tools/blocksconvert/builder/tsdb_test.go index 8224f06d773..8ce6c00b452 100644 --- a/tools/blocksconvert/builder/tsdb_test.go +++ b/tools/blocksconvert/builder/tsdb_test.go @@ -6,15 +6,20 @@ import ( "io/ioutil" "math/rand" "os" + "path/filepath" "sync" "testing" "time" + "github.com/oklog/ulid" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb" "github.com/stretchr/testify/require" + "github.com/thanos-io/thanos/pkg/block" + "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/thanos-io/thanos/pkg/extprom" "go.uber.org/atomic" "github.com/cortexproject/cortex/pkg/chunk" @@ -108,6 +113,33 @@ func TestTsdbBuilder(t *testing.T) { require.NoError(t, q.Close()) require.NoError(t, db.Close()) + + m, err := metadata.Read(filepath.Join(dir, id.String())) + require.NoError(t, err) + + otherId := ulid.MustNew(ulid.Now(), nil) + + // Make sure that deduplicate filter doesn't remove this block (thanks to correct sources). + df := block.NewDeduplicateFilter() + inp := map[ulid.ULID]*metadata.Meta{ + otherId: { + BlockMeta: tsdb.BlockMeta{ + ULID: otherId, + MinTime: 0, + MaxTime: 0, + Compaction: tsdb.BlockMetaCompaction{ + Sources: []ulid.ULID{otherId}, + }, + Version: 0, + }, + }, + + id: m, + } + + err = df.Filter(context.Background(), inp, extprom.NewTxGaugeVec(nil, prometheus.GaugeOpts{}, []string{"state"})) + require.NoError(t, err) + require.NotNil(t, inp[id]) } func metricInfo(ix int) (labels.Labels, int) { From 95dbe151d5e41fb2b0be7967322d954b0a4c5ed3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Wed, 2 Sep 2020 18:47:35 +0200 Subject: [PATCH 3/3] Make lint happy. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- tools/blocksconvert/builder/tsdb_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/blocksconvert/builder/tsdb_test.go b/tools/blocksconvert/builder/tsdb_test.go index 8ce6c00b452..b6fd374e48f 100644 --- a/tools/blocksconvert/builder/tsdb_test.go +++ b/tools/blocksconvert/builder/tsdb_test.go @@ -117,18 +117,18 @@ func TestTsdbBuilder(t *testing.T) { m, err := metadata.Read(filepath.Join(dir, id.String())) require.NoError(t, err) - otherId := ulid.MustNew(ulid.Now(), nil) + otherID := ulid.MustNew(ulid.Now(), nil) // Make sure that deduplicate filter doesn't remove this block (thanks to correct sources). df := block.NewDeduplicateFilter() inp := map[ulid.ULID]*metadata.Meta{ - otherId: { + otherID: { BlockMeta: tsdb.BlockMeta{ - ULID: otherId, + ULID: otherID, MinTime: 0, MaxTime: 0, Compaction: tsdb.BlockMetaCompaction{ - Sources: []ulid.ULID{otherId}, + Sources: []ulid.ULID{otherID}, }, Version: 0, },