add test on BucketStores

Signed-off-by: Alan Protasio <[email protected]>

Author: alanprot

pkg/storage/tsdb/testutil/objstore.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"io"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/pkg/errors"
@@ -46,6 +47,11 @@ func (m *MockBucketFailure) Delete(ctx context.Context, name string) error {
 }
 
 func (m *MockBucketFailure) Get(ctx context.Context, name string) (io.ReadCloser, error) {
+	for prefix, err := range m.GetFailures {
+		if strings.HasPrefix(name, prefix) {
+			return nil, err
+		}
+	}
 	if e, ok := m.GetFailures[name]; ok {
 		return nil, e
 	}
@@ -58,7 +64,7 @@ func (m *MockBucketFailure) WithExpectedErrs(expectedFunc objstore.IsOpFailureEx
 		return &MockBucketFailure{Bucket: ibkt.WithExpectedErrs(expectedFunc), DeleteFailures: m.DeleteFailures, GetFailures: m.GetFailures}
 	}
 
-	return m.WithExpectedErrs(expectedFunc)
+	return m
 }
 
 func (m *MockBucketFailure) ReaderWithExpectedErrs(expectedFunc objstore.IsOpFailureExpectedFunc) objstore.BucketReader {

pkg/storegateway/bucket_index_metadata_fetcher.go

@@ -68,7 +68,7 @@ func (f *BucketIndexMetadataFetcher) Fetch(ctx context.Context) (metas map[ulid.
 	start := time.Now()
 	defer func() {
 		f.metrics.SyncDuration.Observe(time.Since(start).Seconds())
-		if err != nil {
+		if err != nil && !errors.Is(err, bucketindex.ErrCustomerManagedKeyError) {
 			f.metrics.SyncFailures.Inc()
 		}
 	}()

pkg/storegateway/bucket_stores_test.go

@@ -13,6 +13,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cortexproject/cortex/pkg/storage/tsdb/bucketindex"
+	cortex_testutil "github.com/cortexproject/cortex/pkg/storage/tsdb/testutil"
 	"github.com/go-kit/log"
 	"github.com/oklog/ulid"
 	"github.com/prometheus/client_golang/prometheus"
@@ -40,6 +42,51 @@ import (
 	"github.com/cortexproject/cortex/pkg/util/flagext"
 )
 
+func TestBucketStores_CustomerKeyError(t *testing.T) {
+	userToMetric := map[string]string{
+		"user-1": "series_1",
+		"user-2": "series_2",
+	}
+
+	ctx := context.Background()
+	cfg := prepareStorageConfig(t)
+	cfg.BucketStore.BucketIndex.Enabled = true
+
+	storageDir := t.TempDir()
+
+	for userID, metricName := range userToMetric {
+		generateStorageBlock(t, storageDir, userID, metricName, 10, 100, 15)
+	}
+
+	bucket, err := filesystem.NewBucketClient(filesystem.Config{Directory: storageDir})
+
+	mBucket := &cortex_testutil.MockBucketFailure{
+		Bucket: bucket,
+		GetFailures: map[string]error{
+			"user-1": cortex_testutil.ErrKeyAccessDeniedError,
+		},
+	}
+	require.NoError(t, err)
+
+	reg := prometheus.NewPedanticRegistry()
+	stores, err := NewBucketStores(cfg, NewNoShardingStrategy(), mBucket, defaultLimitsOverrides(t), mockLoggingLevel(), log.NewNopLogger(), reg)
+	require.NoError(t, err)
+
+	// Should set the error on user-1
+	require.NoError(t, stores.InitialSync(ctx))
+	require.ErrorIs(t, stores.stores["user-1"].err, bucketindex.ErrCustomerManagedKeyError)
+	require.ErrorIs(t, stores.stores["user-2"].err, nil)
+	require.NoError(t, stores.SyncBlocks(context.Background()))
+	require.ErrorIs(t, stores.stores["user-1"].err, bucketindex.ErrCustomerManagedKeyError)
+	require.ErrorIs(t, stores.stores["user-2"].err, nil)
+
+	// Cleaning the error
+	mBucket.GetFailures = map[string]error{}
+	require.NoError(t, stores.SyncBlocks(context.Background()))
+	require.ErrorIs(t, stores.stores["user-1"].err, nil)
+	require.ErrorIs(t, stores.stores["user-2"].err, nil)
+}
+
 func TestBucketStores_InitialSync(t *testing.T) {
 	t.Parallel()
 	userToMetric := map[string]string{