Skip to content

Commit 35a621c

Browse files
authored
ObjectClient List refactoring (#3238)
* Make delimeter parameter to List method. Signed-off-by: Peter Štibraný <[email protected]> * Make FSObjectClient support recursive List and delimiter parameter. Signed-off-by: Peter Štibraný <[email protected]> * Fix errors and lint issues. Signed-off-by: Peter Štibraný <[email protected]> * Allow listing non-existant prefix, or single file. Signed-off-by: Peter Štibraný <[email protected]> * Review feedback. Signed-off-by: Peter Štibraný <[email protected]> * Fixed formatting error. 🤦 Signed-off-by: Peter Štibraný <[email protected]> * Added comment to List method. Signed-off-by: Peter Štibraný <[email protected]>
1 parent 2390d26 commit 35a621c

File tree

19 files changed

+201
-169
lines changed

19 files changed

+201
-169
lines changed

integration/s3_storage_client_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ func TestS3Client(t *testing.T) {
6767

6868
for _, tt := range tests {
6969
t.Run(tt.name, func(t *testing.T) {
70-
client, err := s3.NewS3ObjectClient(tt.cfg, "/")
70+
client, err := s3.NewS3ObjectClient(tt.cfg)
7171

7272
require.NoError(t, err)
7373

pkg/alertmanager/alerts/objectclient/store.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ func NewAlertStore(client chunk.ObjectClient) *AlertStore {
3333

3434
// ListAlertConfigs returns all of the active alert configs in this store
3535
func (a *AlertStore) ListAlertConfigs(ctx context.Context) (map[string]alerts.AlertConfigDesc, error) {
36-
objs, _, err := a.client.List(ctx, alertPrefix)
36+
objs, _, err := a.client.List(ctx, alertPrefix, "")
3737
if err != nil {
3838
return nil, err
3939
}

pkg/alertmanager/storage.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,9 @@ func NewAlertStore(cfg AlertStoreConfig) (AlertStore, error) {
5555
case "local":
5656
return local.NewStore(cfg.Local)
5757
case "gcs":
58-
return newObjAlertStore(gcp.NewGCSObjectClient(context.Background(), cfg.GCS, ""))
58+
return newObjAlertStore(gcp.NewGCSObjectClient(context.Background(), cfg.GCS))
5959
case "s3":
60-
return newObjAlertStore(aws.NewS3ObjectClient(cfg.S3, ""))
60+
return newObjAlertStore(aws.NewS3ObjectClient(cfg.S3))
6161
default:
6262
return nil, fmt.Errorf("unrecognized alertmanager storage backend %v, choose one of: azure, configdb, gcs, local, s3", cfg.Type)
6363
}

pkg/chunk/aws/fixtures.go

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,7 @@ var Fixtures = []testutils.Fixture{
4444
schemaCfg: schemaConfig,
4545
metrics: newMetrics(nil),
4646
}
47-
object := objectclient.NewClient(&S3ObjectClient{
48-
S3: newMockS3(),
49-
delimiter: chunk.DirDelim,
50-
}, nil)
47+
object := objectclient.NewClient(&S3ObjectClient{S3: newMockS3()}, nil)
5148
return index, object, table, schemaConfig, testutils.CloserFunc(func() error {
5249
table.Stop()
5350
index.Stop()

pkg/chunk/aws/s3_storage_client.go

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -94,12 +94,11 @@ func (cfg *S3Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
9494
type S3ObjectClient struct {
9595
bucketNames []string
9696
S3 s3iface.S3API
97-
delimiter string
9897
sseEncryption *string
9998
}
10099

101100
// NewS3ObjectClient makes a new S3-backed ObjectClient.
102-
func NewS3ObjectClient(cfg S3Config, delimiter string) (*S3ObjectClient, error) {
101+
func NewS3ObjectClient(cfg S3Config) (*S3ObjectClient, error) {
103102
s3Config, bucketNames, err := buildS3Config(cfg)
104103
if err != nil {
105104
return nil, errors.Wrap(err, "failed to build s3 config")
@@ -120,7 +119,6 @@ func NewS3ObjectClient(cfg S3Config, delimiter string) (*S3ObjectClient, error)
120119
client := S3ObjectClient{
121120
S3: s3Client,
122121
bucketNames: bucketNames,
123-
delimiter: delimiter,
124122
sseEncryption: sseEncryption,
125123
}
126124
return &client, nil
@@ -289,8 +287,8 @@ func (a *S3ObjectClient) PutObject(ctx context.Context, objectKey string, object
289287
})
290288
}
291289

292-
// List objects and common-prefixes i.e synthetic directories from the store non-recursively
293-
func (a *S3ObjectClient) List(ctx context.Context, prefix string) ([]chunk.StorageObject, []chunk.StorageCommonPrefix, error) {
290+
// List implements chunk.ObjectClient.
291+
func (a *S3ObjectClient) List(ctx context.Context, prefix, delimiter string) ([]chunk.StorageObject, []chunk.StorageCommonPrefix, error) {
294292
var storageObjects []chunk.StorageObject
295293
var commonPrefixes []chunk.StorageCommonPrefix
296294

@@ -299,7 +297,7 @@ func (a *S3ObjectClient) List(ctx context.Context, prefix string) ([]chunk.Stora
299297
input := s3.ListObjectsV2Input{
300298
Bucket: aws.String(a.bucketNames[i]),
301299
Prefix: aws.String(prefix),
302-
Delimiter: aws.String(a.delimiter),
300+
Delimiter: aws.String(delimiter),
303301
}
304302

305303
for {
@@ -337,7 +335,3 @@ func (a *S3ObjectClient) List(ctx context.Context, prefix string) ([]chunk.Stora
337335

338336
return storageObjects, commonPrefixes, nil
339337
}
340-
341-
func (a *S3ObjectClient) PathSeparator() string {
342-
return a.delimiter
343-
}

pkg/chunk/aws/s3_storage_client_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ func TestRequestMiddleware(t *testing.T) {
5959
for _, tt := range tests {
6060
t.Run(tt.name, func(t *testing.T) {
6161
cfg.Inject = tt.fn
62-
client, err := NewS3ObjectClient(cfg, "/")
62+
client, err := NewS3ObjectClient(cfg)
6363
require.NoError(t, err)
6464

6565
readCloser, err := client.GetObject(context.Background(), "key")

pkg/chunk/azure/blob_storage_client.go

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -88,15 +88,13 @@ type BlobStorage struct {
8888
//blobService storage.Serv
8989
cfg *BlobStorageConfig
9090
containerURL azblob.ContainerURL
91-
delimiter string
9291
}
9392

9493
// NewBlobStorage creates a new instance of the BlobStorage struct.
95-
func NewBlobStorage(cfg *BlobStorageConfig, delimiter string) (*BlobStorage, error) {
94+
func NewBlobStorage(cfg *BlobStorageConfig) (*BlobStorage, error) {
9695
util.WarnExperimentalUse("Azure Blob Storage")
9796
blobStorage := &BlobStorage{
98-
cfg: cfg,
99-
delimiter: delimiter,
97+
cfg: cfg,
10098
}
10199

102100
var err error
@@ -196,8 +194,8 @@ func (b *BlobStorage) newPipeline() (pipeline.Pipeline, error) {
196194
}), nil
197195
}
198196

199-
// List objects and common-prefixes i.e synthetic directories from the store non-recursively
200-
func (b *BlobStorage) List(ctx context.Context, prefix string) ([]chunk.StorageObject, []chunk.StorageCommonPrefix, error) {
197+
// List implements chunk.ObjectClient.
198+
func (b *BlobStorage) List(ctx context.Context, prefix, delimiter string) ([]chunk.StorageObject, []chunk.StorageCommonPrefix, error) {
201199
var storageObjects []chunk.StorageObject
202200
var commonPrefixes []chunk.StorageCommonPrefix
203201

@@ -206,7 +204,7 @@ func (b *BlobStorage) List(ctx context.Context, prefix string) ([]chunk.StorageO
206204
return nil, nil, ctx.Err()
207205
}
208206

209-
listBlob, err := b.containerURL.ListBlobsHierarchySegment(ctx, marker, b.delimiter, azblob.ListBlobsSegmentOptions{Prefix: prefix})
207+
listBlob, err := b.containerURL.ListBlobsHierarchySegment(ctx, marker, delimiter, azblob.ListBlobsSegmentOptions{Prefix: prefix})
210208
if err != nil {
211209
return nil, nil, err
212210
}
@@ -240,10 +238,6 @@ func (b *BlobStorage) DeleteObject(ctx context.Context, blobID string) error {
240238
return err
241239
}
242240

243-
func (b *BlobStorage) PathSeparator() string {
244-
return b.delimiter
245-
}
246-
247241
// Validate the config.
248242
func (c *BlobStorageConfig) Validate() error {
249243
if !util.StringsContain(supportedEnvironments, c.Environment) {

pkg/chunk/gcp/fixtures.go

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,9 +78,7 @@ func (f *fixture) Clients() (
7878
}
7979

8080
if f.gcsObjectClient {
81-
cClient = objectclient.NewClient(newGCSObjectClient(GCSConfig{
82-
BucketName: "chunks",
83-
}, f.gcssrv.Client(), chunk.DirDelim), nil)
81+
cClient = objectclient.NewClient(newGCSObjectClient(GCSConfig{BucketName: "chunks"}, f.gcssrv.Client()), nil)
8482
} else {
8583
cClient = newBigtableObjectClient(Config{}, schemaConfig, client)
8684
}

pkg/chunk/gcp/gcs_object_client.go

Lines changed: 12 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,9 @@ import (
1313
)
1414

1515
type GCSObjectClient struct {
16-
cfg GCSConfig
17-
client *storage.Client
18-
bucket *storage.BucketHandle
19-
delimiter string
16+
cfg GCSConfig
17+
client *storage.Client
18+
bucket *storage.BucketHandle
2019
}
2120

2221
// GCSConfig is config for the GCS Chunk Client.
@@ -39,7 +38,7 @@ func (cfg *GCSConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
3938
}
4039

4140
// NewGCSObjectClient makes a new chunk.Client that writes chunks to GCS.
42-
func NewGCSObjectClient(ctx context.Context, cfg GCSConfig, delimiter string) (*GCSObjectClient, error) {
41+
func NewGCSObjectClient(ctx context.Context, cfg GCSConfig) (*GCSObjectClient, error) {
4342
option, err := gcsInstrumentation(ctx, storage.ScopeReadWrite)
4443
if err != nil {
4544
return nil, err
@@ -49,16 +48,15 @@ func NewGCSObjectClient(ctx context.Context, cfg GCSConfig, delimiter string) (*
4948
if err != nil {
5049
return nil, err
5150
}
52-
return newGCSObjectClient(cfg, client, delimiter), nil
51+
return newGCSObjectClient(cfg, client), nil
5352
}
5453

55-
func newGCSObjectClient(cfg GCSConfig, client *storage.Client, delimiter string) *GCSObjectClient {
54+
func newGCSObjectClient(cfg GCSConfig, client *storage.Client) *GCSObjectClient {
5655
bucket := client.Bucket(cfg.BucketName)
5756
return &GCSObjectClient{
58-
cfg: cfg,
59-
client: client,
60-
bucket: bucket,
61-
delimiter: delimiter,
57+
cfg: cfg,
58+
client: client,
59+
bucket: bucket,
6260
}
6361
}
6462

@@ -107,12 +105,12 @@ func (s *GCSObjectClient) PutObject(ctx context.Context, objectKey string, objec
107105
return nil
108106
}
109107

110-
// List objects and common-prefixes i.e synthetic directories from the store non-recursively
111-
func (s *GCSObjectClient) List(ctx context.Context, prefix string) ([]chunk.StorageObject, []chunk.StorageCommonPrefix, error) {
108+
// List implements chunk.ObjectClient.
109+
func (s *GCSObjectClient) List(ctx context.Context, prefix, delimiter string) ([]chunk.StorageObject, []chunk.StorageCommonPrefix, error) {
112110
var storageObjects []chunk.StorageObject
113111
var commonPrefixes []chunk.StorageCommonPrefix
114112

115-
iter := s.bucket.Objects(ctx, &storage.Query{Prefix: prefix, Delimiter: s.delimiter})
113+
iter := s.bucket.Objects(ctx, &storage.Query{Prefix: prefix, Delimiter: delimiter})
116114
for {
117115
if ctx.Err() != nil {
118116
return nil, nil, ctx.Err()
@@ -155,7 +153,3 @@ func (s *GCSObjectClient) DeleteObject(ctx context.Context, objectKey string) er
155153

156154
return nil
157155
}
158-
159-
func (s *GCSObjectClient) PathSeparator() string {
160-
return s.delimiter
161-
}

pkg/chunk/inmemory_storage_client.go

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -425,7 +425,8 @@ func (m *MockStorage) DeleteObject(ctx context.Context, objectKey string) error
425425
return nil
426426
}
427427

428-
func (m *MockStorage) List(ctx context.Context, prefix string) ([]StorageObject, []StorageCommonPrefix, error) {
428+
// List implements chunk.ObjectClient.
429+
func (m *MockStorage) List(ctx context.Context, prefix, delimiter string) ([]StorageObject, []StorageCommonPrefix, error) {
429430
m.mtx.RLock()
430431
defer m.mtx.RUnlock()
431432

@@ -442,10 +443,6 @@ func (m *MockStorage) List(ctx context.Context, prefix string) ([]StorageObject,
442443
return storageObjects, []StorageCommonPrefix{}, nil
443444
}
444445

445-
func (m *MockStorage) PathSeparator() string {
446-
return DirDelim
447-
}
448-
449446
type mockWriteBatch struct {
450447
inserts []struct {
451448
tableName, hashValue string

0 commit comments

Comments
 (0)