diff --git a/sdk/storage/azblob/CHANGELOG.md b/sdk/storage/azblob/CHANGELOG.md index a9040137b0b2..c77b6ca69821 100644 --- a/sdk/storage/azblob/CHANGELOG.md +++ b/sdk/storage/azblob/CHANGELOG.md @@ -5,6 +5,7 @@ ### Features Added * Added support for [Cold tier](https://learn.microsoft.com/azure/storage/blobs/access-tiers-overview?tabs=azure-portal). * Added `CopySourceTag` option for `UploadBlobFromURLOptions` +* Added [FilterBlobs](https://learn.microsoft.com/rest/api/storageservices/find-blobs-by-tags-container) by tags API for container client. * Added `System` option to `ListContainersInclude` to allow listing of system containers. ### Breaking Changes diff --git a/sdk/storage/azblob/container/client.go b/sdk/storage/azblob/container/client.go index 243fe09b560e..3058b5d49c00 100644 --- a/sdk/storage/azblob/container/client.go +++ b/sdk/storage/azblob/container/client.go @@ -421,3 +421,12 @@ func (c *Client) SubmitBatch(ctx context.Context, bb *BatchBuilder, options *Sub Version: resp.Version, }, nil } + +// FilterBlobs operation finds all blobs in the container whose tags match a given search expression. +// https://docs.microsoft.com/en-us/rest/api/storageservices/find-blobs-by-tags-container +// eg. "dog='germanshepherd' and penguin='emperorpenguin'" +func (c *Client) FilterBlobs(ctx context.Context, where string, o *FilterBlobsOptions) (FilterBlobsResponse, error) { + containerClientFilterBlobsOptions := o.format() + resp, err := c.generated().FilterBlobs(ctx, where, containerClientFilterBlobsOptions) + return resp, err +} diff --git a/sdk/storage/azblob/container/client_test.go b/sdk/storage/azblob/container/client_test.go index 7777f320749f..32bbb8c8f920 100644 --- a/sdk/storage/azblob/container/client_test.go +++ b/sdk/storage/azblob/container/client_test.go @@ -9,6 +9,7 @@ package container_test import ( "context" "fmt" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/appendblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" "net/url" "os" @@ -2340,6 +2341,149 @@ func (s *ContainerUnrecordedTestsSuite) TestSASContainerClient() { _require.Nil(err) } +func (s *ContainerUnrecordedTestsSuite) TestFilterBlobsByTags() { + _require := require.New(s.T()) + testName := s.T().Name() + svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil) + _require.NoError(err) + + containerClient := testcommon.CreateNewContainer(context.Background(), _require, testcommon.GenerateContainerName(testName), svcClient) + defer testcommon.DeleteContainer(context.Background(), _require, containerClient) + + // Adding SAS and options + permissions := sas.ContainerPermissions{ + Read: true, + Add: true, + Write: true, + Create: true, + Delete: true, + Tag: true, + FilterByTags: true, + } + expiry := time.Now().Add(time.Hour) + + // ContainerSASURL is created with GetSASURL + sasUrl, err := containerClient.GetSASURL(permissions, expiry, nil) + _require.Nil(err) + + // Create container client with sasUrl + containerSasClient, err := container.NewClientWithNoCredential(sasUrl, nil) + _require.Nil(err) + + abClient := containerSasClient.NewAppendBlobClient(testcommon.GenerateBlobName(testName)) + + createAppendBlobOptions := appendblob.CreateOptions{ + Tags: testcommon.BasicBlobTagsMap, + } + createResp, err := abClient.Create(context.Background(), &createAppendBlobOptions) + _require.Nil(err) + _require.NotNil(createResp.VersionID) + time.Sleep(10 * time.Second) + + // Use container client to filter blobs by tag + where := "\"azure\"='blob'" + opts := container.FilterBlobsOptions{MaxResults: to.Ptr(int32(10)), Marker: to.Ptr("")} + lResp, err := containerSasClient.FilterBlobs(context.Background(), where, &opts) + _require.Nil(err) + _require.Len(lResp.FilterBlobSegment.Blobs[0].Tags.BlobTagSet, 1) + _require.Equal(*lResp.FilterBlobSegment.Blobs[0].Tags.BlobTagSet[0].Key, "azure") + _require.Equal(*lResp.FilterBlobSegment.Blobs[0].Tags.BlobTagSet[0].Value, "blob") +} + +func (s *ContainerUnrecordedTestsSuite) TestFilterBlobsByTagsNegative() { + _require := require.New(s.T()) + testName := s.T().Name() + svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil) + _require.NoError(err) + + containerClient := testcommon.CreateNewContainer(context.Background(), _require, testcommon.GenerateContainerName(testName), svcClient) + defer testcommon.DeleteContainer(context.Background(), _require, containerClient) + + // Adding SAS and options + permissions := sas.ContainerPermissions{ + Read: true, + Add: true, + Write: true, + Create: true, + Delete: true, + Tag: true, + } + expiry := time.Now().Add(time.Hour) + + // ContainerSASURL is created with GetSASURL + sasUrl, err := containerClient.GetSASURL(permissions, expiry, nil) + _require.Nil(err) + + // Create container client with sasUrl + containerSasClient, err := container.NewClientWithNoCredential(sasUrl, nil) + _require.Nil(err) + + abClient := containerSasClient.NewAppendBlobClient(testcommon.GenerateBlobName(testName)) + + createAppendBlobOptions := appendblob.CreateOptions{ + Tags: testcommon.BasicBlobTagsMap, + } + createResp, err := abClient.Create(context.Background(), &createAppendBlobOptions) + _require.Nil(err) + _require.NotNil(createResp.VersionID) + time.Sleep(10 * time.Second) + + // Use container client to filter blobs by tag + where := "\"azure\"='blob'" + _, err = containerSasClient.FilterBlobs(context.Background(), where, nil) + _require.Error(err) +} + +func (s *ContainerUnrecordedTestsSuite) TestFilterBlobsOnContainer() { + _require := require.New(s.T()) + testName := s.T().Name() + svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil) + _require.NoError(err) + + containerClient := testcommon.CreateNewContainer(context.Background(), _require, testcommon.GenerateContainerName(testName)+"1", svcClient) + defer testcommon.DeleteContainer(context.Background(), _require, containerClient) + + blobTagsMap1 := map[string]string{ + "tag2": "tagsecond", + "tag3": "tagthird", + } + blobTagsMap2 := map[string]string{ + "tag1": "firsttag", + "tag2": "secondtag", + "tag3": "thirdtag", + } + + blobName1 := testcommon.GenerateBlobName(testName) + "1" + blobClient1 := testcommon.CreateNewBlockBlob(context.Background(), _require, blobName1, containerClient) + _, err = blobClient1.SetTags(context.Background(), blobTagsMap1, nil) + _require.Nil(err) + + blobName2 := testcommon.GenerateBlobName(testName) + "2" + blobClient2 := testcommon.CreateNewBlockBlob(context.Background(), _require, blobName2, containerClient) + _, err = blobClient2.SetTags(context.Background(), blobTagsMap2, nil) + _require.Nil(err) + time.Sleep(10 * time.Second) + + blobTagsResp, err := blobClient2.GetTags(context.Background(), nil) + _require.Nil(err) + blobTagsSet := blobTagsResp.BlobTagSet + _require.NotNil(blobTagsSet) + + // Test invalid tag + where := "\"tag4\"='fourthtag'" + lResp, err := containerClient.FilterBlobs(context.Background(), where, nil) + _require.Nil(err) + _require.Equal(len(lResp.Blobs), 0) + + // Test multiple valid tags + where = "\"tag1\"='firsttag'AND\"tag2\"='secondtag'" + lResp, err = containerClient.FilterBlobs(context.Background(), where, nil) + _require.Nil(err) + _require.Len(lResp.FilterBlobSegment.Blobs[0].Tags.BlobTagSet, 2) + _require.Equal(lResp.FilterBlobSegment.Blobs[0].Tags.BlobTagSet[0], blobTagsSet[0]) + _require.Equal(lResp.FilterBlobSegment.Blobs[0].Tags.BlobTagSet[1], blobTagsSet[1]) +} + func (s *ContainerUnrecordedTestsSuite) TestSASContainerClientTags() { _require := require.New(s.T()) testName := s.T().Name() diff --git a/sdk/storage/azblob/container/models.go b/sdk/storage/azblob/container/models.go index f1724861ef9e..61d936ab73db 100644 --- a/sdk/storage/azblob/container/models.go +++ b/sdk/storage/azblob/container/models.go @@ -397,3 +397,31 @@ type SubmitBatchOptions struct { func (o *SubmitBatchOptions) format() *generated.ContainerClientSubmitBatchOptions { return nil } + +// --------------------------------------------------------------------------------------------------------------------- + +// FilterBlobsOptions provides set of options for Client.FilterBlobs. +type FilterBlobsOptions struct { + // A string value that identifies the portion of the list of containers to be returned with the next listing operation. The + // operation returns the NextMarker value within the response body if the listing + // operation did not return all containers remaining to be listed with the current page. The NextMarker value can be used + // as the value for the marker parameter in a subsequent call to request the next + // page of list items. The marker value is opaque to the client. + Marker *string + // Specifies the maximum number of containers to return. If the request does not specify maxresults, or specifies a value + // greater than 5000, the server will return up to 5000 items. Note that if the + // listing operation crosses a partition boundary, then the service will return a continuation token for retrieving the remainder + // of the results. For this reason, it is possible that the service will + // return fewer results than specified by maxresults, or than the default of 5000. + MaxResults *int32 +} + +func (o *FilterBlobsOptions) format() *generated.ContainerClientFilterBlobsOptions { + if o == nil { + return nil + } + return &generated.ContainerClientFilterBlobsOptions{ + Marker: o.Marker, + Maxresults: o.MaxResults, + } +} diff --git a/sdk/storage/azblob/container/responses.go b/sdk/storage/azblob/container/responses.go index 4d1e406ea22b..9aaefe277fb9 100644 --- a/sdk/storage/azblob/container/responses.go +++ b/sdk/storage/azblob/container/responses.go @@ -64,3 +64,6 @@ type SubmitBatchResponse struct { // BatchResponseItem contains the response for the individual sub-requests. type BatchResponseItem = exported.BatchResponseItem + +// FilterBlobsResponse contains the response from method Client.FilterBlobs. +type FilterBlobsResponse = generated.ContainerClientFilterBlobsResponse diff --git a/sdk/storage/azblob/internal/testcommon/clients_auth.go b/sdk/storage/azblob/internal/testcommon/clients_auth.go index c4179f73d05d..235c00220e9d 100644 --- a/sdk/storage/azblob/internal/testcommon/clients_auth.go +++ b/sdk/storage/azblob/internal/testcommon/clients_auth.go @@ -14,6 +14,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/appendblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" "strings" "testing" @@ -277,6 +278,18 @@ func CreateNewBlockBlobWithCPK(ctx context.Context, _require *require.Assertions return } +func GetAppendBlobClient(appendBlobName string, containerClient *container.Client) *appendblob.Client { + return containerClient.NewAppendBlobClient(appendBlobName) +} + +func CreateNewAppendBlob(ctx context.Context, _require *require.Assertions, appendBlobName string, containerClient *container.Client) *appendblob.Client { + abClient := GetAppendBlobClient(appendBlobName, containerClient) + + _, err := abClient.Create(ctx, nil) + _require.Nil(err) + return abClient +} + // Some tests require setting service properties. It can take up to 30 seconds for the new properties to be reflected across all FEs. // We will enable the necessary property and try to run the test implementation. If it fails with an error that should be due to // those changes not being reflected yet, we will wait 30 seconds and try the test again. If it fails this time for any reason, diff --git a/sdk/storage/azblob/sas/service.go b/sdk/storage/azblob/sas/service.go index b900e2b06d33..ba66d46f3557 100644 --- a/sdk/storage/azblob/sas/service.go +++ b/sdk/storage/azblob/sas/service.go @@ -55,13 +55,7 @@ func (v BlobSignatureValues) SignWithSharedKey(sharedKeyCredential *SharedKeyCre return QueryParameters{}, errors.New("service SAS is missing at least one of these: ExpiryTime or Permissions") } - //Make sure the permission characters are in the correct order - perms, err := parseBlobPermissions(v.Permissions) - if err != nil { - return QueryParameters{}, err - } - v.Permissions = perms.String() - + // Parse the resource resource := "c" if !v.SnapshotTime.IsZero() { resource = "bs" @@ -76,6 +70,21 @@ func (v BlobSignatureValues) SignWithSharedKey(sharedKeyCredential *SharedKeyCre resource = "b" } + // make sure the permission characters are in the correct order + if resource == "c" { + perms, err := parseContainerPermissions(v.Permissions) + if err != nil { + return QueryParameters{}, err + } + v.Permissions = perms.String() + } else { + perms, err := parseBlobPermissions(v.Permissions) + if err != nil { + return QueryParameters{}, err + } + v.Permissions = perms.String() + } + if v.Version == "" { v.Version = Version } diff --git a/sdk/storage/azblob/service/client_test.go b/sdk/storage/azblob/service/client_test.go index 57098e6c94a8..f823c58f2fac 100644 --- a/sdk/storage/azblob/service/client_test.go +++ b/sdk/storage/azblob/service/client_test.go @@ -10,6 +10,7 @@ import ( "context" "fmt" "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/appendblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/internal/exported" "io" @@ -1085,6 +1086,78 @@ func (s *ServiceRecordedTestsSuite) TestAccountFilterBlobs() { _require.Len(resp.FilterBlobSegment.Blobs, 0) } +func (s *ServiceUnrecordedTestsSuite) TestFilterBlobsTagsWithServiceSAS() { + _require := require.New(s.T()) + testName := s.T().Name() + cred, _ := testcommon.GetGenericSharedKeyCredential(testcommon.TestAccountDefault) + + serviceClient, err := service.NewClientWithSharedKeyCredential(fmt.Sprintf("https://%s.blob.core.windows.net/", cred.AccountName()), cred, nil) + _require.Nil(err) + + // Note: Always set all permissions, services, types to true to ensure order of string formed is correct. + resources := sas.AccountResourceTypes{ + Object: true, + Service: true, + Container: true, + } + permissions := sas.AccountPermissions{ + Read: true, + Write: true, + Delete: true, + DeletePreviousVersion: true, + List: true, + Add: true, + Create: true, + Update: true, + Process: true, + Tag: true, + FilterByTags: true, + PermanentDelete: true, + } + expiry := time.Now().Add(time.Hour) + sasUrl, err := serviceClient.GetSASURL(resources, permissions, expiry, nil) + _require.Nil(err) + + svcClient, err := testcommon.GetServiceClientNoCredential(s.T(), sasUrl, nil) + _require.Nil(err) + + containerName := testcommon.GenerateContainerName(testName) + containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient) + defer testcommon.DeleteContainer(context.Background(), _require, containerClient) + + abClient := testcommon.GetAppendBlobClient(testcommon.GenerateBlobName(testName), containerClient) + + createAppendBlobOptions := appendblob.CreateOptions{ + Tags: testcommon.SpecialCharBlobTagsMap, + } + createResp, err := abClient.Create(context.Background(), &createAppendBlobOptions) + _require.Nil(err) + _require.NotNil(createResp.VersionID) + time.Sleep(10 * time.Second) + + _, err = abClient.GetProperties(context.Background(), nil) + _require.Nil(err) + + blobGetTagsResponse, err := abClient.GetTags(context.Background(), nil) + _require.Nil(err) + blobTagsSet := blobGetTagsResponse.BlobTagSet + _require.NotNil(blobTagsSet) + _require.Len(blobTagsSet, len(testcommon.SpecialCharBlobTagsMap)) + for _, blobTag := range blobTagsSet { + _require.Equal(testcommon.SpecialCharBlobTagsMap[*blobTag.Key], *blobTag.Value) + } + + // Tags with spaces + where := "\"GO \"='.Net'" + lResp, err := svcClient.FilterBlobs(context.Background(), where, nil) + _require.Nil(err) + _require.Len(lResp.FilterBlobSegment.Blobs[0].Tags.BlobTagSet, 1) + _require.Equal(lResp.FilterBlobSegment.Blobs[0].Tags.BlobTagSet[0], blobTagsSet[2]) + + _, err = svcClient.DeleteContainer(context.Background(), containerName, nil) + _require.Nil(err) +} + func batchSetup(containerName string, svcClient *service.Client, bb *service.BatchBuilder, operationType exported.BlobBatchOperationType) ([]*container.Client, error) { var cntClients []*container.Client for i := 0; i < 5; i++ {