From 8ce5ae0330e42787614365288f53736ec6bc1f3c Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Thu, 21 Mar 2024 08:37:00 +0530 Subject: [PATCH 1/7] Paginated deletes --- sdk/storage/azdatalake/CHANGELOG.md | 1 + sdk/storage/azdatalake/directory/client.go | 11 ++- .../azdatalake/directory/client_test.go | 69 +++++++++++++++++++ .../azdatalake/internal/generated/autorest.md | 14 ++++ .../internal/generated/constants.go | 9 +++ .../generated/zz_filesystem_client.go | 24 +++---- .../internal/generated/zz_path_client.go | 48 ++++++------- .../internal/generated/zz_service_client.go | 4 +- .../azdatalake/internal/path/models.go | 4 ++ .../internal/testcommon/clients_auth.go | 1 + sdk/storage/azdatalake/test-resources.json | 5 ++ 11 files changed, 149 insertions(+), 41 deletions(-) create mode 100644 sdk/storage/azdatalake/internal/generated/constants.go diff --git a/sdk/storage/azdatalake/CHANGELOG.md b/sdk/storage/azdatalake/CHANGELOG.md index b0208b151dd8..3b6d21b34f0c 100644 --- a/sdk/storage/azdatalake/CHANGELOG.md +++ b/sdk/storage/azdatalake/CHANGELOG.md @@ -8,6 +8,7 @@ * Append API with acquire lease, release lease and renewal of lease support. * Flush API bundled with release lease option. * HNS Encryption Context support +* Pagination Support for recursive directory deletion ### Breaking Changes diff --git a/sdk/storage/azdatalake/directory/client.go b/sdk/storage/azdatalake/directory/client.go index 2d9dfe6dfa9d..be93f3eb20c5 100644 --- a/sdk/storage/azdatalake/directory/client.go +++ b/sdk/storage/azdatalake/directory/client.go @@ -262,9 +262,14 @@ func (d *Client) Create(ctx context.Context, options *CreateOptions) (CreateResp // Delete deletes directory and any path under it. func (d *Client) Delete(ctx context.Context, options *DeleteOptions) (DeleteResponse, error) { lac, mac, deleteOpts := path.FormatDeleteOptions(options, true) - resp, err := d.generatedDirClientWithDFS().Delete(ctx, deleteOpts, lac, mac) - err = exported.ConvertToDFSError(err) - return resp, err + for { + resp, err := d.generatedDirClientWithDFS().Delete(ctx, deleteOpts, lac, mac) + if resp.Continuation == nil { + err = exported.ConvertToDFSError(err) + return resp, err + } + deleteOpts.Continuation = resp.Continuation + } } // GetProperties gets the properties of a directory. diff --git a/sdk/storage/azdatalake/directory/client_test.go b/sdk/storage/azdatalake/directory/client_test.go index 0c85757f0cda..315363a52473 100644 --- a/sdk/storage/azdatalake/directory/client_test.go +++ b/sdk/storage/azdatalake/directory/client_test.go @@ -8,7 +8,9 @@ package directory_test import ( "context" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake" "net/http" + "strconv" "testing" "time" @@ -767,6 +769,73 @@ func (s *RecordedTestSuite) TestDeleteDirWithNilAccessConditions() { _require.NotNil(resp) } +func (s *UnrecordedTestSuite) TestDeleteDirWithPaginatedDelete() { + _require := require.New(s.T()) + testName := s.T().Name() + user := "user" + readWriteExecutePermission := "rwx" + + objectId, err := testcommon.GetRequiredEnv(testcommon.ObjectIdEnvVar) + _require.Nil(err) + + accountName, accountKey := testcommon.GetGenericAccountInfo(testcommon.TestAccountDatalake) + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + rootDirectory := fsClient.NewDirectoryClient("/") + + dirName := testcommon.GenerateDirName(testName) + dirURL := "https://" + accountName + ".dfs.core.windows.net/" + filesystemName + "/" + dirName + credential, err := azdatalake.NewSharedKeyCredential(accountName, accountKey) + + dirClient, err := directory.NewClientWithSharedKeyCredential(dirURL, credential, nil) + _require.NoError(err) + + resp, err := dirClient.Create(context.Background(), nil) + _require.NoError(err) + _require.NotNil(resp) + + for i := 0; i < 5020; i++ { + fileClient, err := dirClient.NewFileClient(testcommon.GenerateFileName(testName) + strconv.Itoa(i)) + _require.NoError(err) + _require.NotNil(fileClient) + + _, err = fileClient.Create(context.Background(), nil) + _require.NoError(err) + } + + accessControlResp, err := dirClient.GetAccessControl(context.Background(), nil) + _require.NoError(err) + + newAcl := *accessControlResp.ACL + "," + user + ":" + objectId + ":" + readWriteExecutePermission + + _, err = rootDirectory.SetAccessControlRecursive(context.Background(), newAcl, nil) + _require.NoError(err) + + cred, err := testcommon.GetGenericTokenCredential() + _require.NoError(err) + + directoryURL := "https://" + accountName + ".dfs.core.windows.net/" + filesystemName + "/" + dirName + + newDirClient, err := directory.NewClient(directoryURL, cred, nil) + _require.NoError(err) + + deleteOpts := &directory.DeleteOptions{ + Paginated: to.Ptr(true), + } + + response, err := newDirClient.Delete(context.Background(), deleteOpts) + _require.NoError(err) + _require.Nil(response.Continuation) + _require.NotNil(response) +} + func (s *RecordedTestSuite) TestDeleteDirIfModifiedSinceTrue() { _require := require.New(s.T()) testName := s.T().Name() diff --git a/sdk/storage/azdatalake/internal/generated/autorest.md b/sdk/storage/azdatalake/internal/generated/autorest.md index 9ffc88ee61a5..7cbcffe65522 100644 --- a/sdk/storage/azdatalake/internal/generated/autorest.md +++ b/sdk/storage/azdatalake/internal/generated/autorest.md @@ -298,3 +298,17 @@ directive: replace(/err = unpopulate\((.*), "ContentLength", &p\.ContentLength\)/g, 'var rawVal string\nerr = unpopulate(val, "ContentLength", &rawVal)\nintVal, _ := strconv.ParseInt(rawVal, 10, 64)\np.ContentLength = &intVal'). replace(/err = unpopulate\((.*), "IsDirectory", &p\.IsDirectory\)/g, 'var rawVal string\nerr = unpopulate(val, "IsDirectory", &rawVal)\nboolVal, _ := strconv.ParseBool(rawVal)\np.IsDirectory = &boolVal'); ``` + +### Updating service version to 2023-08-03 +```yaml +directive: +- from: + - zz_service_client.go + - zz_filesystem_client.go + - zz_path_client.go + where: $ + transform: >- + return $. + replaceAll(`[]string{"2023-05-03"}`, `[]string{ServiceVersion}`). + replaceAll(`2023-05-03`, `2023-08-03`); +``` diff --git a/sdk/storage/azdatalake/internal/generated/constants.go b/sdk/storage/azdatalake/internal/generated/constants.go new file mode 100644 index 000000000000..8c13c44116b6 --- /dev/null +++ b/sdk/storage/azdatalake/internal/generated/constants.go @@ -0,0 +1,9 @@ +//go:build go1.18 +// +build go1.18 + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +package generated + +const ServiceVersion = "2023-08-03" diff --git a/sdk/storage/azdatalake/internal/generated/zz_filesystem_client.go b/sdk/storage/azdatalake/internal/generated/zz_filesystem_client.go index f1c396e6c45f..32e338dd9918 100644 --- a/sdk/storage/azdatalake/internal/generated/zz_filesystem_client.go +++ b/sdk/storage/azdatalake/internal/generated/zz_filesystem_client.go @@ -32,7 +32,7 @@ type FileSystemClient struct { // operation does not support conditional HTTP requests. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2023-05-03 +// Generated from API version 2023-08-03 // - options - FileSystemClientCreateOptions contains the optional parameters for the FileSystemClient.Create method. func (client *FileSystemClient) Create(ctx context.Context, options *FileSystemClientCreateOptions) (FileSystemClientCreateResponse, error) { req, err := client.createCreateRequest(ctx, options) @@ -64,7 +64,7 @@ func (client *FileSystemClient) createCreateRequest(ctx context.Context, options if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2023-05-03"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if options != nil && options.Properties != nil { req.Raw().Header["x-ms-properties"] = []string{*options.Properties} } @@ -114,7 +114,7 @@ func (client *FileSystemClient) createHandleResponse(resp *http.Response) (FileS // [https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations]. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2023-05-03 +// Generated from API version 2023-08-03 // - options - FileSystemClientDeleteOptions contains the optional parameters for the FileSystemClient.Delete method. // - ModifiedAccessConditions - ModifiedAccessConditions contains a group of parameters for the FileSystemClient.SetProperties // method. @@ -148,7 +148,7 @@ func (client *FileSystemClient) deleteCreateRequest(ctx context.Context, options if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2023-05-03"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if modifiedAccessConditions != nil && modifiedAccessConditions.IfModifiedSince != nil { req.Raw().Header["If-Modified-Since"] = []string{(*modifiedAccessConditions.IfModifiedSince).In(gmt).Format(time.RFC1123)} } @@ -181,7 +181,7 @@ func (client *FileSystemClient) deleteHandleResponse(resp *http.Response) (FileS // GetProperties - All system and user-defined filesystem properties are specified in the response headers. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2023-05-03 +// Generated from API version 2023-08-03 // - options - FileSystemClientGetPropertiesOptions contains the optional parameters for the FileSystemClient.GetProperties // method. func (client *FileSystemClient) GetProperties(ctx context.Context, options *FileSystemClientGetPropertiesOptions) (FileSystemClientGetPropertiesResponse, error) { @@ -214,7 +214,7 @@ func (client *FileSystemClient) getPropertiesCreateRequest(ctx context.Context, if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2023-05-03"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} req.Raw().Header["Accept"] = []string{"application/json"} return req, nil } @@ -256,7 +256,7 @@ func (client *FileSystemClient) getPropertiesHandleResponse(resp *http.Response) // NewListBlobHierarchySegmentPager - The List Blobs operation returns a list of the blobs under the specified container // -// Generated from API version 2023-05-03 +// Generated from API version 2023-08-03 // - options - FileSystemClientListBlobHierarchySegmentOptions contains the optional parameters for the FileSystemClient.NewListBlobHierarchySegmentPager // method. // @@ -291,7 +291,7 @@ func (client *FileSystemClient) ListBlobHierarchySegmentCreateRequest(ctx contex reqQP.Set("timeout", strconv.FormatInt(int64(*options.Timeout), 10)) } req.Raw().URL.RawQuery = reqQP.Encode() - req.Raw().Header["x-ms-version"] = []string{"2023-05-03"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } @@ -329,7 +329,7 @@ func (client *FileSystemClient) ListBlobHierarchySegmentHandleResponse(resp *htt // NewListPathsPager - List FileSystem paths and their properties. // -// Generated from API version 2023-05-03 +// Generated from API version 2023-08-03 // - recursive - Required // - options - FileSystemClientListPathsOptions contains the optional parameters for the FileSystemClient.NewListPathsPager // method. @@ -362,7 +362,7 @@ func (client *FileSystemClient) ListPathsCreateRequest(ctx context.Context, recu if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2023-05-03"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} req.Raw().Header["Accept"] = []string{"application/json"} return req, nil } @@ -407,7 +407,7 @@ func (client *FileSystemClient) ListPathsHandleResponse(resp *http.Response) (Fi // [https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations]. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2023-05-03 +// Generated from API version 2023-08-03 // - options - FileSystemClientSetPropertiesOptions contains the optional parameters for the FileSystemClient.SetProperties // method. // - ModifiedAccessConditions - ModifiedAccessConditions contains a group of parameters for the FileSystemClient.SetProperties @@ -442,7 +442,7 @@ func (client *FileSystemClient) setPropertiesCreateRequest(ctx context.Context, if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2023-05-03"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if options != nil && options.Properties != nil { req.Raw().Header["x-ms-properties"] = []string{*options.Properties} } diff --git a/sdk/storage/azdatalake/internal/generated/zz_path_client.go b/sdk/storage/azdatalake/internal/generated/zz_path_client.go index 2160a94f678b..0868238f7ed0 100644 --- a/sdk/storage/azdatalake/internal/generated/zz_path_client.go +++ b/sdk/storage/azdatalake/internal/generated/zz_path_client.go @@ -32,7 +32,7 @@ type PathClient struct { // AppendData - Append data to the file. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2023-05-03 +// Generated from API version 2023-08-03 // - body - Initial data // - options - PathClientAppendDataOptions contains the optional parameters for the PathClient.AppendData method. // - PathHTTPHeaders - PathHTTPHeaders contains a group of parameters for the PathClient.Create method. @@ -95,7 +95,7 @@ func (client *PathClient) appendDataCreateRequest(ctx context.Context, body io.R if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2023-05-03"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if cpkInfo != nil && cpkInfo.EncryptionKey != nil { req.Raw().Header["x-ms-encryption-key"] = []string{*cpkInfo.EncryptionKey} } @@ -176,7 +176,7 @@ func (client *PathClient) appendDataHandleResponse(resp *http.Response) (PathCli // If-None-Match: "*". // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2023-05-03 +// Generated from API version 2023-08-03 // - options - PathClientCreateOptions contains the optional parameters for the PathClient.Create method. // - PathHTTPHeaders - PathHTTPHeaders contains a group of parameters for the PathClient.Create method. // - LeaseAccessConditions - LeaseAccessConditions contains a group of parameters for the PathClient.Create method. @@ -223,7 +223,7 @@ func (client *PathClient) createCreateRequest(ctx context.Context, options *Path if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2023-05-03"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if pathHTTPHeaders != nil && pathHTTPHeaders.CacheControl != nil { req.Raw().Header["x-ms-cache-control"] = []string{*pathHTTPHeaders.CacheControl} } @@ -372,7 +372,7 @@ func (client *PathClient) createHandleResponse(resp *http.Response) (PathClientC // [https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations]. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2023-05-03 +// Generated from API version 2023-08-03 // - options - PathClientDeleteOptions contains the optional parameters for the PathClient.Delete method. // - LeaseAccessConditions - LeaseAccessConditions contains a group of parameters for the PathClient.Create method. // - ModifiedAccessConditions - ModifiedAccessConditions contains a group of parameters for the FileSystemClient.SetProperties @@ -415,7 +415,7 @@ func (client *PathClient) deleteCreateRequest(ctx context.Context, options *Path if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2023-05-03"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if leaseAccessConditions != nil && leaseAccessConditions.LeaseID != nil { req.Raw().Header["x-ms-lease-id"] = []string{*leaseAccessConditions.LeaseID} } @@ -472,7 +472,7 @@ func (client *PathClient) deleteHandleResponse(resp *http.Response) (PathClientD // FlushData - Set the owner, group, permissions, or access control list for a path. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2023-05-03 +// Generated from API version 2023-08-03 // - options - PathClientFlushDataOptions contains the optional parameters for the PathClient.FlushData method. // - PathHTTPHeaders - PathHTTPHeaders contains a group of parameters for the PathClient.Create method. // - LeaseAccessConditions - LeaseAccessConditions contains a group of parameters for the PathClient.Create method. @@ -563,7 +563,7 @@ func (client *PathClient) flushDataCreateRequest(ctx context.Context, options *P if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2023-05-03"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if cpkInfo != nil && cpkInfo.EncryptionKey != nil { req.Raw().Header["x-ms-encryption-key"] = []string{*cpkInfo.EncryptionKey} } @@ -640,7 +640,7 @@ func (client *PathClient) flushDataHandleResponse(resp *http.Response) (PathClie // [https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations]. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2023-05-03 +// Generated from API version 2023-08-03 // - options - PathClientGetPropertiesOptions contains the optional parameters for the PathClient.GetProperties method. // - LeaseAccessConditions - LeaseAccessConditions contains a group of parameters for the PathClient.Create method. // - ModifiedAccessConditions - ModifiedAccessConditions contains a group of parameters for the FileSystemClient.SetProperties @@ -680,7 +680,7 @@ func (client *PathClient) getPropertiesCreateRequest(ctx context.Context, option if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2023-05-03"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if leaseAccessConditions != nil && leaseAccessConditions.LeaseID != nil { req.Raw().Header["x-ms-lease-id"] = []string{*leaseAccessConditions.LeaseID} } @@ -792,7 +792,7 @@ func (client *PathClient) getPropertiesHandleResponse(resp *http.Response) (Path // Operations [https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations]. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2023-05-03 +// Generated from API version 2023-08-03 // - xmsLeaseAction - There are five lease actions: "acquire", "break", "change", "renew", and "release". Use "acquire" and // specify the "x-ms-proposed-lease-id" and "x-ms-lease-duration" to acquire a new lease. Use "break" // to break an existing lease. When a lease is broken, the lease break period is allowed to elapse, during which time no lease @@ -835,7 +835,7 @@ func (client *PathClient) leaseCreateRequest(ctx context.Context, xmsLeaseAction if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2023-05-03"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} req.Raw().Header["x-ms-lease-action"] = []string{string(xmsLeaseAction)} req.Raw().Header["x-ms-lease-duration"] = []string{strconv.FormatInt(int64(client.xmsLeaseDuration), 10)} if options != nil && options.XMSLeaseBreakPeriod != nil { @@ -903,7 +903,7 @@ func (client *PathClient) leaseHandleResponse(resp *http.Response) (PathClientLe // Service Operations [https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations]. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2023-05-03 +// Generated from API version 2023-08-03 // - options - PathClientReadOptions contains the optional parameters for the PathClient.Read method. // - LeaseAccessConditions - LeaseAccessConditions contains a group of parameters for the PathClient.Create method. // - ModifiedAccessConditions - ModifiedAccessConditions contains a group of parameters for the FileSystemClient.SetProperties @@ -939,7 +939,7 @@ func (client *PathClient) readCreateRequest(ctx context.Context, options *PathCl if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2023-05-03"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if options != nil && options.Range != nil { req.Raw().Header["Range"] = []string{*options.Range} } @@ -1065,7 +1065,7 @@ func (client *PathClient) readHandleResponse(resp *http.Response) (PathClientRea // SetAccessControl - Set the owner, group, permissions, or access control list for a path. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2023-05-03 +// Generated from API version 2023-08-03 // - options - PathClientSetAccessControlOptions contains the optional parameters for the PathClient.SetAccessControl method. // - LeaseAccessConditions - LeaseAccessConditions contains a group of parameters for the PathClient.Create method. // - ModifiedAccessConditions - ModifiedAccessConditions contains a group of parameters for the FileSystemClient.SetProperties @@ -1127,7 +1127,7 @@ func (client *PathClient) setAccessControlCreateRequest(ctx context.Context, opt if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2023-05-03"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} req.Raw().Header["Accept"] = []string{"application/json"} return req, nil } @@ -1167,7 +1167,7 @@ func (client *PathClient) setAccessControlHandleResponse(resp *http.Response) (P // SetAccessControlRecursive - Set the access control list for a path and sub-paths. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2023-05-03 +// Generated from API version 2023-08-03 // - mode - Mode "set" sets POSIX access control rights on files and directories, "modify" modifies one or more POSIX access // control rights that pre-exist on files and directories, "remove" removes one or more // POSIX access control rights that were present earlier on files and directories @@ -1216,7 +1216,7 @@ func (client *PathClient) SetAccessControlRecursiveCreateRequest(ctx context.Con if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2023-05-03"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} req.Raw().Header["Accept"] = []string{"application/json"} return req, nil } @@ -1252,7 +1252,7 @@ func (client *PathClient) SetAccessControlRecursiveHandleResponse(resp *http.Res // SetExpiry - Sets the time a blob will expire and be deleted. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2023-05-03 +// Generated from API version 2023-08-03 // - expiryOptions - Required. Indicates mode of the expiry time // - options - PathClientSetExpiryOptions contains the optional parameters for the PathClient.SetExpiry method. func (client *PathClient) SetExpiry(ctx context.Context, expiryOptions ExpiryOptions, options *PathClientSetExpiryOptions) (PathClientSetExpiryResponse, error) { @@ -1282,7 +1282,7 @@ func (client *PathClient) setExpiryCreateRequest(ctx context.Context, expiryOpti reqQP.Set("timeout", strconv.FormatInt(int64(*options.Timeout), 10)) } req.Raw().URL.RawQuery = reqQP.Encode() - req.Raw().Header["x-ms-version"] = []string{"2023-05-03"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } @@ -1329,7 +1329,7 @@ func (client *PathClient) setExpiryHandleResponse(resp *http.Response) (PathClie // Undelete - Undelete a path that was previously soft deleted // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2023-05-03 +// Generated from API version 2023-08-03 // - options - PathClientUndeleteOptions contains the optional parameters for the PathClient.Undelete method. func (client *PathClient) Undelete(ctx context.Context, options *PathClientUndeleteOptions) (PathClientUndeleteResponse, error) { req, err := client.undeleteCreateRequest(ctx, options) @@ -1361,7 +1361,7 @@ func (client *PathClient) undeleteCreateRequest(ctx context.Context, options *Pa if options != nil && options.UndeleteSource != nil { req.Raw().Header["x-ms-undelete-source"] = []string{*options.UndeleteSource} } - req.Raw().Header["x-ms-version"] = []string{"2023-05-03"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } @@ -1401,7 +1401,7 @@ func (client *PathClient) undeleteHandleResponse(resp *http.Response) (PathClien // Headers for Blob Service Operations [https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations]. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2023-05-03 +// Generated from API version 2023-08-03 // - action - The action must be "append" to upload data to be appended to a file, "flush" to flush previously uploaded data // to a file, "setProperties" to set the properties of a file or directory, // "setAccessControl" to set the owner, group, permissions, or access control list for a file or directory, or "setAccessControlRecursive" @@ -1467,7 +1467,7 @@ func (client *PathClient) updateCreateRequest(ctx context.Context, action PathUp if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2023-05-03"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if options != nil && options.ContentLength != nil { req.Raw().Header["Content-Length"] = []string{strconv.FormatInt(*options.ContentLength, 10)} } diff --git a/sdk/storage/azdatalake/internal/generated/zz_service_client.go b/sdk/storage/azdatalake/internal/generated/zz_service_client.go index a70d131c2c27..88cdd31d9f46 100644 --- a/sdk/storage/azdatalake/internal/generated/zz_service_client.go +++ b/sdk/storage/azdatalake/internal/generated/zz_service_client.go @@ -29,7 +29,7 @@ type ServiceClient struct { // NewListFileSystemsPager - List filesystems and their properties in given account. // -// Generated from API version 2023-05-03 +// Generated from API version 2023-08-03 // - options - ServiceClientListFileSystemsOptions contains the optional parameters for the ServiceClient.NewListFileSystemsPager // method. // @@ -57,7 +57,7 @@ func (client *ServiceClient) ListFileSystemsCreateRequest(ctx context.Context, o if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2023-05-03"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} req.Raw().Header["Accept"] = []string{"application/json"} return req, nil } diff --git a/sdk/storage/azdatalake/internal/path/models.go b/sdk/storage/azdatalake/internal/path/models.go index 64cb04cc1aa5..bdf1e7e6b177 100644 --- a/sdk/storage/azdatalake/internal/path/models.go +++ b/sdk/storage/azdatalake/internal/path/models.go @@ -19,6 +19,7 @@ import ( type DeleteOptions struct { // AccessConditions contains parameters for accessing the path. AccessConditions *AccessConditions + Paginated *bool } func FormatDeleteOptions(o *DeleteOptions, recursive bool) (*generated.LeaseAccessConditions, *generated.ModifiedAccessConditions, *generated.PathClientDeleteOptions) { @@ -28,6 +29,9 @@ func FormatDeleteOptions(o *DeleteOptions, recursive bool) (*generated.LeaseAcce if o == nil { return nil, nil, deleteOpts } + if o.Paginated != nil { + deleteOpts.Paginated = o.Paginated + } leaseAccessConditions, modifiedAccessConditions := exported.FormatPathAccessConditions(o.AccessConditions) return leaseAccessConditions, modifiedAccessConditions, deleteOpts } diff --git a/sdk/storage/azdatalake/internal/testcommon/clients_auth.go b/sdk/storage/azdatalake/internal/testcommon/clients_auth.go index b4587455aa88..d29158b91330 100644 --- a/sdk/storage/azdatalake/internal/testcommon/clients_auth.go +++ b/sdk/storage/azdatalake/internal/testcommon/clients_auth.go @@ -36,6 +36,7 @@ const ( DataLakeEncryptionScopeEnvVar = "DATALAKE_AZURE_STORAGE_ENCRYPTION_SCOPE" SubscriptionID = "SUBSCRIPTION_ID" ResourceGroupName = "RESOURCE_GROUP_NAME" + ObjectIdEnvVar = "AZURE_APP_OBJECT_ID" ) const ( diff --git a/sdk/storage/azdatalake/test-resources.json b/sdk/storage/azdatalake/test-resources.json index a85273bfea5d..54cac38795be 100644 --- a/sdk/storage/azdatalake/test-resources.json +++ b/sdk/storage/azdatalake/test-resources.json @@ -33,6 +33,7 @@ "dataLakeEncryptionScopeName": "encryptionScope", "dataLakeEncryptionScope": "[concat(parameters('baseName'), 'dtlk', concat('/', variables('dataLakeEncryptionScopeName')))]", "softDeleteAccountName": "[concat(parameters('baseName'), 'sftdl')]", + "objectId": "[parameters('objectId')]", "premiumFileAccountName": "[concat(parameters('baseName'), 'pfile')]", "webjobsPrimaryAccountName": "[concat(parameters('baseName'), 'wjprim')]", "webjobsSecondaryAccountName": "[concat(parameters('baseName'), 'wjsec')]", @@ -578,6 +579,10 @@ "DATALAKE_AZURE_STORAGE_ENCRYPTION_SCOPE": { "type": "string", "value": "[variables('dataLakeEncryptionScopeName')]" + }, + "AZURE_APP_OBJECT_ID": { + "type": "string", + "value": "[variables('ObjectId')]" } } } From 0d2f45c76acbcaa64e63fdc3f9f542ae1e6358e5 Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Thu, 21 Mar 2024 08:50:39 +0530 Subject: [PATCH 2/7] add error check --- sdk/storage/azdatalake/directory/client_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/storage/azdatalake/directory/client_test.go b/sdk/storage/azdatalake/directory/client_test.go index 315363a52473..76b9c0bca714 100644 --- a/sdk/storage/azdatalake/directory/client_test.go +++ b/sdk/storage/azdatalake/directory/client_test.go @@ -793,6 +793,7 @@ func (s *UnrecordedTestSuite) TestDeleteDirWithPaginatedDelete() { dirName := testcommon.GenerateDirName(testName) dirURL := "https://" + accountName + ".dfs.core.windows.net/" + filesystemName + "/" + dirName credential, err := azdatalake.NewSharedKeyCredential(accountName, accountKey) + _require.NoError(err) dirClient, err := directory.NewClientWithSharedKeyCredential(dirURL, credential, nil) _require.NoError(err) From 227a4263291faeb2113ae79efcd11358623abf45 Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Fri, 22 Mar 2024 11:50:14 +0530 Subject: [PATCH 3/7] removed env variable of object id --- sdk/storage/azdatalake/directory/client_test.go | 6 ++---- sdk/storage/azdatalake/internal/testcommon/clients_auth.go | 1 - sdk/storage/azdatalake/test-resources.json | 5 ----- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/sdk/storage/azdatalake/directory/client_test.go b/sdk/storage/azdatalake/directory/client_test.go index 76b9c0bca714..16de8ee21a9e 100644 --- a/sdk/storage/azdatalake/directory/client_test.go +++ b/sdk/storage/azdatalake/directory/client_test.go @@ -775,9 +775,7 @@ func (s *UnrecordedTestSuite) TestDeleteDirWithPaginatedDelete() { user := "user" readWriteExecutePermission := "rwx" - objectId, err := testcommon.GetRequiredEnv(testcommon.ObjectIdEnvVar) - _require.Nil(err) - + objectId := "48965e63-fd19-4438-b9d0-dc585a8baf57" accountName, accountKey := testcommon.GetGenericAccountInfo(testcommon.TestAccountDatalake) filesystemName := testcommon.GenerateFileSystemName(testName) @@ -802,7 +800,7 @@ func (s *UnrecordedTestSuite) TestDeleteDirWithPaginatedDelete() { _require.NoError(err) _require.NotNil(resp) - for i := 0; i < 5020; i++ { + for i := 0; i < 5; i++ { fileClient, err := dirClient.NewFileClient(testcommon.GenerateFileName(testName) + strconv.Itoa(i)) _require.NoError(err) _require.NotNil(fileClient) diff --git a/sdk/storage/azdatalake/internal/testcommon/clients_auth.go b/sdk/storage/azdatalake/internal/testcommon/clients_auth.go index d29158b91330..b4587455aa88 100644 --- a/sdk/storage/azdatalake/internal/testcommon/clients_auth.go +++ b/sdk/storage/azdatalake/internal/testcommon/clients_auth.go @@ -36,7 +36,6 @@ const ( DataLakeEncryptionScopeEnvVar = "DATALAKE_AZURE_STORAGE_ENCRYPTION_SCOPE" SubscriptionID = "SUBSCRIPTION_ID" ResourceGroupName = "RESOURCE_GROUP_NAME" - ObjectIdEnvVar = "AZURE_APP_OBJECT_ID" ) const ( diff --git a/sdk/storage/azdatalake/test-resources.json b/sdk/storage/azdatalake/test-resources.json index 54cac38795be..a85273bfea5d 100644 --- a/sdk/storage/azdatalake/test-resources.json +++ b/sdk/storage/azdatalake/test-resources.json @@ -33,7 +33,6 @@ "dataLakeEncryptionScopeName": "encryptionScope", "dataLakeEncryptionScope": "[concat(parameters('baseName'), 'dtlk', concat('/', variables('dataLakeEncryptionScopeName')))]", "softDeleteAccountName": "[concat(parameters('baseName'), 'sftdl')]", - "objectId": "[parameters('objectId')]", "premiumFileAccountName": "[concat(parameters('baseName'), 'pfile')]", "webjobsPrimaryAccountName": "[concat(parameters('baseName'), 'wjprim')]", "webjobsSecondaryAccountName": "[concat(parameters('baseName'), 'wjsec')]", @@ -579,10 +578,6 @@ "DATALAKE_AZURE_STORAGE_ENCRYPTION_SCOPE": { "type": "string", "value": "[variables('dataLakeEncryptionScopeName')]" - }, - "AZURE_APP_OBJECT_ID": { - "type": "string", - "value": "[variables('ObjectId')]" } } } From 77e5735cb33b33e23d4cec2134f305a2269fdbbc Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Fri, 22 Mar 2024 13:47:10 +0530 Subject: [PATCH 4/7] address comments --- sdk/storage/azdatalake/directory/client.go | 2 +- sdk/storage/azdatalake/directory/client_test.go | 9 ++++++--- sdk/storage/azdatalake/internal/path/models.go | 4 +--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/sdk/storage/azdatalake/directory/client.go b/sdk/storage/azdatalake/directory/client.go index be93f3eb20c5..02671d07222e 100644 --- a/sdk/storage/azdatalake/directory/client.go +++ b/sdk/storage/azdatalake/directory/client.go @@ -264,7 +264,7 @@ func (d *Client) Delete(ctx context.Context, options *DeleteOptions) (DeleteResp lac, mac, deleteOpts := path.FormatDeleteOptions(options, true) for { resp, err := d.generatedDirClientWithDFS().Delete(ctx, deleteOpts, lac, mac) - if resp.Continuation == nil { + if resp.Continuation == nil || err != nil { err = exported.ConvertToDFSError(err) return resp, err } diff --git a/sdk/storage/azdatalake/directory/client_test.go b/sdk/storage/azdatalake/directory/client_test.go index 16de8ee21a9e..a9de337c3ed8 100644 --- a/sdk/storage/azdatalake/directory/client_test.go +++ b/sdk/storage/azdatalake/directory/client_test.go @@ -769,7 +769,10 @@ func (s *RecordedTestSuite) TestDeleteDirWithNilAccessConditions() { _require.NotNil(resp) } -func (s *UnrecordedTestSuite) TestDeleteDirWithPaginatedDelete() { +// To run this test, the NamespaceTenant AAD info needs to be set to an AAD app that does not have any RBAC permissions, +// and entityId needs to be set to the entity ID of the application. +func (s *RecordedTestSuite) TestDeleteDirWithPaginatedDelete() { + s.T().Skip("AAD app not configured for this test, this will be skipped") _require := require.New(s.T()) testName := s.T().Name() user := "user" @@ -800,7 +803,7 @@ func (s *UnrecordedTestSuite) TestDeleteDirWithPaginatedDelete() { _require.NoError(err) _require.NotNil(resp) - for i := 0; i < 5; i++ { + for i := 0; i < 5020; i++ { fileClient, err := dirClient.NewFileClient(testcommon.GenerateFileName(testName) + strconv.Itoa(i)) _require.NoError(err) _require.NotNil(fileClient) @@ -809,7 +812,7 @@ func (s *UnrecordedTestSuite) TestDeleteDirWithPaginatedDelete() { _require.NoError(err) } - accessControlResp, err := dirClient.GetAccessControl(context.Background(), nil) + accessControlResp, err := rootDirectory.GetAccessControl(context.Background(), nil) _require.NoError(err) newAcl := *accessControlResp.ACL + "," + user + ":" + objectId + ":" + readWriteExecutePermission diff --git a/sdk/storage/azdatalake/internal/path/models.go b/sdk/storage/azdatalake/internal/path/models.go index bdf1e7e6b177..96ab9f55def6 100644 --- a/sdk/storage/azdatalake/internal/path/models.go +++ b/sdk/storage/azdatalake/internal/path/models.go @@ -29,9 +29,7 @@ func FormatDeleteOptions(o *DeleteOptions, recursive bool) (*generated.LeaseAcce if o == nil { return nil, nil, deleteOpts } - if o.Paginated != nil { - deleteOpts.Paginated = o.Paginated - } + deleteOpts.Paginated = o.Paginated leaseAccessConditions, modifiedAccessConditions := exported.FormatPathAccessConditions(o.AccessConditions) return leaseAccessConditions, modifiedAccessConditions, deleteOpts } From 154c5e73d55729b296428e8e692749e0438ae68f Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Fri, 22 Mar 2024 14:15:36 +0530 Subject: [PATCH 5/7] recordings --- sdk/storage/azdatalake/assets.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/azdatalake/assets.json b/sdk/storage/azdatalake/assets.json index 1d3e6dc5aeaf..8058fab5265c 100644 --- a/sdk/storage/azdatalake/assets.json +++ b/sdk/storage/azdatalake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "go", "TagPrefix": "go/storage/azdatalake", - "Tag": "go/storage/azdatalake_daa40618fa" + "Tag": "go/storage/azdatalake_001d68d962" } From 7a1886c62867d23bd280fe6b2c82ded6c6545118 Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Fri, 22 Mar 2024 14:52:58 +0530 Subject: [PATCH 6/7] skip this test in live mode --- sdk/storage/azdatalake/directory/client_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdk/storage/azdatalake/directory/client_test.go b/sdk/storage/azdatalake/directory/client_test.go index a9de337c3ed8..613da4354003 100644 --- a/sdk/storage/azdatalake/directory/client_test.go +++ b/sdk/storage/azdatalake/directory/client_test.go @@ -772,7 +772,10 @@ func (s *RecordedTestSuite) TestDeleteDirWithNilAccessConditions() { // To run this test, the NamespaceTenant AAD info needs to be set to an AAD app that does not have any RBAC permissions, // and entityId needs to be set to the entity ID of the application. func (s *RecordedTestSuite) TestDeleteDirWithPaginatedDelete() { - s.T().Skip("AAD app not configured for this test, this will be skipped") + + if recording.GetRecordMode() == recording.LiveMode { + s.T().Skip("AAD app not configured for this test, this will be skipped") + } _require := require.New(s.T()) testName := s.T().Name() user := "user" From 2f52c547476b5850c8baa248becc90c1f13f3c0e Mon Sep 17 00:00:00 2001 From: tanyasethi-msft <124860586+tanyasethi-msft@users.noreply.github.com> Date: Tue, 26 Mar 2024 11:00:03 +0530 Subject: [PATCH 7/7] ignore the test --- sdk/storage/azdatalake/directory/client_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sdk/storage/azdatalake/directory/client_test.go b/sdk/storage/azdatalake/directory/client_test.go index 613da4354003..3dcc454ed75c 100644 --- a/sdk/storage/azdatalake/directory/client_test.go +++ b/sdk/storage/azdatalake/directory/client_test.go @@ -773,15 +773,13 @@ func (s *RecordedTestSuite) TestDeleteDirWithNilAccessConditions() { // and entityId needs to be set to the entity ID of the application. func (s *RecordedTestSuite) TestDeleteDirWithPaginatedDelete() { - if recording.GetRecordMode() == recording.LiveMode { - s.T().Skip("AAD app not configured for this test, this will be skipped") - } + s.T().Skip("AAD app not configured for this test, this will be skipped") _require := require.New(s.T()) testName := s.T().Name() user := "user" readWriteExecutePermission := "rwx" - objectId := "48965e63-fd19-4438-b9d0-dc585a8baf57" + objectId := "" // object ID of an AAD app which has no RBAC permissions accountName, accountKey := testcommon.GetGenericAccountInfo(testcommon.TestAccountDatalake) filesystemName := testcommon.GenerateFileSystemName(testName)