diff --git a/sdk/storage/azdatalake/CHANGELOG.md b/sdk/storage/azdatalake/CHANGELOG.md index 1fd06a3cb4ae..c63ec5d3a2ed 100644 --- a/sdk/storage/azdatalake/CHANGELOG.md +++ b/sdk/storage/azdatalake/CHANGELOG.md @@ -3,6 +3,16 @@ ## 1.1.2 (Unreleased) ### Features Added +* Append API Bundled with Flush functionality +* HNS Encryption Scope support +* Append API with acquire lease, release lease and renewal of lease support. +* Flush API bundled with release lease option. +* HNS Encryption Context support +* Pagination Support for recursive directory deletion +* Bundle ability to set permission, owner, group, acl, lease, expiry time and umask along with FileSystem.CreateFile and FileSystem.CreateDirectory APIs. +* Added support for AAD Audience when OAuth is used. +* Updated service version to `2023-11-03` +* Integrate `InsecureAllowCredentialWithHTTP` client options. ### Breaking Changes @@ -10,6 +20,7 @@ * Fixed an issue where GetSASURL() was providing HTTPS SAS, instead of the default http+https SAS. Fixes [#22448](https://github.com/Azure/azure-sdk-for-go/issues/22448) ### Other Changes +* Updated azcore version to `1.11.1` ## 1.1.1 (2024-02-29) diff --git a/sdk/storage/azdatalake/assets.json b/sdk/storage/azdatalake/assets.json index da2dfd857383..7c9f73ffb5c5 100644 --- a/sdk/storage/azdatalake/assets.json +++ b/sdk/storage/azdatalake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "go", "TagPrefix": "go/storage/azdatalake", - "Tag": "go/storage/azdatalake_6e4e5b7c87" + "Tag": "go/storage/azdatalake_8cf0ce4c24" } diff --git a/sdk/storage/azdatalake/directory/client.go b/sdk/storage/azdatalake/directory/client.go index 2d9dfe6dfa9d..c3cc6c30fd18 100644 --- a/sdk/storage/azdatalake/directory/client.go +++ b/sdk/storage/azdatalake/directory/client.go @@ -42,8 +42,9 @@ type Client base.CompositeClient[generated.PathClient, generated_blob.BlobClient func NewClient(directoryURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) { blobURL, directoryURL := shared.GetURLs(directoryURL) - authPolicy := runtime.NewBearerTokenPolicy(cred, []string{shared.TokenScope}, nil) + audience := base.GetAudience((*base.ClientOptions)(options)) conOptions := shared.GetClientOptions(options) + authPolicy := shared.NewStorageChallengePolicy(cred, audience, conOptions.InsecureAllowCredentialWithHTTP) plOpts := runtime.PipelineOptions{ PerRetry: []policy.Policy{authPolicy}, } @@ -262,9 +263,14 @@ func (d *Client) Create(ctx context.Context, options *CreateOptions) (CreateResp // Delete deletes directory and any path under it. func (d *Client) Delete(ctx context.Context, options *DeleteOptions) (DeleteResponse, error) { lac, mac, deleteOpts := path.FormatDeleteOptions(options, true) - resp, err := d.generatedDirClientWithDFS().Delete(ctx, deleteOpts, lac, mac) - err = exported.ConvertToDFSError(err) - return resp, err + for { + resp, err := d.generatedDirClientWithDFS().Delete(ctx, deleteOpts, lac, mac) + if resp.Continuation == nil || err != nil { + err = exported.ConvertToDFSError(err) + return resp, err + } + deleteOpts.Continuation = resp.Continuation + } } // GetProperties gets the properties of a directory. diff --git a/sdk/storage/azdatalake/directory/client_test.go b/sdk/storage/azdatalake/directory/client_test.go index 0c85757f0cda..6cad9521cd5c 100644 --- a/sdk/storage/azdatalake/directory/client_test.go +++ b/sdk/storage/azdatalake/directory/client_test.go @@ -8,7 +8,9 @@ package directory_test import ( "context" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake" "net/http" + "strconv" "testing" "time" @@ -767,6 +769,76 @@ func (s *RecordedTestSuite) TestDeleteDirWithNilAccessConditions() { _require.NotNil(resp) } +// To run this test, the NamespaceTenant AAD info needs to be set to an AAD app that does not have any RBAC permissions, +// and entityId needs to be set to the entity ID of the application. +func (s *RecordedTestSuite) TestDeleteDirWithPaginatedDelete() { + + s.T().Skip("AAD app not configured for this test, this will be skipped") + _require := require.New(s.T()) + testName := s.T().Name() + user := "user" + readWriteExecutePermission := "rwx" + + objectId := "" // object ID of an AAD app which has no RBAC permissions + accountName, accountKey := testcommon.GetGenericAccountInfo(testcommon.TestAccountDatalake) + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + rootDirectory := fsClient.NewDirectoryClient("/") + + dirName := testcommon.GenerateDirName(testName) + dirURL := "https://" + accountName + ".dfs.core.windows.net/" + filesystemName + "/" + dirName + credential, err := azdatalake.NewSharedKeyCredential(accountName, accountKey) + _require.NoError(err) + + dirClient, err := directory.NewClientWithSharedKeyCredential(dirURL, credential, nil) + _require.NoError(err) + + resp, err := dirClient.Create(context.Background(), nil) + _require.NoError(err) + _require.NotNil(resp) + + for i := 0; i < 5020; i++ { + fileClient, err := dirClient.NewFileClient(testcommon.GenerateFileName(testName) + strconv.Itoa(i)) + _require.NoError(err) + _require.NotNil(fileClient) + + _, err = fileClient.Create(context.Background(), nil) + _require.NoError(err) + } + + accessControlResp, err := rootDirectory.GetAccessControl(context.Background(), nil) + _require.NoError(err) + + newAcl := *accessControlResp.ACL + "," + user + ":" + objectId + ":" + readWriteExecutePermission + + _, err = rootDirectory.SetAccessControlRecursive(context.Background(), newAcl, nil) + _require.NoError(err) + + cred, err := testcommon.GetGenericTokenCredential() + _require.NoError(err) + + directoryURL := "https://" + accountName + ".dfs.core.windows.net/" + filesystemName + "/" + dirName + + newDirClient, err := directory.NewClient(directoryURL, cred, nil) + _require.NoError(err) + + deleteOpts := &directory.DeleteOptions{ + Paginated: to.Ptr(true), + } + + response, err := newDirClient.Delete(context.Background(), deleteOpts) + _require.NoError(err) + _require.Nil(response.Continuation) + _require.NotNil(response) +} + func (s *RecordedTestSuite) TestDeleteDirIfModifiedSinceTrue() { _require := require.New(s.T()) testName := s.T().Name() @@ -2850,3 +2922,73 @@ func (s *UnrecordedTestSuite) TestDirCreateDeleteUsingOAuth() { _, err = dirClient.GetProperties(context.Background(), nil) _require.NoError(err) } + +func (s *RecordedTestSuite) TestCreateDirectoryClientDefaultAudience() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + cred, err := testcommon.GetGenericTokenCredential() + _require.NoError(err) + + accountName, _ := testcommon.GetGenericAccountInfo(testcommon.TestAccountDatalake) + _require.Greater(len(accountName), 0) + + dirName := testcommon.GenerateDirName(testName) + dirURL := "https://" + accountName + ".dfs.core.windows.net/" + filesystemName + "/" + dirName + + options := &directory.ClientOptions{Audience: "https://storage.azure.com/"} + testcommon.SetClientOptions(s.T(), &options.ClientOptions) + + dirClient, err := directory.NewClient(dirURL, cred, options) + _require.NoError(err) + + _, err = dirClient.Create(context.Background(), nil) + _require.NoError(err) + + _, err = dirClient.GetProperties(context.Background(), nil) + _require.NoError(err) + +} + +func (s *RecordedTestSuite) TestCreateDirectoryClientCustomAudience() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + cred, err := testcommon.GetGenericTokenCredential() + _require.NoError(err) + + accountName, _ := testcommon.GetGenericAccountInfo(testcommon.TestAccountDatalake) + _require.Greater(len(accountName), 0) + + dirName := testcommon.GenerateDirName(testName) + dirURL := "https://" + accountName + ".dfs.core.windows.net/" + filesystemName + "/" + dirName + + options := &directory.ClientOptions{Audience: "https://" + accountName + ".blob.core.windows.net"} + testcommon.SetClientOptions(s.T(), &options.ClientOptions) + + dirClient, err := directory.NewClient(dirURL, cred, options) + _require.NoError(err) + + _, err = dirClient.Create(context.Background(), nil) + _require.NoError(err) + + _, err = dirClient.GetProperties(context.Background(), nil) + _require.NoError(err) + +} diff --git a/sdk/storage/azdatalake/file/client.go b/sdk/storage/azdatalake/file/client.go index 75dbd8fccbfb..42795df065ae 100644 --- a/sdk/storage/azdatalake/file/client.go +++ b/sdk/storage/azdatalake/file/client.go @@ -48,8 +48,9 @@ type Client base.CompositeClient[generated.PathClient, generated_blob.BlobClient // - options - client options; pass nil to accept the default values func NewClient(fileURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) { blobURL, fileURL := shared.GetURLs(fileURL) - authPolicy := runtime.NewBearerTokenPolicy(cred, []string{shared.TokenScope}, nil) + audience := base.GetAudience((*base.ClientOptions)(options)) conOptions := shared.GetClientOptions(options) + authPolicy := shared.NewStorageChallengePolicy(cred, audience, conOptions.InsecureAllowCredentialWithHTTP) plOpts := runtime.PipelineOptions{ PerRetry: []policy.Policy{authPolicy}, } @@ -411,13 +412,6 @@ func (f *Client) AppendData(ctx context.Context, offset int64, body io.ReadSeekC return AppendDataResponse{}, err } resp, err := f.generatedFileClientWithDFS().AppendData(ctx, body, appendDataOptions, nil, leaseAccessConditions, cpkInfo) - // TODO: check and uncomment this - //if err != nil { - // _, err1 := body.Seek(0, io.SeekStart) - // if err1 != nil { - // return AppendDataResponse{}, err1 - // } - //} return resp, exported.ConvertToDFSError(err) } @@ -451,6 +445,13 @@ func (f *Client) uploadFromReader(ctx context.Context, reader io.ReaderAt, actua } } + if o.EncryptionContext != nil { + _, err := f.Create(ctx, &CreateOptions{EncryptionContext: o.EncryptionContext}) + if err != nil { + return err + } + } + progress := int64(0) progressLock := &sync.Mutex{} @@ -490,6 +491,12 @@ func (f *Client) uploadFromReader(ctx context.Context, reader io.ReaderAt, actua }) if err != nil { + if o.EncryptionContext != nil { + _, err2 := f.Delete(ctx, nil) + if err2 != nil { + return exported.ConvertToDFSError(err2) + } + } return exported.ConvertToDFSError(err) } // All appends were successful, call to flush @@ -527,7 +534,20 @@ func (f *Client) UploadStream(ctx context.Context, body io.Reader, options *Uplo options = &UploadStreamOptions{} } + if options.EncryptionContext != nil { + _, err := f.Create(ctx, &CreateOptions{EncryptionContext: options.EncryptionContext}) + if err != nil { + return err + } + } err := copyFromReader(ctx, body, f, *options, newMMBPool) + + if err != nil && options.EncryptionContext != nil { + _, err2 := f.Delete(ctx, nil) + if err2 != nil { + return exported.ConvertToDFSError(err2) + } + } return exported.ConvertToDFSError(err) } @@ -538,8 +558,10 @@ func (f *Client) DownloadStream(ctx context.Context, o *DownloadStreamOptions) ( o = &DownloadStreamOptions{} } opts := o.format() - resp, err := f.blobClient().DownloadStream(ctx, opts) - newResp := FormatDownloadStreamResponse(&resp) + var respFromCtx *http.Response + ctxWithResp := shared.WithCaptureBlobResponse(ctx, &respFromCtx) + resp, err := f.blobClient().DownloadStream(ctxWithResp, opts) + newResp := FormatDownloadStreamResponse(&resp, respFromCtx) fullResp := DownloadStreamResponse{ client: f, DownloadResponse: newResp, diff --git a/sdk/storage/azdatalake/file/client_test.go b/sdk/storage/azdatalake/file/client_test.go index fbbd1e920e9e..802cdcb80d73 100644 --- a/sdk/storage/azdatalake/file/client_test.go +++ b/sdk/storage/azdatalake/file/client_test.go @@ -13,6 +13,8 @@ import ( "encoding/binary" "fmt" "github.com/Azure/azure-sdk-for-go/sdk/azcore/log" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/path" "hash/crc64" "io" "math/rand" @@ -2875,6 +2877,42 @@ func (s *UnrecordedTestSuite) TestFileUploadDownloadStreamWithCPK() { _require.Equal(testcommon.TestCPKByValue.EncryptionKeySHA256, dResp.EncryptionKeySHA256) } +func (s *UnrecordedTestSuite) TestFileUploadDownloadStreamWithEncryptionContext() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + var fileSize int64 = 1 * 1024 + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + content := make([]byte, fileSize) + _, err = rand.Read(content) + _require.NoError(err) + + err = fClient.UploadStream(context.Background(), streaming.NopCloser(bytes.NewReader(content)), &file.UploadStreamOptions{ + EncryptionContext: &testcommon.TestEncryptionContext, + }) + _require.NoError(err) + + gResp2, err := fClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(*gResp2.EncryptionContext, testcommon.TestEncryptionContext) + + dResp, err := fClient.DownloadStream(context.Background(), nil) + _require.NoError(err) + + _require.Equal(testcommon.TestEncryptionContext, *dResp.EncryptionContext) +} + func (s *UnrecordedTestSuite) TestFileUploadDownloadStreamWithCPKNegative() { _require := require.New(s.T()) testName := s.T().Name() @@ -2982,6 +3020,156 @@ func (s *UnrecordedTestSuite) TestFileUploadFile() { _require.EqualValues(downloadedContentMD5, contentMD5) } +func (s *UnrecordedTestSuite) TestFileUploadBufferEncryptionContext() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + var fileSize int64 = 10 * 1024 + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + // create local file + content := make([]byte, fileSize) + _, err = rand.Read(content) + _require.NoError(err) + err = os.WriteFile("testFile", content, 0644) + _require.NoError(err) + + defer func() { + err = os.Remove("testFile") + _require.NoError(err) + }() + + fh, err := os.Open("testFile") + _require.NoError(err) + + defer func(fh *os.File) { + err := fh.Close() + _require.NoError(err) + }(fh) + + err = fClient.UploadBuffer(context.Background(), content, &file.UploadBufferOptions{ + Concurrency: 5, + ChunkSize: 4 * 1024, + EncryptionContext: &testcommon.TestEncryptionContext, + }) + _require.NoError(err) + + gResp2, err := fClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(testcommon.TestEncryptionContext, *gResp2.EncryptionContext) + +} + +func (s *UnrecordedTestSuite) TestFileUploadFileEncryptionContext() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + var fileSize int64 = 10 * 1024 + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + // create local file + content := make([]byte, fileSize) + _, err = rand.Read(content) + _require.NoError(err) + err = os.WriteFile("testFile", content, 0644) + _require.NoError(err) + + defer func() { + err = os.Remove("testFile") + _require.NoError(err) + }() + + fh, err := os.Open("testFile") + _require.NoError(err) + + defer func(fh *os.File) { + err := fh.Close() + _require.NoError(err) + }(fh) + + err = fClient.UploadFile(context.Background(), fh, &file.UploadFileOptions{ + Concurrency: 5, + ChunkSize: 4 * 1024, + EncryptionContext: &testcommon.TestEncryptionContext, + }) + _require.NoError(err) + + gResp2, err := fClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(testcommon.TestEncryptionContext, *gResp2.EncryptionContext) + + dResp, err := fClient.DownloadStream(context.Background(), nil) + _require.NoError(err) + _require.Equal(testcommon.TestEncryptionContext, *dResp.EncryptionContext) +} + +func (s *UnrecordedTestSuite) TestFileDownloadStreamEncryptionContext() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + var fileSize int64 = 10 * 1024 + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + resp, err := fClient.Create(context.Background(), &file.CreateOptions{EncryptionContext: &testcommon.TestEncryptionContext}) + _require.NoError(err) + _require.NotNil(resp) + + content := make([]byte, fileSize) + _, err = rand.Read(content) + _require.NoError(err) + md5Value := md5.Sum(content) + contentMD5 := md5Value[:] + + err = fClient.UploadStream(context.Background(), streaming.NopCloser(bytes.NewReader(content)), nil) + _require.NoError(err) + + gResp2, err := fClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(*gResp2.ContentLength, fileSize) + + dResp, err := fClient.DownloadStream(context.Background(), nil) + _require.NoError(err) + + data, err := io.ReadAll(dResp.Body) + _require.NoError(err) + + downloadedMD5Value := md5.Sum(data) + downloadedContentMD5 := downloadedMD5Value[:] + + _require.EqualValues(downloadedContentMD5, contentMD5) + _require.Equal(testcommon.TestEncryptionContext, *dResp.EncryptionContext) +} + func (s *RecordedTestSuite) TestSmallFileUploadFile() { _require := require.New(s.T()) testName := s.T().Name() @@ -3049,6 +3237,48 @@ func (s *RecordedTestSuite) TestSmallFileUploadFile() { _require.EqualValues(downloadedContentMD5, contentMD5) } +func (s *UnrecordedTestSuite) TestFileGetPropertiesWithEncryptionContext() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + createFileOpts := &file.CreateOptions{ + EncryptionContext: &testcommon.TestEncryptionContext, + } + + resp, err := fClient.Create(context.Background(), createFileOpts) + _require.NoError(err) + _require.NotNil(resp) + + response, err := fClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.NotNil(response) + _require.Equal(testcommon.TestEncryptionContext, *response.EncryptionContext) + + fileClient, err := testcommon.GetFileClient(filesystemName, fileName+"test", s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + resp2, err := fileClient.Create(context.Background(), nil) + _require.NoError(err) + _require.NotNil(resp2) + + response2, err := fileClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.NotNil(response) + _require.Nil(response2.EncryptionContext) +} + func (s *RecordedTestSuite) TestSmallFileUploadFileWithAccessConditionsAndHTTPHeaders() { _require := require.New(s.T()) testName := s.T().Name() @@ -3461,6 +3691,197 @@ func (s *RecordedTestSuite) TestDownloadDataContentMD5() { _require.Equal(resp1.ContentMD5, mdf[:]) } +func (s *RecordedTestSuite) TestFileAppendDataWithAcquireLease() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + srcFileName := "src" + testcommon.GenerateFileName(testName) + + srcFClient, err := testcommon.GetFileClient(filesystemName, srcFileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + resp, err := srcFClient.Create(context.Background(), nil) + _require.NoError(err) + _require.NotNil(resp) + + contentSize := 1024 * 8 // 8KB + rsc, _ := testcommon.GenerateData(contentSize) + + opts := &file.AppendDataOptions{ + LeaseAction: &file.LeaseActionAcquire, + LeaseDuration: to.Ptr(int64(15)), + ProposedLeaseID: proposedLeaseIDs[1], + } + _, err = srcFClient.AppendData(context.Background(), 0, rsc, opts) + _require.NoError(err) + + gResp2, err := srcFClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(lease.StateTypeLeased, *gResp2.LeaseState) + + time.Sleep(time.Second * 15) + + //Check if the lease was acquired for the right duration + gResp, err := srcFClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(lease.StateTypeExpired, *gResp.LeaseState) +} + +func (s *RecordedTestSuite) TestFileAppendDataWithRenewLease() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + srcFileName := "src" + testcommon.GenerateFileName(testName) + + srcFClient, err := testcommon.GetFileClient(filesystemName, srcFileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + createOpts := file.CreateOptions{ + ProposedLeaseID: proposedLeaseIDs[0], + LeaseDuration: to.Ptr(int64(15)), + } + + resp, err := srcFClient.Create(context.Background(), &createOpts) + _require.NoError(err) + _require.NotNil(resp) + + gResp2, err := srcFClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(lease.StateTypeLeased, *gResp2.LeaseState) + + //Wait for 15 seconds for lease to expire + time.Sleep(15 * time.Second) + + gResp, err := srcFClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(lease.StateTypeExpired, *gResp.LeaseState) + + contentSize := 1024 * 8 // 8KB + rsc, _ := testcommon.GenerateData(contentSize) + + opts := &file.AppendDataOptions{ + LeaseAction: &file.LeaseActionRenew, + LeaseAccessConditions: &file.LeaseAccessConditions{LeaseID: proposedLeaseIDs[0]}, + LeaseDuration: to.Ptr(int64(-1)), + } + _, err = srcFClient.AppendData(context.Background(), 0, rsc, opts) + _require.NoError(err) + + gResp2, err = srcFClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(lease.StateTypeLeased, *gResp2.LeaseState) +} + +func (s *RecordedTestSuite) TestFileAppendDataWithReleaseLease() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + srcFileName := "src" + testcommon.GenerateFileName(testName) + + srcFClient, err := testcommon.GetFileClient(filesystemName, srcFileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + createOpts := file.CreateOptions{ + ProposedLeaseID: proposedLeaseIDs[0], + LeaseDuration: to.Ptr(int64(15)), + } + + resp, err := srcFClient.Create(context.Background(), &createOpts) + _require.NoError(err) + _require.NotNil(resp) + + contentSize := 1024 * 8 // 8KB + rsc, _ := testcommon.GenerateData(contentSize) + + opts := &file.AppendDataOptions{ + LeaseAction: &file.LeaseActionRelease, + LeaseAccessConditions: &file.LeaseAccessConditions{LeaseID: proposedLeaseIDs[0]}, + Flush: to.Ptr(true), + } + + _, err = srcFClient.AppendData(context.Background(), 0, rsc, opts) + _require.NoError(err) + + gResp, err := srcFClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(lease.StateTypeAvailable, *gResp.LeaseState) +} + +func (s *RecordedTestSuite) TestFileAppendWithFlushReleaseLease() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + srcFileName := "src" + testcommon.GenerateFileName(testName) + + srcFClient, err := testcommon.GetFileClient(filesystemName, srcFileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + createOpts := file.CreateOptions{ + ProposedLeaseID: proposedLeaseIDs[0], + LeaseDuration: to.Ptr(int64(15)), + } + + resp, err := srcFClient.Create(context.Background(), &createOpts) + _require.NoError(err) + _require.NotNil(resp) + + contentSize := 1024 * 8 // 8KB + rsc, _ := testcommon.GenerateData(contentSize) + + _, err = srcFClient.AppendData(context.Background(), 0, rsc, + &file.AppendDataOptions{ + LeaseAccessConditions: &file.LeaseAccessConditions{LeaseID: proposedLeaseIDs[0]}, + }) + _require.NoError(err) + + opts := &file.FlushDataOptions{ + LeaseAction: &file.LeaseActionRelease, + AccessConditions: &path.AccessConditions{ + LeaseAccessConditions: &path.LeaseAccessConditions{LeaseID: proposedLeaseIDs[0]}, + }, + } + + _, err = srcFClient.FlushData(context.Background(), int64(contentSize), opts) + _require.NoError(err) + + gResp2, err := srcFClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(*gResp2.ContentLength, int64(contentSize)) + _require.Equal(lease.StateTypeAvailable, *gResp2.LeaseState) +} + func (s *RecordedTestSuite) TestFileAppendAndFlushData() { _require := require.New(s.T()) testName := s.T().Name() @@ -5063,3 +5484,71 @@ func TestUploadSmallChunkSize(t *testing.T) { _require.Equal(atomic.LoadUint64(&fbb.numChunks), numChunks) } + +func (s *RecordedTestSuite) TestFileClientCustomAudience() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + cred, err := testcommon.GetGenericTokenCredential() + _require.NoError(err) + + accountName, _ := testcommon.GetGenericAccountInfo(testcommon.TestAccountDatalake) + _require.Greater(len(accountName), 0) + + fileName := testcommon.GenerateFileName(testName) + fileURL := "https://" + accountName + ".dfs.core.windows.net/" + filesystemName + "/" + fileName + + options := &file.ClientOptions{Audience: "https://" + accountName + ".blob.core.windows.net"} + testcommon.SetClientOptions(s.T(), &options.ClientOptions) + + fClient, err := file.NewClient(fileURL, cred, options) + _require.NoError(err) + + _, err = fClient.Create(context.Background(), nil) + _require.NoError(err) + + _, err = fClient.GetProperties(context.Background(), nil) + _require.NoError(err) +} + +func (s *RecordedTestSuite) TestFileClientDefaultAudience() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + cred, err := testcommon.GetGenericTokenCredential() + _require.NoError(err) + + accountName, _ := testcommon.GetGenericAccountInfo(testcommon.TestAccountDatalake) + _require.Greater(len(accountName), 0) + + fileName := testcommon.GenerateFileName(testName) + fileURL := "https://" + accountName + ".dfs.core.windows.net/" + filesystemName + "/" + fileName + + options := &file.ClientOptions{Audience: "https://storage.azure.com/"} + testcommon.SetClientOptions(s.T(), &options.ClientOptions) + + fClient, err := file.NewClient(fileURL, cred, options) + _require.NoError(err) + + _, err = fClient.Create(context.Background(), nil) + _require.NoError(err) + + _, err = fClient.GetProperties(context.Background(), nil) + _require.NoError(err) +} diff --git a/sdk/storage/azdatalake/file/constants.go b/sdk/storage/azdatalake/file/constants.go index 77729179d3d4..357472077845 100644 --- a/sdk/storage/azdatalake/file/constants.go +++ b/sdk/storage/azdatalake/file/constants.go @@ -124,3 +124,13 @@ const ( StateTypeBreaking StateType = azdatalake.StateTypeBreaking StateTypeBroken StateType = azdatalake.StateTypeBroken ) + +// LeaseAction Describes actions that can be performed on a lease. +type LeaseAction = path.LeaseAction + +var ( + LeaseActionAcquire LeaseAction = path.LeaseActionAcquire + LeaseActionRelease LeaseAction = path.LeaseActionRelease + LeaseActionAcquireRelease LeaseAction = path.LeaseActionAcquireRelease + LeaseActionRenew LeaseAction = path.LeaseActionRenew +) diff --git a/sdk/storage/azdatalake/file/examples_test.go b/sdk/storage/azdatalake/file/examples_test.go index 213adfb3f173..0bb43c557216 100644 --- a/sdk/storage/azdatalake/file/examples_test.go +++ b/sdk/storage/azdatalake/file/examples_test.go @@ -14,9 +14,11 @@ import ( "fmt" "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease" "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake" "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/directory" "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/file" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/path" "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/shared" "hash/crc64" "io" @@ -387,3 +389,42 @@ func Example_file_AppendAndFlushDataWithValidation() { handleError(err) fmt.Println(*gResp2.ContentLength, int64(contentSize)) } + +func Example_file_AppendAndFlushDataWithAcquireAndReleaseLease() { + accountName, accountKey := os.Getenv("AZURE_STORAGE_ACCOUNT_NAME"), os.Getenv("AZURE_STORAGE_ACCOUNT_KEY") + + // Create a file client + u := fmt.Sprintf("https://%s.dfs.core.windows.net/fs/file.txt", accountName) + credential, err := azdatalake.NewSharedKeyCredential(accountName, accountKey) + handleError(err) + fClient, err := file.NewClientWithSharedKeyCredential(u, credential, nil) + handleError(err) + + contentSize := 1024 * 8 // 8KB + content := make([]byte, contentSize) + body := bytes.NewReader(content) + rsc := streaming.NopCloser(body) + + // Acquire lease during append data + opts := &file.AppendDataOptions{ + LeaseAction: &file.LeaseActionAcquire, + LeaseDuration: to.Ptr(int64(15)), + ProposedLeaseID: proposedLeaseIDs[1], + } + _, err = fClient.AppendData(context.Background(), 0, rsc, opts) + handleError(err) + + _, err = fClient.FlushData(context.Background(), int64(contentSize), &file.FlushDataOptions{ + LeaseAction: &file.LeaseActionRelease, + AccessConditions: &path.AccessConditions{ + LeaseAccessConditions: &path.LeaseAccessConditions{LeaseID: proposedLeaseIDs[0]}, + }, + }) + handleError(err) + + gResp2, err := fClient.GetProperties(context.Background(), nil) + handleError(err) + // Check if the lease is released + fmt.Println(lease.StateTypeAvailable, *gResp2.LeaseState) + +} diff --git a/sdk/storage/azdatalake/file/models.go b/sdk/storage/azdatalake/file/models.go index 27fa5e163166..4a75c795a02f 100644 --- a/sdk/storage/azdatalake/file/models.go +++ b/sdk/storage/azdatalake/file/models.go @@ -55,6 +55,8 @@ type CreateOptions struct { Group *string // ACL is the access control list for the file. ACL *string + // EncryptionContext stores non-encrypted data that can be used to derive the customer-provided key for a file. + EncryptionContext *string } // CreateExpiryValues describes when a newly created file should expire. @@ -91,6 +93,7 @@ func (o *CreateOptions) format() (*generated.LeaseAccessConditions, *generated.M createOpts.Permissions = o.Permissions createOpts.ProposedLeaseID = o.ProposedLeaseID createOpts.LeaseDuration = o.LeaseDuration + createOpts.EncryptionContext = o.EncryptionContext var httpHeaders *generated.PathHTTPHeaders var cpkOpts *generated.CPKInfo @@ -149,6 +152,8 @@ type uploadFromReaderOptions struct { HTTPHeaders *HTTPHeaders // CPKInfo contains optional parameters to perform encryption using customer-provided key. CPKInfo *CPKInfo + // EncryptionContext contains the information returned from the x-ms-encryption-context header response. + EncryptionContext *string } // UploadStreamOptions provides set of configurations for Client.UploadStream operation. @@ -163,6 +168,8 @@ type UploadStreamOptions struct { HTTPHeaders *HTTPHeaders // CPKInfo contains optional parameters to perform encryption using customer-provided key. CPKInfo *CPKInfo + // EncryptionContext contains the information returned from the x-ms-encryption-context header response. + EncryptionContext *string } // UploadBufferOptions provides set of configurations for Client.UploadBuffer operation. @@ -185,6 +192,13 @@ type FlushDataOptions struct { // RetainUncommittedData if "true", uncommitted data is retained after the flush operation // completes, otherwise, the uncommitted data is deleted after the flush operation. RetainUncommittedData *bool + // LeaseAction Describes actions that can be performed on a lease. + LeaseAction *LeaseAction + // LeaseDuration specifies the duration of the lease, in seconds, or negative one + // (-1) for a lease that never expires. A non-infinite lease can be between 15 and 60 seconds. + LeaseDuration *int64 + // ProposedLeaseID specifies the proposed lease ID for the file. + ProposedLeaseID *string } func (o *FlushDataOptions) format(offset int64) (*generated.PathClientFlushDataOptions, *generated.ModifiedAccessConditions, *generated.LeaseAccessConditions, *generated.PathHTTPHeaders, *generated.CPKInfo, error) { @@ -230,6 +244,9 @@ func (o *FlushDataOptions) format(offset int64) (*generated.PathClientFlushDataO cpkInfoOpts.EncryptionKeySHA256 = o.CPKInfo.EncryptionKeySHA256 cpkInfoOpts.EncryptionAlgorithm = o.CPKInfo.EncryptionAlgorithm } + flushDataOpts.LeaseAction = o.LeaseAction + flushDataOpts.LeaseDuration = o.LeaseDuration + flushDataOpts.ProposedLeaseID = o.ProposedLeaseID } return flushDataOpts, modifiedAccessConditions, leaseAccessConditions, httpHeaderOpts, cpkInfoOpts, nil } @@ -241,11 +258,22 @@ type AppendDataOptions struct { TransactionalValidation TransferValidationType // LeaseAccessConditions contains optional parameters to access leased entity. LeaseAccessConditions *LeaseAccessConditions + // LeaseAction describes actions that can be performed on a lease. + LeaseAction *LeaseAction + // LeaseDuration specifies the duration of the lease, in seconds, or negative one + // (-1) for a lease that never expires. A non-infinite lease can be between 15 and 60 seconds. + LeaseDuration *int64 + // ProposedLeaseID specifies the proposed lease ID for the file. + ProposedLeaseID *string // CPKInfo contains optional parameters to perform encryption using customer-provided key. CPKInfo *CPKInfo + //Flush Optional. If true, the file will be flushed after append. + Flush *bool } -func (o *AppendDataOptions) format(offset int64, body io.ReadSeekCloser) (*generated.PathClientAppendDataOptions, *generated.LeaseAccessConditions, *generated.CPKInfo, error) { +func (o *AppendDataOptions) format(offset int64, body io.ReadSeekCloser) (*generated.PathClientAppendDataOptions, + *generated.LeaseAccessConditions, *generated.CPKInfo, error) { + if offset < 0 || body == nil { return nil, nil, nil, errors.New("invalid argument: offset must be >= 0 and body must not be nil") } @@ -280,11 +308,17 @@ func (o *AppendDataOptions) format(offset int64, body io.ReadSeekCloser) (*gener cpkInfoOpts.EncryptionKeySHA256 = o.CPKInfo.EncryptionKeySHA256 cpkInfoOpts.EncryptionAlgorithm = o.CPKInfo.EncryptionAlgorithm } - } - if o != nil && o.TransactionalValidation != nil { - _, err = o.TransactionalValidation.Apply(body, appendDataOptions) - if err != nil { - return nil, nil, nil, err + + appendDataOptions.LeaseAction = o.LeaseAction + appendDataOptions.LeaseDuration = o.LeaseDuration + appendDataOptions.ProposedLeaseID = o.ProposedLeaseID + appendDataOptions.Flush = o.Flush + + if o.TransactionalValidation != nil { + _, err = o.TransactionalValidation.Apply(body, appendDataOptions) + if err != nil { + return nil, nil, nil, err + } } } diff --git a/sdk/storage/azdatalake/file/responses.go b/sdk/storage/azdatalake/file/responses.go index f72747584ab7..38d956226847 100644 --- a/sdk/storage/azdatalake/file/responses.go +++ b/sdk/storage/azdatalake/file/responses.go @@ -14,6 +14,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/generated_blob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/path" "io" + "net/http" "time" ) @@ -135,6 +136,9 @@ type DownloadResponse struct { // EncryptionScope contains the information returned from the x-ms-encryption-scope header response. EncryptionScope *string + // EncryptionContext contains the information returned from the x-ms-encryption-context header response. + EncryptionContext *string + // ErrorCode contains the information returned from the x-ms-error-code header response. ErrorCode *string @@ -193,7 +197,7 @@ type DownloadResponse struct { VersionID *string } -func FormatDownloadStreamResponse(r *blob.DownloadStreamResponse) DownloadResponse { +func FormatDownloadStreamResponse(r *blob.DownloadStreamResponse, rawResponse *http.Response) DownloadResponse { newResp := DownloadResponse{} if r != nil { newResp.AcceptRanges = r.AcceptRanges @@ -238,6 +242,9 @@ func FormatDownloadStreamResponse(r *blob.DownloadStreamResponse) DownloadRespon newResp.Version = r.Version newResp.VersionID = r.VersionID } + if val := rawResponse.Header.Get("x-ms-encryption-context"); val != "" { + newResp.EncryptionContext = &val + } return newResp } diff --git a/sdk/storage/azdatalake/filesystem/client.go b/sdk/storage/azdatalake/filesystem/client.go index f816ac67bd36..e79367171a7b 100644 --- a/sdk/storage/azdatalake/filesystem/client.go +++ b/sdk/storage/azdatalake/filesystem/client.go @@ -40,8 +40,9 @@ type Client base.CompositeClient[generated.FileSystemClient, generated.FileSyste // - options - client options; pass nil to accept the default values func NewClient(filesystemURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) { containerURL, filesystemURL := shared.GetURLs(filesystemURL) - authPolicy := runtime.NewBearerTokenPolicy(cred, []string{shared.TokenScope}, nil) + audience := base.GetAudience((*base.ClientOptions)(options)) conOptions := shared.GetClientOptions(options) + authPolicy := shared.NewStorageChallengePolicy(cred, audience, conOptions.InsecureAllowCredentialWithHTTP) plOpts := runtime.PipelineOptions{ PerRetry: []policy.Policy{authPolicy}, } @@ -370,3 +371,21 @@ func (fs *Client) GetSASURL(permissions sas.FileSystemPermissions, expiry time.T return endpoint, nil } + +// CreateFile Creates a new file within a file system. +// For more information, see the Azure Docs. +func (fs *Client) CreateFile(ctx context.Context, filePath string, options *CreateFileOptions) (CreateFileResponse, error) { + fileClient := fs.NewFileClient(filePath) + resp, err := fileClient.Create(ctx, options) + err = exported.ConvertToDFSError(err) + return resp, err +} + +// CreateDirectory Creates a new directory within a file system. +// For more information, see the Azure Docs. +func (fs *Client) CreateDirectory(ctx context.Context, filePath string, options *CreateDirectoryOptions) (CreateDirectoryResponse, error) { + dirClient := fs.NewDirectoryClient(filePath) + resp, err := dirClient.Create(ctx, options) + err = exported.ConvertToDFSError(err) + return resp, err +} diff --git a/sdk/storage/azdatalake/filesystem/client_test.go b/sdk/storage/azdatalake/filesystem/client_test.go index a1f42713d8cd..695548b14598 100644 --- a/sdk/storage/azdatalake/filesystem/client_test.go +++ b/sdk/storage/azdatalake/filesystem/client_test.go @@ -102,8 +102,9 @@ func (s *RecordedTestSuite) TestCreateFilesystemWithOptions() { metadata := map[string]*string{"foo": &testStr, "bar": &testStr} access := filesystem.FileSystem opts := filesystem.CreateOptions{ - Metadata: metadata, - Access: &access, + Metadata: metadata, + Access: &access, + CPKScopeInfo: &testcommon.TestCPKScopeInfo, } fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) _require.NoError(err) @@ -116,6 +117,7 @@ func (s *RecordedTestSuite) TestCreateFilesystemWithOptions() { _require.NoError(err) _require.NotNil(props.Metadata) _require.Equal(*props.PublicAccess, filesystem.FileSystem) + _require.Equal(props.DefaultEncryptionScope, &testcommon.TestEncryptionScope) } func (s *RecordedTestSuite) TestCreateFilesystemWithFileAccess() { @@ -262,6 +264,34 @@ func (s *RecordedTestSuite) TestFilesystemGetPropertiesWithLease() { _require.NoError(err) } +func (s *RecordedTestSuite) TestFilesystemGetPropertiesDefaultEncryptionScopeAndOverride() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + testStr := "hello" + metadata := map[string]*string{"foo": &testStr, "bar": &testStr} + access := filesystem.FileSystem + + opts := filesystem.CreateOptions{ + Metadata: metadata, + Access: &access, + CPKScopeInfo: &testcommon.TestCPKScopeInfo, + } + _, err = fsClient.Create(context.Background(), &opts) + _require.NoError(err) + + resp, err := fsClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(resp.DenyEncryptionScopeOverride, to.Ptr(false)) + _require.Equal(resp.DefaultEncryptionScope, &testcommon.TestEncryptionScope) + +} + func (s *RecordedTestSuite) TestFilesystemDelete() { _require := require.New(s.T()) testName := s.T().Name() @@ -1640,6 +1670,48 @@ func (s *RecordedTestSuite) TestFilesystemListPathsWithContinuation() { _require.Nil(resp.Continuation) } +func (s *RecordedTestSuite) TestFilesystemListPathsWithEncryptionContext() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + client := fsClient.NewFileClient(testName + "file1") + _, err = client.Create(context.Background(), &file.CreateOptions{EncryptionContext: &testcommon.TestEncryptionContext}) + _require.NoError(err) + client = fsClient.NewFileClient(testName + "file2") + _, err = client.Create(context.Background(), &file.CreateOptions{EncryptionContext: &testcommon.TestEncryptionContext}) + _require.NoError(err) + dirClient := fsClient.NewDirectoryClient(testName + "dir1") + _, err = dirClient.Create(context.Background(), nil) + _require.NoError(err) + dirClient = fsClient.NewDirectoryClient(testName + "dir2") + _, err = dirClient.Create(context.Background(), nil) + _require.NoError(err) + + pager := fsClient.NewListPathsPager(true, nil) + for pager.More() { + resp, err := pager.NextPage(context.Background()) + _require.NoError(err) + _require.Equal(5, len(resp.Paths)) + _require.Equal(resp.PathList.Paths[2].IsDirectory, to.Ptr(true)) + _require.Nil(resp.PathList.Paths[3].IsDirectory) + _require.Nil(resp.PathList.Paths[2].EncryptionContext) + // Encryption context is only applicable on files, not directories. + _require.Equal(resp.PathList.Paths[3].EncryptionContext, &testcommon.TestEncryptionContext) + + if err != nil { + break + } + } +} + func (s *RecordedTestSuite) TestFilesystemListDeletedPaths() { _require := require.New(s.T()) testName := s.T().Name() @@ -1928,3 +2000,141 @@ func (s *UnrecordedTestSuite) TestFSCreateDeleteUsingOAuth() { _require.NoError(err) } + +func (s *RecordedTestSuite) TestCreateFileInFileSystemSetOptions() { + _require := require.New(s.T()) + testName := s.T().Name() + + umask := "0000" + user := "4cf4e284-f6a8-4540-b53e-c3469af032dc" + group := user + acl := "user::rwx,group::r-x,other::rwx" + leaseDuration := to.Ptr(int64(15)) + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + createFileOptions := &filesystem.CreateFileOptions{ + Umask: &umask, + Owner: &user, + Group: &group, + ACL: &acl, + Expiry: file.CreateExpiryValues{ + ExpiryType: file.CreateExpiryTypeNeverExpire, + }, + LeaseDuration: leaseDuration, + ProposedLeaseID: proposedLeaseIDs[0], + } + resp, err := fsClient.CreateFile(context.Background(), testName, createFileOptions) + _require.NoError(err) + _require.NotNil(resp) + + fClient := fsClient.NewFileClient(testName) + + response, err := fClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal("4cf4e284-f6a8-4540-b53e-c3469af032dc", *response.Owner) + _require.Equal("rwxr-xrwx", *response.Permissions) + _require.Equal(filesystem.StateTypeLeased, *response.LeaseState) + +} + +func (s *RecordedTestSuite) TestCreateDirectoryInFileSystemSetOptions() { + _require := require.New(s.T()) + testName := s.T().Name() + + perms := "0777" + umask := "0000" + owner := "4cf4e284-f6a8-4540-b53e-c3469af032dc" + group := owner + leaseDuration := to.Ptr(int64(-1)) + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + createDirOptions := &filesystem.CreateDirectoryOptions{ + Permissions: &perms, + Umask: &umask, + Owner: &owner, + Group: &group, + LeaseDuration: leaseDuration, + ProposedLeaseID: proposedLeaseIDs[0], + } + + resp, err := fsClient.CreateDirectory(context.Background(), testName, createDirOptions) + _require.NoError(err) + _require.NotNil(resp) + + dirClient := fsClient.NewDirectoryClient(testName) + + response, err := dirClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(*response.Owner, "4cf4e284-f6a8-4540-b53e-c3469af032dc") + _require.Equal("rwxrwxrwx", *response.Permissions) + _require.Equal(filesystem.StateTypeLeased, *response.LeaseState) + +} + +func (s *RecordedTestSuite) TestFSCreateDefaultAudience() { + _require := require.New(s.T()) + testName := s.T().Name() + + cred, err := testcommon.GetGenericTokenCredential() + _require.NoError(err) + + accountName, _ := testcommon.GetGenericAccountInfo(testcommon.TestAccountDatalake) + _require.Greater(len(accountName), 0) + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsURL := "https://" + accountName + ".dfs.core.windows.net/" + filesystemName + + options := &filesystem.ClientOptions{Audience: "https://storage.azure.com/"} + testcommon.SetClientOptions(s.T(), &options.ClientOptions) + fsClient, err := filesystem.NewClient(fsURL, cred, options) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + _, err = fsClient.GetProperties(context.Background(), nil) + _require.NoError(err) + +} + +func (s *RecordedTestSuite) TestFSCreateCustomAudience() { + _require := require.New(s.T()) + testName := s.T().Name() + + cred, err := testcommon.GetGenericTokenCredential() + _require.NoError(err) + + accountName, _ := testcommon.GetGenericAccountInfo(testcommon.TestAccountDatalake) + _require.Greater(len(accountName), 0) + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsURL := "https://" + accountName + ".dfs.core.windows.net/" + filesystemName + + options := &filesystem.ClientOptions{Audience: "https://" + accountName + ".blob.core.windows.net"} + testcommon.SetClientOptions(s.T(), &options.ClientOptions) + fsClient, err := filesystem.NewClient(fsURL, cred, options) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + _, err = fsClient.GetProperties(context.Background(), nil) + _require.NoError(err) + +} diff --git a/sdk/storage/azdatalake/filesystem/examples_test.go b/sdk/storage/azdatalake/filesystem/examples_test.go index ad8e7b293c0f..94dd9390c1a1 100644 --- a/sdk/storage/azdatalake/filesystem/examples_test.go +++ b/sdk/storage/azdatalake/filesystem/examples_test.go @@ -10,6 +10,7 @@ import ( "bytes" "context" "fmt" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/file" "io" "log" "net/http" @@ -328,3 +329,71 @@ func Example_fs_ClientSetMetadata() { _, err = fsClient.SetMetadata(context.TODO(), &filesystem.SetMetadataOptions{Metadata: fsGetPropertiesResponse.Metadata}) handleError(err) } + +func Example_fs_ClientCreateFile() { + accountName, ok := os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME") + if !ok { + panic("AZURE_STORAGE_ACCOUNT_NAME could not be found") + } + fsName := "testfs" + fsURL := fmt.Sprintf("https://%s.dfs.core.windows.net/%s", accountName, fsName) + filePath := "testFile" + + cred, err := azidentity.NewDefaultAzureCredential(nil) + handleError(err) + + fsClient, err := filesystem.NewClient(fsURL, cred, nil) + handleError(err) + + fsCreateResponse, err := fsClient.Create(context.TODO(), &filesystem.CreateOptions{ + Metadata: map[string]*string{"Foo": to.Ptr("Bar")}, + }) + handleError(err) + fmt.Println(fsCreateResponse) + + createFileOptions := &filesystem.CreateFileOptions{ + Umask: to.Ptr("0000"), + ACL: to.Ptr("user::rwx,group::r-x,other::rwx"), + Expiry: file.CreateExpiryValues{ + ExpiryType: file.CreateExpiryTypeAbsolute, + ExpiresOn: time.Now().Add(20 * time.Second).UTC().Format(http.TimeFormat), + }, + LeaseDuration: to.Ptr(int64(15)), + ProposedLeaseID: to.Ptr("c820a799-76d7-4ee2-6e15-546f19325c2c"), + } + resp, err := fsClient.CreateFile(context.Background(), filePath, createFileOptions) + handleError(err) + fmt.Println(resp) +} + +func Example_fs_ClientCreateDirectory() { + accountName, ok := os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME") + if !ok { + panic("AZURE_STORAGE_ACCOUNT_NAME could not be found") + } + fsName := "testfs" + fsURL := fmt.Sprintf("https://%s.dfs.core.windows.net/%s", accountName, fsName) + dirPath := "testDir" + + cred, err := azidentity.NewDefaultAzureCredential(nil) + handleError(err) + + fsClient, err := filesystem.NewClient(fsURL, cred, nil) + handleError(err) + + fsCreateResponse, err := fsClient.Create(context.TODO(), &filesystem.CreateOptions{ + Metadata: map[string]*string{"Foo": to.Ptr("Bar")}, + }) + handleError(err) + fmt.Println(fsCreateResponse) + + options := &filesystem.CreateDirectoryOptions{ + Umask: to.Ptr("0000"), + ACL: to.Ptr("user::rwx,group::r-x,other::rwx"), + LeaseDuration: to.Ptr(int64(15)), + ProposedLeaseID: to.Ptr("c820a799-76d7-4ee2-6e15-546f19325c2c"), + } + resp, err := fsClient.CreateDirectory(context.Background(), dirPath, options) + handleError(err) + fmt.Println(resp) +} diff --git a/sdk/storage/azdatalake/filesystem/models.go b/sdk/storage/azdatalake/filesystem/models.go index 05c395fc20c3..6b3d708d6591 100644 --- a/sdk/storage/azdatalake/filesystem/models.go +++ b/sdk/storage/azdatalake/filesystem/models.go @@ -8,6 +8,8 @@ package filesystem import ( "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/directory" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/file" "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/exported" "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/generated" "time" @@ -250,3 +252,9 @@ type PathProperties = generated.PathPropertiesInternal // PathPrefix contains the response from method FileSystemClient.ListPathsHierarchySegment. type PathPrefix = generated.PathPrefix + +// CreateFileOptions contains the optional parameters when calling the CreateFile operation. +type CreateFileOptions = file.CreateOptions + +// CreateDirectoryOptions contains the optional parameters when calling the CreateDirectory operation. +type CreateDirectoryOptions = directory.CreateOptions diff --git a/sdk/storage/azdatalake/filesystem/responses.go b/sdk/storage/azdatalake/filesystem/responses.go index 15b97decc321..cf7ecb521e4a 100644 --- a/sdk/storage/azdatalake/filesystem/responses.go +++ b/sdk/storage/azdatalake/filesystem/responses.go @@ -150,3 +150,9 @@ type ListPathsHierarchySegmentResponse = generated.ListPathsHierarchySegmentResp // PathHierarchyListSegment contains the response from method FileSystemClient.ListPathsHierarchySegment. type PathHierarchyListSegment = generated.PathHierarchyListSegment + +// CreateFileResponse contains the response from method FileSystemClient.CreateFile. +type CreateFileResponse = generated.PathClientCreateResponse + +// CreateDirectoryResponse contains the response from method FileSystemClient.CreateDirectory. +type CreateDirectoryResponse = generated.PathClientCreateResponse diff --git a/sdk/storage/azdatalake/go.mod b/sdk/storage/azdatalake/go.mod index 469cd5ce4c6d..6a2ac55d65ae 100644 --- a/sdk/storage/azdatalake/go.mod +++ b/sdk/storage/azdatalake/go.mod @@ -3,10 +3,10 @@ module github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake go 1.18 require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 github.com/stretchr/testify v1.8.4 ) @@ -19,9 +19,9 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/crypto v0.18.0 // indirect - golang.org/x/net v0.20.0 // indirect - golang.org/x/sys v0.16.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/net v0.22.0 // indirect + golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/sdk/storage/azdatalake/go.sum b/sdk/storage/azdatalake/go.sum index f7e8b87df681..1e3746a61567 100644 --- a/sdk/storage/azdatalake/go.sum +++ b/sdk/storage/azdatalake/go.sum @@ -1,12 +1,12 @@ -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 h1:c4k2FIYIh4xtwqrQwV0Ct1v5+ehlNXj5NI/MWVsiTkQ= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2/go.mod h1:5FDJtLEO/GxwNgUxbwrY3LP0pEoThTQJtk2oysdXHxM= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 h1:fXPMAmuh0gDuRDey0atC8cXBuKIlqCzCkL8sm1n9Ov0= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 h1:YUUxeiOWgdAQE3pXt2H7QXzZs0q8UBjgRbl56qo8GYM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2/go.mod h1:dmXQgZuiSubAecswZE+Sm8jkvEa7kQgTPVRvwL/nd0E= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -26,13 +26,13 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/sdk/storage/azdatalake/internal/base/clients.go b/sdk/storage/azdatalake/internal/base/clients.go index e5b9e2a300c5..ad6de2d5583a 100644 --- a/sdk/storage/azdatalake/internal/base/clients.go +++ b/sdk/storage/azdatalake/internal/base/clients.go @@ -15,12 +15,18 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/exported" "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/generated" "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/generated_blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/shared" + "strings" ) // ClientOptions contains the optional parameters when creating a Client. type ClientOptions struct { azcore.ClientOptions pipelineOptions *runtime.PipelineOptions + // Audience to use when requesting tokens for Azure Active Directory authentication. + // Only has an effect when credential is of type TokenCredential. The value could be + // https://storage.azure.com/ (default) or https://.blob.core.windows.net. + Audience string } func GetPipelineOptions(clOpts *ClientOptions) *runtime.PipelineOptions { @@ -91,3 +97,11 @@ func NewPathClient(pathURL string, pathURLWithBlobEndpoint string, client *block func GetCompositeClientOptions[T, K, U any](client *CompositeClient[T, K, U]) *ClientOptions { return client.options } + +func GetAudience(clOpts *ClientOptions) string { + if clOpts == nil || len(strings.TrimSpace(clOpts.Audience)) == 0 { + return shared.TokenScope + } else { + return strings.TrimRight(clOpts.Audience, "/") + "/.default" + } +} diff --git a/sdk/storage/azdatalake/internal/generated/autorest.md b/sdk/storage/azdatalake/internal/generated/autorest.md index d0fcbb9a408a..e282c8e9a865 100644 --- a/sdk/storage/azdatalake/internal/generated/autorest.md +++ b/sdk/storage/azdatalake/internal/generated/autorest.md @@ -7,7 +7,7 @@ go: true clear-output-folder: false version: "^3.0.0" license-header: MICROSOFT_MIT_NO_VERSION -input-file: "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/3f3b51edf8fd0eb65004df390d6ee98e0e23c53d/specification/storage/data-plane/Azure.Storage.Files.DataLake/preview/2021-06-08/DataLakeStorage.json" +input-file: "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/688a906172823628e75b19ea8964d998cb7560fd/specification/storage/data-plane/Azure.Storage.Files.DataLake/preview/2023-05-03/DataLakeStorage.json" credential-scope: "https://storage.azure.com/.default" output-folder: ../generated file-prefix: "zz_" @@ -298,3 +298,17 @@ directive: replace(/err = unpopulate\((.*), "ContentLength", &p\.ContentLength\)/g, 'var rawVal string\nerr = unpopulate(val, "ContentLength", &rawVal)\nintVal, _ := strconv.ParseInt(rawVal, 10, 64)\np.ContentLength = &intVal'). replace(/err = unpopulate\((.*), "IsDirectory", &p\.IsDirectory\)/g, 'var rawVal string\nerr = unpopulate(val, "IsDirectory", &rawVal)\nboolVal, _ := strconv.ParseBool(rawVal)\np.IsDirectory = &boolVal'); ``` + +### Updating service version to 2023-11-03 +```yaml +directive: +- from: + - zz_service_client.go + - zz_filesystem_client.go + - zz_path_client.go + where: $ + transform: >- + return $. + replaceAll(`[]string{"2023-05-03"}`, `[]string{ServiceVersion}`). + replaceAll(`2023-05-03`, `2023-11-03`); +``` diff --git a/sdk/storage/azdatalake/internal/generated/zz_filesystem_client.go b/sdk/storage/azdatalake/internal/generated/zz_filesystem_client.go index 115de5aeaa9d..f76c33f3fbe5 100644 --- a/sdk/storage/azdatalake/internal/generated/zz_filesystem_client.go +++ b/sdk/storage/azdatalake/internal/generated/zz_filesystem_client.go @@ -32,7 +32,7 @@ type FileSystemClient struct { // operation does not support conditional HTTP requests. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2021-06-08 +// Generated from API version 2023-11-03 // - options - FileSystemClientCreateOptions contains the optional parameters for the FileSystemClient.Create method. func (client *FileSystemClient) Create(ctx context.Context, options *FileSystemClientCreateOptions) (FileSystemClientCreateResponse, error) { req, err := client.createCreateRequest(ctx, options) @@ -64,7 +64,7 @@ func (client *FileSystemClient) createCreateRequest(ctx context.Context, options if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2021-06-08"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if options != nil && options.Properties != nil { req.Raw().Header["x-ms-properties"] = []string{*options.Properties} } @@ -114,7 +114,7 @@ func (client *FileSystemClient) createHandleResponse(resp *http.Response) (FileS // [https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations]. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2021-06-08 +// Generated from API version 2023-11-03 // - options - FileSystemClientDeleteOptions contains the optional parameters for the FileSystemClient.Delete method. // - ModifiedAccessConditions - ModifiedAccessConditions contains a group of parameters for the FileSystemClient.SetProperties // method. @@ -148,7 +148,7 @@ func (client *FileSystemClient) deleteCreateRequest(ctx context.Context, options if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2021-06-08"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if modifiedAccessConditions != nil && modifiedAccessConditions.IfModifiedSince != nil { req.Raw().Header["If-Modified-Since"] = []string{(*modifiedAccessConditions.IfModifiedSince).In(gmt).Format(time.RFC1123)} } @@ -181,7 +181,7 @@ func (client *FileSystemClient) deleteHandleResponse(resp *http.Response) (FileS // GetProperties - All system and user-defined filesystem properties are specified in the response headers. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2021-06-08 +// Generated from API version 2023-11-03 // - options - FileSystemClientGetPropertiesOptions contains the optional parameters for the FileSystemClient.GetProperties // method. func (client *FileSystemClient) GetProperties(ctx context.Context, options *FileSystemClientGetPropertiesOptions) (FileSystemClientGetPropertiesResponse, error) { @@ -214,7 +214,7 @@ func (client *FileSystemClient) getPropertiesCreateRequest(ctx context.Context, if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2021-06-08"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} req.Raw().Header["Accept"] = []string{"application/json"} return req, nil } @@ -256,7 +256,7 @@ func (client *FileSystemClient) getPropertiesHandleResponse(resp *http.Response) // NewListBlobHierarchySegmentPager - The List Blobs operation returns a list of the blobs under the specified container // -// Generated from API version 2021-06-08 +// Generated from API version 2023-11-03 // - options - FileSystemClientListBlobHierarchySegmentOptions contains the optional parameters for the FileSystemClient.NewListBlobHierarchySegmentPager // method. // @@ -291,7 +291,7 @@ func (client *FileSystemClient) ListBlobHierarchySegmentCreateRequest(ctx contex reqQP.Set("timeout", strconv.FormatInt(int64(*options.Timeout), 10)) } req.Raw().URL.RawQuery = reqQP.Encode() - req.Raw().Header["x-ms-version"] = []string{"2021-06-08"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } @@ -329,7 +329,7 @@ func (client *FileSystemClient) ListBlobHierarchySegmentHandleResponse(resp *htt // NewListPathsPager - List FileSystem paths and their properties. // -// Generated from API version 2021-06-08 +// Generated from API version 2023-11-03 // - recursive - Required // - options - FileSystemClientListPathsOptions contains the optional parameters for the FileSystemClient.NewListPathsPager // method. @@ -362,7 +362,7 @@ func (client *FileSystemClient) ListPathsCreateRequest(ctx context.Context, recu if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2021-06-08"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} req.Raw().Header["Accept"] = []string{"application/json"} return req, nil } @@ -407,7 +407,7 @@ func (client *FileSystemClient) ListPathsHandleResponse(resp *http.Response) (Fi // [https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations]. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2021-06-08 +// Generated from API version 2023-11-03 // - options - FileSystemClientSetPropertiesOptions contains the optional parameters for the FileSystemClient.SetProperties // method. // - ModifiedAccessConditions - ModifiedAccessConditions contains a group of parameters for the FileSystemClient.SetProperties @@ -442,7 +442,7 @@ func (client *FileSystemClient) setPropertiesCreateRequest(ctx context.Context, if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2021-06-08"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if options != nil && options.Properties != nil { req.Raw().Header["x-ms-properties"] = []string{*options.Properties} } diff --git a/sdk/storage/azdatalake/internal/generated/zz_models.go b/sdk/storage/azdatalake/internal/generated/zz_models.go index 1e24dca4257d..f9767726fc69 100644 --- a/sdk/storage/azdatalake/internal/generated/zz_models.go +++ b/sdk/storage/azdatalake/internal/generated/zz_models.go @@ -386,6 +386,11 @@ type PathClientDeleteOptions struct { // this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation // of the delete operation to continue deleting the directory. Continuation *string + // If true, paginated behavior will be seen. Pagination is for the recursive ACL checks as a POSIX requirement in the server + // and Delete in an atomic operation once the ACL checks are completed. If false + // or missing, normal default behavior will kick in, which may timeout in case of very large directories due to recursive + // ACL checks. This new parameter is introduced for backward compatibility. + Paginated *bool // Required Recursive *bool // Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the analytics logs when storage diff --git a/sdk/storage/azdatalake/internal/generated/zz_path_client.go b/sdk/storage/azdatalake/internal/generated/zz_path_client.go index c110ad790150..8788b2750a9d 100644 --- a/sdk/storage/azdatalake/internal/generated/zz_path_client.go +++ b/sdk/storage/azdatalake/internal/generated/zz_path_client.go @@ -32,7 +32,7 @@ type PathClient struct { // AppendData - Append data to the file. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2021-06-08 +// Generated from API version 2023-11-03 // - body - Initial data // - options - PathClientAppendDataOptions contains the optional parameters for the PathClient.AppendData method. // - PathHTTPHeaders - PathHTTPHeaders contains a group of parameters for the PathClient.Create method. @@ -95,7 +95,7 @@ func (client *PathClient) appendDataCreateRequest(ctx context.Context, body io.R if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2021-06-08"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if cpkInfo != nil && cpkInfo.EncryptionKey != nil { req.Raw().Header["x-ms-encryption-key"] = []string{*cpkInfo.EncryptionKey} } @@ -176,7 +176,7 @@ func (client *PathClient) appendDataHandleResponse(resp *http.Response) (PathCli // If-None-Match: "*". // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2021-06-08 +// Generated from API version 2023-11-03 // - options - PathClientCreateOptions contains the optional parameters for the PathClient.Create method. // - PathHTTPHeaders - PathHTTPHeaders contains a group of parameters for the PathClient.Create method. // - LeaseAccessConditions - LeaseAccessConditions contains a group of parameters for the PathClient.Create method. @@ -223,7 +223,7 @@ func (client *PathClient) createCreateRequest(ctx context.Context, options *Path if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2021-06-08"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if pathHTTPHeaders != nil && pathHTTPHeaders.CacheControl != nil { req.Raw().Header["x-ms-cache-control"] = []string{*pathHTTPHeaders.CacheControl} } @@ -372,7 +372,7 @@ func (client *PathClient) createHandleResponse(resp *http.Response) (PathClientC // [https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations]. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2021-06-08 +// Generated from API version 2023-11-03 // - options - PathClientDeleteOptions contains the optional parameters for the PathClient.Delete method. // - LeaseAccessConditions - LeaseAccessConditions contains a group of parameters for the PathClient.Create method. // - ModifiedAccessConditions - ModifiedAccessConditions contains a group of parameters for the FileSystemClient.SetProperties @@ -386,7 +386,7 @@ func (client *PathClient) Delete(ctx context.Context, options *PathClientDeleteO if err != nil { return PathClientDeleteResponse{}, err } - if !runtime.HasStatusCode(resp, http.StatusOK) { + if !runtime.HasStatusCode(resp, http.StatusOK, http.StatusAccepted) { return PathClientDeleteResponse{}, runtime.NewResponseError(resp) } return client.deleteHandleResponse(resp) @@ -408,11 +408,14 @@ func (client *PathClient) deleteCreateRequest(ctx context.Context, options *Path if options != nil && options.Continuation != nil { reqQP.Set("continuation", *options.Continuation) } + if options != nil && options.Paginated != nil { + reqQP.Set("paginated", strconv.FormatBool(*options.Paginated)) + } req.Raw().URL.RawQuery = reqQP.Encode() if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2021-06-08"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if leaseAccessConditions != nil && leaseAccessConditions.LeaseID != nil { req.Raw().Header["x-ms-lease-id"] = []string{*leaseAccessConditions.LeaseID} } @@ -454,13 +457,22 @@ func (client *PathClient) deleteHandleResponse(resp *http.Response) (PathClientD if val := resp.Header.Get("x-ms-deletion-id"); val != "" { result.DeletionID = &val } + if val := resp.Header.Get("x-ms-request-id"); val != "" { + result.XMSRequestID = &val + } + if val := resp.Header.Get("x-ms-version"); val != "" { + result.XMSVersion = &val + } + if val := resp.Header.Get("x-ms-continuation"); val != "" { + result.XMSContinuation = &val + } return result, nil } // FlushData - Set the owner, group, permissions, or access control list for a path. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2021-06-08 +// Generated from API version 2023-11-03 // - options - PathClientFlushDataOptions contains the optional parameters for the PathClient.FlushData method. // - PathHTTPHeaders - PathHTTPHeaders contains a group of parameters for the PathClient.Create method. // - LeaseAccessConditions - LeaseAccessConditions contains a group of parameters for the PathClient.Create method. @@ -551,7 +563,7 @@ func (client *PathClient) flushDataCreateRequest(ctx context.Context, options *P if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2021-06-08"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if cpkInfo != nil && cpkInfo.EncryptionKey != nil { req.Raw().Header["x-ms-encryption-key"] = []string{*cpkInfo.EncryptionKey} } @@ -628,7 +640,7 @@ func (client *PathClient) flushDataHandleResponse(resp *http.Response) (PathClie // [https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations]. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2021-06-08 +// Generated from API version 2023-11-03 // - options - PathClientGetPropertiesOptions contains the optional parameters for the PathClient.GetProperties method. // - LeaseAccessConditions - LeaseAccessConditions contains a group of parameters for the PathClient.Create method. // - ModifiedAccessConditions - ModifiedAccessConditions contains a group of parameters for the FileSystemClient.SetProperties @@ -668,7 +680,7 @@ func (client *PathClient) getPropertiesCreateRequest(ctx context.Context, option if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2021-06-08"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if leaseAccessConditions != nil && leaseAccessConditions.LeaseID != nil { req.Raw().Header["x-ms-lease-id"] = []string{*leaseAccessConditions.LeaseID} } @@ -780,7 +792,7 @@ func (client *PathClient) getPropertiesHandleResponse(resp *http.Response) (Path // Operations [https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations]. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2021-06-08 +// Generated from API version 2023-11-03 // - xmsLeaseAction - There are five lease actions: "acquire", "break", "change", "renew", and "release". Use "acquire" and // specify the "x-ms-proposed-lease-id" and "x-ms-lease-duration" to acquire a new lease. Use "break" // to break an existing lease. When a lease is broken, the lease break period is allowed to elapse, during which time no lease @@ -823,7 +835,7 @@ func (client *PathClient) leaseCreateRequest(ctx context.Context, xmsLeaseAction if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2021-06-08"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} req.Raw().Header["x-ms-lease-action"] = []string{string(xmsLeaseAction)} req.Raw().Header["x-ms-lease-duration"] = []string{strconv.FormatInt(int64(client.xmsLeaseDuration), 10)} if options != nil && options.XMSLeaseBreakPeriod != nil { @@ -891,7 +903,7 @@ func (client *PathClient) leaseHandleResponse(resp *http.Response) (PathClientLe // Service Operations [https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations]. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2021-06-08 +// Generated from API version 2023-11-03 // - options - PathClientReadOptions contains the optional parameters for the PathClient.Read method. // - LeaseAccessConditions - LeaseAccessConditions contains a group of parameters for the PathClient.Create method. // - ModifiedAccessConditions - ModifiedAccessConditions contains a group of parameters for the FileSystemClient.SetProperties @@ -927,7 +939,7 @@ func (client *PathClient) readCreateRequest(ctx context.Context, options *PathCl if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2021-06-08"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if options != nil && options.Range != nil { req.Raw().Header["Range"] = []string{*options.Range} } @@ -1053,7 +1065,7 @@ func (client *PathClient) readHandleResponse(resp *http.Response) (PathClientRea // SetAccessControl - Set the owner, group, permissions, or access control list for a path. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2021-06-08 +// Generated from API version 2023-11-03 // - options - PathClientSetAccessControlOptions contains the optional parameters for the PathClient.SetAccessControl method. // - LeaseAccessConditions - LeaseAccessConditions contains a group of parameters for the PathClient.Create method. // - ModifiedAccessConditions - ModifiedAccessConditions contains a group of parameters for the FileSystemClient.SetProperties @@ -1115,7 +1127,7 @@ func (client *PathClient) setAccessControlCreateRequest(ctx context.Context, opt if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2021-06-08"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} req.Raw().Header["Accept"] = []string{"application/json"} return req, nil } @@ -1155,7 +1167,7 @@ func (client *PathClient) setAccessControlHandleResponse(resp *http.Response) (P // SetAccessControlRecursive - Set the access control list for a path and sub-paths. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2021-06-08 +// Generated from API version 2023-11-03 // - mode - Mode "set" sets POSIX access control rights on files and directories, "modify" modifies one or more POSIX access // control rights that pre-exist on files and directories, "remove" removes one or more // POSIX access control rights that were present earlier on files and directories @@ -1204,7 +1216,7 @@ func (client *PathClient) SetAccessControlRecursiveCreateRequest(ctx context.Con if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2021-06-08"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} req.Raw().Header["Accept"] = []string{"application/json"} return req, nil } @@ -1240,7 +1252,7 @@ func (client *PathClient) SetAccessControlRecursiveHandleResponse(resp *http.Res // SetExpiry - Sets the time a blob will expire and be deleted. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2021-06-08 +// Generated from API version 2023-11-03 // - expiryOptions - Required. Indicates mode of the expiry time // - options - PathClientSetExpiryOptions contains the optional parameters for the PathClient.SetExpiry method. func (client *PathClient) SetExpiry(ctx context.Context, expiryOptions ExpiryOptions, options *PathClientSetExpiryOptions) (PathClientSetExpiryResponse, error) { @@ -1270,7 +1282,7 @@ func (client *PathClient) setExpiryCreateRequest(ctx context.Context, expiryOpti reqQP.Set("timeout", strconv.FormatInt(int64(*options.Timeout), 10)) } req.Raw().URL.RawQuery = reqQP.Encode() - req.Raw().Header["x-ms-version"] = []string{"2021-06-08"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } @@ -1317,7 +1329,7 @@ func (client *PathClient) setExpiryHandleResponse(resp *http.Response) (PathClie // Undelete - Undelete a path that was previously soft deleted // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2021-06-08 +// Generated from API version 2023-11-03 // - options - PathClientUndeleteOptions contains the optional parameters for the PathClient.Undelete method. func (client *PathClient) Undelete(ctx context.Context, options *PathClientUndeleteOptions) (PathClientUndeleteResponse, error) { req, err := client.undeleteCreateRequest(ctx, options) @@ -1349,7 +1361,7 @@ func (client *PathClient) undeleteCreateRequest(ctx context.Context, options *Pa if options != nil && options.UndeleteSource != nil { req.Raw().Header["x-ms-undelete-source"] = []string{*options.UndeleteSource} } - req.Raw().Header["x-ms-version"] = []string{"2021-06-08"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } @@ -1389,7 +1401,7 @@ func (client *PathClient) undeleteHandleResponse(resp *http.Response) (PathClien // Headers for Blob Service Operations [https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations]. // If the operation fails it returns an *azcore.ResponseError type. // -// Generated from API version 2021-06-08 +// Generated from API version 2023-11-03 // - action - The action must be "append" to upload data to be appended to a file, "flush" to flush previously uploaded data // to a file, "setProperties" to set the properties of a file or directory, // "setAccessControl" to set the owner, group, permissions, or access control list for a file or directory, or "setAccessControlRecursive" @@ -1455,7 +1467,7 @@ func (client *PathClient) updateCreateRequest(ctx context.Context, action PathUp if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2021-06-08"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} if options != nil && options.ContentLength != nil { req.Raw().Header["Content-Length"] = []string{strconv.FormatInt(*options.ContentLength, 10)} } diff --git a/sdk/storage/azdatalake/internal/generated/zz_response_types.go b/sdk/storage/azdatalake/internal/generated/zz_response_types.go index fdf0d25fdc74..d8f3911c6a02 100644 --- a/sdk/storage/azdatalake/internal/generated/zz_response_types.go +++ b/sdk/storage/azdatalake/internal/generated/zz_response_types.go @@ -210,6 +210,15 @@ type PathClientDeleteResponse struct { // Version contains the information returned from the x-ms-version header response. Version *string + + // XMSContinuation contains the information returned from the x-ms-continuation header response. + XMSContinuation *string + + // XMSRequestID contains the information returned from the x-ms-request-id header response. + XMSRequestID *string + + // XMSVersion contains the information returned from the x-ms-version header response. + XMSVersion *string } // PathClientFlushDataResponse contains the response from method PathClient.FlushData. diff --git a/sdk/storage/azdatalake/internal/generated/zz_service_client.go b/sdk/storage/azdatalake/internal/generated/zz_service_client.go index a441fa581105..7a2040766276 100644 --- a/sdk/storage/azdatalake/internal/generated/zz_service_client.go +++ b/sdk/storage/azdatalake/internal/generated/zz_service_client.go @@ -29,7 +29,7 @@ type ServiceClient struct { // NewListFileSystemsPager - List filesystems and their properties in given account. // -// Generated from API version 2021-06-08 +// Generated from API version 2023-11-03 // - options - ServiceClientListFileSystemsOptions contains the optional parameters for the ServiceClient.NewListFileSystemsPager // method. // @@ -57,7 +57,7 @@ func (client *ServiceClient) ListFileSystemsCreateRequest(ctx context.Context, o if options != nil && options.RequestID != nil { req.Raw().Header["x-ms-client-request-id"] = []string{*options.RequestID} } - req.Raw().Header["x-ms-version"] = []string{"2021-06-08"} + req.Raw().Header["x-ms-version"] = []string{ServiceVersion} req.Raw().Header["Accept"] = []string{"application/json"} return req, nil } diff --git a/sdk/storage/azdatalake/internal/path/constants.go b/sdk/storage/azdatalake/internal/path/constants.go index 0744d15e9205..6062851c4e78 100644 --- a/sdk/storage/azdatalake/internal/path/constants.go +++ b/sdk/storage/azdatalake/internal/path/constants.go @@ -73,3 +73,12 @@ const ( StateTypeBreaking StateType = azdatalake.StateTypeBreaking StateTypeBroken StateType = azdatalake.StateTypeBroken ) + +type LeaseAction = generated.LeaseAction + +const ( + LeaseActionAcquire = generated.LeaseActionAcquire + LeaseActionRelease = generated.LeaseActionRelease + LeaseActionAcquireRelease = generated.LeaseActionAcquireRelease + LeaseActionRenew = generated.LeaseActionAutoRenew +) diff --git a/sdk/storage/azdatalake/internal/path/models.go b/sdk/storage/azdatalake/internal/path/models.go index 64cb04cc1aa5..96ab9f55def6 100644 --- a/sdk/storage/azdatalake/internal/path/models.go +++ b/sdk/storage/azdatalake/internal/path/models.go @@ -19,6 +19,7 @@ import ( type DeleteOptions struct { // AccessConditions contains parameters for accessing the path. AccessConditions *AccessConditions + Paginated *bool } func FormatDeleteOptions(o *DeleteOptions, recursive bool) (*generated.LeaseAccessConditions, *generated.ModifiedAccessConditions, *generated.PathClientDeleteOptions) { @@ -28,6 +29,7 @@ func FormatDeleteOptions(o *DeleteOptions, recursive bool) (*generated.LeaseAcce if o == nil { return nil, nil, deleteOpts } + deleteOpts.Paginated = o.Paginated leaseAccessConditions, modifiedAccessConditions := exported.FormatPathAccessConditions(o.AccessConditions) return leaseAccessConditions, modifiedAccessConditions, deleteOpts } diff --git a/sdk/storage/azdatalake/internal/path/responses.go b/sdk/storage/azdatalake/internal/path/responses.go index bc65cdf6d069..6033974f3464 100644 --- a/sdk/storage/azdatalake/internal/path/responses.go +++ b/sdk/storage/azdatalake/internal/path/responses.go @@ -157,6 +157,9 @@ type GetPropertiesResponse struct { // EncryptionScope contains the information returned from the x-ms-encryption-scope header response. EncryptionScope *string + // EncryptionContext contains the information returned from the x-ms-encryption-context header response. + EncryptionContext *string + // ExpiresOn contains the information returned from the x-ms-expiry-time header response. ExpiresOn *time.Time @@ -293,6 +296,9 @@ func FormatGetPropertiesResponse(r *blob.GetPropertiesResponse, rawResponse *htt if val := rawResponse.Header.Get("x-ms-resource-type"); val != "" { newResp.ResourceType = &val } + if val := rawResponse.Header.Get("x-ms-encryption-context"); val != "" { + newResp.EncryptionContext = &val + } return newResp } diff --git a/sdk/storage/azdatalake/internal/shared/challenge_policy.go b/sdk/storage/azdatalake/internal/shared/challenge_policy.go new file mode 100644 index 000000000000..4aea4ee83b9a --- /dev/null +++ b/sdk/storage/azdatalake/internal/shared/challenge_policy.go @@ -0,0 +1,113 @@ +//go:build go1.18 +// +build go1.18 + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +package shared + +import ( + "errors" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "net/http" + "strings" +) + +type storageAuthorizer struct { + scopes []string + tenantID string +} + +func NewStorageChallengePolicy(cred azcore.TokenCredential, audience string, allowHTTP bool) policy.Policy { + s := storageAuthorizer{scopes: []string{audience}} + return runtime.NewBearerTokenPolicy(cred, []string{audience}, &policy.BearerTokenOptions{ + AuthorizationHandler: policy.AuthorizationHandler{ + OnRequest: s.onRequest, + OnChallenge: s.onChallenge, + }, + InsecureAllowCredentialWithHTTP: allowHTTP, + }) +} + +func (s *storageAuthorizer) onRequest(req *policy.Request, authNZ func(policy.TokenRequestOptions) error) error { + return authNZ(policy.TokenRequestOptions{Scopes: s.scopes}) +} + +func (s *storageAuthorizer) onChallenge(req *policy.Request, resp *http.Response, authNZ func(policy.TokenRequestOptions) error) error { + // parse the challenge + err := s.parseChallenge(resp) + if err != nil { + return err + } + // TODO: Set tenantID when policy.TokenRequestOptions supports it. https://github.com/Azure/azure-sdk-for-go/issues/19841 + return authNZ(policy.TokenRequestOptions{Scopes: s.scopes}) +} + +type challengePolicyError struct { + err error +} + +func (c *challengePolicyError) Error() string { + return c.err.Error() +} + +func (*challengePolicyError) NonRetriable() { + // marker method +} + +func (c *challengePolicyError) Unwrap() error { + return c.err +} + +// parses Tenant ID from auth challenge +// https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000/oauth2/authorize +func parseTenant(url string) string { + if url == "" { + return "" + } + parts := strings.Split(url, "/") + if len(parts) >= 3 { + tenant := parts[3] + tenant = strings.ReplaceAll(tenant, ",", "") + return tenant + } else { + return "" + } +} + +func (s *storageAuthorizer) parseChallenge(resp *http.Response) error { + authHeader := resp.Header.Get("WWW-Authenticate") + if authHeader == "" { + return &challengePolicyError{err: errors.New("response has no WWW-Authenticate header for challenge authentication")} + } + + // Strip down to auth and resource + // Format is "Bearer authorization_uri=\"\" resource_id=\"\"" + authHeader = strings.ReplaceAll(authHeader, "Bearer ", "") + + parts := strings.Split(authHeader, " ") + + vals := map[string]string{} + for _, part := range parts { + subParts := strings.Split(part, "=") + if len(subParts) == 2 { + stripped := strings.ReplaceAll(subParts[1], "\"", "") + stripped = strings.TrimSuffix(stripped, ",") + vals[subParts[0]] = stripped + } + } + + s.tenantID = parseTenant(vals["authorization_uri"]) + + scope := vals["resource_id"] + if scope == "" { + return &challengePolicyError{err: errors.New("could not find a valid resource in the WWW-Authenticate header")} + } + + if !strings.HasSuffix(scope, "/.default") { + scope += "/.default" + } + s.scopes = []string{scope} + return nil +} diff --git a/sdk/storage/azdatalake/internal/shared/challenge_policy_test.go b/sdk/storage/azdatalake/internal/shared/challenge_policy_test.go new file mode 100644 index 000000000000..8eb25d8fa050 --- /dev/null +++ b/sdk/storage/azdatalake/internal/shared/challenge_policy_test.go @@ -0,0 +1,114 @@ +//go:build go1.18 +// +build go1.18 + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +package shared + +import ( + "context" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/internal/mock" + "github.com/stretchr/testify/require" + "strings" + "testing" + "time" +) + +type credentialFunc func(context.Context, policy.TokenRequestOptions) (azcore.AccessToken, error) + +func (cf credentialFunc) GetToken(ctx context.Context, options policy.TokenRequestOptions) (azcore.AccessToken, error) { + return cf(ctx, options) +} + +func TestChallengePolicyStorage(t *testing.T) { + accessToken := "***" + storageScope := "https://storage.azure.com/.default" + + srv, close := mock.NewServer(mock.WithTransformAllRequestsToTestServerUrl()) + defer close() + srv.AppendResponse( + mock.WithStatusCode(200), + ) + authenticated := false + cred := credentialFunc(func(ctx context.Context, tro policy.TokenRequestOptions) (azcore.AccessToken, error) { + authenticated = true + require.Equal(t, []string{storageScope}, tro.Scopes) + return azcore.AccessToken{Token: accessToken, ExpiresOn: time.Now().Add(time.Hour)}, nil + }) + p := NewStorageChallengePolicy(cred, storageScope, false) + pl := runtime.NewPipeline("", "", + runtime.PipelineOptions{PerRetry: []policy.Policy{p}}, + &policy.ClientOptions{Transport: srv}, + ) + req, err := runtime.NewRequest(context.Background(), "GET", "https://localhost") + require.NoError(t, err) + _, err = pl.Do(req) + require.NoError(t, err) + require.True(t, authenticated, "policy should have authenticated") +} + +func TestChallengePolicyDisk(t *testing.T) { + accessToken := "***" + diskResource := "https://disk.azure.com/" + diskScope := "https://disk.azure.com//.default" + challenge := `Bearer authorization_uri="https://login.microsoftonline.com/{tenant}", resource_id="{storageResource}"` + + srv, close := mock.NewServer(mock.WithTransformAllRequestsToTestServerUrl()) + defer close() + srv.AppendResponse( + mock.WithHeader("WWW-Authenticate", strings.ReplaceAll(challenge, "{storageResource}", diskResource)), + mock.WithStatusCode(401), + ) + srv.AppendResponse( + mock.WithStatusCode(200), + ) + attemptedAuthentication := false + authenticated := false + cred := credentialFunc(func(ctx context.Context, tro policy.TokenRequestOptions) (azcore.AccessToken, error) { + if attemptedAuthentication { + authenticated = true + require.Equal(t, []string{diskScope}, tro.Scopes) + return azcore.AccessToken{Token: accessToken, ExpiresOn: time.Now().Add(time.Hour)}, nil + } + attemptedAuthentication = true + return azcore.AccessToken{}, nil + }) + p := NewStorageChallengePolicy(cred, "https://storage.azure.com/.default", false) + pl := runtime.NewPipeline("", "", + runtime.PipelineOptions{PerRetry: []policy.Policy{p}}, + &policy.ClientOptions{Transport: srv}, + ) + req, err := runtime.NewRequest(context.Background(), "GET", "https://localhost") + require.NoError(t, err) + _, err = pl.Do(req) + require.NoError(t, err) + require.True(t, authenticated, "policy should have authenticated") +} + +func TestParseTenant(t *testing.T) { + actual := parseTenant("") + require.Empty(t, actual) + + expected := "00000000-0000-0000-0000-000000000000" + sampleURL := "https://login.microsoftonline.com/" + expected + actual = parseTenant(sampleURL) + require.Equal(t, expected, actual, "tenant was not properly parsed") +} + +func TestParseTenantNegative(t *testing.T) { + actual := parseTenant("") + require.Empty(t, actual) + + expected := "" + sampleURL := "https://login.microsoftonline.com/" + expected + actual = parseTenant(sampleURL) + require.Equal(t, expected, actual) + + sampleURL = "" + actual = parseTenant(sampleURL) + require.Equal(t, expected, actual) +} diff --git a/sdk/storage/azdatalake/internal/testcommon/clients_auth.go b/sdk/storage/azdatalake/internal/testcommon/clients_auth.go index 7a45a06c5385..b4587455aa88 100644 --- a/sdk/storage/azdatalake/internal/testcommon/clients_auth.go +++ b/sdk/storage/azdatalake/internal/testcommon/clients_auth.go @@ -12,6 +12,7 @@ import ( "fmt" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "testing" "time" @@ -58,11 +59,17 @@ var ( testEncryptedKey = "MDEyMzQ1NjcwMTIzNDU2NzAxMjM0NTY3MDEyMzQ1Njc=" testEncryptedHash = "3QFFFpRA5+XANHqwwbT4yXDmrT/2JaLt/FKHjzhOdoE=" testEncryptionAlgorithm = file.EncryptionAlgorithmTypeAES256 + TestEncryptionContext = "test_encryption_context" TestCPKByValue = file.CPKInfo{ EncryptionKey: &testEncryptedKey, EncryptionKeySHA256: &testEncryptedHash, EncryptionAlgorithm: &testEncryptionAlgorithm, } + TestEncryptionScope = "datalaketestencryptionscope" + TestCPKScopeInfo = container.CPKScopeInfo{ + DefaultEncryptionScope: &TestEncryptionScope, + PreventEncryptionScopeOverride: to.Ptr(false), + } ) var BasicHeaders = file.HTTPHeaders{ diff --git a/sdk/storage/azdatalake/service/client.go b/sdk/storage/azdatalake/service/client.go index 327871cb4f56..fa2546c7bdb1 100644 --- a/sdk/storage/azdatalake/service/client.go +++ b/sdk/storage/azdatalake/service/client.go @@ -39,8 +39,9 @@ type Client base.CompositeClient[generated.ServiceClient, generated_blob.Service // - options - client options; pass nil to accept the default values func NewClient(serviceURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) { blobServiceURL, datalakeServiceURL := shared.GetURLs(serviceURL) - authPolicy := runtime.NewBearerTokenPolicy(cred, []string{shared.TokenScope}, nil) + audience := base.GetAudience((*base.ClientOptions)(options)) conOptions := shared.GetClientOptions(options) + authPolicy := shared.NewStorageChallengePolicy(cred, audience, conOptions.InsecureAllowCredentialWithHTTP) plOpts := runtime.PipelineOptions{ PerRetry: []policy.Policy{authPolicy}, } diff --git a/sdk/storage/azdatalake/service/client_test.go b/sdk/storage/azdatalake/service/client_test.go index 7ac500c0fc9c..1af7e1491b57 100644 --- a/sdk/storage/azdatalake/service/client_test.go +++ b/sdk/storage/azdatalake/service/client_test.go @@ -92,6 +92,35 @@ func (s *ServiceUnrecordedTestsSuite) TestServiceClientFromConnectionString() { defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) } +func (s *ServiceRecordedTestsSuite) TestCreateFilesystemsWithOptions() { + _require := require.New(s.T()) + testName := s.T().Name() + svcClient, err := testcommon.GetServiceClientFromConnectionString(s.T(), testcommon.TestAccountDefault, nil) + _require.NoError(err) + md := map[string]*string{ + "foo": to.Ptr("foovalue"), + "bar": to.Ptr("barvalue"), + } + cpkScopeInfo := &testcommon.TestCPKScopeInfo + + fsName := testcommon.GenerateFileSystemName(testName) + fsClient := testcommon.ServiceGetFileSystemClient(fsName, svcClient) + + _, err = fsClient.Create(context.Background(), &filesystem.CreateOptions{Metadata: md, CPKScopeInfo: cpkScopeInfo}) + defer func(fsClient *filesystem.Client, ctx context.Context, options *filesystem.DeleteOptions) { + _, err := fsClient.Delete(ctx, options) + if err != nil { + _require.NoError(err) + } + }(fsClient, context.Background(), nil) + + _require.NoError(err) + resp, err := fsClient.GetProperties(context.Background(), nil) + + _require.NoError(err) + _require.Equal(resp.DefaultEncryptionScope, &testcommon.TestEncryptionScope) +} + func (s *ServiceRecordedTestsSuite) TestSetPropertiesLogging() { _require := require.New(s.T()) svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil) @@ -831,3 +860,75 @@ func (s *ServiceRecordedTestsSuite) TestServiceClientWithNilSharedKey() { _require.Error(err) _require.Nil(svcClient) } + +func (s *ServiceRecordedTestsSuite) TestServiceClientUsingOauth() { + _require := require.New(s.T()) + + accountName, _ := testcommon.GetGenericAccountInfo(testcommon.TestAccountDatalake) + _require.Greater(len(accountName), 0) + + cred, err := testcommon.GetGenericTokenCredential() + _require.NoError(err) + + serviceUrl := "https://" + accountName + ".dfs.core.windows.net/" + + svcClient, err := service.NewClient(serviceUrl, cred, nil) + _require.NoError(err) + _require.NotNil(svcClient) + + fs, _ := svcClient.CreateFileSystem(context.Background(), "test", nil) + _require.NotNil(fs) + _require.NoError(err) +} + +func (s *ServiceRecordedTestsSuite) TestServiceClientUsingOauthWithDefaultAudience() { + _require := require.New(s.T()) + + accountName, _ := testcommon.GetGenericAccountInfo(testcommon.TestAccountDatalake) + _require.Greater(len(accountName), 0) + + cred, err := testcommon.GetGenericTokenCredential() + _require.NoError(err) + + serviceUrl := "https://" + accountName + ".dfs.core.windows.net/" + + options := service.ClientOptions{ + Audience: "https://storage.azure.com/", + } + + testcommon.SetClientOptions(s.T(), &options.ClientOptions) + svcClient, err := service.NewClient(serviceUrl, cred, &options) + _require.NoError(err) + _require.NotNil(svcClient) + + fs, _ := svcClient.CreateFileSystem(context.Background(), "test", nil) + _require.NotNil(fs) + _require.NoError(err) + +} + +func (s *ServiceRecordedTestsSuite) TestServiceClientUsingOauthWithCustomAudience() { + _require := require.New(s.T()) + + accountName, _ := testcommon.GetGenericAccountInfo(testcommon.TestAccountDatalake) + _require.Greater(len(accountName), 0) + + serviceUrl := "https://" + accountName + ".dfs.core.windows.net/" + + cred, err := testcommon.GetGenericTokenCredential() + _require.NoError(err) + + options := service.ClientOptions{ + Audience: "https://" + accountName + ".blob.core.windows.net", + } + + testcommon.SetClientOptions(s.T(), &options.ClientOptions) + svcClient, err := service.NewClient(serviceUrl, cred, &options) + _require.NoError(err) + _require.NotNil(svcClient) + + fs, _ := svcClient.CreateFileSystem(context.Background(), "test", nil) + _require.NotNil(fs) + _require.NoError(err) + +}