From 5e73ff65d5b86b9a12b426d1d4ef1243d41196ef Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Thu, 29 Feb 2024 15:52:06 +0530 Subject: [PATCH 1/9] append and flush lease operations --- sdk/storage/azdatalake/file/client.go | 7 - sdk/storage/azdatalake/file/client_test.go | 192 ++++++++++++++++++ sdk/storage/azdatalake/file/models.go | 56 ++++- .../azdatalake/internal/exported/exported.go | 3 + .../azdatalake/internal/path/models.go | 2 + .../internal/testcommon/clients_auth.go | 7 + 6 files changed, 254 insertions(+), 13 deletions(-) diff --git a/sdk/storage/azdatalake/file/client.go b/sdk/storage/azdatalake/file/client.go index 75dbd8fccbfb..2724ce67c7b2 100644 --- a/sdk/storage/azdatalake/file/client.go +++ b/sdk/storage/azdatalake/file/client.go @@ -411,13 +411,6 @@ func (f *Client) AppendData(ctx context.Context, offset int64, body io.ReadSeekC return AppendDataResponse{}, err } resp, err := f.generatedFileClientWithDFS().AppendData(ctx, body, appendDataOptions, nil, leaseAccessConditions, cpkInfo) - // TODO: check and uncomment this - //if err != nil { - // _, err1 := body.Seek(0, io.SeekStart) - // if err1 != nil { - // return AppendDataResponse{}, err1 - // } - //} return resp, exported.ConvertToDFSError(err) } diff --git a/sdk/storage/azdatalake/file/client_test.go b/sdk/storage/azdatalake/file/client_test.go index 7532b83f7d7f..e789d6c884f9 100644 --- a/sdk/storage/azdatalake/file/client_test.go +++ b/sdk/storage/azdatalake/file/client_test.go @@ -12,6 +12,8 @@ import ( "crypto/md5" "encoding/binary" "fmt" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/path" "hash/crc64" "io" "math/rand" @@ -3459,6 +3461,196 @@ func (s *RecordedTestSuite) TestDownloadDataContentMD5() { _require.Equal(resp1.ContentMD5, mdf[:]) } +func (s *RecordedTestSuite) TestFileAppendDataWithAcquireLease() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + srcFileName := "src" + testcommon.GenerateFileName(testName) + + srcFClient, err := testcommon.GetFileClient(filesystemName, srcFileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + resp, err := srcFClient.Create(context.Background(), nil) + _require.NoError(err) + _require.NotNil(resp) + + contentSize := 1024 * 8 // 8KB + rsc, _ := testcommon.GenerateData(contentSize) + + opts := &file.AppendDataOptions{ + LeaseAction: &testcommon.TestLeaseActionAcquire, + LeaseDuration: to.Ptr(int64(15)), + ProposedLeaseID: proposedLeaseIDs[1], + } + _, err = srcFClient.AppendData(context.Background(), 0, rsc, opts) + _require.NoError(err) + + gResp2, err := srcFClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(lease.StateTypeLeased, *gResp2.LeaseState) + + time.Sleep(time.Second * 15) + + //Check if the lease was acquired for the right duration + gResp, err := srcFClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(lease.StateTypeExpired, *gResp.LeaseState) +} + +func (s *RecordedTestSuite) TestFileAppendDataWithRenewLease() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + srcFileName := "src" + testcommon.GenerateFileName(testName) + + srcFClient, err := testcommon.GetFileClient(filesystemName, srcFileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + createOpts := file.CreateOptions{ + ProposedLeaseID: proposedLeaseIDs[0], + LeaseDuration: to.Ptr(int64(15)), + } + + resp, err := srcFClient.Create(context.Background(), &createOpts) + _require.NoError(err) + _require.NotNil(resp) + + gResp2, err := srcFClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(lease.StateTypeLeased, *gResp2.LeaseState) + + //Wait for 15 seconds for lease to expire + time.Sleep(15 * time.Second) + + gResp, err := srcFClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(lease.StateTypeExpired, *gResp.LeaseState) + + contentSize := 1024 * 8 // 8KB + rsc, _ := testcommon.GenerateData(contentSize) + + opts := &file.AppendDataOptions{ + //LeaseAction: &testcommon.TestLeaseActionRenew, + LeaseAccessConditions: &file.LeaseAccessConditions{LeaseID: proposedLeaseIDs[0]}, + //LeaseDuration: to.Ptr(int64(-1)), + } + _, err = srcFClient.AppendData(context.Background(), 0, rsc, opts) + _require.NoError(err) + + gResp2, err = srcFClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(lease.StateTypeLeased, *gResp2.LeaseState) +} + +func (s *RecordedTestSuite) TestFileAppendDataWithReleaseLease() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + srcFileName := "src" + testcommon.GenerateFileName(testName) + + srcFClient, err := testcommon.GetFileClient(filesystemName, srcFileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + createOpts := file.CreateOptions{ + ProposedLeaseID: proposedLeaseIDs[0], + LeaseDuration: to.Ptr(int64(15)), + } + + resp, err := srcFClient.Create(context.Background(), &createOpts) + _require.NoError(err) + _require.NotNil(resp) + + contentSize := 1024 * 8 // 8KB + rsc, _ := testcommon.GenerateData(contentSize) + + opts := &file.AppendDataOptions{ + //LeaseAction: &testcommon.TestLeaseActionRelease, + LeaseAccessConditions: &file.LeaseAccessConditions{LeaseID: proposedLeaseIDs[0]}, + //Flush: to.Ptr(true), + } + _, err = srcFClient.AppendData(context.Background(), 0, rsc, opts) + _require.NoError(err) + + gResp, err := srcFClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(lease.StateTypeAvailable, *gResp.LeaseState) +} + +func (s *RecordedTestSuite) TestFileAppendWithFlushReleaseLease() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + srcFileName := "src" + testcommon.GenerateFileName(testName) + + srcFClient, err := testcommon.GetFileClient(filesystemName, srcFileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + createOpts := file.CreateOptions{ + ProposedLeaseID: proposedLeaseIDs[0], + LeaseDuration: to.Ptr(int64(15)), + } + + resp, err := srcFClient.Create(context.Background(), &createOpts) + _require.NoError(err) + _require.NotNil(resp) + + contentSize := 1024 * 8 // 8KB + rsc, _ := testcommon.GenerateData(contentSize) + + _, err = srcFClient.AppendData(context.Background(), 0, rsc, + &file.AppendDataOptions{ + LeaseAccessConditions: &file.LeaseAccessConditions{LeaseID: proposedLeaseIDs[0]}, + }) + _require.NoError(err) + + opts := &file.FlushDataOptions{ + LeaseAction: &testcommon.TestLeaseActionRelease, + AccessConditions: &path.AccessConditions{ + LeaseAccessConditions: &path.LeaseAccessConditions{LeaseID: proposedLeaseIDs[0]}, + }, + } + + _, err = srcFClient.FlushData(context.Background(), int64(contentSize), opts) + _require.NoError(err) + + gResp2, err := srcFClient.GetProperties(context.Background(), nil) + _require.NoError(err) + _require.Equal(*gResp2.ContentLength, int64(contentSize)) + _require.Equal(lease.StateTypeAvailable, *gResp2.LeaseState) +} + func (s *RecordedTestSuite) TestFileAppendAndFlushData() { _require := require.New(s.T()) testName := s.T().Name() diff --git a/sdk/storage/azdatalake/file/models.go b/sdk/storage/azdatalake/file/models.go index 27fa5e163166..382d654df0ac 100644 --- a/sdk/storage/azdatalake/file/models.go +++ b/sdk/storage/azdatalake/file/models.go @@ -185,6 +185,13 @@ type FlushDataOptions struct { // RetainUncommittedData if "true", uncommitted data is retained after the flush operation // completes, otherwise, the uncommitted data is deleted after the flush operation. RetainUncommittedData *bool + // LeaseAction Describes actions that can be performed on a lease. + LeaseAction *LeaseAction + // LeaseDuration specifies the duration of the lease, in seconds, or negative one + // (-1) for a lease that never expires. A non-infinite lease can be between 15 and 60 seconds. + LeaseDuration *int64 + // ProposedLeaseID specifies the proposed lease ID for the file. + ProposedLeaseID *string } func (o *FlushDataOptions) format(offset int64) (*generated.PathClientFlushDataOptions, *generated.ModifiedAccessConditions, *generated.LeaseAccessConditions, *generated.PathHTTPHeaders, *generated.CPKInfo, error) { @@ -230,6 +237,15 @@ func (o *FlushDataOptions) format(offset int64) (*generated.PathClientFlushDataO cpkInfoOpts.EncryptionKeySHA256 = o.CPKInfo.EncryptionKeySHA256 cpkInfoOpts.EncryptionAlgorithm = o.CPKInfo.EncryptionAlgorithm } + if o.LeaseAction != nil { + flushDataOpts.LeaseAction = o.LeaseAction + } + if o.LeaseDuration != nil { + flushDataOpts.LeaseDuration = o.LeaseDuration + } + if o.ProposedLeaseID != nil { + flushDataOpts.ProposedLeaseID = o.ProposedLeaseID + } } return flushDataOpts, modifiedAccessConditions, leaseAccessConditions, httpHeaderOpts, cpkInfoOpts, nil } @@ -241,11 +257,22 @@ type AppendDataOptions struct { TransactionalValidation TransferValidationType // LeaseAccessConditions contains optional parameters to access leased entity. LeaseAccessConditions *LeaseAccessConditions + // If file should be flushed after append + Flush *bool + // LeaseAction Describes actions that can be performed on a lease. + LeaseAction *LeaseAction + // LeaseDuration specifies the duration of the lease, in seconds, or negative one + // (-1) for a lease that never expires. A non-infinite lease can be between 15 and 60 seconds. + LeaseDuration *int64 + // ProposedLeaseID specifies the proposed lease ID for the file. + ProposedLeaseID *string // CPKInfo contains optional parameters to perform encryption using customer-provided key. CPKInfo *CPKInfo } -func (o *AppendDataOptions) format(offset int64, body io.ReadSeekCloser) (*generated.PathClientAppendDataOptions, *generated.LeaseAccessConditions, *generated.CPKInfo, error) { +func (o *AppendDataOptions) format(offset int64, body io.ReadSeekCloser) (*generated.PathClientAppendDataOptions, + *generated.LeaseAccessConditions, *generated.CPKInfo, error) { + if offset < 0 || body == nil { return nil, nil, nil, errors.New("invalid argument: offset must be >= 0 and body must not be nil") } @@ -260,6 +287,9 @@ func (o *AppendDataOptions) format(offset int64, body io.ReadSeekCloser) (*gener } appendDataOptions := &generated.PathClientAppendDataOptions{} + if o == nil { + return appendDataOptions, nil, nil, nil + } httpRange := exported.FormatHTTPRange(HTTPRange{ Offset: offset, Count: count, @@ -280,11 +310,23 @@ func (o *AppendDataOptions) format(offset int64, body io.ReadSeekCloser) (*gener cpkInfoOpts.EncryptionKeySHA256 = o.CPKInfo.EncryptionKeySHA256 cpkInfoOpts.EncryptionAlgorithm = o.CPKInfo.EncryptionAlgorithm } - } - if o != nil && o.TransactionalValidation != nil { - _, err = o.TransactionalValidation.Apply(body, appendDataOptions) - if err != nil { - return nil, nil, nil, err + if o.LeaseAction != nil { + appendDataOptions.LeaseAction = o.LeaseAction + } + if o.LeaseDuration != nil { + appendDataOptions.LeaseDuration = o.LeaseDuration + } + if o.ProposedLeaseID != nil { + appendDataOptions.ProposedLeaseID = o.ProposedLeaseID + } + if o.Flush != nil { + appendDataOptions.Flush = o.Flush + } + if o.TransactionalValidation != nil { + _, err = o.TransactionalValidation.Apply(body, appendDataOptions) + if err != nil { + return nil, nil, nil, err + } } } @@ -506,6 +548,8 @@ type SetExpiryValues struct { // ACLFailedEntry contains the failed ACL entry (response model). type ACLFailedEntry = path.ACLFailedEntry +type LeaseAction = path.LeaseAction + // SetAccessControlRecursiveResponse contains part of the response data returned by the []OP_AccessControl operations. type SetAccessControlRecursiveResponse = generated.SetAccessControlRecursiveResponse diff --git a/sdk/storage/azdatalake/internal/exported/exported.go b/sdk/storage/azdatalake/internal/exported/exported.go index 3b32ee961c65..fe33eaea4bcc 100644 --- a/sdk/storage/azdatalake/internal/exported/exported.go +++ b/sdk/storage/azdatalake/internal/exported/exported.go @@ -11,6 +11,7 @@ import ( "fmt" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/generated" "strconv" "strings" ) @@ -22,6 +23,8 @@ const SnapshotTimeFormat = "2006-01-02T15:04:05.0000000Z07:00" // which has an offset and zero value count indicates from the offset to the resource's end. type HTTPRange = blob.HTTPRange +type LeaseAction = generated.LeaseAction + // FormatHTTPRange converts an HTTPRange to its string format. func FormatHTTPRange(r HTTPRange) *string { if r.Offset == 0 && r.Count == 0 { diff --git a/sdk/storage/azdatalake/internal/path/models.go b/sdk/storage/azdatalake/internal/path/models.go index 64cb04cc1aa5..9641519ab090 100644 --- a/sdk/storage/azdatalake/internal/path/models.go +++ b/sdk/storage/azdatalake/internal/path/models.go @@ -315,3 +315,5 @@ type CPKScopeInfo = blob.CPKScopeInfo // ACLFailedEntry contains the failed ACL entry (response model). type ACLFailedEntry = generated.ACLFailedEntry + +type LeaseAction = generated.LeaseAction diff --git a/sdk/storage/azdatalake/internal/testcommon/clients_auth.go b/sdk/storage/azdatalake/internal/testcommon/clients_auth.go index 7a45a06c5385..f8441317d840 100644 --- a/sdk/storage/azdatalake/internal/testcommon/clients_auth.go +++ b/sdk/storage/azdatalake/internal/testcommon/clients_auth.go @@ -74,6 +74,13 @@ var BasicHeaders = file.HTTPHeaders{ ContentEncoding: &DatalakeContentEncoding, } +var ( + TestLeaseActionAcquire = file.LeaseAction("acquire") + TestLeaseActionRelease = file.LeaseAction("release") + TestLeaseActionRenew = file.LeaseAction("auto-renew") + TestLeaseActionAcquireRelease = file.LeaseAction("acquire-release") +) + type TestAccountType string const ( From 2feadbe91213e172c5fc31a789d84292b381a983 Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Thu, 29 Feb 2024 17:15:16 +0530 Subject: [PATCH 2/9] recording, and test update --- sdk/storage/azdatalake/CHANGELOG.md | 2 ++ sdk/storage/azdatalake/assets.json | 2 +- sdk/storage/azdatalake/file/client_test.go | 8 ++++---- sdk/storage/azdatalake/file/models.go | 3 --- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/sdk/storage/azdatalake/CHANGELOG.md b/sdk/storage/azdatalake/CHANGELOG.md index 43e1aedce9ee..ca38bede64d5 100644 --- a/sdk/storage/azdatalake/CHANGELOG.md +++ b/sdk/storage/azdatalake/CHANGELOG.md @@ -3,6 +3,8 @@ ## 1.1.1 (Unreleased) ### Features Added +* Append with acquire, release lease and renewal of lease. +* Flush with release lease ### Breaking Changes diff --git a/sdk/storage/azdatalake/assets.json b/sdk/storage/azdatalake/assets.json index da2dfd857383..a51aed3097d4 100644 --- a/sdk/storage/azdatalake/assets.json +++ b/sdk/storage/azdatalake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "go", "TagPrefix": "go/storage/azdatalake", - "Tag": "go/storage/azdatalake_6e4e5b7c87" + "Tag": "go/storage/azdatalake_0c9e3c6f2e" } diff --git a/sdk/storage/azdatalake/file/client_test.go b/sdk/storage/azdatalake/file/client_test.go index e789d6c884f9..d59945aa922d 100644 --- a/sdk/storage/azdatalake/file/client_test.go +++ b/sdk/storage/azdatalake/file/client_test.go @@ -3546,9 +3546,9 @@ func (s *RecordedTestSuite) TestFileAppendDataWithRenewLease() { rsc, _ := testcommon.GenerateData(contentSize) opts := &file.AppendDataOptions{ - //LeaseAction: &testcommon.TestLeaseActionRenew, + LeaseAction: &testcommon.TestLeaseActionRenew, LeaseAccessConditions: &file.LeaseAccessConditions{LeaseID: proposedLeaseIDs[0]}, - //LeaseDuration: to.Ptr(int64(-1)), + LeaseDuration: to.Ptr(int64(-1)), } _, err = srcFClient.AppendData(context.Background(), 0, rsc, opts) _require.NoError(err) @@ -3588,9 +3588,9 @@ func (s *RecordedTestSuite) TestFileAppendDataWithReleaseLease() { rsc, _ := testcommon.GenerateData(contentSize) opts := &file.AppendDataOptions{ - //LeaseAction: &testcommon.TestLeaseActionRelease, + LeaseAction: &testcommon.TestLeaseActionRelease, LeaseAccessConditions: &file.LeaseAccessConditions{LeaseID: proposedLeaseIDs[0]}, - //Flush: to.Ptr(true), + Flush: to.Ptr(true), } _, err = srcFClient.AppendData(context.Background(), 0, rsc, opts) _require.NoError(err) diff --git a/sdk/storage/azdatalake/file/models.go b/sdk/storage/azdatalake/file/models.go index 382d654df0ac..9fabec977756 100644 --- a/sdk/storage/azdatalake/file/models.go +++ b/sdk/storage/azdatalake/file/models.go @@ -287,9 +287,6 @@ func (o *AppendDataOptions) format(offset int64, body io.ReadSeekCloser) (*gener } appendDataOptions := &generated.PathClientAppendDataOptions{} - if o == nil { - return appendDataOptions, nil, nil, nil - } httpRange := exported.FormatHTTPRange(HTTPRange{ Offset: offset, Count: count, From da2b18fcb9fcab861d9db15dcb23775ea65577ba Mon Sep 17 00:00:00 2001 From: tanyasethi-msft <124860586+tanyasethi-msft@users.noreply.github.com> Date: Thu, 29 Feb 2024 17:41:21 +0530 Subject: [PATCH 3/9] fix indentation issue in test file --- sdk/storage/azdatalake/file/client_test.go | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/sdk/storage/azdatalake/file/client_test.go b/sdk/storage/azdatalake/file/client_test.go index 0d5e033443d6..a1f3a6d2b497 100644 --- a/sdk/storage/azdatalake/file/client_test.go +++ b/sdk/storage/azdatalake/file/client_test.go @@ -47,17 +47,7 @@ var proposedLeaseIDs = []*string{to.Ptr("c820a799-76d7-4ee2-6e15-546f19325c2c"), func Test(t *testing.T) { recordMode := recording.GetRecordMode() - t. - - - - - - - - - - f("Running datalake Tests in %s mode\n", recordMode) + t.Logf("Running datalake Tests in %s mode\n", recordMode) if recordMode == recording.LiveMode { suite.Run(t, &RecordedTestSuite{}) suite.Run(t, &UnrecordedTestSuite{}) From e769ffa64ff28a9ec01b381e0c5b955d81e7d4d7 Mon Sep 17 00:00:00 2001 From: tanyasethi-msft <124860586+tanyasethi-msft@users.noreply.github.com> Date: Thu, 29 Feb 2024 17:59:54 +0530 Subject: [PATCH 4/9] remove redeclaration --- sdk/storage/azdatalake/file/models.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/sdk/storage/azdatalake/file/models.go b/sdk/storage/azdatalake/file/models.go index 0501fbb4d6b5..d195a9e0b8eb 100644 --- a/sdk/storage/azdatalake/file/models.go +++ b/sdk/storage/azdatalake/file/models.go @@ -257,8 +257,6 @@ type AppendDataOptions struct { TransactionalValidation TransferValidationType // LeaseAccessConditions contains optional parameters to access leased entity. LeaseAccessConditions *LeaseAccessConditions - // If file should be flushed after append - Flush *bool // LeaseAction Describes actions that can be performed on a lease. LeaseAction *LeaseAction // LeaseDuration specifies the duration of the lease, in seconds, or negative one From e00ee4a9f6e0c9dfafaf02051df4b9c944c634c8 Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Fri, 1 Mar 2024 11:50:03 +0530 Subject: [PATCH 5/9] fix formatting errors --- sdk/storage/azdatalake/assets.json | 2 +- sdk/storage/azdatalake/file/client_test.go | 6 +++--- sdk/storage/azdatalake/file/models.go | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/sdk/storage/azdatalake/assets.json b/sdk/storage/azdatalake/assets.json index a51aed3097d4..a15e5dbe8505 100644 --- a/sdk/storage/azdatalake/assets.json +++ b/sdk/storage/azdatalake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "go", "TagPrefix": "go/storage/azdatalake", - "Tag": "go/storage/azdatalake_0c9e3c6f2e" + "Tag": "go/storage/azdatalake_9d160e2359" } diff --git a/sdk/storage/azdatalake/file/client_test.go b/sdk/storage/azdatalake/file/client_test.go index 60ef1037ac85..ec030efbe843 100644 --- a/sdk/storage/azdatalake/file/client_test.go +++ b/sdk/storage/azdatalake/file/client_test.go @@ -12,9 +12,9 @@ import ( "crypto/md5" "encoding/binary" "fmt" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/log" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease" "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/path" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/log" "hash/crc64" "io" "math/rand" @@ -3464,7 +3464,6 @@ func (s *RecordedTestSuite) TestDownloadDataContentMD5() { } func (s *RecordedTestSuite) TestFileAppendDataWithAcquireLease() { - _require := require.New(s.T()) testName := s.T().Name() @@ -3590,11 +3589,12 @@ func (s *RecordedTestSuite) TestFileAppendDataWithReleaseLease() { contentSize := 1024 * 8 // 8KB rsc, _ := testcommon.GenerateData(contentSize) - opts := &file.AppendDataOptions { + opts := &file.AppendDataOptions{ LeaseAction: &testcommon.TestLeaseActionRelease, LeaseAccessConditions: &file.LeaseAccessConditions{LeaseID: proposedLeaseIDs[0]}, Flush: to.Ptr(true), } + _, err = srcFClient.AppendData(context.Background(), 0, rsc, opts) _require.NoError(err) diff --git a/sdk/storage/azdatalake/file/models.go b/sdk/storage/azdatalake/file/models.go index d195a9e0b8eb..1a7ad03bcbc6 100644 --- a/sdk/storage/azdatalake/file/models.go +++ b/sdk/storage/azdatalake/file/models.go @@ -257,7 +257,7 @@ type AppendDataOptions struct { TransactionalValidation TransferValidationType // LeaseAccessConditions contains optional parameters to access leased entity. LeaseAccessConditions *LeaseAccessConditions - // LeaseAction Describes actions that can be performed on a lease. + // LeaseAction describes actions that can be performed on a lease. LeaseAction *LeaseAction // LeaseDuration specifies the duration of the lease, in seconds, or negative one // (-1) for a lease that never expires. A non-infinite lease can be between 15 and 60 seconds. @@ -546,6 +546,7 @@ type SetExpiryValues struct { // ACLFailedEntry contains the failed ACL entry (response model). type ACLFailedEntry = path.ACLFailedEntry +// LeaseAction describes actions that can be performed on a lease. type LeaseAction = path.LeaseAction // SetAccessControlRecursiveResponse contains part of the response data returned by the []OP_AccessControl operations. From 9f87cac093362372ebd42a8d1055b11daafc93fa Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Fri, 1 Mar 2024 18:34:40 +0530 Subject: [PATCH 6/9] code refactor --- sdk/storage/azdatalake/file/client_test.go | 8 ++++---- sdk/storage/azdatalake/file/constants.go | 9 +++++++++ sdk/storage/azdatalake/file/models.go | 3 --- sdk/storage/azdatalake/internal/exported/exported.go | 3 --- sdk/storage/azdatalake/internal/path/constants.go | 9 +++++++++ sdk/storage/azdatalake/internal/path/models.go | 2 -- .../azdatalake/internal/testcommon/clients_auth.go | 7 ------- 7 files changed, 22 insertions(+), 19 deletions(-) diff --git a/sdk/storage/azdatalake/file/client_test.go b/sdk/storage/azdatalake/file/client_test.go index ec030efbe843..00de8adbf399 100644 --- a/sdk/storage/azdatalake/file/client_test.go +++ b/sdk/storage/azdatalake/file/client_test.go @@ -3488,7 +3488,7 @@ func (s *RecordedTestSuite) TestFileAppendDataWithAcquireLease() { rsc, _ := testcommon.GenerateData(contentSize) opts := &file.AppendDataOptions{ - LeaseAction: &testcommon.TestLeaseActionAcquire, + LeaseAction: &file.LeaseActionAcquire, LeaseDuration: to.Ptr(int64(15)), ProposedLeaseID: proposedLeaseIDs[1], } @@ -3548,7 +3548,7 @@ func (s *RecordedTestSuite) TestFileAppendDataWithRenewLease() { rsc, _ := testcommon.GenerateData(contentSize) opts := &file.AppendDataOptions{ - LeaseAction: &testcommon.TestLeaseActionRenew, + LeaseAction: &file.LeaseActionRenew, LeaseAccessConditions: &file.LeaseAccessConditions{LeaseID: proposedLeaseIDs[0]}, LeaseDuration: to.Ptr(int64(-1)), } @@ -3590,7 +3590,7 @@ func (s *RecordedTestSuite) TestFileAppendDataWithReleaseLease() { rsc, _ := testcommon.GenerateData(contentSize) opts := &file.AppendDataOptions{ - LeaseAction: &testcommon.TestLeaseActionRelease, + LeaseAction: &file.LeaseActionRelease, LeaseAccessConditions: &file.LeaseAccessConditions{LeaseID: proposedLeaseIDs[0]}, Flush: to.Ptr(true), } @@ -3639,7 +3639,7 @@ func (s *RecordedTestSuite) TestFileAppendWithFlushReleaseLease() { _require.NoError(err) opts := &file.FlushDataOptions{ - LeaseAction: &testcommon.TestLeaseActionRelease, + LeaseAction: &file.LeaseActionRelease, AccessConditions: &path.AccessConditions{ LeaseAccessConditions: &path.LeaseAccessConditions{LeaseID: proposedLeaseIDs[0]}, }, diff --git a/sdk/storage/azdatalake/file/constants.go b/sdk/storage/azdatalake/file/constants.go index 77729179d3d4..7e2952f56aad 100644 --- a/sdk/storage/azdatalake/file/constants.go +++ b/sdk/storage/azdatalake/file/constants.go @@ -124,3 +124,12 @@ const ( StateTypeBreaking StateType = azdatalake.StateTypeBreaking StateTypeBroken StateType = azdatalake.StateTypeBroken ) + +type LeaseAction = path.LeaseAction + +var ( + LeaseActionAcquire LeaseAction = path.LeaseActionAcquire + LeaseActionRelease LeaseAction = path.LeaseActionRelease + LeaseActionAcquireRelease LeaseAction = path.LeaseActionAcquireRelease + LeaseActionRenew LeaseAction = path.LeaseActionRenew +) diff --git a/sdk/storage/azdatalake/file/models.go b/sdk/storage/azdatalake/file/models.go index 1a7ad03bcbc6..dd583804e0eb 100644 --- a/sdk/storage/azdatalake/file/models.go +++ b/sdk/storage/azdatalake/file/models.go @@ -546,9 +546,6 @@ type SetExpiryValues struct { // ACLFailedEntry contains the failed ACL entry (response model). type ACLFailedEntry = path.ACLFailedEntry -// LeaseAction describes actions that can be performed on a lease. -type LeaseAction = path.LeaseAction - // SetAccessControlRecursiveResponse contains part of the response data returned by the []OP_AccessControl operations. type SetAccessControlRecursiveResponse = generated.SetAccessControlRecursiveResponse diff --git a/sdk/storage/azdatalake/internal/exported/exported.go b/sdk/storage/azdatalake/internal/exported/exported.go index fe33eaea4bcc..3b32ee961c65 100644 --- a/sdk/storage/azdatalake/internal/exported/exported.go +++ b/sdk/storage/azdatalake/internal/exported/exported.go @@ -11,7 +11,6 @@ import ( "fmt" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/generated" "strconv" "strings" ) @@ -23,8 +22,6 @@ const SnapshotTimeFormat = "2006-01-02T15:04:05.0000000Z07:00" // which has an offset and zero value count indicates from the offset to the resource's end. type HTTPRange = blob.HTTPRange -type LeaseAction = generated.LeaseAction - // FormatHTTPRange converts an HTTPRange to its string format. func FormatHTTPRange(r HTTPRange) *string { if r.Offset == 0 && r.Count == 0 { diff --git a/sdk/storage/azdatalake/internal/path/constants.go b/sdk/storage/azdatalake/internal/path/constants.go index 0744d15e9205..6062851c4e78 100644 --- a/sdk/storage/azdatalake/internal/path/constants.go +++ b/sdk/storage/azdatalake/internal/path/constants.go @@ -73,3 +73,12 @@ const ( StateTypeBreaking StateType = azdatalake.StateTypeBreaking StateTypeBroken StateType = azdatalake.StateTypeBroken ) + +type LeaseAction = generated.LeaseAction + +const ( + LeaseActionAcquire = generated.LeaseActionAcquire + LeaseActionRelease = generated.LeaseActionRelease + LeaseActionAcquireRelease = generated.LeaseActionAcquireRelease + LeaseActionRenew = generated.LeaseActionAutoRenew +) diff --git a/sdk/storage/azdatalake/internal/path/models.go b/sdk/storage/azdatalake/internal/path/models.go index 9641519ab090..64cb04cc1aa5 100644 --- a/sdk/storage/azdatalake/internal/path/models.go +++ b/sdk/storage/azdatalake/internal/path/models.go @@ -315,5 +315,3 @@ type CPKScopeInfo = blob.CPKScopeInfo // ACLFailedEntry contains the failed ACL entry (response model). type ACLFailedEntry = generated.ACLFailedEntry - -type LeaseAction = generated.LeaseAction diff --git a/sdk/storage/azdatalake/internal/testcommon/clients_auth.go b/sdk/storage/azdatalake/internal/testcommon/clients_auth.go index 4463120cbe5a..db202ebf0909 100644 --- a/sdk/storage/azdatalake/internal/testcommon/clients_auth.go +++ b/sdk/storage/azdatalake/internal/testcommon/clients_auth.go @@ -80,13 +80,6 @@ var BasicHeaders = file.HTTPHeaders{ ContentEncoding: &DatalakeContentEncoding, } -var ( - TestLeaseActionAcquire = file.LeaseAction("acquire") - TestLeaseActionRelease = file.LeaseAction("release") - TestLeaseActionRenew = file.LeaseAction("auto-renew") - TestLeaseActionAcquireRelease = file.LeaseAction("acquire-release") -) - type TestAccountType string const ( From 95f964ad2369b8df01baf52c52848cdeb8539f07 Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Mon, 4 Mar 2024 15:03:49 +0530 Subject: [PATCH 7/9] add documentation --- sdk/storage/azdatalake/file/constants.go | 1 + sdk/storage/azdatalake/file/examples_test.go | 41 ++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/sdk/storage/azdatalake/file/constants.go b/sdk/storage/azdatalake/file/constants.go index 7e2952f56aad..357472077845 100644 --- a/sdk/storage/azdatalake/file/constants.go +++ b/sdk/storage/azdatalake/file/constants.go @@ -125,6 +125,7 @@ const ( StateTypeBroken StateType = azdatalake.StateTypeBroken ) +// LeaseAction Describes actions that can be performed on a lease. type LeaseAction = path.LeaseAction var ( diff --git a/sdk/storage/azdatalake/file/examples_test.go b/sdk/storage/azdatalake/file/examples_test.go index 213adfb3f173..0bb43c557216 100644 --- a/sdk/storage/azdatalake/file/examples_test.go +++ b/sdk/storage/azdatalake/file/examples_test.go @@ -14,9 +14,11 @@ import ( "fmt" "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease" "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake" "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/directory" "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/file" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/path" "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/shared" "hash/crc64" "io" @@ -387,3 +389,42 @@ func Example_file_AppendAndFlushDataWithValidation() { handleError(err) fmt.Println(*gResp2.ContentLength, int64(contentSize)) } + +func Example_file_AppendAndFlushDataWithAcquireAndReleaseLease() { + accountName, accountKey := os.Getenv("AZURE_STORAGE_ACCOUNT_NAME"), os.Getenv("AZURE_STORAGE_ACCOUNT_KEY") + + // Create a file client + u := fmt.Sprintf("https://%s.dfs.core.windows.net/fs/file.txt", accountName) + credential, err := azdatalake.NewSharedKeyCredential(accountName, accountKey) + handleError(err) + fClient, err := file.NewClientWithSharedKeyCredential(u, credential, nil) + handleError(err) + + contentSize := 1024 * 8 // 8KB + content := make([]byte, contentSize) + body := bytes.NewReader(content) + rsc := streaming.NopCloser(body) + + // Acquire lease during append data + opts := &file.AppendDataOptions{ + LeaseAction: &file.LeaseActionAcquire, + LeaseDuration: to.Ptr(int64(15)), + ProposedLeaseID: proposedLeaseIDs[1], + } + _, err = fClient.AppendData(context.Background(), 0, rsc, opts) + handleError(err) + + _, err = fClient.FlushData(context.Background(), int64(contentSize), &file.FlushDataOptions{ + LeaseAction: &file.LeaseActionRelease, + AccessConditions: &path.AccessConditions{ + LeaseAccessConditions: &path.LeaseAccessConditions{LeaseID: proposedLeaseIDs[0]}, + }, + }) + handleError(err) + + gResp2, err := fClient.GetProperties(context.Background(), nil) + handleError(err) + // Check if the lease is released + fmt.Println(lease.StateTypeAvailable, *gResp2.LeaseState) + +} From bd1f0b7cd01aaf3893351e95d13a10d25525add8 Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Wed, 6 Mar 2024 13:53:10 +0530 Subject: [PATCH 8/9] comments address --- sdk/storage/azdatalake/CHANGELOG.md | 4 ++-- sdk/storage/azdatalake/file/models.go | 29 ++++++++------------------- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/sdk/storage/azdatalake/CHANGELOG.md b/sdk/storage/azdatalake/CHANGELOG.md index 75a0ccccec7e..484b017092eb 100644 --- a/sdk/storage/azdatalake/CHANGELOG.md +++ b/sdk/storage/azdatalake/CHANGELOG.md @@ -3,8 +3,8 @@ ## 1.1.2 (Unreleased) ### Features Added -* Append with acquire, release lease and renewal of lease. -* Flush with release lease +* Append API with acquire, release lease and renewal of lease support. +* Flush API bundles with release lease option ### Breaking Changes diff --git a/sdk/storage/azdatalake/file/models.go b/sdk/storage/azdatalake/file/models.go index dd583804e0eb..1f9eac9d12d9 100644 --- a/sdk/storage/azdatalake/file/models.go +++ b/sdk/storage/azdatalake/file/models.go @@ -237,15 +237,9 @@ func (o *FlushDataOptions) format(offset int64) (*generated.PathClientFlushDataO cpkInfoOpts.EncryptionKeySHA256 = o.CPKInfo.EncryptionKeySHA256 cpkInfoOpts.EncryptionAlgorithm = o.CPKInfo.EncryptionAlgorithm } - if o.LeaseAction != nil { - flushDataOpts.LeaseAction = o.LeaseAction - } - if o.LeaseDuration != nil { - flushDataOpts.LeaseDuration = o.LeaseDuration - } - if o.ProposedLeaseID != nil { - flushDataOpts.ProposedLeaseID = o.ProposedLeaseID - } + flushDataOpts.LeaseAction = o.LeaseAction + flushDataOpts.LeaseDuration = o.LeaseDuration + flushDataOpts.ProposedLeaseID = o.ProposedLeaseID } return flushDataOpts, modifiedAccessConditions, leaseAccessConditions, httpHeaderOpts, cpkInfoOpts, nil } @@ -308,18 +302,11 @@ func (o *AppendDataOptions) format(offset int64, body io.ReadSeekCloser) (*gener cpkInfoOpts.EncryptionAlgorithm = o.CPKInfo.EncryptionAlgorithm } - if o.LeaseAction != nil { - appendDataOptions.LeaseAction = o.LeaseAction - } - if o.LeaseDuration != nil { - appendDataOptions.LeaseDuration = o.LeaseDuration - } - if o.ProposedLeaseID != nil { - appendDataOptions.ProposedLeaseID = o.ProposedLeaseID - } - if o.Flush != nil { - appendDataOptions.Flush = o.Flush - } + appendDataOptions.LeaseAction = o.LeaseAction + appendDataOptions.LeaseDuration = o.LeaseDuration + appendDataOptions.ProposedLeaseID = o.ProposedLeaseID + appendDataOptions.Flush = o.Flush + if o.TransactionalValidation != nil { _, err = o.TransactionalValidation.Apply(body, appendDataOptions) if err != nil { From cd661331a16842d44d29345053d4da802227efb2 Mon Sep 17 00:00:00 2001 From: tanyasethi-msft <124860586+tanyasethi-msft@users.noreply.github.com> Date: Wed, 6 Mar 2024 14:04:28 +0530 Subject: [PATCH 9/9] Update CHANGELOG.md --- sdk/storage/azdatalake/CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/storage/azdatalake/CHANGELOG.md b/sdk/storage/azdatalake/CHANGELOG.md index 484b017092eb..81ea8529ef3a 100644 --- a/sdk/storage/azdatalake/CHANGELOG.md +++ b/sdk/storage/azdatalake/CHANGELOG.md @@ -3,8 +3,8 @@ ## 1.1.2 (Unreleased) ### Features Added -* Append API with acquire, release lease and renewal of lease support. -* Flush API bundles with release lease option +* Append API with acquire lease, release lease and renewal of lease support. +* Flush API bundled with release lease option. ### Breaking Changes