Skip to content

Commit

Permalink
What if I completely redid everything for fun
Browse files Browse the repository at this point in the history
  • Loading branch information
@adreed-msft
adreed-msft committed Oct 3, 2024
1 parent 3efbd5a commit 343d581
Show file tree
Hide file tree
Showing 11 changed files with 210 additions and 140 deletions.
49 changes: 17 additions & 32 deletions cmd/copyEnumeratorInit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -279,39 +279,24 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde

if cca.dryrunMode && shouldSendToSte {
glcm.Dryrun(func(format common.OutputFormat) string {
if format == common.EOutputFormat.Json() {
jsonOutput, err := json.Marshal(transfer)
common.PanicIfErr(err)
return string(jsonOutput)
} else {
if cca.FromTo.From() == common.ELocation.Local() {
// formatting from local source
dryrunValue := fmt.Sprintf("DRYRUN: copy %v", common.ToShortPath(cca.Source.Value))
if runtime.GOOS == "windows" {
dryrunValue += strings.ReplaceAll(srcRelPath, "/", "\\")
} else { // linux and mac
dryrunValue += srcRelPath
}
dryrunValue += fmt.Sprintf(" to %v%v", strings.Trim(cca.Destination.Value, "/"), dstRelPath)
return dryrunValue
} else if cca.FromTo.To() == common.ELocation.Local() {
// formatting to local source
dryrunValue := fmt.Sprintf("DRYRUN: copy %v%v to %v",
strings.Trim(cca.Source.Value, "/"), srcRelPath,
common.ToShortPath(cca.Destination.Value))
if runtime.GOOS == "windows" {
dryrunValue += strings.ReplaceAll(dstRelPath, "/", "\\")
} else { // linux and mac
dryrunValue += dstRelPath
}
return dryrunValue
} else {
return fmt.Sprintf("DRYRUN: copy %v%v to %v%v",
cca.Source.Value,
srcRelPath,
cca.Destination.Value,
dstRelPath)
src := common.GenerateFullPath(cca.Source.Value, srcRelPath)
dst := common.GenerateFullPath(cca.Destination.Value, dstRelPath)

switch format {
case common.EOutputFormat.Json():
tx := DryrunTransfer{
EntityType: transfer.EntityType,
BlobType: common.FromBlobType(transfer.BlobType),
FromTo: cca.FromTo,
Source: src,
Destination: dst,
}

buf, _ := json.Marshal(tx)
return string(buf)
default:
return fmt.Sprintf("DRYRUN: copy %v to %v",
src, dst)
}
})
return nil
Expand Down
57 changes: 51 additions & 6 deletions cmd/removeEnumerator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ package cmd

import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake"
Expand Down Expand Up @@ -229,8 +230,22 @@ func removeBfsResources(cca *CookedCopyCmdArgs) (err error) {
func dryrunRemoveSingleDFSResource(ctx context.Context, dsc *service.Client, datalakeURLParts azdatalake.URLParts, recursive bool) error {
//deleting a filesystem
if datalakeURLParts.PathName == "" {
glcm.Dryrun(func(_ common.OutputFormat) string {
return fmt.Sprintf("DRYRUN: remove filesystem %s", datalakeURLParts.FileSystemName)
glcm.Dryrun(func(of common.OutputFormat) string {
switch of {
case of.Text():
return fmt.Sprintf("DRYRUN: remove %s", dsc.NewFileSystemClient(datalakeURLParts.FileSystemName).DFSURL())
case of.Json():
tx := DryrunTransfer{
EntityType: common.EEntityType.Folder(),
FromTo: common.EFromTo.BlobFSTrash(),
Source: dsc.NewFileSystemClient(datalakeURLParts.FileSystemName).DFSURL(),
}

buf, _ := json.Marshal(tx)
return string(buf)
default:
panic("unsupported output format " + of.String())
}
})
return nil
}
Expand All @@ -246,8 +261,22 @@ func dryrunRemoveSingleDFSResource(ctx context.Context, dsc *service.Client, dat
// then we should short-circuit and simply remove that file
resourceType := common.IffNotNil(props.ResourceType, "")
if strings.EqualFold(resourceType, "file") {
glcm.Dryrun(func(_ common.OutputFormat) string {
return fmt.Sprintf("DRYRUN: remove file %s", datalakeURLParts.PathName)
glcm.Dryrun(func(of common.OutputFormat) string {
switch of {
case of.Text():
return fmt.Sprintf("DRYRUN: remove %s", directoryClient.DFSURL())
case of.Json():
tx := DryrunTransfer{
EntityType: common.EEntityType.File(),
FromTo: common.EFromTo.BlobFSTrash(),
Source: directoryClient.DFSURL(),
}

buf, _ := json.Marshal(tx)
return string(buf)
default:
panic("unsupported output format " + of.String())
}
})
return nil
}
Expand All @@ -267,8 +296,24 @@ func dryrunRemoveSingleDFSResource(ctx context.Context, dsc *service.Client, dat
entityType = "file"
}

glcm.Dryrun(func(_ common.OutputFormat) string {
return fmt.Sprintf("DRYRUN: remove %s %s", entityType, *v.Name)
glcm.Dryrun(func(of common.OutputFormat) string {
uri := dsc.NewFileSystemClient(datalakeURLParts.FileSystemName).NewFileClient(*v.Name).DFSURL()

switch of {
case of.Text():
return fmt.Sprintf("DRYRUN: remove %s", uri)
case of.Json():
tx := DryrunTransfer{
EntityType: common.Iff(entityType == "directory", common.EEntityType.Folder(), common.EEntityType.File()),
FromTo: common.EFromTo.BlobFSTrash(),
Source: uri,
}

buf, _ := json.Marshal(tx)
return string(buf)
default:
panic("unsupported output format " + of.String())
}
})
}
}
Expand Down
12 changes: 11 additions & 1 deletion cmd/sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,17 @@ func (raw *rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) {
jobsAdmin.JobsAdmin.LogToJobLog(LocalToFileShareWarnMsg, common.LogWarning)
}
if raw.dryrun {
glcm.Dryrun(func(_ common.OutputFormat) string {
glcm.Dryrun(func(of common.OutputFormat) string {
if of == common.EOutputFormat.Json() {
var out struct {
Warn string `json:"warn"`
}

out.Warn = LocalToFileShareWarnMsg
buf, _ := json.Marshal(out)
return string(buf)
}

return fmt.Sprintf("DRYRUN: warn %s", LocalToFileShareWarnMsg)
})
}
Expand Down
51 changes: 18 additions & 33 deletions cmd/syncProcessor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (
"net/url"
"os"
"path"
"runtime"
"strings"

"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
Expand Down Expand Up @@ -75,24 +74,6 @@ type interactiveDeleteProcessor struct {
dryrunMode bool
}

func newDeleteTransfer(object StoredObject) (newDeleteTransfer common.CopyTransfer) {
return common.CopyTransfer{
Source: object.relativePath,
EntityType: object.entityType,
LastModifiedTime: object.lastModifiedTime,
SourceSize: object.size,
ContentType: object.contentType,
ContentEncoding: object.contentEncoding,
ContentDisposition: object.contentDisposition,
ContentLanguage: object.contentLanguage,
CacheControl: object.cacheControl,
Metadata: object.Metadata,
BlobType: object.blobType,
BlobVersionID: object.blobVersionID,
BlobTags: object.blobTags,
}
}

func (d *interactiveDeleteProcessor) removeImmediately(object StoredObject) (err error) {
if d.shouldPromptUser {
d.shouldDelete, d.shouldPromptUser = d.promptForConfirmation(object) // note down the user's decision
Expand All @@ -105,22 +86,24 @@ func (d *interactiveDeleteProcessor) removeImmediately(object StoredObject) (err
if d.dryrunMode {
glcm.Dryrun(func(format common.OutputFormat) string {
if format == common.EOutputFormat.Json() {
jsonOutput, err := json.Marshal(newDeleteTransfer(object))
deleteTarget := common.ELocation.Local()
if d.objectTypeToDisplay != LocalFileObjectType {
_ = deleteTarget.Parse(d.objectTypeToDisplay)
}

tx := DryrunTransfer{
Source: common.GenerateFullPath(d.objectLocationToDisplay, object.relativePath),
BlobType: common.FromBlobType(object.blobType),
EntityType: object.entityType,
FromTo: common.FromToValue(deleteTarget, common.ELocation.Unknown()),
}

jsonOutput, err := json.Marshal(tx)
common.PanicIfErr(err)
return string(jsonOutput)
} else { // remove for sync
if d.objectTypeToDisplay == "local file" { // removing from local src
dryrunValue := fmt.Sprintf("DRYRUN: remove %v", common.ToShortPath(d.objectLocationToDisplay))
if runtime.GOOS == "windows" {
dryrunValue += "\\" + strings.ReplaceAll(object.relativePath, "/", "\\")
} else { // linux and mac
dryrunValue += "/" + object.relativePath
}
return dryrunValue
}
return fmt.Sprintf("DRYRUN: remove %v/%v",
d.objectLocationToDisplay,
object.relativePath)
return fmt.Sprintf("DRYRUN: remove %v",
common.GenerateFullPath(d.objectLocationToDisplay, object.relativePath))
}
})
return nil
Expand Down Expand Up @@ -189,9 +172,11 @@ func newInteractiveDeleteProcessor(deleter objectProcessor, deleteDestination co
}
}

const LocalFileObjectType = "local file"

func newSyncLocalDeleteProcessor(cca *cookedSyncCmdArgs, fpo common.FolderPropertyOption) *interactiveDeleteProcessor {
localDeleter := localFileDeleter{rootPath: cca.destination.ValueLocal(), fpo: fpo, folderManager: common.NewFolderDeletionManager(context.Background(), fpo, azcopyScanningLogger)}
return newInteractiveDeleteProcessor(localDeleter.deleteFile, cca.deleteDestination, "local file", cca.destination, cca.incrementDeletionCount, cca.dryrunMode)
return newInteractiveDeleteProcessor(localDeleter.deleteFile, cca.deleteDestination, LocalFileObjectType, cca.destination, cca.incrementDeletionCount, cca.dryrunMode)
}

type localFileDeleter struct {
Expand Down
25 changes: 22 additions & 3 deletions cmd/zc_processor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,17 @@ func newCopyTransferProcessor(copyJobTemplate *common.CopyJobPartOrderRequest, n
}

type DryrunTransfer struct {
EntityType common.EntityType
BlobType common.BlobType
FromTo common.FromTo
Source string
Destination string
}

func (d *DryrunTransfer) UnmarshalJSON(bytes []byte) error {
var surrogate struct {
EntityType string
BlobType string
FromTo string
Source string
Destination string
Expand All @@ -86,6 +90,16 @@ func (d *DryrunTransfer) UnmarshalJSON(bytes []byte) error {
return fmt.Errorf("failed to parse fromto: %w", err)
}

err = d.EntityType.Parse(surrogate.EntityType)
if err != nil {
return fmt.Errorf("failed to parse entity type: %w", err)
}

err = d.BlobType.Parse(surrogate.BlobType)
if err != nil {
return fmt.Errorf("failed to parse entity type: %w", err)
}

d.Source = surrogate.Source
d.Destination = surrogate.Destination

Expand All @@ -94,10 +108,14 @@ func (d *DryrunTransfer) UnmarshalJSON(bytes []byte) error {

func (d DryrunTransfer) MarshalJSON() ([]byte, error) {
surrogate := struct {
EntityType string
BlobType string
FromTo string
Source string
Destination string
}{
d.EntityType.String(),
d.BlobType.String(),
d.FromTo.String(),
d.Source,
d.Destination,
Expand Down Expand Up @@ -161,8 +179,10 @@ func (s *copyTransferProcessor) scheduleCopyTransfer(storedObject StoredObject)

if format == common.EOutputFormat.Json() {
tx := DryrunTransfer{
FromTo: s.copyJobTemplate.FromTo,
Source: common.GenerateFullPath(s.copyJobTemplate.SourceRoot.Value, prettySrcRelativePath),
BlobType: common.FromBlobType(storedObject.blobType),
EntityType: storedObject.entityType,
FromTo: s.copyJobTemplate.FromTo,
Source: common.GenerateFullPath(s.copyJobTemplate.SourceRoot.Value, prettySrcRelativePath),
}

if fromTo.To() != common.ELocation.None() && fromTo.To() != common.ELocation.Unknown() {
Expand All @@ -173,7 +193,6 @@ func (s *copyTransferProcessor) scheduleCopyTransfer(storedObject StoredObject)
common.PanicIfErr(err)
return string(jsonOutput)
} else {

// if remove then To() will equal to common.ELocation.Unknown()
if s.copyJobTemplate.FromTo.To() == common.ELocation.Unknown() { // remove
return fmt.Sprintf("DRYRUN: remove %v",
Expand Down
9 changes: 5 additions & 4 deletions cmd/zt_remove_blob_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -606,14 +606,15 @@ func TestDryrunRemoveBlobsUnderContainerJson(t *testing.T) {
a.Zero(len(mockedRPC.transfers))

msg := <-mockedLcm.dryrunLog
deleteTransfer := common.CopyTransfer{}
deleteTransfer := DryrunTransfer{}
errMarshal := json.Unmarshal([]byte(msg), &deleteTransfer)
a.Nil(errMarshal)
// comparing some values of deleteTransfer
a.Equal(deleteTransfer.Source, "/"+blobName[0])
a.Equal(deleteTransfer.Destination, "/"+blobName[0])
targetUri := cc.NewBlobClient(blobName[0]).URL()
a.Equal(targetUri, deleteTransfer.Source)
a.Equal("", deleteTransfer.Destination)
a.Equal("File", deleteTransfer.EntityType.String())
a.Equal("BlockBlob", string(deleteTransfer.BlobType))
a.Equal("BlockBlob", deleteTransfer.BlobType.String())
})
}

Expand Down
8 changes: 8 additions & 0 deletions common/fe-ste-models.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1575,6 +1575,14 @@ func (e EntityType) String() string {
return enum.StringInt(e, reflect.TypeOf(e))
}

func (e *EntityType) Parse(s string) error {
val, err := enum.ParseInt(reflect.TypeOf(e), s, true, true)
if err == nil {
*e = val.(EntityType)
}
return err
}

////////////////////////////////////////////////////////////////

var EFolderPropertiesOption = FolderPropertyOption(0)
Expand Down
9 changes: 7 additions & 2 deletions e2etest/newe2e_resource_manager_getter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,13 @@ func GetRootResource(a Asserter, location common.Location, varOpts ...GetResourc

return NewLocalContainer(a)
case common.ELocation.Blob(), common.ELocation.BlobFS(), common.ELocation.File():
// acct handles the dryrun case for us
acct := GetAccount(a, DerefOrDefault(opts.PreferredAccount, PrimaryStandardAcct))
// do we have a hns acct attached, if so, and we're requesting blobfs, let's use it
defaultacct := PrimaryStandardAcct
if _, ok := AccountRegistry[PrimaryHNSAcct]; ok {
defaultacct = PrimaryHNSAcct
}

acct := GetAccount(a, DerefOrDefault(opts.PreferredAccount, defaultacct))
return acct.GetService(a, location)
default:
a.Error(fmt.Sprintf("TODO: Location %s is not yet supported", location))
Expand Down
Loading

0 comments on commit 343d581

Please sign in to comment.