diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6ba53ab3f..e2ee89e01 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -134,6 +134,7 @@ jobs: imageName: 'windows-2019' build_name: 'azcopy_windows_amd64.exe' display_name: "Windows" + type: 'windows' go_path: 'C:\Users\VssAdministrator\go\bin\' suffix: '.exe' run_e2e: 'go test -timeout=1h -v ./e2etest > test.txt' @@ -152,6 +153,12 @@ jobs: vmImage: $(imageName) steps: + - task: PowerShell@2 + condition: eq(variables.type, 'windows' ) + inputs: + targetType: 'inline' + script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force' + displayName: 'Install Powershell Az Module' - task: GoTool@0 inputs: version: $(AZCOPY_GOLANG_VERSION_COVERAGE) diff --git a/cmd/credentialUtil.go b/cmd/credentialUtil.go index b8397b2c6..3a28f1692 100644 --- a/cmd/credentialUtil.go +++ b/cmd/credentialUtil.go @@ -88,7 +88,7 @@ func GetOAuthTokenManagerInstance() (*common.UserOAuthTokenManager, error) { return } - if autoLoginType != "SPN" && autoLoginType != "MSI" && autoLoginType != "DEVICE" { + if autoLoginType != "SPN" && autoLoginType != "MSI" && autoLoginType != "DEVICE" && autoLoginType != "AZCLI" && autoLoginType != "PSCRED" { glcm.Error("Invalid Auto-login type specified.") return } @@ -118,6 +118,18 @@ func GetOAuthTokenManagerInstance() (*common.UserOAuthTokenManager, error) { case "DEVICE": lca.identity = false + + case "AZCLI": + lca.identity = false + lca.servicePrincipal = false + lca.psCred = false + lca.azCliCred = true + + case "PSCRED": + lca.identity = false + lca.servicePrincipal = false + lca.azCliCred = false + lca.psCred = true } lca.persistToken = false diff --git a/cmd/login.go b/cmd/login.go index 7cfe03501..8472cca29 100644 --- a/cmd/login.go +++ b/cmd/login.go @@ -32,7 +32,7 @@ import ( var loginCmdArg = loginCmdArgs{tenantID: common.DefaultTenantID} var loginNotice = "'azcopy %s' command will be deprecated starting release 10.22. " + - "Use auto-login instead. Visit %s to know more." + "Use auto-login instead. Visit %s to know more." var autoLoginURL = "https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-authorize-azure-active-directory#authorize-without-a-secret-store " var lgCmd = &cobra.Command{ @@ -85,7 +85,7 @@ func init() { lgCmd.PersistentFlags().StringVar(&loginCmdArg.certPath, "certificate-path", "", "Path to certificate for SPN authentication. Required for certificate-based service principal auth.") // Deprecate the identity-object-id flag - _ = lgCmd.PersistentFlags().MarkHidden("identity-object-id") // Object ID of user-assigned identity. + _ = lgCmd.PersistentFlags().MarkHidden("identity-object-id") // Object ID of user-assigned identity. lgCmd.PersistentFlags().StringVar(&loginCmdArg.identityObjectID, "identity-object-id", "", "Object ID of user-assigned identity. This parameter is deprecated. Please use client id or resource id") } @@ -97,6 +97,8 @@ type loginCmdArgs struct { identity bool // Whether to use MSI. servicePrincipal bool + azCliCred bool + psCred bool // Info of VM's user assigned identity, client or object ids of the service identity are required if // your VM has multiple user-assigned managed identities. @@ -191,6 +193,16 @@ func (lca loginCmdArgs) process() error { } // For MSI login, info success message to user. glcm.Info("Login with identity succeeded.") + case lca.azCliCred: + if err := uotm.AzCliLogin(lca.tenantID); err != nil { + return err + } + glcm.Info("Login with AzCliCreds succeeded") + case lca.psCred: + if err := uotm.PSContextToken(lca.tenantID); err != nil { + return err + } + glcm.Info("Login with Powershell context succeeded") default: if err := uotm.UserLogin(lca.tenantID, lca.aadEndpoint, lca.persistToken); err != nil { return err diff --git a/common/azure_ps_context_credential.go b/common/azure_ps_context_credential.go new file mode 100644 index 000000000..81fa82d4a --- /dev/null +++ b/common/azure_ps_context_credential.go @@ -0,0 +1,172 @@ +//go:build go1.18 +// +build go1.18 + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package common + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "regexp" + "sync" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" +) + +const credNamePSContext = "PSContextCredential" + +type PSTokenProvider func(ctx context.Context, resource string, tenant string) ([]byte, error) +func validTenantID(tenantID string) bool { + match, err := regexp.MatchString("^[0-9a-zA-Z-.]+$", tenantID) + if err != nil { + return false + } + return match +} + +func resolveTenant(defaultTenant, specified, credName string, additionalTenants []string) (string, error) { + if specified == "" || specified == defaultTenant { + return defaultTenant, nil + } + if defaultTenant == "adfs" { + return "", errors.New("ADFS doesn't support tenants") + } + if !validTenantID(specified) { + return "", errors.New("Invalid tenant") + } + for _, t := range additionalTenants { + if t == "*" || t == specified { + return specified, nil + } + } + return "", fmt.Errorf(`%s isn't configured to acquire tokens for tenant %q. To enable acquiring tokens for this tenant add it to the AdditionallyAllowedTenants on the credential options, or add "*" to allow acquiring tokens for any tenant`, credName, specified) +} +// PowershellContextCredentialOptions contains optional parameters for AzureDeveloperCLICredential. +type PowershellContextCredentialOptions struct { + // TenantID identifies the tenant the credential should authenticate in. Defaults to the azd environment, + // which is the tenant of the selected Azure subscription. + TenantID string + + tokenProvider PSTokenProvider +} + +// PowershellContextCredential authenticates as the identity logged in to the [Azure Developer CLI]. +// +// [Azure Developer CLI]: https://learn.microsoft.com/azure/developer/azure-developer-cli/overview +type PowershellContextCredential struct { + mu *sync.Mutex + opts PowershellContextCredentialOptions +} + +// NewPowershellContextCredential constructs an AzureDeveloperCLICredential. Pass nil to accept default options. +func NewPowershellContextCredential(options *PowershellContextCredentialOptions) (*PowershellContextCredential, error) { + cp := PowershellContextCredentialOptions{} + if options != nil { + cp = *options + } + if cp.TenantID != "" && !validTenantID(cp.TenantID) { + return nil, errors.New("invalid tenant id") + } + if cp.tokenProvider == nil { + cp.tokenProvider = defaultAzdTokenProvider + } + return &PowershellContextCredential{mu: &sync.Mutex{}, opts: cp}, nil +} + +// GetToken requests a token from the Azure Developer CLI. This credential doesn't cache tokens, so every call invokes azd. +// This method is called automatically by Azure SDK clients. +func (c *PowershellContextCredential) GetToken(ctx context.Context, opts policy.TokenRequestOptions) (azcore.AccessToken, error) { + at := azcore.AccessToken{} + if len(opts.Scopes) != 1 { + return at, errors.New(credNamePSContext + ": GetToken() exactly one scope") + } + + tenant, err := resolveTenant(c.opts.TenantID, opts.TenantID, credNamePSContext, nil) + if err != nil { + return at, err + } + c.mu.Lock() + defer c.mu.Unlock() + b, err := c.opts.tokenProvider(ctx, opts.Scopes[0], tenant) + if err == nil { + at, err = c.createAccessToken(b) + } + if err != nil { + return at, err + } + //msg := fmt.Sprintf("%s.GetToken() acquired a token for scope %q", credNamePSContext, strings.Join(opts.Scopes, ", ")) + return at, nil +} + +// We ignore resource because PS does not support all Resources. Disk scope is not supported +// and we are here only with Storage scope +var defaultAzdTokenProvider PSTokenProvider = func(ctx context.Context, _ string, tenantID string) ([]byte, error) { + // set a default timeout for this authentication iff the application hasn't done so already + var cancel context.CancelFunc + if _, hasDeadline := ctx.Deadline(); !hasDeadline { + ctx, cancel = context.WithTimeout(ctx, 10 * time.Minute) + defer cancel() + } + + r := regexp.MustCompile(`\{"token".*"expiresOn".*\}`) + + if tenantID != "" { + tenantID += " -TenantId " + tenantID + } + cmd := `$token = Get-AzAccessToken -ResourceUrl https://storage.azure.com` + tenantID + ";" + cmd += `$output = -join('{"token":','"',$token.Token,'"', ',"expiresOn":', '"',$token.ExpiresOn.ToString("yyyy-MM-ddTHH:mm:ss.fffK"),'"',"}");` + cmd += "echo $output" + + cliCmd := exec.CommandContext(ctx, "powershell", cmd) + cliCmd.Env = os.Environ() + var stderr bytes.Buffer + cliCmd.Stderr = &stderr + + output, err := cliCmd.Output() + if err != nil { + msg := stderr.String() + if msg == "" { + msg = err.Error() + } + return nil, errors.New(credNamePSContext + msg) + } + + output = []byte(r.FindString(string(output))) + if string(output) == "" { + return nil, errors.New(credNamePSContext + "Invalid output while retrving token") + } + return output, nil +} + +func (c *PowershellContextCredential) createAccessToken(tk []byte) (azcore.AccessToken, error) { + t := struct { + AccessToken string `json:"token"` + ExpiresOn string `json:"expiresOn"` + }{} + + err := json.Unmarshal(tk, &t) + if err != nil { + return azcore.AccessToken{}, errors.New(err.Error()) + } + + parseErr := "error parsing token expiration time %q: %v" + exp, err := time.Parse(time.RFC3339, t.ExpiresOn) + if err != nil { + return azcore.AccessToken{}, fmt.Errorf(parseErr, t.ExpiresOn, err) + } + return azcore.AccessToken{ + ExpiresOn: exp.UTC(), + Token: t.AccessToken, + }, nil +} + +var _ azcore.TokenCredential = (*PowershellContextCredential)(nil) \ No newline at end of file diff --git a/common/oauthTokenManager.go b/common/oauthTokenManager.go index 60a05d193..15cb728a0 100644 --- a/common/oauthTokenManager.go +++ b/common/oauthTokenManager.go @@ -25,11 +25,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" - "github.com/Azure/azure-sdk-for-go/sdk/azidentity" - "github.com/Azure/go-autorest/autorest/date" "net" "net/http" "net/url" @@ -39,6 +34,12 @@ import ( "strings" "time" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/go-autorest/autorest/date" + "github.com/Azure/go-autorest/autorest/adal" ) @@ -82,7 +83,7 @@ func newAzcopyHTTPClient() *http.Client { Timeout: 10 * time.Second, KeepAlive: 10 * time.Second, DualStack: true, - }).Dial, /*Context*/ + }).Dial, /*Context*/ MaxIdleConns: 0, // No limit MaxIdleConnsPerHost: 1000, IdleConnTimeout: 180 * time.Second, @@ -160,6 +161,24 @@ func (uotm *UserOAuthTokenManager) validateAndPersistLogin(oAuthTokenInfo *OAuth return nil } +func (uotm *UserOAuthTokenManager) AzCliLogin(tenantID string) error { + oAuthTokenInfo := &OAuthTokenInfo{ + AzCLICred: true, + Tenant: tenantID, + } + + // CLI creds will not be persisted. AzCLI would have already persistd that + return uotm.validateAndPersistLogin(oAuthTokenInfo, false) +} + +func (uotm *UserOAuthTokenManager) PSContextToken(tenantID string) error { + oAuthTokenInfo := &OAuthTokenInfo { + PSCred: true, + Tenant: tenantID, + } + + return uotm.validateAndPersistLogin(oAuthTokenInfo, false) +} // MSILogin tries to get token from MSI, persist indicates whether to cache the token on local disk. func (uotm *UserOAuthTokenManager) MSILogin(identityInfo IdentityInfo, persist bool) error { if err := identityInfo.Validate(); err != nil { @@ -175,12 +194,12 @@ func (uotm *UserOAuthTokenManager) MSILogin(identityInfo IdentityInfo, persist b } // SecretLogin is a UOTM shell for secretLoginNoUOTM. -func (uotm *UserOAuthTokenManager) SecretLogin(tenantID, activeDirectoryEndpoint, secret, applicationID string, persist bool) (error) { +func (uotm *UserOAuthTokenManager) SecretLogin(tenantID, activeDirectoryEndpoint, secret, applicationID string, persist bool) error { oAuthTokenInfo := &OAuthTokenInfo{ - ServicePrincipalName: true, + ServicePrincipalName: true, Tenant: tenantID, ActiveDirectoryEndpoint: activeDirectoryEndpoint, - ApplicationID: applicationID, + ApplicationID: applicationID, SPNInfo: SPNInfo{ Secret: secret, CertPath: "", @@ -201,10 +220,10 @@ func (uotm *UserOAuthTokenManager) CertLogin(tenantID, activeDirectoryEndpoint, } absCertPath, _ := filepath.Abs(certPath) oAuthTokenInfo := &OAuthTokenInfo{ - ServicePrincipalName: true, + ServicePrincipalName: true, Tenant: tenantID, ActiveDirectoryEndpoint: activeDirectoryEndpoint, - ApplicationID: applicationID, + ApplicationID: applicationID, SPNInfo: SPNInfo{ Secret: certPass, CertPath: absCertPath, @@ -262,7 +281,7 @@ func (uotm *UserOAuthTokenManager) UserLogin(tenantID, activeDirectoryEndpoint s Token: *token, Tenant: tenantID, ActiveDirectoryEndpoint: activeDirectoryEndpoint, - ApplicationID: ApplicationID, + ApplicationID: ApplicationID, } uotm.stashedInfo = &oAuthTokenInfo @@ -404,6 +423,8 @@ type OAuthTokenInfo struct { IdentityInfo IdentityInfo ServicePrincipalName bool `json:"_spn"` SPNInfo SPNInfo + AzCLICred bool + PSCred bool // Note: ClientID should be only used for internal integrations through env var with refresh token. // It indicates the Application ID assigned to your app when you registered it with Azure AD. // In this case AzCopy refresh token on behalf of caller. @@ -461,8 +482,8 @@ func (credInfo *OAuthTokenInfo) Refresh(ctx context.Context) (*adal.Token, error return nil, err } return &adal.Token{ - AccessToken: t.Token, - ExpiresOn: json.Number(strconv.FormatInt(int64(t.ExpiresOn.Sub(date.UnixEpoch())/time.Second), 10)), + AccessToken: t.Token, + ExpiresOn: json.Number(strconv.FormatInt(int64(t.ExpiresOn.Sub(date.UnixEpoch())/time.Second), 10)), }, nil } else { if dcc, ok := tc.(*DeviceCodeCredential); ok { @@ -516,11 +537,10 @@ func (tsc *TokenStoreCredential) GetToken(_ context.Context, _ policy.TokenReque } return azcore.AccessToken{ - Token: tokenInfo.AccessToken, + Token: tokenInfo.AccessToken, ExpiresOn: tokenInfo.Expires(), }, nil - } // GetNewTokenFromTokenStore gets token from token store. (Credential Manager in Windows, keyring in Linux and keychain in MacOS.) @@ -598,11 +618,29 @@ func (credInfo *OAuthTokenInfo) GetClientSecretCredential() (azcore.TokenCredent return tc, nil } +func (credInfo *OAuthTokenInfo) GetAzCliCredential() (azcore.TokenCredential, error) { + tc, err := azidentity.NewAzureCLICredential(&azidentity.AzureCLICredentialOptions{TenantID: credInfo.Tenant}) + if err != nil { + return nil, err + } + credInfo.TokenCredential = tc + return tc, nil +} + +func (credInfo *OAuthTokenInfo) GetPSContextCredential() (azcore.TokenCredential, error) { + tc, err := NewPowershellContextCredential(nil) + if err != nil { + return nil, err + } + credInfo.TokenCredential = tc + return tc, nil +} + type DeviceCodeCredential struct { - token adal.Token + token adal.Token aadEndpoint string - tenantID string - clientID string + tenantID string + clientID string } func (dcc *DeviceCodeCredential) GetToken(ctx context.Context, options policy.TokenRequestOptions) (azcore.AccessToken, error) { @@ -677,6 +715,13 @@ func (credInfo *OAuthTokenInfo) GetTokenCredential() (azcore.TokenCredential, er } } + if credInfo.AzCLICred { + return credInfo.GetAzCliCredential() + } + + if credInfo.PSCred { + return credInfo.GetPSContextCredential() + } return credInfo.GetDeviceCodeCredential() } diff --git a/e2etest/runner.go b/e2etest/runner.go index 5948ef5ac..9c830a499 100644 --- a/e2etest/runner.go +++ b/e2etest/runner.go @@ -247,8 +247,8 @@ func (t *TestRunner) ExecuteAzCopyCommand(operation Operation, src, dst string, env := make([]string, len(os.Environ())) copy(env, os.Environ()) - // paste in OAuth environment variables - if needsOAuth { + // paste in OAuth environment variables if not specified + if needsOAuth && os.Getenv("AZCOPY_AUTO_LOGIN_TYPE") == "" { tenId, appId, clientSecret := GlobalInputManager{}.GetServicePrincipalAuth() env = append(env, diff --git a/e2etest/zt_basic_cli_ps_auth_test.go b/e2etest/zt_basic_cli_ps_auth_test.go new file mode 100644 index 000000000..bd3dda9a6 --- /dev/null +++ b/e2etest/zt_basic_cli_ps_auth_test.go @@ -0,0 +1,109 @@ +// Copyright © Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package e2etest + +import ( + "fmt" + "os" + "os/exec" + "runtime" + "testing" + + "github.com/Azure/azure-storage-azcopy/v10/common" +) + +// Purpose: Tests AZCLI and powershell auth tests + +func TestBasic_AzCLIAuth(t *testing.T) { + RunScenarios(t, eOperation.Copy(), eTestFromTo.Other(common.EFromTo.BlobBlob()), eValidate.Auto(), oAuthOnly, oAuthOnly, params{ // Pass flag values that the test requires. The params struct is a superset of Copy and Sync params + recursive: true, + }, &hooks{ + beforeTestRun: func(h hookHelper) { + tenId, appId, clientSecret := GlobalInputManager{}.GetServicePrincipalAuth() + args := []string{ + "login", + "--service-principal", + "-u=" + appId, + "-p=" + clientSecret, + } + if tenId != "" { + args = append(args, "--tenant=" + tenId) + } + + out, err := exec.Command("az", args...).Output() + if err != nil { + e := err.(*exec.ExitError) + t.Logf(string(e.Stderr)) + t.Logf(string(out)) + t.Logf("Failed to login with AzCLI " + err.Error()) + t.FailNow() + } + os.Setenv("AZCOPY_AUTO_LOGIN_TYPE", "AZCLI") + }, + }, testFiles{ + defaultSize: "1K", + shouldTransfer: []interface{}{ + "wantedfile", + folder("sub/subsub"), + "sub/subsub/filea", + "sub/subsub/filec", + }, + }, EAccountType.Standard(), EAccountType.Standard(), "") +} + + +func TestBasic_PSAuth(t *testing.T) { + RunScenarios(t, eOperation.Copy(), eTestFromTo.Other(common.EFromTo.BlobBlob()), eValidate.Auto(), oAuthOnly, oAuthOnly, params{ // Pass flag values that the test requires. The params struct is a superset of Copy and Sync params + recursive: true, + }, &hooks{ + beforeTestRun: func(h hookHelper) { + if runtime.GOOS != "windows" { + h.SkipTest() + } + tenId, appId, clientSecret := GlobalInputManager{}.GetServicePrincipalAuth() + cmd := `$secret = ConvertTo-SecureString -String %s -AsPlainText -Force; + $cred = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList %s, $secret; + Connect-AzAccount -ServicePrincipal -Credential $cred` + if tenId != "" { + cmd += " -Tenant " + tenId + } + + script := fmt.Sprintf(cmd, clientSecret, appId) + out, err := exec.Command("powershell", script).Output() + if err != nil { + e := err.(*exec.ExitError) + t.Logf(string(e.Stderr)) + t.Logf(string(out)) + t.Logf("Failed to login with Powershell " + err.Error()) + t.FailNow() + } + os.Setenv("AZCOPY_AUTO_LOGIN_TYPE", "PSCRED") + }, + }, testFiles{ + defaultSize: "1K", + shouldTransfer: []interface{}{ + "wantedfile", + folder("sub/subsub"), + "sub/subsub/filea", + "sub/subsub/filec", + }, + }, EAccountType.Standard(), EAccountType.Standard(), "") +} diff --git a/go.sum b/go.sum index 4b198c7c7..f59d13db2 100644 --- a/go.sum +++ b/go.sum @@ -92,7 +92,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=