Replace go-azure-helpers with azidentity

.github/workflows/azcore-scheduled.yml

@@ -19,4 +19,4 @@ jobs:
       version: ${{ needs.version.outputs.version }}
       short_test: false
       retention_days: 7
-      use_autorest: false
+      use_azcore: true

.github/workflows/build-test.yml

@@ -22,10 +22,10 @@ on:
         type: number
         description: The number of days for which we retain assets
         default: 90
-      use_autorest:
+      use_azcore:
         type: boolean
-        description: Whether to use autorest, not azcore
-        default: true
+        description: Whether to use the newer azcore+azidentity backend for REST and auth, or the older autorest
+        default: false
 
 env:
   GITHUB_TOKEN: ${{ secrets.PULUMI_BOT_TOKEN }}
@@ -45,7 +45,19 @@ env:
   ARM_SUBSCRIPTION_ID: 0282681f-7a9e-424b-80b2-96babd57a8a1
   ARM_TENANT_ID: 706143bc-e1d4-4593-aee2-c9dc60ab9be7
   PULUMI_API: https://api.pulumi-staging.io
-  PULUMI_USE_AUTOREST: ${{ inputs.use_autorest }}
+  # Feature toggle that's read in provider.go enableAzcoreBackend()
+  PULUMI_ENABLE_AZCORE_BACKEND: ${{ inputs.use_azcore }}
+  # This is the content of a ~/.azure/ folder, zipped and base64-encoded, for CLI auth.
+  # If/when the contained refresh token expires, someone with access to our subscription needs to
+  # `az login` on their own computer and repeat the steps below.
+  # Generated by using @mikhail's .azure folder and running:
+  #     cp -R ~/.azure ~/azure
+  #     cd ~/azure
+  #     rm -rf .DS_Store logs/ commands/* cliextensions/ extensionCommandTree.json
+  #     zip -v azure.zip *
+  #     base64 --input azure.zip | clipcopy
+  #     Paste into repo secret
+  AZURE_CLI_FOLDER: ${{ secrets.AZURE_CLI_FOLDER }}
 
 jobs:
   prerequisites:
@@ -172,6 +184,7 @@ jobs:
     needs: build_sdks
     # Use big runner for dotnet and nodejs because we need more memory and more compute, respectively
     runs-on: ${{ (matrix.language == 'dotnet' || matrix.language == 'nodejs' || matrix.language == 'go') && 'pulumi-ubuntu-8core' || 'ubuntu-latest' }}
+    environment: env-ci
     strategy:
       fail-fast: false
       matrix:
@@ -202,6 +215,18 @@ jobs:
         run: |
           echo "${{ secrets.ARM_CLIENT_CERTIFICATE }}" | base64 -d > "${{ runner.temp }}/azure-client-certificate.pfx"
 
+      - name: Write .azure.tmp folder
+        # We write to .azure.tmp, not directly to .azure, because we want only one test to use
+        # this folder. The test needs to rename it to .azure and then back. This is to avoid other
+        # tests using it unintentionally since CLI is the fallback auth method.
+        run: |
+          set -euxo pipefail
+          echo "${{ secrets.AZURE_CLI_FOLDER }}" | base64 -d > "${{ runner.temp }}/azure-cli-folder.zip"
+          # Unzip it to a temp folder to avoid other tests using it unintentionally (since CLI auth is the fallback method).
+          # We only want one specific test to use it.
+          unzip -d "$HOME/.azure.tmp" "${{ runner.temp }}/azure-cli-folder.zip"
+          rm "${{ runner.temp }}/azure-cli-folder.zip"
+
       - name: Run tests
         if: ${{ ! inputs.short_test }}
         env:

examples/azure-native-sdk-v2/go-azure-in-azure/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"os"
+	"os/exec"
 
 	"github.com/pulumi/pulumi-azure-native-sdk/authorization/v2"
 	"github.com/pulumi/pulumi-azure-native-sdk/compute/v2"
@@ -115,6 +116,7 @@ func main() {
 		vmIdentity := &compute.VirtualMachineIdentityArgs{Type: compute.ResourceIdentityTypeSystemAssigned}
 
 		var umi *managedidentity.UserAssignedIdentity
+		var umiClientId pulumi.StringOutput = pulumi.String("").ToStringOutput()
 		if os.Getenv("PULUMI_TEST_USER_IDENTITY") == "true" {
 			fmt.Printf("go-azure-in-azure: using user-assigned identity\n")
 
@@ -124,10 +126,20 @@ func main() {
 			if err != nil {
 				return err
 			}
+			umiClientId = umi.ClientId
+
+			// Create a second user-assigned identity to test multiple identities. With multiple identities, the one to
+			// use needs to be specified via clientId.
+			umi2, err := managedidentity.NewUserAssignedIdentity(ctx, "umi2", &managedidentity.UserAssignedIdentityArgs{
+				ResourceGroupName: rg.Name,
+			})
+			if err != nil {
+				return err
+			}
 
 			vmIdentity = &compute.VirtualMachineIdentityArgs{
 				Type:                   compute.ResourceIdentityTypeUserAssigned,
-				UserAssignedIdentities: pulumi.StringArray{umi.ID()},
+				UserAssignedIdentities: pulumi.StringArray{umi.ID(), umi2.ID()},
 			}
 		}
 
@@ -228,6 +240,7 @@ func main() {
 			User:       pulumi.String("pulumi"),
 			PrivateKey: privateKey.PrivateKeyOpenssh,
 		}
+
 		copy, err := remote.NewCopyToRemote(ctx, "copy", &remote.CopyToRemoteArgs{
 			Connection: sshConn,
 			Source:     pulumi.NewFileArchive(innerProgram + "/"),
@@ -247,30 +260,62 @@ func main() {
 			return err
 		}
 
+		// Copy the provider binary under test (the one on PATH) to the VM.
+		// We put it in the same directory as the pulumi binary which is on the PATH.
+		// Note that this can only work if the VM has the same architecture as the local machine.
+		providerBinaryPath, err := exec.LookPath("pulumi-resource-azure-native")
+		if err != nil {
+			return err
+		}
+		copyProvider, err := remote.NewCopyToRemote(ctx, "copyProvider", &remote.CopyToRemoteArgs{
+			Connection: sshConn,
+			Source:     pulumi.NewFileAsset(providerBinaryPath),
+			RemotePath: pulumi.String("/home/pulumi/.pulumi/bin/"),
+			Triggers:   pulumi.ToArray([]any{vm.ID()}),
+		}, pulumi.DependsOn([]pulumi.Resource{poll, installPulumi}))
+		if err != nil {
+			return err
+		}
+		chmodProvider, err := remote.NewCommand(ctx, "chmodProvider", &remote.CommandArgs{
+			Connection: sshConn,
+			Create:     pulumi.String("chmod +x /home/pulumi/.pulumi/bin/pulumi-resource-azure-native"),
+			Triggers:   pulumi.ToArray([]any{vm.ID()}),
+		}, pulumi.DependsOn([]pulumi.Resource{copyProvider}))
+		if err != nil {
+			return err
+		}
+
+		// Pass feature flags into the VM.
+		useAutorest := os.Getenv("PULUMI_USE_AUTOREST")
+		useLegacyAuth := os.Getenv("PULUMI_USE_LEGACY_AUTH")
+
 		// We pass the resource group's ID into the inner program via config so the program can
 		// create a resource in the resource group.
 		create := pulumi.Sprintf(`cd %s && \
 set -euxo pipefail && \
 export ARM_USE_MSI=true && \
 export ARM_SUBSCRIPTION_ID=%s && \
-export PATH=~/.pulumi/bin:$PATH && \
+export PATH="$HOME/.pulumi/bin:$PATH" && \
 export PULUMI_CONFIG_PASSPHRASE=pass && \
+export PULUMI_USE_AUTOREST=%s && \
+export PULUMI_USE_LEGACY_AUTH=%s && \
 rand=$(openssl rand -hex 4) && \
 stackname="%s-$rand" && \
 pulumi login --local && \
 pulumi stack init $stackname && \
+pulumi config set azure-native:clientId "%s" -s $stackname && \
 pulumi config set rgId "%s" -s $stackname && \
 pulumi config -s $stackname && \
 pulumi up -s $stackname --skip-preview --logtostderr --logflow -v=9 && \
 pulumi down -s $stackname --skip-preview --logtostderr --logflow -v=9 && \
 pulumi stack rm --yes $stackname && \
-pulumi logout --local`, innerProgram, clientConf.SubscriptionId, innerProgram, rg.ID())
+pulumi logout --local`, innerProgram, clientConf.SubscriptionId, useAutorest, useLegacyAuth, innerProgram, umiClientId, rg.ID())
 
-		pulumiPreview, err := remote.NewCommand(ctx, "pulumiPreview", &remote.CommandArgs{
+		pulumiPreview, err := remote.NewCommand(ctx, "pulumiUpDown", &remote.CommandArgs{
 			Connection: sshConn,
 			Triggers:   pulumi.ToArray([]any{vm.ID(), principalId, roleAssignment.ID()}),
 			Create:     create,
-		}, pulumi.DependsOn([]pulumi.Resource{roleAssignment, copy, installPulumi}))
+		}, pulumi.DependsOn([]pulumi.Resource{roleAssignment, copy, copyProvider, chmodProvider, installPulumi}))
 		if err != nil {
 			return err
 		}
@@ -281,8 +326,9 @@ pulumi logout --local`, innerProgram, clientConf.SubscriptionId, innerProgram, r
 		ctx.Export("publicIpAddress", ipLookup.IpAddress().Elem())
 		ctx.Export("installPulumi", installPulumi.Stdout)
 		ctx.Export("installPulumiStderr", installPulumi.Stderr)
-		ctx.Export("pulumiPreview", pulumiPreview.Stdout)
-		ctx.Export("pulumiPreviewStderr", pulumiPreview.Stderr)
+		ctx.Export("providerBinary", copyProvider.Source)
+		ctx.Export("pulumiStdout", pulumiPreview.Stdout)
+		ctx.Export("pulumiStderr", pulumiPreview.Stderr)
 
 		return nil
 	})

examples/examples_nodejs_keyvault_test.go

@@ -5,10 +5,12 @@ package examples
 
 import (
 	"os"
+	"os/user"
 	"path/filepath"
 	"testing"
 
 	"github.com/pulumi/pulumi/pkg/v3/testing/integration"
+	"github.com/stretchr/testify/require"
 )
 
 func TestAccKeyVaultTs(t *testing.T) {
@@ -80,9 +82,44 @@ func TestAccKeyVaultTs_ClientCert(t *testing.T) {
 				"ARM_CLIENT_CERTIFICATE_PATH=" + os.Getenv("ARM_CLIENT_CERTIFICATE_PATH_FOR_TEST"),
 				"ARM_CLIENT_CERTIFICATE_PASSWORD=" + os.Getenv("ARM_CLIENT_CERTIFICATE_PASSWORD_FOR_TEST"),
 				// Make sure we test the client cert path
+				"ACTIONS_ID_TOKEN_REQUEST_TOKEN=",
+				"ACTIONS_ID_TOKEN_REQUEST_URL=",
+				"ARM_CLIENT_SECRET=",
+				"ARM_CLIENT_CERTIFICATE_PATH=",
+			},
+		})
+
+	integration.ProgramTest(t, &test)
+}
+
+func TestAccKeyVaultTs_CLI(t *testing.T) {
+	skipIfShort(t)
+
+	usr, err := user.Current()
+	require.NoError(t, err)
+	// .azure.tmp is created by the GH workflow build-test.yml, from the GH secret AZURE_CLI_FOLDER
+	// which is also documented in the workflow. We rename it to .azure so the `az` CLI can find it.
+	err = os.Rename(filepath.Join(usr.HomeDir, ".azure.tmp"), filepath.Join(usr.HomeDir, ".azure"))
+	require.NoError(t, err)
+
+	// Prevent later tests from accidentally picking up the .azure folder because authentication
+	// falls back to CLI when other methods are misconfigured.
+	defer func() {
+		_ = os.Rename(filepath.Join(usr.HomeDir, ".azure"), filepath.Join(usr.HomeDir, ".azure.tmp"))
+	}()
+
+	test := getJSBaseOptions(t).
+		With(integration.ProgramTestOptions{
+			Dir: filepath.Join(getCwd(t), "keyvault"),
+			Env: []string{
+				// Unset auth variables to make sure we're testing the CLI auth path
 				"ARM_CLIENT_SECRET=",
+				"ARM_CLIENT_CERTIFICATE_PATH=",
+				"ARM_USE_MSI=false",
+				"ARM_USE_OIDC=false",
 			},
 		})
 
 	integration.ProgramTest(t, &test)
+
 }

provider/go.mod

@@ -4,7 +4,8 @@ go 1.21.0
 
 require (
 	github.com/Azure/azure-sdk-for-go v68.0.0+incompatible
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.15.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0
 	github.com/Azure/go-autorest/autorest v0.11.29
 	github.com/blang/semver v3.5.1+incompatible
@@ -42,7 +43,6 @@ require (
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/longrunning v0.5.5 // indirect
 	dario.cat/mergo v1.0.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
@@ -217,14 +217,14 @@ require (
 	go.uber.org/atomic v1.9.0 // indirect
 	gocloud.dev v0.37.0 // indirect
 	gocloud.dev/secrets/hashivault v0.37.0 // indirect
-	golang.org/x/crypto v0.26.0 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
 	golang.org/x/mod v0.18.0 // indirect
-	golang.org/x/net v0.28.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
 	golang.org/x/oauth2 v0.18.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/sys v0.23.0 // indirect
-	golang.org/x/term v0.23.0 // indirect
-	golang.org/x/text v0.17.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/term v0.25.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.22.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect