Fix spurious resource replacements during API version upgrades with refresh

CLAUDE.md

@@ -208,6 +208,103 @@ This metadata is used by the provider at runtime for constructing API calls.
 - Use `TEST_NAME` to run specific tests
 - Full test suite can take 2+ hours
 
+### Manual Testing for API Version Upgrades
+
+When testing scenarios involving API version changes (e.g., state migration, refresh behavior), it's important to understand how `azureApiVersion` works:
+
+**How `azureApiVersion` is Determined:**
+
+1. **NOT from SDK**: The `azureApiVersion` field is computed by the provider, not supplied as an input from the SDK
+2. **From Provider Metadata**: The API version comes from `metadata-compact.json` baked into the provider binary
+3. **Lookup Process**:
+   - SDK sends URN with type token (e.g., `azure-native:containerservice/v20241002preview:ManagedCluster`)
+   - Provider looks up this token in its embedded metadata
+   - Provider uses `res.APIVersion` from metadata for all operations
+   - Provider sets `azureApiVersion` output after every CRUD operation
+
+**API Version Format Conversion:**
+
+The provider uses two different formats for API versions:
+
+1. **SDK Version Format** (used in type tokens): `v20241002preview`
+   - Example: `azure-native:containerservice/v20241002preview:ManagedCluster`
+   - Format: `v` + `YYYYMMDD` + optional suffix (preview, beta, privatepreview)
+
+2. **API Version Format** (used in Azure ARM API calls and state): `2024-10-02-preview`
+   - Stored in `azureApiVersion` output property
+   - Format: `YYYY-MM-DD` + optional `-suffix`
+
+**Conversion Functions** (in `provider/pkg/openapi/versioner.go`):
+- `openapi.ApiToSdkVersion()` - Converts `2024-10-02-preview` → `v20241002preview`
+- `openapi.SdkToApiVersion()` - Converts `v20241002preview` → `2024-10-02-preview`
+
+When working with API versions in provider code, always use these canonical conversion functions rather than reimplementing the logic.
+
+**Testing Approaches for API Version Changes:**
+
+**Option 1: Modify Version Configuration (Most Realistic)**
+```bash
+# 1. Modify default versions for a resource
+vi versions/v3-config.yaml  # Change StorageAccount default version
+
+# 2. Regenerate schema and metadata
+make schema
+
+# 3. Rebuild provider with new metadata
+make provider
+
+# 4. Deploy resources, then repeat steps 1-3 with different version
+# 5. Run pulumi refresh to test migration behavior
+```
+
+**Option 2: State Manipulation (Quick & Effective)**
+```bash
+# 1. Deploy resources with current provider
+pulumi up
+
+# 2. Export and modify state to simulate old API version
+pulumi stack export > state.json
+jq '(.deployment.resources[] | select(.type == "azure-native:storage:StorageAccount")) |= (.outputs.azureApiVersion = "2023-01-01")' state.json > state-old.json
+
+# 3. Import modified state
+pulumi stack import --file state-old.json
+
+# 4. Run refresh (provider reads with new API, compares with old)
+pulumi refresh --yes
+
+# 5. Verify no spurious changes detected
+```
+
+**Option 3: Two Provider Binaries (Most Thorough)**
+```bash
+# 1. Checkout and build provider at earlier version
+git checkout v3.50.0
+make provider
+cp bin/pulumi-resource-azure-native ~/provider-old
+
+# 2. Deploy resources using old provider
+PATH="$HOME:$PATH" pulumi up
+
+# 3. Checkout and build provider at newer version
+git checkout v3.100.0
+make provider
+cp bin/pulumi-resource-azure-native ~/provider-new
+
+# 4. Run refresh with new provider
+PATH="$HOME:$PATH" pulumi refresh --yes
+
+# 5. Verify no spurious changes or replacements
+```
+
+**Why This Matters:**
+
+- The SDK version does NOT control API versions - the provider binary's metadata does
+- When users upgrade provider versions, default API versions can change
+- The provider must correctly handle state migration when `azureApiVersion` differs between old state and new metadata
+- Test scenarios should validate that refresh operations don't report spurious property changes when only the API version changed
+
+See issue #4400 for an example of a bug that required this type of testing to validate the fix.
+
 ## Submodule Management
 
 ```bash

provider/pkg/provider/provider.go

@@ -31,6 +31,7 @@ import (
 	"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/azure"
 	"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/azure/cloud"
 	"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/convert"
+	"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/openapi"
 	"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/openapi/defaults"
 	"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/provider/crud"
 	"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/resources"
@@ -175,6 +176,50 @@ func (k *azureNativeProvider) lookupResourceFromURN(urn resource.URN) (*resource
 	return &res, nil
 }
 
+// lookupResourceWithAPIVersion attempts to look up resource metadata for a specific API version.
+// It finds a resource with the same ARM path as the current resource but with the requested API version.
+// This works because the ARM path is stable across API versions for the same resource type.
+func (k *azureNativeProvider) lookupResourceWithAPIVersion(urn resource.URN, apiVersion string) (*resources.AzureAPIResource, error) {
+	// Get current resource to extract its ARM path for verification
+	currentRes, err := k.lookupResourceFromURN(urn)
+	if err != nil {
+		return nil, err
+	}
+
+	// Parse the current type token to extract module and resource name
+	currentType := string(urn.Type())
+	module, _, resourceName, err := resources.ParseToken(currentType)
+	if err != nil {
+		return nil, errors.Wrapf(err, "parsing resource type %s", currentType)
+	}
+
+	// Convert API version from ISO format (2024-01-02) to SDK version format (v20240102)
+	sdkVersion := openapi.ApiToSdkVersion(openapi.ApiVersion(apiVersion))
+
+	// Construct candidate type token with the target API version
+	candidateType := resources.BuildToken(module, string(sdkVersion), resourceName)
+
+	// Look up the resource with this type token
+	res, ok, err := k.LookupResource(candidateType)
+	if err != nil {
+		return nil, errors.Wrapf(err, "looking up resource type %s", candidateType)
+	}
+	if !ok {
+		return nil, errors.Errorf("Resource type %s not found (API version %s not available in this provider version)",
+			candidateType, apiVersion)
+	}
+
+	// Verify this resource has the same path and correct API version
+	if res.Path != currentRes.Path {
+		return nil, errors.Errorf("Resource path mismatch: expected %s, found %s", currentRes.Path, res.Path)
+	}
+	if res.APIVersion != apiVersion {
+		return nil, errors.Errorf("API version mismatch: expected %s, found %s", apiVersion, res.APIVersion)
+	}
+
+	return &res, nil
+}
+
 // newCrudClient implements crud.ResourceCrudClientFactory
 func (p *azureNativeProvider) newCrudClient(res *resources.AzureAPIResource) crud.ResourceCrudClient {
 	return crud.NewResourceCrudClient(p.azureClient, p.lookupType, p.converter, p.subscriptionID, res)
@@ -920,7 +965,7 @@ func (k *azureNativeProvider) DiffConfig(context.Context, *rpc.DiffRequest) (*rp
 }
 
 // Diff checks what impacts a hypothetical update will have on the resource's properties.
-func (k *azureNativeProvider) Diff(_ context.Context, req *rpc.DiffRequest) (*rpc.DiffResponse, error) {
+func (k *azureNativeProvider) Diff(ctx context.Context, req *rpc.DiffRequest) (*rpc.DiffResponse, error) {
 	urn := resource.URN(req.GetUrn())
 	label := fmt.Sprintf("%s.Diff(%s)", k.name, urn)
 
@@ -974,6 +1019,22 @@ func (k *azureNativeProvider) Diff(_ context.Context, req *rpc.DiffRequest) (*rp
 		return nil, err
 	}
 
+	// Check if API version has changed between state and current provider metadata.
+	// If so, warn the user to run `pulumi refresh` for proper schema alignment.
+	var oldApiVersion string
+	if azureApiVersion, ok := oldState["azureApiVersion"]; ok && azureApiVersion.IsString() {
+		oldApiVersion = azureApiVersion.StringValue()
+		if oldApiVersion != res.APIVersion {
+			logging.V(5).Infof("%s: API version in state (%s) differs from provider metadata (%s)",
+				label, oldApiVersion, res.APIVersion)
+
+			k.host.Log(ctx, diag.Warning, urn, fmt.Sprintf(
+				"Resource API version has changed from %s to %s. "+
+					"Run 'pulumi refresh' to update the resource state with the new API version schema.",
+				oldApiVersion, res.APIVersion))
+		}
+	}
+
 	detailedDiff := diff(k.lookupType, *res, oldInputs, newResInputs)
 	if detailedDiff == nil {
 		return &rpc.DiffResponse{
@@ -1312,6 +1373,24 @@ func (k *azureNativeProvider) Read(ctx context.Context, req *rpc.ReadRequest) (*
 		return nil, err
 	}
 
+	// Check if API version has changed (e.g., via alias during upgrade).
+	// If so, we need to use the old API version's schema for normalizing old state.
+	var oldApiVersion string
+	var oldRes *resources.AzureAPIResource
+	if azureApiVersion, ok := oldState["azureApiVersion"]; ok && azureApiVersion.IsString() {
+		oldApiVersion = azureApiVersion.StringValue()
+		if oldApiVersion != res.APIVersion {
+			// API version has changed. Try to look up the old resource metadata.
+			logging.V(5).Infof("%s: API version changed from %s to %s", label, oldApiVersion, res.APIVersion)
+			oldRes, err = k.lookupResourceWithAPIVersion(urn, oldApiVersion)
+			if err != nil {
+				logging.V(5).Infof("%s: could not look up old resource metadata for API version %s: %v", label, oldApiVersion, err)
+				// Continue with current resource metadata but log the issue
+				oldRes = nil
+			}
+		}
+	}
+
 	crudClient := crud.NewResourceCrudClient(k.azureClient, k.lookupType, k.converter, k.subscriptionID, res)
 
 	var outputs map[string]interface{}
@@ -1379,21 +1458,45 @@ func (k *azureNativeProvider) Read(ctx context.Context, req *rpc.ReadRequest) (*
 
 		// 1. If we previously reset inputs to their default value, remove them so we don't get them in
 		// the projected output. This would cause unnecessary changes on refresh.
-		removeDefaults(*res, plainOldState, previousInputs.Mappable())
+		// Use old resource metadata if API version changed to ensure we use the correct defaults.
+		resForDefaults := res
+		if oldRes != nil {
+			resForDefaults = oldRes
+		}
+		removeDefaults(*resForDefaults, plainOldState, previousInputs.Mappable())
+
 		// 2. Project old outputs to their corresponding input shape (exclude read-only properties).
-		oldInputProjection := k.converter.SdkOutputsToSdkInputs(res.PutParameters, plainOldState)
+		// Use old resource metadata if API version changed to ensure schema-correct conversion.
+		var oldInputProjection map[string]interface{}
+		if oldRes != nil {
+			// API version changed: use old schema for converting old state
+			oldInputProjection = k.converter.SdkOutputsToSdkInputs(oldRes.PutParameters, plainOldState)
+		} else {
+			// Same API version: use current schema
+			oldInputProjection = k.converter.SdkOutputsToSdkInputs(res.PutParameters, plainOldState)
+		}
+
 		// 3a. Remove sub-resource properties from new outputs which weren't set in the old inputs.
 		// If the user didn't specify them inline originally, we don't want to push them into the inputs now.
 		outputsWithoutIgnores := k.removeUnsetSubResourceProperties(ctx, urn, outputs, inputs, res)
 		// 3b. Project new outputs to their corresponding input shape (exclude read-only properties).
+		// Always use new schema for new outputs from Azure.
 		newInputProjection := k.converter.SdkOutputsToSdkInputs(res.PutParameters, outputsWithoutIgnores)
+
 		// 4. Calculate the difference between two projections. This should give us actual significant changes
 		// that happened in Azure between the last resource update and its current state.
 		oldInputPropertyMap := resource.NewPropertyMapFromMap(oldInputProjection)
 		newInputPropertyMap := resource.NewPropertyMapFromMap(newInputProjection)
 		diff := oldInputPropertyMap.Diff(newInputPropertyMap)
+
 		// 5. Apply this difference to the actual inputs (not a projection) that we have in state.
-		inputs = applyDiff(inputs, diff)
+		// If API version changed and we couldn't find old metadata, preserve old inputs to avoid spurious diffs.
+		if oldApiVersion != "" && oldApiVersion != res.APIVersion && oldRes == nil {
+			logging.V(5).Infof("%s: API version changed but old metadata not found, preserving old inputs", label)
+			// Don't apply the diff - keep old inputs as-is since we can't reliably calculate changes
+		} else {
+			inputs = applyDiff(inputs, diff)
+		}
 	}
 
 	// Store both outputs and inputs into the state.

provider/pkg/provider/provider_e2e_test.go

@@ -359,6 +359,12 @@ func TestUpgradeContainerServiceAgentPool_2_90_0(t *testing.T) {
 	upgradeTest(t, "upgrade-containerservice-agentpool", "2.90.0")
 }
 
+func TestUpgradeAksApiVersion_2_90_0(t *testing.T) {
+	upgradeTest(t, "upgrade-aks-api-version", "2.90.0",
+		// v2 uses versioned type (containerservice/v20240102preview), v3 uses unversioned with alias
+		optproviderupgrade.NewSourcePath(filepath.Join("test-programs", "upgrade-aks-api-version", "v3")))
+}
+
 func upgradeTest(t *testing.T, testProgramDir string, upgradeFromVersion string, opts ...optproviderupgrade.PreviewProviderUpgradeOpt) {
 	t.Helper()
 	if testing.Short() {