Resolve remote references during plan for "skip" targets

acceptance/bundle/resource_deps/jobs_update_remote/output.txt

@@ -58,6 +58,9 @@ Deployment complete!
   }
 }
 
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+
 === Update trigger.periodic.unit remotely and re-deploy; jobs.bar is unchanged
 >>> envsubst
 

acceptance/bundle/resource_deps/jobs_update_remote/script

@@ -3,6 +3,8 @@ $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json
 trace $CLI bundle deploy
 trace print_requests.py //jobs
 
+trace $CLI bundle plan
+
 foo_id=`read_id.py jobs foo`
 echo "$foo_id:FOO_ID" >> ACC_REPLS
 export foo_id

acceptance/bundle/resource_deps/remote_field_storage_location/out.plan_after_deploy.direct.txt

@@ -1,3 +1 @@
-update volumes.foo
-
-Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged
+Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged

acceptance/bundle/resource_deps/remote_field_storage_location/script

@@ -13,3 +13,4 @@ print_requests.py --get //unity &> out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE
 
 # Should show empty plan for direct, but shows update in volumes.foo (permanent drift)
 $CLI bundle plan &> out.plan_after_deploy.$DATABRICKS_BUNDLE_ENGINE.txt
+$CLI bundle plan -o json &> out.plan_after_deploy.$DATABRICKS_BUNDLE_ENGINE.json

acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json

@@ -37,21 +37,7 @@
           "label": "${resources.pipelines.foo1.creator_user_name}"
         }
       ],
-      "action": "update",
-      "new_state": {
-        "value": {
-          "channel": "CURRENT",
-          "deployment": {
-            "kind": "BUNDLE",
-            "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
-          },
-          "edition": "ADVANCED",
-          "name": "foo2 foo1.creator_user_name=${resources.pipelines.foo1.creator_user_name}"
-        },
-        "vars": {
-          "name": "foo2 foo1.creator_user_name=${resources.pipelines.foo1.creator_user_name}"
-        }
-      },
+      "action": "skip",
       "remote_state": {
         "creator_user_name": "[USERNAME]",
         "last_modified": [UNIX_TIME_MILLIS],
@@ -72,11 +58,6 @@
         "state": "IDLE"
       },
       "changes": {
-        "local": {
-          "name": {
-            "action": "update"
-          }
-        },
         "remote": {
           "storage": {
             "action": "skip",
@@ -92,21 +73,7 @@
           "label": "${resources.pipelines.foo2.state}"
         }
       ],
-      "action": "update",
-      "new_state": {
-        "value": {
-          "channel": "CURRENT",
-          "deployment": {
-            "kind": "BUNDLE",
-            "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
-          },
-          "edition": "ADVANCED",
-          "name": "foo3 foo2.state=${resources.pipelines.foo2.state}"
-        },
-        "vars": {
-          "name": "foo3 foo2.state=${resources.pipelines.foo2.state}"
-        }
-      },
+      "action": "skip",
       "remote_state": {
         "creator_user_name": "[USERNAME]",
         "last_modified": [UNIX_TIME_MILLIS],
@@ -127,11 +94,6 @@
         "state": "IDLE"
       },
       "changes": {
-        "local": {
-          "name": {
-            "action": "update"
-          }
-        },
         "remote": {
           "storage": {
             "action": "skip",
@@ -147,21 +109,7 @@
           "label": "${resources.pipelines.foo3.last_modified}"
         }
       ],
-      "action": "update",
-      "new_state": {
-        "value": {
-          "channel": "CURRENT",
-          "deployment": {
-            "kind": "BUNDLE",
-            "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
-          },
-          "edition": "ADVANCED",
-          "name": "foo4 foo3.last_modified=${resources.pipelines.foo3.last_modified}"
-        },
-        "vars": {
-          "name": "foo4 foo3.last_modified=${resources.pipelines.foo3.last_modified}"
-        }
-      },
+      "action": "skip",
       "remote_state": {
         "creator_user_name": "[USERNAME]",
         "last_modified": [UNIX_TIME_MILLIS],
@@ -182,11 +130,6 @@
         "state": "IDLE"
       },
       "changes": {
-        "local": {
-          "name": {
-            "action": "update"
-          }
-        },
         "remote": {
           "storage": {
             "action": "skip",

bundle/direct/bundle_apply.go

@@ -25,6 +25,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa
 	}
 
 	b.StateDB.AssertOpened()
+	b.RemoteStateCache.Clear()
 
 	g, err := makeGraph(plan)
 	if err != nil {

bundle/direct/bundle_plan.go

@@ -196,7 +196,13 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks
 			remoteAction, remoteChangeMap = interpretOldStateVsRemoteState(ctx, adapter, remoteDiff, remoteState)
 		}
 
-		entry.Action = max(localAction, remoteAction).String()
+		action := max(localAction, remoteAction)
+		if action == deployplan.ActionTypeSkip {
+			// resource is not going to change, can use remoteState to resolve references
+			b.RemoteStateCache.Store(resourceKey, remoteState)
+		}
+
+		entry.Action = action.String()
 
 		if len(localChangeMap) > 0 || len(remoteChangeMap) > 0 {
 			entry.Changes = &deployplan.Changes{
@@ -276,9 +282,11 @@ func interpretOldStateVsRemoteState(ctx context.Context, adapter *dresources.Ada
 	return action, m
 }
 
+// TODO: calling this "Local" is not right, it can resolve "id" and remote refrences for "skip" targets
 func (b *DeploymentBundle) LookupReferenceLocal(ctx context.Context, path *structpath.PathNode) (any, error) {
 	// TODO: Prefix(3) assumes resources.jobs.foo but not resources.jobs.foo.permissions
 	targetResourceKey := path.Prefix(3).String()
+
 	fieldPath := path.SkipPrefix(3)
 	fieldPathS := fieldPath.String()
 
@@ -349,19 +357,32 @@ func (b *DeploymentBundle) LookupReferenceLocal(ctx context.Context, path *struc
 
 	if configValidErr != nil && remoteValidErr == nil {
 		// The field is only present in remote state schema.
-		// TODO: If resource is unchanged in this plan, we can proceed with resolution.
-		// If resource is going to change, we need to postpone this until deploy.
+		if targetAction != deployplan.ActionTypeSkip {
+			// The resource is going to be updated, so remoteState can change
+			return nil, errDelayed
+		}
+		remoteState, ok := b.RemoteStateCache.Load(targetResourceKey)
+		if ok {
+			return structaccess.Get(remoteState, fieldPath)
+		}
 		return nil, errDelayed
 	}
 
-	// Field is present in both: try local, fallback to delayed.
+	// Field is present in both: try local, fallback to remote (if skip) then delayed.
 
 	value, err := structaccess.Get(localConfig, fieldPath)
 
 	if err == nil && value != nil {
 		return value, nil
 	}
 
+	if targetAction == deployplan.ActionTypeSkip {
+		remoteState, ok := b.RemoteStateCache.Load(targetResourceKey)
+		if ok {
+			return structaccess.Get(remoteState, fieldPath)
+		}
+	}
+
 	return nil, errDelayed
 }