From aa9ac67174eef0768fc2d55bf04f02727d663239 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 24 Nov 2025 15:34:09 +0100 Subject: [PATCH 1/2] Resolve remote references during plan for "skip" targets --- .../jobs_update_remote/output.txt | 3 + .../resource_deps/jobs_update_remote/script | 2 + .../out.plan_after_deploy.direct.txt | 4 +- .../remote_field_storage_location/script | 1 + .../remote_pipeline/out.plan_skip.direct.json | 63 +------------------ bundle/direct/bundle_apply.go | 1 + bundle/direct/bundle_plan.go | 29 +++++++-- 7 files changed, 36 insertions(+), 67 deletions(-) diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/output.txt b/acceptance/bundle/resource_deps/jobs_update_remote/output.txt index 4d8ed5a468..0d8f9b0c71 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/output.txt +++ b/acceptance/bundle/resource_deps/jobs_update_remote/output.txt @@ -58,6 +58,9 @@ Deployment complete! } } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + === Update trigger.periodic.unit remotely and re-deploy; jobs.bar is unchanged >>> envsubst diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/script b/acceptance/bundle/resource_deps/jobs_update_remote/script index 5f2f8cf7eb..42b7b9751e 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/script +++ b/acceptance/bundle/resource_deps/jobs_update_remote/script @@ -3,6 +3,8 @@ $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //jobs +trace $CLI bundle plan + foo_id=`read_id.py jobs foo` echo "$foo_id:FOO_ID" >> ACC_REPLS export foo_id diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/out.plan_after_deploy.direct.txt b/acceptance/bundle/resource_deps/remote_field_storage_location/out.plan_after_deploy.direct.txt index 6cb8c9d929..80d724bd01 100644 --- a/acceptance/bundle/resource_deps/remote_field_storage_location/out.plan_after_deploy.direct.txt +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/out.plan_after_deploy.direct.txt @@ -1,3 +1 @@ -update volumes.foo - -Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/script b/acceptance/bundle/resource_deps/remote_field_storage_location/script index 3b274eaecc..3591dbffd2 100644 --- a/acceptance/bundle/resource_deps/remote_field_storage_location/script +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/script @@ -13,3 +13,4 @@ print_requests.py --get //unity &> out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE # Should show empty plan for direct, but shows update in volumes.foo (permanent drift) $CLI bundle plan &> out.plan_after_deploy.$DATABRICKS_BUNDLE_ENGINE.txt +$CLI bundle plan -o json &> out.plan_after_deploy.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json b/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json index c55b0edbd3..f17b29e2a8 100644 --- a/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json +++ b/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json @@ -37,21 +37,7 @@ "label": "${resources.pipelines.foo1.creator_user_name}" } ], - "action": "update", - "new_state": { - "value": { - "channel": "CURRENT", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" - }, - "edition": "ADVANCED", - "name": "foo2 foo1.creator_user_name=${resources.pipelines.foo1.creator_user_name}" - }, - "vars": { - "name": "foo2 foo1.creator_user_name=${resources.pipelines.foo1.creator_user_name}" - } - }, + "action": "skip", "remote_state": { "creator_user_name": "[USERNAME]", "last_modified": [UNIX_TIME_MILLIS], @@ -72,11 +58,6 @@ "state": "IDLE" }, "changes": { - "local": { - "name": { - "action": "update" - } - }, "remote": { "storage": { "action": "skip", @@ -92,21 +73,7 @@ "label": "${resources.pipelines.foo2.state}" } ], - "action": "update", - "new_state": { - "value": { - "channel": "CURRENT", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" - }, - "edition": "ADVANCED", - "name": "foo3 foo2.state=${resources.pipelines.foo2.state}" - }, - "vars": { - "name": "foo3 foo2.state=${resources.pipelines.foo2.state}" - } - }, + "action": "skip", "remote_state": { "creator_user_name": "[USERNAME]", "last_modified": [UNIX_TIME_MILLIS], @@ -127,11 +94,6 @@ "state": "IDLE" }, "changes": { - "local": { - "name": { - "action": "update" - } - }, "remote": { "storage": { "action": "skip", @@ -147,21 +109,7 @@ "label": "${resources.pipelines.foo3.last_modified}" } ], - "action": "update", - "new_state": { - "value": { - "channel": "CURRENT", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" - }, - "edition": "ADVANCED", - "name": "foo4 foo3.last_modified=${resources.pipelines.foo3.last_modified}" - }, - "vars": { - "name": "foo4 foo3.last_modified=${resources.pipelines.foo3.last_modified}" - } - }, + "action": "skip", "remote_state": { "creator_user_name": "[USERNAME]", "last_modified": [UNIX_TIME_MILLIS], @@ -182,11 +130,6 @@ "state": "IDLE" }, "changes": { - "local": { - "name": { - "action": "update" - } - }, "remote": { "storage": { "action": "skip", diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 07d2d64fbf..508b7383f1 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -25,6 +25,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } b.StateDB.AssertOpened() + b.RemoteStateCache.Clear() g, err := makeGraph(plan) if err != nil { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index f02094a2d4..8103fde30f 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -196,7 +196,13 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks remoteAction, remoteChangeMap = interpretOldStateVsRemoteState(ctx, adapter, remoteDiff, remoteState) } - entry.Action = max(localAction, remoteAction).String() + action := max(localAction, remoteAction) + if action == deployplan.ActionTypeSkip { + // resource is not going to change, can use remoteState to resolve references + b.RemoteStateCache.Store(resourceKey, remoteState) + } + + entry.Action = action.String() if len(localChangeMap) > 0 || len(remoteChangeMap) > 0 { entry.Changes = &deployplan.Changes{ @@ -276,9 +282,11 @@ func interpretOldStateVsRemoteState(ctx context.Context, adapter *dresources.Ada return action, m } +// TODO: calling this "Local" is not right, it can resolve "id" and remote refrences for "skip" targets func (b *DeploymentBundle) LookupReferenceLocal(ctx context.Context, path *structpath.PathNode) (any, error) { // TODO: Prefix(3) assumes resources.jobs.foo but not resources.jobs.foo.permissions targetResourceKey := path.Prefix(3).String() + fieldPath := path.SkipPrefix(3) fieldPathS := fieldPath.String() @@ -349,12 +357,18 @@ func (b *DeploymentBundle) LookupReferenceLocal(ctx context.Context, path *struc if configValidErr != nil && remoteValidErr == nil { // The field is only present in remote state schema. - // TODO: If resource is unchanged in this plan, we can proceed with resolution. - // If resource is going to change, we need to postpone this until deploy. + if targetAction != deployplan.ActionTypeSkip { + // The resource is going to be updated, so remoteState can change + return nil, errDelayed + } + remoteState, ok := b.RemoteStateCache.Load(targetResourceKey) + if ok { + return structaccess.Get(remoteState, fieldPath) + } return nil, errDelayed } - // Field is present in both: try local, fallback to delayed. + // Field is present in both: try local, fallback to remote (if skip) then delayed. value, err := structaccess.Get(localConfig, fieldPath) @@ -362,6 +376,13 @@ func (b *DeploymentBundle) LookupReferenceLocal(ctx context.Context, path *struc return value, nil } + if targetAction == deployplan.ActionTypeSkip { + remoteState, ok := b.RemoteStateCache.Load(targetResourceKey) + if ok { + return structaccess.Get(remoteState, fieldPath) + } + } + return nil, errDelayed } From 92b58588b39e4195d4abdd66dc9011417addb336 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 24 Nov 2025 15:41:04 +0100 Subject: [PATCH 2/2] add missing files --- .../out.plan_after_deploy.direct.json | 82 +++++++++++++++++++ .../out.plan_after_deploy.terraform.json | 13 +++ 2 files changed, 95 insertions(+) create mode 100644 acceptance/bundle/resource_deps/remote_field_storage_location/out.plan_after_deploy.direct.json create mode 100644 acceptance/bundle/resource_deps/remote_field_storage_location/out.plan_after_deploy.terraform.json diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/out.plan_after_deploy.direct.json b/acceptance/bundle/resource_deps/remote_field_storage_location/out.plan_after_deploy.direct.json new file mode 100644 index 0000000000..6827735126 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/out.plan_after_deploy.direct.json @@ -0,0 +1,82 @@ +{ + "plan": { + "resources.schemas.my": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "main.myschema-[UNIQUE_NAME]", + "name": "myschema-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "updated_at": [UNIX_TIME_MILLIS][0], + "updated_by": "[USERNAME]" + } + }, + "resources.volumes.bar": { + "depends_on": [ + { + "node": "resources.schemas.my", + "label": "${resources.schemas.my.name}" + } + ], + "action": "skip", + "remote_state": { + "catalog_name": "main", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "main.myschema-[UNIQUE_NAME].volumebar-[UNIQUE_NAME]", + "name": "volumebar-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_name": "myschema-[UNIQUE_NAME]", + "storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]", + "updated_at": [UNIX_TIME_MILLIS][1], + "volume_type": "MANAGED" + }, + "changes": { + "remote": { + "storage_location": { + "action": "skip", + "reason": "server_side_default" + } + } + } + }, + "resources.volumes.foo": { + "depends_on": [ + { + "node": "resources.schemas.my", + "label": "${resources.schemas.my.name}" + }, + { + "node": "resources.volumes.bar", + "label": "${resources.volumes.bar.storage_location}" + } + ], + "action": "skip", + "remote_state": { + "catalog_name": "main", + "comment": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]", + "created_at": [UNIX_TIME_MILLIS][2], + "created_by": "[USERNAME]", + "full_name": "main.myschema-[UNIQUE_NAME].volumefoo-[UNIQUE_NAME]", + "name": "volumefoo-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_name": "myschema-[UNIQUE_NAME]", + "storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]", + "updated_at": [UNIX_TIME_MILLIS][2], + "volume_type": "MANAGED" + }, + "changes": { + "remote": { + "storage_location": { + "action": "skip", + "reason": "server_side_default" + } + } + } + } + } +} diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/out.plan_after_deploy.terraform.json b/acceptance/bundle/resource_deps/remote_field_storage_location/out.plan_after_deploy.terraform.json new file mode 100644 index 0000000000..fa2f597e6e --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/out.plan_after_deploy.terraform.json @@ -0,0 +1,13 @@ +{ + "plan": { + "resources.schemas.my": { + "action": "skip" + }, + "resources.volumes.bar": { + "action": "skip" + }, + "resources.volumes.foo": { + "action": "create" + } + } +}