Fixed spark version check for clusters defined in the same bundle

acceptance/bundle/trampoline/warning_message/databricks.yml

@@ -0,0 +1,37 @@
+bundle:
+  name: trampoline_warning_message
+
+targets:
+  dev:
+    mode: development
+    default: true
+
+  prod:
+    resources:
+      clusters:
+        development_cluster:
+          spark_version: 14.2.x-cpu-ml-scala2.12
+
+
+resources:
+  clusters:
+    development_cluster:
+      cluster_name: jobs-as-code-all-purpose-cluster
+      spark_version: 12.2.x-cpu-ml-scala2.12
+      node_type_id: r5d.8xlarge
+      autotermination_minutes: 30
+      autoscale:
+        min_workers: 1
+        max_workers: 1
+      driver_node_type_id: r5d.8xlarge
+  jobs:
+    whl:
+      name: "wheel-job"
+      tasks:
+        - task_key: test_task
+          python_wheel_task:
+            package_name: my_package
+            entry_point: my_module.my_function
+          existing_cluster_id: ${resources.clusters.development_cluster.id}
+          libraries:
+            - whl: ./dist/*.whl

acceptance/bundle/trampoline/warning_message/output.txt

@@ -0,0 +1,18 @@
+Error: Python wheel tasks require compute with DBR 13.3+ to include local libraries. Please change your cluster configuration or use the experimental 'python_wheel_wrapper' setting. See https://docs.databricks.com/dev-tools/bundles/python-wheel.html for more information.
+
+Name: trampoline_warning_message
+Target: dev
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/trampoline_warning_message/dev
+
+Found 1 error
+
+Exit code: 1
+Name: trampoline_warning_message
+Target: prod
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/trampoline_warning_message/prod
+
+Validation OK!

acceptance/bundle/trampoline/warning_message/script

@@ -0,0 +1,2 @@
+errcode $CLI bundle validate -t dev
+errcode $CLI bundle validate -t prod

bundle/trampoline/python_dbr_warning.go

@@ -9,6 +9,7 @@ import (
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/libraries"
 	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn/dynvar"
 	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/databricks-sdk-go"
 	"golang.org/x/mod/semver"
@@ -60,11 +61,32 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool {
 		}
 
 		if task.ExistingClusterId != "" {
-			version, err := getSparkVersionForCluster(ctx, b.WorkspaceClient(), task.ExistingClusterId)
-			// If there's error getting spark version for cluster, do not mark it as incompatible
-			if err != nil {
-				log.Warnf(ctx, "unable to get spark version for cluster %s, err: %s", task.ExistingClusterId, err.Error())
-				return false
+			var version string
+			var err error
+			// If the cluster id is a variable and it's not resolved, it means it references a cluster defined in the same bundle.
+			// So we can get the version from the cluster definition.
+			// It's defined in a form of resources.clusters.<cluster_key>.spark_version while the value here is
+			// ${resources.clusters.<cluster_key>.id}
+			if strings.HasPrefix(task.ExistingClusterId, "${") {
+				p, ok := dynvar.PureReferenceToPath(task.ExistingClusterId)
+				if !ok || len(p) < 3 {
+					log.Warnf(ctx, "unable to parse cluster key from %s", task.ExistingClusterId)
+					return false
+				}
+				clusterKey := p[2].Key()
+				cluster, ok := b.Config.Resources.Clusters[clusterKey]
+				if !ok {
+					log.Warnf(ctx, "unable to find cluster with key %s", clusterKey)
+					return false
+				}
+				version = cluster.SparkVersion
+			} else {
+				version, err = getSparkVersionForCluster(ctx, b.WorkspaceClient(), task.ExistingClusterId)
+				// If there's error getting spark version for cluster, do not mark it as incompatible
+				if err != nil {
+					log.Warnf(ctx, "unable to get spark version for cluster %s, err: %s", task.ExistingClusterId, err.Error())
+					return false
+				}
 			}
 
 			if lowerThanExpectedVersion(version) {
@@ -82,7 +104,7 @@ func lowerThanExpectedVersion(sparkVersion string) bool {
 		return false
 	}
 
-	if parts[1][0] == 'x' { // treat versions like 13.x as the very latest minor (13.99)
+	if len(parts[1]) > 0 && parts[1][0] == 'x' { // treat versions like 13.x as the very latest minor (13.99)
 		parts[1] = "99"
 	}
 

bundle/trampoline/python_dbr_warning_test.go

@@ -346,6 +346,7 @@ func TestSparkVersionLowerThanExpected(t *testing.T) {
 		"13.x-rc-scala-2.12":              false,
 		"client.1.10-scala2.12":           false,
 		"latest-stable-gpu-scala2.11":     false,
+		"1.":                              false,
 		"10.4.x-aarch64-photon-scala2.12": true,
 		"10.4.x-scala2.12":                true,
 		"13.0.x-scala2.12":                true,