Skip to content

Commit

Permalink
Moving position for register cluster info to populate spark version
Browse files Browse the repository at this point in the history
  • Loading branch information
pritishpai committed Jul 17, 2024
1 parent b3d4a68 commit 0c2e3ca
Showing 1 changed file with 4 additions and 3 deletions.
7 changes: 4 additions & 3 deletions src/databricks/labs/ucx/source_code/jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ def build_dependency_graph(self, parent: DependencyGraph) -> list[DependencyProb
return list(self._register_task_dependencies(parent))

def _register_task_dependencies(self, graph: DependencyGraph) -> Iterable[DependencyProblem]:
yield from self._register_cluster_info()
yield from self._register_libraries(graph)
yield from self._register_existing_cluster_id(graph)
yield from self._register_notebook(graph)
Expand All @@ -110,7 +111,6 @@ def _register_task_dependencies(self, graph: DependencyGraph) -> Iterable[Depend
yield from self._register_run_job_task(graph)
yield from self._register_pipeline_task(graph)
yield from self._register_spark_submit_task(graph)
yield from self._register_cluster_info()

def _register_libraries(self, graph: DependencyGraph) -> Iterable[DependencyProblem]:
if not self._task.libraries:
Expand All @@ -125,8 +125,9 @@ def _register_library(self, graph: DependencyGraph, library: compute.Library) ->
yield from problems
if library.egg:
if self.runtime_version > (14, 0):
yield DependencyProblem('not-yet-implemented', 'Installing eggs is no longer supported '
'on Databricks 14.0 or higher')
yield DependencyProblem(
'not-yet-implemented', 'Installing eggs is no longer supported ' 'on Databricks 14.0 or higher'
)
logger.info(f"Registering library from {library.egg}")
with self._ws.workspace.download(library.egg, format=ExportFormat.AUTO) as remote_file:
with tempfile.TemporaryDirectory() as directory:
Expand Down

0 comments on commit 0c2e3ca

Please sign in to comment.