From 5c83060440c5c17782189ad08a09865f0d1cc166 Mon Sep 17 00:00:00 2001 From: Serge Smertin <259697+nfx@users.noreply.github.com> Date: Fri, 6 Oct 2023 23:03:34 +0200 Subject: [PATCH] Detect installations of unreleased versions (#399) Any installations that are done from unreleased versions will get a different format. For example, the latest released version is `0.24.0` and there were 15 commits to `main` branch since the last release. `./install.sh` will tell about `Installing v0.24.1+1520231006230102`. --- src/databricks/labs/ucx/__about__.py | 1 + src/databricks/labs/ucx/install.py | 52 ++++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/src/databricks/labs/ucx/__about__.py b/src/databricks/labs/ucx/__about__.py index 493f7415d7..ebea246dbd 100644 --- a/src/databricks/labs/ucx/__about__.py +++ b/src/databricks/labs/ucx/__about__.py @@ -1 +1,2 @@ +# DO NOT MODIFY THIS FILE __version__ = "0.3.0" diff --git a/src/databricks/labs/ucx/install.py b/src/databricks/labs/ucx/install.py index 5551817c24..18ad64578c 100644 --- a/src/databricks/labs/ucx/install.py +++ b/src/databricks/labs/ucx/install.py @@ -1,6 +1,8 @@ +import datetime import logging import os import re +import shutil import subprocess import sys import tempfile @@ -12,6 +14,7 @@ import yaml from databricks.sdk import WorkspaceClient from databricks.sdk.core import DatabricksError +from databricks.sdk.mixins.compute import SemVer from databricks.sdk.service import compute, jobs from databricks.sdk.service.sql import EndpointInfoWarehouseType, SpotInstancePolicy from databricks.sdk.service.workspace import ImportFormat @@ -74,7 +77,7 @@ def __init__(self, ws: WorkspaceClient, *, prefix: str = "ucx", promtps: bool = self._dashboards = {} def run(self): - logger.info(f"Installing UCX v{__version__}") + logger.info(f"Installing UCX v{self._version}") self._configure() self._run_configured() @@ -89,8 +92,8 @@ def _run_configured(self): def run_for_config( ws: WorkspaceClient, config: WorkspaceConfig, *, prefix="ucx", override_clusters: dict[str, str] | None = None ) -> "WorkspaceInstaller": - logger.info(f"Installing UCX v{__version__} on {ws.config.host}") workspace_installer = WorkspaceInstaller(ws, prefix=prefix, promtps=False) + logger.info(f"Installing UCX v{workspace_installer._version} on {ws.config.host}") workspace_installer._config = config workspace_installer._write_config() workspace_installer._override_clusters = override_clusters @@ -418,7 +421,7 @@ def _job_settings(self, step_name: str, dbfs_path: str): tasks = sorted([t for t in _TASKS.values() if t.workflow == step_name], key=lambda _: _.name) return { "name": self._name(step_name), - "tags": {TAG_APP: self._app, TAG_STEP: step_name, "version": f"v{__version__}"}, + "tags": {TAG_APP: self._app, TAG_STEP: step_name, "version": f"v{self._version}"}, "job_clusters": self._job_clusters({t.job_cluster for t in tasks}), "email_notifications": email_notifications, "tasks": [self._job_task(task, dbfs_path) for task in tasks], @@ -515,6 +518,38 @@ def _job_clusters(self, names: set[str]): ) return clusters + @property + def _version(self): + if hasattr(self, "__version"): + return self.__version + project_root = self._find_project_root() + if not (project_root / ".git/config").exists(): + # normal install, downloaded releases won't have the .git folder + return __version__ + try: + out = subprocess.run(["git", "describe", "--tags"], stdout=subprocess.PIPE, check=True) # noqa S607 + git_detached_version = out.stdout.decode("utf8") + dv = SemVer.parse(git_detached_version) + datestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S") + # new commits on main branch since the last tag + new_commits = dv.pre_release.split("-")[0] + # show that it's a version different from the released one in stats + bump_patch = dv.patch + 1 + # create something that is both https://semver.org and https://peps.python.org/pep-0440/ + semver_and_pep0440 = f"{dv.major}.{dv.minor}.{bump_patch}+{new_commits}{datestamp}" + # validate the semver + SemVer.parse(semver_and_pep0440) + self.__version = semver_and_pep0440 + return semver_and_pep0440 + except Exception as err: + msg = ( + f"Cannot determine unreleased version. Please report this error " + f"message that you see on https://github.com/databrickslabs/ucx/issues/new. " + f"Meanwhile, download, unpack, and install the latest released version from " + f"https://github.com/databrickslabs/ucx/releases. Original error is: {err!s}" + ) + raise OSError(msg) from None + def _build_wheel(self, tmp_dir: str, *, verbose: bool = False): """Helper to build the wheel package""" streams = {} @@ -524,6 +559,17 @@ def _build_wheel(self, tmp_dir: str, *, verbose: bool = False): "stderr": subprocess.DEVNULL, } project_root = self._find_project_root() + is_non_released_version = "+" in self._version + if (project_root / ".git" / "config").exists() and is_non_released_version: + tmp_dir_path = Path(tmp_dir) / "working-copy" + # copy everything to a temporary directory + shutil.copytree(project_root, tmp_dir_path) + # and override the version file + version_file = tmp_dir_path / "src/databricks/labs/ucx/__about__.py" + with version_file.open("w") as f: + f.write(f'__version__ = "{self._version}"') + # working copy becomes project root for building a wheel + project_root = tmp_dir_path logger.debug(f"Building wheel for {project_root} in {tmp_dir}") subprocess.run( [sys.executable, "-m", "pip", "wheel", "--no-deps", "--wheel-dir", tmp_dir, project_root],