Merge branch 'main' into whitelist-pulp

.github/workflows/acceptance.yml

@@ -39,12 +39,16 @@ jobs:
       - name: Install hatch
         run: pip install hatch==1.9.4
 
+      - name: Fetch relevant branches
+        run: |
+          git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF
+          git fetch origin $GITHUB_HEAD_REF:$GITHUB_HEAD_REF
+
       - name: Run integration tests
-        # ...
         uses: databrickslabs/sandbox/acceptance@acceptance/v0.2.2
         with:
           vault_uri: ${{ secrets.VAULT_URI }}
-          timeout: 45m
+          timeout: 55m
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}

README.md

@@ -618,6 +618,9 @@ in the Migration dashboard.
 The `experimental-workflow-linter` workflow lints accessible code belonging to all workflows/jobs present in the
 workspace. The linting emits problems indicating what to resolve for making the code Unity Catalog compatible.
 
+Once the workflow completes, the output will be stored in `$inventory_database.workflow_problems` table, and displayed
+in the Migration dashboard.
+
 ![code compatibility problems](docs/code_compatibility_problems.png)
 
 [[back to top](#databricks-labs-ucx)]

src/databricks/labs/ucx/config.py

@@ -41,7 +41,7 @@ class WorkspaceConfig:  # pylint: disable=too-many-instance-attributes
     uber_spn_id: str | None = None
     uber_instance_profile: str | None = None
 
-    is_terraform_used: bool = False  # Not used, keep for backwards compatability
+    is_terraform_used: bool = False  # Not used, keep for backwards compatibility
 
     # Whether the assessment should capture a specific list of databases, if not specified, it will list all databases.
     include_databases: list[str] | None = None
@@ -52,6 +52,9 @@ class WorkspaceConfig:  # pylint: disable=too-many-instance-attributes
     exclude_paths_in_mount: list[str] | None = None
     include_paths_in_mount: list[str] | None = None
 
+    # Used for limiting the number of jobs to be analysed
+    include_job_ids: list[int] | None = None
+
     # Whether to trigger assessment job after installation
     trigger_job: bool = False
 

src/databricks/labs/ucx/contexts/application.py

@@ -402,6 +402,7 @@ def workflow_linter(self):
             self.dependency_resolver,
             self.path_lookup,
             MigrationIndex([]),  # TODO: bring back self.tables_migrator.index()
+            self.config.include_job_ids,
         )
 
     @cached_property

src/databricks/labs/ucx/mixins/fixtures.py

@@ -13,8 +13,6 @@
 from typing import BinaryIO
 
 import pytest
-from databricks.labs.blueprint.entrypoint import is_in_debug
-from databricks.labs.blueprint.wheels import ProductInfo
 from databricks.labs.lsql.backends import StatementExecutionBackend
 from databricks.labs.blueprint.commands import CommandExecutor
 from databricks.sdk import AccountClient, WorkspaceClient
@@ -53,7 +51,6 @@
 )
 from databricks.sdk.service.workspace import ImportFormat, Language
 
-from databricks.labs.ucx.config import WorkspaceConfig
 from databricks.labs.ucx.workspace_access.groups import MigratedGroup
 
 # this file will get to databricks-labs-pytester project and be maintained/refactored there
@@ -1400,33 +1397,3 @@ def get_test_purge_time() -> str:
 def get_purge_suffix() -> str:
     """HEX-encoded purge time suffix for test objects."""
     return f'ra{int(get_test_purge_time()):x}'
-
-
-@pytest.fixture
-def modified_or_skip():
-    product_info = ProductInfo.from_class(WorkspaceConfig)
-    checkout_root = product_info.checkout_root()
-
-    def run_command(command: str) -> str:
-        with subprocess.Popen(
-            command.split(),
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            cwd=checkout_root,
-        ) as process:
-            output, error = process.communicate()
-            if process.returncode != 0:
-                pytest.fail(f"Command failed: {command}\n{error.decode('utf-8')}", pytrace=False)
-            return output.decode("utf-8").strip()
-
-    def inner(package: str):
-        if is_in_debug():
-            return  # not skipping, as we're debugging
-        if 'TEST_NIGHTLY' in os.environ:
-            return  # or during nightly runs
-        current_branch = run_command("git branch --show-current")
-        changed_files = run_command(f"git diff origin/main..{current_branch} --name-only")
-        if package not in changed_files:
-            pytest.skip(f"Skipping long test as {package} was not modified in branch {current_branch}")
-
-    return inner

src/databricks/labs/ucx/source_code/jobs.py

@@ -287,17 +287,22 @@ def __init__(
         resolver: DependencyResolver,
         path_lookup: PathLookup,
         migration_index: MigrationIndex,
+        include_job_ids: list[int] | None = None,
     ):
         self._ws = ws
         self._resolver = resolver
         self._path_lookup = path_lookup
         self._migration_index = migration_index
+        self._include_job_ids = include_job_ids
 
     def refresh_report(self, sql_backend: SqlBackend, inventory_database: str):
         tasks = []
         all_jobs = list(self._ws.jobs.list())
         logger.info(f"Preparing {len(all_jobs)} linting jobs...")
         for job in all_jobs:
+            if self._include_job_ids and job.job_id not in self._include_job_ids:
+                logger.info(f"Skipping job {job.job_id}...")
+                continue
             tasks.append(functools.partial(self.lint_job, job.job_id))
         logger.info(f"Running {tasks} linting tasks in parallel...")
         job_problems, errors = Threads.gather('linting workflows', tasks)