From 033bd76cfd74e647ede7260ccf30b0be3250110b Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Wed, 6 Sep 2023 16:19:52 +0100 Subject: [PATCH 1/3] Fix #8544: Parse the correct schema version from manifest --- core/dbt/contracts/graph/manifest.py | 3 ++- .../artifacts/test_previous_version_state.py | 19 ++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index 2786529dbb0..9adfabe49ee 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -1528,7 +1528,8 @@ def get_manifest_schema_version(dct: dict) -> int: schema_version = dct.get("metadata", {}).get("dbt_schema_version", None) if not schema_version: raise ValueError("Manifest doesn't have schema version") - return int(schema_version.split(".")[-2][-1]) + + return int(schema_version.split("/")[-1].split(".")[0][1:]) def _check_duplicates(value: BaseNode, src: Mapping[str, BaseNode]): diff --git a/tests/functional/artifacts/test_previous_version_state.py b/tests/functional/artifacts/test_previous_version_state.py index 5a15040bae1..4a9aebbc05c 100644 --- a/tests/functional/artifacts/test_previous_version_state.py +++ b/tests/functional/artifacts/test_previous_version_state.py @@ -1,9 +1,12 @@ -import pytest +import json import os import shutil -from dbt.tests.util import run_dbt, get_manifest + +import pytest + +from dbt.contracts.graph.manifest import WritableManifest, get_manifest_schema_version from dbt.exceptions import IncompatibleSchemaError -from dbt.contracts.graph.manifest import WritableManifest +from dbt.tests.util import run_dbt, get_manifest # This project must have one of each kind of node type, plus disabled versions, for # test coverage to be complete. @@ -352,3 +355,13 @@ def test_nonbackwards_compatible_versions(self, project): # schema versions 1, 2, 3 are all not forward compatible for schema_version in range(1, 4): self.compare_previous_state(project, schema_version, False, 0) + + def test_get_manifest_schema_version(self, project): + for schema_version in range(1, self.CURRENT_EXPECTED_MANIFEST_VERSION): + manifest_path = os.path.join( + project.test_data_dir, f"state/v{schema_version}/manifest.json" + ) + manifest = json.load(open(manifest_path)) + + manifest_version = get_manifest_schema_version(manifest) + assert manifest_version == schema_version From 41c24d1edbbadc4e0137a7e51fc093bd7551b54d Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Wed, 6 Sep 2023 16:24:40 +0100 Subject: [PATCH 2/3] add changie --- .changes/unreleased/Fixes-20230906-162427.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Fixes-20230906-162427.yaml diff --git a/.changes/unreleased/Fixes-20230906-162427.yaml b/.changes/unreleased/Fixes-20230906-162427.yaml new file mode 100644 index 00000000000..6b45ab72b59 --- /dev/null +++ b/.changes/unreleased/Fixes-20230906-162427.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Parse the correct schema version from manifest +time: 2023-09-06T16:24:27.849069+01:00 +custom: + Author: aranke + Issue: "8544" From 00d9fcf0177027ccc5c6cffb5e561729442d6464 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Wed, 6 Sep 2023 18:37:26 +0100 Subject: [PATCH 3/3] add comment --- core/dbt/contracts/graph/manifest.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index 9adfabe49ee..d4bdcbddcc1 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -1529,6 +1529,13 @@ def get_manifest_schema_version(dct: dict) -> int: if not schema_version: raise ValueError("Manifest doesn't have schema version") + # schema_version is in this format: https://schemas.getdbt.com/dbt/manifest/v10.json + # What the code below is doing: + # 1. Split on "/" – v10.json + # 2. Split on "." – v10 + # 3. Skip first character – 10 + # 4. Convert to int + # TODO: If this gets more complicated, turn into a regex return int(schema_version.split("/")[-1].split(".")[0][1:])