diff --git a/src/ingestion/connectors/task-tracking/jira/dbt/jira__bronze_promoted.sql b/src/ingestion/connectors/task-tracking/jira/dbt/jira__bronze_promoted.sql index 64e8f1813..5b20f4ea2 100644 --- a/src/ingestion/connectors/task-tracking/jira/dbt/jira__bronze_promoted.sql +++ b/src/ingestion/connectors/task-tracking/jira/dbt/jira__bronze_promoted.sql @@ -30,10 +30,18 @@ ------------------------------------------------------------------------- #} -- @cpt-principle:cpt-dataflow-principle-promote-bronze:p1 +{# `staging` tag (not just `jira`): the prod pipeline's staging step selects + `tag:staging,tag:jira` (an AND-intersection — see render_cronworkflow.py / + render_sync_trigger.py). Tagged only `jira`, this promote model was excluded + from that selection (and, with no `+` in the selector, not pulled in as an + upstream either), so on a real Airbyte sync bronze stayed plain MergeTree and + the downstream `jira-enrich` step crashed with `Storage MergeTree doesn't + support FINAL` (issue #1886). `schema='staging'` sets the target DATABASE, not + a dbt tag, so it does not participate in tag selection. #} {{ config( materialized='view', schema='staging', - tags=['jira'] + tags=['jira', 'staging'] ) }} {# All Jira bronze tables carry a `unique_key` column added by the connector diff --git a/src/ingestion/tests/e2e/meta/test_dbt_runner.py b/src/ingestion/tests/e2e/meta/test_dbt_runner.py index 83338f0a5..06ef48e1f 100644 --- a/src/ingestion/tests/e2e/meta/test_dbt_runner.py +++ b/src/ingestion/tests/e2e/meta/test_dbt_runner.py @@ -9,14 +9,11 @@ from __future__ import annotations import json -from pathlib import Path import pytest - from lib.dbt_runner import DbtError, DbtRunner from lib.worker import WorkerContext - pytestmark = pytest.mark.smoke @@ -42,13 +39,49 @@ def test_dbt_profiles_written(dbt_runner: DbtRunner) -> None: assert "ReplacingMergeTree" in body +def test_jira_staging_selector_includes_bronze_promoted(dbt_runner: DbtRunner) -> None: + """Regression guard for issue #1886. + + The prod jira pipeline runs its staging dbt step with the selector + ``tag:staging,tag:jira`` (an AND-intersection of both tags — hardcoded in + ``reconcile-connectors/python/render_cronworkflow.py`` and + ``render_sync_trigger.py``). The MergeTree -> ReplacingMergeTree promotion + lives in the ``jira__bronze_promoted`` model, and the enrich step that runs + right after reads ``bronze_jira.jira_issue FINAL`` — illegal unless that + promotion already flipped bronze to ReplacingMergeTree. + + When ``jira__bronze_promoted`` was tagged only ``['jira']`` the AND-selector + excluded it (and, with no ``+`` in the selector, it was not pulled in as an + upstream either), so prod never promoted, bronze stayed MergeTree, and enrich + crashed with ``Storage MergeTree doesn't support FINAL`` on every real sync. + Assert the promote model carries BOTH tags so the prod selector picks it up. + """ + manifest = dbt_runner.target_dir / "manifest.json" + data = json.loads(manifest.read_text(encoding="utf-8")) + + # Reproduce `tag:staging,tag:jira` (AND) against the parsed manifest: a model + # is selected iff its config tags contain every tag in the intersection. + required = {"staging", "jira"} + selected = { + node["name"] + for node in data["nodes"].values() + if node.get("resource_type") == "model" and required.issubset(set(node.get("config", {}).get("tags", []))) + } + + assert "jira__bronze_promoted" in selected, ( + "jira__bronze_promoted is not selected by the prod staging selector " + "'tag:staging,tag:jira'; it must be tagged both 'jira' and 'staging' or " + "the MergeTree->ReplacingMergeTree promotion never runs on a real sync " + "and jira-enrich crashes with ILLEGAL_FINAL (issue #1886)." + ) + + def test_dbt_build_unknown_selector_raises(dbt_runner: DbtRunner) -> None: """A selector that matches no models surfaces a clear DbtError.""" # `dbt build --select ` is NOT an error in dbt — it just runs # zero models. So we instead pass an invalid selector syntax that dbt # rejects. The point of the test is that the wrapper surfaces failures # without swallowing the dbt output. - runner = dbt_runner # Use a deliberately broken --vars to force a non-zero exit # (more reliable than guessing bad selector syntax across dbt versions). with pytest.raises(DbtError): @@ -74,7 +107,6 @@ def test_dbt_build_with_worker_context_passes_var(dbt_runner: DbtRunner) -> None Running a real dbt build is exercised end-to-end by feature-yaml-rig; here we only verify that worker context produces a deterministic --vars payload. """ - runner = dbt_runner ctx = WorkerContext(worker_id="gw0", schema_suffix="_w0") # We don't call .build() (which would shell out); we just check the worker # id translation works as advertised.