Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
engine='ReplacingMergeTree(_version)',
order_by=['unique_key'],
settings={'allow_nullable_key': 1},
tags=['jira', 'silver:class_task_comments']
tags=['jira', 'staging', 'silver:class_task_comments']
) }}

-- `body` is raw ADF JSON at Bronze level; plaintext extraction deferred.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
materialized='view',
alias='jira__task_projects',
schema='staging',
tags=['jira', 'silver:class_task_projects']
tags=['jira', 'staging', 'silver:class_task_projects']
) }}

-- View, not table: bronze `jira_projects` is MergeTree (full_refresh + overwrite),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
materialized='view',
alias='jira__task_sprints',
schema='staging',
tags=['jira', 'silver:class_task_sprints']
tags=['jira', 'staging', 'silver:class_task_sprints']
) }}

-- View, not table: bronze `jira_sprints` is MergeTree (full_refresh + overwrite),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
materialized='view',
alias='jira__task_statuses',
schema='staging',
tags=['jira', 'silver:class_task_statuses']
tags=['jira', 'staging', 'silver:class_task_statuses']
) }}

-- Per-source status dimension; unioned into `silver.class_task_statuses` via `union_by_tag`.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
materialized='view',
alias='jira__task_users',
schema='staging',
tags=['jira', 'silver:class_task_users']
tags=['jira', 'staging', 'silver:class_task_users']
) }}

-- Per-source staging view; unioned into `silver.class_task_users` via
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
engine='ReplacingMergeTree(_version)',
order_by=['unique_key'],
settings={'allow_nullable_key': 1},
tags=['jira', 'silver:class_task_worklogs']
tags=['jira', 'staging', 'silver:class_task_worklogs']
) }}

SELECT
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,17 @@
-- depends_on: {{ ref('jira__bronze_promoted') }}
{# `staging` tag (issue #1893): the prod jira pipeline materializes staging
feeders only in its staging step (`tag:staging,tag:jira`) and builds silver
with `tag:silver,tag:jira+`. Tagged only `jira`, this model matched neither
pass, so on a fresh install `staging.jira__users_snapshot` was never built and
the silver model `jira__users_fields_history` — which ref()s it — failed with
`code: 60 Unknown table expression identifier 'staging.jira__users_snapshot'`.
`schema='staging'` is the target DATABASE, not a dbt tag, so it does not
participate in tag selection. #}
{{ config(
materialized='incremental',
incremental_strategy='append',
schema='staging',
tags=['jira']
tags=['jira', 'staging']
) }}

{{ snapshot(
Expand Down
82 changes: 82 additions & 0 deletions src/ingestion/tests/e2e/meta/test_dbt_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,88 @@ def test_jira_staging_selector_includes_bronze_promoted(dbt_runner: DbtRunner) -
)


def test_every_jira_model_is_built_by_some_prod_pass(dbt_runner: DbtRunner) -> None:
"""Regression guard for issue #1893 (and the whole #1886/#1893 class).

Unlike other connectors — which build via the legacy single pass
``tag:<connector>+`` that pulls every connector-tagged model plus its
descendants — the jira pipeline builds in three narrow tag-scoped passes and
has NO ``tag:jira`` catch-all:

* staging: ``tag:staging,tag:jira`` (render_*.py, no ``+``)
* silver: ``tag:silver,tag:jira+`` (jira descriptor dbt_select)
* gold: ``tag:gold,tag:jira+`` (jira descriptor dbt_select)

A jira model tagged only ``['jira']`` (or ``['jira', 'silver:class_*']`` —
note ``silver:class_*`` is a routing tag, not the bare ``silver`` tag) matches
none of these passes, so it is never materialized. Its silver consumers then
fail: ``jira__users_fields_history`` ``ref()``s ``jira__users_snapshot`` and
dies with ``code: 60 Unknown table 'staging.jira__users_snapshot'`` (#1893),
and the ``class_task_*`` silver models lose their jira rows (or hit the
``union_by_tag`` empty-target compiler error on a fresh install).

Assert every jira connector model is selected by at least one prod pass.
Ephemeral models are exempt: they have no DB relation and are inlined into
their consumers via ``ref()``, so no pass needs to build them standalone.
"""
manifest = dbt_runner.target_dir / "manifest.json"
data = json.loads(manifest.read_text(encoding="utf-8"))
nodes = data["nodes"]
child_map = data.get("child_map", {})

def _tags(uid: str) -> set[str]:
return set(nodes[uid].get("config", {}).get("tags", []) or [])

def _descendants(seed: set[str]) -> set[str]:
seen: set[str] = set()
stack = list(seed)
while stack:
cur = stack.pop()
for child in child_map.get(cur, []):
if child not in seen:
seen.add(child)
stack.append(child)
return seen

def _select(expr: str) -> set[str]:
# Minimal dbt selector semantics: space = union of terms, comma =
# intersection of atoms, atom = ``tag:X`` optionally suffixed ``+``
# (the node plus all its graph descendants).
union: set[str] = set()
for term in expr.split():
inter: set[str] | None = None
for atom in term.split(","):
plus = atom.endswith("+")
tag = atom[:-1] if plus else atom
assert tag.startswith("tag:"), atom
base = {uid for uid in nodes if tag[4:] in _tags(uid)}
atom_set = base | _descendants(base) if plus else base
inter = atom_set if inter is None else (inter & atom_set)
union |= inter or set()
return union

covered = _select("tag:staging,tag:jira") | _select("tag:silver,tag:jira+") | _select("tag:gold,tag:jira+")

def _is_jira_model(node: dict) -> bool:
return node.get("resource_type") == "model" and (
"/connectors/task-tracking/jira/dbt/" in node.get("original_file_path", "")
)
Comment on lines +141 to +144

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟡 Minor | ⚡ Quick win

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
fd -a '^manifest\.json$' . -E target -E node_modules | while read -r manifest; do
  echo "== $manifest"
  jq -r '.nodes[]?.original_file_path // empty' "$manifest" |
    grep 'task-tracking/jira/dbt/' | head -5
done

Repository: constructorfabric/insight

Length of output: 163


🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "== candidate test file =="
fd -a 'test_dbt_runner.py$' . | sed 's#^\./##'

FILE="$(fd 'test_dbt_runner.py$' src/ingestion/tests/e2e/meta -1 || true)"
if [ -n "${FILE:-}" ]; then
  echo "== outline for $FILE =="
  ast-grep outline "$FILE" || true
  echo "== relevant lines =="
  sed -n '110,170p' "$FILE" | cat -n
fi

echo "== dbt-related files =="
fd -a 'dbt_project\.ya?ml|manifest\.json|model' . -E target -E node_modules -E .venv | sed 's#^\./##' | head -200

echo "== Jira connector references =="
rg -n 'task-tracking/jira/dbt|connectors/task-tracking/jira/dbt|/connectors/task-tracking/jira/dbt' . -E target -E node_modules -E .venv || true

Repository: constructorfabric/insight

Length of output: 4476


🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "== test full context =="
sed -n '1,120p' src/ingestion/tests/e2e/meta/test_dbt_runner.py | cat -n

echo "== dbt_project.yml =="
cat -n src/ingestion/dbt/dbt_project.yml

echo "== files under ingestion/dbt containing jira or task-tracking =="
find src/ingestion/dbt -maxdepth 6 \( -type f -o -type d \) | grep -E 'jira|task-tracking' | sort | sed -n '1,200p'

echo "== search repository (no binary) for jira paths =="
python3 - <<'PY'
import os, re
root='src/ingestion/dbt'
for dirpath, dirs, files in os.walk(root):
    dirs[:] = [d for d in dirs if not d.startswith('.')]
    for f in files:
        if f.startswith('.'): 
            continue
        p=os.path.join(dirpath,f)
        if not os.path.isfile(p):
            continue
        try:
            text=open(p,'r',encoding='utf-8',errors='ignore').read()
        except Exception as e:
            print(f"SKIP {p}: {e}")
            continue
        if 'jira' in text.lower() or 'task-tracking' in text:
            print(p)
PY

echo "== dbt package/source references =="
python3 - <<'PY'
import os, re
root='src/ingestion/dbt'
pattern=re.compile(r'connectors/task-tracking/jira/dbt|/connectors/task-tracking/jira/dbt|task-tracking/jira/dbt', re.I)
for dirpath, dirs, files in os.walk(root):
    dirs[:] = [d for d in dirs if not d.startswith('.')]
    for f in files:
        if f.startswith('.'): continue
        p=os.path.join(dirpath,f)
        try:
            s=open(p,'rb').readline(1000)
        except Exception:
            continue
        if b'connections' in s.lower() or b'jira' in s.lower() or b'task-tracking' in s.lower() or b'model' in s.lower():
            try:
                txt=open(p,'r',encoding='utf-8',errors='ignore').read()
            except Exception:
                continue
            if pattern.search(txt):
                for i,line in enumerate(txt.splitlines(),1):
                    if pattern.search(line):
                        print(f"{p}:{i}:{line}")
PY

Repository: constructorfabric/insight

Length of output: 11111


🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "== all files under src/ingestion/dbt that reference task-tracking/jira/dbt or jira models =="
python3 - <<'PY'
import os, re

def ok(path):
    root = 'src/ingestion/dbt'
    return path.startswith(root + os.sep) and not any(part.startswith(('.', 'node_modules', 'target', '.venv', '__pycache__')) for part in path.split(os.sep))

root = 'src/ingestion/dbt'
patterns = [
    re.compile(r'connectors/task-tracking/jira/dbt'),
    re.compile(r'/connectors/task-tracking/jira/dbt'),
    re.compile(r'jira__', re.I),
    re.compile(r'task-tracking/jira/dbt', re.I),
]
matches = {}
for dirpath, dirs, files in os.walk(root):
    dirs[:] = [d for d in dirs if not d.startswith('.') and d not in ('node_modules', 'target', '.venv', '__pycache__')]
    for f in files:
        if f.startswith('.'):
            continue
        path = os.path.join(dirpath, f)
        try:
            text = open(path, 'r', encoding='utf-8', errors='ignore').read()
        except Exception as e:
            print(f"SKIP {path}: {e}")
            continue
        for p in patterns:
            if not p.search(text):
                continue
            matches.setdefault(path, []).append(p.pattern)
            break

for path in sorted(matches):
    print(f"== {path} ==")
    lines = open(path, 'r', encoding='utf-8', errors='ignore').readlines()
    hits = [(i+1, line.rstrip('\n')) for i, line in enumerate(lines) if any(re.search(p, line) for p in matches[path])]
    for line_no, line in hits[:40]:
        print(f"{line_no}: {line[:300]}")
PY

echo "== files under entire repo that reference Jira path pattern =="
rg -n 'connectors/task-tracking/jira/dbt|/connectors/task-tracking/jira/dbt' -g '!target/**' -g '!node_modules/**' -g '!dist/**' -g '!build/**' . || true

echo "== test after current snippet from src/ingestion/tests/e2e/meta/test_dbt_runner.py =="
sed -n '120,150p' src/ingestion/tests/e2e/meta/test_dbt_runner.py | cat -n

Repository: constructorfabric/insight

Length of output: 3968


🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "== lock/package references to Jira dbt package =="
rg -n 'task-tracking/jira/dbt|jira/dbt|connectors/task-tracking/jira/dbt|src/ingestion/connectors/task-tracking/jira/dbt' . \
  --glob '!target/**' --glob '!node_modules/**' --glob '!dist/**' --glob '!build/**' --glob '!*.lock' --glob '!*.json' \
| sort -u || true

echo "== dbt config/package files =="
git ls-files | rg '(^|/)(dbt_project\.yml|packages\.yml|package-lock\.json|packages-config\.yml|dbt_project_exclude|packages\.yml)$' | sed -n '1,200p'

echo "== src/ingestion/connectors task-tracking entries =="
find src/ingestion/connectors -maxdepth 3 -type f -o -type d | sort | sed -n '1,200p'

Repository: constructorfabric/insight

Length of output: 10010


Avoid the leading slash in the Jira-model check.

original_file_path is dbt-project-relative for this package, so paths under the Jira connector are connectors/task-tracking/jira/dbt/.... Keeping / makes the filter miss Jira models entirely.

Proposed fix
-            "/connectors/task-tracking/jira/dbt/" in node.get("original_file_path", "")
+            "connectors/task-tracking/jira/dbt/" in node.get("original_file_path", "")
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
def _is_jira_model(node: dict) -> bool:
return node.get("resource_type") == "model" and (
"/connectors/task-tracking/jira/dbt/" in node.get("original_file_path", "")
)
def _is_jira_model(node: dict) -> bool:
return node.get("resource_type") == "model" and (
"connectors/task-tracking/jira/dbt/" in node.get("original_file_path", "")
)
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/ingestion/tests/e2e/meta/test_dbt_runner.py` around lines 141 - 144,
Update the path substring checked by _is_jira_model to remove the leading slash,
matching dbt-project-relative paths under connectors/task-tracking/jira/dbt/
while preserving the existing resource_type model check.


gaps = sorted(
node["name"]
for uid, node in nodes.items()
if _is_jira_model(node) and node.get("config", {}).get("materialized") != "ephemeral" and uid not in covered
)

assert not gaps, (
"These non-ephemeral jira models are built by no prod pass "
"(tag:staging,tag:jira / tag:silver,tag:jira+ / tag:gold,tag:jira+), so on "
f"a real sync they are never materialized and their silver consumers fail: {gaps}. "
"Add the 'staging' tag (like jira__issue_field_snapshot) so the staging pass "
"builds them before enrich/silver (issue #1893)."
)


def test_dbt_build_unknown_selector_raises(dbt_runner: DbtRunner) -> None:
"""A selector that matches no models surfaces a clear DbtError."""
# `dbt build --select <nonsense>` is NOT an error in dbt — it just runs
Expand Down
Loading