Dbt subdirectory (#36)

Reading manifest file through dbt project

Author: mcnick

CHANGELOG.md

@@ -15,8 +15,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+### Breaking Changes
+* Path to dbt project.yml file is provided instead of manifest.json
+
 ### Fixed
 * Fixed generation of CTE names from references with hyphens
+* Fixed query paths if dbt project is in subdirectory
 
 ## [0.5.0]
 

docs/dbt.md

@@ -11,14 +11,14 @@ This guide will provide a quick start on how to use SQLMock with dbt (data build
 
 ## Configuration
 
-### Setting the dbt Manifest Path
+### Setting the dbt Project Path
 
-Initialize your testing environment by setting the global path to your dbt manifest file:
+Initialize your testing environment by setting the global path to your dbt project file:
 
 ```python
 from sql_mock.config import SQLMockConfig
 
-SQLMockConfig.set_dbt_manifest_path('/path/to/your/dbt/manifest.json')
+SQLMockConfig.set_dbt_project_path('/path/to/your/dbt_project.yml')
 ```
 
 ## Creating Table Mocks

examples/dbt/test_example.py

@@ -3,7 +3,7 @@
 from sql_mock.config import SQLMockConfig
 from sql_mock.dbt import dbt_model_meta, dbt_seed_meta, dbt_source_meta
 
-SQLMockConfig.set_dbt_manifest_path("./tests/resources/dbt/dbt_manifest.json")
+SQLMockConfig.set_dbt_project_path("./tests/resources/dbt/dbt_project.yml")
 
 
 # NOTE: The Source and Seed classes will not be used in the example test. They are only here for demonstration purpose.

poetry.lock

@@ -41,6 +41,7 @@ python = "^3.9"
 jinja2 = "^3.1.2"
 pydantic-settings = "^2.0.3"
 sqlglot = "^20.5.0"
+pyyaml = "^6.0.1"
 
 # Clickhouse specific
 clickhouse-driver = {extras = ["numpy"], version = "^0.2.6", optional = true}

pyproject.toml

@@ -1,12 +1,12 @@
 class SQLMockConfig:
-    _dbt_manifest_path = None
+    _dbt_project_path = None
 
     @classmethod
-    def set_dbt_manifest_path(cls, path: str):
-        cls._dbt_manifest_path = path
+    def set_dbt_project_path(cls, path: str):
+        cls._dbt_project_path = path
 
     @classmethod
-    def get_dbt_manifest_path(cls):
-        if cls._dbt_manifest_path is None:
-            raise ValueError("DBT manifest path is not set. Please set it using set_dbt_manifest_path()")
-        return cls._dbt_manifest_path
+    def get_dbt_project_path(cls):
+        if cls._dbt_project_path is None:
+            raise ValueError("DBT project path is not set. Please set it using set_dbt_project_path()")
+        return cls._dbt_project_path

src/sql_mock/config.py

@@ -1,6 +1,9 @@
 import json
+import os
 from typing import TYPE_CHECKING
 
+import yaml
+
 from sql_mock.config import SQLMockConfig
 from sql_mock.helpers import parse_table_refs, validate_input_mocks
 from sql_mock.table_mocks import TableMockMeta
@@ -10,44 +13,53 @@
     from sql_mock.table_mocks import BaseTableMock
 
 
-def _get_model_metadata_from_dbt_manifest(manifest_path: str, model_name: str) -> dict:
+def _get_manifest_from_project_file(project_path: str) -> dict:
+    with open(project_path, "r") as f:
+        dbt_project = yaml.safe_load(f)
+    target_path = dbt_project["target-path"]
+    project_dir = os.path.dirname(project_path)
+    with open(os.path.join(project_dir, target_path, "manifest.json"), "r") as file:
+        manifest = json.load(file)
+    return manifest
+
+
+def _get_model_metadata(project_path: str, model_name: str) -> dict:
     """
     Extracts the rendered SQL query for a specified model from the dbt manifest file.
 
     Args:
-        manifest_path (str): Path to the dbt manifest.json file.
+        project_path (str): Path to the dbt_project.yml file.
         model_name (str): Name of the dbt model.
 
     Returns:
         dict: Dictionary of metadata from dbt (path to compiled sql query and table ref)
     """
-    with open(manifest_path, "r") as file:
-        manifest = json.load(file)
+    manifest = _get_manifest_from_project_file(project_path)
+    project_dir = os.path.dirname(project_path)
 
     for node in manifest["nodes"].values():
         if node["resource_type"] == "model" and node["name"] == model_name:
             return {
-                "query_path": node["compiled_path"],
+                "query_path": os.path.join(project_dir, node["compiled_path"]),
                 "table_ref": node["relation_name"],
             }
 
     raise ValueError(f"Model '{model_name}' not found in dbt manifest.")
 
 
-def _get_source_metadata_from_dbt_manifest(manifest_path: str, source_name: str, table_name: str) -> dict:
+def _get_source_metadata(project_path: str, source_name: str, table_name: str) -> dict:
     """
     Extracts the table metadata for dbt source from the manifest file.
 
     Args:
-        manifest_path (str): Path to the dbt manifest.json file.
+        project_path (str): Path to the dbt_project.yml file.
         source_name (str): Name of the dbt source.
         table_name (str): Name of the table in the dbt source.
 
     Returns:
         dict: Dictionary of metadata from dbt
     """
-    with open(manifest_path, "r") as file:
-        manifest = json.load(file)
+    manifest = _get_manifest_from_project_file(project_path)
 
     for node in manifest["sources"].values():
         if (
@@ -62,19 +74,18 @@ def _get_source_metadata_from_dbt_manifest(manifest_path: str, source_name: str,
     raise ValueError(f"Source '{source_name}' not found in dbt manifest.")
 
 
-def _get_seed_metadata_from_dbt_manifest(manifest_path: str, seed_name: str) -> dict:
+def _get_seed_metadata(project_path: str, seed_name: str) -> dict:
     """
     Extracts the table metadata for dbt seed from the manifest file.
 
     Args:
-        manifest_path (str): Path to the dbt manifest.json file.
+        project_path (str): Path to the dbt_project.yml file.
         seed_name (str): Name of the dbt seed.
 
     Returns:
         dict: Dictionary of metadata from dbt
     """
-    with open(manifest_path, "r") as file:
-        manifest = json.load(file)
+    manifest = _get_manifest_from_project_file(project_path)
 
     for node in manifest["nodes"].values():
         if node["resource_type"] == "seed" and node["name"] == seed_name:
@@ -85,20 +96,20 @@ def _get_seed_metadata_from_dbt_manifest(manifest_path: str, seed_name: str) ->
     raise ValueError(f"Seed '{seed_name}' not found in dbt manifest.")
 
 
-def dbt_model_meta(model_name: str, manifest_path: str = None, default_inputs: ["BaseTableMock"] = None):
+def dbt_model_meta(model_name: str, project_path: str = None, default_inputs: ["BaseTableMock"] = None):
     """
     Decorator that is used to define TableMock metadata for dbt models.
 
     Args:
         model_name (string) : Name of the dbt model
-        manifest_path (string): Path to the dbt manifest file
+        project_path (string): Path to the dbt manifest file
         default_inputs: List of default input mock instances that serve as default input if no other instance of that class is provided.
     """
 
     def decorator(cls):
-        path = manifest_path or SQLMockConfig.get_dbt_manifest_path()
+        path = project_path or SQLMockConfig.get_dbt_project_path()
 
-        dbt_meta = _get_model_metadata_from_dbt_manifest(manifest_path=path, model_name=model_name)
+        dbt_meta = _get_model_metadata(project_path=path, model_name=model_name)
 
         parsed_query = ""
         with open(dbt_meta["query_path"]) as f:
@@ -118,23 +129,23 @@ def decorator(cls):
 
 
 def dbt_source_meta(
-    source_name: str, table_name: str, manifest_path: str = None, default_inputs: ["BaseTableMock"] = None
+    source_name: str, table_name: str, project_path: str = None, default_inputs: ["BaseTableMock"] = None
 ):
     """
     Decorator that is used to define TableMock metadata for dbt sources.
 
     Args:
         source_name (string) : Name of source
         table_name (string): Name of the table in the source
-        manifest_path (string): Path to the dbt manifest file
+        project_path (string): Path to the dbt manifest file
         default_inputs: List of default input mock instances that serve as default input if no other instance of that class is provided.
     """
 
     def decorator(cls):
-        path = manifest_path or SQLMockConfig.get_dbt_manifest_path()
+        path = project_path or SQLMockConfig.get_dbt_project_path()
 
-        dbt_meta = _get_source_metadata_from_dbt_manifest(
-            manifest_path=path, source_name=source_name, table_name=table_name
+        dbt_meta = _get_source_metadata(
+            project_path=path, source_name=source_name, table_name=table_name
         )
 
         if default_inputs:
@@ -149,21 +160,21 @@ def decorator(cls):
     return decorator
 
 
-def dbt_seed_meta(seed_name: str, manifest_path: str = None, default_inputs: ["BaseTableMock"] = None):
+def dbt_seed_meta(seed_name: str, project_path: str = None, default_inputs: ["BaseTableMock"] = None):
     """
     Decorator that is used to define TableMock metadata for dbt sources.
 
     Args:
         seed_name (string) : Name of the dbt seed
-        manifest_path (string): Path to the dbt manifest file
+        project_path (string): Path to the dbt manifest file
         default_inputs: List of default input mock instances that serve as default input if no other instance of that class is provided.
     """
 
     def decorator(cls):
-        path = manifest_path or SQLMockConfig.get_dbt_manifest_path()
+        path = project_path or SQLMockConfig.get_dbt_project_path()
 
-        dbt_meta = _get_seed_metadata_from_dbt_manifest(
-            manifest_path=path,
+        dbt_meta = _get_seed_metadata(
+            project_path=path,
             seed_name=seed_name,
         )
 

src/sql_mock/dbt.py

@@ -0,0 +1 @@
+target-path: "dbt_target"