From b1603c2da483068e96de82ca348eb28a551ec56c Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Tue, 6 Jan 2026 13:44:51 -0500 Subject: [PATCH 01/24] Port implementation without tests or optimizations --- cosmos/config.py | 30 +++ cosmos/dbt/graph.py | 41 ++- cosmos/dbt/selector.py | 233 ++++++++++++++++++ .../cosmos_manifest_yaml_selectors_example.py | 121 +++++++++ dev/dags/dbt/jaffle_shop/selectors.yml | 50 ++++ 5 files changed, 462 insertions(+), 13 deletions(-) create mode 100644 dev/dags/cosmos_manifest_yaml_selectors_example.py create mode 100644 dev/dags/dbt/jaffle_shop/selectors.yml diff --git a/cosmos/config.py b/cosmos/config.py index de07e59250..5f03c71bae 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -169,6 +169,8 @@ class ProjectConfig: :param snapshots_relative_path: The relative path to the dbt snapshots directory within the project. Defaults to snapshots :param manifest_path: The absolute path to the dbt manifest file. Defaults to None :param manifest_conn_id: Name of the Airflow connection used to access the manifest file if it is not stored locally. Defaults to None + :param selectors_path: The absolute path to the dbt selectors file. Defaults to None + :param selectors_conn_id: Name of the Airflow connection used to access the selectors file if it is not stored locally. Defaults to None :param project_name: Allows the user to define the project name. Required if dbt_project_path is not defined. Defaults to the folder name of dbt_project_path. :param env_vars: Dictionary of environment variables that are used for both rendering and execution. Rendering with @@ -185,6 +187,7 @@ class ProjectConfig: install_dbt_deps: bool = True copy_dbt_packages: bool = settings.default_copy_dbt_packages manifest_path: Path | ObjectStoragePath | None = None + selectors_path: Path | ObjectStoragePath | None = None models_path: Path | None = None seeds_path: Path | None = None snapshots_path: Path | None = None @@ -200,6 +203,8 @@ def __init__( snapshots_relative_path: str | Path = "snapshots", manifest_path: str | Path | None = None, manifest_conn_id: str | None = None, + selectors_path: str | Path | None = None, + selectors_conn_id: str | None = None, project_name: str | None = None, env_vars: dict[str, str] | None = None, dbt_vars: dict[str, str] | None = None, @@ -243,6 +248,25 @@ def __init__( else: self.manifest_path = Path(manifest_path_str) + if selectors_path: + selectors_path_str = str(selectors_path) + if not selectors_conn_id: + selectors_scheme = selectors_path_str.split("://")[0] + # Use the default Airflow connection ID for the scheme if it is not provided. + selectors_conn_id = FILE_SCHEME_AIRFLOW_DEFAULT_CONN_ID_MAP.get(selectors_scheme, lambda: None)() + + if selectors_conn_id is not None and not settings.AIRFLOW_IO_AVAILABLE: + raise CosmosValueError( + f"The selectors path {selectors_path_str} uses a remote file scheme, but the required Object " + f"Storage feature is unavailable in Airflow version {airflow_version}. Please upgrade to " + f"Airflow 2.8 or later." + ) + + if settings.AIRFLOW_IO_AVAILABLE: + self.selectors_path = ObjectStoragePath(selectors_path_str, conn_id=selectors_conn_id) + else: + self.selectors_path = Path(selectors_path_str) + self.env_vars = env_vars self.dbt_vars = dbt_vars self.partial_parse = partial_parse @@ -287,6 +311,12 @@ def is_manifest_available(self) -> bool: """ return self.manifest_path.exists() if self.manifest_path else False + def is_selectors_available(self) -> bool: + """ + Check if the `dbt` selectors file is set and if the file exists. + """ + return self.selectors_path.exists() if self.selectors_path else False + @dataclass class ProfileConfig: diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index 7df33d9338..741a7204fb 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -9,6 +9,7 @@ import platform import tempfile import warnings +import yaml import zlib from dataclasses import dataclass, field from functools import cached_property @@ -57,7 +58,7 @@ get_partial_parse_path, has_non_empty_dependencies_file, ) -from cosmos.dbt.selector import select_nodes +from cosmos.dbt.selector import select_nodes, parse_yaml_selectors from cosmos.log import get_logger logger = get_logger(__name__) @@ -920,10 +921,8 @@ def load_from_dbt_manifest(self) -> None: self.load_method = LoadMode.DBT_MANIFEST logger.info("Trying to parse the dbt project `%s` using a dbt manifest...", self.project.project_name) - if self.render_config.selector: - raise CosmosLoadDbtException( - "RenderConfig.selector is not yet supported when loading dbt projects using the LoadMode.DBT_MANIFEST parser." - ) + if self.render_config.selector and not self.project.is_selectors_available: + raise CosmosLoadDbtException(f"Unable to load selectors using {self.project.selectors_path}") if not self.project.is_manifest_available(): raise CosmosLoadDbtException(f"Unable to load manifest using {self.project.manifest_path}") @@ -957,14 +956,30 @@ def load_from_dbt_manifest(self) -> None: ) nodes[node.unique_id] = node - - self.nodes = nodes - self.filtered_nodes = select_nodes( - project_dir=self.execution_config.project_path, - nodes=nodes, - select=self.render_config.select, - exclude=self.render_config.exclude, - ) + + if self.render_config.selector: + if TYPE_CHECKING: + assert self.project.selectors_path is not None # pragma: no cover + + with self.project.selectors_path.open() as fp2: + selectors = yaml.safe_load(fp2) + selections = parse_yaml_selectors(selectors)[self.render_config.selector] + + self.nodes = nodes + self.filtered_nodes = select_nodes( + project_dir=self.execution_config.project_path, + nodes=nodes, + select=selections["select"], + exclude=selections["exclude"], + ) + else: + self.nodes = nodes + self.filtered_nodes = select_nodes( + project_dir=self.execution_config.project_path, + nodes=nodes, + select=self.render_config.select, + exclude=self.render_config.exclude, + ) def update_node_dependency(self) -> None: """ diff --git a/cosmos/dbt/selector.py b/cosmos/dbt/selector.py index 7dcef3c066..6b58b11414 100644 --- a/cosmos/dbt/selector.py +++ b/cosmos/dbt/selector.py @@ -682,6 +682,239 @@ def retrieve_by_label(statement_list: list[str], label: str) -> set[str]: return label_values +def _parse_selector_for_selections( + selector_definition: dict[str, Any], + all_selector_definitions: dict[str, dict[str, Any]], + resolved_cache: dict[str, Any], + visiting: set[str] | None = None, +): + """ + Convert a selector definition from selectors.yml to dbt command-line syntax. + + This function recursively processes selector definitions and converts them into + dbt-compatible select/exclude criteria. + + Args: + selector_definition (dict): The selector definition to convert. + all_selector_definitions (dict): A mapping of all selector names to their definitions. + resolved_cache (dict): Cache for already resolved selectors to avoid redundant processing. + visiting (set, optional): Set of currently visiting selector names to detect cycles. + Returns: + dict: A dictionary with "select" and "exclude" keys containing lists of dbt selection strings. + """ + resolved_cache = resolved_cache or {} + visiting = visiting or set() + + def resolve_selector_reference(selector_name: str) -> tuple: + """ + Recursively resolve a selector reference to its base components. + + Returns: + Tuple of (select_parts, exclude_parts) + """ + # Check cache first + if selector_name in resolved_cache: + cached = resolved_cache[selector_name] + return (cached.get("select") or [], cached.get("exclude") or []) + + # Check for circular reference + if selector_name in visiting: + raise CosmosValueError(f"Circular reference detected in selector '{selector_name}'") + + # Get the selector definition + if selector_name not in all_selector_definitions: + raise CosmosValueError(f"Selector '{selector_name}' not found") + + # Mark as visiting + visiting.add(selector_name) + + # Recursively resolve the referenced selector + selector_def = all_selector_definitions[selector_name] + result = _parse_selector_for_selections(selector_def, all_selector_definitions, resolved_cache, visiting) + + # Remove from visiting + visiting.discard(selector_name) + + # Cache the result + resolved_cache[selector_name] = result + + return (result.get("select") or [], result.get("exclude") or []) + + def process_method(method_def: dict[str, Any]) -> tuple: + """ + Process a single method definition. + Returns tuple of (select_parts, exclude_parts) + """ + method = method_def.get("method") + value = method_def.get("value") + parents = method_def.get("parents", False) + children = method_def.get("children", False) + parents_depth = method_def.get("parents_depth", 0) + children_depth = method_def.get("children_depth", 0) + childrens_parents = method_def.get("childrens_parents", False) + + # Build the selector string + if method.startswith(TAG_SELECTOR[:-1]): + selector_str = f"{TAG_SELECTOR}{value}" + elif method.startswith(PATH_SELECTOR[:-1]): + selector_str = f"{PATH_SELECTOR}{value}" + elif method.startswith(SOURCE_SELECTOR[:-1]): + selector_str = f"{SOURCE_SELECTOR}{value}" + elif method.startswith(EXPOSURE_SELECTOR[:-1]): + selector_str = f"{EXPOSURE_SELECTOR}{value}" + elif method.startswith(RESOURCE_TYPE_SELECTOR[:-1]): + selector_str = f"{RESOURCE_TYPE_SELECTOR}{value}" + elif method.startswith(EXCLUDE_RESOURCE_TYPE_SELECTOR[:-1]): + selector_str = f"{EXCLUDE_RESOURCE_TYPE_SELECTOR}{value}" + elif any([method.startswith(CONFIG_SELECTOR + config) for config in SUPPORTED_CONFIG]): + selector_str = f"{CONFIG_SELECTOR}:{value}" + elif method == "fqn": + if value == "*": + return ([], []) + selector_str = value + elif method == "selector": + # Recursively resolve the selector reference + return resolve_selector_reference(value) + else: + raise CosmosValueError(f"Unsupported selector method: '{method}'") + + # Add modifiers + if parents: + if parents_depth and parents_depth > 0: + selector_str = f"{parents_depth}+{selector_str}" + else: + selector_str = f"+{selector_str}" + if children: + if children_depth and children_depth > 0: + selector_str = f"{selector_str}+{children_depth}" + else: + selector_str = f"{selector_str}+" + if childrens_parents: + if parents_depth or children_depth: + raise CosmosValueError("childrens_parents cannot be combined with parents_depth or children_depth.") + else: + selector_str = f"@{selector_str}" + + return ([selector_str], []) + + def process_definition(definition: dict[str, Any]) -> tuple: + """ + Process a selector definition recursively. + + Handles union, intersection, exclude, and method definitions. + Does not handle CLI-style or key-value definitions. + Raises any parse errors encountered during processing. + + Args: + definition: Selector definition dictionary + + Returns: + Tuple of (select_parts, exclude_parts) + """ + select_parts = [] + exclude_parts = [] + + # Reject CLI-style string definitions (e.g., definition: "tag:my_tag") + # These should be structured as method-value definitions instead + if isinstance(definition, str): + raise CosmosValueError(f"CLI-style string definitions are not supported: '{definition}'. Use method-value definitions instead.") + + # Reject key-value definitions (e.g., tag: my_tag, path: models/, etc.) + # These are deprecated in favor of explicit method definitions + # Valid structural keys are: method, union, intersection, exclude, and method modifiers + supported_definitions = {"method", "union", "intersection", "exclude", "value", "parents", "children", "parents_depth", "children_depth", "childrens_parents"} + key_value_definitions = set(definition.keys()) - supported_definitions + + if key_value_definitions: + key_value_strings = ", ".join(f"{k}: {definition[k]}" for k in key_value_definitions) + raise CosmosValueError(f"Key-value definitions are not supported: {key_value_strings}. Use method-value definitions instead.") + + # Handle method + value + if "method" in definition: + method_select, method_exclude = process_method(definition) + select_parts.extend(method_select) + exclude_parts.extend(method_exclude) + + # Handle exclude + if "exclude" in definition: + exclude_items = definition["exclude"] + for exclude_item in exclude_items: + ex_select, ex_exclude = process_definition(exclude_item) + # Items in exclude list should go to exclude_parts + exclude_parts.extend(ex_select) + exclude_parts.extend(ex_exclude) + + # Handle union + if "union" in definition: + union_items = definition["union"] + + for item in union_items: + if isinstance(item, dict): + item_select, item_exclude = process_definition(item) + select_parts.extend(item_select) + exclude_parts.extend(item_exclude) + + # Handle intersection + if "intersection" in definition: + intersection_items = definition["intersection"] + intersection_selects = [] + for item in intersection_items: + item_select, item_exclude = process_definition(item) + intersection_selects.extend(item_select) + exclude_parts.extend(item_exclude) + + if intersection_selects: + # Join with commas for intersection + select_parts = [",".join(intersection_selects)] + + return (select_parts, exclude_parts) + + select_parts, exclude_parts = process_definition(selector_definition) + + # Remove None values, empty strings, and duplicates while preserving order + select_parts = list(dict.fromkeys([s for s in select_parts if s])) + exclude_parts = list(dict.fromkeys([e for e in exclude_parts if e])) + + result = { + "select": select_parts if select_parts else None, + "exclude": exclude_parts if exclude_parts else None, + } + + return result + +# TODO: Cache so its only calculated once (invalidate on file change) +def parse_yaml_selectors(selectors: dict[str, list[str]]) -> dict[str, dict[str, Any]]: + """ + Parse dbt YAML selectors into a dictionary of dbt command-line syntax. + + Args: + selectors (dict): The selectors dictionary loaded from selectors.yml. + Returns: + dict: A dictionary mapping selector names to their dbt selection syntax. + + """ + selectors_root = selectors.get("selectors", []) + + # Build a map of all selector definitions for reference resolution + all_selectors = {} + for selector in selectors_root: + name = selector.get("name", "") + definition = selector.get("definition", {}) + all_selectors[name] = definition + + # Build result dictionary with resolved selectors + result = {} + resolved_cache = {} + + for selector in selectors_root: + name = selector.get("name", "") + definition = selector.get("definition", {}) + + dbt_syntax = _parse_selector_for_selections(definition, all_selectors, resolved_cache) + result[name] = dbt_syntax + + return result + def select_nodes( project_dir: Path | None, diff --git a/dev/dags/cosmos_manifest_yaml_selectors_example.py b/dev/dags/cosmos_manifest_yaml_selectors_example.py new file mode 100644 index 0000000000..7180063350 --- /dev/null +++ b/dev/dags/cosmos_manifest_yaml_selectors_example.py @@ -0,0 +1,121 @@ +""" +An example DAG that uses Cosmos to render a dbt project into Airflow using a dbt manifest file with YAML selectors. +""" + +import os +from datetime import datetime +from pathlib import Path + +from airflow import DAG + +try: + from airflow.providers.standard.operators.empty import EmptyOperator +except ImportError: + from airflow.operators.empty import EmptyOperator + +from cosmos import DbtTaskGroup, ExecutionConfig, LoadMode, ProfileConfig, ProjectConfig, RenderConfig +from cosmos.profiles import DbtProfileConfigVars, PostgresUserPasswordProfileMapping + +DEFAULT_DBT_ROOT_PATH = Path(__file__).parent / "dbt" +DBT_ROOT_PATH = Path(os.getenv("DBT_ROOT_PATH", DEFAULT_DBT_ROOT_PATH)) + + +execution_config = ExecutionConfig(dbt_project_path=DBT_ROOT_PATH / "jaffle_shop") + +profile_config = ProfileConfig( + profile_name="default", + target_name="dev", + profile_mapping=PostgresUserPasswordProfileMapping( + conn_id="example_conn", + profile_args={"schema": "public"}, + dbt_config_vars=DbtProfileConfigVars(send_anonymous_usage_stats=True), + ), +) + +render_config = RenderConfig(load_method=LoadMode.DBT_MANIFEST, selector="critical_path") + + +with DAG( + dag_id="cosmos_manifest_yaml_selectors_example", + schedule="@daily", + start_date=datetime(2023, 1, 1), + catchup=False, + default_args={"retries": 0}, +): + pre_dbt = EmptyOperator(task_id="pre_dbt") + + # [START local_example] + local_example = DbtTaskGroup( + group_id="local_example", + project_config=ProjectConfig( + manifest_path=DBT_ROOT_PATH / "jaffle_shop" / "target" / "manifest.json", + selectors_path=DBT_ROOT_PATH / "jaffle_shop" / "selectors.yml", + project_name="jaffle_shop", + ), + profile_config=profile_config, + render_config=render_config, + execution_config=execution_config, + operator_args={"install_deps": True}, + ) + # [END local_example] + + # [START aws_s3_example] + aws_s3_example = DbtTaskGroup( + group_id="aws_s3_example", + project_config=ProjectConfig( + manifest_path="s3://cosmos-manifest-test/manifest.json", + manifest_conn_id="aws_s3_conn", + # `manifest_conn_id` is optional. If not provided, the default connection ID `aws_default` is used. + selectors_path="s3://cosmos-manifest-test/selectors.yml", + selectors_conn_id="aws_s3_conn", + # `selectors_conn_id` is optional. If not provided, the default connection ID ` + project_name="jaffle_shop", + ), + profile_config=profile_config, + render_config=render_config, + execution_config=execution_config, + operator_args={"install_deps": True}, + ) + # [END aws_s3_example] + + # [START gcp_gs_example] + gcp_gs_example = DbtTaskGroup( + group_id="gcp_gs_example", + project_config=ProjectConfig( + manifest_path="gs://cosmos_remote_target/manifest.json", + manifest_conn_id="gcp_gs_conn", + # `manifest_conn_id` is optional. If not provided, the default connection ID `google_cloud_default` is used. + selectors_path="gs://cosmos_remote_target/selectors.yml", + selectors_conn_id="gcp_gs_conn", + # `selectors_conn_id` is optional. If not provided, the default connection ID `google_cloud_default` is used. + project_name="jaffle_shop", + ), + profile_config=profile_config, + render_config=render_config, + execution_config=execution_config, + operator_args={"install_deps": True}, + ) + # [END gcp_gs_example] + + # [START azure_abfs_example] + azure_abfs_example = DbtTaskGroup( + group_id="azure_abfs_example", + project_config=ProjectConfig( + manifest_path="abfs://cosmos-manifest-test/manifest.json", + manifest_conn_id="azure_abfs_conn", + # `manifest_conn_id` is optional. If not provided, the default connection ID `wasb_default` is used. + selectors_path="abfs://cosmos-manifest-test/selectors.yml", + selectors_conn_id="azure_abfs_conn", + # `selectors_conn_id` is optional. If not provided, the default connection ID `wasb_default` is used. + project_name="jaffle_shop", + ), + profile_config=profile_config, + render_config=render_config, + execution_config=execution_config, + operator_args={"install_deps": True}, + ) + # [END azure_abfs_example] + + post_dbt = EmptyOperator(task_id="post_dbt") + + (pre_dbt >> local_example >> aws_s3_example >> gcp_gs_example >> azure_abfs_example >> post_dbt) diff --git a/dev/dags/dbt/jaffle_shop/selectors.yml b/dev/dags/dbt/jaffle_shop/selectors.yml new file mode 100644 index 0000000000..d442b030e2 --- /dev/null +++ b/dev/dags/dbt/jaffle_shop/selectors.yml @@ -0,0 +1,50 @@ +selectors: + - name: staging_models + description: "Select all staging models" + definition: + method: path + value: models/staging + + - name: core_models + description: "Select core business logic models (non-staging)" + definition: + method: tag + value: core + + - name: customers_and_downstream + description: "Select customers model and all downstream dependencies" + definition: + method: fqn + value: customers + childrens_parents: true + + - name: staging_orders + description: "Select staging orders model specifically" + definition: + method: fqn + value: jaffle_shop.staging.stg_orders + + - name: nightly_models + description: "Models tagged for nightly runs" + definition: + method: tag + value: nightly + + - name: exclude_staging + description: "All models except staging" + definition: + union: + - method: fqn + value: "*" + - exclude: + - method: path + value: models/staging + + - name: critical_path + description: "Critical business models" + definition: + union: + - method: fqn + value: customers + - method: fqn + value: orders From f1c09927a3e9358c24864edc801fca62a827c2ca Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Tue, 6 Jan 2026 15:30:18 -0500 Subject: [PATCH 02/24] Implement caching for parsed yaml selectors --- cosmos/cache.py | 33 +++- cosmos/config.py | 32 +-- cosmos/dbt/graph.py | 186 ++++++++++++++++-- cosmos/dbt/selector.py | 86 ++++---- cosmos/settings.py | 1 + .../cosmos_manifest_yaml_selectors_example.py | 20 +- 6 files changed, 277 insertions(+), 81 deletions(-) diff --git a/cosmos/cache.py b/cosmos/cache.py index 096a7fa489..85cf85e65f 100644 --- a/cosmos/cache.py +++ b/cosmos/cache.py @@ -149,8 +149,8 @@ def _create_cache_identifier(dag: DAG, task_group: TaskGroup | None) -> str: return "__".join(cache_identifiers_list) -def create_cache_key(cache_identifier: str) -> str: - return f"{VAR_KEY_CACHE_PREFIX}{cache_identifier}" +def create_cache_key(cache_identifier: str, cache_suffix: str) -> str: + return f"{VAR_KEY_CACHE_PREFIX}{cache_identifier}_{cache_suffix}" def _obtain_cache_dir_path(cache_identifier: str, base_dir: Path = settings.cache_dir) -> Path: @@ -287,6 +287,35 @@ def _copy_partial_parse_to_project(partial_parse_filepath: Path, project_path: P shutil.copy(str(source_manifest_filepath), str(target_manifest_filepath)) +def _calculate_selectors_yaml_cache_current_version( + cache_identifier: str, project_dir: Path, selectors_yaml_path: Path +) -> str: + """ + Taking into account the project directory contents and the selectors yaml file, calculate the + hash that represents the "dbt selectors yaml" version - to be used to decide if the cache should be refreshed or not. + + :param cache_identifier: Unique identifier of the cache (may include DbtDag or DbtTaskGroup information) + :param project_path: Path to the target dbt project directory + :param selectors_yaml_path: Path to the selectors yaml file + """ + + start_time = time.perf_counter() + + # Combined value for when the dbt project directory files were last modified + # This is fast (e.g. 0.01s for jaffle shop, 0.135s for a 5k models dbt folder) + dbt_project_hash = _create_folder_version_hash(project_dir) + + with selectors_yaml_path.open("r") as f: + selectors_yaml_content = yaml.safe_load(f) + hash_selectors_yaml = hashlib.md5(yaml.dump(selectors_yaml_content).encode()).hexdigest() + + elapsed_time = time.perf_counter() - start_time + logger.info( + f"Cosmos performance: time to calculate cache identifier {cache_identifier} for current version: {elapsed_time}" + ) + return f"{dbt_project_hash},{hash_selectors_yaml}" + + def _calculate_dbt_ls_cache_current_version(cache_identifier: str, project_dir: Path, cmd_args: list[str]) -> str: """ Taking into account the project directory contents and the command arguments, calculate the diff --git a/cosmos/config.py b/cosmos/config.py index 5f03c71bae..53ee64b79f 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -187,7 +187,7 @@ class ProjectConfig: install_dbt_deps: bool = True copy_dbt_packages: bool = settings.default_copy_dbt_packages manifest_path: Path | ObjectStoragePath | None = None - selectors_path: Path | ObjectStoragePath | None = None + selectors_yaml_path: Path | ObjectStoragePath | None = None models_path: Path | None = None seeds_path: Path | None = None snapshots_path: Path | None = None @@ -203,8 +203,8 @@ def __init__( snapshots_relative_path: str | Path = "snapshots", manifest_path: str | Path | None = None, manifest_conn_id: str | None = None, - selectors_path: str | Path | None = None, - selectors_conn_id: str | None = None, + selectors_yaml_path: str | Path | None = None, + selectors_yaml_conn_id: str | None = None, project_name: str | None = None, env_vars: dict[str, str] | None = None, dbt_vars: dict[str, str] | None = None, @@ -248,24 +248,26 @@ def __init__( else: self.manifest_path = Path(manifest_path_str) - if selectors_path: - selectors_path_str = str(selectors_path) - if not selectors_conn_id: - selectors_scheme = selectors_path_str.split("://")[0] + if selectors_yaml_path: + selectors_yaml_path_str = str(selectors_yaml_path) + if not selectors_yaml_conn_id: + selectors_yaml_scheme = selectors_yaml_path_str.split("://")[0] # Use the default Airflow connection ID for the scheme if it is not provided. - selectors_conn_id = FILE_SCHEME_AIRFLOW_DEFAULT_CONN_ID_MAP.get(selectors_scheme, lambda: None)() + selectors_yaml_conn_id = FILE_SCHEME_AIRFLOW_DEFAULT_CONN_ID_MAP.get( + selectors_yaml_scheme, lambda: None + )() - if selectors_conn_id is not None and not settings.AIRFLOW_IO_AVAILABLE: + if selectors_yaml_conn_id is not None and not settings.AIRFLOW_IO_AVAILABLE: raise CosmosValueError( - f"The selectors path {selectors_path_str} uses a remote file scheme, but the required Object " + f"The selectors yaml path {selectors_yaml_path_str} uses a remote file scheme, but the required Object " f"Storage feature is unavailable in Airflow version {airflow_version}. Please upgrade to " f"Airflow 2.8 or later." ) if settings.AIRFLOW_IO_AVAILABLE: - self.selectors_path = ObjectStoragePath(selectors_path_str, conn_id=selectors_conn_id) + self.selectors_yaml_path = ObjectStoragePath(selectors_yaml_path_str, conn_id=selectors_yaml_conn_id) else: - self.selectors_path = Path(selectors_path_str) + self.selectors_yaml_path = Path(selectors_yaml_path_str) self.env_vars = env_vars self.dbt_vars = dbt_vars @@ -311,11 +313,11 @@ def is_manifest_available(self) -> bool: """ return self.manifest_path.exists() if self.manifest_path else False - def is_selectors_available(self) -> bool: + def is_selectors_yaml_available(self) -> bool: """ - Check if the `dbt` selectors file is set and if the file exists. + Check if the `dbt` selectors YAML file is set and if the file exists. """ - return self.selectors_path.exists() if self.selectors_path else False + return self.selectors_yaml_path.exists() if self.selectors_yaml_path else False @dataclass diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index 741a7204fb..9d82259d53 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -9,7 +9,6 @@ import platform import tempfile import warnings -import yaml import zlib from dataclasses import dataclass, field from functools import cached_property @@ -17,6 +16,7 @@ from subprocess import PIPE, Popen from typing import TYPE_CHECKING, Any +import yaml from airflow.models import Variable if TYPE_CHECKING: @@ -58,7 +58,7 @@ get_partial_parse_path, has_non_empty_dependencies_file, ) -from cosmos.dbt.selector import select_nodes, parse_yaml_selectors +from cosmos.dbt.selector import parse_yaml_selectors, select_nodes from cosmos.log import get_logger logger = get_logger(__name__) @@ -380,9 +380,11 @@ def __init__( self.cache_dir = cache_dir self.airflow_metadata = airflow_metadata or {} if cache_identifier: - self.dbt_ls_cache_key = cache.create_cache_key(cache_identifier) + self.dbt_ls_cache_key = cache.create_cache_key(cache_identifier, "dbt_ls") + self.selectors_yaml_cache_key = cache.create_cache_key(cache_identifier, "selectors_yaml") else: self.dbt_ls_cache_key = "" + self.selectors_yaml_cache_key = "" self.dbt_vars = dbt_vars or {} self.operator_args = operator_args or {} self.log_dir: Path | None = None @@ -639,6 +641,11 @@ def should_use_dbt_ls_cache(self) -> bool: """Identify if Cosmos should use/store dbt ls cache or not.""" return settings.enable_cache and settings.enable_cache_dbt_ls and bool(self.dbt_ls_cache_key) + @functools.lru_cache + def should_use_selectors_yaml_cache(self) -> bool: + """Identify if Cosmos should use/store selectors YAML cache or not.""" + return settings.enable_cache and settings.enable_cache_selectors_yaml and bool(self.selectors_yaml_cache_key) + def load_via_dbt_ls_cache(self) -> bool: """(Try to) load dbt ls cache from an Airflow Variable""" logger.info(f"Trying to parse the dbt project using dbt ls cache {self.dbt_ls_cache_key}...") @@ -907,6 +914,146 @@ def load_via_custom_parser(self) -> None: exclude=self.render_config.exclude, ) + def _get_dbt_ls_remote_cache(self, remote_cache_dir: Path | ObjectStoragePath) -> dict[str, str]: + """Loads the remote cache for selectors yaml.""" + cache_dict: dict[str, str] = {} + remote_cache_key_path = remote_cache_dir / self.selectors_yaml_cache_key / "selectors_yaml_cache.json" + if remote_cache_key_path.exists(): + with remote_cache_key_path.open("r") as fp: + cache_dict = json.load(fp) + return cache_dict + + def get_selectors_yaml_cache(self) -> dict[str, Any]: + """ + Retrieve previously saved selectors YAML cache from an Airflow Variable. + + Outputs: + { + "version": "cache-version", + "selectors": uncompressed selectors dictionary, + "last_modified": "Isoformat timestamp" + } + """ + cache_dict: dict[str, Any] = {} + + airflow_variable_exceptions: list[type[BaseException]] = [json.decoder.JSONDecodeError, KeyError] + try: + from airflow.sdk.exceptions import AirflowRuntimeError + except ImportError: + pass + else: + airflow_variable_exceptions.append(AirflowRuntimeError) + + try: + remote_cache_dir = _configure_remote_cache_dir() + cache_dict = ( + self._get_selectors_yaml_remote_cache(remote_cache_dir) + if remote_cache_dir + else Variable.get(self.selectors_yaml_cache_key, deserialize_json=True) + ) + except tuple(airflow_variable_exceptions): + return cache_dict + else: + selectors_compressed = cache_dict.pop("selectors", None) + if selectors_compressed: + encoded_data = base64.b64decode(selectors_compressed.encode()) + decompressed_data = zlib.decompress(encoded_data).decode("utf-8") + cache_dict["selectors"] = json.loads(decompressed_data) + + return cache_dict + + def save_selectors_yaml_cache(self, selections: dict[str, Any]) -> None: + """ + Store parsed selectors YAML into an Airflow Variable. + + Stores: + { + "version": "cache-version", + "selectors": compressed_selectors, + "last_modified": "Isoformat timestamp" + } + """ + compressed_data = zlib.compress(json.dumps(selections).encode("utf-8")) + encoded_data = base64.b64encode(compressed_data) + selections_compressed = encoded_data.decode("utf-8") + + cache_dict = { + "version": cache._calculate_selectors_yaml_cache_current_version( + self.selectors_yaml_cache_key, self.project_path, self.project.selectors_yaml_path + ), + "selectors": selections_compressed, + "last_modified": datetime.datetime.now(datetime.timezone.utc).isoformat(), + **self.airflow_metadata, + } + remote_cache_dir = _configure_remote_cache_dir() + if remote_cache_dir: + remote_cache_key_path = remote_cache_dir / self.selectors_yaml_cache_key / "selectors_yaml_cache.json" + with remote_cache_key_path.open("w") as fp: + json.dump(cache_dict, fp) + else: + Variable.set(self.selectors_yaml_cache_key, cache_dict, serialize_json=True) + + def parse_selectors_yaml(self) -> dict[str, Any]: + """ + Parse the selectors YAML file and return the selections for all selectors. + + Returns: + A dictionary containing all selectors and their selections. + """ + if TYPE_CHECKING: + assert self.project.selectors_yaml_path is not None # pragma: no cover + + with self.project.selectors_yaml_path.open() as fp: + selectors = yaml.safe_load(fp) + selections = parse_yaml_selectors(selectors) + + if self.should_use_selectors_yaml_cache(): + self.save_selectors_yaml_cache(selections) + + return selections + + def load_parsed_selectors(self) -> dict[str, Any]: + """ + Parse the selectors YAML file and return the selections for the specified selector. + + Returns: + A dictionary containing the selections for the specified selector. + """ + logger.info(f"Trying to parse the dbt selector yamls using {self.selectors_yaml_cache_key}...") + + def get_selections(): + selections = self.parse_selectors_yaml() + + if self.should_use_selectors_yaml_cache(): + self.save_selectors_yaml_cache(selections) + + return selections + + if self.should_use_selectors_yaml_cache(): + cache_dict = self.get_selectors_yaml_cache() + if not cache_dict: + logger.info(f"Cosmos performance: Cache miss for {self.selectors_yaml_cache_key}") + return get_selections() + + cache_version = cache_dict.get("version") + selectors_cache = cache_dict.get("selectors") + + current_version = cache._calculate_selectors_yaml_cache_current_version( + self.selectors_yaml_cache_key, self.project_path, self.project.selectors_yaml_path + ) + + if selectors_cache and not cache.was_project_modified(cache_version, current_version): + logger.info( + f"Cosmos performance [{platform.node()}|{os.getpid()}]: The cache size for {self.selectors_yaml_cache_key} is {len(selectors_cache)}" + ) + logger.info(f"Cosmos performance: Cache hit for {self.selectors_yaml_cache_key} - {current_version}") + + return selectors_cache + + logger.info(f"Cosmos performance: Cache miss for {self.selectors_yaml_cache_key} - skipped") + + return get_selections() + def load_from_dbt_manifest(self) -> None: """ This approach accurately loads `dbt` projects using the `manifest.yml` file. @@ -921,8 +1068,8 @@ def load_from_dbt_manifest(self) -> None: self.load_method = LoadMode.DBT_MANIFEST logger.info("Trying to parse the dbt project `%s` using a dbt manifest...", self.project.project_name) - if self.render_config.selector and not self.project.is_selectors_available: - raise CosmosLoadDbtException(f"Unable to load selectors using {self.project.selectors_path}") + if self.render_config.selector and not self.project.is_selectors_yaml_available(): + raise CosmosLoadDbtException(f"Unable to load selectors using {self.project.selectors_yaml_path}") if not self.project.is_manifest_available(): raise CosmosLoadDbtException(f"Unable to load manifest using {self.project.manifest_path}") @@ -956,22 +1103,23 @@ def load_from_dbt_manifest(self) -> None: ) nodes[node.unique_id] = node - + if self.render_config.selector: - if TYPE_CHECKING: - assert self.project.selectors_path is not None # pragma: no cover - - with self.project.selectors_path.open() as fp2: - selectors = yaml.safe_load(fp2) - selections = parse_yaml_selectors(selectors)[self.render_config.selector] - - self.nodes = nodes - self.filtered_nodes = select_nodes( - project_dir=self.execution_config.project_path, - nodes=nodes, - select=selections["select"], - exclude=selections["exclude"], + selectors = self.load_parsed_selectors() + selections = selectors.get(self.render_config.selector) + + if not selections: + raise CosmosLoadDbtException( + f"Selector `{self.render_config.selector}` not found in selectors YAML `{self.project.selectors_yaml_path}`" ) + + self.nodes = nodes + self.filtered_nodes = select_nodes( + project_dir=self.execution_config.project_path, + nodes=nodes, + select=selections["select"], + exclude=selections["exclude"], + ) else: self.nodes = nodes self.filtered_nodes = select_nodes( diff --git a/cosmos/dbt/selector.py b/cosmos/dbt/selector.py index 6b58b11414..c0f301016b 100644 --- a/cosmos/dbt/selector.py +++ b/cosmos/dbt/selector.py @@ -682,18 +682,19 @@ def retrieve_by_label(statement_list: list[str], label: str) -> set[str]: return label_values + def _parse_selector_for_selections( - selector_definition: dict[str, Any], - all_selector_definitions: dict[str, dict[str, Any]], - resolved_cache: dict[str, Any], - visiting: set[str] | None = None, -): + selector_definition: dict[str, Any], + all_selector_definitions: dict[str, dict[str, Any]], + resolved_cache: dict[str, Any], + visiting: set[str] | None = None, +): """ Convert a selector definition from selectors.yml to dbt command-line syntax. - + This function recursively processes selector definitions and converts them into dbt-compatible select/exclude criteria. - + Args: selector_definition (dict): The selector definition to convert. all_selector_definitions (dict): A mapping of all selector names to their definitions. @@ -708,7 +709,7 @@ def _parse_selector_for_selections( def resolve_selector_reference(selector_name: str) -> tuple: """ Recursively resolve a selector reference to its base components. - + Returns: Tuple of (select_parts, exclude_parts) """ @@ -716,30 +717,30 @@ def resolve_selector_reference(selector_name: str) -> tuple: if selector_name in resolved_cache: cached = resolved_cache[selector_name] return (cached.get("select") or [], cached.get("exclude") or []) - + # Check for circular reference if selector_name in visiting: raise CosmosValueError(f"Circular reference detected in selector '{selector_name}'") - + # Get the selector definition if selector_name not in all_selector_definitions: raise CosmosValueError(f"Selector '{selector_name}' not found") - + # Mark as visiting visiting.add(selector_name) - + # Recursively resolve the referenced selector selector_def = all_selector_definitions[selector_name] result = _parse_selector_for_selections(selector_def, all_selector_definitions, resolved_cache, visiting) # Remove from visiting visiting.discard(selector_name) - + # Cache the result resolved_cache[selector_name] = result return (result.get("select") or [], result.get("exclude") or []) - + def process_method(method_def: dict[str, Any]) -> tuple: """ Process a single method definition. @@ -752,7 +753,7 @@ def process_method(method_def: dict[str, Any]) -> tuple: parents_depth = method_def.get("parents_depth", 0) children_depth = method_def.get("children_depth", 0) childrens_parents = method_def.get("childrens_parents", False) - + # Build the selector string if method.startswith(TAG_SELECTOR[:-1]): selector_str = f"{TAG_SELECTOR}{value}" @@ -775,7 +776,7 @@ def process_method(method_def: dict[str, Any]) -> tuple: elif method == "selector": # Recursively resolve the selector reference return resolve_selector_reference(value) - else: + else: raise CosmosValueError(f"Unsupported selector method: '{method}'") # Add modifiers @@ -801,40 +802,55 @@ def process_definition(definition: dict[str, Any]) -> tuple: """ Process a selector definition recursively. - Handles union, intersection, exclude, and method definitions. - Does not handle CLI-style or key-value definitions. + Handles union, intersection, exclude, and method definitions. + Does not handle CLI-style or key-value definitions. Raises any parse errors encountered during processing. Args: definition: Selector definition dictionary - + Returns: Tuple of (select_parts, exclude_parts) """ select_parts = [] exclude_parts = [] - + # Reject CLI-style string definitions (e.g., definition: "tag:my_tag") # These should be structured as method-value definitions instead if isinstance(definition, str): - raise CosmosValueError(f"CLI-style string definitions are not supported: '{definition}'. Use method-value definitions instead.") - + raise CosmosValueError( + f"CLI-style string definitions are not supported: '{definition}'. Use method-value definitions instead." + ) + # Reject key-value definitions (e.g., tag: my_tag, path: models/, etc.) # These are deprecated in favor of explicit method definitions # Valid structural keys are: method, union, intersection, exclude, and method modifiers - supported_definitions = {"method", "union", "intersection", "exclude", "value", "parents", "children", "parents_depth", "children_depth", "childrens_parents"} + supported_definitions = { + "method", + "union", + "intersection", + "exclude", + "value", + "parents", + "children", + "parents_depth", + "children_depth", + "childrens_parents", + } key_value_definitions = set(definition.keys()) - supported_definitions if key_value_definitions: key_value_strings = ", ".join(f"{k}: {definition[k]}" for k in key_value_definitions) - raise CosmosValueError(f"Key-value definitions are not supported: {key_value_strings}. Use method-value definitions instead.") + raise CosmosValueError( + f"Key-value definitions are not supported: {key_value_strings}. Use method-value definitions instead." + ) # Handle method + value if "method" in definition: method_select, method_exclude = process_method(definition) select_parts.extend(method_select) exclude_parts.extend(method_exclude) - + # Handle exclude if "exclude" in definition: exclude_items = definition["exclude"] @@ -843,17 +859,17 @@ def process_definition(definition: dict[str, Any]) -> tuple: # Items in exclude list should go to exclude_parts exclude_parts.extend(ex_select) exclude_parts.extend(ex_exclude) - + # Handle union if "union" in definition: union_items = definition["union"] - + for item in union_items: if isinstance(item, dict): item_select, item_exclude = process_definition(item) select_parts.extend(item_select) exclude_parts.extend(item_exclude) - + # Handle intersection if "intersection" in definition: intersection_items = definition["intersection"] @@ -882,7 +898,7 @@ def process_definition(definition: dict[str, Any]) -> tuple: return result -# TODO: Cache so its only calculated once (invalidate on file change) + def parse_yaml_selectors(selectors: dict[str, list[str]]) -> dict[str, dict[str, Any]]: """ Parse dbt YAML selectors into a dictionary of dbt command-line syntax. @@ -894,27 +910,27 @@ def parse_yaml_selectors(selectors: dict[str, list[str]]) -> dict[str, dict[str, """ selectors_root = selectors.get("selectors", []) - + # Build a map of all selector definitions for reference resolution all_selectors = {} for selector in selectors_root: name = selector.get("name", "") definition = selector.get("definition", {}) all_selectors[name] = definition - + # Build result dictionary with resolved selectors result = {} resolved_cache = {} - + for selector in selectors_root: name = selector.get("name", "") definition = selector.get("definition", {}) - + dbt_syntax = _parse_selector_for_selections(definition, all_selectors, resolved_cache) result[name] = dbt_syntax - + return result - + def select_nodes( project_dir: Path | None, diff --git a/cosmos/settings.py b/cosmos/settings.py index a4f35aa74d..8ef9319c08 100644 --- a/cosmos/settings.py +++ b/cosmos/settings.py @@ -27,6 +27,7 @@ enable_cache_partial_parse = conf.getboolean("cosmos", "enable_cache_partial_parse", fallback=True) enable_cache_package_lockfile = conf.getboolean("cosmos", "enable_cache_package_lockfile", fallback=True) enable_cache_dbt_ls = conf.getboolean("cosmos", "enable_cache_dbt_ls", fallback=True) +enable_cache_selectors_yaml = conf.getboolean("cosmos", "enable_cache_selectors_yaml", fallback=True) rich_logging = conf.getboolean("cosmos", "rich_logging", fallback=False) dbt_docs_dir = conf.get("cosmos", "dbt_docs_dir", fallback=None) dbt_docs_conn_id = conf.get("cosmos", "dbt_docs_conn_id", fallback=None) diff --git a/dev/dags/cosmos_manifest_yaml_selectors_example.py b/dev/dags/cosmos_manifest_yaml_selectors_example.py index 7180063350..f67fdc79c0 100644 --- a/dev/dags/cosmos_manifest_yaml_selectors_example.py +++ b/dev/dags/cosmos_manifest_yaml_selectors_example.py @@ -49,7 +49,7 @@ group_id="local_example", project_config=ProjectConfig( manifest_path=DBT_ROOT_PATH / "jaffle_shop" / "target" / "manifest.json", - selectors_path=DBT_ROOT_PATH / "jaffle_shop" / "selectors.yml", + selectors_yaml_path=DBT_ROOT_PATH / "jaffle_shop" / "selectors.yml", project_name="jaffle_shop", ), profile_config=profile_config, @@ -66,9 +66,9 @@ manifest_path="s3://cosmos-manifest-test/manifest.json", manifest_conn_id="aws_s3_conn", # `manifest_conn_id` is optional. If not provided, the default connection ID `aws_default` is used. - selectors_path="s3://cosmos-manifest-test/selectors.yml", - selectors_conn_id="aws_s3_conn", - # `selectors_conn_id` is optional. If not provided, the default connection ID ` + selectors_yaml_path="s3://cosmos-manifest-test/selectors.yml", + selectors_yaml_conn_id="aws_s3_conn", + # `selectors_yaml_conn_id` is optional. If not provided, the default connection ID ` project_name="jaffle_shop", ), profile_config=profile_config, @@ -85,9 +85,9 @@ manifest_path="gs://cosmos_remote_target/manifest.json", manifest_conn_id="gcp_gs_conn", # `manifest_conn_id` is optional. If not provided, the default connection ID `google_cloud_default` is used. - selectors_path="gs://cosmos_remote_target/selectors.yml", - selectors_conn_id="gcp_gs_conn", - # `selectors_conn_id` is optional. If not provided, the default connection ID `google_cloud_default` is used. + selectors_yaml_path="gs://cosmos_remote_target/selectors.yml", + selectors_yaml_conn_id="gcp_gs_conn", + # `selectors_yaml_conn_id` is optional. If not provided, the default connection ID `google_cloud_default` is used. project_name="jaffle_shop", ), profile_config=profile_config, @@ -104,9 +104,9 @@ manifest_path="abfs://cosmos-manifest-test/manifest.json", manifest_conn_id="azure_abfs_conn", # `manifest_conn_id` is optional. If not provided, the default connection ID `wasb_default` is used. - selectors_path="abfs://cosmos-manifest-test/selectors.yml", - selectors_conn_id="azure_abfs_conn", - # `selectors_conn_id` is optional. If not provided, the default connection ID `wasb_default` is used. + selectors_yaml_path="abfs://cosmos-manifest-test/selectors.yml", + selectors_yaml_conn_id="azure_abfs_conn", + # `selectors_yaml_conn_id` is optional. If not provided, the default connection ID `wasb_default` is used. project_name="jaffle_shop", ), profile_config=profile_config, From 2bd3804b8af5fd932b121f27efbc8a322f53c579 Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Wed, 7 Jan 2026 12:19:57 -0500 Subject: [PATCH 03/24] Fix static typing errors --- cosmos/cache.py | 2 +- cosmos/dbt/graph.py | 19 +++++++++++----- cosmos/dbt/selector.py | 31 +++++++++++++++----------- dev/dags/dbt/jaffle_shop/selectors.yml | 2 +- 4 files changed, 33 insertions(+), 21 deletions(-) diff --git a/cosmos/cache.py b/cosmos/cache.py index 85cf85e65f..e34553884b 100644 --- a/cosmos/cache.py +++ b/cosmos/cache.py @@ -288,7 +288,7 @@ def _copy_partial_parse_to_project(partial_parse_filepath: Path, project_path: P def _calculate_selectors_yaml_cache_current_version( - cache_identifier: str, project_dir: Path, selectors_yaml_path: Path + cache_identifier: str, project_dir: Path, selectors_yaml_path: Path | ObjectStoragePath ) -> str: """ Taking into account the project directory contents and the selectors yaml file, calculate the diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index 9d82259d53..848b8065d7 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -914,9 +914,9 @@ def load_via_custom_parser(self) -> None: exclude=self.render_config.exclude, ) - def _get_dbt_ls_remote_cache(self, remote_cache_dir: Path | ObjectStoragePath) -> dict[str, str]: + def _get_selectors_yaml_remote_cache(self, remote_cache_dir: Path | ObjectStoragePath) -> dict[str, Any]: """Loads the remote cache for selectors yaml.""" - cache_dict: dict[str, str] = {} + cache_dict: dict[str, Any] = {} remote_cache_key_path = remote_cache_dir / self.selectors_yaml_cache_key / "selectors_yaml_cache.json" if remote_cache_key_path.exists(): with remote_cache_key_path.open("r") as fp: @@ -973,6 +973,9 @@ def save_selectors_yaml_cache(self, selections: dict[str, Any]) -> None: "last_modified": "Isoformat timestamp" } """ + if TYPE_CHECKING: + assert self.project.selectors_yaml_path is not None # pragma: no cover + compressed_data = zlib.compress(json.dumps(selections).encode("utf-8")) encoded_data = base64.b64encode(compressed_data) selections_compressed = encoded_data.decode("utf-8") @@ -1021,7 +1024,10 @@ def load_parsed_selectors(self) -> dict[str, Any]: """ logger.info(f"Trying to parse the dbt selector yamls using {self.selectors_yaml_cache_key}...") - def get_selections(): + if TYPE_CHECKING: + assert self.project.selectors_yaml_path is not None # pragma: no cover + + def get_selections() -> dict[str, Any]: selections = self.parse_selectors_yaml() if self.should_use_selectors_yaml_cache(): @@ -1033,10 +1039,11 @@ def get_selections(): cache_dict = self.get_selectors_yaml_cache() if not cache_dict: logger.info(f"Cosmos performance: Cache miss for {self.selectors_yaml_cache_key}") - return get_selections() - cache_version = cache_dict.get("version") - selectors_cache = cache_dict.get("selectors") + cache_dict = get_selections() + + cache_version = cache_dict.get("version", "") + selectors_cache: dict[str, Any] = cache_dict.get("selectors", {}) current_version = cache._calculate_selectors_yaml_cache_current_version( self.selectors_yaml_cache_key, self.project_path, self.project.selectors_yaml_path diff --git a/cosmos/dbt/selector.py b/cosmos/dbt/selector.py index c0f301016b..243c3f1980 100644 --- a/cosmos/dbt/selector.py +++ b/cosmos/dbt/selector.py @@ -686,9 +686,9 @@ def retrieve_by_label(statement_list: list[str], label: str) -> set[str]: def _parse_selector_for_selections( selector_definition: dict[str, Any], all_selector_definitions: dict[str, dict[str, Any]], - resolved_cache: dict[str, Any], + resolved_cache: dict[str, dict[str, list[str] | None]], visiting: set[str] | None = None, -): +) -> dict[str, list[str] | None]: """ Convert a selector definition from selectors.yml to dbt command-line syntax. @@ -706,7 +706,7 @@ def _parse_selector_for_selections( resolved_cache = resolved_cache or {} visiting = visiting or set() - def resolve_selector_reference(selector_name: str) -> tuple: + def resolve_selector_reference(selector_name: str) -> tuple[list[str], list[str]]: """ Recursively resolve a selector reference to its base components. @@ -741,19 +741,24 @@ def resolve_selector_reference(selector_name: str) -> tuple: return (result.get("select") or [], result.get("exclude") or []) - def process_method(method_def: dict[str, Any]) -> tuple: + def process_method(method_def: dict[str, Any]) -> tuple[list[str], list[str]]: """ Process a single method definition. Returns tuple of (select_parts, exclude_parts) """ - method = method_def.get("method") - value = method_def.get("value") + method = method_def.get("method", "METHOD_KEY_NOT_FOUND") + value = method_def.get("value", "VALUE_KEY_NOT_FOUND") parents = method_def.get("parents", False) children = method_def.get("children", False) parents_depth = method_def.get("parents_depth", 0) children_depth = method_def.get("children_depth", 0) childrens_parents = method_def.get("childrens_parents", False) + if method == "METHOD_KEY_NOT_FOUND": + raise CosmosValueError(f"Selector method is missing in method definition: {method_def}.") + if value == "VALUE_KEY_NOT_FOUND": + raise CosmosValueError(f"Selector value is missing in method definition: {method_def}.") + # Build the selector string if method.startswith(TAG_SELECTOR[:-1]): selector_str = f"{TAG_SELECTOR}{value}" @@ -798,7 +803,7 @@ def process_method(method_def: dict[str, Any]) -> tuple: return ([selector_str], []) - def process_definition(definition: dict[str, Any]) -> tuple: + def process_definition(definition: dict[str, Any]) -> tuple[list[str], list[str]]: """ Process a selector definition recursively. @@ -873,7 +878,7 @@ def process_definition(definition: dict[str, Any]) -> tuple: # Handle intersection if "intersection" in definition: intersection_items = definition["intersection"] - intersection_selects = [] + intersection_selects: list[str] = [] for item in intersection_items: item_select, item_exclude = process_definition(item) intersection_selects.extend(item_select) @@ -899,7 +904,7 @@ def process_definition(definition: dict[str, Any]) -> tuple: return result -def parse_yaml_selectors(selectors: dict[str, list[str]]) -> dict[str, dict[str, Any]]: +def parse_yaml_selectors(selectors: dict[str, list[dict[str, Any]]]) -> dict[str, dict[str, Any]]: """ Parse dbt YAML selectors into a dictionary of dbt command-line syntax. @@ -909,18 +914,18 @@ def parse_yaml_selectors(selectors: dict[str, list[str]]) -> dict[str, dict[str, dict: A dictionary mapping selector names to their dbt selection syntax. """ - selectors_root = selectors.get("selectors", []) + selectors_root: list[dict[str, Any]] = selectors.get("selectors", []) # Build a map of all selector definitions for reference resolution - all_selectors = {} + all_selectors: dict[str, Any] = {} for selector in selectors_root: name = selector.get("name", "") definition = selector.get("definition", {}) all_selectors[name] = definition # Build result dictionary with resolved selectors - result = {} - resolved_cache = {} + result: dict[str, dict[str, list[str] | None]] = {} + resolved_cache: dict[str, dict[str, list[str] | None]] = {} for selector in selectors_root: name = selector.get("name", "") diff --git a/dev/dags/dbt/jaffle_shop/selectors.yml b/dev/dags/dbt/jaffle_shop/selectors.yml index d442b030e2..9e8d30b2bc 100644 --- a/dev/dags/dbt/jaffle_shop/selectors.yml +++ b/dev/dags/dbt/jaffle_shop/selectors.yml @@ -10,7 +10,7 @@ selectors: definition: method: tag value: core - + - name: customers_and_downstream description: "Select customers model and all downstream dependencies" definition: From bc08169125a08570dbdda8cd7fa3be62c566a9de Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Wed, 7 Jan 2026 18:46:43 -0500 Subject: [PATCH 04/24] Fix code complexity linting errors --- cosmos/config.py | 48 ++++- cosmos/dbt/selector.py | 455 ++++++++++++++++++++++++----------------- 2 files changed, 306 insertions(+), 197 deletions(-) diff --git a/cosmos/config.py b/cosmos/config.py index 53ee64b79f..45947e3d6b 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -210,6 +210,35 @@ def __init__( dbt_vars: dict[str, str] | None = None, partial_parse: bool = True, ): + if project_name: + self.project_name = project_name + + self.env_vars = env_vars + self.dbt_vars = dbt_vars + self.partial_parse = partial_parse + self.install_dbt_deps = install_dbt_deps + self.copy_dbt_packages = copy_dbt_packages + + self.validate_dbt_project_paths( + project_name, + dbt_project_path, + models_relative_path, + seeds_relative_path, + snapshots_relative_path, + manifest_path, + ) + self.validate_manifest_path(manifest_path, manifest_conn_id) + self.validate_selectors_yaml_path(selectors_yaml_path, selectors_yaml_conn_id) + + def validate_dbt_project_paths( + self, + project_name: str | None, + dbt_project_path: str | Path | None, + models_relative_path: str | Path, + seeds_relative_path: str | Path, + snapshots_relative_path: str | Path, + manifest_path: str | Path | None, + ) -> None: # Since we allow dbt_project_path to be defined in ExecutionConfig and RenderConfig # dbt_project_path may not always be defined here. # We do, however, still require that both manifest_path and project_name be defined, or neither be defined. @@ -218,9 +247,6 @@ def __init__( raise CosmosValueError( "If ProjectConfig.dbt_project_path is not defined, ProjectConfig.manifest_path and ProjectConfig.project_name must be defined together, or both left undefined." ) - if project_name: - self.project_name = project_name - if dbt_project_path: self.dbt_project_path = Path(dbt_project_path) self.models_path = self.dbt_project_path / Path(models_relative_path) @@ -229,6 +255,11 @@ def __init__( if not project_name: self.project_name = self.dbt_project_path.stem + def validate_manifest_path( + self, + manifest_path: str | Path | None, + manifest_conn_id: str | None, + ) -> None: if manifest_path: manifest_path_str = str(manifest_path) if not manifest_conn_id: @@ -248,6 +279,11 @@ def __init__( else: self.manifest_path = Path(manifest_path_str) + def validate_selectors_yaml_path( + self, + selectors_yaml_path: str | Path | None, + selectors_yaml_conn_id: str | None, + ) -> None: if selectors_yaml_path: selectors_yaml_path_str = str(selectors_yaml_path) if not selectors_yaml_conn_id: @@ -269,12 +305,6 @@ def __init__( else: self.selectors_yaml_path = Path(selectors_yaml_path_str) - self.env_vars = env_vars - self.dbt_vars = dbt_vars - self.partial_parse = partial_parse - self.install_dbt_deps = install_dbt_deps - self.copy_dbt_packages = copy_dbt_packages - def validate_project(self) -> None: """ Validates necessary context is present for a project. diff --git a/cosmos/dbt/selector.py b/cosmos/dbt/selector.py index 243c3f1980..5b014fc013 100644 --- a/cosmos/dbt/selector.py +++ b/cosmos/dbt/selector.py @@ -683,225 +683,294 @@ def retrieve_by_label(statement_list: list[str], label: str) -> set[str]: return label_values -def _parse_selector_for_selections( - selector_definition: dict[str, Any], +def _resolve_selector_reference( + selector_name: str, all_selector_definitions: dict[str, dict[str, Any]], - resolved_cache: dict[str, dict[str, list[str] | None]], - visiting: set[str] | None = None, -) -> dict[str, list[str] | None]: + resolved_cache: dict[str, tuple[list[str], list[str]]], + visiting: set[str], +) -> tuple[list[str], list[str]]: """ - Convert a selector definition from selectors.yml to dbt command-line syntax. + Resolve a selector reference recursively into select/exclude dbt command-line syntax. + Args: + selector_name (str): The name of the selector to resolve. + all_selector_definitions (dict): All selector definitions for reference resolution. + resolved_cache (dict): Cache for already resolved selectors. + visiting (set): Set of currently visiting selectors to detect circular references. + Returns: + tuple: A tuple containing two lists - select parts and exclude parts. + Raises: + CosmosValueError: If a circular reference is detected or if the selector is not found. + """ + # Check cache first + if selector_name in resolved_cache: + return resolved_cache[selector_name] + + # Check for circular reference + if selector_name in visiting: + raise CosmosValueError(f"Circular reference detected in selector '{selector_name}'") + + # Get the selector definition + if selector_name not in all_selector_definitions: + raise CosmosValueError(f"Selector '{selector_name}' not found") + + # Mark as visiting + visiting.add(selector_name) + + # Recursively resolve the referenced selector + selector_def = all_selector_definitions[selector_name] + result = _parse_selector_definition(selector_def, all_selector_definitions, resolved_cache, visiting) + + # Remove from visiting + visiting.discard(selector_name) - This function recursively processes selector definitions and converts them into - dbt-compatible select/exclude criteria. + # Cache the result + resolved_cache[selector_name] = result + return result + + +def _evaluate_selector_method_definition( + method: str, + value: str, +) -> str: + """ + Evaluate a selector method definition into dbt command-line syntax. Args: - selector_definition (dict): The selector definition to convert. - all_selector_definitions (dict): A mapping of all selector names to their definitions. - resolved_cache (dict): Cache for already resolved selectors to avoid redundant processing. - visiting (set, optional): Set of currently visiting selector names to detect cycles. + method (str): The selector method (e.g., "tag", "path", "source", etc.). + value (str): The value associated with the method. Returns: - dict: A dictionary with "select" and "exclude" keys containing lists of dbt selection strings. + str: The dbt command-line syntax for the selector. + Raises: + CosmosValueError: If the method is unsupported. """ - resolved_cache = resolved_cache or {} - visiting = visiting or set() + if method.startswith(TAG_SELECTOR[:-1]): + return f"{TAG_SELECTOR}{value}" + elif method.startswith(PATH_SELECTOR[:-1]): + return f"{PATH_SELECTOR}{value}" + elif method.startswith(SOURCE_SELECTOR[:-1]): + return f"{SOURCE_SELECTOR}{value}" + elif method.startswith(EXPOSURE_SELECTOR[:-1]): + return f"{EXPOSURE_SELECTOR}{value}" + elif method.startswith(RESOURCE_TYPE_SELECTOR[:-1]): + return f"{RESOURCE_TYPE_SELECTOR}{value}" + elif method.startswith(EXCLUDE_RESOURCE_TYPE_SELECTOR[:-1]): + return f"{EXCLUDE_RESOURCE_TYPE_SELECTOR}{value}" + elif any([method.startswith(CONFIG_SELECTOR + config) for config in SUPPORTED_CONFIG]): + return f"{CONFIG_SELECTOR}:{value}" + elif method == "fqn": + if value == "*": + return "" + else: + return value + else: + raise CosmosValueError(f"Unsupported selector method: '{method}'") - def resolve_selector_reference(selector_name: str) -> tuple[list[str], list[str]]: - """ - Recursively resolve a selector reference to its base components. - Returns: - Tuple of (select_parts, exclude_parts) - """ - # Check cache first - if selector_name in resolved_cache: - cached = resolved_cache[selector_name] - return (cached.get("select") or [], cached.get("exclude") or []) +def _evaluate_selector_method_definition_modifiers( + selector: str, parents: bool, children: bool, parents_depth: int, children_depth: int, childrens_parents: bool +) -> str: + """ + Apply graph operator modifiers to a selector string. - # Check for circular reference - if selector_name in visiting: - raise CosmosValueError(f"Circular reference detected in selector '{selector_name}'") + Args: + selector (str): The base selector string. + parents (bool): Whether to include parents. + children (bool): Whether to include children. + parents_depth (int): Depth of parents to include. + children_depth (int): Depth of children to include. + childrens_parents (bool): Whether to include children's parents. + Returns: + str: The modified selector string with graph operators applied. + """ + if parents: + if parents_depth and parents_depth > 0: + selector = f"{parents_depth}+{selector}" + else: + selector = f"+{selector}" + if children: + if children_depth and children_depth > 0: + selector = f"{selector}+{children_depth}" + else: + selector = f"{selector}+" + if childrens_parents: + if parents_depth or children_depth: + raise CosmosValueError("childrens_parents cannot be combined with parents_depth or children_depth.") + else: + selector = f"@{selector}" + return selector - # Get the selector definition - if selector_name not in all_selector_definitions: - raise CosmosValueError(f"Selector '{selector_name}' not found") - # Mark as visiting - visiting.add(selector_name) +def _parse_selector_method_definition( + method_def: dict[str, Any], + all_selector_definitions: dict[str, dict[str, Any]], + resolved_cache: dict[str, tuple[list[str], list[str]]], + visiting: set[str], +) -> tuple[list[str], list[str]]: + """ + Parse a single selector method definition into select/exclude dbt command-line syntax. + Args: + method_def (dict): The selector method definition to parse. + all_selector_definitions (dict): All selector definitions for reference resolution. + resolved_cache (dict): Cache for already resolved selectors. + visiting (set): Set of currently visiting selectors to detect circular references. + Returns: + tuple: A tuple containing two lists - select parts and exclude parts. + """ + method = method_def.get("method", "METHOD_KEY_NOT_FOUND") + value = method_def.get("value", "VALUE_KEY_NOT_FOUND") + parents = method_def.get("parents", False) + children = method_def.get("children", False) + parents_depth = method_def.get("parents_depth", 0) + children_depth = method_def.get("children_depth", 0) + childrens_parents = method_def.get("childrens_parents", False) - # Recursively resolve the referenced selector - selector_def = all_selector_definitions[selector_name] - result = _parse_selector_for_selections(selector_def, all_selector_definitions, resolved_cache, visiting) + if method == "METHOD_KEY_NOT_FOUND": + raise CosmosValueError(f"Selector method is missing in method definition: {method_def}.") + if value == "VALUE_KEY_NOT_FOUND": + raise CosmosValueError(f"Selector value is missing in method definition: {method_def}.") - # Remove from visiting - visiting.discard(selector_name) + if method == "selector": + return _resolve_selector_reference(value, all_selector_definitions, resolved_cache, visiting) - # Cache the result - resolved_cache[selector_name] = result + selector = _evaluate_selector_method_definition(method, value) - return (result.get("select") or [], result.get("exclude") or []) + if not selector: + return ([], []) - def process_method(method_def: dict[str, Any]) -> tuple[list[str], list[str]]: - """ - Process a single method definition. - Returns tuple of (select_parts, exclude_parts) - """ - method = method_def.get("method", "METHOD_KEY_NOT_FOUND") - value = method_def.get("value", "VALUE_KEY_NOT_FOUND") - parents = method_def.get("parents", False) - children = method_def.get("children", False) - parents_depth = method_def.get("parents_depth", 0) - children_depth = method_def.get("children_depth", 0) - childrens_parents = method_def.get("childrens_parents", False) - - if method == "METHOD_KEY_NOT_FOUND": - raise CosmosValueError(f"Selector method is missing in method definition: {method_def}.") - if value == "VALUE_KEY_NOT_FOUND": - raise CosmosValueError(f"Selector value is missing in method definition: {method_def}.") - - # Build the selector string - if method.startswith(TAG_SELECTOR[:-1]): - selector_str = f"{TAG_SELECTOR}{value}" - elif method.startswith(PATH_SELECTOR[:-1]): - selector_str = f"{PATH_SELECTOR}{value}" - elif method.startswith(SOURCE_SELECTOR[:-1]): - selector_str = f"{SOURCE_SELECTOR}{value}" - elif method.startswith(EXPOSURE_SELECTOR[:-1]): - selector_str = f"{EXPOSURE_SELECTOR}{value}" - elif method.startswith(RESOURCE_TYPE_SELECTOR[:-1]): - selector_str = f"{RESOURCE_TYPE_SELECTOR}{value}" - elif method.startswith(EXCLUDE_RESOURCE_TYPE_SELECTOR[:-1]): - selector_str = f"{EXCLUDE_RESOURCE_TYPE_SELECTOR}{value}" - elif any([method.startswith(CONFIG_SELECTOR + config) for config in SUPPORTED_CONFIG]): - selector_str = f"{CONFIG_SELECTOR}:{value}" - elif method == "fqn": - if value == "*": - return ([], []) - selector_str = value - elif method == "selector": - # Recursively resolve the selector reference - return resolve_selector_reference(value) - else: - raise CosmosValueError(f"Unsupported selector method: '{method}'") + selector = _evaluate_selector_method_definition_modifiers( + selector, parents, children, parents_depth, children_depth, childrens_parents + ) - # Add modifiers - if parents: - if parents_depth and parents_depth > 0: - selector_str = f"{parents_depth}+{selector_str}" - else: - selector_str = f"+{selector_str}" - if children: - if children_depth and children_depth > 0: - selector_str = f"{selector_str}+{children_depth}" - else: - selector_str = f"{selector_str}+" - if childrens_parents: - if parents_depth or children_depth: - raise CosmosValueError("childrens_parents cannot be combined with parents_depth or children_depth.") - else: - selector_str = f"@{selector_str}" + return ([selector], []) - return ([selector_str], []) - def process_definition(definition: dict[str, Any]) -> tuple[list[str], list[str]]: - """ - Process a selector definition recursively. +def _validate_selector_definition(selectors: dict[str, Any]) -> None: + """ + Validate the structure of a selector definition. - Handles union, intersection, exclude, and method definitions. - Does not handle CLI-style or key-value definitions. - Raises any parse errors encountered during processing. + Args: + selectors (dict): The selector definition to validate. + Raises: + CosmosValueError: If the selector definition contains CLI-style string definitions + or key-value definitions. + """ - Args: - definition: Selector definition dictionary + # Reject CLI-style string definitions (e.g., definition: "tag:my_tag") + # These should be structured as method-value definitions instead + if isinstance(selectors, str): + raise CosmosValueError( + f"CLI-style string definitions are not supported: '{selectors}'. Use method-value definitions instead." + ) - Returns: - Tuple of (select_parts, exclude_parts) - """ - select_parts = [] - exclude_parts = [] - - # Reject CLI-style string definitions (e.g., definition: "tag:my_tag") - # These should be structured as method-value definitions instead - if isinstance(definition, str): - raise CosmosValueError( - f"CLI-style string definitions are not supported: '{definition}'. Use method-value definitions instead." - ) + # Reject key-value definitions (e.g., tag: my_tag, path: models/, etc.) + # These are deprecated in favor of explicit method definitions + # Valid structural keys are: method, union, intersection, exclude, and method modifiers + supported_definitions = { + "method", + "union", + "intersection", + "exclude", + "value", + "parents", + "children", + "parents_depth", + "children_depth", + "childrens_parents", + } + key_value_definitions = set(selectors.keys()) - supported_definitions - # Reject key-value definitions (e.g., tag: my_tag, path: models/, etc.) - # These are deprecated in favor of explicit method definitions - # Valid structural keys are: method, union, intersection, exclude, and method modifiers - supported_definitions = { - "method", - "union", - "intersection", - "exclude", - "value", - "parents", - "children", - "parents_depth", - "children_depth", - "childrens_parents", - } - key_value_definitions = set(definition.keys()) - supported_definitions + if key_value_definitions: + key_value_strings = ", ".join(f"{k}: {selectors[k]}" for k in key_value_definitions) + raise CosmosValueError( + f"Key-value definitions are not supported: {key_value_strings}. Use method-value definitions instead." + ) - if key_value_definitions: - key_value_strings = ", ".join(f"{k}: {definition[k]}" for k in key_value_definitions) - raise CosmosValueError( - f"Key-value definitions are not supported: {key_value_strings}. Use method-value definitions instead." - ) - # Handle method + value - if "method" in definition: - method_select, method_exclude = process_method(definition) - select_parts.extend(method_select) - exclude_parts.extend(method_exclude) - - # Handle exclude - if "exclude" in definition: - exclude_items = definition["exclude"] - for exclude_item in exclude_items: - ex_select, ex_exclude = process_definition(exclude_item) - # Items in exclude list should go to exclude_parts - exclude_parts.extend(ex_select) - exclude_parts.extend(ex_exclude) - - # Handle union - if "union" in definition: - union_items = definition["union"] - - for item in union_items: - if isinstance(item, dict): - item_select, item_exclude = process_definition(item) - select_parts.extend(item_select) - exclude_parts.extend(item_exclude) - - # Handle intersection - if "intersection" in definition: - intersection_items = definition["intersection"] - intersection_selects: list[str] = [] - for item in intersection_items: - item_select, item_exclude = process_definition(item) - intersection_selects.extend(item_select) - exclude_parts.extend(item_exclude) +def _parse_selector_definition( + selectors: dict[str, Any], + all_selector_definitions: dict[str, dict[str, Any]], + resolved_cache: dict[str, tuple[list[str], list[str]]], + visiting: set[str], +) -> tuple[list[str], list[str]]: + """ + Parse a selector definition into select/exclude dbt command-line syntax. - if intersection_selects: - # Join with commas for intersection - select_parts = [",".join(intersection_selects)] + Args: + selectors (dict): The selector definition to parse. + all_selector_definitions (dict): All selector definitions for reference resolution. + resolved_cache (dict): Cache for already resolved selectors. + visiting (set): Set of currently visiting selectors to detect circular references. + Returns: + tuple: A tuple containing two lists - select parts and exclude parts. + """ + select_parts = [] + exclude_parts = [] + + _validate_selector_definition(selectors) + + # Handle method + value + if "method" in selectors: + method_select, method_exclude = _parse_selector_method_definition( + selectors, + all_selector_definitions, + resolved_cache, + visiting, + ) + select_parts.extend(method_select) + exclude_parts.extend(method_exclude) + + # Handle exclude + if "exclude" in selectors: + exclude_items = selectors["exclude"] + for exclude_item in exclude_items: + ex_select, ex_exclude = _parse_selector_definition( + exclude_item, + all_selector_definitions, + resolved_cache, + visiting, + ) + # Items in exclude list should go to exclude_parts + exclude_parts.extend(ex_select) + exclude_parts.extend(ex_exclude) + + # Handle union + if "union" in selectors: + union_items = selectors["union"] + + for item in union_items: + if isinstance(item, dict): + item_select, item_exclude = _parse_selector_definition( + item, + all_selector_definitions, + resolved_cache, + visiting, + ) + select_parts.extend(item_select) + exclude_parts.extend(item_exclude) - return (select_parts, exclude_parts) + # Handle intersection + if "intersection" in selectors: + intersection_items = selectors["intersection"] + intersection_selects: list[str] = [] + for item in intersection_items: + item_select, item_exclude = _parse_selector_definition( + item, + all_selector_definitions, + resolved_cache, + visiting, + ) + intersection_selects.extend(item_select) + exclude_parts.extend(item_exclude) - select_parts, exclude_parts = process_definition(selector_definition) + if intersection_selects: + # Join with commas for intersection + select_parts = [",".join(intersection_selects)] # Remove None values, empty strings, and duplicates while preserving order select_parts = list(dict.fromkeys([s for s in select_parts if s])) exclude_parts = list(dict.fromkeys([e for e in exclude_parts if e])) - result = { - "select": select_parts if select_parts else None, - "exclude": exclude_parts if exclude_parts else None, - } - - return result + return (select_parts, exclude_parts) def parse_yaml_selectors(selectors: dict[str, list[dict[str, Any]]]) -> dict[str, dict[str, Any]]: @@ -925,14 +994,24 @@ def parse_yaml_selectors(selectors: dict[str, list[dict[str, Any]]]) -> dict[str # Build result dictionary with resolved selectors result: dict[str, dict[str, list[str] | None]] = {} - resolved_cache: dict[str, dict[str, list[str] | None]] = {} + resolved_cache: dict[str, tuple[list[str], list[str]]] = {} + visiting: set[str] = set() for selector in selectors_root: name = selector.get("name", "") definition = selector.get("definition", {}) - dbt_syntax = _parse_selector_for_selections(definition, all_selectors, resolved_cache) - result[name] = dbt_syntax + select, exclude = _parse_selector_definition( + definition, + all_selectors, + resolved_cache, + visiting, + ) + + result[name] = { + "select": select if select else None, + "exclude": exclude if exclude else None, + } return result From 728a30a0cdcaeafc347caf3b49f6b62d58f2d638 Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Thu, 8 Jan 2026 11:39:02 -0500 Subject: [PATCH 05/24] Update comments --- cosmos/dbt/selector.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/cosmos/dbt/selector.py b/cosmos/dbt/selector.py index 5b014fc013..ae6eba233e 100644 --- a/cosmos/dbt/selector.py +++ b/cosmos/dbt/selector.py @@ -683,6 +683,9 @@ def retrieve_by_label(statement_list: list[str], label: str) -> set[str]: return label_values +# YAML Selectors + + def _resolve_selector_reference( selector_name: str, all_selector_definitions: dict[str, dict[str, Any]], @@ -808,6 +811,7 @@ def _parse_selector_method_definition( ) -> tuple[list[str], list[str]]: """ Parse a single selector method definition into select/exclude dbt command-line syntax. + Args: method_def (dict): The selector method definition to parse. all_selector_definitions (dict): All selector definitions for reference resolution. @@ -895,6 +899,9 @@ def _parse_selector_definition( """ Parse a selector definition into select/exclude dbt command-line syntax. + Handles union, intersection, exclude, and method definitions. Collects any + parse errors encountered during processing. + Args: selectors (dict): The selector definition to parse. all_selector_definitions (dict): All selector definitions for reference resolution. @@ -902,13 +909,15 @@ def _parse_selector_definition( visiting (set): Set of currently visiting selectors to detect circular references. Returns: tuple: A tuple containing two lists - select parts and exclude parts. + Raises: + CosmosValueError: If there are issues parsing the selector definition. + """ select_parts = [] exclude_parts = [] _validate_selector_definition(selectors) - # Handle method + value if "method" in selectors: method_select, method_exclude = _parse_selector_method_definition( selectors, @@ -975,7 +984,7 @@ def _parse_selector_definition( def parse_yaml_selectors(selectors: dict[str, list[dict[str, Any]]]) -> dict[str, dict[str, Any]]: """ - Parse dbt YAML selectors into a dictionary of dbt command-line syntax. + Parse dbt YAML selectors into a dictionary of select/exclude dbt command-line syntax. Args: selectors (dict): The selectors dictionary loaded from selectors.yml. From 795f964b055baf795d9ceb3fcc5404f551907129 Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Thu, 8 Jan 2026 12:05:59 -0500 Subject: [PATCH 06/24] Converge on single graph cache --- cosmos/cache.py | 4 +-- cosmos/dbt/graph.py | 56 ++++++++++++++++++++--------------------- tests/dbt/test_graph.py | 2 +- 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/cosmos/cache.py b/cosmos/cache.py index e34553884b..431de4e945 100644 --- a/cosmos/cache.py +++ b/cosmos/cache.py @@ -149,8 +149,8 @@ def _create_cache_identifier(dag: DAG, task_group: TaskGroup | None) -> str: return "__".join(cache_identifiers_list) -def create_cache_key(cache_identifier: str, cache_suffix: str) -> str: - return f"{VAR_KEY_CACHE_PREFIX}{cache_identifier}_{cache_suffix}" +def create_cache_key(cache_identifier: str) -> str: + return f"{VAR_KEY_CACHE_PREFIX}{cache_identifier}" def _obtain_cache_dir_path(cache_identifier: str, base_dir: Path = settings.cache_dir) -> Path: diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index 848b8065d7..9d00e27368 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -380,11 +380,9 @@ def __init__( self.cache_dir = cache_dir self.airflow_metadata = airflow_metadata or {} if cache_identifier: - self.dbt_ls_cache_key = cache.create_cache_key(cache_identifier, "dbt_ls") - self.selectors_yaml_cache_key = cache.create_cache_key(cache_identifier, "selectors_yaml") + self.cache_key = cache.create_cache_key(cache_identifier) else: - self.dbt_ls_cache_key = "" - self.selectors_yaml_cache_key = "" + self.cache_key = "" self.dbt_vars = dbt_vars or {} self.operator_args = operator_args or {} self.log_dir: Path | None = None @@ -459,7 +457,7 @@ def dbt_ls_cache_key_args(self) -> list[str]: airflow_vars = [var_name, Variable.get(var_name, "")] cache_args.extend(airflow_vars) - logger.debug(f"Value of `dbt_ls_cache_key_args` for <{self.dbt_ls_cache_key}>: {cache_args}") + logger.debug(f"Value of `dbt_ls_cache_key_args` for <{self.cache_key}>: {cache_args}") return cache_args def save_dbt_ls_cache(self, dbt_ls_output: str) -> None: @@ -479,7 +477,7 @@ def save_dbt_ls_cache(self, dbt_ls_output: str) -> None: dbt_ls_compressed = encoded_data.decode("utf-8") cache_dict = { "version": cache._calculate_dbt_ls_cache_current_version( - self.dbt_ls_cache_key, self.project_path, self.dbt_ls_cache_key_args + self.cache_key, self.project_path, self.dbt_ls_cache_key_args ), "dbt_ls_compressed": dbt_ls_compressed, "last_modified": datetime.datetime.now(datetime.timezone.utc).isoformat(), @@ -487,16 +485,16 @@ def save_dbt_ls_cache(self, dbt_ls_output: str) -> None: } remote_cache_dir = _configure_remote_cache_dir() if remote_cache_dir: - remote_cache_key_path = remote_cache_dir / self.dbt_ls_cache_key / "dbt_ls_cache.json" + remote_cache_key_path = remote_cache_dir / self.cache_key / "dbt_ls_cache.json" with remote_cache_key_path.open("w") as fp: json.dump(cache_dict, fp) else: - Variable.set(self.dbt_ls_cache_key, cache_dict, serialize_json=True) + Variable.set(self.cache_key, cache_dict, serialize_json=True) def _get_dbt_ls_remote_cache(self, remote_cache_dir: Path | ObjectStoragePath) -> dict[str, str]: """Loads the remote cache for dbt ls.""" cache_dict: dict[str, str] = {} - remote_cache_key_path = remote_cache_dir / self.dbt_ls_cache_key / "dbt_ls_cache.json" + remote_cache_key_path = remote_cache_dir / self.cache_key / "dbt_ls_cache.json" if remote_cache_key_path.exists(): with remote_cache_key_path.open("r") as fp: cache_dict = json.load(fp) @@ -528,7 +526,7 @@ def get_dbt_ls_cache(self) -> dict[str, str]: cache_dict = ( self._get_dbt_ls_remote_cache(remote_cache_dir) if remote_cache_dir - else Variable.get(self.dbt_ls_cache_key, deserialize_json=True) + else Variable.get(self.cache_key, deserialize_json=True) ) except tuple(airflow_variable_exceptions): return cache_dict @@ -639,43 +637,43 @@ def load_via_dbt_ls(self) -> None: @functools.lru_cache def should_use_dbt_ls_cache(self) -> bool: """Identify if Cosmos should use/store dbt ls cache or not.""" - return settings.enable_cache and settings.enable_cache_dbt_ls and bool(self.dbt_ls_cache_key) + return settings.enable_cache and settings.enable_cache_dbt_ls and bool(self.cache_key) @functools.lru_cache def should_use_selectors_yaml_cache(self) -> bool: """Identify if Cosmos should use/store selectors YAML cache or not.""" - return settings.enable_cache and settings.enable_cache_selectors_yaml and bool(self.selectors_yaml_cache_key) + return settings.enable_cache and settings.enable_cache_selectors_yaml and bool(self.cache_key) def load_via_dbt_ls_cache(self) -> bool: """(Try to) load dbt ls cache from an Airflow Variable""" - logger.info(f"Trying to parse the dbt project using dbt ls cache {self.dbt_ls_cache_key}...") + logger.info(f"Trying to parse the dbt project using dbt ls cache {self.cache_key}...") if self.should_use_dbt_ls_cache(): project_path = self.project_path cache_dict = self.get_dbt_ls_cache() if not cache_dict: - logger.info(f"Cosmos performance: Cache miss for {self.dbt_ls_cache_key}") + logger.info(f"Cosmos performance: Cache miss for {self.cache_key}") return False cache_version = cache_dict.get("version") dbt_ls_cache = cache_dict.get("dbt_ls") current_version = cache._calculate_dbt_ls_cache_current_version( - self.dbt_ls_cache_key, project_path, self.dbt_ls_cache_key_args + self.cache_key, project_path, self.dbt_ls_cache_key_args ) if dbt_ls_cache and not cache.was_project_modified(cache_version, current_version): logger.info( - f"Cosmos performance [{platform.node()}|{os.getpid()}]: The cache size for {self.dbt_ls_cache_key} is {len(dbt_ls_cache)}" + f"Cosmos performance [{platform.node()}|{os.getpid()}]: The cache size for {self.cache_key} is {len(dbt_ls_cache)}" ) self.load_method = LoadMode.DBT_LS_CACHE nodes = parse_dbt_ls_output(project_path=project_path, ls_stdout=dbt_ls_cache) self.nodes = nodes self.filtered_nodes = nodes - logger.info(f"Cosmos performance: Cache hit for {self.dbt_ls_cache_key} - {current_version}") + logger.info(f"Cosmos performance: Cache hit for {self.cache_key} - {current_version}") return True - logger.info(f"Cosmos performance: Cache miss for {self.dbt_ls_cache_key} - skipped") + logger.info(f"Cosmos performance: Cache miss for {self.cache_key} - skipped") return False def should_use_partial_parse_cache(self) -> bool: @@ -917,7 +915,7 @@ def load_via_custom_parser(self) -> None: def _get_selectors_yaml_remote_cache(self, remote_cache_dir: Path | ObjectStoragePath) -> dict[str, Any]: """Loads the remote cache for selectors yaml.""" cache_dict: dict[str, Any] = {} - remote_cache_key_path = remote_cache_dir / self.selectors_yaml_cache_key / "selectors_yaml_cache.json" + remote_cache_key_path = remote_cache_dir / self.cache_key / "selectors_yaml_cache.json" if remote_cache_key_path.exists(): with remote_cache_key_path.open("r") as fp: cache_dict = json.load(fp) @@ -949,7 +947,7 @@ def get_selectors_yaml_cache(self) -> dict[str, Any]: cache_dict = ( self._get_selectors_yaml_remote_cache(remote_cache_dir) if remote_cache_dir - else Variable.get(self.selectors_yaml_cache_key, deserialize_json=True) + else Variable.get(self.cache_key, deserialize_json=True) ) except tuple(airflow_variable_exceptions): return cache_dict @@ -982,7 +980,7 @@ def save_selectors_yaml_cache(self, selections: dict[str, Any]) -> None: cache_dict = { "version": cache._calculate_selectors_yaml_cache_current_version( - self.selectors_yaml_cache_key, self.project_path, self.project.selectors_yaml_path + self.cache_key, self.project_path, self.project.selectors_yaml_path ), "selectors": selections_compressed, "last_modified": datetime.datetime.now(datetime.timezone.utc).isoformat(), @@ -990,11 +988,11 @@ def save_selectors_yaml_cache(self, selections: dict[str, Any]) -> None: } remote_cache_dir = _configure_remote_cache_dir() if remote_cache_dir: - remote_cache_key_path = remote_cache_dir / self.selectors_yaml_cache_key / "selectors_yaml_cache.json" + remote_cache_key_path = remote_cache_dir / self.cache_key / "selectors_yaml_cache.json" with remote_cache_key_path.open("w") as fp: json.dump(cache_dict, fp) else: - Variable.set(self.selectors_yaml_cache_key, cache_dict, serialize_json=True) + Variable.set(self.cache_key, cache_dict, serialize_json=True) def parse_selectors_yaml(self) -> dict[str, Any]: """ @@ -1022,7 +1020,7 @@ def load_parsed_selectors(self) -> dict[str, Any]: Returns: A dictionary containing the selections for the specified selector. """ - logger.info(f"Trying to parse the dbt selector yamls using {self.selectors_yaml_cache_key}...") + logger.info(f"Trying to parse the dbt selector yamls using {self.cache_key}...") if TYPE_CHECKING: assert self.project.selectors_yaml_path is not None # pragma: no cover @@ -1038,7 +1036,7 @@ def get_selections() -> dict[str, Any]: if self.should_use_selectors_yaml_cache(): cache_dict = self.get_selectors_yaml_cache() if not cache_dict: - logger.info(f"Cosmos performance: Cache miss for {self.selectors_yaml_cache_key}") + logger.info(f"Cosmos performance: Cache miss for {self.cache_key}") cache_dict = get_selections() @@ -1046,18 +1044,18 @@ def get_selections() -> dict[str, Any]: selectors_cache: dict[str, Any] = cache_dict.get("selectors", {}) current_version = cache._calculate_selectors_yaml_cache_current_version( - self.selectors_yaml_cache_key, self.project_path, self.project.selectors_yaml_path + self.cache_key, self.project_path, self.project.selectors_yaml_path ) if selectors_cache and not cache.was_project_modified(cache_version, current_version): logger.info( - f"Cosmos performance [{platform.node()}|{os.getpid()}]: The cache size for {self.selectors_yaml_cache_key} is {len(selectors_cache)}" + f"Cosmos performance [{platform.node()}|{os.getpid()}]: The cache size for {self.cache_key} is {len(selectors_cache)}" ) - logger.info(f"Cosmos performance: Cache hit for {self.selectors_yaml_cache_key} - {current_version}") + logger.info(f"Cosmos performance: Cache hit for {self.cache_key} - {current_version}") return selectors_cache - logger.info(f"Cosmos performance: Cache miss for {self.selectors_yaml_cache_key} - skipped") + logger.info(f"Cosmos performance: Cache miss for {self.cache_key} - skipped") return get_selections() diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index 7849bccaf5..91ee465885 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -2056,7 +2056,7 @@ def test_save_dbt_ls_cache_remote_cache_dir( dbt_graph.save_dbt_ls_cache(dbt_ls_output) - mock_remote_cache_key_path = mock_remote_cache_dir_path / dbt_graph.dbt_ls_cache_key / "dbt_ls_cache.json" + mock_remote_cache_key_path = mock_remote_cache_dir_path / dbt_graph.cache_key / "dbt_ls_cache.json" mock_remote_cache_key_path.open.assert_called_once_with("w") From f609ada1a40162a4fcd71e781ebbc0d5f985ec27 Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Thu, 8 Jan 2026 12:25:21 -0500 Subject: [PATCH 07/24] Use proper function to check for selectors yaml cache invalidation --- cosmos/cache.py | 9 +++++++++ cosmos/dbt/graph.py | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/cosmos/cache.py b/cosmos/cache.py index 431de4e945..7da1c9ae10 100644 --- a/cosmos/cache.py +++ b/cosmos/cache.py @@ -350,6 +350,15 @@ def was_project_modified(previous_version: str, current_version: str) -> bool: return previous_version != current_version +@functools.lru_cache +def was_selectors_yaml_modified(previous_version: str, current_version: str) -> bool: + """ + Given the cache version of a project's selectors.yaml and the latest version + of the project's selectors.yaml, decides if the selectors.yaml was modified or not. + """ + return previous_version != current_version + + @provide_session def delete_unused_dbt_ls_cache( max_age_last_usage: timedelta = timedelta(days=30), session: Session | None = None diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index 9d00e27368..3caed7280b 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -1040,14 +1040,14 @@ def get_selections() -> dict[str, Any]: cache_dict = get_selections() - cache_version = cache_dict.get("version", "") + cache_version: tuple[str, str] = cache_dict.get("version", ()) selectors_cache: dict[str, Any] = cache_dict.get("selectors", {}) current_version = cache._calculate_selectors_yaml_cache_current_version( self.cache_key, self.project_path, self.project.selectors_yaml_path ) - if selectors_cache and not cache.was_project_modified(cache_version, current_version): + if selectors_cache and not cache.was_selectors_yaml_modified(cache_version, current_version): logger.info( f"Cosmos performance [{platform.node()}|{os.getpid()}]: The cache size for {self.cache_key} is {len(selectors_cache)}" ) From f88b344d337458b29ee41e821317a8c195a53541 Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Thu, 8 Jan 2026 14:01:38 -0500 Subject: [PATCH 08/24] Update comments and docstrings --- cosmos/dbt/selector.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cosmos/dbt/selector.py b/cosmos/dbt/selector.py index ae6eba233e..8426213c8f 100644 --- a/cosmos/dbt/selector.py +++ b/cosmos/dbt/selector.py @@ -694,6 +694,7 @@ def _resolve_selector_reference( ) -> tuple[list[str], list[str]]: """ Resolve a selector reference recursively into select/exclude dbt command-line syntax. + Args: selector_name (str): The name of the selector to resolve. all_selector_definitions (dict): All selector definitions for reference resolution. @@ -738,6 +739,7 @@ def _evaluate_selector_method_definition( ) -> str: """ Evaluate a selector method definition into dbt command-line syntax. + Args: method (str): The selector method (e.g., "tag", "path", "source", etc.). value (str): The value associated with the method. @@ -774,6 +776,7 @@ def _evaluate_selector_method_definition_modifiers( ) -> str: """ Apply graph operator modifiers to a selector string. + This does NOT support the indirect_selection keyword. Args: selector (str): The base selector string. From 389d5e11d9f091268bf86e1c35b6e982af6f73e1 Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Fri, 9 Jan 2026 10:31:52 -0500 Subject: [PATCH 09/24] Update comments and docstrings --- cosmos/dbt/graph.py | 2 +- cosmos/dbt/selector.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index 3caed7280b..44e928d66d 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -1015,7 +1015,7 @@ def parse_selectors_yaml(self) -> dict[str, Any]: def load_parsed_selectors(self) -> dict[str, Any]: """ - Parse the selectors YAML file and return the selections for the specified selector. + Load the parsed selectors from cache if available, otherwise parse the selectors YAML file. Returns: A dictionary containing the selections for the specified selector. diff --git a/cosmos/dbt/selector.py b/cosmos/dbt/selector.py index 8426213c8f..a7f19143ec 100644 --- a/cosmos/dbt/selector.py +++ b/cosmos/dbt/selector.py @@ -921,6 +921,7 @@ def _parse_selector_definition( _validate_selector_definition(selectors) + # Handle method + value if "method" in selectors: method_select, method_exclude = _parse_selector_method_definition( selectors, From 195eb0c3f86e7fe0f5324be7245b19943bec7fb7 Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Thu, 15 Jan 2026 14:06:41 -0500 Subject: [PATCH 10/24] Refactor and reorganize yaml selectors logic --- cosmos/cache.py | 16 +- cosmos/config.py | 38 +- cosmos/dbt/graph.py | 123 +-- cosmos/dbt/selector.py | 796 +++++++++++------- cosmos/settings.py | 2 +- .../cosmos_manifest_yaml_selectors_example.py | 20 +- 6 files changed, 598 insertions(+), 397 deletions(-) diff --git a/cosmos/cache.py b/cosmos/cache.py index 7da1c9ae10..ed161e5260 100644 --- a/cosmos/cache.py +++ b/cosmos/cache.py @@ -287,8 +287,8 @@ def _copy_partial_parse_to_project(partial_parse_filepath: Path, project_path: P shutil.copy(str(source_manifest_filepath), str(target_manifest_filepath)) -def _calculate_selectors_yaml_cache_current_version( - cache_identifier: str, project_dir: Path, selectors_yaml_path: Path | ObjectStoragePath +def _calculate_yaml_selectors_cache_current_version( + cache_identifier: str, project_dir: Path, yaml_selectors_path: Path | ObjectStoragePath ) -> str: """ Taking into account the project directory contents and the selectors yaml file, calculate the @@ -296,7 +296,7 @@ def _calculate_selectors_yaml_cache_current_version( :param cache_identifier: Unique identifier of the cache (may include DbtDag or DbtTaskGroup information) :param project_path: Path to the target dbt project directory - :param selectors_yaml_path: Path to the selectors yaml file + :param yaml_selectors_path: Path to the selectors yaml file """ start_time = time.perf_counter() @@ -305,15 +305,15 @@ def _calculate_selectors_yaml_cache_current_version( # This is fast (e.g. 0.01s for jaffle shop, 0.135s for a 5k models dbt folder) dbt_project_hash = _create_folder_version_hash(project_dir) - with selectors_yaml_path.open("r") as f: - selectors_yaml_content = yaml.safe_load(f) - hash_selectors_yaml = hashlib.md5(yaml.dump(selectors_yaml_content).encode()).hexdigest() + with yaml_selectors_path.open("r") as f: + yaml_selector_content = yaml.safe_load(f) + yaml_selector_hash = hashlib.md5(yaml.dump(yaml_selector_content).encode()).hexdigest() elapsed_time = time.perf_counter() - start_time logger.info( f"Cosmos performance: time to calculate cache identifier {cache_identifier} for current version: {elapsed_time}" ) - return f"{dbt_project_hash},{hash_selectors_yaml}" + return f"{dbt_project_hash},{yaml_selector_hash}" def _calculate_dbt_ls_cache_current_version(cache_identifier: str, project_dir: Path, cmd_args: list[str]) -> str: @@ -351,7 +351,7 @@ def was_project_modified(previous_version: str, current_version: str) -> bool: @functools.lru_cache -def was_selectors_yaml_modified(previous_version: str, current_version: str) -> bool: +def were_yaml_selectors_modified(previous_version: str, current_version: str) -> bool: """ Given the cache version of a project's selectors.yaml and the latest version of the project's selectors.yaml, decides if the selectors.yaml was modified or not. diff --git a/cosmos/config.py b/cosmos/config.py index 45947e3d6b..ef353f8f8a 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -187,7 +187,7 @@ class ProjectConfig: install_dbt_deps: bool = True copy_dbt_packages: bool = settings.default_copy_dbt_packages manifest_path: Path | ObjectStoragePath | None = None - selectors_yaml_path: Path | ObjectStoragePath | None = None + yaml_selectors_path: Path | ObjectStoragePath | None = None models_path: Path | None = None seeds_path: Path | None = None snapshots_path: Path | None = None @@ -203,8 +203,8 @@ def __init__( snapshots_relative_path: str | Path = "snapshots", manifest_path: str | Path | None = None, manifest_conn_id: str | None = None, - selectors_yaml_path: str | Path | None = None, - selectors_yaml_conn_id: str | None = None, + yaml_selectors_path: str | Path | None = None, + yaml_selectors_conn_id: str | None = None, project_name: str | None = None, env_vars: dict[str, str] | None = None, dbt_vars: dict[str, str] | None = None, @@ -228,7 +228,7 @@ def __init__( manifest_path, ) self.validate_manifest_path(manifest_path, manifest_conn_id) - self.validate_selectors_yaml_path(selectors_yaml_path, selectors_yaml_conn_id) + self.validate_yaml_selectors_path(yaml_selectors_path, yaml_selectors_conn_id) def validate_dbt_project_paths( self, @@ -279,31 +279,31 @@ def validate_manifest_path( else: self.manifest_path = Path(manifest_path_str) - def validate_selectors_yaml_path( + def validate_yaml_selectors_path( self, - selectors_yaml_path: str | Path | None, - selectors_yaml_conn_id: str | None, + yaml_selectors_path: str | Path | None, + yaml_selectors_conn_id: str | None, ) -> None: - if selectors_yaml_path: - selectors_yaml_path_str = str(selectors_yaml_path) - if not selectors_yaml_conn_id: - selectors_yaml_scheme = selectors_yaml_path_str.split("://")[0] + if yaml_selectors_path: + yaml_selectors_path_str = str(yaml_selectors_path) + if not yaml_selectors_conn_id: + yaml_selectors_scheme = yaml_selectors_path_str.split("://")[0] # Use the default Airflow connection ID for the scheme if it is not provided. - selectors_yaml_conn_id = FILE_SCHEME_AIRFLOW_DEFAULT_CONN_ID_MAP.get( - selectors_yaml_scheme, lambda: None + yaml_selectors_conn_id = FILE_SCHEME_AIRFLOW_DEFAULT_CONN_ID_MAP.get( + yaml_selectors_scheme, lambda: None )() - if selectors_yaml_conn_id is not None and not settings.AIRFLOW_IO_AVAILABLE: + if yaml_selectors_conn_id is not None and not settings.AIRFLOW_IO_AVAILABLE: raise CosmosValueError( - f"The selectors yaml path {selectors_yaml_path_str} uses a remote file scheme, but the required Object " + f"The selectors yaml path {yaml_selectors_path_str} uses a remote file scheme, but the required Object " f"Storage feature is unavailable in Airflow version {airflow_version}. Please upgrade to " f"Airflow 2.8 or later." ) if settings.AIRFLOW_IO_AVAILABLE: - self.selectors_yaml_path = ObjectStoragePath(selectors_yaml_path_str, conn_id=selectors_yaml_conn_id) + self.yaml_selectors_path = ObjectStoragePath(yaml_selectors_path_str, conn_id=yaml_selectors_conn_id) else: - self.selectors_yaml_path = Path(selectors_yaml_path_str) + self.yaml_selectors_path = Path(yaml_selectors_path_str) def validate_project(self) -> None: """ @@ -343,11 +343,11 @@ def is_manifest_available(self) -> bool: """ return self.manifest_path.exists() if self.manifest_path else False - def is_selectors_yaml_available(self) -> bool: + def is_yaml_selectors_available(self) -> bool: """ Check if the `dbt` selectors YAML file is set and if the file exists. """ - return self.selectors_yaml_path.exists() if self.selectors_yaml_path else False + return self.yaml_selectors_path.exists() if self.yaml_selectors_path else False @dataclass diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index 44e928d66d..03ceacbf33 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -6,6 +6,7 @@ import itertools import json import os +import pickle import platform import tempfile import warnings @@ -58,7 +59,7 @@ get_partial_parse_path, has_non_empty_dependencies_file, ) -from cosmos.dbt.selector import parse_yaml_selectors, select_nodes +from cosmos.dbt.selector import YamlSelectors, select_nodes from cosmos.log import get_logger logger = get_logger(__name__) @@ -640,9 +641,9 @@ def should_use_dbt_ls_cache(self) -> bool: return settings.enable_cache and settings.enable_cache_dbt_ls and bool(self.cache_key) @functools.lru_cache - def should_use_selectors_yaml_cache(self) -> bool: - """Identify if Cosmos should use/store selectors YAML cache or not.""" - return settings.enable_cache and settings.enable_cache_selectors_yaml and bool(self.cache_key) + def should_use_yaml_selectors_cache(self) -> bool: + """Identify if Cosmos should use/store YAML selectors cache or not.""" + return settings.enable_cache and settings.enable_cache_yaml_selectors and bool(self.cache_key) def load_via_dbt_ls_cache(self) -> bool: """(Try to) load dbt ls cache from an Airflow Variable""" @@ -912,23 +913,23 @@ def load_via_custom_parser(self) -> None: exclude=self.render_config.exclude, ) - def _get_selectors_yaml_remote_cache(self, remote_cache_dir: Path | ObjectStoragePath) -> dict[str, Any]: - """Loads the remote cache for selectors yaml.""" + def _get_yaml_selectors_remote_cache(self, remote_cache_dir: Path | ObjectStoragePath) -> dict[str, Any]: + """Loads the remote cache for the yaml selectors.""" cache_dict: dict[str, Any] = {} - remote_cache_key_path = remote_cache_dir / self.cache_key / "selectors_yaml_cache.json" + remote_cache_key_path = remote_cache_dir / self.cache_key / "yaml_selectors_cache.json" if remote_cache_key_path.exists(): with remote_cache_key_path.open("r") as fp: cache_dict = json.load(fp) return cache_dict - def get_selectors_yaml_cache(self) -> dict[str, Any]: + def get_yaml_selector_cache(self) -> dict[str, Any]: """ - Retrieve previously saved selectors YAML cache from an Airflow Variable. + Retrieve previously saved YAML selectors from an Airflow Variable. Outputs: { "version": "cache-version", - "selectors": uncompressed selectors dictionary, + "yaml_selectors": uncompressed YamlSelectors instance, "last_modified": "Isoformat timestamp" } """ @@ -945,119 +946,121 @@ def get_selectors_yaml_cache(self) -> dict[str, Any]: try: remote_cache_dir = _configure_remote_cache_dir() cache_dict = ( - self._get_selectors_yaml_remote_cache(remote_cache_dir) + self._get_yaml_selectors_remote_cache(remote_cache_dir) if remote_cache_dir else Variable.get(self.cache_key, deserialize_json=True) ) except tuple(airflow_variable_exceptions): return cache_dict else: - selectors_compressed = cache_dict.pop("selectors", None) + selectors_compressed = cache_dict.pop("yaml_selectors", None) if selectors_compressed: - encoded_data = base64.b64decode(selectors_compressed.encode()) - decompressed_data = zlib.decompress(encoded_data).decode("utf-8") - cache_dict["selectors"] = json.loads(decompressed_data) + encoded_data = base64.b64decode(selectors_compressed.encode("utf-8")) + decompressed_data = zlib.decompress(encoded_data) + cache_dict["yaml_selectors"] = pickle.loads(decompressed_data) return cache_dict - def save_selectors_yaml_cache(self, selections: dict[str, Any]) -> None: + def save_yaml_selector_cache(self, yaml_selectors: YamlSelectors) -> None: """ - Store parsed selectors YAML into an Airflow Variable. + Store parsed YAML selectors into an Airflow Variable. Stores: { "version": "cache-version", - "selectors": compressed_selectors, + "yaml_selectors": compressed YamlSelectors instance, "last_modified": "Isoformat timestamp" } """ if TYPE_CHECKING: - assert self.project.selectors_yaml_path is not None # pragma: no cover + assert self.project.yaml_selectors_path is not None # pragma: no cover - compressed_data = zlib.compress(json.dumps(selections).encode("utf-8")) + serialized_data = pickle.dumps(yaml_selectors) + compressed_data = zlib.compress(serialized_data) encoded_data = base64.b64encode(compressed_data) selections_compressed = encoded_data.decode("utf-8") cache_dict = { - "version": cache._calculate_selectors_yaml_cache_current_version( - self.cache_key, self.project_path, self.project.selectors_yaml_path + "version": cache._calculate_yaml_selectors_cache_current_version( + self.cache_key, self.project_path, self.project.yaml_selectors_path ), - "selectors": selections_compressed, + "yaml_selectors": selections_compressed, "last_modified": datetime.datetime.now(datetime.timezone.utc).isoformat(), **self.airflow_metadata, } remote_cache_dir = _configure_remote_cache_dir() if remote_cache_dir: - remote_cache_key_path = remote_cache_dir / self.cache_key / "selectors_yaml_cache.json" + remote_cache_key_path = remote_cache_dir / self.cache_key / "yaml_selector_cache.json" with remote_cache_key_path.open("w") as fp: json.dump(cache_dict, fp) else: Variable.set(self.cache_key, cache_dict, serialize_json=True) - def parse_selectors_yaml(self) -> dict[str, Any]: + def parse_yaml_selectors(self) -> YamlSelectors: """ - Parse the selectors YAML file and return the selections for all selectors. + Parse the YAML selectors file into a YamlSelctor Returns: - A dictionary containing all selectors and their selections. + A YamlSelectors instance """ if TYPE_CHECKING: - assert self.project.selectors_yaml_path is not None # pragma: no cover + assert self.project.yaml_selectors_path is not None # pragma: no cover - with self.project.selectors_yaml_path.open() as fp: - selectors = yaml.safe_load(fp) - selections = parse_yaml_selectors(selectors) + with self.project.yaml_selectors_path.open() as fp: + yaml_selectors_definition = yaml.safe_load(fp) + yaml_selectors = YamlSelectors.parse(yaml_selectors_definition) - if self.should_use_selectors_yaml_cache(): - self.save_selectors_yaml_cache(selections) + if self.should_use_yaml_selectors_cache(): + self.save_yaml_selector_cache(yaml_selectors) - return selections + return yaml_selectors - def load_parsed_selectors(self) -> dict[str, Any]: + def load_parsed_selectors(self) -> YamlSelectors: """ - Load the parsed selectors from cache if available, otherwise parse the selectors YAML file. + Load the YamlSelectors from cache if available, otherwise parse the YAML selectors file. Returns: - A dictionary containing the selections for the specified selector. + A dictionary containing the selections for the specified selectors. """ - logger.info(f"Trying to parse the dbt selector yamls using {self.cache_key}...") + logger.info(f"Trying to parse the dbt yaml selectors using {self.cache_key}...") if TYPE_CHECKING: - assert self.project.selectors_yaml_path is not None # pragma: no cover + assert self.project.yaml_selectors_path is not None # pragma: no cover - def get_selections() -> dict[str, Any]: - selections = self.parse_selectors_yaml() + def get_yaml_selectors() -> YamlSelectors: + yaml_selectors = self.parse_yaml_selectors() - if self.should_use_selectors_yaml_cache(): - self.save_selectors_yaml_cache(selections) + if self.should_use_yaml_selectors_cache(): + self.save_yaml_selector_cache(yaml_selectors) - return selections + return yaml_selectors + + if self.should_use_yaml_selectors_cache(): + cache_dict = self.get_yaml_selector_cache() - if self.should_use_selectors_yaml_cache(): - cache_dict = self.get_selectors_yaml_cache() if not cache_dict: logger.info(f"Cosmos performance: Cache miss for {self.cache_key}") - cache_dict = get_selections() + return get_yaml_selectors() - cache_version: tuple[str, str] = cache_dict.get("version", ()) - selectors_cache: dict[str, Any] = cache_dict.get("selectors", {}) + cache_version: str = cache_dict["version"] + yaml_selectors: YamlSelectors = cache_dict["yaml_selectors"] - current_version = cache._calculate_selectors_yaml_cache_current_version( - self.cache_key, self.project_path, self.project.selectors_yaml_path + current_version = cache._calculate_yaml_selectors_cache_current_version( + self.cache_key, self.project_path, self.project.yaml_selectors_path ) - if selectors_cache and not cache.was_selectors_yaml_modified(cache_version, current_version): + if cache_dict and not cache.were_yaml_selectors_modified(cache_version, current_version): logger.info( - f"Cosmos performance [{platform.node()}|{os.getpid()}]: The cache size for {self.cache_key} is {len(selectors_cache)}" + f"Cosmos performance [{platform.node()}|{os.getpid()}]: The cache size for {self.cache_key} is {len(yaml_selectors.parsed)}" ) logger.info(f"Cosmos performance: Cache hit for {self.cache_key} - {current_version}") - return selectors_cache + return yaml_selectors logger.info(f"Cosmos performance: Cache miss for {self.cache_key} - skipped") - return get_selections() + return get_yaml_selectors() def load_from_dbt_manifest(self) -> None: """ @@ -1073,8 +1076,8 @@ def load_from_dbt_manifest(self) -> None: self.load_method = LoadMode.DBT_MANIFEST logger.info("Trying to parse the dbt project `%s` using a dbt manifest...", self.project.project_name) - if self.render_config.selector and not self.project.is_selectors_yaml_available(): - raise CosmosLoadDbtException(f"Unable to load selectors using {self.project.selectors_yaml_path}") + if self.render_config.selector and not self.project.is_yaml_selectors_available(): + raise CosmosLoadDbtException(f"Unable to load yaml selectors using {self.project.yaml_selectors_path}") if not self.project.is_manifest_available(): raise CosmosLoadDbtException(f"Unable to load manifest using {self.project.manifest_path}") @@ -1110,12 +1113,12 @@ def load_from_dbt_manifest(self) -> None: nodes[node.unique_id] = node if self.render_config.selector: - selectors = self.load_parsed_selectors() - selections = selectors.get(self.render_config.selector) + yaml_selectors = self.load_parsed_selectors() + selections = yaml_selectors.get_parsed(self.render_config.selector) if not selections: raise CosmosLoadDbtException( - f"Selector `{self.render_config.selector}` not found in selectors YAML `{self.project.selectors_yaml_path}`" + f"Selector `{self.render_config.selector}` not found in parsed YAML selectors `{self.project.yaml_selectors_path}`" ) self.nodes = nodes diff --git a/cosmos/dbt/selector.py b/cosmos/dbt/selector.py index a7f19143ec..cf18bf9838 100644 --- a/cosmos/dbt/selector.py +++ b/cosmos/dbt/selector.py @@ -7,6 +7,8 @@ from pathlib import Path from typing import TYPE_CHECKING, Any +import yaml + from cosmos.constants import DbtResourceType from cosmos.exceptions import CosmosValueError from cosmos.log import get_logger @@ -683,350 +685,546 @@ def retrieve_by_label(statement_list: list[str], label: str) -> set[str]: return label_values -# YAML Selectors +class YamlSelectors: + """ + Parses and manages dbt YAML selector definitions. + + This class handles the parsing of dbt selector YAML files, converting them from their raw + YAML format into a format compatible with Cosmos node selection. It supports union, + intersection, and method-based selector definitions as specified in dbt's selector syntax. + Instances of this class should be created using the `parse` class method. -def _resolve_selector_reference( - selector_name: str, - all_selector_definitions: dict[str, dict[str, Any]], - resolved_cache: dict[str, tuple[list[str], list[str]]], - visiting: set[str], -) -> tuple[list[str], list[str]]: - """ - Resolve a selector reference recursively into select/exclude dbt command-line syntax. - - Args: - selector_name (str): The name of the selector to resolve. - all_selector_definitions (dict): All selector definitions for reference resolution. - resolved_cache (dict): Cache for already resolved selectors. - visiting (set): Set of currently visiting selectors to detect circular references. - Returns: - tuple: A tuple containing two lists - select parts and exclude parts. - Raises: - CosmosValueError: If a circular reference is detected or if the selector is not found. + :param raw_selectors: dict[str, list[dict[str, Any]]] - The original, unparsed selector definitions from the YAML file + :param parsed_selectors: dict[str, dict[str, Any]] - The processed selector definitions converted to Cosmos format + + :property raw: dict[str, list[dict[str, Any]]] + The original unparsed selector definitions from the YAML file. + :property parsed: dict[str, dict[str, Any]] + The parsed selector definitions in Cosmos format. + :property dbt_spec_version: int + The dbt selector specification version being used. + + References: + https://docs.getdbt.com/reference/node-selection/yaml-selectors """ - # Check cache first - if selector_name in resolved_cache: - return resolved_cache[selector_name] - # Check for circular reference - if selector_name in visiting: - raise CosmosValueError(f"Circular reference detected in selector '{selector_name}'") + def __init__(self, raw_selectors: dict[str, list[dict[str, Any]]], parsed_selectors: dict[str, dict[str, Any]]): + self._raw = raw_selectors + self._parsed = parsed_selectors + self._dbt_spec_version = 2 - # Get the selector definition - if selector_name not in all_selector_definitions: - raise CosmosValueError(f"Selector '{selector_name}' not found") + @property + def raw(self) -> dict[str, list[dict[str, Any]]]: + """ + Get the original unparsed selector definitions. - # Mark as visiting - visiting.add(selector_name) + :return: dict[str, list[dict[str, Any]]] - Dictionary containing the raw YAML structure + """ + return self._raw - # Recursively resolve the referenced selector - selector_def = all_selector_definitions[selector_name] - result = _parse_selector_definition(selector_def, all_selector_definitions, resolved_cache, visiting) + @property + def parsed(self) -> dict[str, dict[str, Any]]: + """ + Get the parsed selector definitions in Cosmos format. - # Remove from visiting - visiting.discard(selector_name) + :return: dict[str, dict[str, Any]] - Dictionary mapping selector names to their parsed definitions with select/exclude lists + """ + return self._parsed - # Cache the result - resolved_cache[selector_name] = result + @property + def dbt_spec_version(self) -> int: + """ + Get the dbt selector specification version being used. - return result + :return: int - The dbt spec version + """ + return self._dbt_spec_version + def get_raw(self, selector_name: str, default: Any = None) -> dict[str, Any] | Any: + """ + Retrieve a raw selector definition by name. -def _evaluate_selector_method_definition( - method: str, - value: str, -) -> str: - """ - Evaluate a selector method definition into dbt command-line syntax. - - Args: - method (str): The selector method (e.g., "tag", "path", "source", etc.). - value (str): The value associated with the method. - Returns: - str: The dbt command-line syntax for the selector. - Raises: - CosmosValueError: If the method is unsupported. - """ - if method.startswith(TAG_SELECTOR[:-1]): - return f"{TAG_SELECTOR}{value}" - elif method.startswith(PATH_SELECTOR[:-1]): - return f"{PATH_SELECTOR}{value}" - elif method.startswith(SOURCE_SELECTOR[:-1]): - return f"{SOURCE_SELECTOR}{value}" - elif method.startswith(EXPOSURE_SELECTOR[:-1]): - return f"{EXPOSURE_SELECTOR}{value}" - elif method.startswith(RESOURCE_TYPE_SELECTOR[:-1]): - return f"{RESOURCE_TYPE_SELECTOR}{value}" - elif method.startswith(EXCLUDE_RESOURCE_TYPE_SELECTOR[:-1]): - return f"{EXCLUDE_RESOURCE_TYPE_SELECTOR}{value}" - elif any([method.startswith(CONFIG_SELECTOR + config) for config in SUPPORTED_CONFIG]): - return f"{CONFIG_SELECTOR}:{value}" - elif method == "fqn": - if value == "*": - return "" - else: - return value - else: - raise CosmosValueError(f"Unsupported selector method: '{method}'") + :param selector_name: str - Name of the selector to retrieve + :param default: Any - Default value to return if selector is not found + :return: dict[str, Any] | Any - The raw selector definition or the default value + """ + return self.raw.get(selector_name, default) + def get_parsed(self, selector_name: str, default: Any = None) -> dict[str, Any] | Any: + """ + Retrieve a parsed selector definition by name. -def _evaluate_selector_method_definition_modifiers( - selector: str, parents: bool, children: bool, parents_depth: int, children_depth: int, childrens_parents: bool -) -> str: - """ - Apply graph operator modifiers to a selector string. - This does NOT support the indirect_selection keyword. - - Args: - selector (str): The base selector string. - parents (bool): Whether to include parents. - children (bool): Whether to include children. - parents_depth (int): Depth of parents to include. - children_depth (int): Depth of children to include. - childrens_parents (bool): Whether to include children's parents. - Returns: - str: The modified selector string with graph operators applied. - """ - if parents: - if parents_depth and parents_depth > 0: - selector = f"{parents_depth}+{selector}" - else: - selector = f"+{selector}" - if children: - if children_depth and children_depth > 0: - selector = f"{selector}+{children_depth}" - else: - selector = f"{selector}+" - if childrens_parents: - if parents_depth or children_depth: + :param selector_name: str - Name of the selector to retrieve + :param default: Any - Default value to return if selector is not found + :return: dict[str, Any] | Any - The parsed selector definition or the default value + """ + return self.parsed.get(selector_name, default) + + @staticmethod + def _get_list_dicts(dct: dict[str, Any], key: str) -> list[dict[str, Any]]: + """ + Extract and validate a list of dictionaries from a given key in a dictionary. + + This helper method retrieves a value from a dictionary and ensures it's a list containing + either dictionaries with string keys or string values. + + :param dct: dict[str, Any] - The dictionary to extract from + :param key: str - The key to look up in the dictionary + :return: list[dict[str, Any]] - List of dictionaries extracted from the specified key + :raises CosmosValueError: If the key is missing, the value is not a list, or contains invalid types + """ + result: list[dict[str, Any]] = [] + if key not in dct: + raise CosmosValueError(f"Expected to find key {key} in dict, only found {list(dct)}") + values = dct[key] + if not isinstance(values, list): + raise CosmosValueError(f'Invalid value for key "{key}". Expected a list.') + for value in values: + if isinstance(value, dict): + for value_key in value: + if not isinstance(value_key, str): + raise CosmosValueError( + f'Expected all keys to "{key}" dict to be strings, ' + f'but "{value_key}" is a "{type(value_key)}"' + ) + result.append(value) + else: + raise CosmosValueError( + f'Invalid value type {type(value)} in key "{key}", expected ' f"dict or str (value: {value})." + ) + + return result + + @staticmethod + def _parse_selection_graph_operators( + selector: str, parents: bool, children: bool, parents_depth: int, children_depth: int, childrens_parents: bool + ) -> str: + """ + Apply graph operator syntax to a selector string. + + Transforms a selector by adding graph operators (+, @) based on the specified parameters. + These operators control traversal of the DAG to include parents, children, or both. + + :param selector: str - The base selector string to modify + :param parents: bool - Whether to include parent nodes + :param children: bool - Whether to include child nodes + :param parents_depth: int - Number of parent generations to include (0 for all) + :param children_depth: int - Number of child generations to include (0 for all) + :param childrens_parents: bool - Whether to use the @ operator (all ancestors of children) + :return: str - The selector string with graph operators applied + :raises CosmosValueError: If childrens_parents is combined with depth parameters + + Examples: + - selector="my_model", parents=True, children=False -> "+my_model" + - selector="my_model", parents=True, children=True, parents_depth=2 -> "2+my_model+" + - selector="my_model", childrens_parents=True -> "@my_model" + """ + if not selector: + return selector + + if childrens_parents and (parents_depth or children_depth): raise CosmosValueError("childrens_parents cannot be combined with parents_depth or children_depth.") - else: - selector = f"@{selector}" - return selector + if childrens_parents: + return f"@{selector}" -def _parse_selector_method_definition( - method_def: dict[str, Any], - all_selector_definitions: dict[str, dict[str, Any]], - resolved_cache: dict[str, tuple[list[str], list[str]]], - visiting: set[str], -) -> tuple[list[str], list[str]]: - """ - Parse a single selector method definition into select/exclude dbt command-line syntax. - - Args: - method_def (dict): The selector method definition to parse. - all_selector_definitions (dict): All selector definitions for reference resolution. - resolved_cache (dict): Cache for already resolved selectors. - visiting (set): Set of currently visiting selectors to detect circular references. - Returns: - tuple: A tuple containing two lists - select parts and exclude parts. - """ - method = method_def.get("method", "METHOD_KEY_NOT_FOUND") - value = method_def.get("value", "VALUE_KEY_NOT_FOUND") - parents = method_def.get("parents", False) - children = method_def.get("children", False) - parents_depth = method_def.get("parents_depth", 0) - children_depth = method_def.get("children_depth", 0) - childrens_parents = method_def.get("childrens_parents", False) + if parents: + prefix = f"{parents_depth}+" if parents_depth > 0 else "+" + selector = f"{prefix}{selector}" - if method == "METHOD_KEY_NOT_FOUND": - raise CosmosValueError(f"Selector method is missing in method definition: {method_def}.") - if value == "VALUE_KEY_NOT_FOUND": - raise CosmosValueError(f"Selector value is missing in method definition: {method_def}.") + if children: + suffix = f"+{children_depth}" if children_depth > 0 else "+" + selector = f"{selector}{suffix}" - if method == "selector": - return _resolve_selector_reference(value, all_selector_definitions, resolved_cache, visiting) + return selector - selector = _evaluate_selector_method_definition(method, value) + @staticmethod + def _parse_selection_from_cosmos_spec(method: str, value: str) -> str: + """ + Convert a dbt YAML selector method and value into Cosmos selector syntax. - if not selector: - return ([], []) + Maps dbt's method-based selector definitions to Cosmos's string-based selector format. + For example, method="tag" and value="nightly" becomes "tag:nightly". - selector = _evaluate_selector_method_definition_modifiers( - selector, parents, children, parents_depth, children_depth, childrens_parents - ) + :param method: str - The selector method (e.g., "tag", "path", "config.materialized") + :param value: str - The value for the selector method + :return: str - A Cosmos-formatted selector string + :raises CosmosValueError: If the method is not supported - return ([selector], []) + Examples: + - method="tag", value="nightly" -> "tag:nightly" + - method="path", value="models/" -> "path:models/" + - method="fqn", value="*" -> "" + - method="config.materialized", value="view" -> "config.materialized:view" + """ + if method == "fqn": + return "" if value == "*" else value + + method_mappings = { + TAG_SELECTOR[:-1]: TAG_SELECTOR, + PATH_SELECTOR[:-1]: PATH_SELECTOR, + SOURCE_SELECTOR[:-1]: SOURCE_SELECTOR, + EXPOSURE_SELECTOR[:-1]: EXPOSURE_SELECTOR, + RESOURCE_TYPE_SELECTOR[:-1]: RESOURCE_TYPE_SELECTOR, + EXCLUDE_RESOURCE_TYPE_SELECTOR[:-1]: EXCLUDE_RESOURCE_TYPE_SELECTOR, + } + for method_prefix, selector_prefix in method_mappings.items(): + if method.startswith(method_prefix): + return f"{selector_prefix}{value}" -def _validate_selector_definition(selectors: dict[str, Any]) -> None: - """ - Validate the structure of a selector definition. + if any(method.startswith(f"{CONFIG_SELECTOR}{config}") for config in SUPPORTED_CONFIG): + return f"{CONFIG_SELECTOR}:{value}" - Args: - selectors (dict): The selector definition to validate. - Raises: - CosmosValueError: If the selector definition contains CLI-style string definitions - or key-value definitions. - """ + raise CosmosValueError(f"Unsupported selector method: '{method}'") - # Reject CLI-style string definitions (e.g., definition: "tag:my_tag") - # These should be structured as method-value definitions instead - if isinstance(selectors, str): - raise CosmosValueError( - f"CLI-style string definitions are not supported: '{selectors}'. Use method-value definitions instead." - ) + @classmethod + def _parse_selector(cls, dct: dict[str, Any]) -> str: + """ + Parse a single selector definition dictionary into a Cosmos selector string. + + Processes a selector definition that includes method, value, and optional graph operators + (parents, children, etc.) and converts it to Cosmos selector syntax. + + :param dct: dict[str, Any] - Dictionary containing selector definition with keys like 'method', 'value', + 'parents', 'children', 'parents_depth', 'children_depth', 'childrens_parents' + :return: str - A Cosmos-formatted selector string + :raises CosmosValueError: If required keys are missing or depth values are not integers + + Example Input: + { + "method": "tag", + "value": "nightly", + "children": True, + "children_depth": 2 + } - # Reject key-value definitions (e.g., tag: my_tag, path: models/, etc.) - # These are deprecated in favor of explicit method definitions - # Valid structural keys are: method, union, intersection, exclude, and method modifiers - supported_definitions = { - "method", - "union", - "intersection", - "exclude", - "value", - "parents", - "children", - "parents_depth", - "children_depth", - "childrens_parents", - } - key_value_definitions = set(selectors.keys()) - supported_definitions - - if key_value_definitions: - key_value_strings = ", ".join(f"{k}: {selectors[k]}" for k in key_value_definitions) - raise CosmosValueError( - f"Key-value definitions are not supported: {key_value_strings}. Use method-value definitions instead." + Example Output: + "tag:nightly+2" + """ + method = dct.get("method", "METHOD_KEY_NOT_FOUND") + value = dct.get("value", "VALUE_KEY_NOT_FOUND") + parents = dct.get("parents", False) + children = dct.get("children", False) + parents_depth = dct.get("parents_depth", 0) + children_depth = dct.get("children_depth", 0) + childrens_parents = dct.get("childrens_parents", False) + indirect_selection = dct.get("indirect_selection", False) + + if method == "METHOD_KEY_NOT_FOUND": + raise CosmosValueError(f"Selector method is missing in method definition: {dct}.") + if value == "VALUE_KEY_NOT_FOUND": + raise CosmosValueError(f"Selector value is missing in method definition: {dct}.") + + if not isinstance(parents_depth, int): + raise CosmosValueError(f"parents_depth must be an integer, got {type(parents_depth)}: {parents_depth}") + if not isinstance(children_depth, int): + raise CosmosValueError(f"children_depth must be an integer, got {type(children_depth)}: {children_depth}") + + if indirect_selection: + logger.warning("The 'indirect_selection' parameter is not supported and will be ignored.") + + selector = cls._parse_selection_from_cosmos_spec(method, value) + + selector = cls._parse_selection_graph_operators( + selector, parents, children, parents_depth, children_depth, childrens_parents ) + return selector -def _parse_selector_definition( - selectors: dict[str, Any], - all_selector_definitions: dict[str, dict[str, Any]], - resolved_cache: dict[str, tuple[list[str], list[str]]], - visiting: set[str], -) -> tuple[list[str], list[str]]: - """ - Parse a selector definition into select/exclude dbt command-line syntax. + @classmethod + def _parse_exclusions( + cls, definition: dict[str, Any], cache: dict[str, tuple[list[str], list[str]]] = {} + ) -> list[str]: + """ + Parse exclusion definitions from a selector definition. - Handles union, intersection, exclude, and method definitions. Collects any - parse errors encountered during processing. + Extracts and processes 'exclude' clauses from a selector definition, recursively + parsing any nested definitions. - Args: - selectors (dict): The selector definition to parse. - all_selector_definitions (dict): All selector definitions for reference resolution. - resolved_cache (dict): Cache for already resolved selectors. - visiting (set): Set of currently visiting selectors to detect circular references. - Returns: - tuple: A tuple containing two lists - select parts and exclude parts. - Raises: - CosmosValueError: If there are issues parsing the selector definition. + :param definition: dict[str, Any] - Dictionary containing the selector definition with an 'exclude' key + :param cache: dict[str, tuple[list[str], list[str]]] - Cache of previously parsed selectors for reference resolution + :return: list[str] - List of exclusion selector strings + """ + exclusions = cls._get_list_dicts(definition, "exclude") + exclude: list[str] = [] - """ - select_parts = [] - exclude_parts = [] - - _validate_selector_definition(selectors) - - # Handle method + value - if "method" in selectors: - method_select, method_exclude = _parse_selector_method_definition( - selectors, - all_selector_definitions, - resolved_cache, - visiting, - ) - select_parts.extend(method_select) - exclude_parts.extend(method_exclude) - - # Handle exclude - if "exclude" in selectors: - exclude_items = selectors["exclude"] - for exclude_item in exclude_items: - ex_select, ex_exclude = _parse_selector_definition( - exclude_item, - all_selector_definitions, - resolved_cache, - visiting, + for exclusion in exclusions: + excl, _ = cls._parse_from_definition(exclusion, cache=cache) + exclude.extend(excl) + + return exclude + + @classmethod + def _parse_include_exclude_subdefs( + cls, + definitions: list[dict[str, Any]], + cache: dict[str, tuple[list[str], list[str]]], + ) -> tuple[list[str], list[str]]: + """ + Parse a list of selector subdefinitions into include and exclude lists. + + Processes a list of selector definitions, separating them into inclusions and exclusions. + Handles nested definitions and prevents multiple exclude clauses at the same level. + + :param definitions: list[dict[str, Any]] - List of selector definition dictionaries + :param cache: dict[str, tuple[list[str], list[str]]] - Cache of previously parsed selectors for reference resolution + :return: tuple[list[str], list[str]] - Tuple of (include_list, exclude_list) containing selector strings + :raises CosmosValueError: If multiple exclude clauses are found at the same level + """ + include: list[str] = [] + exclude: list[str] = [] + + for definition in definitions: + if isinstance(definition, dict) and "exclude" in definition: + # Do not allow multiple exclude: defs at the same level + if exclude: + yaml_sel_cfg = yaml.dump(definition) + raise CosmosValueError( + f"You cannot provide multiple exclude arguments to the " + f"same selector set operator:\n{yaml_sel_cfg}" + ) + definition_exclude = cls._parse_exclusions(definition, cache=cache) + exclude.extend(definition_exclude) + else: + definition_include, _ = cls._parse_from_definition(definition, cache=cache) + include.extend(definition_include) + + return (include, exclude) + + @classmethod + def _parse_union_definition( + cls, + definition: dict[str, Any], + cache: dict[str, tuple[list[str], list[str]]] = {}, + ) -> tuple[list[str], list[str]]: + """ + Parse a union selector definition. + + A union combines multiple selectors with OR logic - nodes matching any of the + subdefinitions will be included. + + :param definition: dict[str, Any] - Dictionary containing a 'union' key with a list of selector definitions + :param cache: dict[str, tuple[list[str], list[str]]] - Cache of previously parsed selectors for reference resolution + :return: tuple[list[str], list[str]] - Tuple of (include_list, exclude_list) containing selector strings + + Example Input: + { + "union": [ + {"method": "tag", "value": "nightly"}, + {"method": "path", "value": "models/staging"} + ] + } + + Example Output: + (["tag:nightly", "path:models/staging"], []) + """ + union_def_parts = cls._get_list_dicts(definition, "union") + return cls._parse_include_exclude_subdefs(union_def_parts, cache=cache) + + @classmethod + def _parse_intersection_definition( + cls, + definition: dict[str, Any], + cache: dict[str, tuple[list[str], list[str]]], + ) -> tuple[list[str], list[str]]: + """ + Parse an intersection selector definition. + + An intersection combines multiple selectors with AND logic - only nodes matching all + subdefinitions will be included. In Cosmos syntax, this is represented by joining + selectors with commas. + + :param definition: dict[str, Any] - Dictionary containing an 'intersection' key with a list of selector definitions + :param cache: dict[str, tuple[list[str], list[str]]] - Cache of previously parsed selectors for reference resolution + :return: tuple[list[str], list[str]] - Tuple of (include_list, exclude_list) with intersected selectors joined by commas + + Example Input: + { + "intersection": [ + {"method": "tag", "value": "nightly"}, + {"method": "config.materialized", "value": "view"} + ] + } + + Example Output: + (["tag:nightly,config.materialized:view"], []) + """ + intersection_def_parts = cls._get_list_dicts(definition, "intersection") + include, exclude = cls._parse_include_exclude_subdefs(intersection_def_parts, cache=cache) + + intersection = [",".join(include)] + + return (intersection, exclude) + + @classmethod + def _parse_method_definition( + cls, + definition: dict[str, Any], + cache: dict[str, tuple[list[str], list[str]]], + ) -> tuple[list[str], list[str]]: + """ + Parse a method-based selector definition. + + Handles selector definitions that use the 'method' and 'value' keys, including + special handling for the 'selector' method which references other named selectors. + + :param definition: dict[str, Any] - Dictionary containing 'method' and 'value' keys, and optionally 'exclude' + :param cache: dict[str, tuple[list[str], list[str]]] - Cache of previously parsed selectors for reference resolution + :return: tuple[list[str], list[str]] - Tuple of (include_list, exclude_list) containing selector strings + :raises CosmosValueError: If 'method' or 'value' keys are missing, or if a referenced + selector doesn't exist + + Example Input (method-based): + { + "method": "tag", + "value": "nightly", + "exclude": [{"method": "path", "value": "models/archived"}] + } + + Example Output: + (["tag:nightly"], ["path:models/archived"]) + + Example Input (selector reference): + { + "method": "selector", + "value": "my_existing_selector" + } + + Example Output: + Returns the cached tuple of (include_list, exclude_list) from the referenced selector + """ + exclude: list[str] = [] + + if definition.get("method") == "selector": + sel_def = definition.get("value") + if sel_def not in cache: + raise CosmosValueError(f"Existing selector definition for {sel_def} not found.") + return cache[definition["value"]] + elif "method" in definition and "value" in definition: + dct = definition + if "exclude" in definition: + exclude = cls._parse_exclusions(definition, cache=cache) + dct = {k: v for k, v in dct.items() if k != "exclude"} + else: + raise CosmosValueError(f'Expected "method" and "value" keys, but got {list(definition)}') + + selector = cls._parse_selector(dct) + + include = [selector] if selector else [] + + return (include, exclude) + + @classmethod + def _parse_from_definition( + cls, + definition: dict[str, Any], + cache: dict[str, tuple[list[str], list[str]]], + rootlevel: bool = False, + ) -> tuple[list[str], list[str]]: + """ + Recursively parse a selector definition into include and exclude lists. + + This is the main entry point for parsing selector definitions. It handles union, + intersection, and method-based definitions, delegating to the appropriate + specialized parsing method. + + :param definition: dict[str, Any] - The selector definition dictionary to parse + :param cache: dict[str, tuple[list[str], list[str]]] - Cache of previously parsed selectors for reference resolution + :param rootlevel: bool - Whether this is a root-level selector (affects validation rules) + :return: tuple[list[str], list[str]] - Tuple of (include_list, exclude_list) containing selector strings + :raises CosmosValueError: If the definition structure is invalid or uses unsupported formats + + Note: + Root-level selectors can only have a single 'union' or 'intersection' key. + CLI-style string definitions and key-value pairs are not supported - use + method-based definitions instead. + """ + if ( + isinstance(definition, dict) + and ("union" in definition or "intersection" in definition) + and rootlevel + and len(definition) > 1 + ): + keys = ",".join(definition.keys()) + raise CosmosValueError( + f"Only a single 'union' or 'intersection' key is allowed " + f"in a root level selector definition; found {keys}." ) - # Items in exclude list should go to exclude_parts - exclude_parts.extend(ex_select) - exclude_parts.extend(ex_exclude) - - # Handle union - if "union" in selectors: - union_items = selectors["union"] - - for item in union_items: - if isinstance(item, dict): - item_select, item_exclude = _parse_selector_definition( - item, - all_selector_definitions, - resolved_cache, - visiting, - ) - select_parts.extend(item_select) - exclude_parts.extend(item_exclude) - - # Handle intersection - if "intersection" in selectors: - intersection_items = selectors["intersection"] - intersection_selects: list[str] = [] - for item in intersection_items: - item_select, item_exclude = _parse_selector_definition( - item, - all_selector_definitions, - resolved_cache, - visiting, + + if "union" in definition: + select, exclude = cls._parse_union_definition(definition, cache=cache) + elif "intersection" in definition: + select, exclude = cls._parse_intersection_definition(definition, cache=cache) + elif "method" in definition: + select, exclude = cls._parse_method_definition(definition, cache=cache) + else: + # Rejects CLI-style string definitions (e.g., definition: "tag:my_tag") + # These should be structured as method-value definitions instead + + # Rejects key-value definitions (e.g., tag: my_tag, path: models/, etc.) + # These should be structured as method-value definitions instead + raise CosmosValueError( + f"Expected to find union, intersection, or method definition, instead " + f"found {type(definition)}: {definition}" ) - intersection_selects.extend(item_select) - exclude_parts.extend(item_exclude) - if intersection_selects: - # Join with commas for intersection - select_parts = [",".join(intersection_selects)] + return (select, exclude) - # Remove None values, empty strings, and duplicates while preserving order - select_parts = list(dict.fromkeys([s for s in select_parts if s])) - exclude_parts = list(dict.fromkeys([e for e in exclude_parts if e])) + @classmethod + def parse(cls, selectors: dict[str, list[dict[str, Any]]]) -> YamlSelectors: + """ + Parse a complete dbt selectors YAML file into a YamlSelectors instance. - return (select_parts, exclude_parts) + This is the main entry point for parsing YAML selectors. It processes all selector + definitions in the YAML file and converts them to Cosmos format. + :param selectors: dict[str, list[dict[str, Any]]] - The complete selectors YAML content as a dictionary with a 'selectors' key + :return: YamlSelectors - A YamlSelectors instance containing both raw and parsed selector definitions -def parse_yaml_selectors(selectors: dict[str, list[dict[str, Any]]]) -> dict[str, dict[str, Any]]: - """ - Parse dbt YAML selectors into a dictionary of select/exclude dbt command-line syntax. + Example Input: + { + "selectors": [ + { + "name": "nightly_models", + "definition": { + "method": "tag", + "value": "nightly" + }, + "default": False + } + ] + } + + Example Output: + YamlSelectors instance with parsed definitions: + { + "nightly_models": { + "select": ["tag:nightly"], + "exclude": None, + "default": False + } + } + """ + result: dict[str, dict[str, Any]] = {} + cache: dict[str, tuple[list[str], list[str]]] = {} + selectors_root: list[dict[str, Any]] = selectors.get("selectors", []) - Args: - selectors (dict): The selectors dictionary loaded from selectors.yml. - Returns: - dict: A dictionary mapping selector names to their dbt selection syntax. + for selector in selectors_root: + name = selector.get("name", "") + definition = selector.get("definition", {}) + default = selector.get("default", False) - """ - selectors_root: list[dict[str, Any]] = selectors.get("selectors", []) - - # Build a map of all selector definitions for reference resolution - all_selectors: dict[str, Any] = {} - for selector in selectors_root: - name = selector.get("name", "") - definition = selector.get("definition", {}) - all_selectors[name] = definition - - # Build result dictionary with resolved selectors - result: dict[str, dict[str, list[str] | None]] = {} - resolved_cache: dict[str, tuple[list[str], list[str]]] = {} - visiting: set[str] = set() - - for selector in selectors_root: - name = selector.get("name", "") - definition = selector.get("definition", {}) - - select, exclude = _parse_selector_definition( - definition, - all_selectors, - resolved_cache, - visiting, - ) + select, exclude = cls._parse_from_definition(definition, cache=copy.deepcopy(cache), rootlevel=True) - result[name] = { - "select": select if select else None, - "exclude": exclude if exclude else None, - } + result[name] = { + "select": select if select else None, + "exclude": exclude if exclude else None, + "default": default, + } - return result + return cls(selectors, result) def select_nodes( diff --git a/cosmos/settings.py b/cosmos/settings.py index 8ef9319c08..c243126f0d 100644 --- a/cosmos/settings.py +++ b/cosmos/settings.py @@ -27,7 +27,7 @@ enable_cache_partial_parse = conf.getboolean("cosmos", "enable_cache_partial_parse", fallback=True) enable_cache_package_lockfile = conf.getboolean("cosmos", "enable_cache_package_lockfile", fallback=True) enable_cache_dbt_ls = conf.getboolean("cosmos", "enable_cache_dbt_ls", fallback=True) -enable_cache_selectors_yaml = conf.getboolean("cosmos", "enable_cache_selectors_yaml", fallback=True) +enable_cache_yaml_selectors = conf.getboolean("cosmos", "enable_cache_yaml_selectors", fallback=True) rich_logging = conf.getboolean("cosmos", "rich_logging", fallback=False) dbt_docs_dir = conf.get("cosmos", "dbt_docs_dir", fallback=None) dbt_docs_conn_id = conf.get("cosmos", "dbt_docs_conn_id", fallback=None) diff --git a/dev/dags/cosmos_manifest_yaml_selectors_example.py b/dev/dags/cosmos_manifest_yaml_selectors_example.py index f67fdc79c0..61d95ef656 100644 --- a/dev/dags/cosmos_manifest_yaml_selectors_example.py +++ b/dev/dags/cosmos_manifest_yaml_selectors_example.py @@ -49,7 +49,7 @@ group_id="local_example", project_config=ProjectConfig( manifest_path=DBT_ROOT_PATH / "jaffle_shop" / "target" / "manifest.json", - selectors_yaml_path=DBT_ROOT_PATH / "jaffle_shop" / "selectors.yml", + yaml_selectors_path=DBT_ROOT_PATH / "jaffle_shop" / "selectors.yml", project_name="jaffle_shop", ), profile_config=profile_config, @@ -66,9 +66,9 @@ manifest_path="s3://cosmos-manifest-test/manifest.json", manifest_conn_id="aws_s3_conn", # `manifest_conn_id` is optional. If not provided, the default connection ID `aws_default` is used. - selectors_yaml_path="s3://cosmos-manifest-test/selectors.yml", - selectors_yaml_conn_id="aws_s3_conn", - # `selectors_yaml_conn_id` is optional. If not provided, the default connection ID ` + yaml_selectors_path="s3://cosmos-manifest-test/selectors.yml", + yaml_selectors_conn_id="aws_s3_conn", + # `yaml_selectors_conn_id` is optional. If not provided, the default connection ID ` project_name="jaffle_shop", ), profile_config=profile_config, @@ -85,9 +85,9 @@ manifest_path="gs://cosmos_remote_target/manifest.json", manifest_conn_id="gcp_gs_conn", # `manifest_conn_id` is optional. If not provided, the default connection ID `google_cloud_default` is used. - selectors_yaml_path="gs://cosmos_remote_target/selectors.yml", - selectors_yaml_conn_id="gcp_gs_conn", - # `selectors_yaml_conn_id` is optional. If not provided, the default connection ID `google_cloud_default` is used. + yaml_selectors_path="gs://cosmos_remote_target/selectors.yml", + yaml_selectors_conn_id="gcp_gs_conn", + # `yaml_selectors_conn_id` is optional. If not provided, the default connection ID `google_cloud_default` is used. project_name="jaffle_shop", ), profile_config=profile_config, @@ -104,9 +104,9 @@ manifest_path="abfs://cosmos-manifest-test/manifest.json", manifest_conn_id="azure_abfs_conn", # `manifest_conn_id` is optional. If not provided, the default connection ID `wasb_default` is used. - selectors_yaml_path="abfs://cosmos-manifest-test/selectors.yml", - selectors_yaml_conn_id="azure_abfs_conn", - # `selectors_yaml_conn_id` is optional. If not provided, the default connection ID `wasb_default` is used. + yaml_selectors_path="abfs://cosmos-manifest-test/selectors.yml", + yaml_selectors_conn_id="azure_abfs_conn", + # `yaml_selectors_conn_id` is optional. If not provided, the default connection ID `wasb_default` is used. project_name="jaffle_shop", ), profile_config=profile_config, From 93078dd0aa38e11ac15c4919ad217259183fe140 Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Thu, 15 Jan 2026 17:06:21 -0500 Subject: [PATCH 11/24] Pull preprocessed selector definitions from the manifest --- cosmos/cache.py | 26 +++--- cosmos/config.py | 38 --------- cosmos/dbt/graph.py | 60 +++++++------- cosmos/dbt/selector.py | 50 +++++------- .../cosmos_manifest_yaml_selectors_example.py | 1 - dev/dags/dbt/jaffle_shop/selectors.yml | 2 + dev/dags/dbt/jaffle_shop/target/manifest.json | 80 ++++++++++++++++++- 7 files changed, 145 insertions(+), 112 deletions(-) diff --git a/cosmos/cache.py b/cosmos/cache.py index ed161e5260..2bb85d8a7e 100644 --- a/cosmos/cache.py +++ b/cosmos/cache.py @@ -10,7 +10,7 @@ from collections import defaultdict from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import msgpack import yaml @@ -288,15 +288,16 @@ def _copy_partial_parse_to_project(partial_parse_filepath: Path, project_path: P def _calculate_yaml_selectors_cache_current_version( - cache_identifier: str, project_dir: Path, yaml_selectors_path: Path | ObjectStoragePath + cache_identifier: str, project_dir: Path, selector_definitions: dict[str, dict[str, Any]] ) -> str: """ - Taking into account the project directory contents and the selectors yaml file, calculate the - hash that represents the "dbt selectors yaml" version - to be used to decide if the cache should be refreshed or not. + Taking into account the project directory contents and the selectors definitions, calculate the + hash that represents the "dbt selectors" version - to be used to decide if the cache should be refreshed or not. - :param cache_identifier: Unique identifier of the cache (may include DbtDag or DbtTaskGroup information) - :param project_path: Path to the target dbt project directory - :param yaml_selectors_path: Path to the selectors yaml file + :param cache_identifier: str - Unique identifier of the cache (may include DbtDag or DbtTaskGroup information) + :param project_dir: Path - Path to the target dbt project directory + :param selector_definitions: dict[str, dict[str, Any]] - Dictionary containing the selectors definitions from the manifest + :return: str - Combined hash string of project and selectors """ start_time = time.perf_counter() @@ -305,9 +306,7 @@ def _calculate_yaml_selectors_cache_current_version( # This is fast (e.g. 0.01s for jaffle shop, 0.135s for a 5k models dbt folder) dbt_project_hash = _create_folder_version_hash(project_dir) - with yaml_selectors_path.open("r") as f: - yaml_selector_content = yaml.safe_load(f) - yaml_selector_hash = hashlib.md5(yaml.dump(yaml_selector_content).encode()).hexdigest() + yaml_selector_hash = hashlib.md5(yaml.dump(selector_definitions).encode()).hexdigest() elapsed_time = time.perf_counter() - start_time logger.info( @@ -321,9 +320,10 @@ def _calculate_dbt_ls_cache_current_version(cache_identifier: str, project_dir: Taking into account the project directory contents and the command arguments, calculate the hash that represents the "dbt ls" command version - to be used to decide if the cache should be refreshed or not. - :param cache_identifier: Unique identifier of the cache (may include DbtDag or DbtTaskGroup information) - :param project_path: Path to the target dbt project directory - :param cmd_args: List containing the arguments passed to the dbt ls command that would affect its output + :param cache_identifier: str - Unique identifier of the cache (may include DbtDag or DbtTaskGroup information) + :param project_dir: Path - Path to the target dbt project directory + :param cmd_args: list[str] - List containing the arguments passed to the dbt ls command that would affect its output + :return: str - Combined hash string of project and command arguments """ start_time = time.perf_counter() diff --git a/cosmos/config.py b/cosmos/config.py index ef353f8f8a..d25db183ab 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -169,8 +169,6 @@ class ProjectConfig: :param snapshots_relative_path: The relative path to the dbt snapshots directory within the project. Defaults to snapshots :param manifest_path: The absolute path to the dbt manifest file. Defaults to None :param manifest_conn_id: Name of the Airflow connection used to access the manifest file if it is not stored locally. Defaults to None - :param selectors_path: The absolute path to the dbt selectors file. Defaults to None - :param selectors_conn_id: Name of the Airflow connection used to access the selectors file if it is not stored locally. Defaults to None :param project_name: Allows the user to define the project name. Required if dbt_project_path is not defined. Defaults to the folder name of dbt_project_path. :param env_vars: Dictionary of environment variables that are used for both rendering and execution. Rendering with @@ -187,7 +185,6 @@ class ProjectConfig: install_dbt_deps: bool = True copy_dbt_packages: bool = settings.default_copy_dbt_packages manifest_path: Path | ObjectStoragePath | None = None - yaml_selectors_path: Path | ObjectStoragePath | None = None models_path: Path | None = None seeds_path: Path | None = None snapshots_path: Path | None = None @@ -203,8 +200,6 @@ def __init__( snapshots_relative_path: str | Path = "snapshots", manifest_path: str | Path | None = None, manifest_conn_id: str | None = None, - yaml_selectors_path: str | Path | None = None, - yaml_selectors_conn_id: str | None = None, project_name: str | None = None, env_vars: dict[str, str] | None = None, dbt_vars: dict[str, str] | None = None, @@ -228,7 +223,6 @@ def __init__( manifest_path, ) self.validate_manifest_path(manifest_path, manifest_conn_id) - self.validate_yaml_selectors_path(yaml_selectors_path, yaml_selectors_conn_id) def validate_dbt_project_paths( self, @@ -279,32 +273,6 @@ def validate_manifest_path( else: self.manifest_path = Path(manifest_path_str) - def validate_yaml_selectors_path( - self, - yaml_selectors_path: str | Path | None, - yaml_selectors_conn_id: str | None, - ) -> None: - if yaml_selectors_path: - yaml_selectors_path_str = str(yaml_selectors_path) - if not yaml_selectors_conn_id: - yaml_selectors_scheme = yaml_selectors_path_str.split("://")[0] - # Use the default Airflow connection ID for the scheme if it is not provided. - yaml_selectors_conn_id = FILE_SCHEME_AIRFLOW_DEFAULT_CONN_ID_MAP.get( - yaml_selectors_scheme, lambda: None - )() - - if yaml_selectors_conn_id is not None and not settings.AIRFLOW_IO_AVAILABLE: - raise CosmosValueError( - f"The selectors yaml path {yaml_selectors_path_str} uses a remote file scheme, but the required Object " - f"Storage feature is unavailable in Airflow version {airflow_version}. Please upgrade to " - f"Airflow 2.8 or later." - ) - - if settings.AIRFLOW_IO_AVAILABLE: - self.yaml_selectors_path = ObjectStoragePath(yaml_selectors_path_str, conn_id=yaml_selectors_conn_id) - else: - self.yaml_selectors_path = Path(yaml_selectors_path_str) - def validate_project(self) -> None: """ Validates necessary context is present for a project. @@ -343,12 +311,6 @@ def is_manifest_available(self) -> bool: """ return self.manifest_path.exists() if self.manifest_path else False - def is_yaml_selectors_available(self) -> bool: - """ - Check if the `dbt` selectors YAML file is set and if the file exists. - """ - return self.yaml_selectors_path.exists() if self.yaml_selectors_path else False - @dataclass class ProfileConfig: diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index 03ceacbf33..1d34da68f0 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -17,7 +17,6 @@ from subprocess import PIPE, Popen from typing import TYPE_CHECKING, Any -import yaml from airflow.models import Variable if TYPE_CHECKING: @@ -972,8 +971,6 @@ def save_yaml_selector_cache(self, yaml_selectors: YamlSelectors) -> None: "last_modified": "Isoformat timestamp" } """ - if TYPE_CHECKING: - assert self.project.yaml_selectors_path is not None # pragma: no cover serialized_data = pickle.dumps(yaml_selectors) compressed_data = zlib.compress(serialized_data) @@ -982,7 +979,7 @@ def save_yaml_selector_cache(self, yaml_selectors: YamlSelectors) -> None: cache_dict = { "version": cache._calculate_yaml_selectors_cache_current_version( - self.cache_key, self.project_path, self.project.yaml_selectors_path + self.cache_key, self.project_path, yaml_selectors.raw ), "yaml_selectors": selections_compressed, "last_modified": datetime.datetime.now(datetime.timezone.utc).isoformat(), @@ -996,39 +993,38 @@ def save_yaml_selector_cache(self, yaml_selectors: YamlSelectors) -> None: else: Variable.set(self.cache_key, cache_dict, serialize_json=True) - def parse_yaml_selectors(self) -> YamlSelectors: + def parse_yaml_selectors(self, selector_definitions: dict[str, Any]) -> YamlSelectors: """ - Parse the YAML selectors file into a YamlSelctor + Parse the YAML selectors definitions into a YamlSelectors instance. + + Args: + selector_definitions (dict[str, Any]): The selector definitions from the dbt manifest. Returns: - A YamlSelectors instance + YamlSelectors: A YamlSelectors instance """ - if TYPE_CHECKING: - assert self.project.yaml_selectors_path is not None # pragma: no cover - with self.project.yaml_selectors_path.open() as fp: - yaml_selectors_definition = yaml.safe_load(fp) - yaml_selectors = YamlSelectors.parse(yaml_selectors_definition) + yaml_selectors = YamlSelectors.parse(selector_definitions) - if self.should_use_yaml_selectors_cache(): - self.save_yaml_selector_cache(yaml_selectors) + if self.should_use_yaml_selectors_cache(): + self.save_yaml_selector_cache(yaml_selectors) - return yaml_selectors + return yaml_selectors - def load_parsed_selectors(self) -> YamlSelectors: + def load_parsed_selectors(self, selector_definitions: dict[str, Any]) -> YamlSelectors: """ - Load the YamlSelectors from cache if available, otherwise parse the YAML selectors file. + Load the YamlSelectors from cache if available, otherwise parse the YAML selectors definitions. + + Args: + selector_definitions (dict[str, Any]): The selector definitions from the dbt manifest. Returns: - A dictionary containing the selections for the specified selectors. + YamlSelectors: A YamlSelectors instance """ logger.info(f"Trying to parse the dbt yaml selectors using {self.cache_key}...") - if TYPE_CHECKING: - assert self.project.yaml_selectors_path is not None # pragma: no cover - - def get_yaml_selectors() -> YamlSelectors: - yaml_selectors = self.parse_yaml_selectors() + def get_yaml_selectors(selector_definitions: dict[str, Any]) -> YamlSelectors: + yaml_selectors = self.parse_yaml_selectors(selector_definitions) if self.should_use_yaml_selectors_cache(): self.save_yaml_selector_cache(yaml_selectors) @@ -1041,13 +1037,13 @@ def get_yaml_selectors() -> YamlSelectors: if not cache_dict: logger.info(f"Cosmos performance: Cache miss for {self.cache_key}") - return get_yaml_selectors() + return get_yaml_selectors(selector_definitions) cache_version: str = cache_dict["version"] yaml_selectors: YamlSelectors = cache_dict["yaml_selectors"] current_version = cache._calculate_yaml_selectors_cache_current_version( - self.cache_key, self.project_path, self.project.yaml_selectors_path + self.cache_key, self.project_path, selector_definitions ) if cache_dict and not cache.were_yaml_selectors_modified(cache_version, current_version): @@ -1060,7 +1056,7 @@ def get_yaml_selectors() -> YamlSelectors: logger.info(f"Cosmos performance: Cache miss for {self.cache_key} - skipped") - return get_yaml_selectors() + return get_yaml_selectors(selector_definitions) def load_from_dbt_manifest(self) -> None: """ @@ -1076,9 +1072,6 @@ def load_from_dbt_manifest(self) -> None: self.load_method = LoadMode.DBT_MANIFEST logger.info("Trying to parse the dbt project `%s` using a dbt manifest...", self.project.project_name) - if self.render_config.selector and not self.project.is_yaml_selectors_available(): - raise CosmosLoadDbtException(f"Unable to load yaml selectors using {self.project.yaml_selectors_path}") - if not self.project.is_manifest_available(): raise CosmosLoadDbtException(f"Unable to load manifest using {self.project.manifest_path}") @@ -1113,12 +1106,17 @@ def load_from_dbt_manifest(self) -> None: nodes[node.unique_id] = node if self.render_config.selector: - yaml_selectors = self.load_parsed_selectors() + selector_definitions = manifest.get("selectors", {}) + + if not selector_definitions: + raise CosmosLoadDbtException(f"Selectors not found in manifest file `{self.project.manifest_path}`") + + yaml_selectors = self.load_parsed_selectors(selector_definitions) selections = yaml_selectors.get_parsed(self.render_config.selector) if not selections: raise CosmosLoadDbtException( - f"Selector `{self.render_config.selector}` not found in parsed YAML selectors `{self.project.yaml_selectors_path}`" + f"Selector `{self.render_config.selector}` not found in parsed YAML selectors `{selector_definitions}`" ) self.nodes = nodes diff --git a/cosmos/dbt/selector.py b/cosmos/dbt/selector.py index cf18bf9838..2d6b104e55 100644 --- a/cosmos/dbt/selector.py +++ b/cosmos/dbt/selector.py @@ -689,37 +689,39 @@ class YamlSelectors: """ Parses and manages dbt YAML selector definitions. - This class handles the parsing of dbt selector YAML files, converting them from their raw - YAML format into a format compatible with Cosmos node selection. It supports union, + This class handles the parsing of dbt selector YAML definitions, converting them from a dbt manifest + preprocessed format into syntax compatible with Cosmos node selection. Attempts to parse a selectors.yaml + may lead to errors. This version of the parser expects method-value selector definitions and does not have + support for the `default` or `indirect_selection` properties. It supports union, intersection, and method-based selector definitions as specified in dbt's selector syntax. Instances of this class should be created using the `parse` class method. - :param raw_selectors: dict[str, list[dict[str, Any]]] - The original, unparsed selector definitions from the YAML file + :param raw_selectors: dict[str, dict[str, Any]] - The original, unparsed selector definitions from the manifest :param parsed_selectors: dict[str, dict[str, Any]] - The processed selector definitions converted to Cosmos format - :property raw: dict[str, list[dict[str, Any]]] - The original unparsed selector definitions from the YAML file. + :property raw: dict[str, dict[str, Any]] + The original unparsed selector definitions from the manifest. :property parsed: dict[str, dict[str, Any]] The parsed selector definitions in Cosmos format. :property dbt_spec_version: int - The dbt selector specification version being used. + The dbt selector file specification version being used. References: https://docs.getdbt.com/reference/node-selection/yaml-selectors """ - def __init__(self, raw_selectors: dict[str, list[dict[str, Any]]], parsed_selectors: dict[str, dict[str, Any]]): + def __init__(self, raw_selectors: dict[str, dict[str, Any]], parsed_selectors: dict[str, dict[str, Any]]): self._raw = raw_selectors self._parsed = parsed_selectors self._dbt_spec_version = 2 @property - def raw(self) -> dict[str, list[dict[str, Any]]]: + def raw(self) -> dict[str, dict[str, Any]]: """ Get the original unparsed selector definitions. - :return: dict[str, list[dict[str, Any]]] - Dictionary containing the raw YAML structure + :return: dict[str, dict[str, Any]] - Dictionary containing the raw YAML structure """ return self._raw @@ -735,7 +737,7 @@ def parsed(self) -> dict[str, dict[str, Any]]: @property def dbt_spec_version(self) -> int: """ - Get the dbt selector specification version being used. + Get the current dbt selector file specification version. :return: int - The dbt spec version """ @@ -910,7 +912,6 @@ def _parse_selector(cls, dct: dict[str, Any]) -> str: parents_depth = dct.get("parents_depth", 0) children_depth = dct.get("children_depth", 0) childrens_parents = dct.get("childrens_parents", False) - indirect_selection = dct.get("indirect_selection", False) if method == "METHOD_KEY_NOT_FOUND": raise CosmosValueError(f"Selector method is missing in method definition: {dct}.") @@ -922,9 +923,6 @@ def _parse_selector(cls, dct: dict[str, Any]) -> str: if not isinstance(children_depth, int): raise CosmosValueError(f"children_depth must be an integer, got {type(children_depth)}: {children_depth}") - if indirect_selection: - logger.warning("The 'indirect_selection' parameter is not supported and will be ignored.") - selector = cls._parse_selection_from_cosmos_spec(method, value) selector = cls._parse_selection_graph_operators( @@ -1173,14 +1171,14 @@ def _parse_from_definition( return (select, exclude) @classmethod - def parse(cls, selectors: dict[str, list[dict[str, Any]]]) -> YamlSelectors: + def parse(cls, selectors: dict[str, dict[str, Any]]) -> YamlSelectors: """ Parse a complete dbt selectors YAML file into a YamlSelectors instance. This is the main entry point for parsing YAML selectors. It processes all selector definitions in the YAML file and converts them to Cosmos format. - :param selectors: dict[str, list[dict[str, Any]]] - The complete selectors YAML content as a dictionary with a 'selectors' key + :param selectors: dict[str, dict[str, Any]] - The complete selectors YAML content as a dictionary with a 'selectors' key :return: YamlSelectors - A YamlSelectors instance containing both raw and parsed selector definitions Example Input: @@ -1191,8 +1189,7 @@ def parse(cls, selectors: dict[str, list[dict[str, Any]]]) -> YamlSelectors: "definition": { "method": "tag", "value": "nightly" - }, - "default": False + } } ] } @@ -1202,26 +1199,23 @@ def parse(cls, selectors: dict[str, list[dict[str, Any]]]) -> YamlSelectors: { "nightly_models": { "select": ["tag:nightly"], - "exclude": None, - "default": False - } + "exclude": None } } """ result: dict[str, dict[str, Any]] = {} cache: dict[str, tuple[list[str], list[str]]] = {} - selectors_root: list[dict[str, Any]] = selectors.get("selectors", []) - for selector in selectors_root: - name = selector.get("name", "") - definition = selector.get("definition", {}) - default = selector.get("default", False) + for name, definition in selectors.items(): + name = definition.get("name", "") + selector_definition = definition.get("definition", {}) - select, exclude = cls._parse_from_definition(definition, cache=copy.deepcopy(cache), rootlevel=True) + select, exclude = cls._parse_from_definition( + selector_definition, cache=copy.deepcopy(cache), rootlevel=True + ) result[name] = { "select": select if select else None, "exclude": exclude if exclude else None, - "default": default, } return cls(selectors, result) diff --git a/dev/dags/cosmos_manifest_yaml_selectors_example.py b/dev/dags/cosmos_manifest_yaml_selectors_example.py index 61d95ef656..410569d700 100644 --- a/dev/dags/cosmos_manifest_yaml_selectors_example.py +++ b/dev/dags/cosmos_manifest_yaml_selectors_example.py @@ -49,7 +49,6 @@ group_id="local_example", project_config=ProjectConfig( manifest_path=DBT_ROOT_PATH / "jaffle_shop" / "target" / "manifest.json", - yaml_selectors_path=DBT_ROOT_PATH / "jaffle_shop" / "selectors.yml", project_name="jaffle_shop", ), profile_config=profile_config, diff --git a/dev/dags/dbt/jaffle_shop/selectors.yml b/dev/dags/dbt/jaffle_shop/selectors.yml index 9e8d30b2bc..04573a3700 100644 --- a/dev/dags/dbt/jaffle_shop/selectors.yml +++ b/dev/dags/dbt/jaffle_shop/selectors.yml @@ -7,6 +7,7 @@ selectors: - name: core_models description: "Select core business logic models (non-staging)" + default: true definition: method: tag value: core @@ -48,3 +49,4 @@ selectors: value: customers - method: fqn value: orders + indirect_selection: cautious diff --git a/dev/dags/dbt/jaffle_shop/target/manifest.json b/dev/dags/dbt/jaffle_shop/target/manifest.json index e46748c438..fc70ecf8dc 100644 --- a/dev/dags/dbt/jaffle_shop/target/manifest.json +++ b/dev/dags/dbt/jaffle_shop/target/manifest.json @@ -15852,7 +15852,85 @@ ] }, "saved_queries": {}, - "selectors": {}, + "selectors": { + "core_models": { + "definition": { + "method": "tag", + "value": "core" + }, + "description": "Select core business logic models (non-staging)", + "name": "core_models" + }, + "critical_path": { + "definition": { + "union": [ + { + "method": "fqn", + "value": "customers" + }, + { + "method": "fqn", + "value": "orders" + } + ] + }, + "description": "Critical business models", + "name": "critical_path" + }, + "customers_and_downstream": { + "definition": { + "childrens_parents": true, + "method": "fqn", + "value": "customers" + }, + "description": "Select customers model and all downstream dependencies", + "name": "customers_and_downstream" + }, + "exclude_staging": { + "definition": { + "union": [ + { + "method": "fqn", + "value": "*" + }, + { + "exclude": [ + { + "method": "path", + "value": "models/staging" + } + ] + } + ] + }, + "description": "All models except staging", + "name": "exclude_staging" + }, + "nightly_models": { + "definition": { + "method": "tag", + "value": "nightly" + }, + "description": "Models tagged for nightly runs", + "name": "nightly_models" + }, + "staging_models": { + "definition": { + "method": "path", + "value": "models/staging" + }, + "description": "Select all staging models", + "name": "staging_models" + }, + "staging_orders": { + "definition": { + "method": "fqn", + "value": "jaffle_shop.staging.stg_orders" + }, + "description": "Select staging orders model specifically", + "name": "staging_orders" + } + }, "semantic_models": {}, "sources": {}, "unit_tests": {} From 2b410a45909fb127a5920469ea6d3f2411462303 Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Thu, 15 Jan 2026 17:29:10 -0500 Subject: [PATCH 12/24] Remove dbt_spec_version --- cosmos/dbt/selector.py | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/cosmos/dbt/selector.py b/cosmos/dbt/selector.py index 2d6b104e55..7655457af9 100644 --- a/cosmos/dbt/selector.py +++ b/cosmos/dbt/selector.py @@ -687,13 +687,20 @@ def retrieve_by_label(statement_list: list[str], label: str) -> set[str]: class YamlSelectors: """ - Parses and manages dbt YAML selector definitions. + Parses and manages dbt YAML selector definitions from the manifest. - This class handles the parsing of dbt selector YAML definitions, converting them from a dbt manifest - preprocessed format into syntax compatible with Cosmos node selection. Attempts to parse a selectors.yaml - may lead to errors. This version of the parser expects method-value selector definitions and does not have - support for the `default` or `indirect_selection` properties. It supports union, - intersection, and method-based selector definitions as specified in dbt's selector syntax. + This class handles the parsing of selector definitions that come from a dbt manifest file, + converting them into syntax compatible with Cosmos node selection. The manifest provides + selectors in a preprocessed format (dict keyed by selector name) rather than the original + selectors.yml file format (list of selector objects). + + **Important**: This parser expects the manifest's preprocessed selector format and was NOT + designed to support parsing selectors.yml files directly. Attempting to parse a selectors.yml + file may result in errors. + + **Limitations**: This implementation does not support the `default` or `indirect_selection` + properties from the dbt selector specification. It focuses on the core selection logic: + union, intersection, and method-based selector definitions. Instances of this class should be created using the `parse` class method. @@ -704,8 +711,6 @@ class YamlSelectors: The original unparsed selector definitions from the manifest. :property parsed: dict[str, dict[str, Any]] The parsed selector definitions in Cosmos format. - :property dbt_spec_version: int - The dbt selector file specification version being used. References: https://docs.getdbt.com/reference/node-selection/yaml-selectors @@ -714,7 +719,6 @@ class YamlSelectors: def __init__(self, raw_selectors: dict[str, dict[str, Any]], parsed_selectors: dict[str, dict[str, Any]]): self._raw = raw_selectors self._parsed = parsed_selectors - self._dbt_spec_version = 2 @property def raw(self) -> dict[str, dict[str, Any]]: @@ -734,15 +738,6 @@ def parsed(self) -> dict[str, dict[str, Any]]: """ return self._parsed - @property - def dbt_spec_version(self) -> int: - """ - Get the current dbt selector file specification version. - - :return: int - The dbt spec version - """ - return self._dbt_spec_version - def get_raw(self, selector_name: str, default: Any = None) -> dict[str, Any] | Any: """ Retrieve a raw selector definition by name. From 52c698296ada8e398d8d2323544830dc84c1640b Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Fri, 16 Jan 2026 12:18:42 -0500 Subject: [PATCH 13/24] Invalidate cosmos cache when YamlSelectors implementation changes --- cosmos/cache.py | 9 +++++++-- cosmos/dbt/graph.py | 4 ++-- cosmos/dbt/selector.py | 16 ++++++++++++++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/cosmos/cache.py b/cosmos/cache.py index 2bb85d8a7e..4c4b49ed7a 100644 --- a/cosmos/cache.py +++ b/cosmos/cache.py @@ -288,7 +288,10 @@ def _copy_partial_parse_to_project(partial_parse_filepath: Path, project_path: P def _calculate_yaml_selectors_cache_current_version( - cache_identifier: str, project_dir: Path, selector_definitions: dict[str, dict[str, Any]] + cache_identifier: str, + project_dir: Path, + selector_definitions: dict[str, dict[str, Any]], + implementation_version: str, ) -> str: """ Taking into account the project directory contents and the selectors definitions, calculate the @@ -297,6 +300,7 @@ def _calculate_yaml_selectors_cache_current_version( :param cache_identifier: str - Unique identifier of the cache (may include DbtDag or DbtTaskGroup information) :param project_dir: Path - Path to the target dbt project directory :param selector_definitions: dict[str, dict[str, Any]] - Dictionary containing the selectors definitions from the manifest + :param implementation_version: str - The implementation version of the YamlSelectors class :return: str - Combined hash string of project and selectors """ @@ -307,12 +311,13 @@ def _calculate_yaml_selectors_cache_current_version( dbt_project_hash = _create_folder_version_hash(project_dir) yaml_selector_hash = hashlib.md5(yaml.dump(selector_definitions).encode()).hexdigest() + implementation_hash = hashlib.md5(implementation_version.encode()).hexdigest() elapsed_time = time.perf_counter() - start_time logger.info( f"Cosmos performance: time to calculate cache identifier {cache_identifier} for current version: {elapsed_time}" ) - return f"{dbt_project_hash},{yaml_selector_hash}" + return f"{dbt_project_hash},{yaml_selector_hash},{implementation_hash}" def _calculate_dbt_ls_cache_current_version(cache_identifier: str, project_dir: Path, cmd_args: list[str]) -> str: diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index 1d34da68f0..bc29f3e623 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -979,7 +979,7 @@ def save_yaml_selector_cache(self, yaml_selectors: YamlSelectors) -> None: cache_dict = { "version": cache._calculate_yaml_selectors_cache_current_version( - self.cache_key, self.project_path, yaml_selectors.raw + self.cache_key, self.project_path, yaml_selectors.raw, yaml_selectors.impl_version ), "yaml_selectors": selections_compressed, "last_modified": datetime.datetime.now(datetime.timezone.utc).isoformat(), @@ -1043,7 +1043,7 @@ def get_yaml_selectors(selector_definitions: dict[str, Any]) -> YamlSelectors: yaml_selectors: YamlSelectors = cache_dict["yaml_selectors"] current_version = cache._calculate_yaml_selectors_cache_current_version( - self.cache_key, self.project_path, selector_definitions + self.cache_key, self.project_path, selector_definitions, yaml_selectors.impl_version ) if cache_dict and not cache.were_yaml_selectors_modified(cache_version, current_version): diff --git a/cosmos/dbt/selector.py b/cosmos/dbt/selector.py index 7655457af9..e8fae811fa 100644 --- a/cosmos/dbt/selector.py +++ b/cosmos/dbt/selector.py @@ -1,6 +1,8 @@ from __future__ import annotations import copy +import functools +import inspect import re from collections import defaultdict from dataclasses import dataclass @@ -738,6 +740,20 @@ def parsed(self) -> dict[str, dict[str, Any]]: """ return self._parsed + @property + @functools.lru_cache(maxsize=1) + def impl_version(self) -> str: + """ + Get the source code of the YamlSelectors implementation. + + This property retrieves the complete source code of the YamlSelectors class, which can be + used to detect changes in the selector parsing logic (e.g., for cache invalidation). + The source code is retrieved once and cached for the lifetime of the instance. + + :return: str - Complete source code of the YamlSelectors class + """ + return inspect.getsource(self.__class__) + def get_raw(self, selector_name: str, default: Any = None) -> dict[str, Any] | Any: """ Retrieve a raw selector definition by name. From 6c1402ec3a5e76289aadf0ec91d6089c58d8fa00 Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Wed, 21 Jan 2026 17:15:41 -0500 Subject: [PATCH 14/24] Remove changes from previous approach --- cosmos/config.py | 42 ++++--------------- .../cosmos_manifest_yaml_selectors_example.py | 9 ---- 2 files changed, 9 insertions(+), 42 deletions(-) diff --git a/cosmos/config.py b/cosmos/config.py index d25db183ab..de07e59250 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -205,34 +205,6 @@ def __init__( dbt_vars: dict[str, str] | None = None, partial_parse: bool = True, ): - if project_name: - self.project_name = project_name - - self.env_vars = env_vars - self.dbt_vars = dbt_vars - self.partial_parse = partial_parse - self.install_dbt_deps = install_dbt_deps - self.copy_dbt_packages = copy_dbt_packages - - self.validate_dbt_project_paths( - project_name, - dbt_project_path, - models_relative_path, - seeds_relative_path, - snapshots_relative_path, - manifest_path, - ) - self.validate_manifest_path(manifest_path, manifest_conn_id) - - def validate_dbt_project_paths( - self, - project_name: str | None, - dbt_project_path: str | Path | None, - models_relative_path: str | Path, - seeds_relative_path: str | Path, - snapshots_relative_path: str | Path, - manifest_path: str | Path | None, - ) -> None: # Since we allow dbt_project_path to be defined in ExecutionConfig and RenderConfig # dbt_project_path may not always be defined here. # We do, however, still require that both manifest_path and project_name be defined, or neither be defined. @@ -241,6 +213,9 @@ def validate_dbt_project_paths( raise CosmosValueError( "If ProjectConfig.dbt_project_path is not defined, ProjectConfig.manifest_path and ProjectConfig.project_name must be defined together, or both left undefined." ) + if project_name: + self.project_name = project_name + if dbt_project_path: self.dbt_project_path = Path(dbt_project_path) self.models_path = self.dbt_project_path / Path(models_relative_path) @@ -249,11 +224,6 @@ def validate_dbt_project_paths( if not project_name: self.project_name = self.dbt_project_path.stem - def validate_manifest_path( - self, - manifest_path: str | Path | None, - manifest_conn_id: str | None, - ) -> None: if manifest_path: manifest_path_str = str(manifest_path) if not manifest_conn_id: @@ -273,6 +243,12 @@ def validate_manifest_path( else: self.manifest_path = Path(manifest_path_str) + self.env_vars = env_vars + self.dbt_vars = dbt_vars + self.partial_parse = partial_parse + self.install_dbt_deps = install_dbt_deps + self.copy_dbt_packages = copy_dbt_packages + def validate_project(self) -> None: """ Validates necessary context is present for a project. diff --git a/dev/dags/cosmos_manifest_yaml_selectors_example.py b/dev/dags/cosmos_manifest_yaml_selectors_example.py index 410569d700..490ca450ca 100644 --- a/dev/dags/cosmos_manifest_yaml_selectors_example.py +++ b/dev/dags/cosmos_manifest_yaml_selectors_example.py @@ -65,9 +65,6 @@ manifest_path="s3://cosmos-manifest-test/manifest.json", manifest_conn_id="aws_s3_conn", # `manifest_conn_id` is optional. If not provided, the default connection ID `aws_default` is used. - yaml_selectors_path="s3://cosmos-manifest-test/selectors.yml", - yaml_selectors_conn_id="aws_s3_conn", - # `yaml_selectors_conn_id` is optional. If not provided, the default connection ID ` project_name="jaffle_shop", ), profile_config=profile_config, @@ -84,9 +81,6 @@ manifest_path="gs://cosmos_remote_target/manifest.json", manifest_conn_id="gcp_gs_conn", # `manifest_conn_id` is optional. If not provided, the default connection ID `google_cloud_default` is used. - yaml_selectors_path="gs://cosmos_remote_target/selectors.yml", - yaml_selectors_conn_id="gcp_gs_conn", - # `yaml_selectors_conn_id` is optional. If not provided, the default connection ID `google_cloud_default` is used. project_name="jaffle_shop", ), profile_config=profile_config, @@ -103,9 +97,6 @@ manifest_path="abfs://cosmos-manifest-test/manifest.json", manifest_conn_id="azure_abfs_conn", # `manifest_conn_id` is optional. If not provided, the default connection ID `wasb_default` is used. - yaml_selectors_path="abfs://cosmos-manifest-test/selectors.yml", - yaml_selectors_conn_id="azure_abfs_conn", - # `yaml_selectors_conn_id` is optional. If not provided, the default connection ID `wasb_default` is used. project_name="jaffle_shop", ), profile_config=profile_config, From ec12d86d510939e12df8880067f8a325ca34a6c9 Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Fri, 23 Jan 2026 11:31:27 -0500 Subject: [PATCH 15/24] Implement unit/integration tests --- .github/workflows/test.yml | 2 + cosmos/cache.py | 7 +- cosmos/dbt/graph.py | 26 +- cosmos/dbt/selector.py | 37 +- tests/dbt/test_graph.py | 406 +- tests/dbt/test_selector.py | 523 +- tests/sample/manifest_selectors.json | 10365 +++++++++++++++++++++++++ tests/test_cache.py | 32 + 8 files changed, 11354 insertions(+), 44 deletions(-) create mode 100644 tests/sample/manifest_selectors.json diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 34fdd2222b..e7cb0a1692 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -197,6 +197,7 @@ jobs: hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}:test-integration env: AIRFLOW__COSMOS__ENABLE_CACHE_DBT_LS: 0 + AIRFLOW__COSMOS__ENABLE_CACHE_YAML_SELECTORS: 0 AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres AIRFLOW_CONN_AWS_S3_CONN: ${{ secrets.AIRFLOW_CONN_AWS_S3_CONN }} @@ -521,6 +522,7 @@ jobs: hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}:test-integration-dbtf env: AIRFLOW__COSMOS__ENABLE_CACHE_DBT_LS: 0 + AIRFLOW__COSMOS__ENABLE_CACHE_YAML_SELECTORS: 0 AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH diff --git a/cosmos/cache.py b/cosmos/cache.py index 4c4b49ed7a..01deb247b4 100644 --- a/cosmos/cache.py +++ b/cosmos/cache.py @@ -325,10 +325,9 @@ def _calculate_dbt_ls_cache_current_version(cache_identifier: str, project_dir: Taking into account the project directory contents and the command arguments, calculate the hash that represents the "dbt ls" command version - to be used to decide if the cache should be refreshed or not. - :param cache_identifier: str - Unique identifier of the cache (may include DbtDag or DbtTaskGroup information) - :param project_dir: Path - Path to the target dbt project directory - :param cmd_args: list[str] - List containing the arguments passed to the dbt ls command that would affect its output - :return: str - Combined hash string of project and command arguments + :param cache_identifier: Unique identifier of the cache (may include DbtDag or DbtTaskGroup information) + :param project_path: Path to the target dbt project directory + :param cmd_args: List containing the arguments passed to the dbt ls command that would affect its output """ start_time = time.perf_counter() diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index bc29f3e623..d7c23879d8 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -921,7 +921,7 @@ def _get_yaml_selectors_remote_cache(self, remote_cache_dir: Path | ObjectStorag cache_dict = json.load(fp) return cache_dict - def get_yaml_selector_cache(self) -> dict[str, Any]: + def get_yaml_selectors_cache(self) -> dict[str, Any]: """ Retrieve previously saved YAML selectors from an Airflow Variable. @@ -960,7 +960,7 @@ def get_yaml_selector_cache(self) -> dict[str, Any]: return cache_dict - def save_yaml_selector_cache(self, yaml_selectors: YamlSelectors) -> None: + def save_yaml_selectors_cache(self, yaml_selectors: YamlSelectors) -> None: """ Store parsed YAML selectors into an Airflow Variable. @@ -975,19 +975,19 @@ def save_yaml_selector_cache(self, yaml_selectors: YamlSelectors) -> None: serialized_data = pickle.dumps(yaml_selectors) compressed_data = zlib.compress(serialized_data) encoded_data = base64.b64encode(compressed_data) - selections_compressed = encoded_data.decode("utf-8") + selectors_encoded = encoded_data.decode("utf-8") cache_dict = { "version": cache._calculate_yaml_selectors_cache_current_version( self.cache_key, self.project_path, yaml_selectors.raw, yaml_selectors.impl_version ), - "yaml_selectors": selections_compressed, + "yaml_selectors": selectors_encoded, "last_modified": datetime.datetime.now(datetime.timezone.utc).isoformat(), **self.airflow_metadata, } remote_cache_dir = _configure_remote_cache_dir() if remote_cache_dir: - remote_cache_key_path = remote_cache_dir / self.cache_key / "yaml_selector_cache.json" + remote_cache_key_path = remote_cache_dir / self.cache_key / "yaml_selectors_cache.json" with remote_cache_key_path.open("w") as fp: json.dump(cache_dict, fp) else: @@ -1007,7 +1007,7 @@ def parse_yaml_selectors(self, selector_definitions: dict[str, Any]) -> YamlSele yaml_selectors = YamlSelectors.parse(selector_definitions) if self.should_use_yaml_selectors_cache(): - self.save_yaml_selector_cache(yaml_selectors) + self.save_yaml_selectors_cache(yaml_selectors) return yaml_selectors @@ -1023,21 +1023,13 @@ def load_parsed_selectors(self, selector_definitions: dict[str, Any]) -> YamlSel """ logger.info(f"Trying to parse the dbt yaml selectors using {self.cache_key}...") - def get_yaml_selectors(selector_definitions: dict[str, Any]) -> YamlSelectors: - yaml_selectors = self.parse_yaml_selectors(selector_definitions) - - if self.should_use_yaml_selectors_cache(): - self.save_yaml_selector_cache(yaml_selectors) - - return yaml_selectors - if self.should_use_yaml_selectors_cache(): - cache_dict = self.get_yaml_selector_cache() + cache_dict = self.get_yaml_selectors_cache() if not cache_dict: logger.info(f"Cosmos performance: Cache miss for {self.cache_key}") - return get_yaml_selectors(selector_definitions) + return self.parse_yaml_selectors(selector_definitions) cache_version: str = cache_dict["version"] yaml_selectors: YamlSelectors = cache_dict["yaml_selectors"] @@ -1056,7 +1048,7 @@ def get_yaml_selectors(selector_definitions: dict[str, Any]) -> YamlSelectors: logger.info(f"Cosmos performance: Cache miss for {self.cache_key} - skipped") - return get_yaml_selectors(selector_definitions) + return self.parse_yaml_selectors(selector_definitions) def load_from_dbt_manifest(self) -> None: """ diff --git a/cosmos/dbt/selector.py b/cosmos/dbt/selector.py index e8fae811fa..74f970bf40 100644 --- a/cosmos/dbt/selector.py +++ b/cosmos/dbt/selector.py @@ -713,6 +713,8 @@ class YamlSelectors: The original unparsed selector definitions from the manifest. :property parsed: dict[str, dict[str, Any]] The parsed selector definitions in Cosmos format. + :property impl_version: str + The source code of the YamlSelectors implementation, used for change detection. References: https://docs.getdbt.com/reference/node-selection/yaml-selectors @@ -789,10 +791,10 @@ def _get_list_dicts(dct: dict[str, Any], key: str) -> list[dict[str, Any]]: """ result: list[dict[str, Any]] = [] if key not in dct: - raise CosmosValueError(f"Expected to find key {key} in dict, only found {list(dct)}") + raise CosmosValueError(f"Expected to find key '{key}' in dict, only found {list(dct)}") values = dct[key] if not isinstance(values, list): - raise CosmosValueError(f'Invalid value for key "{key}". Expected a list.') + raise CosmosValueError(f"Invalid value for key '{key}'. Expected a list.") for value in values: if isinstance(value, dict): for value_key in value: @@ -840,14 +842,14 @@ def _parse_selection_graph_operators( raise CosmosValueError("childrens_parents cannot be combined with parents_depth or children_depth.") if childrens_parents: - return f"@{selector}" + return f"{AT_SELECTOR}{selector}" if parents: - prefix = f"{parents_depth}+" if parents_depth > 0 else "+" + prefix = f"{parents_depth}{PLUS_SELECTOR}" if parents_depth > 0 else PLUS_SELECTOR selector = f"{prefix}{selector}" if children: - suffix = f"+{children_depth}" if children_depth > 0 else "+" + suffix = f"{PLUS_SELECTOR}{children_depth}" if children_depth > 0 else PLUS_SELECTOR selector = f"{selector}{suffix}" return selector @@ -884,11 +886,11 @@ def _parse_selection_from_cosmos_spec(method: str, value: str) -> str: } for method_prefix, selector_prefix in method_mappings.items(): - if method.startswith(method_prefix): + if method == method_prefix: return f"{selector_prefix}{value}" if any(method.startswith(f"{CONFIG_SELECTOR}{config}") for config in SUPPORTED_CONFIG): - return f"{CONFIG_SELECTOR}:{value}" + return f"{method}:{value}" raise CosmosValueError(f"Unsupported selector method: '{method}'") @@ -903,7 +905,7 @@ def _parse_selector(cls, dct: dict[str, Any]) -> str: :param dct: dict[str, Any] - Dictionary containing selector definition with keys like 'method', 'value', 'parents', 'children', 'parents_depth', 'children_depth', 'childrens_parents' :return: str - A Cosmos-formatted selector string - :raises CosmosValueError: If required keys are missing or depth values are not integers + :raises CosmosValueError: If depth values are not integers Example Input: { @@ -916,19 +918,14 @@ def _parse_selector(cls, dct: dict[str, Any]) -> str: Example Output: "tag:nightly+2" """ - method = dct.get("method", "METHOD_KEY_NOT_FOUND") - value = dct.get("value", "VALUE_KEY_NOT_FOUND") + method = dct["method"] + value = dct["value"] parents = dct.get("parents", False) children = dct.get("children", False) parents_depth = dct.get("parents_depth", 0) children_depth = dct.get("children_depth", 0) childrens_parents = dct.get("childrens_parents", False) - if method == "METHOD_KEY_NOT_FOUND": - raise CosmosValueError(f"Selector method is missing in method definition: {dct}.") - if value == "VALUE_KEY_NOT_FOUND": - raise CosmosValueError(f"Selector value is missing in method definition: {dct}.") - if not isinstance(parents_depth, int): raise CosmosValueError(f"parents_depth must be an integer, got {type(parents_depth)}: {parents_depth}") if not isinstance(children_depth, int): @@ -1117,7 +1114,7 @@ def _parse_method_definition( exclude = cls._parse_exclusions(definition, cache=cache) dct = {k: v for k, v in dct.items() if k != "exclude"} else: - raise CosmosValueError(f'Expected "method" and "value" keys, but got {list(definition)}') + raise CosmosValueError(f"Expected 'method' and 'value' keys, but got {list(definition)}") selector = cls._parse_selector(dct) @@ -1171,7 +1168,7 @@ def _parse_from_definition( else: # Rejects CLI-style string definitions (e.g., definition: "tag:my_tag") # These should be structured as method-value definitions instead - + # # Rejects key-value definitions (e.g., tag: my_tag, path: models/, etc.) # These should be structured as method-value definitions instead raise CosmosValueError( @@ -1220,15 +1217,15 @@ def parse(cls, selectors: dict[str, dict[str, Any]]) -> YamlSelectors: name = definition.get("name", "") selector_definition = definition.get("definition", {}) - select, exclude = cls._parse_from_definition( - selector_definition, cache=copy.deepcopy(cache), rootlevel=True - ) + select, exclude = cls._parse_from_definition(selector_definition, cache=cache, rootlevel=True) result[name] = { "select": select if select else None, "exclude": exclude if exclude else None, } + cache[name] = (select, exclude) + return cls(selectors, result) diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index 91ee465885..8c3fd07321 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -3,6 +3,7 @@ import json import logging import os +import pickle import shutil import sys import tempfile @@ -36,6 +37,7 @@ parse_dbt_ls_output, run_command, ) +from cosmos.dbt.selector import YamlSelectors from cosmos.profiles import PostgresUserPasswordProfileMapping from cosmos.settings import AIRFLOW_IO_AVAILABLE @@ -47,6 +49,7 @@ SAMPLE_MANIFEST_PY = Path(__file__).parent.parent / "sample/manifest_python.json" SAMPLE_MANIFEST_MODEL_VERSION = Path(__file__).parent.parent / "sample/manifest_model_version.json" SAMPLE_MANIFEST_SOURCE = Path(__file__).parent.parent / "sample/manifest_source.json" +SAMPLE_MANIFEST_SELECTORS = Path(__file__).parent.parent / "sample/manifest_selectors.json" SAMPLE_DBT_LS_OUTPUT = Path(__file__).parent.parent / "sample/sample_dbt_ls.txt" SOURCE_RENDERING_BEHAVIOR = SourceRenderingBehavior(os.getenv("SOURCE_RENDERING_BEHAVIOR", "none")) @@ -369,6 +372,144 @@ def test_load_via_manifest_with_select(project_name, manifest_filepath, model_fi assert sample_node.file_path == DBT_PROJECTS_ROOT_DIR / f"{project_name}/models/{model_filepath}" +def test_load_via_manifest_with_selectors_and_missing_definitions(): + project_config = ProjectConfig( + dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME, manifest_path=SAMPLE_MANIFEST_MODEL_VERSION + ) + profile_config = ProfileConfig( + profile_name="test", + target_name="test", + profiles_yml_filepath=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME / "profiles.yml", + ) + render_config = RenderConfig( + load_method=LoadMode.DBT_MANIFEST, + selector="my_selector", + source_rendering_behavior=SOURCE_RENDERING_BEHAVIOR, + ) + execution_config = ExecutionConfig(dbt_project_path=project_config.dbt_project_path) + dbt_graph = DbtGraph( + project=project_config, + execution_config=execution_config, + profile_config=profile_config, + render_config=render_config, + ) + with pytest.raises(CosmosLoadDbtException) as err_info: + dbt_graph.load_from_dbt_manifest() + assert err_info.value.args[0] == f"Selectors not found in the manifest file {project_config.manifest_path}" + + +def test_load_via_manifest_with_selectors_and_missing_selector(): + project_config = ProjectConfig( + dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME, manifest_path=SAMPLE_MANIFEST_SELECTORS + ) + profile_config = ProfileConfig( + profile_name="test", + target_name="test", + profiles_yml_filepath=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME / "profiles.yml", + ) + render_config = RenderConfig( + load_method=LoadMode.DBT_MANIFEST, + selector="my_selector", + source_rendering_behavior=SOURCE_RENDERING_BEHAVIOR, + ) + execution_config = ExecutionConfig(dbt_project_path=project_config.dbt_project_path) + dbt_graph = DbtGraph( + project=project_config, + execution_config=execution_config, + profile_config=profile_config, + render_config=render_config, + ) + with pytest.raises(CosmosLoadDbtException) as err_info: + dbt_graph.load_from_dbt_manifest() + assert "Selector 'my_selector' not found in the manifest file" in err_info.value.args[0] + + +def test_load_via_manifest_with_selectors(): + project_config = ProjectConfig( + dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME, manifest_path=SAMPLE_MANIFEST_SELECTORS + ) + profile_config = ProfileConfig( + profile_name="test", + target_name="test", + profiles_yml_filepath=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME / "profiles.yml", + ) + render_config = RenderConfig( + load_method=LoadMode.DBT_MANIFEST, + selector="staging_models", + source_rendering_behavior=SOURCE_RENDERING_BEHAVIOR, + ) + execution_config = ExecutionConfig(dbt_project_path=project_config.dbt_project_path) + dbt_graph = DbtGraph( + project=project_config, + execution_config=execution_config, + profile_config=profile_config, + render_config=render_config, + ) + + dbt_graph.load_from_dbt_manifest() + + expected_keys = [ + "model.jaffle_shop.customers", + "model.jaffle_shop.orders", + "model.jaffle_shop.stg_customers", + "model.jaffle_shop.stg_orders", + "model.jaffle_shop.stg_payments", + "seed.jaffle_shop.raw_customers", + "seed.jaffle_shop.raw_orders", + "seed.jaffle_shop.raw_payments", + "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3", + "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad", + "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278", + "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d", + "test.jaffle_shop.not_null_orders_amount.106140f9fd", + "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49", + "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625", + "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59", + "test.jaffle_shop.not_null_orders_customer_id.c5f02694af", + "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a", + "test.jaffle_shop.not_null_orders_order_id.cf6c17daed", + "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa", + "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64", + "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075", + "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2", + "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1", + "test.jaffle_shop.unique_orders_order_id.fed79b3a6e", + "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada", + "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a", + "test.jaffle_shop.unique_stg_payments_payment_id.3744510712", + ] + expected_filtered_keys = [ + "model.jaffle_shop.stg_customers", + "model.jaffle_shop.stg_orders", + "model.jaffle_shop.stg_payments", + "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad", + "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278", + "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa", + "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64", + "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075", + "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada", + "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a", + "test.jaffle_shop.unique_stg_payments_payment_id.3744510712", + ] + + assert sorted(dbt_graph.nodes.keys()) == expected_keys + assert sorted(dbt_graph.filtered_nodes.keys()) == expected_filtered_keys + + assert len(dbt_graph.nodes) == 28 + assert len(dbt_graph.filtered_nodes) == 11 + + sample_node = dbt_graph.nodes["model.jaffle_shop.customers"] + assert sample_node.name == "customers" + assert sample_node.unique_id == "model.jaffle_shop.customers" + assert sample_node.resource_type == DbtResourceType.MODEL + assert sample_node.depends_on == [ + "model.jaffle_shop.stg_customers", + "model.jaffle_shop.stg_orders", + "model.jaffle_shop.stg_payments", + ] + assert sample_node.file_path == DBT_PROJECTS_ROOT_DIR / f"{DBT_PROJECT_NAME}/models/customers.sql" + + @patch("cosmos.dbt.graph.DbtGraph.load_from_dbt_manifest", return_value=None) def test_load_automatic_manifest_is_available(mock_load_from_dbt_manifest): project_config = ProjectConfig( @@ -1708,7 +1849,7 @@ def test_load_via_dbt_ls_render_config_no_partial_parse( assert "--no-partial-parse" in ls_command -@pytest.mark.parametrize("load_method", [LoadMode.DBT_MANIFEST, LoadMode.CUSTOM]) +@pytest.mark.parametrize("load_method", [LoadMode.CUSTOM]) def test_load_method_with_unsupported_render_config_selector_arg(load_method): """Tests that error is raised when RenderConfig.selector is used with LoadMode.DBT_MANIFEST or LoadMode.CUSTOM.""" @@ -1924,7 +2065,44 @@ def test_save_dbt_ls_cache(mock_variable_set, mock_datetime, tmp_dbt_project_dir assert hash_args == "d41d8cd98f00b204e9800998ecf8427e" if sys.platform == "darwin": # We faced inconsistent hashing versions depending on the version of MacOS/Linux - the following line aims to address these. - assert hash_dir in ("7abb868ed1c22e78de1c00429d950a77", "85cba4ef17dd7c161938da6980a6ff85") + assert hash_dir in ( + "7abb868ed1c22e78de1c00429d950a77", + "85cba4ef17dd7c161938da6980a6ff85", + "7e273d9b7569e959af96f4368b9a036e", + ) + else: + assert hash_dir == "85cba4ef17dd7c161938da6980a6ff85" + + +@patch("cosmos.dbt.graph.datetime") +@patch("cosmos.dbt.graph.Variable.set") +def test_save_yamls_selector_cache(mock_variable_set, mock_datetime, tmp_dbt_project_dir): + mock_datetime.datetime.now.return_value = datetime(2022, 1, 1, 12, 0, 0) + graph = DbtGraph(cache_identifier="something", project=ProjectConfig(dbt_project_path=tmp_dbt_project_dir)) + selectors = YamlSelectors( + {"staging_orders": {"name": "staging_orders", "definition": {"method": "tag", "value": "tag_a"}}}, + {"select": ["tag:tag_a"], "exclude": None}, + ) + graph.save_yaml_selectors_cache(selectors) + assert mock_variable_set.call_args[0][0] == "cosmos_cache__something" + assert ( + mock_variable_set.call_args[0][1]["yaml_selectors"] + == "eJwtjjsKAkEQRAU/i4ImXkKTPYCHMDEykKF3p90dmI/MR00EzTuzvYync9ixkq6iXkM9J5/vqIjWrQvGhVo2sQ6osY3OMy2PYPThHwO/efviB29oIjzcsqNViNAp2wnnJWZiKC0Y5L6ihcSzsioqZ4diZjD2TjKN8xPT9Ao6Yb45CeCUAlXiAj6gLHjZwSemeUZ2BQOq8N7qJJH3KTX1D+xVTPQ=" + ) + assert mock_variable_set.call_args[0][1]["last_modified"] == "2022-01-01T12:00:00" + version = mock_variable_set.call_args[0][1].get("version") + hash_dir, hash_selectors, hash_impl = version.split(",") + + assert hash_selectors == "fbfa164dd765f83c2941eeb019a7f7b4" + assert hash_impl == "535bf463068da8658a5183094a72545c" + + if sys.platform == "darwin": + # We faced inconsistent hashing versions depending on the version of MacOS/Linux - the following line aims to address these. + assert hash_dir in ( + "7abb868ed1c22e78de1c00429d950a77", + "85cba4ef17dd7c161938da6980a6ff85", + "7e273d9b7569e959af96f4368b9a036e", + ) else: assert hash_dir == "85cba4ef17dd7c161938da6980a6ff85" @@ -1935,12 +2113,37 @@ def test_get_dbt_ls_cache_returns_empty_if_non_json_var(airflow_variable): assert graph.get_dbt_ls_cache() == {} +@pytest.mark.integration +def test_get_yaml_selectors_cache_returns_empty_if_non_json_var(airflow_variable): + graph = DbtGraph(project=ProjectConfig()) + assert graph.get_yaml_selectors_cache() == {} + + @patch("cosmos.dbt.graph.Variable.get", return_value={"dbt_ls_compressed": "eJwrzs9NVcgvLSkoLQEAGpAEhg=="}) def test_get_dbt_ls_cache_returns_decoded_and_decompressed_value(mock_variable_get): graph = DbtGraph(project=ProjectConfig()) assert graph.get_dbt_ls_cache() == {"dbt_ls": "some output"} +@patch( + "cosmos.dbt.graph.Variable.get", + return_value={ + "yaml_selectors": "eJwtjjsKAkEQRAU/i4ImXkKTPYCHMDEykKF3p90dmI/MR00EzTuzvYync9ixkq6iXkM9J5/vqIjWrQvGhVo2sQ6osY3OMy2PYPThHwO/efviB29oIjzcsqNViNAp2wnnJWZiKC0Y5L6ihcSzsioqZ4diZjD2TjKN8xPT9Ao6Yb45CeCUAlXiAj6gLHjZwSemeUZ2BQOq8N7qJJH3KTX1D+xVTPQ=" + }, +) +def test_get_yaml_selectors_cache_returns_decoded_and_decompressed_value(mock_variable_get): + graph = DbtGraph(project=ProjectConfig()) + selectors = YamlSelectors( + {"staging_orders": {"name": "staging_orders", "definition": {"method": "tag", "value": "tag_a"}}}, + {"select": ["tag:tag_a"], "exclude": None}, + ) + result = graph.get_yaml_selectors_cache().get("yaml_selectors") + + assert result.raw == selectors.raw + assert result.parsed == selectors.parsed + assert result.impl_version == selectors.impl_version + + @patch("cosmos.dbt.graph.Variable.get", return_value={}) def test_get_dbt_ls_cache_returns_empty_dict_if_empty_dict_var(mock_variable_get): graph = DbtGraph(project=ProjectConfig()) @@ -2036,6 +2239,30 @@ def test_should_use_dbt_ls_cache(enable_cache, enable_cache_dbt_ls, cache_id, sh assert graph.should_use_dbt_ls_cache() == should_use +@pytest.mark.parametrize( + "enable_cache,enable_cache_yaml_selectors,cache_id,should_use", + [ + (False, True, "id", False), + (True, False, "id", False), + (False, False, "id", False), + (True, True, "", False), + (True, True, "id", True), + ], +) +def test_should_use_yaml_selectors_cache(enable_cache, enable_cache_yaml_selectors, cache_id, should_use): + with patch.dict( + os.environ, + { + "AIRFLOW__COSMOS__ENABLE_CACHE": str(enable_cache), + "AIRFLOW__COSMOS__ENABLE_CACHE_YAML_SELECTORS": str(enable_cache_yaml_selectors), + }, + ): + importlib.reload(settings) + graph = DbtGraph(cache_identifier=cache_id, project=ProjectConfig(dbt_project_path="/tmp")) + graph.should_use_yaml_selectors_cache.cache_clear() + assert graph.should_use_yaml_selectors_cache() == should_use + + @pytest.mark.skipif(not AIRFLOW_IO_AVAILABLE, reason="Airflow did not have Object Storage until the 2.8 release") @patch(object_storage_path) @patch("cosmos.config.ProjectConfig") @@ -2060,6 +2287,33 @@ def test_save_dbt_ls_cache_remote_cache_dir( mock_remote_cache_key_path.open.assert_called_once_with("w") +@pytest.mark.skipif(not AIRFLOW_IO_AVAILABLE, reason="Airflow did not have Object Storage until the 2.8 release") +@patch(object_storage_path) +@patch("cosmos.config.ProjectConfig") +@patch("cosmos.dbt.graph._configure_remote_cache_dir") +def test_save_yaml_selectors_remote_cache_dir( + mock_configure_remote_cache_dir, mock_project_config, mock_object_storage_path +): + mock_remote_cache_dir_path = mock_object_storage_path.return_value + mock_remote_cache_dir_path.exists.return_value = True + + mock_configure_remote_cache_dir.return_value = mock_remote_cache_dir_path + + yaml_selectors = YamlSelectors( + {"staging_orders": {"name": "staging_orders", "definition": {"method": "tag", "value": "tag_a"}}}, + {"select": ["tag:tag_a"], "exclude": None}, + ) + mock_project_config.dbt_vars = {"var1": "value1"} + mock_project_config.env_vars = {"var1": "value1"} + mock_project_config._calculate_yaml_selectors_current_version.return_value = "mock_version" + dbt_graph = DbtGraph(project=mock_project_config) + + dbt_graph.save_yaml_selectors_cache(yaml_selectors) + + mock_remote_cache_key_path = mock_remote_cache_dir_path / dbt_graph.cache_key / "yaml_selectors_cache.json" + mock_remote_cache_key_path.open.assert_called_once_with("w") + + @pytest.mark.skipif(not AIRFLOW_IO_AVAILABLE, reason="Airflow did not have Object Storage until the 2.8 release") @patch(object_storage_path) @patch("cosmos.config.ProjectConfig") @@ -2098,6 +2352,154 @@ def test_get_dbt_ls_cache_remote_cache_dir( assert result == expected_result +@pytest.mark.skipif(not AIRFLOW_IO_AVAILABLE, reason="Airflow did not have Object Storage until the 2.8 release") +@patch(object_storage_path) +@patch("cosmos.config.ProjectConfig") +@patch("cosmos.dbt.graph._configure_remote_cache_dir") +def test_get_yaml_selectors_remote_cache_dir( + mock_configure_remote_cache_dir, mock_project_config, mock_object_storage_path +): + mock_remote_cache_dir_path = mock_object_storage_path.return_value + mock_remote_cache_dir_path.exists.return_value = True + mock_configure_remote_cache_dir.return_value = mock_remote_cache_dir_path + + yaml_selectors = YamlSelectors( + {"staging_orders": {"name": "staging_orders", "definition": {"method": "tag", "value": "tag_a"}}}, + {"select": ["tag:tag_a"], "exclude": None}, + ) + + serialized_data = pickle.dumps(yaml_selectors) + compressed_data = zlib.compress(serialized_data) + encoded_data = base64.b64encode(compressed_data).decode("utf-8") + + cache_dict = { + "version": "cache-version", + "yaml_selectors": encoded_data, + "last_modified": "2024-08-13T12:34:56Z", + } + + mock_remote_cache_key_path = mock_remote_cache_dir_path / "some_cache_key" / "yaml_selectors_cache.json" + mock_remote_cache_key_path.exists.return_value = True + mock_remote_cache_key_path.open.return_value.__enter__.return_value.read.return_value = json.dumps(cache_dict) + + dbt_graph = DbtGraph(project=mock_project_config) + + result = dbt_graph.get_yaml_selectors_cache() + yaml_selectors_result = result.get("yaml_selectors") + + assert result["version"] == "cache-version" + assert yaml_selectors_result.raw == yaml_selectors.raw + assert yaml_selectors_result.parsed == yaml_selectors.parsed + assert yaml_selectors_result.impl_version == yaml_selectors.impl_version + assert result["last_modified"] == "2024-08-13T12:34:56Z" + + +@pytest.mark.parametrize( + "enable_cache,enable_cache_yaml_selectors,cache_id,should_use", + [ + (False, True, "id", False), + (True, False, "id", False), + (False, False, "id", False), + (True, True, "", False), + (True, True, "id", True), + ], +) +def test_parse_yaml_selectors_saves_cache(enable_cache, enable_cache_yaml_selectors, cache_id, should_use): + selector_definitions = { + "staging_orders": {"name": "staging_orders", "definition": {"method": "tag", "value": "tag_a"}} + } + + with patch.dict( + os.environ, + { + "AIRFLOW__COSMOS__ENABLE_CACHE": str(enable_cache), + "AIRFLOW__COSMOS__ENABLE_CACHE_YAML_SELECTORS": str(enable_cache_yaml_selectors), + }, + ): + importlib.reload(settings) + graph = DbtGraph(cache_identifier=cache_id, project=ProjectConfig(dbt_project_path="/tmp")) + graph.save_yaml_selectors_cache = MagicMock() + graph.should_use_yaml_selectors_cache.cache_clear() + + yaml_selectors = graph.parse_yaml_selectors(selector_definitions) + + if should_use: + graph.save_yaml_selectors_cache.assert_called_once_with(yaml_selectors) + else: + graph.save_yaml_selectors_cache.assert_not_called() + + +def test_load_parsed_selectors_with_cache_miss_skipped(caplog): + selector_definitions = { + "staging_orders": {"name": "staging_orders", "definition": {"method": "tag", "value": "tag_a"}} + } + + yaml_selectors = YamlSelectors( + selector_definitions, + {"select": ["tag:tag_a"], "exclude": None}, + ) + + graph = DbtGraph(project=ProjectConfig(dbt_project_path="/tmp")) + graph.parse_yaml_selectors = MagicMock(return_value=yaml_selectors) + graph.should_use_yaml_selectors_cache = MagicMock(return_value=False) + + result = graph.load_parsed_selectors(selector_definitions) + + graph.parse_yaml_selectors.assert_called_once() + assert result == yaml_selectors + assert "Cosmos performance: Cache miss" in caplog.text and " - skipped" in caplog.text + + +def test_load_parsed_selectors_with_cache_miss(caplog): + selector_definitions = { + "staging_orders": {"name": "staging_orders", "definition": {"method": "tag", "value": "tag_a"}} + } + + yaml_selectors = YamlSelectors( + selector_definitions, + {"select": ["tag:tag_a"], "exclude": None}, + ) + + graph = DbtGraph(project=ProjectConfig(dbt_project_path="/tmp")) + graph.parse_yaml_selectors = MagicMock(return_value=yaml_selectors) + graph.should_use_yaml_selectors_cache = MagicMock(return_value=True) + graph.get_yaml_selectors_cache = MagicMock(return_value=None) + + result = graph.load_parsed_selectors(selector_definitions) + + graph.parse_yaml_selectors.assert_called_once() + assert result == yaml_selectors + + assert "Cosmos performance: Cache miss" in caplog.text and " - skipped" not in caplog.text + + +def test_load_parsed_selectors_with_cache_hit(caplog): + selector_definitions = { + "staging_orders": {"name": "staging_orders", "definition": {"method": "tag", "value": "tag_a"}} + } + yaml_selectors = YamlSelectors( + selector_definitions, + {"select": ["tag:tag_a"], "exclude": None}, + ) + + graph = DbtGraph(project=ProjectConfig(dbt_project_path="/tmp")) + graph.parse_yaml_selectors = MagicMock() + graph.should_use_yaml_selectors_cache = MagicMock(return_value=True) + graph.get_yaml_selectors_cache = MagicMock( + return_value={"version": "dbt_project_hash_v1,yamlselectors_v1,impl_hash_v1", "yaml_selectors": yaml_selectors} + ) + + with patch( + "cosmos.cache._calculate_yaml_selectors_cache_current_version", + return_value="dbt_project_hash_v1,yamlselectors_v1,impl_hash_v1", + ): + result = graph.load_parsed_selectors(selector_definitions) + + graph.parse_yaml_selectors.assert_not_called() + assert result == yaml_selectors + assert "Cosmos performance: Cache hit" in caplog.text and "The cache size" in caplog.text + + def test__normalize_path(): """ This normalizes the path (e.g. declared inside a manifest.json file) when it was created using MS Windows instead diff --git a/tests/dbt/test_selector.py b/tests/dbt/test_selector.py index 86c1e7278b..94a1a8d403 100644 --- a/tests/dbt/test_selector.py +++ b/tests/dbt/test_selector.py @@ -4,7 +4,7 @@ from cosmos.constants import DbtResourceType from cosmos.dbt.graph import DbtNode -from cosmos.dbt.selector import NodeSelector, SelectorConfig, select_nodes +from cosmos.dbt.selector import NodeSelector, SelectorConfig, YamlSelectors, select_nodes from cosmos.exceptions import CosmosValueError SAMPLE_PROJ_PATH = Path("/home/user/path/dbt-proj/") @@ -1117,3 +1117,524 @@ def test_exposure_selector(): exposure_node_match.unique_id: exposure_node_match, } assert selected == expected, f"Expected only {exposure_node_match.unique_id} to match" + + +@pytest.mark.parametrize( + "selector_name, selector_definition, parsed_selector", + [ + ( + "path_method", + {"name": "path_method", "definition": {"method": "path", "value": "models/staging"}}, + {"select": ["path:models/staging"], "exclude": None}, + ), + ( + "tag_method", + {"name": "tag_method", "definition": {"method": "tag", "value": "nightly"}}, + {"select": ["tag:nightly"], "exclude": None}, + ), + ( + "fqn_method", + {"name": "fqn_method", "definition": {"method": "fqn", "value": "customers"}}, + {"select": ["customers"], "exclude": None}, + ), + ( + "fqn_star_method", + {"name": "fqn_star_method", "definition": {"method": "fqn", "value": "*"}}, + {"select": None, "exclude": None}, + ), + ( + "config_method_materialized", + { + "name": "config_method_materialized", + "definition": {"method": "config.materialized", "value": "incremental"}, + }, + {"select": ["config.materialized:incremental"], "exclude": None}, + ), + ( + "config_method_schema", + {"name": "config_method_schema", "definition": {"method": "config.schema", "value": "audit"}}, + {"select": ["config.schema:audit"], "exclude": None}, + ), + ( + "config_method_tags", + {"name": "config_method_tags", "definition": {"method": "config.tags", "value": "nightly"}}, + {"select": ["config.tags:nightly"], "exclude": None}, + ), + ( + "config_method_meta", + {"name": "config_method_meta", "definition": {"method": "config.meta.allow_pii", "value": "true"}}, + {"select": ["config.meta.allow_pii:true"], "exclude": None}, + ), + ( + "source_method", + {"name": "source_method", "definition": {"method": "source", "value": "raw_*"}}, + {"select": ["source:raw_*"], "exclude": None}, + ), + ( + "exposure_method", + {"name": "exposure_method", "definition": {"method": "exposure", "value": "daily_report"}}, + {"select": ["exposure:daily_report"], "exclude": None}, + ), + ( + "resource_type_selector_method", + {"name": "resource_type_selector_method", "definition": {"method": "resource_type", "value": "source"}}, + {"select": ["resource_type:source"], "exclude": None}, + ), + ( + "exclude_resource_type_selector_method", + { + "name": "exclude_resource_type_selector_method", + "definition": {"method": "exclude_resource_type", "value": "model"}, + }, + {"select": ["exclude_resource_type:model"], "exclude": None}, + ), + ( + "intersection_method", + { + "name": "intersection_method", + "definition": { + "intersection": [ + {"method": "tag", "value": "nightly"}, + {"method": "config.materialized", "value": "table"}, + ] + }, + }, + {"select": ["tag:nightly,config.materialized:table"], "exclude": None}, + ), + ( + "union_method", + { + "name": "union_method", + "definition": { + "union": [{"method": "tag", "value": "nightly"}, {"method": "config.materialized", "value": "view"}] + }, + }, + {"select": ["tag:nightly", "config.materialized:view"], "exclude": None}, + ), + ( + "exclude_method", + { + "name": "exclude_method", + "definition": {"method": "tag", "value": "", "exclude": [{"method": "tag", "value": "deprecated"}]}, + }, + {"select": ["tag:"], "exclude": ["tag:deprecated"]}, + ), + ( + "complex_method", + { + "name": "complex_method", + "definition": { + "union": [ + {"method": "tag", "value": "nightly"}, + {"method": "path", "value": "models/staging"}, + {"exclude": [{"method": "config.materialized", "value": "ephemeral"}]}, + ] + }, + }, + {"select": ["tag:nightly", "path:models/staging"], "exclude": ["config.materialized:ephemeral"]}, + ), + ], +) +def test_valid_method_yaml_selectors(selector_name, selector_definition, parsed_selector): + selectors = {selector_name: selector_definition} + yaml_selectors = YamlSelectors.parse(selectors) + + result = yaml_selectors.get_parsed(selector_name) + + assert result == parsed_selector + + +@pytest.mark.parametrize( + "selector_name, selector_definition, parsed_selector", + [ + ( + "with_parents_no_children", + {"name": "with_parents_no_children", "definition": {"method": "tag", "value": "nightly", "parents": True}}, + {"select": ["+tag:nightly"], "exclude": None}, + ), + ( + "with_children_no_parents", + {"name": "with_children_no_parents", "definition": {"method": "tag", "value": "nightly", "children": True}}, + {"select": ["tag:nightly+"], "exclude": None}, + ), + ( + "with_parents_and_children", + { + "name": "with_parents_and_children", + "definition": {"method": "tag", "value": "nightly", "parents": True, "children": True}, + }, + {"select": ["+tag:nightly+"], "exclude": None}, + ), + ( + "with_parents_depth", + { + "name": "with_parents_depth", + "definition": {"method": "tag", "value": "nightly", "parents": True, "parents_depth": 2}, + }, + {"select": ["2+tag:nightly"], "exclude": None}, + ), + ( + "with_parents_depth_and_children", + { + "name": "with_parents_depth_and_children", + "definition": { + "method": "tag", + "value": "nightly", + "parents": True, + "parents_depth": 2, + "children": True, + }, + }, + {"select": ["2+tag:nightly+"], "exclude": None}, + ), + ( + "with_children_depth", + { + "name": "with_children_depth", + "definition": {"method": "tag", "value": "nightly", "children": True, "children_depth": 3}, + }, + {"select": ["tag:nightly+3"], "exclude": None}, + ), + ( + "with_children_depth_and_parents", + { + "name": "with_children_depth_and_parents", + "definition": { + "method": "tag", + "value": "nightly", + "parents": True, + "children": True, + "children_depth": 3, + }, + }, + {"select": ["+tag:nightly+3"], "exclude": None}, + ), + ( + "with_parents_depth_and_children_depth", + { + "name": "with_parents_depth_and_children_depth", + "definition": { + "method": "tag", + "value": "nightly", + "parents": True, + "parents_depth": 2, + "children": True, + "children_depth": 3, + }, + }, + {"select": ["2+tag:nightly+3"], "exclude": None}, + ), + ( + "with_childrens_parents", + { + "name": "with_childrens_parents", + "definition": {"method": "tag", "value": "nightly", "childrens_parents": True}, + }, + {"select": ["@tag:nightly"], "exclude": None}, + ), + ], +) +def test_valid_graph_operator_yaml_selectors(selector_name, selector_definition, parsed_selector): + selectors = {selector_name: selector_definition} + yaml_selectors = YamlSelectors.parse(selectors) + + result = yaml_selectors.get_parsed(selector_name) + + assert result == parsed_selector + + +@pytest.mark.parametrize( + "selector_name, selector_definition, exception_msg", + [ + # Node selector methods supported by dbt but not by Cosmos filter selection + ( + "access_method", + {"name": "access_method", "definition": {"method": "access", "value": "public"}}, + "Unsupported selector method: 'access'", + ), + ( + "file_method", + {"name": "file_method", "definition": {"method": "file", "value": "my_model.sql"}}, + "Unsupported selector method: 'file'", + ), + ( + "group_method", + {"name": "group_method", "definition": {"method": "group", "value": "finance"}}, + "Unsupported selector method: 'group'", + ), + ( + "metric_method", + {"name": "metric_method", "definition": {"method": "metric", "value": "revenue"}}, + "Unsupported selector method: 'metric'", + ), + ( + "package_method", + {"name": "package_method", "definition": {"method": "package", "value": "dbt_utils"}}, + "Unsupported selector method: 'package'", + ), + ( + "result_method", + {"name": "result_method", "definition": {"method": "result", "value": "error"}}, + "Unsupported selector method: 'result'", + ), + ( + "saved_query_method", + {"name": "saved_query_method", "definition": {"method": "saved_query", "value": "quarterly_report"}}, + "Unsupported selector method: 'saved_query'", + ), + ( + "semantic_model_method", + {"name": "semantic_model_method", "definition": {"method": "semantic_model", "value": "customer_metrics"}}, + "Unsupported selector method: 'semantic_model'", + ), + ( + "source_status_method", + {"name": "source_status_method", "definition": {"method": "source_status", "value": "fresher"}}, + "Unsupported selector method: 'source_status'", + ), + ( + "state_method", + {"name": "state_method", "definition": {"method": "state", "value": "modified"}}, + "Unsupported selector method: 'state'", + ), + ( + "test_name_method", + {"name": "test_name_method", "definition": {"method": "test_name", "value": "unique_id"}}, + "Unsupported selector method: 'test_name'", + ), + ( + "test_type_method", + {"name": "test_type_method", "definition": {"method": "test_type", "value": "generic"}}, + "Unsupported selector method: 'test_type'", + ), + ( + "unit_test_method", + {"name": "unit_test_method", "definition": {"method": "unit_test", "value": "test_my_model"}}, + "Unsupported selector method: 'unit_test'", + ), + ( + "version_method", + {"name": "version_method", "definition": {"method": "version", "value": "latest"}}, + "Unsupported selector method: 'version'", + ), + ], +) +def test_invalid_cosmos_method_yaml_selectors(selector_name, selector_definition, exception_msg): + selectors = {selector_name: selector_definition} + + with pytest.raises(CosmosValueError) as err_info: + _ = YamlSelectors.parse(selectors) + + assert err_info.value.args[0] == exception_msg + + +@pytest.mark.parametrize( + "selector_name, selector_definition, exception_msg", + [ + ( + "multi_root_union", + { + "name": "multi_root_union", + "definition": { + "union": [{"method": "tag", "value": "nightly"}], + "intersection": [{"method": "tag", "value": "daily"}], + }, + }, + "Only a single 'union' or 'intersection' key is allowed in a root level selector definition; found union,intersection.", + ), + ( + "multi_root_intersection", + { + "name": "multi_root_intersection", + "definition": { + "intersection": [{"method": "tag", "value": "nightly"}], + "union": [{"method": "tag", "value": "daily"}], + }, + }, + "Only a single 'union' or 'intersection' key is allowed in a root level selector definition; found intersection,union.", + ), + ( + "cli_selector", + {"name": "cli_selector", "definition": "tag:nightly"}, + "Expected to find union, intersection, or method definition, instead found : tag:nightly", + ), + ( + "key_value_selector", + {"name": "key_value_selector", "definition": {"tag": "nightly"}}, + "Expected to find union, intersection, or method definition, instead found : {'tag': 'nightly'}", + ), + ( + "circular_reference_selector", + { + "name": "circular_reference_selector", + "definition": {"method": "selector", "value": "circular_reference_selector"}, + }, + "Existing selector definition for circular_reference_selector not found.", + ), + ( + "invalid_selector_keys", + {"name": "invalid_selector_keys", "definition": {"invalid_key": "invalid_value"}}, + "Expected to find union, intersection, or method definition, instead found : {'invalid_key': 'invalid_value'}", + ), + ( + "missing_method_key", + {"name": "missing_method_key", "definition": {"value": "nightly"}}, + "Expected to find union, intersection, or method definition, instead found : {'value': 'nightly'}", + ), + ( + "missing_value_key", + {"name": "missing_value_key", "definition": {"method": "tag"}}, + "Expected 'method' and 'value' keys, but got ['method']", + ), + ( + "missing_union_key", + {"name": "missing_union_key", "definition": {"union": {"method": "tag"}}}, + "Invalid value for key 'union'. Expected a list.", + ), + ( + "non_int_parents_depth", + { + "name": "non_int_parents_depth", + "definition": {"method": "tag", "value": "nightly", "parents": True, "parents_depth": "two"}, + }, + "parents_depth must be an integer, got : two", + ), + ( + "non_int_children_depth", + { + "name": "non_int_children_depth", + "definition": {"method": "tag", "value": "nightly", "children": True, "children_depth": "three"}, + }, + "children_depth must be an integer, got : three", + ), + ( + "childrens_parents_and_parents_depth", + { + "name": "childrens_parents_and_parents_depth", + "definition": {"method": "tag", "value": "nightly", "childrens_parents": True, "parents_depth": 2}, + }, + "childrens_parents cannot be combined with parents_depth or children_depth.", + ), + ( + "childrens_parents_and_children_depth", + { + "name": "childrens_parents_and_children_depth", + "definition": {"method": "tag", "value": "nightly", "childrens_parents": True, "children_depth": 2}, + }, + "childrens_parents cannot be combined with parents_depth or children_depth.", + ), + ( + "multiple_excludes_in_union", + { + "name": "multiple_excludes_in_union", + "definition": { + "union": [ + {"method": "tag", "value": "nightly", "exclude": [{"method": "tag", "value": "deprecated"}]}, + {"method": "path", "value": "models/", "exclude": [{"method": "tag", "value": "archived"}]}, + ] + }, + }, + "You cannot provide multiple exclude arguments to the same selector set operator:\nexclude:\n- method: tag\n value: archived\nmethod: path\nvalue: models/\n", + ), + ( + "unknown_random_method", + {"name": "unknown_random_method", "definition": {"method": "unknown_random", "value": "test"}}, + "Unsupported selector method: 'unknown_random'", + ), + ], +) +def test_invalid_dbt_method_yaml_selectors(selector_name, selector_definition, exception_msg): + selectors = {selector_name: selector_definition} + + with pytest.raises(CosmosValueError) as err_info: + _ = YamlSelectors.parse(selectors) + + assert err_info.value.args[0] == exception_msg + + +@pytest.mark.parametrize( + "selector_name, definition_key, invalid_value, expected_error_fragment", + [ + ("union_not_a_list_string", "union", "not_a_list", "Invalid value for key 'union'. Expected a list."), + ("intersection_not_a_list_int", "intersection", 42, "Invalid value for key 'intersection'. Expected a list."), + ("union_not_a_list_dict", "union", {"nested": "dict"}, "Invalid value for key 'union'. Expected a list."), + ], +) +def test_invalid_value_type_for_list_keys(selector_name, definition_key, invalid_value, expected_error_fragment): + selectors = {selector_name: {"name": selector_name, "definition": {definition_key: invalid_value}}} + + with pytest.raises(CosmosValueError) as err_info: + _ = YamlSelectors.parse(selectors) + + assert expected_error_fragment in err_info.value.args[0] + + +@pytest.mark.parametrize( + "selector_name, definition_key, invalid_list_item, item_type_str", + [ + ("string_in_union_list", "union", "tag:daily", ""), + ("integer_in_union_list", "union", 42, ""), + ("none_in_intersection_list", "intersection", None, ""), + ("boolean_in_union_list", "union", True, ""), + ("list_in_union_list", "union", ["nested", "list"], ""), + ("float_in_intersection_list", "intersection", 3.14, ""), + ], +) +def test_invalid_items_in_selector_lists(selector_name, definition_key, invalid_list_item, item_type_str): + selectors = { + selector_name: { + "name": selector_name, + "definition": {definition_key: [{"method": "tag", "value": "nightly"}, invalid_list_item]}, + } + } + + with pytest.raises(CosmosValueError) as err_info: + _ = YamlSelectors.parse(selectors) + + assert ( + f'Invalid value type {item_type_str} in key "{definition_key}", expected dict or str (value: {invalid_list_item}).' + in err_info.value.args[0] + ) + + +@pytest.mark.parametrize( + "selector_name, definition_key, non_string_key, key_type_str", + [ + ("int_key_in_union", "union", 123, ""), + ("float_key_in_intersection", "intersection", 3.14, ""), + ("tuple_key_in_union", "union", (1, 2), ""), + ("bool_key_in_intersection", "intersection", True, ""), + ("none_key_in_union", "union", None, ""), + ], +) +def test_non_string_keys_in_selector_dicts(selector_name, definition_key, non_string_key, key_type_str): + selectors = { + selector_name: { + "name": selector_name, + "definition": {definition_key: [{non_string_key: "value", "method": "tag", "value": "nightly"}]}, + } + } + + with pytest.raises(CosmosValueError) as err_info: + _ = YamlSelectors.parse(selectors) + + assert f'Expected all keys to "{definition_key}" dict to be strings' in err_info.value.args[0] + assert f'but "{non_string_key}" is a "{key_type_str}"' in err_info.value.args[0] + + +def test_selector_reference_resolves_from_cache(): + selectors = { + "base_selector": {"name": "base_selector", "definition": {"method": "tag", "value": "nightly"}}, + "reference_selector": { + "name": "reference_selector", + "definition": {"method": "selector", "value": "base_selector"}, + }, + } + + yaml_selectors = YamlSelectors.parse(selectors) + + base_result = yaml_selectors.get_parsed("base_selector") + reference_result = yaml_selectors.get_parsed("reference_selector") + + assert base_result == {"select": ["tag:nightly"], "exclude": None} + assert reference_result == base_result diff --git a/tests/sample/manifest_selectors.json b/tests/sample/manifest_selectors.json new file mode 100644 index 0000000000..751be357bf --- /dev/null +++ b/tests/sample/manifest_selectors.json @@ -0,0 +1,10365 @@ +{ + "child_map": { + "model.jaffle_shop.customers": [ + "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d", + "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2", + "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1" + ], + "model.jaffle_shop.orders": [ + "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3", + "test.jaffle_shop.not_null_orders_amount.106140f9fd", + "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49", + "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625", + "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59", + "test.jaffle_shop.not_null_orders_customer_id.c5f02694af", + "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a", + "test.jaffle_shop.not_null_orders_order_id.cf6c17daed", + "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2", + "test.jaffle_shop.unique_orders_order_id.fed79b3a6e" + ], + "model.jaffle_shop.stg_customers": [ + "model.jaffle_shop.customers", + "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa", + "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada" + ], + "model.jaffle_shop.stg_orders": [ + "model.jaffle_shop.customers", + "model.jaffle_shop.orders", + "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad", + "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64", + "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a" + ], + "model.jaffle_shop.stg_payments": [ + "model.jaffle_shop.customers", + "model.jaffle_shop.orders", + "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278", + "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075", + "test.jaffle_shop.unique_stg_payments_payment_id.3744510712" + ], + "seed.jaffle_shop.raw_customers": [ + "model.jaffle_shop.stg_customers" + ], + "seed.jaffle_shop.raw_orders": [ + "model.jaffle_shop.stg_orders" + ], + "seed.jaffle_shop.raw_payments": [ + "model.jaffle_shop.stg_payments" + ], + "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3": [], + "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad": [], + "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278": [], + "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d": [], + "test.jaffle_shop.not_null_orders_amount.106140f9fd": [], + "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49": [], + "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625": [], + "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59": [], + "test.jaffle_shop.not_null_orders_customer_id.c5f02694af": [], + "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a": [], + "test.jaffle_shop.not_null_orders_order_id.cf6c17daed": [], + "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa": [], + "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64": [], + "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075": [], + "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2": [], + "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1": [], + "test.jaffle_shop.unique_orders_order_id.fed79b3a6e": [], + "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada": [], + "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a": [], + "test.jaffle_shop.unique_stg_payments_payment_id.3744510712": [] + }, + "disabled": {}, + "docs": { + "doc.dbt.__overview__": { + "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion", + "name": "__overview__", + "original_file_path": "docs/overview.md", + "package_name": "dbt", + "path": "overview.md", + "resource_type": "doc", + "unique_id": "doc.dbt.__overview__" + }, + "doc.jaffle_shop.__overview__": { + "block_contents": "## Data Documentation for Jaffle Shop\n\n`jaffle_shop` is a fictional ecommerce store.\n\nThis [dbt](https://www.getdbt.com/) project is for testing out code.\n\nThe source code can be found [here](https://github.com/clrcrl/jaffle_shop).", + "name": "__overview__", + "original_file_path": "models/overview.md", + "package_name": "jaffle_shop", + "path": "overview.md", + "resource_type": "doc", + "unique_id": "doc.jaffle_shop.__overview__" + }, + "doc.jaffle_shop.orders_status": { + "block_contents": "Orders can be one of the following statuses:\n\n| status | description |\n|----------------|------------------------------------------------------------------------------------------------------------------------|\n| placed | The order has been placed but has not yet left the warehouse |\n| shipped | The order has ben shipped to the customer and is currently in transit |\n| completed | The order has been received by the customer |\n| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse |\n| returned | The order has been returned by the customer and received at the warehouse |", + "name": "orders_status", + "original_file_path": "models/docs.md", + "package_name": "jaffle_shop", + "path": "docs.md", + "resource_type": "doc", + "unique_id": "doc.jaffle_shop.orders_status" + } + }, + "exposures": {}, + "group_map": {}, + "groups": {}, + "macros": { + "macro.dbt._split_part_negative": { + "arguments": [], + "created_at": 1687942823.00502, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 {{ part_number }}\n )\n\n{% endmacro %}", + "meta": {}, + "name": "_split_part_negative", + "original_file_path": "macros/utils/split_part.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/split_part.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt._split_part_negative" + }, + "macro.dbt.after_commit": { + "arguments": [], + "created_at": 1687942822.871392, + "depends_on": { + "macros": [ + "macro.dbt.make_hook_config" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", + "meta": {}, + "name": "after_commit", + "original_file_path": "macros/materializations/hooks.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/hooks.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.after_commit" + }, + "macro.dbt.alter_column_comment": { + "arguments": [], + "created_at": 1687942823.028165, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__alter_column_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", + "meta": {}, + "name": "alter_column_comment", + "original_file_path": "macros/adapters/persist_docs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/persist_docs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.alter_column_comment" + }, + "macro.dbt.alter_column_type": { + "arguments": [], + "created_at": 1687942823.039871, + "depends_on": { + "macros": [ + "macro.dbt.default__alter_column_type" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", + "meta": {}, + "name": "alter_column_type", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.alter_column_type" + }, + "macro.dbt.alter_relation_add_remove_columns": { + "arguments": [], + "created_at": 1687942823.040888, + "depends_on": { + "macros": [ + "macro.dbt.default__alter_relation_add_remove_columns" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", + "meta": {}, + "name": "alter_relation_add_remove_columns", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.alter_relation_add_remove_columns" + }, + "macro.dbt.alter_relation_comment": { + "arguments": [], + "created_at": 1687942823.0285828, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__alter_relation_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", + "meta": {}, + "name": "alter_relation_comment", + "original_file_path": "macros/adapters/persist_docs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/persist_docs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.alter_relation_comment" + }, + "macro.dbt.any_value": { + "arguments": [], + "created_at": 1687942822.997621, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__any_value" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro any_value(expression) -%}\n {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}\n{% endmacro %}", + "meta": {}, + "name": "any_value", + "original_file_path": "macros/utils/any_value.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/any_value.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.any_value" + }, + "macro.dbt.apply_grants": { + "arguments": [], + "created_at": 1687942823.025873, + "depends_on": { + "macros": [ + "macro.dbt.default__apply_grants" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro apply_grants(relation, grant_config, should_revoke) %}\n {{ return(adapter.dispatch(\"apply_grants\", \"dbt\")(relation, grant_config, should_revoke)) }}\n{% endmacro %}", + "meta": {}, + "name": "apply_grants", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.apply_grants" + }, + "macro.dbt.array_append": { + "arguments": [], + "created_at": 1687942823.0065181, + "depends_on": { + "macros": [ + "macro.dbt.default__array_append" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro array_append(array, new_element) -%}\n {{ return(adapter.dispatch('array_append', 'dbt')(array, new_element)) }}\n{%- endmacro %}", + "meta": {}, + "name": "array_append", + "original_file_path": "macros/utils/array_append.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/array_append.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.array_append" + }, + "macro.dbt.array_concat": { + "arguments": [], + "created_at": 1687942823.002666, + "depends_on": { + "macros": [ + "macro.dbt.default__array_concat" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro array_concat(array_1, array_2) -%}\n {{ return(adapter.dispatch('array_concat', 'dbt')(array_1, array_2)) }}\n{%- endmacro %}", + "meta": {}, + "name": "array_concat", + "original_file_path": "macros/utils/array_concat.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/array_concat.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.array_concat" + }, + "macro.dbt.array_construct": { + "arguments": [], + "created_at": 1687942823.00587, + "depends_on": { + "macros": [ + "macro.dbt.default__array_construct" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro array_construct(inputs=[], data_type=api.Column.translate_type('integer')) -%}\n {{ return(adapter.dispatch('array_construct', 'dbt')(inputs, data_type)) }}\n{%- endmacro %}", + "meta": {}, + "name": "array_construct", + "original_file_path": "macros/utils/array_construct.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/array_construct.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.array_construct" + }, + "macro.dbt.assert_columns_equivalent": { + "arguments": [], + "created_at": 1687942822.948245, + "depends_on": { + "macros": [ + "macro.dbt.get_column_schema_from_query", + "macro.dbt.get_empty_schema_sql", + "macro.dbt.format_columns" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro assert_columns_equivalent(sql) %}\n {#-- Obtain the column schema provided by sql file. #}\n {%- set sql_file_provided_columns = get_column_schema_from_query(sql, config.get('sql_header', none)) -%}\n {#--Obtain the column schema provided by the schema file by generating an 'empty schema' query from the model's columns. #}\n {%- set schema_file_provided_columns = get_column_schema_from_query(get_empty_schema_sql(model['columns'])) -%}\n\n {#-- create dictionaries with name and formatted data type and strings for exception #}\n {%- set sql_columns = format_columns(sql_file_provided_columns) -%}\n {%- set yaml_columns = format_columns(schema_file_provided_columns) -%}\n\n {%- if sql_columns|length != yaml_columns|length -%}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n\n {%- for sql_col in sql_columns -%}\n {%- set yaml_col = [] -%}\n {%- for this_col in yaml_columns -%}\n {%- if this_col['name'] == sql_col['name'] -%}\n {%- do yaml_col.append(this_col) -%}\n {%- break -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if not yaml_col -%}\n {#-- Column with name not found in yaml #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- if sql_col['formatted'] != yaml_col[0]['formatted'] -%}\n {#-- Column data types don't match #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- endfor -%}\n\n{% endmacro %}", + "meta": {}, + "name": "assert_columns_equivalent", + "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.assert_columns_equivalent" + }, + "macro.dbt.before_begin": { + "arguments": [], + "created_at": 1687942822.871079, + "depends_on": { + "macros": [ + "macro.dbt.make_hook_config" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", + "meta": {}, + "name": "before_begin", + "original_file_path": "macros/materializations/hooks.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/hooks.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.before_begin" + }, + "macro.dbt.bool_or": { + "arguments": [], + "created_at": 1687942823.00308, + "depends_on": { + "macros": [ + "macro.dbt.default__bool_or" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro bool_or(expression) -%}\n {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}\n{% endmacro %}", + "meta": {}, + "name": "bool_or", + "original_file_path": "macros/utils/bool_or.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/bool_or.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.bool_or" + }, + "macro.dbt.build_config_dict": { + "arguments": [], + "created_at": 1687942823.045846, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro build_config_dict(model) %}\n {%- set config_dict = {} -%}\n {% set config_dbt_used = zip(model.config.config_keys_used, model.config.config_keys_defaults) | list %}\n {%- for key, default in config_dbt_used -%}\n {# weird type testing with enum, would be much easier to write this logic in Python! #}\n {%- if key == \"language\" -%}\n {%- set value = \"python\" -%}\n {%- endif -%}\n {%- set value = model.config.get(key, default) -%}\n {%- do config_dict.update({key: value}) -%}\n {%- endfor -%}\nconfig_dict = {{ config_dict }}\n{% endmacro %}", + "meta": {}, + "name": "build_config_dict", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.build_config_dict" + }, + "macro.dbt.build_ref_function": { + "arguments": [], + "created_at": 1687942823.0447218, + "depends_on": { + "macros": [ + "macro.dbt.resolve_model_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro build_ref_function(model) %}\n\n {%- set ref_dict = {} -%}\n {%- for _ref in model.refs -%}\n {% set _ref_args = [_ref.get('package'), _ref['name']] if _ref.get('package') else [_ref['name'],] %}\n {%- set resolved = ref(*_ref_args, v=_ref.get('version')) -%}\n {%- if _ref.get('version') -%}\n {% do _ref_args.extend([\"v\" ~ _ref['version']]) %}\n {%- endif -%}\n {%- do ref_dict.update({_ref_args | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef ref(*args, **kwargs):\n refs = {{ ref_dict | tojson }}\n key = '.'.join(args)\n version = kwargs.get(\"v\") or kwargs.get(\"version\")\n if version:\n key += f\".v{version}\"\n dbt_load_df_function = kwargs.get(\"dbt_load_df_function\")\n return dbt_load_df_function(refs[key])\n\n{% endmacro %}", + "meta": {}, + "name": "build_ref_function", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.build_ref_function" + }, + "macro.dbt.build_snapshot_staging_table": { + "arguments": [], + "created_at": 1687942822.892981, + "depends_on": { + "macros": [ + "macro.dbt.make_temp_relation", + "macro.dbt.snapshot_staging_table", + "macro.dbt.statement", + "macro.dbt.create_table_as" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set temp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, temp_relation, select) }}\n {% endcall %}\n\n {% do return(temp_relation) %}\n{% endmacro %}", + "meta": {}, + "name": "build_snapshot_staging_table", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.build_snapshot_staging_table" + }, + "macro.dbt.build_snapshot_table": { + "arguments": [], + "created_at": 1687942822.892051, + "depends_on": { + "macros": [ + "macro.dbt.default__build_snapshot_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", + "meta": {}, + "name": "build_snapshot_table", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.build_snapshot_table" + }, + "macro.dbt.build_source_function": { + "arguments": [], + "created_at": 1687942823.045177, + "depends_on": { + "macros": [ + "macro.dbt.resolve_model_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro build_source_function(model) %}\n\n {%- set source_dict = {} -%}\n {%- for _source in model.sources -%}\n {%- set resolved = source(*_source) -%}\n {%- do source_dict.update({_source | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef source(*args, dbt_load_df_function):\n sources = {{ source_dict | tojson }}\n key = '.'.join(args)\n return dbt_load_df_function(sources[key])\n\n{% endmacro %}", + "meta": {}, + "name": "build_source_function", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.build_source_function" + }, + "macro.dbt.call_dcl_statements": { + "arguments": [], + "created_at": 1687942823.0253298, + "depends_on": { + "macros": [ + "macro.dbt.default__call_dcl_statements" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro call_dcl_statements(dcl_statement_list) %}\n {{ return(adapter.dispatch(\"call_dcl_statements\", \"dbt\")(dcl_statement_list)) }}\n{% endmacro %}", + "meta": {}, + "name": "call_dcl_statements", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.call_dcl_statements" + }, + "macro.dbt.cast_bool_to_text": { + "arguments": [], + "created_at": 1687942822.9970849, + "depends_on": { + "macros": [ + "macro.dbt.default__cast_bool_to_text" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro cast_bool_to_text(field) %}\n {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}\n{% endmacro %}", + "meta": {}, + "name": "cast_bool_to_text", + "original_file_path": "macros/utils/cast_bool_to_text.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/cast_bool_to_text.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.cast_bool_to_text" + }, + "macro.dbt.check_for_schema_changes": { + "arguments": [], + "created_at": 1687942822.9420822, + "depends_on": { + "macros": [ + "macro.dbt.diff_columns", + "macro.dbt.diff_column_data_types" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", + "meta": {}, + "name": "check_for_schema_changes", + "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/on_schema_change.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.check_for_schema_changes" + }, + "macro.dbt.check_schema_exists": { + "arguments": [], + "created_at": 1687942823.032896, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__check_schema_exists" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", + "meta": {}, + "name": "check_schema_exists", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.check_schema_exists" + }, + "macro.dbt.collect_freshness": { + "arguments": [], + "created_at": 1687942823.019376, + "depends_on": { + "macros": [ + "macro.dbt.default__collect_freshness" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", + "meta": {}, + "name": "collect_freshness", + "original_file_path": "macros/adapters/freshness.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/freshness.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.collect_freshness" + }, + "macro.dbt.concat": { + "arguments": [], + "created_at": 1687942822.991219, + "depends_on": { + "macros": [ + "macro.dbt.default__concat" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro concat(fields) -%}\n {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}\n{%- endmacro %}", + "meta": {}, + "name": "concat", + "original_file_path": "macros/utils/concat.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/concat.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.concat" + }, + "macro.dbt.convert_datetime": { + "arguments": [], + "created_at": 1687942822.9874282, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", + "meta": {}, + "name": "convert_datetime", + "original_file_path": "macros/etc/datetime.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/datetime.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.convert_datetime" + }, + "macro.dbt.copy_grants": { + "arguments": [], + "created_at": 1687942823.0217059, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__copy_grants" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro copy_grants() %}\n {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "copy_grants", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.copy_grants" + }, + "macro.dbt.create_columns": { + "arguments": [], + "created_at": 1687942822.88876, + "depends_on": { + "macros": [ + "macro.dbt.default__create_columns" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", + "meta": {}, + "name": "create_columns", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_columns" + }, + "macro.dbt.create_csv_table": { + "arguments": [], + "created_at": 1687942822.9726899, + "depends_on": { + "macros": [ + "macro.dbt.default__create_csv_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", + "meta": {}, + "name": "create_csv_table", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_csv_table" + }, + "macro.dbt.create_indexes": { + "arguments": [], + "created_at": 1687942823.010463, + "depends_on": { + "macros": [ + "macro.dbt.default__create_indexes" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", + "meta": {}, + "name": "create_indexes", + "original_file_path": "macros/adapters/indexes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/indexes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_indexes" + }, + "macro.dbt.create_or_replace_view": { + "arguments": [], + "created_at": 1687942822.961272, + "depends_on": { + "macros": [ + "macro.dbt.run_hooks", + "macro.dbt.handle_existing_table", + "macro.dbt.should_full_refresh", + "macro.dbt.statement", + "macro.dbt.get_create_view_as_sql", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", + "meta": {}, + "name": "create_or_replace_view", + "original_file_path": "macros/materializations/models/view/create_or_replace_view.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/view/create_or_replace_view.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_or_replace_view" + }, + "macro.dbt.create_schema": { + "arguments": [], + "created_at": 1687942823.007079, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__create_schema" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", + "meta": {}, + "name": "create_schema", + "original_file_path": "macros/adapters/schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_schema" + }, + "macro.dbt.create_table_as": { + "arguments": [], + "created_at": 1687942822.954165, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__create_table_as" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {# backward compatibility for create_table_as that does not support language #}\n {% if language == \"sql\" %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code)}}\n {% else %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code, language) }}\n {% endif %}\n\n{%- endmacro %}", + "meta": {}, + "name": "create_table_as", + "original_file_path": "macros/materializations/models/table/create_table_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/create_table_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_table_as" + }, + "macro.dbt.create_view_as": { + "arguments": [], + "created_at": 1687942822.962175, + "depends_on": { + "macros": [ + "macro.dbt.default__create_view_as" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", + "meta": {}, + "name": "create_view_as", + "original_file_path": "macros/materializations/models/view/create_view_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/view/create_view_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_view_as" + }, + "macro.dbt.current_timestamp": { + "arguments": [], + "created_at": 1687942823.008286, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__current_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "current_timestamp", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.current_timestamp" + }, + "macro.dbt.current_timestamp_backcompat": { + "arguments": [], + "created_at": 1687942823.008908, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__current_timestamp_backcompat" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro current_timestamp_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "current_timestamp_backcompat", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.current_timestamp_backcompat" + }, + "macro.dbt.current_timestamp_in_utc_backcompat": { + "arguments": [], + "created_at": 1687942823.009182, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_in_utc_backcompat', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "current_timestamp_in_utc_backcompat", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.current_timestamp_in_utc_backcompat" + }, + "macro.dbt.date_trunc": { + "arguments": [], + "created_at": 1687942823.005321, + "depends_on": { + "macros": [ + "macro.dbt.default__date_trunc" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro date_trunc(datepart, date) -%}\n {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}\n{%- endmacro %}", + "meta": {}, + "name": "date_trunc", + "original_file_path": "macros/utils/date_trunc.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/date_trunc.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.date_trunc" + }, + "macro.dbt.dateadd": { + "arguments": [], + "created_at": 1687942822.992304, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__dateadd" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro dateadd(datepart, interval, from_date_or_timestamp) %}\n {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}\n{% endmacro %}", + "meta": {}, + "name": "dateadd", + "original_file_path": "macros/utils/dateadd.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/dateadd.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.dateadd" + }, + "macro.dbt.datediff": { + "arguments": [], + "created_at": 1687942822.995577, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__datediff" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro datediff(first_date, second_date, datepart) %}\n {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}\n{% endmacro %}", + "meta": {}, + "name": "datediff", + "original_file_path": "macros/utils/datediff.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/datediff.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.datediff" + }, + "macro.dbt.dates_in_range": { + "arguments": [], + "created_at": 1687942822.9888482, + "depends_on": { + "macros": [ + "macro.dbt.convert_datetime" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", + "meta": {}, + "name": "dates_in_range", + "original_file_path": "macros/etc/datetime.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/datetime.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.dates_in_range" + }, + "macro.dbt.default__alter_column_comment": { + "arguments": [], + "created_at": 1687942823.028358, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", + "meta": {}, + "name": "default__alter_column_comment", + "original_file_path": "macros/adapters/persist_docs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/persist_docs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__alter_column_comment" + }, + "macro.dbt.default__alter_column_type": { + "arguments": [], + "created_at": 1687942823.040595, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__alter_column_type", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__alter_column_type" + }, + "macro.dbt.default__alter_relation_add_remove_columns": { + "arguments": [], + "created_at": 1687942823.041811, + "depends_on": { + "macros": [ + "macro.dbt.run_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__alter_relation_add_remove_columns", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__alter_relation_add_remove_columns" + }, + "macro.dbt.default__alter_relation_comment": { + "arguments": [], + "created_at": 1687942823.02877, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", + "meta": {}, + "name": "default__alter_relation_comment", + "original_file_path": "macros/adapters/persist_docs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/persist_docs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__alter_relation_comment" + }, + "macro.dbt.default__any_value": { + "arguments": [], + "created_at": 1687942822.997757, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__any_value(expression) -%}\n\n any_value({{ expression }})\n\n{%- endmacro %}", + "meta": {}, + "name": "default__any_value", + "original_file_path": "macros/utils/any_value.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/any_value.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__any_value" + }, + "macro.dbt.default__apply_grants": { + "arguments": [], + "created_at": 1687942823.027256, + "depends_on": { + "macros": [ + "macro.dbt.run_query", + "macro.dbt.get_show_grant_sql", + "macro.dbt.get_dcl_statement_list", + "macro.dbt.call_dcl_statements" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "default__apply_grants", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__apply_grants" + }, + "macro.dbt.default__array_append": { + "arguments": [], + "created_at": 1687942823.006666, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__array_append(array, new_element) -%}\n array_append({{ array }}, {{ new_element }})\n{%- endmacro %}", + "meta": {}, + "name": "default__array_append", + "original_file_path": "macros/utils/array_append.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/array_append.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__array_append" + }, + "macro.dbt.default__array_concat": { + "arguments": [], + "created_at": 1687942823.002811, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__array_concat(array_1, array_2) -%}\n array_cat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", + "meta": {}, + "name": "default__array_concat", + "original_file_path": "macros/utils/array_concat.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/array_concat.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__array_concat" + }, + "macro.dbt.default__array_construct": { + "arguments": [], + "created_at": 1687942823.006149, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__array_construct(inputs, data_type) -%}\n {% if inputs|length > 0 %}\n array[ {{ inputs|join(' , ') }} ]\n {% else %}\n array[]::{{data_type}}[]\n {% endif %}\n{%- endmacro %}", + "meta": {}, + "name": "default__array_construct", + "original_file_path": "macros/utils/array_construct.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/array_construct.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__array_construct" + }, + "macro.dbt.default__bool_or": { + "arguments": [], + "created_at": 1687942823.003194, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__bool_or(expression) -%}\n\n bool_or({{ expression }})\n\n{%- endmacro %}", + "meta": {}, + "name": "default__bool_or", + "original_file_path": "macros/utils/bool_or.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/bool_or.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__bool_or" + }, + "macro.dbt.default__build_snapshot_table": { + "arguments": [], + "created_at": 1687942822.8924742, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n\n select *,\n {{ strategy.scd_id }} as dbt_scd_id,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", + "meta": {}, + "name": "default__build_snapshot_table", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__build_snapshot_table" + }, + "macro.dbt.default__call_dcl_statements": { + "arguments": [], + "created_at": 1687942823.02561, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__call_dcl_statements(dcl_statement_list) %}\n {#\n -- By default, supply all grant + revoke statements in a single semicolon-separated block,\n -- so that they're all processed together.\n\n -- Some databases do not support this. Those adapters will need to override this macro\n -- to run each statement individually.\n #}\n {% call statement('grants') %}\n {% for dcl_statement in dcl_statement_list %}\n {{ dcl_statement }};\n {% endfor %}\n {% endcall %}\n{% endmacro %}", + "meta": {}, + "name": "default__call_dcl_statements", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__call_dcl_statements" + }, + "macro.dbt.default__cast_bool_to_text": { + "arguments": [], + "created_at": 1687942822.9972851, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__cast_bool_to_text(field) %}\n cast({{ field }} as {{ api.Column.translate_type('string') }})\n{% endmacro %}", + "meta": {}, + "name": "default__cast_bool_to_text", + "original_file_path": "macros/utils/cast_bool_to_text.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/cast_bool_to_text.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__cast_bool_to_text" + }, + "macro.dbt.default__check_schema_exists": { + "arguments": [], + "created_at": 1687942823.033236, + "depends_on": { + "macros": [ + "macro.dbt.replace", + "macro.dbt.run_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "default__check_schema_exists", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__check_schema_exists" + }, + "macro.dbt.default__collect_freshness": { + "arguments": [], + "created_at": 1687942823.019836, + "depends_on": { + "macros": [ + "macro.dbt.statement", + "macro.dbt.current_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness')) }}\n{% endmacro %}", + "meta": {}, + "name": "default__collect_freshness", + "original_file_path": "macros/adapters/freshness.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/freshness.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__collect_freshness" + }, + "macro.dbt.default__concat": { + "arguments": [], + "created_at": 1687942822.9913611, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__concat(fields) -%}\n {{ fields|join(' || ') }}\n{%- endmacro %}", + "meta": {}, + "name": "default__concat", + "original_file_path": "macros/utils/concat.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/concat.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__concat" + }, + "macro.dbt.default__copy_grants": { + "arguments": [], + "created_at": 1687942823.021837, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__copy_grants() %}\n {{ return(True) }}\n{% endmacro %}", + "meta": {}, + "name": "default__copy_grants", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__copy_grants" + }, + "macro.dbt.default__create_columns": { + "arguments": [], + "created_at": 1687942822.8890939, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", + "meta": {}, + "name": "default__create_columns", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_columns" + }, + "macro.dbt.default__create_csv_table": { + "arguments": [], + "created_at": 1687942822.973732, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", + "meta": {}, + "name": "default__create_csv_table", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_csv_table" + }, + "macro.dbt.default__create_indexes": { + "arguments": [], + "created_at": 1687942823.0109131, + "depends_on": { + "macros": [ + "macro.dbt.get_create_index_sql", + "macro.dbt.run_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", + "meta": {}, + "name": "default__create_indexes", + "original_file_path": "macros/adapters/indexes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/indexes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_indexes" + }, + "macro.dbt.default__create_schema": { + "arguments": [], + "created_at": 1687942823.0072918, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", + "meta": {}, + "name": "default__create_schema", + "original_file_path": "macros/adapters/schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_schema" + }, + "macro.dbt.default__create_table_as": { + "arguments": [], + "created_at": 1687942822.954907, + "depends_on": { + "macros": [ + "macro.dbt.get_assert_columns_equivalent", + "macro.dbt.get_table_columns_and_constraints", + "macro.dbt.get_select_subquery" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", + "meta": {}, + "name": "default__create_table_as", + "original_file_path": "macros/materializations/models/table/create_table_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/create_table_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_table_as" + }, + "macro.dbt.default__create_view_as": { + "arguments": [], + "created_at": 1687942822.962664, + "depends_on": { + "macros": [ + "macro.dbt.get_assert_columns_equivalent" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", + "meta": {}, + "name": "default__create_view_as", + "original_file_path": "macros/materializations/models/view/create_view_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/view/create_view_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_view_as" + }, + "macro.dbt.default__current_timestamp": { + "arguments": [], + "created_at": 1687942823.0084531, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter ' + adapter.type()) }}\n{%- endmacro %}", + "meta": {}, + "name": "default__current_timestamp", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__current_timestamp" + }, + "macro.dbt.default__current_timestamp_backcompat": { + "arguments": [], + "created_at": 1687942823.008993, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__current_timestamp_backcompat() %}\n current_timestamp::timestamp\n{% endmacro %}", + "meta": {}, + "name": "default__current_timestamp_backcompat", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__current_timestamp_backcompat" + }, + "macro.dbt.default__current_timestamp_in_utc_backcompat": { + "arguments": [], + "created_at": 1687942823.0093691, + "depends_on": { + "macros": [ + "macro.dbt.current_timestamp_backcompat", + "macro.dbt_postgres.postgres__current_timestamp_backcompat" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "default__current_timestamp_in_utc_backcompat", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__current_timestamp_in_utc_backcompat" + }, + "macro.dbt.default__date_trunc": { + "arguments": [], + "created_at": 1687942823.005462, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__date_trunc(datepart, date) -%}\n date_trunc('{{datepart}}', {{date}})\n{%- endmacro %}", + "meta": {}, + "name": "default__date_trunc", + "original_file_path": "macros/utils/date_trunc.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/date_trunc.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__date_trunc" + }, + "macro.dbt.default__dateadd": { + "arguments": [], + "created_at": 1687942822.992487, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n dateadd(\n {{ datepart }},\n {{ interval }},\n {{ from_date_or_timestamp }}\n )\n\n{% endmacro %}", + "meta": {}, + "name": "default__dateadd", + "original_file_path": "macros/utils/dateadd.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/dateadd.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__dateadd" + }, + "macro.dbt.default__datediff": { + "arguments": [], + "created_at": 1687942822.9957619, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__datediff(first_date, second_date, datepart) -%}\n\n datediff(\n {{ datepart }},\n {{ first_date }},\n {{ second_date }}\n )\n\n{%- endmacro %}", + "meta": {}, + "name": "default__datediff", + "original_file_path": "macros/utils/datediff.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/datediff.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__datediff" + }, + "macro.dbt.default__drop_relation": { + "arguments": [], + "created_at": 1687942823.016405, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n {%- endcall %}\n{% endmacro %}", + "meta": {}, + "name": "default__drop_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__drop_relation" + }, + "macro.dbt.default__drop_schema": { + "arguments": [], + "created_at": 1687942823.0077598, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", + "meta": {}, + "name": "default__drop_schema", + "original_file_path": "macros/adapters/schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__drop_schema" + }, + "macro.dbt.default__escape_single_quotes": { + "arguments": [], + "created_at": 1687942822.9933708, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"''\") }}\n{%- endmacro %}", + "meta": {}, + "name": "default__escape_single_quotes", + "original_file_path": "macros/utils/escape_single_quotes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/escape_single_quotes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__escape_single_quotes" + }, + "macro.dbt.default__except": { + "arguments": [], + "created_at": 1687942822.990392, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__except() %}\n\n except\n\n{% endmacro %}", + "meta": {}, + "name": "default__except", + "original_file_path": "macros/utils/except.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/except.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__except" + }, + "macro.dbt.default__format_column": { + "arguments": [], + "created_at": 1687942822.949109, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__format_column(column) -%}\n {% set data_type = column.dtype %}\n {% set formatted = column.column.lower() ~ \" \" ~ data_type %}\n {{ return({'name': column.name, 'data_type': data_type, 'formatted': formatted}) }}\n{%- endmacro -%}", + "meta": {}, + "name": "default__format_column", + "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__format_column" + }, + "macro.dbt.default__generate_alias_name": { + "arguments": [], + "created_at": 1687942822.979058, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name -%}\n\n {{ custom_alias_name | trim }}\n\n {%- elif node.version -%}\n\n {{ return(node.name ~ \"_v\" ~ (node.version | replace(\".\", \"_\"))) }}\n\n {%- else -%}\n\n {{ node.name }}\n\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "default__generate_alias_name", + "original_file_path": "macros/get_custom_name/get_custom_alias.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_alias.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__generate_alias_name" + }, + "macro.dbt.default__generate_database_name": { + "arguments": [], + "created_at": 1687942822.981025, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "default__generate_database_name", + "original_file_path": "macros/get_custom_name/get_custom_database.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_database.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__generate_database_name" + }, + "macro.dbt.default__generate_schema_name": { + "arguments": [], + "created_at": 1687942822.979957, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "default__generate_schema_name", + "original_file_path": "macros/get_custom_name/get_custom_schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__generate_schema_name" + }, + "macro.dbt.default__get_assert_columns_equivalent": { + "arguments": [], + "created_at": 1687942822.946909, + "depends_on": { + "macros": [ + "macro.dbt.assert_columns_equivalent" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_assert_columns_equivalent(sql) -%}\n {{ return(assert_columns_equivalent(sql)) }}\n{%- endmacro %}", + "meta": {}, + "name": "default__get_assert_columns_equivalent", + "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_assert_columns_equivalent" + }, + "macro.dbt.default__get_batch_size": { + "arguments": [], + "created_at": 1687942822.975518, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_batch_size", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_batch_size" + }, + "macro.dbt.default__get_binding_char": { + "arguments": [], + "created_at": 1687942822.975202, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_binding_char", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_binding_char" + }, + "macro.dbt.default__get_catalog": { + "arguments": [], + "created_at": 1687942823.031789, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_catalog", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_catalog" + }, + "macro.dbt.default__get_column_names": { + "arguments": [], + "created_at": 1687942822.95544, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_column_names() %}\n {#- loop through user_provided_columns to get column names -#}\n {%- set user_provided_columns = model['columns'] -%}\n {%- for i in user_provided_columns %}\n {%- set col = user_provided_columns[i] -%}\n {%- set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] -%}\n {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n{% endmacro %}", + "meta": {}, + "name": "default__get_column_names", + "original_file_path": "macros/materializations/models/table/create_table_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/create_table_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_column_names" + }, + "macro.dbt.default__get_columns_in_query": { + "arguments": [], + "created_at": 1687942823.039603, + "depends_on": { + "macros": [ + "macro.dbt.statement", + "macro.dbt.get_empty_subquery_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n {{ get_empty_subquery_sql(select_sql) }}\n {% endcall %}\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_columns_in_query", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_columns_in_query" + }, + "macro.dbt.default__get_columns_in_relation": { + "arguments": [], + "created_at": 1687942823.036652, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_columns_in_relation", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_columns_in_relation" + }, + "macro.dbt.default__get_create_index_sql": { + "arguments": [], + "created_at": 1687942823.010286, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", + "meta": {}, + "name": "default__get_create_index_sql", + "original_file_path": "macros/adapters/indexes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/indexes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_create_index_sql" + }, + "macro.dbt.default__get_create_table_as_sql": { + "arguments": [], + "created_at": 1687942822.953666, + "depends_on": { + "macros": [ + "macro.dbt.create_table_as" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_create_table_as_sql", + "original_file_path": "macros/materializations/models/table/create_table_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/create_table_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_create_table_as_sql" + }, + "macro.dbt.default__get_create_view_as_sql": { + "arguments": [], + "created_at": 1687942822.961925, + "depends_on": { + "macros": [ + "macro.dbt.create_view_as" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_create_view_as_sql", + "original_file_path": "macros/materializations/models/view/create_view_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/view/create_view_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_create_view_as_sql" + }, + "macro.dbt.default__get_csv_sql": { + "arguments": [], + "created_at": 1687942822.9749131, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ create_or_truncate_sql }};\n -- dbt seed --\n {{ insert_sql }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_csv_sql", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_csv_sql" + }, + "macro.dbt.default__get_dcl_statement_list": { + "arguments": [], + "created_at": 1687942823.0251, + "depends_on": { + "macros": [ + "macro.dbt.support_multiple_grantees_per_dcl_statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}\n {#\n -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.\n -- Depending on whether this database supports multiple grantees per statement, pass in the list of\n -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.\n -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`\n #}\n {%- set dcl_statements = [] -%}\n {%- for privilege, grantees in grant_config.items() %}\n {%- if support_multiple_grantees_per_dcl_statement() and grantees -%}\n {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}\n {%- do dcl_statements.append(dcl) -%}\n {%- else -%}\n {%- for grantee in grantees -%}\n {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}\n {%- do dcl_statements.append(dcl) -%}\n {% endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {{ return(dcl_statements) }}\n{%- endmacro %}", + "meta": {}, + "name": "default__get_dcl_statement_list", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_dcl_statement_list" + }, + "macro.dbt.default__get_delete_insert_merge_sql": { + "arguments": [], + "created_at": 1687942822.922552, + "depends_on": { + "macros": [ + "macro.dbt.get_quoted_csv" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not string %}\n delete from {{target }}\n using {{ source }}\n where (\n {% for key in unique_key %}\n {{ source }}.{{ key }} = {{ target }}.{{ key }}\n {{ \"and \" if not loop.last}}\n {% endfor %}\n {% if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {% endif %}\n );\n {% else %}\n delete from {{ target }}\n where (\n {{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n )\n {%- if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {%- endif -%};\n\n {% endif %}\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", + "meta": {}, + "name": "default__get_delete_insert_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_delete_insert_merge_sql" + }, + "macro.dbt.default__get_empty_schema_sql": { + "arguments": [], + "created_at": 1687942823.038543, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_empty_schema_sql(columns) %}\n {%- set col_err = [] -%}\n select\n {% for i in columns %}\n {%- set col = columns[i] -%}\n {%- if col['data_type'] is not defined -%}\n {{ col_err.append(col['name']) }}\n {%- endif -%}\n {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %}\n cast(null as {{ col['data_type'] }}) as {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n {%- if (col_err | length) > 0 -%}\n {{ exceptions.column_type_missing(column_names=col_err) }}\n {%- endif -%}\n{% endmacro %}", + "meta": {}, + "name": "default__get_empty_schema_sql", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_empty_schema_sql" + }, + "macro.dbt.default__get_empty_subquery_sql": { + "arguments": [], + "created_at": 1687942823.037494, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_empty_subquery_sql(select_sql, select_sql_header=none) %}\n {%- if select_sql_header is not none -%}\n {{ select_sql_header }}\n {%- endif -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n{% endmacro %}", + "meta": {}, + "name": "default__get_empty_subquery_sql", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_empty_subquery_sql" + }, + "macro.dbt.default__get_grant_sql": { + "arguments": [], + "created_at": 1687942823.0234182, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "default__get_grant_sql", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_grant_sql" + }, + "macro.dbt.default__get_incremental_append_sql": { + "arguments": [], + "created_at": 1687942822.9257421, + "depends_on": { + "macros": [ + "macro.dbt.get_insert_into_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_incremental_append_sql(arg_dict) %}\n\n {% do return(get_insert_into_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_incremental_append_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_incremental_append_sql" + }, + "macro.dbt.default__get_incremental_default_sql": { + "arguments": [], + "created_at": 1687942822.927765, + "depends_on": { + "macros": [ + "macro.dbt.get_incremental_append_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_incremental_default_sql(arg_dict) %}\n\n {% do return(get_incremental_append_sql(arg_dict)) %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_incremental_default_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_incremental_default_sql" + }, + "macro.dbt.default__get_incremental_delete_insert_sql": { + "arguments": [], + "created_at": 1687942822.926282, + "depends_on": { + "macros": [ + "macro.dbt.get_delete_insert_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_incremental_delete_insert_sql(arg_dict) %}\n\n {% do return(get_delete_insert_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_incremental_delete_insert_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_incremental_delete_insert_sql" + }, + "macro.dbt.default__get_incremental_insert_overwrite_sql": { + "arguments": [], + "created_at": 1687942822.9273689, + "depends_on": { + "macros": [ + "macro.dbt.get_insert_overwrite_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {% do return(get_insert_overwrite_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_incremental_insert_overwrite_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_incremental_insert_overwrite_sql" + }, + "macro.dbt.default__get_incremental_merge_sql": { + "arguments": [], + "created_at": 1687942822.926823, + "depends_on": { + "macros": [ + "macro.dbt.get_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_incremental_merge_sql(arg_dict) %}\n\n {% do return(get_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_incremental_merge_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_incremental_merge_sql" + }, + "macro.dbt.default__get_insert_overwrite_merge_sql": { + "arguments": [], + "created_at": 1687942822.923584, + "depends_on": { + "macros": [ + "macro.dbt.get_quoted_csv" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {#-- The only time include_sql_header is True: --#}\n {#-- BigQuery + insert_overwrite strategy + \"static\" partitions config --#}\n {#-- We should consider including the sql header at the materialization level instead --#}\n\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_insert_overwrite_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql" + }, + "macro.dbt.default__get_merge_sql": { + "arguments": [], + "created_at": 1687942822.921111, + "depends_on": { + "macros": [ + "macro.dbt.get_quoted_csv", + "macro.dbt.get_merge_update_columns" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{\"(\" ~ predicates | join(\") and (\") ~ \")\"}}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_merge_sql" + }, + "macro.dbt.default__get_merge_update_columns": { + "arguments": [], + "created_at": 1687942822.911655, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = dest_columns | map(attribute=\"quoted\") | list -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_merge_update_columns", + "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/column_helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_merge_update_columns" + }, + "macro.dbt.default__get_or_create_relation": { + "arguments": [], + "created_at": 1687942823.018283, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", + "meta": {}, + "name": "default__get_or_create_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_or_create_relation" + }, + "macro.dbt.default__get_revoke_sql": { + "arguments": [], + "created_at": 1687942823.023883, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "default__get_revoke_sql", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_revoke_sql" + }, + "macro.dbt.default__get_select_subquery": { + "arguments": [], + "created_at": 1687942822.955848, + "depends_on": { + "macros": [ + "macro.dbt.default__get_column_names" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_select_subquery(sql) %}\n select {{ adapter.dispatch('get_column_names', 'dbt')() }}\n from (\n {{ sql }}\n ) as model_subq\n{%- endmacro %}", + "meta": {}, + "name": "default__get_select_subquery", + "original_file_path": "macros/materializations/models/table/create_table_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/create_table_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_select_subquery" + }, + "macro.dbt.default__get_show_grant_sql": { + "arguments": [], + "created_at": 1687942823.02295, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_show_grant_sql", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_show_grant_sql" + }, + "macro.dbt.default__get_table_columns_and_constraints": { + "arguments": [], + "created_at": 1687942822.945963, + "depends_on": { + "macros": [ + "macro.dbt.table_columns_and_constraints" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_table_columns_and_constraints() -%}\n {{ return(table_columns_and_constraints()) }}\n{%- endmacro %}", + "meta": {}, + "name": "default__get_table_columns_and_constraints", + "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_table_columns_and_constraints" + }, + "macro.dbt.default__get_test_sql": { + "arguments": [], + "created_at": 1687942822.904812, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", + "meta": {}, + "name": "default__get_test_sql", + "original_file_path": "macros/materializations/tests/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/tests/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_test_sql" + }, + "macro.dbt.default__get_true_sql": { + "arguments": [], + "created_at": 1687942822.890411, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_true_sql", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_true_sql" + }, + "macro.dbt.default__get_where_subquery": { + "arguments": [], + "created_at": 1687942822.905828, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", + "meta": {}, + "name": "default__get_where_subquery", + "original_file_path": "macros/materializations/tests/where_subquery.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/tests/where_subquery.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_where_subquery" + }, + "macro.dbt.default__handle_existing_table": { + "arguments": [], + "created_at": 1687942822.959599, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "default__handle_existing_table", + "original_file_path": "macros/materializations/models/view/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/view/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__handle_existing_table" + }, + "macro.dbt.default__hash": { + "arguments": [], + "created_at": 1687942822.996808, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", + "meta": {}, + "name": "default__hash", + "original_file_path": "macros/utils/hash.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/hash.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__hash" + }, + "macro.dbt.default__information_schema_name": { + "arguments": [], + "created_at": 1687942823.032175, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", + "meta": {}, + "name": "default__information_schema_name", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__information_schema_name" + }, + "macro.dbt.default__intersect": { + "arguments": [], + "created_at": 1687942822.992857, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", + "meta": {}, + "name": "default__intersect", + "original_file_path": "macros/utils/intersect.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/intersect.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__intersect" + }, + "macro.dbt.default__last_day": { + "arguments": [], + "created_at": 1687942823.004024, + "depends_on": { + "macros": [ + "macro.dbt.default_last_day" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", + "meta": {}, + "name": "default__last_day", + "original_file_path": "macros/utils/last_day.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/last_day.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__last_day" + }, + "macro.dbt.default__length": { + "arguments": [], + "created_at": 1687942822.9919202, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", + "meta": {}, + "name": "default__length", + "original_file_path": "macros/utils/length.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/length.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__length" + }, + "macro.dbt.default__list_relations_without_caching": { + "arguments": [], + "created_at": 1687942823.033619, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", + "meta": {}, + "name": "default__list_relations_without_caching", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__list_relations_without_caching" + }, + "macro.dbt.default__list_schemas": { + "arguments": [], + "created_at": 1687942823.032667, + "depends_on": { + "macros": [ + "macro.dbt.information_schema_name", + "macro.dbt.run_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "default__list_schemas", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__list_schemas" + }, + "macro.dbt.default__listagg": { + "arguments": [], + "created_at": 1687942822.995195, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", + "meta": {}, + "name": "default__listagg", + "original_file_path": "macros/utils/listagg.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/listagg.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__listagg" + }, + "macro.dbt.default__load_csv_rows": { + "arguments": [], + "created_at": 1687942822.978139, + "depends_on": { + "macros": [ + "macro.dbt.get_batch_size", + "macro.dbt.get_seed_column_quoted_csv", + "macro.dbt.get_binding_char" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", + "meta": {}, + "name": "default__load_csv_rows", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__load_csv_rows" + }, + "macro.dbt.default__make_backup_relation": { + "arguments": [], + "created_at": 1687942823.01597, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "default__make_backup_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__make_backup_relation" + }, + "macro.dbt.default__make_intermediate_relation": { + "arguments": [], + "created_at": 1687942823.014782, + "depends_on": { + "macros": [ + "macro.dbt.default__make_temp_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", + "meta": {}, + "name": "default__make_intermediate_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__make_intermediate_relation" + }, + "macro.dbt.default__make_temp_relation": { + "arguments": [], + "created_at": 1687942823.015351, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "default__make_temp_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__make_temp_relation" + }, + "macro.dbt.default__persist_docs": { + "arguments": [], + "created_at": 1687942823.029631, + "depends_on": { + "macros": [ + "macro.dbt.run_query", + "macro.dbt.alter_relation_comment", + "macro.dbt.alter_column_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "default__persist_docs", + "original_file_path": "macros/adapters/persist_docs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/persist_docs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__persist_docs" + }, + "macro.dbt.default__position": { + "arguments": [], + "created_at": 1687942822.99832, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", + "meta": {}, + "name": "default__position", + "original_file_path": "macros/utils/position.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/position.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__position" + }, + "macro.dbt.default__post_snapshot": { + "arguments": [], + "created_at": 1687942822.8900359, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", + "meta": {}, + "name": "default__post_snapshot", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__post_snapshot" + }, + "macro.dbt.default__rename_relation": { + "arguments": [], + "created_at": 1687942823.0173938, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", + "meta": {}, + "name": "default__rename_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__rename_relation" + }, + "macro.dbt.default__replace": { + "arguments": [], + "created_at": 1687942822.990945, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", + "meta": {}, + "name": "default__replace", + "original_file_path": "macros/utils/replace.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__replace" + }, + "macro.dbt.default__reset_csv_table": { + "arguments": [], + "created_at": 1687942822.9745421, + "depends_on": { + "macros": [ + "macro.dbt.create_csv_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", + "meta": {}, + "name": "default__reset_csv_table", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__reset_csv_table" + }, + "macro.dbt.default__resolve_model_name": { + "arguments": [], + "created_at": 1687942823.043811, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro default__resolve_model_name(input_model_name) -%}\n {{ input_model_name | string | replace('\"', '\\\"') }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "default__resolve_model_name", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__resolve_model_name" + }, + "macro.dbt.default__right": { + "arguments": [], + "created_at": 1687942822.993966, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", + "meta": {}, + "name": "default__right", + "original_file_path": "macros/utils/right.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/right.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__right" + }, + "macro.dbt.default__safe_cast": { + "arguments": [], + "created_at": 1687942822.996346, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", + "meta": {}, + "name": "default__safe_cast", + "original_file_path": "macros/utils/safe_cast.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/safe_cast.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__safe_cast" + }, + "macro.dbt.default__snapshot_get_time": { + "arguments": [], + "created_at": 1687942823.008725, + "depends_on": { + "macros": [ + "macro.dbt.current_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__snapshot_get_time() %}\n {{ current_timestamp() }}\n{% endmacro %}", + "meta": {}, + "name": "default__snapshot_get_time", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__snapshot_get_time" + }, + "macro.dbt.default__snapshot_hash_arguments": { + "arguments": [], + "created_at": 1687942822.878295, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", + "meta": {}, + "name": "default__snapshot_hash_arguments", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__snapshot_hash_arguments" + }, + "macro.dbt.default__snapshot_merge_sql": { + "arguments": [], + "created_at": 1687942822.873353, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", + "meta": {}, + "name": "default__snapshot_merge_sql", + "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/snapshot_merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__snapshot_merge_sql" + }, + "macro.dbt.default__snapshot_staging_table": { + "arguments": [], + "created_at": 1687942822.891789, + "depends_on": { + "macros": [ + "macro.dbt.snapshot_get_time" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n\n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", + "meta": {}, + "name": "default__snapshot_staging_table", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__snapshot_staging_table" + }, + "macro.dbt.default__snapshot_string_as_time": { + "arguments": [], + "created_at": 1687942822.879877, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", + "meta": {}, + "name": "default__snapshot_string_as_time", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__snapshot_string_as_time" + }, + "macro.dbt.default__split_part": { + "arguments": [], + "created_at": 1687942823.004779, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", + "meta": {}, + "name": "default__split_part", + "original_file_path": "macros/utils/split_part.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/split_part.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__split_part" + }, + "macro.dbt.default__string_literal": { + "arguments": [], + "created_at": 1687942822.998917, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", + "meta": {}, + "name": "default__string_literal", + "original_file_path": "macros/utils/literal.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/literal.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__string_literal" + }, + "macro.dbt.default__support_multiple_grantees_per_dcl_statement": { + "arguments": [], + "created_at": 1687942823.0221472, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "default__support_multiple_grantees_per_dcl_statement", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement" + }, + "macro.dbt.default__test_accepted_values": { + "arguments": [], + "created_at": 1687942822.982643, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", + "meta": {}, + "name": "default__test_accepted_values", + "original_file_path": "macros/generic_test_sql/accepted_values.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/generic_test_sql/accepted_values.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__test_accepted_values" + }, + "macro.dbt.default__test_not_null": { + "arguments": [], + "created_at": 1687942822.98174, + "depends_on": { + "macros": [ + "macro.dbt.should_store_failures" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", + "meta": {}, + "name": "default__test_not_null", + "original_file_path": "macros/generic_test_sql/not_null.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/generic_test_sql/not_null.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__test_not_null" + }, + "macro.dbt.default__test_relationships": { + "arguments": [], + "created_at": 1687942822.981408, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", + "meta": {}, + "name": "default__test_relationships", + "original_file_path": "macros/generic_test_sql/relationships.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/generic_test_sql/relationships.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__test_relationships" + }, + "macro.dbt.default__test_unique": { + "arguments": [], + "created_at": 1687942822.9820108, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", + "meta": {}, + "name": "default__test_unique", + "original_file_path": "macros/generic_test_sql/unique.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/generic_test_sql/unique.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__test_unique" + }, + "macro.dbt.default__truncate_relation": { + "arguments": [], + "created_at": 1687942823.016861, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", + "meta": {}, + "name": "default__truncate_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__truncate_relation" + }, + "macro.dbt.default__type_bigint": { + "arguments": [], + "created_at": 1687942823.0016088, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", + "meta": {}, + "name": "default__type_bigint", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_bigint" + }, + "macro.dbt.default__type_boolean": { + "arguments": [], + "created_at": 1687942823.0023592, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro default__type_boolean() -%}\n {{ return(api.Column.translate_type(\"boolean\")) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "default__type_boolean", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_boolean" + }, + "macro.dbt.default__type_float": { + "arguments": [], + "created_at": 1687942823.0008872, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", + "meta": {}, + "name": "default__type_float", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_float" + }, + "macro.dbt.default__type_int": { + "arguments": [], + "created_at": 1687942823.002025, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "default__type_int", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_int" + }, + "macro.dbt.default__type_numeric": { + "arguments": [], + "created_at": 1687942823.001258, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", + "meta": {}, + "name": "default__type_numeric", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_numeric" + }, + "macro.dbt.default__type_string": { + "arguments": [], + "created_at": 1687942823.000193, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", + "meta": {}, + "name": "default__type_string", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_string" + }, + "macro.dbt.default__type_timestamp": { + "arguments": [], + "created_at": 1687942823.000544, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", + "meta": {}, + "name": "default__type_timestamp", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_timestamp" + }, + "macro.dbt.default_last_day": { + "arguments": [], + "created_at": 1687942823.00386, + "depends_on": { + "macros": [ + "macro.dbt.dateadd", + "macro.dbt.date_trunc" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "default_last_day", + "original_file_path": "macros/utils/last_day.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/last_day.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default_last_day" + }, + "macro.dbt.diff_column_data_types": { + "arguments": [], + "created_at": 1687942822.910557, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type and not sc.can_expand_to(other_column=tc) %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", + "meta": {}, + "name": "diff_column_data_types", + "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/column_helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.diff_column_data_types" + }, + "macro.dbt.diff_columns": { + "arguments": [], + "created_at": 1687942822.908992, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", + "meta": {}, + "name": "diff_columns", + "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/column_helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.diff_columns" + }, + "macro.dbt.drop_relation": { + "arguments": [], + "created_at": 1687942823.016174, + "depends_on": { + "macros": [ + "macro.dbt.default__drop_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "drop_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.drop_relation" + }, + "macro.dbt.drop_relation_if_exists": { + "arguments": [], + "created_at": 1687942823.018911, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "drop_relation_if_exists", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.drop_relation_if_exists" + }, + "macro.dbt.drop_schema": { + "arguments": [], + "created_at": 1687942823.007472, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__drop_schema" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", + "meta": {}, + "name": "drop_schema", + "original_file_path": "macros/adapters/schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.drop_schema" + }, + "macro.dbt.escape_single_quotes": { + "arguments": [], + "created_at": 1687942822.993175, + "depends_on": { + "macros": [ + "macro.dbt.default__escape_single_quotes" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", + "meta": {}, + "name": "escape_single_quotes", + "original_file_path": "macros/utils/escape_single_quotes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/escape_single_quotes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.escape_single_quotes" + }, + "macro.dbt.except": { + "arguments": [], + "created_at": 1687942822.990304, + "depends_on": { + "macros": [ + "macro.dbt.default__except" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "except", + "original_file_path": "macros/utils/except.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/except.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.except" + }, + "macro.dbt.format_columns": { + "arguments": [], + "created_at": 1687942822.9486878, + "depends_on": { + "macros": [ + "macro.dbt.default__format_column" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro format_columns(columns) %}\n {% set formatted_columns = [] %}\n {% for column in columns %}\n {%- set formatted_column = adapter.dispatch('format_column', 'dbt')(column) -%}\n {%- do formatted_columns.append(formatted_column) -%}\n {% endfor %}\n {{ return(formatted_columns) }}\n{% endmacro %}", + "meta": {}, + "name": "format_columns", + "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.format_columns" + }, + "macro.dbt.generate_alias_name": { + "arguments": [], + "created_at": 1687942822.9786382, + "depends_on": { + "macros": [ + "macro.dbt.default__generate_alias_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", + "meta": {}, + "name": "generate_alias_name", + "original_file_path": "macros/get_custom_name/get_custom_alias.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_alias.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.generate_alias_name" + }, + "macro.dbt.generate_database_name": { + "arguments": [], + "created_at": 1687942822.980743, + "depends_on": { + "macros": [ + "macro.dbt.default__generate_database_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", + "meta": {}, + "name": "generate_database_name", + "original_file_path": "macros/get_custom_name/get_custom_database.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_database.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.generate_database_name" + }, + "macro.dbt.generate_schema_name": { + "arguments": [], + "created_at": 1687942822.97963, + "depends_on": { + "macros": [ + "macro.dbt.default__generate_schema_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", + "meta": {}, + "name": "generate_schema_name", + "original_file_path": "macros/get_custom_name/get_custom_schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.generate_schema_name" + }, + "macro.dbt.generate_schema_name_for_env": { + "arguments": [], + "created_at": 1687942822.9802911, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "generate_schema_name_for_env", + "original_file_path": "macros/get_custom_name/get_custom_schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.generate_schema_name_for_env" + }, + "macro.dbt.get_assert_columns_equivalent": { + "arguments": [], + "created_at": 1687942822.946754, + "depends_on": { + "macros": [ + "macro.dbt.default__get_assert_columns_equivalent" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro get_assert_columns_equivalent(sql) -%}\n {{ adapter.dispatch('get_assert_columns_equivalent', 'dbt')(sql) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "get_assert_columns_equivalent", + "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_assert_columns_equivalent" + }, + "macro.dbt.get_batch_size": { + "arguments": [], + "created_at": 1687942822.975379, + "depends_on": { + "macros": [ + "macro.dbt.default__get_batch_size" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_batch_size", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_batch_size" + }, + "macro.dbt.get_binding_char": { + "arguments": [], + "created_at": 1687942822.975074, + "depends_on": { + "macros": [ + "macro.dbt.default__get_binding_char" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", + "meta": {}, + "name": "get_binding_char", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_binding_char" + }, + "macro.dbt.get_catalog": { + "arguments": [], + "created_at": 1687942823.031494, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_catalog" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_catalog", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_catalog" + }, + "macro.dbt.get_column_schema_from_query": { + "arguments": [], + "created_at": 1687942823.0389261, + "depends_on": { + "macros": [ + "macro.dbt.get_empty_subquery_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_column_schema_from_query(select_sql, select_sql_header=none) -%}\n {% set columns = [] %}\n {# -- Using an 'empty subquery' here to get the same schema as the given select_sql statement, without necessitating a data scan.#}\n {% set sql = get_empty_subquery_sql(select_sql, select_sql_header) %}\n {% set column_schema = adapter.get_column_schema_from_query(sql) %}\n {{ return(column_schema) }}\n{% endmacro %}", + "meta": {}, + "name": "get_column_schema_from_query", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_column_schema_from_query" + }, + "macro.dbt.get_columns_in_query": { + "arguments": [], + "created_at": 1687942823.0391302, + "depends_on": { + "macros": [ + "macro.dbt.default__get_columns_in_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_columns_in_query", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_columns_in_query" + }, + "macro.dbt.get_columns_in_relation": { + "arguments": [], + "created_at": 1687942823.036205, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_columns_in_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_columns_in_relation", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_columns_in_relation" + }, + "macro.dbt.get_create_index_sql": { + "arguments": [], + "created_at": 1687942823.0101209, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_create_index_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_create_index_sql", + "original_file_path": "macros/adapters/indexes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/indexes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_create_index_sql" + }, + "macro.dbt.get_create_table_as_sql": { + "arguments": [], + "created_at": 1687942822.95344, + "depends_on": { + "macros": [ + "macro.dbt.default__get_create_table_as_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_create_table_as_sql", + "original_file_path": "macros/materializations/models/table/create_table_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/create_table_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_create_table_as_sql" + }, + "macro.dbt.get_create_view_as_sql": { + "arguments": [], + "created_at": 1687942822.961741, + "depends_on": { + "macros": [ + "macro.dbt.default__get_create_view_as_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_create_view_as_sql", + "original_file_path": "macros/materializations/models/view/create_view_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/view/create_view_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_create_view_as_sql" + }, + "macro.dbt.get_csv_sql": { + "arguments": [], + "created_at": 1687942822.974763, + "depends_on": { + "macros": [ + "macro.dbt.default__get_csv_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", + "meta": {}, + "name": "get_csv_sql", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_csv_sql" + }, + "macro.dbt.get_dcl_statement_list": { + "arguments": [], + "created_at": 1687942823.024218, + "depends_on": { + "macros": [ + "macro.dbt.default__get_dcl_statement_list" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_dcl_statement_list", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_dcl_statement_list" + }, + "macro.dbt.get_delete_insert_merge_sql": { + "arguments": [], + "created_at": 1687942822.9214232, + "depends_on": { + "macros": [ + "macro.dbt.default__get_delete_insert_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_delete_insert_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_delete_insert_merge_sql" + }, + "macro.dbt.get_empty_schema_sql": { + "arguments": [], + "created_at": 1687942823.037695, + "depends_on": { + "macros": [ + "macro.dbt.default__get_empty_schema_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_empty_schema_sql(columns) -%}\n {{ return(adapter.dispatch('get_empty_schema_sql', 'dbt')(columns)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_empty_schema_sql", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_empty_schema_sql" + }, + "macro.dbt.get_empty_subquery_sql": { + "arguments": [], + "created_at": 1687942823.0372648, + "depends_on": { + "macros": [ + "macro.dbt.default__get_empty_subquery_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_empty_subquery_sql(select_sql, select_sql_header=none) -%}\n {{ return(adapter.dispatch('get_empty_subquery_sql', 'dbt')(select_sql, select_sql_header)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_empty_subquery_sql", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_empty_subquery_sql" + }, + "macro.dbt.get_grant_sql": { + "arguments": [], + "created_at": 1687942823.023207, + "depends_on": { + "macros": [ + "macro.dbt.default__get_grant_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_grant_sql", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_grant_sql" + }, + "macro.dbt.get_incremental_append_sql": { + "arguments": [], + "created_at": 1687942822.9254642, + "depends_on": { + "macros": [ + "macro.dbt.default__get_incremental_append_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_incremental_append_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_incremental_append_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_incremental_append_sql" + }, + "macro.dbt.get_incremental_default_sql": { + "arguments": [], + "created_at": 1687942822.927587, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_incremental_default_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_incremental_default_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_incremental_default_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_incremental_default_sql" + }, + "macro.dbt.get_incremental_delete_insert_sql": { + "arguments": [], + "created_at": 1687942822.925951, + "depends_on": { + "macros": [ + "macro.dbt.default__get_incremental_delete_insert_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_incremental_delete_insert_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_incremental_delete_insert_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_incremental_delete_insert_sql" + }, + "macro.dbt.get_incremental_insert_overwrite_sql": { + "arguments": [], + "created_at": 1687942822.9270568, + "depends_on": { + "macros": [ + "macro.dbt.default__get_incremental_insert_overwrite_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_incremental_insert_overwrite_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_incremental_insert_overwrite_sql" + }, + "macro.dbt.get_incremental_merge_sql": { + "arguments": [], + "created_at": 1687942822.92649, + "depends_on": { + "macros": [ + "macro.dbt.default__get_incremental_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_incremental_merge_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_incremental_merge_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_incremental_merge_sql" + }, + "macro.dbt.get_insert_into_sql": { + "arguments": [], + "created_at": 1687942822.928073, + "depends_on": { + "macros": [ + "macro.dbt.get_quoted_csv" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ temp_relation }}\n )\n\n{% endmacro %}", + "meta": {}, + "name": "get_insert_into_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_insert_into_sql" + }, + "macro.dbt.get_insert_overwrite_merge_sql": { + "arguments": [], + "created_at": 1687942822.922858, + "depends_on": { + "macros": [ + "macro.dbt.default__get_insert_overwrite_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_insert_overwrite_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_insert_overwrite_merge_sql" + }, + "macro.dbt.get_merge_sql": { + "arguments": [], + "created_at": 1687942822.919136, + "depends_on": { + "macros": [ + "macro.dbt.default__get_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n -- back compat for old kwarg name\n {% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_merge_sql" + }, + "macro.dbt.get_merge_update_columns": { + "arguments": [], + "created_at": 1687942822.9108438, + "depends_on": { + "macros": [ + "macro.dbt.default__get_merge_update_columns" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_merge_update_columns", + "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/column_helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_merge_update_columns" + }, + "macro.dbt.get_or_create_relation": { + "arguments": [], + "created_at": 1687942823.017668, + "depends_on": { + "macros": [ + "macro.dbt.default__get_or_create_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_or_create_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_or_create_relation" + }, + "macro.dbt.get_quoted_csv": { + "arguments": [], + "created_at": 1687942822.908025, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_quoted_csv", + "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/column_helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_quoted_csv" + }, + "macro.dbt.get_revoke_sql": { + "arguments": [], + "created_at": 1687942823.023674, + "depends_on": { + "macros": [ + "macro.dbt.default__get_revoke_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_revoke_sql", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_revoke_sql" + }, + "macro.dbt.get_seed_column_quoted_csv": { + "arguments": [], + "created_at": 1687942822.976216, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", + "meta": {}, + "name": "get_seed_column_quoted_csv", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_seed_column_quoted_csv" + }, + "macro.dbt.get_select_subquery": { + "arguments": [], + "created_at": 1687942822.955651, + "depends_on": { + "macros": [ + "macro.dbt.default__get_select_subquery" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_select_subquery(sql) %}\n {{ return(adapter.dispatch('get_select_subquery', 'dbt')(sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_select_subquery", + "original_file_path": "macros/materializations/models/table/create_table_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/create_table_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_select_subquery" + }, + "macro.dbt.get_show_grant_sql": { + "arguments": [], + "created_at": 1687942823.0228322, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_show_grant_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_show_grant_sql", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_show_grant_sql" + }, + "macro.dbt.get_table_columns_and_constraints": { + "arguments": [], + "created_at": 1687942822.945817, + "depends_on": { + "macros": [ + "macro.dbt.default__get_table_columns_and_constraints" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro get_table_columns_and_constraints() -%}\n {{ adapter.dispatch('get_table_columns_and_constraints', 'dbt')() }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "get_table_columns_and_constraints", + "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_table_columns_and_constraints" + }, + "macro.dbt.get_test_sql": { + "arguments": [], + "created_at": 1687942822.904258, + "depends_on": { + "macros": [ + "macro.dbt.default__get_test_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_test_sql", + "original_file_path": "macros/materializations/tests/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/tests/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_test_sql" + }, + "macro.dbt.get_true_sql": { + "arguments": [], + "created_at": 1687942822.890275, + "depends_on": { + "macros": [ + "macro.dbt.default__get_true_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", + "meta": {}, + "name": "get_true_sql", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_true_sql" + }, + "macro.dbt.get_where_subquery": { + "arguments": [], + "created_at": 1687942822.905386, + "depends_on": { + "macros": [ + "macro.dbt.default__get_where_subquery" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", + "meta": {}, + "name": "get_where_subquery", + "original_file_path": "macros/materializations/tests/where_subquery.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/tests/where_subquery.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_where_subquery" + }, + "macro.dbt.handle_existing_table": { + "arguments": [], + "created_at": 1687942822.9593441, + "depends_on": { + "macros": [ + "macro.dbt.default__handle_existing_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "handle_existing_table", + "original_file_path": "macros/materializations/models/view/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/view/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.handle_existing_table" + }, + "macro.dbt.hash": { + "arguments": [], + "created_at": 1687942822.9966311, + "depends_on": { + "macros": [ + "macro.dbt.default__hash" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", + "meta": {}, + "name": "hash", + "original_file_path": "macros/utils/hash.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/hash.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.hash" + }, + "macro.dbt.in_transaction": { + "arguments": [], + "created_at": 1687942822.871236, + "depends_on": { + "macros": [ + "macro.dbt.make_hook_config" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", + "meta": {}, + "name": "in_transaction", + "original_file_path": "macros/materializations/hooks.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/hooks.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.in_transaction" + }, + "macro.dbt.incremental_validate_on_schema_change": { + "arguments": [], + "created_at": 1687942822.9404979, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", + "meta": {}, + "name": "incremental_validate_on_schema_change", + "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/on_schema_change.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.incremental_validate_on_schema_change" + }, + "macro.dbt.information_schema_name": { + "arguments": [], + "created_at": 1687942823.032001, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__information_schema_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", + "meta": {}, + "name": "information_schema_name", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.information_schema_name" + }, + "macro.dbt.intersect": { + "arguments": [], + "created_at": 1687942822.992756, + "depends_on": { + "macros": [ + "macro.dbt.default__intersect" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "intersect", + "original_file_path": "macros/utils/intersect.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/intersect.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.intersect" + }, + "macro.dbt.is_incremental": { + "arguments": [], + "created_at": 1687942822.924459, + "depends_on": { + "macros": [ + "macro.dbt.should_full_refresh" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "is_incremental", + "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/is_incremental.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.is_incremental" + }, + "macro.dbt.last_day": { + "arguments": [], + "created_at": 1687942823.0035672, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__last_day" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", + "meta": {}, + "name": "last_day", + "original_file_path": "macros/utils/last_day.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/last_day.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.last_day" + }, + "macro.dbt.length": { + "arguments": [], + "created_at": 1687942822.9916742, + "depends_on": { + "macros": [ + "macro.dbt.default__length" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", + "meta": {}, + "name": "length", + "original_file_path": "macros/utils/length.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/length.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.length" + }, + "macro.dbt.list_relations_without_caching": { + "arguments": [], + "created_at": 1687942823.03344, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__list_relations_without_caching" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "list_relations_without_caching", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.list_relations_without_caching" + }, + "macro.dbt.list_schemas": { + "arguments": [], + "created_at": 1687942823.032376, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__list_schemas" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", + "meta": {}, + "name": "list_schemas", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.list_schemas" + }, + "macro.dbt.listagg": { + "arguments": [], + "created_at": 1687942822.9947362, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__listagg" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", + "meta": {}, + "name": "listagg", + "original_file_path": "macros/utils/listagg.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/listagg.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.listagg" + }, + "macro.dbt.load_cached_relation": { + "arguments": [], + "created_at": 1687942823.018537, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", + "meta": {}, + "name": "load_cached_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.load_cached_relation" + }, + "macro.dbt.load_csv_rows": { + "arguments": [], + "created_at": 1687942822.976654, + "depends_on": { + "macros": [ + "macro.dbt.default__load_csv_rows" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", + "meta": {}, + "name": "load_csv_rows", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.load_csv_rows" + }, + "macro.dbt.load_relation": { + "arguments": [], + "created_at": 1687942823.018698, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "load_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.load_relation" + }, + "macro.dbt.make_backup_relation": { + "arguments": [], + "created_at": 1687942823.015619, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__make_backup_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", + "meta": {}, + "name": "make_backup_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.make_backup_relation" + }, + "macro.dbt.make_hook_config": { + "arguments": [], + "created_at": 1687942822.8709161, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", + "meta": {}, + "name": "make_hook_config", + "original_file_path": "macros/materializations/hooks.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/hooks.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.make_hook_config" + }, + "macro.dbt.make_intermediate_relation": { + "arguments": [], + "created_at": 1687942823.014596, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__make_intermediate_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", + "meta": {}, + "name": "make_intermediate_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.make_intermediate_relation" + }, + "macro.dbt.make_temp_relation": { + "arguments": [], + "created_at": 1687942823.0150259, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__make_temp_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", + "meta": {}, + "name": "make_temp_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.make_temp_relation" + }, + "macro.dbt.materialization_incremental_default": { + "arguments": [], + "created_at": 1687942822.9339032, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation", + "macro.dbt.make_temp_relation", + "macro.dbt.make_intermediate_relation", + "macro.dbt.make_backup_relation", + "macro.dbt.should_full_refresh", + "macro.dbt.incremental_validate_on_schema_change", + "macro.dbt.drop_relation_if_exists", + "macro.dbt.run_hooks", + "macro.dbt.get_create_table_as_sql", + "macro.dbt.run_query", + "macro.dbt.process_schema_changes", + "macro.dbt.statement", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs", + "macro.dbt.create_indexes" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparison later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", + "meta": {}, + "name": "materialization_incremental_default", + "original_file_path": "macros/materializations/models/incremental/incremental.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/incremental.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_incremental_default" + }, + "macro.dbt.materialization_seed_default": { + "arguments": [], + "created_at": 1687942822.966659, + "depends_on": { + "macros": [ + "macro.dbt.should_full_refresh", + "macro.dbt.run_hooks", + "macro.dbt.reset_csv_table", + "macro.dbt.create_csv_table", + "macro.dbt.load_csv_rows", + "macro.dbt.noop_statement", + "macro.dbt.get_csv_sql", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs", + "macro.dbt.create_indexes" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparison later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", + "meta": {}, + "name": "materialization_seed_default", + "original_file_path": "macros/materializations/seeds/seed.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/seed.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_seed_default" + }, + "macro.dbt.materialization_snapshot_default": { + "arguments": [], + "created_at": 1687942822.901087, + "depends_on": { + "macros": [ + "macro.dbt.get_or_create_relation", + "macro.dbt.run_hooks", + "macro.dbt.strategy_dispatch", + "macro.dbt.build_snapshot_table", + "macro.dbt.create_table_as", + "macro.dbt.build_snapshot_staging_table", + "macro.dbt.create_columns", + "macro.dbt.snapshot_merge_sql", + "macro.dbt.statement", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs", + "macro.dbt.create_indexes", + "macro.dbt.post_snapshot" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparison later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", + "meta": {}, + "name": "materialization_snapshot_default", + "original_file_path": "macros/materializations/snapshots/snapshot.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/snapshot.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_snapshot_default" + }, + "macro.dbt.materialization_table_default": { + "arguments": [], + "created_at": 1687942822.952439, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation", + "macro.dbt.make_intermediate_relation", + "macro.dbt.make_backup_relation", + "macro.dbt.drop_relation_if_exists", + "macro.dbt.run_hooks", + "macro.dbt.statement", + "macro.dbt.get_create_table_as_sql", + "macro.dbt.create_indexes", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparison later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", + "meta": {}, + "name": "materialization_table_default", + "original_file_path": "macros/materializations/models/table/table.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/table.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_table_default" + }, + "macro.dbt.materialization_test_default": { + "arguments": [], + "created_at": 1687942822.903701, + "depends_on": { + "macros": [ + "macro.dbt.should_store_failures", + "macro.dbt.statement", + "macro.dbt.create_table_as", + "macro.dbt.get_test_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type='table') -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ create_table_as(False, target_relation, sql) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql %}\n\n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", + "meta": {}, + "name": "materialization_test_default", + "original_file_path": "macros/materializations/tests/test.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/tests/test.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_test_default" + }, + "macro.dbt.materialization_view_default": { + "arguments": [], + "created_at": 1687942822.9589398, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation", + "macro.dbt.make_intermediate_relation", + "macro.dbt.make_backup_relation", + "macro.dbt.run_hooks", + "macro.dbt.drop_relation_if_exists", + "macro.dbt.statement", + "macro.dbt.get_create_view_as_sql", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparison later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", + "meta": {}, + "name": "materialization_view_default", + "original_file_path": "macros/materializations/models/view/view.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/view/view.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_view_default" + }, + "macro.dbt.noop_statement": { + "arguments": [], + "created_at": 1687942822.984918, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "noop_statement", + "original_file_path": "macros/etc/statement.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/statement.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.noop_statement" + }, + "macro.dbt.partition_range": { + "arguments": [], + "created_at": 1687942822.989682, + "depends_on": { + "macros": [ + "macro.dbt.dates_in_range" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", + "meta": {}, + "name": "partition_range", + "original_file_path": "macros/etc/datetime.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/datetime.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.partition_range" + }, + "macro.dbt.persist_docs": { + "arguments": [], + "created_at": 1687942823.029068, + "depends_on": { + "macros": [ + "macro.dbt.default__persist_docs" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", + "meta": {}, + "name": "persist_docs", + "original_file_path": "macros/adapters/persist_docs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/persist_docs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.persist_docs" + }, + "macro.dbt.position": { + "arguments": [], + "created_at": 1687942822.998125, + "depends_on": { + "macros": [ + "macro.dbt.default__position" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", + "meta": {}, + "name": "position", + "original_file_path": "macros/utils/position.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/position.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.position" + }, + "macro.dbt.post_snapshot": { + "arguments": [], + "created_at": 1687942822.889555, + "depends_on": { + "macros": [ + "macro.dbt.default__post_snapshot" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "post_snapshot", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.post_snapshot" + }, + "macro.dbt.process_schema_changes": { + "arguments": [], + "created_at": 1687942822.9446778, + "depends_on": { + "macros": [ + "macro.dbt.check_for_schema_changes", + "macro.dbt.sync_column_schemas" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n\n Additional troubleshooting context:\n Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}\n Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}\n New column types: {{ schema_changes_dict['new_target_types'] }}\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", + "meta": {}, + "name": "process_schema_changes", + "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/on_schema_change.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.process_schema_changes" + }, + "macro.dbt.py_current_timestring": { + "arguments": [], + "created_at": 1687942822.989971, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", + "meta": {}, + "name": "py_current_timestring", + "original_file_path": "macros/etc/datetime.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/datetime.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.py_current_timestring" + }, + "macro.dbt.py_script_comment": { + "arguments": [], + "created_at": 1687942823.046462, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%macro py_script_comment()%}\n{%endmacro%}", + "meta": {}, + "name": "py_script_comment", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.py_script_comment" + }, + "macro.dbt.py_script_postfix": { + "arguments": [], + "created_at": 1687942823.04638, + "depends_on": { + "macros": [ + "macro.dbt.build_ref_function", + "macro.dbt.build_source_function", + "macro.dbt.build_config_dict", + "macro.dbt.resolve_model_name", + "macro.dbt.is_incremental", + "macro.dbt.py_script_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro py_script_postfix(model) %}\n# This part is user provided model code\n# you will need to copy the next section to run the code\n# COMMAND ----------\n# this part is dbt logic for get ref work, do not modify\n\n{{ build_ref_function(model ) }}\n{{ build_source_function(model ) }}\n{{ build_config_dict(model) }}\n\nclass config:\n def __init__(self, *args, **kwargs):\n pass\n\n @staticmethod\n def get(key, default=None):\n return config_dict.get(key, default)\n\nclass this:\n \"\"\"dbt.this() or dbt.this.identifier\"\"\"\n database = \"{{ this.database }}\"\n schema = \"{{ this.schema }}\"\n identifier = \"{{ this.identifier }}\"\n {% set this_relation_name = resolve_model_name(this) %}\n def __repr__(self):\n return '{{ this_relation_name }}'\n\n\nclass dbtObj:\n def __init__(self, load_df_function) -> None:\n self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function)\n self.ref = lambda *args, **kwargs: ref(*args, **kwargs, dbt_load_df_function=load_df_function)\n self.config = config\n self.this = this()\n self.is_incremental = {{ is_incremental() }}\n\n# COMMAND ----------\n{{py_script_comment()}}\n{% endmacro %}", + "meta": {}, + "name": "py_script_postfix", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.py_script_postfix" + }, + "macro.dbt.rename_relation": { + "arguments": [], + "created_at": 1687942823.017083, + "depends_on": { + "macros": [ + "macro.dbt.default__rename_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "rename_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.rename_relation" + }, + "macro.dbt.replace": { + "arguments": [], + "created_at": 1687942822.990763, + "depends_on": { + "macros": [ + "macro.dbt.default__replace" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", + "meta": {}, + "name": "replace", + "original_file_path": "macros/utils/replace.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.replace" + }, + "macro.dbt.reset_csv_table": { + "arguments": [], + "created_at": 1687942822.974001, + "depends_on": { + "macros": [ + "macro.dbt.default__reset_csv_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", + "meta": {}, + "name": "reset_csv_table", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.reset_csv_table" + }, + "macro.dbt.resolve_model_name": { + "arguments": [], + "created_at": 1687942823.043636, + "depends_on": { + "macros": [ + "macro.dbt.default__resolve_model_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro resolve_model_name(input_model_name) %}\n {{ return(adapter.dispatch('resolve_model_name', 'dbt')(input_model_name)) }}\n{% endmacro %}", + "meta": {}, + "name": "resolve_model_name", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.resolve_model_name" + }, + "macro.dbt.right": { + "arguments": [], + "created_at": 1687942822.9938078, + "depends_on": { + "macros": [ + "macro.dbt.default__right" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", + "meta": {}, + "name": "right", + "original_file_path": "macros/utils/right.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/right.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.right" + }, + "macro.dbt.run_hooks": { + "arguments": [], + "created_at": 1687942822.870699, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", + "meta": {}, + "name": "run_hooks", + "original_file_path": "macros/materializations/hooks.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/hooks.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.run_hooks" + }, + "macro.dbt.run_query": { + "arguments": [], + "created_at": 1687942822.985245, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", + "meta": {}, + "name": "run_query", + "original_file_path": "macros/etc/statement.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/statement.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.run_query" + }, + "macro.dbt.safe_cast": { + "arguments": [], + "created_at": 1687942822.996167, + "depends_on": { + "macros": [ + "macro.dbt.default__safe_cast" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", + "meta": {}, + "name": "safe_cast", + "original_file_path": "macros/utils/safe_cast.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/safe_cast.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.safe_cast" + }, + "macro.dbt.set_sql_header": { + "arguments": [], + "created_at": 1687942822.871808, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", + "meta": {}, + "name": "set_sql_header", + "original_file_path": "macros/materializations/configs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/configs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.set_sql_header" + }, + "macro.dbt.should_full_refresh": { + "arguments": [], + "created_at": 1687942822.8721411, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", + "meta": {}, + "name": "should_full_refresh", + "original_file_path": "macros/materializations/configs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/configs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.should_full_refresh" + }, + "macro.dbt.should_revoke": { + "arguments": [], + "created_at": 1687942823.022611, + "depends_on": { + "macros": [ + "macro.dbt.copy_grants" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", + "meta": {}, + "name": "should_revoke", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.should_revoke" + }, + "macro.dbt.should_store_failures": { + "arguments": [], + "created_at": 1687942822.872481, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", + "meta": {}, + "name": "should_store_failures", + "original_file_path": "macros/materializations/configs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/configs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.should_store_failures" + }, + "macro.dbt.snapshot_check_all_get_existing_columns": { + "arguments": [], + "created_at": 1687942822.881519, + "depends_on": { + "macros": [ + "macro.dbt.get_columns_in_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n {#-- N.B. The whitespace below is necessary to avoid edge case issue with comments --#}\n {#-- See: https://github.com/dbt-labs/dbt-core/issues/6781 --#}\n select {{ check_cols_config | join(', ') }} from (\n {{ node['compiled_code'] }}\n ) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", + "meta": {}, + "name": "snapshot_check_all_get_existing_columns", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns" + }, + "macro.dbt.snapshot_check_strategy": { + "arguments": [], + "created_at": 1687942822.8830268, + "depends_on": { + "macros": [ + "macro.dbt.snapshot_get_time", + "macro.dbt.snapshot_check_all_get_existing_columns", + "macro.dbt.get_true_sql", + "macro.dbt.snapshot_hash_arguments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n {% set updated_at = config.get('updated_at', snapshot_get_time()) %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", + "meta": {}, + "name": "snapshot_check_strategy", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_check_strategy" + }, + "macro.dbt.snapshot_get_time": { + "arguments": [], + "created_at": 1687942823.008611, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__snapshot_get_time" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "snapshot_get_time", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_get_time" + }, + "macro.dbt.snapshot_hash_arguments": { + "arguments": [], + "created_at": 1687942822.877908, + "depends_on": { + "macros": [ + "macro.dbt.default__snapshot_hash_arguments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", + "meta": {}, + "name": "snapshot_hash_arguments", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_hash_arguments" + }, + "macro.dbt.snapshot_merge_sql": { + "arguments": [], + "created_at": 1687942822.872962, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__snapshot_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", + "meta": {}, + "name": "snapshot_merge_sql", + "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/snapshot_merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_merge_sql" + }, + "macro.dbt.snapshot_staging_table": { + "arguments": [], + "created_at": 1687942822.890684, + "depends_on": { + "macros": [ + "macro.dbt.default__snapshot_staging_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "snapshot_staging_table", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_staging_table" + }, + "macro.dbt.snapshot_string_as_time": { + "arguments": [], + "created_at": 1687942822.8794858, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__snapshot_string_as_time" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", + "meta": {}, + "name": "snapshot_string_as_time", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_string_as_time" + }, + "macro.dbt.snapshot_timestamp_strategy": { + "arguments": [], + "created_at": 1687942822.879221, + "depends_on": { + "macros": [ + "macro.dbt.snapshot_hash_arguments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", + "meta": {}, + "name": "snapshot_timestamp_strategy", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_timestamp_strategy" + }, + "macro.dbt.split_part": { + "arguments": [], + "created_at": 1687942823.004601, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__split_part" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", + "meta": {}, + "name": "split_part", + "original_file_path": "macros/utils/split_part.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/split_part.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.split_part" + }, + "macro.dbt.sql_convert_columns_in_relation": { + "arguments": [], + "created_at": 1687942823.0370119, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", + "meta": {}, + "name": "sql_convert_columns_in_relation", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.sql_convert_columns_in_relation" + }, + "macro.dbt.statement": { + "arguments": [], + "created_at": 1687942822.9842792, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%}\n {%- if execute: -%}\n {%- set compiled_code = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime {} for node \"{}\"'.format(language, model['unique_id'])) }}\n {{ write(compiled_code) }}\n {%- endif -%}\n {%- if language == 'sql'-%}\n {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- elif language == 'python' -%}\n {%- set res = submit_python_job(model, compiled_code) -%}\n {#-- TODO: What should table be for python models? --#}\n {%- set table = None -%}\n {%- else -%}\n {% do exceptions.raise_compiler_error(\"statement macro didn't get supported language\") %}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", + "meta": {}, + "name": "statement", + "original_file_path": "macros/etc/statement.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/statement.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.statement" + }, + "macro.dbt.strategy_dispatch": { + "arguments": [], + "created_at": 1687942822.877471, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", + "meta": {}, + "name": "strategy_dispatch", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.strategy_dispatch" + }, + "macro.dbt.string_literal": { + "arguments": [], + "created_at": 1687942822.998741, + "depends_on": { + "macros": [ + "macro.dbt.default__string_literal" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "string_literal", + "original_file_path": "macros/utils/literal.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/literal.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.string_literal" + }, + "macro.dbt.support_multiple_grantees_per_dcl_statement": { + "arguments": [], + "created_at": 1687942823.022024, + "depends_on": { + "macros": [ + "macro.dbt.default__support_multiple_grantees_per_dcl_statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "support_multiple_grantees_per_dcl_statement", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement" + }, + "macro.dbt.sync_column_schemas": { + "arguments": [], + "created_at": 1687942822.943712, + "depends_on": { + "macros": [ + "macro.dbt.alter_relation_add_remove_columns", + "macro.dbt.alter_column_type" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", + "meta": {}, + "name": "sync_column_schemas", + "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/on_schema_change.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.sync_column_schemas" + }, + "macro.dbt.table_columns_and_constraints": { + "arguments": [], + "created_at": 1687942822.9465652, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro table_columns_and_constraints() %}\n {# loop through user_provided_columns to create DDL with data types and constraints #}\n {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%}\n {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%}\n (\n {% for c in raw_column_constraints -%}\n {{ c }}{{ \",\" if not loop.last or raw_model_constraints }}\n {% endfor %}\n {% for c in raw_model_constraints -%}\n {{ c }}{{ \",\" if not loop.last }}\n {% endfor -%}\n )\n{% endmacro %}", + "meta": {}, + "name": "table_columns_and_constraints", + "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.table_columns_and_constraints" + }, + "macro.dbt.test_accepted_values": { + "arguments": [], + "created_at": 1687942823.047759, + "depends_on": { + "macros": [ + "macro.dbt.default__test_accepted_values" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", + "meta": {}, + "name": "test_accepted_values", + "original_file_path": "tests/generic/builtin.sql", + "package_name": "dbt", + "patch_path": null, + "path": "tests/generic/builtin.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.test_accepted_values" + }, + "macro.dbt.test_not_null": { + "arguments": [], + "created_at": 1687942823.047362, + "depends_on": { + "macros": [ + "macro.dbt.default__test_not_null" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", + "meta": {}, + "name": "test_not_null", + "original_file_path": "tests/generic/builtin.sql", + "package_name": "dbt", + "patch_path": null, + "path": "tests/generic/builtin.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.test_not_null" + }, + "macro.dbt.test_relationships": { + "arguments": [], + "created_at": 1687942823.048068, + "depends_on": { + "macros": [ + "macro.dbt.default__test_relationships" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", + "meta": {}, + "name": "test_relationships", + "original_file_path": "tests/generic/builtin.sql", + "package_name": "dbt", + "patch_path": null, + "path": "tests/generic/builtin.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.test_relationships" + }, + "macro.dbt.test_unique": { + "arguments": [], + "created_at": 1687942823.047106, + "depends_on": { + "macros": [ + "macro.dbt.default__test_unique" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", + "meta": {}, + "name": "test_unique", + "original_file_path": "tests/generic/builtin.sql", + "package_name": "dbt", + "patch_path": null, + "path": "tests/generic/builtin.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.test_unique" + }, + "macro.dbt.truncate_relation": { + "arguments": [], + "created_at": 1687942823.0166788, + "depends_on": { + "macros": [ + "macro.dbt.default__truncate_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "truncate_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.truncate_relation" + }, + "macro.dbt.type_bigint": { + "arguments": [], + "created_at": 1687942823.001445, + "depends_on": { + "macros": [ + "macro.dbt.default__type_bigint" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_bigint", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_bigint" + }, + "macro.dbt.type_boolean": { + "arguments": [], + "created_at": 1687942823.002202, + "depends_on": { + "macros": [ + "macro.dbt.default__type_boolean" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_boolean() -%}\n {{ return(adapter.dispatch('type_boolean', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_boolean", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_boolean" + }, + "macro.dbt.type_float": { + "arguments": [], + "created_at": 1687942823.000724, + "depends_on": { + "macros": [ + "macro.dbt.default__type_float" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_float", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_float" + }, + "macro.dbt.type_int": { + "arguments": [], + "created_at": 1687942823.001866, + "depends_on": { + "macros": [ + "macro.dbt.default__type_int" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_int", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_int" + }, + "macro.dbt.type_numeric": { + "arguments": [], + "created_at": 1687942823.0010638, + "depends_on": { + "macros": [ + "macro.dbt.default__type_numeric" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_numeric", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_numeric" + }, + "macro.dbt.type_string": { + "arguments": [], + "created_at": 1687942823.00002, + "depends_on": { + "macros": [ + "macro.dbt.default__type_string" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_string", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_string" + }, + "macro.dbt.type_timestamp": { + "arguments": [], + "created_at": 1687942823.000371, + "depends_on": { + "macros": [ + "macro.dbt.default__type_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_timestamp", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_timestamp" + }, + "macro.dbt_postgres.postgres__alter_column_comment": { + "arguments": [], + "created_at": 1687942822.8619561, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres_escape_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% for column_name in column_dict if (column_name in existing_columns) %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", + "meta": {}, + "name": "postgres__alter_column_comment", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__alter_column_comment" + }, + "macro.dbt_postgres.postgres__alter_relation_comment": { + "arguments": [], + "created_at": 1687942822.861244, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres_escape_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", + "meta": {}, + "name": "postgres__alter_relation_comment", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__alter_relation_comment" + }, + "macro.dbt_postgres.postgres__any_value": { + "arguments": [], + "created_at": 1687942822.868335, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__any_value(expression) -%}\n\n min({{ expression }})\n\n{%- endmacro %}", + "meta": {}, + "name": "postgres__any_value", + "original_file_path": "macros/utils/any_value.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/utils/any_value.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__any_value" + }, + "macro.dbt_postgres.postgres__check_schema_exists": { + "arguments": [], + "created_at": 1687942822.857978, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__check_schema_exists(information_schema, schema) -%}\n {% if information_schema.database -%}\n {{ adapter.verify_database(information_schema.database) }}\n {%- endif -%}\n {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}\n select count(*) from pg_namespace where nspname = '{{ schema }}'\n {% endcall %}\n {{ return(load_result('check_schema_exists').table) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__check_schema_exists", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__check_schema_exists" + }, + "macro.dbt_postgres.postgres__copy_grants": { + "arguments": [], + "created_at": 1687942822.862329, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__copy_grants() %}\n {{ return(False) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__copy_grants", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__copy_grants" + }, + "macro.dbt_postgres.postgres__create_schema": { + "arguments": [], + "created_at": 1687942822.8555672, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__create_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier().include(database=False) }}\n {%- endcall -%}\n{% endmacro %}", + "meta": {}, + "name": "postgres__create_schema", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__create_schema" + }, + "macro.dbt_postgres.postgres__create_table_as": { + "arguments": [], + "created_at": 1687942822.854578, + "depends_on": { + "macros": [ + "macro.dbt.get_assert_columns_equivalent", + "macro.dbt.get_table_columns_and_constraints", + "macro.dbt.default__get_column_names", + "macro.dbt.get_select_subquery" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__create_table_as(temporary, relation, sql) -%}\n {%- set unlogged = config.get('unlogged', default=false) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary -%}\n temporary\n {%- elif unlogged -%}\n unlogged\n {%- endif %} table {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }} ;\n insert into {{ relation }} (\n {{ adapter.dispatch('get_column_names', 'dbt')() }}\n )\n {%- set sql = get_select_subquery(sql) %}\n {% else %}\n as\n {% endif %}\n (\n {{ sql }}\n );\n{%- endmacro %}", + "meta": {}, + "name": "postgres__create_table_as", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__create_table_as" + }, + "macro.dbt_postgres.postgres__current_timestamp": { + "arguments": [], + "created_at": 1687942822.8427362, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__current_timestamp() -%}\n now()\n{%- endmacro %}", + "meta": {}, + "name": "postgres__current_timestamp", + "original_file_path": "macros/timestamps.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__current_timestamp" + }, + "macro.dbt_postgres.postgres__current_timestamp_backcompat": { + "arguments": [], + "created_at": 1687942822.843909, + "depends_on": { + "macros": [ + "macro.dbt.type_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__current_timestamp_backcompat() %}\n current_timestamp::{{ type_timestamp() }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__current_timestamp_backcompat", + "original_file_path": "macros/timestamps.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__current_timestamp_backcompat" + }, + "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat": { + "arguments": [], + "created_at": 1687942822.8440351, + "depends_on": { + "macros": [ + "macro.dbt.type_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__current_timestamp_in_utc_backcompat() %}\n (current_timestamp at time zone 'utc')::{{ type_timestamp() }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__current_timestamp_in_utc_backcompat", + "original_file_path": "macros/timestamps.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat" + }, + "macro.dbt_postgres.postgres__dateadd": { + "arguments": [], + "created_at": 1687942822.863829, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }}))\n\n{% endmacro %}", + "meta": {}, + "name": "postgres__dateadd", + "original_file_path": "macros/utils/dateadd.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/utils/dateadd.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__dateadd" + }, + "macro.dbt_postgres.postgres__datediff": { + "arguments": [], + "created_at": 1687942822.868163, + "depends_on": { + "macros": [ + "macro.dbt.datediff" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__datediff(first_date, second_date, datepart) -%}\n\n {% if datepart == 'year' %}\n (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date))\n {% elif datepart == 'quarter' %}\n ({{ datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date))\n {% elif datepart == 'month' %}\n ({{ datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date))\n {% elif datepart == 'day' %}\n (({{second_date}})::date - ({{first_date}})::date)\n {% elif datepart == 'week' %}\n ({{ datediff(first_date, second_date, 'day') }} / 7 + case\n when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then\n case when {{first_date}} <= {{second_date}} then 0 else -1 end\n else\n case when {{first_date}} <= {{second_date}} then 1 else 0 end\n end)\n {% elif datepart == 'hour' %}\n ({{ datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp))\n {% elif datepart == 'minute' %}\n ({{ datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp))\n {% elif datepart == 'second' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp)))\n {% elif datepart == 'millisecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp)))\n {% elif datepart == 'microsecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp)))\n {% else %}\n {{ exceptions.raise_compiler_error(\"Unsupported datepart for macro datediff in postgres: {!r}\".format(datepart)) }}\n {% endif %}\n\n{%- endmacro %}", + "meta": {}, + "name": "postgres__datediff", + "original_file_path": "macros/utils/datediff.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/utils/datediff.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__datediff" + }, + "macro.dbt_postgres.postgres__drop_schema": { + "arguments": [], + "created_at": 1687942822.855936, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__drop_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade\n {%- endcall -%}\n{% endmacro %}", + "meta": {}, + "name": "postgres__drop_schema", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__drop_schema" + }, + "macro.dbt_postgres.postgres__get_catalog": { + "arguments": [], + "created_at": 1687942822.845242, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_catalog(information_schema, schemas) -%}\n\n {%- call statement('catalog', fetch_result=True) -%}\n {#\n If the user has multiple databases set and the first one is wrong, this will fail.\n But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.\n #}\n {% set database = information_schema.database %}\n {{ adapter.verify_database(database) }}\n\n select\n '{{ database }}' as table_database,\n sch.nspname as table_schema,\n tbl.relname as table_name,\n case tbl.relkind\n when 'v' then 'VIEW'\n else 'BASE TABLE'\n end as table_type,\n tbl_desc.description as table_comment,\n col.attname as column_name,\n col.attnum as column_index,\n pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,\n col_desc.description as column_comment,\n pg_get_userbyid(tbl.relowner) as table_owner\n\n from pg_catalog.pg_namespace sch\n join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid\n join pg_catalog.pg_attribute col on col.attrelid = tbl.oid\n left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)\n left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)\n\n where (\n {%- for schema in schemas -%}\n upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session\n and tbl.relpersistence in ('p', 'u') -- [p]ermanent table or [u]nlogged table. Exclude [t]emporary tables\n and tbl.relkind in ('r', 'v', 'f', 'p') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table, [m]aterialized view\n and col.attnum > 0 -- negative numbers are used for system columns such as oid\n and not col.attisdropped -- column as not been dropped\n\n order by\n sch.nspname,\n tbl.relname,\n col.attnum\n\n {%- endcall -%}\n\n {{ return(load_result('catalog').table) }}\n\n{%- endmacro %}", + "meta": {}, + "name": "postgres__get_catalog", + "original_file_path": "macros/catalog.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/catalog.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_catalog" + }, + "macro.dbt_postgres.postgres__get_columns_in_relation": { + "arguments": [], + "created_at": 1687942822.8564942, + "depends_on": { + "macros": [ + "macro.dbt.statement", + "macro.dbt.sql_convert_columns_in_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n select\n column_name,\n data_type,\n character_maximum_length,\n numeric_precision,\n numeric_scale\n\n from {{ relation.information_schema('columns') }}\n where table_name = '{{ relation.identifier }}'\n {% if relation.schema %}\n and table_schema = '{{ relation.schema }}'\n {% endif %}\n order by ordinal_position\n\n {% endcall %}\n {% set table = load_result('get_columns_in_relation').table %}\n {{ return(sql_convert_columns_in_relation(table)) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__get_columns_in_relation", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_columns_in_relation" + }, + "macro.dbt_postgres.postgres__get_create_index_sql": { + "arguments": [], + "created_at": 1687942822.855187, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_create_index_sql(relation, index_dict) -%}\n {%- set index_config = adapter.parse_index(index_dict) -%}\n {%- set comma_separated_columns = \", \".join(index_config.columns) -%}\n {%- set index_name = index_config.render(relation) -%}\n\n create {% if index_config.unique -%}\n unique\n {%- endif %} index if not exists\n \"{{ index_name }}\"\n on {{ relation }} {% if index_config.type -%}\n using {{ index_config.type }}\n {%- endif %}\n ({{ comma_separated_columns }});\n{%- endmacro %}", + "meta": {}, + "name": "postgres__get_create_index_sql", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_create_index_sql" + }, + "macro.dbt_postgres.postgres__get_incremental_default_sql": { + "arguments": [], + "created_at": 1687942822.8627782, + "depends_on": { + "macros": [ + "macro.dbt.get_incremental_delete_insert_sql", + "macro.dbt.get_incremental_append_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_incremental_default_sql(arg_dict) %}\n\n {% if arg_dict[\"unique_key\"] %}\n {% do return(get_incremental_delete_insert_sql(arg_dict)) %}\n {% else %}\n {% do return(get_incremental_append_sql(arg_dict)) %}\n {% endif %}\n\n{% endmacro %}", + "meta": {}, + "name": "postgres__get_incremental_default_sql", + "original_file_path": "macros/materializations/incremental_strategies.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/materializations/incremental_strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_incremental_default_sql" + }, + "macro.dbt_postgres.postgres__get_show_grant_sql": { + "arguments": [], + "created_at": 1687942822.8621938, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro postgres__get_show_grant_sql(relation) -%}\n select grantee, privilege_type\n from {{ relation.information_schema('role_table_grants') }}\n where grantor = current_role\n and grantee != current_role\n and table_schema = '{{ relation.schema }}'\n and table_name = '{{ relation.identifier }}'\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "postgres__get_show_grant_sql", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_show_grant_sql" + }, + "macro.dbt_postgres.postgres__information_schema_name": { + "arguments": [], + "created_at": 1687942822.8571231, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__information_schema_name(database) -%}\n {% if database_name -%}\n {{ adapter.verify_database(database_name) }}\n {%- endif -%}\n information_schema\n{%- endmacro %}", + "meta": {}, + "name": "postgres__information_schema_name", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__information_schema_name" + }, + "macro.dbt_postgres.postgres__last_day": { + "arguments": [], + "created_at": 1687942822.868885, + "depends_on": { + "macros": [ + "macro.dbt.dateadd", + "macro.dbt.date_trunc", + "macro.dbt.default_last_day" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__last_day(date, datepart) -%}\n\n {%- if datepart == 'quarter' -%}\n -- postgres dateadd does not support quarter interval.\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd('month', '3', dbt.date_trunc(datepart, date))\n )}}\n as date)\n {%- else -%}\n {{dbt.default_last_day(date, datepart)}}\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "postgres__last_day", + "original_file_path": "macros/utils/last_day.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/utils/last_day.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__last_day" + }, + "macro.dbt_postgres.postgres__list_relations_without_caching": { + "arguments": [], + "created_at": 1687942822.856927, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n select\n '{{ schema_relation.database }}' as database,\n tablename as name,\n schemaname as schema,\n 'table' as type\n from pg_tables\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n viewname as name,\n schemaname as schema,\n 'view' as type\n from pg_views\n where schemaname ilike '{{ schema_relation.schema }}'\n {% endcall %}\n {{ return(load_result('list_relations_without_caching').table) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__list_relations_without_caching", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__list_relations_without_caching" + }, + "macro.dbt_postgres.postgres__list_schemas": { + "arguments": [], + "created_at": 1687942822.857524, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__list_schemas(database) %}\n {% if database -%}\n {{ adapter.verify_database(database) }}\n {%- endif -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n select distinct nspname from pg_namespace\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__list_schemas", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__list_schemas" + }, + "macro.dbt_postgres.postgres__listagg": { + "arguments": [], + "created_at": 1687942822.864536, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n (array_agg(\n {{ measure }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n ))[1:{{ limit_num }}],\n {{ delimiter_text }}\n )\n {%- else %}\n string_agg(\n {{ measure }},\n {{ delimiter_text }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n )\n {%- endif %}\n\n{%- endmacro %}", + "meta": {}, + "name": "postgres__listagg", + "original_file_path": "macros/utils/listagg.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/utils/listagg.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__listagg" + }, + "macro.dbt_postgres.postgres__make_backup_relation": { + "arguments": [], + "created_at": 1687942822.860425, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__make_relation_with_suffix" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {% set backup_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=False) %}\n {{ return(backup_relation.incorporate(type=backup_relation_type)) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__make_backup_relation", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__make_backup_relation" + }, + "macro.dbt_postgres.postgres__make_intermediate_relation": { + "arguments": [], + "created_at": 1687942822.859671, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__make_relation_with_suffix" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__make_intermediate_relation(base_relation, suffix) %}\n {{ return(postgres__make_relation_with_suffix(base_relation, suffix, dstring=False)) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__make_intermediate_relation", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__make_intermediate_relation" + }, + "macro.dbt_postgres.postgres__make_relation_with_suffix": { + "arguments": [], + "created_at": 1687942822.8594298, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__make_relation_with_suffix(base_relation, suffix, dstring) %}\n {% if dstring %}\n {% set dt = modules.datetime.datetime.now() %}\n {% set dtstring = dt.strftime(\"%H%M%S%f\") %}\n {% set suffix = suffix ~ dtstring %}\n {% endif %}\n {% set suffix_length = suffix|length %}\n {% set relation_max_name_length = base_relation.relation_max_name_length() %}\n {% if suffix_length > relation_max_name_length %}\n {% do exceptions.raise_compiler_error('Relation suffix is too long (' ~ suffix_length ~ ' characters). Maximum length is ' ~ relation_max_name_length ~ ' characters.') %}\n {% endif %}\n {% set identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix %}\n\n {{ return(base_relation.incorporate(path={\"identifier\": identifier })) }}\n\n {% endmacro %}", + "meta": {}, + "name": "postgres__make_relation_with_suffix", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__make_relation_with_suffix" + }, + "macro.dbt_postgres.postgres__make_temp_relation": { + "arguments": [], + "created_at": 1687942822.86004, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__make_relation_with_suffix" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__make_temp_relation(base_relation, suffix) %}\n {% set temp_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=True) %}\n {{ return(temp_relation.incorporate(path={\"schema\": none,\n \"database\": none})) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__make_temp_relation", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__make_temp_relation" + }, + "macro.dbt_postgres.postgres__snapshot_get_time": { + "arguments": [], + "created_at": 1687942822.84378, + "depends_on": { + "macros": [ + "macro.dbt.current_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__snapshot_get_time() -%}\n {{ current_timestamp() }}::timestamp without time zone\n{%- endmacro %}", + "meta": {}, + "name": "postgres__snapshot_get_time", + "original_file_path": "macros/timestamps.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__snapshot_get_time" + }, + "macro.dbt_postgres.postgres__snapshot_merge_sql": { + "arguments": [], + "created_at": 1687942822.8635678, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n update {{ target }}\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text\n and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)\n and {{ target }}.dbt_valid_to is null;\n\n insert into {{ target }} ({{ insert_cols_csv }})\n select {% for column in insert_cols -%}\n DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;\n{% endmacro %}", + "meta": {}, + "name": "postgres__snapshot_merge_sql", + "original_file_path": "macros/materializations/snapshot_merge.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/materializations/snapshot_merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__snapshot_merge_sql" + }, + "macro.dbt_postgres.postgres__snapshot_string_as_time": { + "arguments": [], + "created_at": 1687942822.8435972, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"'\" ~ timestamp ~ \"'::timestamp without time zone\" -%}\n {{ return(result) }}\n{%- endmacro %}", + "meta": {}, + "name": "postgres__snapshot_string_as_time", + "original_file_path": "macros/timestamps.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__snapshot_string_as_time" + }, + "macro.dbt_postgres.postgres__split_part": { + "arguments": [], + "created_at": 1687942822.86934, + "depends_on": { + "macros": [ + "macro.dbt.default__split_part", + "macro.dbt._split_part_negative" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__split_part(string_text, delimiter_text, part_number) %}\n\n {% if part_number >= 0 %}\n {{ dbt.default__split_part(string_text, delimiter_text, part_number) }}\n {% else %}\n {{ dbt._split_part_negative(string_text, delimiter_text, part_number) }}\n {% endif %}\n\n{% endmacro %}", + "meta": {}, + "name": "postgres__split_part", + "original_file_path": "macros/utils/split_part.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/utils/split_part.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__split_part" + }, + "macro.dbt_postgres.postgres_escape_comment": { + "arguments": [], + "created_at": 1687942822.860971, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres_escape_comment(comment) -%}\n {% if comment is not string %}\n {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}\n {% endif %}\n {%- set magic = '$dbt_comment_literal_block$' -%}\n {%- if magic in comment -%}\n {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}\n {%- endif -%}\n {{ magic }}{{ comment }}{{ magic }}\n{%- endmacro %}", + "meta": {}, + "name": "postgres_escape_comment", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres_escape_comment" + }, + "macro.dbt_postgres.postgres_get_relations": { + "arguments": [], + "created_at": 1687942822.845938, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres_get_relations () -%}\n\n {#\n -- in pg_depend, objid is the dependent, refobjid is the referenced object\n -- > a pg_depend entry indicates that the referenced object cannot be\n -- > dropped without also dropping the dependent object.\n #}\n\n {%- call statement('relations', fetch_result=True) -%}\n with relation as (\n select\n pg_rewrite.ev_class as class,\n pg_rewrite.oid as id\n from pg_rewrite\n ),\n class as (\n select\n oid as id,\n relname as name,\n relnamespace as schema,\n relkind as kind\n from pg_class\n ),\n dependency as (\n select distinct\n pg_depend.objid as id,\n pg_depend.refobjid as ref\n from pg_depend\n ),\n schema as (\n select\n pg_namespace.oid as id,\n pg_namespace.nspname as name\n from pg_namespace\n where nspname != 'information_schema' and nspname not like 'pg\\_%'\n ),\n referenced as (\n select\n relation.id AS id,\n referenced_class.name ,\n referenced_class.schema ,\n referenced_class.kind\n from relation\n join class as referenced_class on relation.class=referenced_class.id\n where referenced_class.kind in ('r', 'v')\n ),\n relationships as (\n select\n referenced.name as referenced_name,\n referenced.schema as referenced_schema_id,\n dependent_class.name as dependent_name,\n dependent_class.schema as dependent_schema_id,\n referenced.kind as kind\n from referenced\n join dependency on referenced.id=dependency.id\n join class as dependent_class on dependency.ref=dependent_class.id\n where\n (referenced.name != dependent_class.name or\n referenced.schema != dependent_class.schema)\n )\n\n select\n referenced_schema.name as referenced_schema,\n relationships.referenced_name as referenced_name,\n dependent_schema.name as dependent_schema,\n relationships.dependent_name as dependent_name\n from relationships\n join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id\n join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id\n group by referenced_schema, referenced_name, dependent_schema, dependent_name\n order by referenced_schema, referenced_name, dependent_schema, dependent_name;\n\n {%- endcall -%}\n\n {{ return(load_result('relations').table) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres_get_relations", + "original_file_path": "macros/relations.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres_get_relations" + }, + "macro.jaffle_shop.drop_table": { + "arguments": [], + "created_at": 1687942822.842345, + "depends_on": { + "macros": [ + "macro.dbt.run_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro drop_table(table_name) -%}\n {%- set drop_query -%}\n DROP TABLE IF EXISTS {{ target.schema }}.{{ table_name }} CASCADE\n {%- endset -%}\n {% do run_query(drop_query) %}\n{%- endmacro -%}", + "meta": {}, + "name": "drop_table", + "original_file_path": "macros/drop_table.sql", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "macros/drop_table.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.jaffle_shop.drop_table" + } + }, + "metadata": { + "adapter_type": "postgres", + "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v9.json", + "dbt_version": "1.5.2", + "env": {}, + "generated_at": "2023-06-28T09:00:22.812217Z", + "invocation_id": "67cf034b-da45-43ca-973c-71bafad9f92c", + "project_id": "06e5b98c2db46f8a72cc4f66410e9b3b", + "send_anonymous_usage_stats": true, + "user_id": "842c63fe-29b4-41e3-b5ef-9e065d34bcc0" + }, + "metrics": {}, + "nodes": { + "model.jaffle_shop.customers": { + "access": "protected", + "alias": "customers", + "build_path": null, + "checksum": { + "checksum": "60bd72e33da43fff3a7e7609135c17cd4468bd22afec0735dd36018bfb5af30a", + "name": "sha256" + }, + "columns": { + "customer_id": { + "constraints": [], + "data_type": null, + "description": "This is a unique identifier for a customer", + "meta": {}, + "name": "customer_id", + "quote": null, + "tags": [] + }, + "first_name": { + "constraints": [], + "data_type": null, + "description": "Customer's first name. PII.", + "meta": {}, + "name": "first_name", + "quote": null, + "tags": [] + }, + "first_order": { + "constraints": [], + "data_type": null, + "description": "Date (UTC) of a customer's first order", + "meta": {}, + "name": "first_order", + "quote": null, + "tags": [] + }, + "last_name": { + "constraints": [], + "data_type": null, + "description": "Customer's last name. PII.", + "meta": {}, + "name": "last_name", + "quote": null, + "tags": [] + }, + "most_recent_order": { + "constraints": [], + "data_type": null, + "description": "Date (UTC) of a customer's most recent order", + "meta": {}, + "name": "most_recent_order", + "quote": null, + "tags": [] + }, + "number_of_orders": { + "constraints": [], + "data_type": null, + "description": "Count of the number of orders a customer has placed", + "meta": {}, + "name": "number_of_orders", + "quote": null, + "tags": [] + }, + "total_order_amount": { + "constraints": [], + "data_type": null, + "description": "Total value (AUD) of a customer's orders", + "meta": {}, + "name": "total_order_amount", + "quote": null, + "tags": [] + } + }, + "compiled_path": null, + "config": { + "alias": null, + "column_types": {}, + "contract": { + "enforced": false + }, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "table", + "meta": {}, + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "schema": null, + "tags": [], + "unique_key": null + }, + "constraints": [], + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.256299, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [], + "nodes": [ + "model.jaffle_shop.stg_customers", + "model.jaffle_shop.stg_orders", + "model.jaffle_shop.stg_payments" + ] + }, + "description": "This table has basic information about a customer, as well as some derived facts based on a customer's orders", + "docs": { + "node_color": null, + "show": true + }, + "fqn": [ + "jaffle_shop", + "customers" + ], + "group": null, + "language": "sql", + "latest_version": null, + "meta": {}, + "metrics": [], + "name": "customers", + "original_file_path": "models/customers.sql", + "package_name": "jaffle_shop", + "patch_path": "jaffle_shop://models/schema.yml", + "path": "customers.sql", + "raw_code": "with customers as (\n\n select * from {{ ref('stg_customers') }}\n\n),\n\norders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\ncustomer_orders as (\n\n select\n customer_id,\n\n min(order_date) as first_order,\n max(order_date) as most_recent_order,\n count(order_id) as number_of_orders\n from orders\n\n group by customer_id\n\n),\n\ncustomer_payments as (\n\n select\n orders.customer_id,\n sum(amount) as total_amount\n\n from payments\n\n left join orders on\n payments.order_id = orders.order_id\n\n group by orders.customer_id\n\n),\n\nfinal as (\n\n select\n customers.customer_id,\n customers.first_name,\n customers.last_name,\n customer_orders.first_order,\n customer_orders.most_recent_order,\n customer_orders.number_of_orders,\n customer_payments.total_amount as customer_lifetime_value\n\n from customers\n\n left join customer_orders\n on customers.customer_id = customer_orders.customer_id\n\n left join customer_payments\n on customers.customer_id = customer_payments.customer_id\n\n)\n\nselect * from final", + "refs": [ + { + "name": "stg_customers", + "package": null, + "version": null + }, + { + "name": "stg_orders", + "package": null, + "version": null + }, + { + "name": "stg_payments", + "package": null, + "version": null + } + ], + "relation_name": "\"postgres\".\"public\".\"customers\"", + "resource_type": "model", + "schema": "public", + "sources": [], + "tags": [ + "test_tag" + ], + "unique_id": "model.jaffle_shop.customers", + "unrendered_config": { + "materialized": "table" + }, + "version": null + }, + "model.jaffle_shop.orders": { + "access": "protected", + "alias": "orders", + "build_path": null, + "checksum": { + "checksum": "27f8c79aad1cfd8411ab9c3d2ce8da1d787f7f05c58bbee1d247510dc426be0f", + "name": "sha256" + }, + "columns": { + "amount": { + "constraints": [], + "data_type": null, + "description": "Total amount (AUD) of the order", + "meta": {}, + "name": "amount", + "quote": null, + "tags": [] + }, + "bank_transfer_amount": { + "constraints": [], + "data_type": null, + "description": "Amount of the order (AUD) paid for by bank transfer", + "meta": {}, + "name": "bank_transfer_amount", + "quote": null, + "tags": [] + }, + "coupon_amount": { + "constraints": [], + "data_type": null, + "description": "Amount of the order (AUD) paid for by coupon", + "meta": {}, + "name": "coupon_amount", + "quote": null, + "tags": [] + }, + "credit_card_amount": { + "constraints": [], + "data_type": null, + "description": "Amount of the order (AUD) paid for by credit card", + "meta": {}, + "name": "credit_card_amount", + "quote": null, + "tags": [] + }, + "customer_id": { + "constraints": [], + "data_type": null, + "description": "Foreign key to the customers table", + "meta": {}, + "name": "customer_id", + "quote": null, + "tags": [] + }, + "gift_card_amount": { + "constraints": [], + "data_type": null, + "description": "Amount of the order (AUD) paid for by gift card", + "meta": {}, + "name": "gift_card_amount", + "quote": null, + "tags": [] + }, + "order_date": { + "constraints": [], + "data_type": null, + "description": "Date (UTC) that the order was placed", + "meta": {}, + "name": "order_date", + "quote": null, + "tags": [] + }, + "order_id": { + "constraints": [], + "data_type": null, + "description": "This is a unique identifier for an order", + "meta": {}, + "name": "order_id", + "quote": null, + "tags": [] + }, + "status": { + "constraints": [], + "data_type": null, + "description": "Orders can be one of the following statuses:\n\n| status | description |\n|----------------|------------------------------------------------------------------------------------------------------------------------|\n| placed | The order has been placed but has not yet left the warehouse |\n| shipped | The order has ben shipped to the customer and is currently in transit |\n| completed | The order has been received by the customer |\n| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse |\n| returned | The order has been returned by the customer and received at the warehouse |", + "meta": {}, + "name": "status", + "quote": null, + "tags": [] + } + }, + "compiled_path": null, + "config": { + "alias": null, + "column_types": {}, + "contract": { + "enforced": false + }, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "table", + "meta": {}, + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "schema": null, + "tags": [], + "unique_key": null + }, + "constraints": [], + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.258613, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [], + "nodes": [ + "model.jaffle_shop.stg_orders", + "model.jaffle_shop.stg_payments" + ] + }, + "description": "This table has basic information about orders, as well as some derived facts based on payments", + "docs": { + "node_color": null, + "show": true + }, + "fqn": [ + "jaffle_shop", + "orders" + ], + "group": null, + "language": "sql", + "latest_version": null, + "meta": {}, + "metrics": [], + "name": "orders", + "original_file_path": "models/orders.sql", + "package_name": "jaffle_shop", + "patch_path": "jaffle_shop://models/schema.yml", + "path": "orders.sql", + "raw_code": "{% set payment_methods = ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] %}\n\nwith orders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\norder_payments as (\n\n select\n order_id,\n\n {% for payment_method in payment_methods -%}\n sum(case when payment_method = '{{ payment_method }}' then amount else 0 end) as {{ payment_method }}_amount,\n {% endfor -%}\n\n sum(amount) as total_amount\n\n from payments\n\n group by order_id\n\n),\n\nfinal as (\n\n select\n orders.order_id,\n orders.customer_id,\n orders.order_date,\n orders.status,\n\n {% for payment_method in payment_methods -%}\n\n order_payments.{{ payment_method }}_amount,\n\n {% endfor -%}\n\n order_payments.total_amount as amount\n\n from orders\n\n\n left join order_payments\n on orders.order_id = order_payments.order_id\n\n)\n\nselect * from final", + "refs": [ + { + "name": "stg_orders", + "package": null, + "version": null + }, + { + "name": "stg_payments", + "package": null, + "version": null + } + ], + "relation_name": "\"postgres\".\"public\".\"orders\"", + "resource_type": "model", + "schema": "public", + "sources": [], + "tags": [ + "test_tag" + ], + "unique_id": "model.jaffle_shop.orders", + "unrendered_config": { + "materialized": "table" + }, + "version": null + }, + "model.jaffle_shop.stg_customers": { + "access": "protected", + "alias": "stg_customers", + "build_path": null, + "checksum": { + "checksum": "80e3223cd54387e11fa16cd0f4cbe15f8ff74dcd9900b93856b9e39416178c9d", + "name": "sha256" + }, + "columns": { + "customer_id": { + "constraints": [], + "data_type": null, + "description": "", + "meta": {}, + "name": "customer_id", + "quote": null, + "tags": [] + } + }, + "compiled_path": null, + "config": { + "alias": null, + "column_types": {}, + "contract": { + "enforced": false + }, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "view", + "meta": {}, + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "schema": null, + "tags": [], + "unique_key": null + }, + "constraints": [], + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.2960231, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [], + "nodes": [ + "seed.jaffle_shop.raw_customers" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "fqn": [ + "jaffle_shop", + "staging", + "stg_customers" + ], + "group": null, + "language": "sql", + "latest_version": null, + "meta": {}, + "metrics": [], + "name": "stg_customers", + "original_file_path": "models/staging/stg_customers.sql", + "package_name": "jaffle_shop", + "patch_path": "jaffle_shop://models/staging/schema.yml", + "path": "staging/stg_customers.sql", + "raw_code": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_customers') }}\n\n),\n\nrenamed as (\n\n select\n id as customer_id,\n first_name,\n last_name\n\n from source\n\n)\n\nselect * from renamed", + "refs": [ + { + "name": "raw_customers", + "package": null, + "version": null + } + ], + "relation_name": "\"postgres\".\"public\".\"stg_customers\"", + "resource_type": "model", + "schema": "public", + "sources": [], + "tags": [], + "unique_id": "model.jaffle_shop.stg_customers", + "unrendered_config": { + "materialized": "view" + }, + "version": null + }, + "model.jaffle_shop.stg_orders": { + "access": "protected", + "alias": "stg_orders", + "build_path": null, + "checksum": { + "checksum": "f4f881cb09d2c4162200fc331d7401df6d1abd4fed492554a7db70dede347108", + "name": "sha256" + }, + "columns": { + "order_id": { + "constraints": [], + "data_type": null, + "description": "", + "meta": {}, + "name": "order_id", + "quote": null, + "tags": [] + }, + "status": { + "constraints": [], + "data_type": null, + "description": "", + "meta": {}, + "name": "status", + "quote": null, + "tags": [] + } + }, + "compiled_path": null, + "config": { + "alias": null, + "column_types": {}, + "contract": { + "enforced": false + }, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "view", + "meta": {}, + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "schema": null, + "tags": [], + "unique_key": null + }, + "constraints": [], + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.296747, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [], + "nodes": [ + "seed.jaffle_shop.raw_orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "fqn": [ + "jaffle_shop", + "staging", + "stg_orders" + ], + "group": null, + "language": "sql", + "latest_version": null, + "meta": {}, + "metrics": [], + "name": "stg_orders", + "original_file_path": "models/staging/stg_orders.sql", + "package_name": "jaffle_shop", + "patch_path": "jaffle_shop://models/staging/schema.yml", + "path": "staging/stg_orders.sql", + "raw_code": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_orders') }}\n\n),\n\nrenamed as (\n\n select\n id as order_id,\n user_id as customer_id,\n order_date,\n status\n\n from source\n\n)\n\nselect * from renamed", + "refs": [ + { + "name": "raw_orders", + "package": null, + "version": null + } + ], + "relation_name": "\"postgres\".\"public\".\"stg_orders\"", + "resource_type": "model", + "schema": "public", + "sources": [], + "tags": [], + "unique_id": "model.jaffle_shop.stg_orders", + "unrendered_config": { + "materialized": "view" + }, + "version": null + }, + "model.jaffle_shop.stg_payments": { + "access": "protected", + "alias": "stg_payments", + "build_path": null, + "checksum": { + "checksum": "30f346f66ef7bca4c8865a471086303720c3daab58870c805b6f45e92d19fd65", + "name": "sha256" + }, + "columns": { + "payment_id": { + "constraints": [], + "data_type": null, + "description": "", + "meta": {}, + "name": "payment_id", + "quote": null, + "tags": [] + }, + "payment_method": { + "constraints": [], + "data_type": null, + "description": "", + "meta": {}, + "name": "payment_method", + "quote": null, + "tags": [] + } + }, + "compiled_path": null, + "config": { + "alias": null, + "column_types": {}, + "contract": { + "enforced": false + }, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "view", + "meta": {}, + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "schema": null, + "tags": [], + "unique_key": null + }, + "constraints": [], + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.297438, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [], + "nodes": [ + "seed.jaffle_shop.raw_payments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "fqn": [ + "jaffle_shop", + "staging", + "stg_payments" + ], + "group": null, + "language": "sql", + "latest_version": null, + "meta": {}, + "metrics": [], + "name": "stg_payments", + "original_file_path": "models/staging/stg_payments.sql", + "package_name": "jaffle_shop", + "patch_path": "jaffle_shop://models/staging/schema.yml", + "path": "staging/stg_payments.sql", + "raw_code": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_payments') }}\n\n),\n\nrenamed as (\n\n select\n id as payment_id,\n order_id,\n payment_method,\n\n -- `amount` is currently stored in cents, so we convert it to dollars\n amount / 100 as amount\n\n from source\n\n)\n\nselect * from renamed", + "refs": [ + { + "name": "raw_payments", + "package": null, + "version": null + } + ], + "relation_name": "\"postgres\".\"public\".\"stg_payments\"", + "resource_type": "model", + "schema": "public", + "sources": [], + "tags": [], + "unique_id": "model.jaffle_shop.stg_payments", + "unrendered_config": { + "materialized": "view" + }, + "version": null + }, + "seed.jaffle_shop.raw_customers": { + "alias": "raw_customers", + "build_path": null, + "checksum": { + "checksum": "357d173dda65a741ad97d6683502286cc2655bb396ab5f4dfad12b8c39bd2a63", + "name": "sha256" + }, + "columns": {}, + "config": { + "alias": null, + "column_types": {}, + "contract": { + "enforced": false + }, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "seed", + "meta": {}, + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quote_columns": null, + "quoting": {}, + "schema": null, + "tags": [], + "unique_key": null + }, + "created_at": 1687942823.236107, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "fqn": [ + "jaffle_shop", + "raw_customers" + ], + "group": null, + "meta": {}, + "name": "raw_customers", + "original_file_path": "seeds/raw_customers.csv", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "raw_customers.csv", + "raw_code": "", + "relation_name": "\"postgres\".\"public\".\"raw_customers\"", + "resource_type": "seed", + "root_path": "/Users/tati/Code/astronomer-cosmos/dev/dags/dbt/jaffle_shop", + "schema": "public", + "tags": [], + "unique_id": "seed.jaffle_shop.raw_customers", + "unrendered_config": {} + }, + "seed.jaffle_shop.raw_orders": { + "alias": "raw_orders", + "build_path": null, + "checksum": { + "checksum": "6228dde8e17b9621f35c13e272ec67d3ff55b55499433f47d303adf2be72c17f", + "name": "sha256" + }, + "columns": {}, + "config": { + "alias": null, + "column_types": {}, + "contract": { + "enforced": false + }, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "seed", + "meta": {}, + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quote_columns": null, + "quoting": {}, + "schema": null, + "tags": [], + "unique_key": null + }, + "created_at": 1687942823.237559, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "fqn": [ + "jaffle_shop", + "raw_orders" + ], + "group": null, + "meta": {}, + "name": "raw_orders", + "original_file_path": "seeds/raw_orders.csv", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "raw_orders.csv", + "raw_code": "", + "relation_name": "\"postgres\".\"public\".\"raw_orders\"", + "resource_type": "seed", + "root_path": "/Users/tati/Code/astronomer-cosmos/dev/dags/dbt/jaffle_shop", + "schema": "public", + "tags": [], + "unique_id": "seed.jaffle_shop.raw_orders", + "unrendered_config": {} + }, + "seed.jaffle_shop.raw_payments": { + "alias": "raw_payments", + "build_path": null, + "checksum": { + "checksum": "6de0626a8db9c1750eefd1b2e17fac4c2a4b9f778eb50532d8b377b90de395e6", + "name": "sha256" + }, + "columns": {}, + "config": { + "alias": null, + "column_types": {}, + "contract": { + "enforced": false + }, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "seed", + "meta": {}, + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quote_columns": null, + "quoting": {}, + "schema": null, + "tags": [], + "unique_key": null + }, + "created_at": 1687942823.2389288, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "fqn": [ + "jaffle_shop", + "raw_payments" + ], + "group": null, + "meta": {}, + "name": "raw_payments", + "original_file_path": "seeds/raw_payments.csv", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "raw_payments.csv", + "raw_code": "", + "relation_name": "\"postgres\".\"public\".\"raw_payments\"", + "resource_type": "seed", + "root_path": "/Users/tati/Code/astronomer-cosmos/dev/dags/dbt/jaffle_shop", + "schema": "public", + "tags": [], + "unique_id": "seed.jaffle_shop.raw_payments", + "unrendered_config": {} + }, + "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3": { + "alias": "accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "status", + "columns": {}, + "compiled_path": null, + "config": { + "alias": "accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758", + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.283273, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_accepted_values", + "macro.dbt.get_where_subquery" + ], + "nodes": [ + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "accepted_values_orders_status__placed__shipped__completed__return_pending__returned" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "accepted_values_orders_status__placed__shipped__completed__return_pending__returned", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758.sql", + "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758\") }}", + "refs": [ + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "status", + "model": "{{ get_where_subquery(ref('orders')) }}", + "values": [ + "placed", + "shipped", + "completed", + "return_pending", + "returned" + ] + }, + "name": "accepted_values", + "namespace": null + }, + "unique_id": "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3", + "unrendered_config": { + "alias": "accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758" + } + }, + "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad": { + "alias": "accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58", + "attached_node": "model.jaffle_shop.stg_orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "status", + "columns": {}, + "compiled_path": null, + "config": { + "alias": "accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58", + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.3024309, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_accepted_values", + "macro.dbt.get_where_subquery" + ], + "nodes": [ + "model.jaffle_shop.stg_orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.stg_orders", + "fqn": [ + "jaffle_shop", + "staging", + "accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned", + "original_file_path": "models/staging/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58.sql", + "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58\") }}", + "refs": [ + { + "name": "stg_orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "status", + "model": "{{ get_where_subquery(ref('stg_orders')) }}", + "values": [ + "placed", + "shipped", + "completed", + "return_pending", + "returned" + ] + }, + "name": "accepted_values", + "namespace": null + }, + "unique_id": "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad", + "unrendered_config": { + "alias": "accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58" + } + }, + "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278": { + "alias": "accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef", + "attached_node": "model.jaffle_shop.stg_payments", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "payment_method", + "columns": {}, + "compiled_path": null, + "config": { + "alias": "accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef", + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.308291, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_accepted_values", + "macro.dbt.get_where_subquery" + ], + "nodes": [ + "model.jaffle_shop.stg_payments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.stg_payments", + "fqn": [ + "jaffle_shop", + "staging", + "accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card", + "original_file_path": "models/staging/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef.sql", + "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef\") }}", + "refs": [ + { + "name": "stg_payments", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "payment_method", + "model": "{{ get_where_subquery(ref('stg_payments')) }}", + "values": [ + "credit_card", + "coupon", + "bank_transfer", + "gift_card" + ] + }, + "name": "accepted_values", + "namespace": null + }, + "unique_id": "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278", + "unrendered_config": { + "alias": "accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef" + } + }, + "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d": { + "alias": "not_null_customers_customer_id", + "attached_node": "model.jaffle_shop.customers", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "customer_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.272043, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.customers" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.customers", + "fqn": [ + "jaffle_shop", + "not_null_customers_customer_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_customers_customer_id", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_customers_customer_id.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "customers", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "customer_id", + "model": "{{ get_where_subquery(ref('customers')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_orders_amount.106140f9fd": { + "alias": "not_null_orders_amount", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "amount", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.28982, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "not_null_orders_amount" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_orders_amount", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_orders_amount.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "amount", + "model": "{{ get_where_subquery(ref('orders')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_orders_amount.106140f9fd", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49": { + "alias": "not_null_orders_bank_transfer_amount", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "bank_transfer_amount", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.2937589, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "not_null_orders_bank_transfer_amount" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_orders_bank_transfer_amount", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_orders_bank_transfer_amount.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "bank_transfer_amount", + "model": "{{ get_where_subquery(ref('orders')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625": { + "alias": "not_null_orders_coupon_amount", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "coupon_amount", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.292507, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "not_null_orders_coupon_amount" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_orders_coupon_amount", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_orders_coupon_amount.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "coupon_amount", + "model": "{{ get_where_subquery(ref('orders')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59": { + "alias": "not_null_orders_credit_card_amount", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "credit_card_amount", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.291426, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "not_null_orders_credit_card_amount" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_orders_credit_card_amount", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_orders_credit_card_amount.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "credit_card_amount", + "model": "{{ get_where_subquery(ref('orders')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_orders_customer_id.c5f02694af": { + "alias": "not_null_orders_customer_id", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "customer_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.275403, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "not_null_orders_customer_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_orders_customer_id", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_orders_customer_id.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "customer_id", + "model": "{{ get_where_subquery(ref('orders')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_orders_customer_id.c5f02694af", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a": { + "alias": "not_null_orders_gift_card_amount", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "gift_card_amount", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.294739, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "not_null_orders_gift_card_amount" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_orders_gift_card_amount", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_orders_gift_card_amount.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "gift_card_amount", + "model": "{{ get_where_subquery(ref('orders')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_orders_order_id.cf6c17daed": { + "alias": "not_null_orders_order_id", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "order_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.274264, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "not_null_orders_order_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_orders_order_id", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_orders_order_id.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "order_id", + "model": "{{ get_where_subquery(ref('orders')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_orders_order_id.cf6c17daed", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa": { + "alias": "not_null_stg_customers_customer_id", + "attached_node": "model.jaffle_shop.stg_customers", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "customer_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.29916, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.stg_customers" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.stg_customers", + "fqn": [ + "jaffle_shop", + "staging", + "not_null_stg_customers_customer_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_stg_customers_customer_id", + "original_file_path": "models/staging/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_stg_customers_customer_id.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "stg_customers", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "customer_id", + "model": "{{ get_where_subquery(ref('stg_customers')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64": { + "alias": "not_null_stg_orders_order_id", + "attached_node": "model.jaffle_shop.stg_orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "order_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.301339, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.stg_orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.stg_orders", + "fqn": [ + "jaffle_shop", + "staging", + "not_null_stg_orders_order_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_stg_orders_order_id", + "original_file_path": "models/staging/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_stg_orders_order_id.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "stg_orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "order_id", + "model": "{{ get_where_subquery(ref('stg_orders')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075": { + "alias": "not_null_stg_payments_payment_id", + "attached_node": "model.jaffle_shop.stg_payments", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "payment_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.3071492, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.stg_payments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.stg_payments", + "fqn": [ + "jaffle_shop", + "staging", + "not_null_stg_payments_payment_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_stg_payments_payment_id", + "original_file_path": "models/staging/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_stg_payments_payment_id.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "stg_payments", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "payment_id", + "model": "{{ get_where_subquery(ref('stg_payments')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075", + "unrendered_config": {} + }, + "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2": { + "alias": "relationships_orders_customer_id__customer_id__ref_customers_", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "customer_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.276394, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_relationships", + "macro.dbt.get_where_subquery" + ], + "nodes": [ + "model.jaffle_shop.customers", + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "relationships_orders_customer_id__customer_id__ref_customers_" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "relationships_orders_customer_id__customer_id__ref_customers_", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "relationships_orders_customer_id__customer_id__ref_customers_.sql", + "raw_code": "{{ test_relationships(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "customers", + "package": null, + "version": null + }, + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "customer_id", + "field": "customer_id", + "model": "{{ get_where_subquery(ref('orders')) }}", + "to": "ref('customers')" + }, + "name": "relationships", + "namespace": null + }, + "unique_id": "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2", + "unrendered_config": {} + }, + "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1": { + "alias": "unique_customers_customer_id", + "attached_node": "model.jaffle_shop.customers", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "customer_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.270796, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_unique" + ], + "nodes": [ + "model.jaffle_shop.customers" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.customers", + "fqn": [ + "jaffle_shop", + "unique_customers_customer_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "unique_customers_customer_id", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "unique_customers_customer_id.sql", + "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "customers", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "customer_id", + "model": "{{ get_where_subquery(ref('customers')) }}" + }, + "name": "unique", + "namespace": null + }, + "unique_id": "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1", + "unrendered_config": {} + }, + "test.jaffle_shop.unique_orders_order_id.fed79b3a6e": { + "alias": "unique_orders_order_id", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "order_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.273182, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_unique" + ], + "nodes": [ + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "unique_orders_order_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "unique_orders_order_id", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "unique_orders_order_id.sql", + "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "order_id", + "model": "{{ get_where_subquery(ref('orders')) }}" + }, + "name": "unique", + "namespace": null + }, + "unique_id": "test.jaffle_shop.unique_orders_order_id.fed79b3a6e", + "unrendered_config": {} + }, + "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada": { + "alias": "unique_stg_customers_customer_id", + "attached_node": "model.jaffle_shop.stg_customers", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "customer_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.298065, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_unique" + ], + "nodes": [ + "model.jaffle_shop.stg_customers" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.stg_customers", + "fqn": [ + "jaffle_shop", + "staging", + "unique_stg_customers_customer_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "unique_stg_customers_customer_id", + "original_file_path": "models/staging/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "unique_stg_customers_customer_id.sql", + "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "stg_customers", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "customer_id", + "model": "{{ get_where_subquery(ref('stg_customers')) }}" + }, + "name": "unique", + "namespace": null + }, + "unique_id": "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada", + "unrendered_config": {} + }, + "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a": { + "alias": "unique_stg_orders_order_id", + "attached_node": "model.jaffle_shop.stg_orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "order_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.3003101, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_unique" + ], + "nodes": [ + "model.jaffle_shop.stg_orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.stg_orders", + "fqn": [ + "jaffle_shop", + "staging", + "unique_stg_orders_order_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "unique_stg_orders_order_id", + "original_file_path": "models/staging/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "unique_stg_orders_order_id.sql", + "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "stg_orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "order_id", + "model": "{{ get_where_subquery(ref('stg_orders')) }}" + }, + "name": "unique", + "namespace": null + }, + "unique_id": "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a", + "unrendered_config": {} + }, + "test.jaffle_shop.unique_stg_payments_payment_id.3744510712": { + "alias": "unique_stg_payments_payment_id", + "attached_node": "model.jaffle_shop.stg_payments", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "payment_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1687942823.3060858, + "database": "postgres", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_unique" + ], + "nodes": [ + "model.jaffle_shop.stg_payments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.stg_payments", + "fqn": [ + "jaffle_shop", + "staging", + "unique_stg_payments_payment_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "unique_stg_payments_payment_id", + "original_file_path": "models/staging/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "unique_stg_payments_payment_id.sql", + "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "stg_payments", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "payment_id", + "model": "{{ get_where_subquery(ref('stg_payments')) }}" + }, + "name": "unique", + "namespace": null + }, + "unique_id": "test.jaffle_shop.unique_stg_payments_payment_id.3744510712", + "unrendered_config": {} + } + }, + "parent_map": { + "model.jaffle_shop.customers": [ + "model.jaffle_shop.stg_customers", + "model.jaffle_shop.stg_orders", + "model.jaffle_shop.stg_payments" + ], + "model.jaffle_shop.orders": [ + "model.jaffle_shop.stg_orders", + "model.jaffle_shop.stg_payments" + ], + "model.jaffle_shop.stg_customers": [ + "seed.jaffle_shop.raw_customers" + ], + "model.jaffle_shop.stg_orders": [ + "seed.jaffle_shop.raw_orders" + ], + "model.jaffle_shop.stg_payments": [ + "seed.jaffle_shop.raw_payments" + ], + "seed.jaffle_shop.raw_customers": [], + "seed.jaffle_shop.raw_orders": [], + "seed.jaffle_shop.raw_payments": [], + "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3": [ + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad": [ + "model.jaffle_shop.stg_orders" + ], + "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278": [ + "model.jaffle_shop.stg_payments" + ], + "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d": [ + "model.jaffle_shop.customers" + ], + "test.jaffle_shop.not_null_orders_amount.106140f9fd": [ + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49": [ + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625": [ + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59": [ + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.not_null_orders_customer_id.c5f02694af": [ + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a": [ + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.not_null_orders_order_id.cf6c17daed": [ + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa": [ + "model.jaffle_shop.stg_customers" + ], + "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64": [ + "model.jaffle_shop.stg_orders" + ], + "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075": [ + "model.jaffle_shop.stg_payments" + ], + "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2": [ + "model.jaffle_shop.customers", + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1": [ + "model.jaffle_shop.customers" + ], + "test.jaffle_shop.unique_orders_order_id.fed79b3a6e": [ + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada": [ + "model.jaffle_shop.stg_customers" + ], + "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a": [ + "model.jaffle_shop.stg_orders" + ], + "test.jaffle_shop.unique_stg_payments_payment_id.3744510712": [ + "model.jaffle_shop.stg_payments" + ] + }, + "selectors": { + "complex_nested_union_intersection": { + "definition": { + "union": [ + { + "intersection": [ + { + "method": "tag", + "value": "nightly" + }, + { + "method": "path", + "value": "models/staging" + } + ] + }, + { + "method": "config.materialized", + "value": "incremental" + } + ] + }, + "description": "Select (staging AND nightly) OR incremental models", + "name": "complex_nested_union_intersection" + }, + "config_materialized_table": { + "definition": { + "method": "config.materialized", + "value": "table" + }, + "description": "Select all models materialized as tables", + "name": "config_materialized_table" + }, + "edge_case_empty_value": { + "definition": { + "method": "tag", + "value": "" + }, + "description": "Edge case: empty string value", + "name": "edge_case_empty_value" + }, + "edge_case_special_chars": { + "definition": { + "method": "tag", + "value": "tag-with.special_chars123" + }, + "description": "Edge case: tag with special characters", + "name": "edge_case_special_chars" + }, + "exclude_staging_except_customers": { + "definition": { + "exclude": [ + { + "method": "fqn", + "value": "stg_customers" + } + ], + "method": "path", + "value": "models/staging" + }, + "description": "Select all staging models except stg_customers", + "name": "exclude_staging_except_customers" + }, + "exposure_selector": { + "definition": { + "method": "exposure", + "value": "*" + }, + "description": "Select all exposures", + "name": "exposure_selector" + }, + "fqn_customers": { + "definition": { + "method": "fqn", + "value": "customers" + }, + "description": "Select models matching FQN 'customers'", + "name": "fqn_customers" + }, + "graph_2plus_customers": { + "definition": { + "children": true, + "children_depth": 2, + "method": "fqn", + "value": "customers" + }, + "description": "Select customers and descendants 2 levels deep (2+ operator)", + "name": "graph_2plus_customers" + }, + "graph_children_customers": { + "definition": { + "children": true, + "childrens_parents": true, + "method": "fqn", + "value": "customers" + }, + "description": "Select customers and all its children (1+ operator)", + "name": "graph_children_customers" + }, + "graph_parents_orders": { + "definition": { + "method": "fqn", + "parents": true, + "value": "orders" + }, + "description": "Select all parents of orders (+1 operator)", + "name": "graph_parents_orders" + }, + "graph_plus2_orders": { + "definition": { + "method": "fqn", + "parents": true, + "parents_depth": 2, + "value": "orders" + }, + "description": "Select orders and ancestors 2 levels deep (+2 operator)", + "name": "graph_plus2_orders" + }, + "graph_wildcard_children": { + "definition": { + "children": true, + "method": "fqn", + "value": "stg_*" + }, + "description": "Select all staging models and their children", + "name": "graph_wildcard_children" + }, + "greedy_graph_operator": { + "definition": { + "children": true, + "childrens_parents": true, + "method": "fqn", + "parents": true, + "value": "customers" + }, + "description": "Select customers, all parents, and all children (@ operator)", + "name": "greedy_graph_operator" + }, + "intersection_staging_and_tagged": { + "definition": { + "intersection": [ + { + "method": "path", + "value": "models/staging" + }, + { + "method": "tag", + "value": "daily" + } + ] + }, + "description": "Select staging models that are also tagged 'daily'", + "name": "intersection_staging_and_tagged" + }, + "resource_type_model": { + "definition": { + "method": "resource_type", + "value": "model" + }, + "description": "Select all models by resource type", + "name": "resource_type_model" + }, + "source_raw": { + "definition": { + "method": "source", + "value": "raw_*" + }, + "description": "Select all sources starting with 'raw_'", + "name": "source_raw" + }, + "staging_models": { + "definition": { + "method": "path", + "value": "models/staging" + }, + "description": "Select all staging models", + "name": "staging_models" + }, + "tagged_nightly": { + "definition": { + "method": "tag", + "value": "nightly" + }, + "description": "Select all models tagged with 'nightly'", + "name": "tagged_nightly" + }, + "union_staging_or_marts": { + "definition": { + "union": [ + { + "method": "path", + "value": "models/staging" + }, + { + "method": "path", + "value": "models/marts" + } + ] + }, + "description": "Select all models in staging OR marts directories", + "name": "union_staging_or_marts" + }, + "valid_indirect": { + "definition": { + "indirect_selection": "eager", + "method": "fqn", + "value": "customers" + }, + "description": "Valid: indirect_selection for tests", + "name": "valid_indirect" + }, + "wildcard_path": { + "definition": { + "method": "path", + "value": "models/*/intermediate" + }, + "description": "Select all intermediate models using wildcard", + "name": "wildcard_path" + } + }, + "sources": {} +} diff --git a/tests/test_cache.py b/tests/test_cache.py index 3cfbf2a470..5cb91cbeb4 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -25,6 +25,7 @@ from airflow.utils.task_group import TaskGroup from cosmos.cache import ( + _calculate_yaml_selectors_cache_current_version, _configure_remote_cache_dir, _copy_partial_parse_to_project, _create_cache_identifier, @@ -38,6 +39,7 @@ get_cached_profile, is_cache_package_lockfile_enabled, is_profile_cache_enabled, + were_yaml_selectors_modified, ) from cosmos.constants import ( DBT_PARTIAL_PARSE_FILE_NAME, @@ -448,3 +450,33 @@ def test_remote_cache_path_initialization_with_conn_id(mock_object_storage_path) configured_remote_cache_dir = _configure_remote_cache_dir() mock_object_storage_path.assert_called_with("s3://some-bucket/cache", conn_id="my_conn_id") assert configured_remote_cache_dir == mock_cache_path + + +def test_calculate_yaml_selectors_cache_current_version_equals(): + mock_create_dbt_folder_version_hash = MagicMock() + mock_create_dbt_folder_version_hash.return_value = "dbt_project_hash_v1" + selector_definitions = { + "core_models": { + "definition": {"method": "tag", "value": "core"}, + "description": "Select core business logic models (non-staging)", + "name": "core_models", + }, + } + + with patch("cosmos.cache._create_folder_version_hash", mock_create_dbt_folder_version_hash): + result = _calculate_yaml_selectors_cache_current_version( + "cosmos_cache__dag_a", Path("/path/to/project"), selector_definitions, "yamlselectors_v1" + ) + assert result == "dbt_project_hash_v1,dcbc007b6ea10b215d0024015f762d93,d450c7fd9f02df15cd10a22ebeeca661" + + +def test_were_yaml_selectors_modified_true(): + previous_version = "dbt_project_hash_v1, yamlselectors_v1, impl_hash_v1" + current_version = "dbt_project_hash_v1, yamlselectors_v2, impl_hash_v1" + assert were_yaml_selectors_modified(previous_version, current_version) is True + + +def test_were_yaml_selectors_modified_false(): + previous_version = "dbt_project_hash_v1, yamlselectors_v1, impl_hash_v1" + current_version = "dbt_project_hash_v1, yamlselectors_v1, impl_hash_v1" + assert were_yaml_selectors_modified(previous_version, current_version) is False From f2a398d1337d01bb24b8056985806722928a38e8 Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Fri, 23 Jan 2026 11:43:32 -0500 Subject: [PATCH 16/24] Return hash of source code for impl_version --- cosmos/cache.py | 3 +-- cosmos/dbt/selector.py | 14 ++++++++------ tests/dbt/test_graph.py | 2 +- tests/test_cache.py | 12 ++++++------ 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/cosmos/cache.py b/cosmos/cache.py index 01deb247b4..249f3b13a4 100644 --- a/cosmos/cache.py +++ b/cosmos/cache.py @@ -311,13 +311,12 @@ def _calculate_yaml_selectors_cache_current_version( dbt_project_hash = _create_folder_version_hash(project_dir) yaml_selector_hash = hashlib.md5(yaml.dump(selector_definitions).encode()).hexdigest() - implementation_hash = hashlib.md5(implementation_version.encode()).hexdigest() elapsed_time = time.perf_counter() - start_time logger.info( f"Cosmos performance: time to calculate cache identifier {cache_identifier} for current version: {elapsed_time}" ) - return f"{dbt_project_hash},{yaml_selector_hash},{implementation_hash}" + return f"{dbt_project_hash},{yaml_selector_hash},{implementation_version}" def _calculate_dbt_ls_cache_current_version(cache_identifier: str, project_dir: Path, cmd_args: list[str]) -> str: diff --git a/cosmos/dbt/selector.py b/cosmos/dbt/selector.py index 74f970bf40..9e0dcc39e6 100644 --- a/cosmos/dbt/selector.py +++ b/cosmos/dbt/selector.py @@ -2,6 +2,7 @@ import copy import functools +import hashlib import inspect import re from collections import defaultdict @@ -746,15 +747,16 @@ def parsed(self) -> dict[str, dict[str, Any]]: @functools.lru_cache(maxsize=1) def impl_version(self) -> str: """ - Get the source code of the YamlSelectors implementation. + Get a hash of the YamlSelectors implementation for version detection. - This property retrieves the complete source code of the YamlSelectors class, which can be - used to detect changes in the selector parsing logic (e.g., for cache invalidation). - The source code is retrieved once and cached for the lifetime of the instance. + This property retrieves the source code of the YamlSelectors class and returns its MD5 hash, + which can be used to detect changes in the selector parsing logic (e.g., for cache invalidation). + The hash is computed once and cached for the lifetime of the instance. - :return: str - Complete source code of the YamlSelectors class + :return: str - MD5 hash (32 hex characters) of the YamlSelectors class source code """ - return inspect.getsource(self.__class__) + source_code = inspect.getsource(self.__class__) + return hashlib.md5(source_code.encode()).hexdigest() def get_raw(self, selector_name: str, default: Any = None) -> dict[str, Any] | Any: """ diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index 8c3fd07321..325c00c185 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -2094,7 +2094,7 @@ def test_save_yamls_selector_cache(mock_variable_set, mock_datetime, tmp_dbt_pro hash_dir, hash_selectors, hash_impl = version.split(",") assert hash_selectors == "fbfa164dd765f83c2941eeb019a7f7b4" - assert hash_impl == "535bf463068da8658a5183094a72545c" + assert hash_impl == "9a2177baf050abaacec7ca1949ea7959" if sys.platform == "darwin": # We faced inconsistent hashing versions depending on the version of MacOS/Linux - the following line aims to address these. diff --git a/tests/test_cache.py b/tests/test_cache.py index 5cb91cbeb4..25e67529e7 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -465,18 +465,18 @@ def test_calculate_yaml_selectors_cache_current_version_equals(): with patch("cosmos.cache._create_folder_version_hash", mock_create_dbt_folder_version_hash): result = _calculate_yaml_selectors_cache_current_version( - "cosmos_cache__dag_a", Path("/path/to/project"), selector_definitions, "yamlselectors_v1" + "cosmos_cache__dag_a", Path("/path/to/project"), selector_definitions, "yamlselectors_hash_v1" ) - assert result == "dbt_project_hash_v1,dcbc007b6ea10b215d0024015f762d93,d450c7fd9f02df15cd10a22ebeeca661" + assert result == "dbt_project_hash_v1,dcbc007b6ea10b215d0024015f762d93,yamlselectors_hash_v1" def test_were_yaml_selectors_modified_true(): - previous_version = "dbt_project_hash_v1, yamlselectors_v1, impl_hash_v1" - current_version = "dbt_project_hash_v1, yamlselectors_v2, impl_hash_v1" + previous_version = "dbt_project_hash_v1, yamlselectors_hash_v1, impl_hash_v1" + current_version = "dbt_project_hash_v1, yamlselectors_hash_v2, impl_hash_v1" assert were_yaml_selectors_modified(previous_version, current_version) is True def test_were_yaml_selectors_modified_false(): - previous_version = "dbt_project_hash_v1, yamlselectors_v1, impl_hash_v1" - current_version = "dbt_project_hash_v1, yamlselectors_v1, impl_hash_v1" + previous_version = "dbt_project_hash_v1, yamlselectors_hash_v1, impl_hash_v1" + current_version = "dbt_project_hash_v1, yamlselectors_hash_v1, impl_hash_v1" assert were_yaml_selectors_modified(previous_version, current_version) is False From c87043bbfecbd00607733a54da63efcd6d3f5524 Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Fri, 23 Jan 2026 13:58:15 -0500 Subject: [PATCH 17/24] Unify dbt_cache implementation (preserving public API) --- cosmos/cache.py | 124 +++++++++++++++--- cosmos/config.py | 2 + cosmos/dbt/graph.py | 32 ++++- .../cosmos_manifest_yaml_selectors_example.py | 7 +- dev/dags/example_cosmos_cleanup_dag.py | 33 ++++- tests/dbt/test_graph.py | 16 ++- tests/test_cache.py | 12 +- 7 files changed, 191 insertions(+), 35 deletions(-) diff --git a/cosmos/cache.py b/cosmos/cache.py index 249f3b13a4..06223a027e 100644 --- a/cosmos/cache.py +++ b/cosmos/cache.py @@ -291,7 +291,7 @@ def _calculate_yaml_selectors_cache_current_version( cache_identifier: str, project_dir: Path, selector_definitions: dict[str, dict[str, Any]], - implementation_version: str, + cache_key: list[str], ) -> str: """ Taking into account the project directory contents and the selectors definitions, calculate the @@ -311,12 +311,13 @@ def _calculate_yaml_selectors_cache_current_version( dbt_project_hash = _create_folder_version_hash(project_dir) yaml_selector_hash = hashlib.md5(yaml.dump(selector_definitions).encode()).hexdigest() + cache_key_hash = hashlib.md5("".join(cache_key).encode()).hexdigest() elapsed_time = time.perf_counter() - start_time logger.info( f"Cosmos performance: time to calculate cache identifier {cache_identifier} for current version: {elapsed_time}" ) - return f"{dbt_project_hash},{yaml_selector_hash},{implementation_version}" + return f"{dbt_project_hash},{yaml_selector_hash},{cache_key_hash}" def _calculate_dbt_ls_cache_current_version(cache_identifier: str, project_dir: Path, cmd_args: list[str]) -> str: @@ -363,16 +364,24 @@ def were_yaml_selectors_modified(previous_version: str, current_version: str) -> @provide_session -def delete_unused_dbt_ls_cache( - max_age_last_usage: timedelta = timedelta(days=30), session: Session | None = None +def delete_unused_dbt_cache( + max_age_last_usage: timedelta = timedelta(days=30), session: Session | None = None, cache_type: str = "cosmos" ) -> int: """ Delete Cosmos cache stored in Airflow Variables based on the last execution of their associated DAGs. + This function handles all types of Cosmos cache (dbt ls, YAML selectors, etc.) and is used by + specific wrapper functions for backwards compatibility. + + :param max_age_last_usage: Delete cache for DAGs not run within this timeframe + :param session: SQLAlchemy session for Airflow metadata database (automatically provided by @provide_session) + :param cache_type: Type of cache being deleted, used in log messages (default: "cosmos") + :return: Number of Airflow Variables deleted + Example usage: There are three Cosmos cache Airflow Variables: - 1. ``cache cosmos_cache__basic_cosmos_dag`` + 1. ``cosmos_cache__basic_cosmos_dag`` 2. ``cosmos_cache__basic_cosmos_task_group__orders`` 3. ``cosmos_cache__basic_cosmos_task_group__customers`` @@ -385,21 +394,21 @@ def delete_unused_dbt_ls_cache( To delete the cache related to ``DbtDags`` and ``DbtTaskGroup`` that were run more than 5 days ago: ..code: python - >>> delete_unused_dbt_ls_cache(max_age_last_usage=timedelta(days=5)) - INFO - Removing the dbt ls cache cosmos_cache__basic_cosmos_dag + >>> delete_unused_dbt_cache(max_age_last_usage=timedelta(days=5)) + INFO - Removing the cosmos cache cosmos_cache__basic_cosmos_dag To delete the cache related to ``DbtDags`` and ``DbtTaskGroup`` that were run more than 10 minutes ago: ..code: python - >>> delete_unused_dbt_ls_cache(max_age_last_usage=timedelta(minutes=10)) - INFO - Removing the dbt ls cache cosmos_cache__basic_cosmos_dag - INFO - Removing the dbt ls cache cosmos_cache__basic_cosmos_task_group__orders - INFO - Removing the dbt ls cache cosmos_cache__basic_cosmos_task_group__orders + >>> delete_unused_dbt_cache(max_age_last_usage=timedelta(minutes=10)) + INFO - Removing the cosmos cache cosmos_cache__basic_cosmos_dag + INFO - Removing the cosmos cache cosmos_cache__basic_cosmos_task_group__orders + INFO - Removing the cosmos cache cosmos_cache__basic_cosmos_task_group__customers - To delete the cache related to ``DbtDags`` and ``DbtTaskGroup`` that were run more than 10 days ago + To delete the cache related to ``DbtDags`` and ``DbtTaskGroup`` that were run more than 10 days ago: ..code: python - >>> delete_unused_dbt_ls_cache(max_age_last_usage=timedelta(days=10)) + >>> delete_unused_dbt_cache(max_age_last_usage=timedelta(days=10)) In this last example, nothing is deleted. """ @@ -431,7 +440,7 @@ def delete_unused_dbt_ls_cache( ) if last_dag_run and last_dag_run.execution_date < (datetime.now(timezone.utc) - max_age_last_usage): for var_key in vars_keys: - logger.info(f"Removing the dbt ls cache {var_key}") + logger.info(f"Removing the {cache_type} cache {var_key}") Variable.delete(var_key) deleted_cosmos_variables += 1 @@ -443,11 +452,19 @@ def delete_unused_dbt_ls_cache( # TODO: Add integration tests once remote cache is supported in the CI pipeline @provide_session -def delete_unused_dbt_ls_remote_cache_files( # pragma: no cover - max_age_last_usage: timedelta = timedelta(days=30), session: Session | None = None +def delete_unused_dbt_remote_cache_files( # pragma: no cover + max_age_last_usage: timedelta = timedelta(days=30), session: Session | None = None, cache_type: str = "cosmos" ) -> int: """ Delete Cosmos cache stored in remote storage based on the last execution of their associated DAGs. + + This function handles all types of Cosmos cache (dbt ls, YAML selectors, etc.) stored in remote + storage and is used by specific wrapper functions for backwards compatibility. + + :param max_age_last_usage: Delete cache for DAGs not run within this timeframe + :param session: SQLAlchemy session for Airflow metadata database (automatically provided by @provide_session) + :param cache_type: Type of cache being deleted, used in log messages (default: "cosmos") + :return: Number of remote cache files deleted """ if session is None: return 0 @@ -458,7 +475,7 @@ def delete_unused_dbt_ls_remote_cache_files( # pragma: no cover configured_remote_cache_dir = _configure_remote_cache_dir() if not configured_remote_cache_dir: logger.info( - "No remote cache directory configured. Skipping the deletion of the dbt ls cache files in remote storage." + f"No remote cache directory configured. Skipping the deletion of the {cache_type} cache files in remote storage." ) return 0 @@ -487,18 +504,87 @@ def delete_unused_dbt_ls_remote_cache_files( # pragma: no cover ) if last_dag_run and last_dag_run.execution_date < (datetime.now(timezone.utc) - max_age_last_usage): for file in files: - logger.info(f"Removing the dbt ls cache remote file {file}") + logger.info(f"Removing the {cache_type} cache remote file {file}") file.unlink() deleted_cosmos_remote_cache_files += 1 logger.info( - "Deleted %s/%s dbt ls cache files in remote storage.", + "Deleted %s/%s %s cache files in remote storage.", deleted_cosmos_remote_cache_files, total_cosmos_remote_cache_files, + cache_type, ) return deleted_cosmos_remote_cache_files +@provide_session +def delete_unused_dbt_ls_cache( + max_age_last_usage: timedelta = timedelta(days=30), session: Session | None = None +) -> int: + """ + Delete all Cosmos cache stored in Airflow Variables (dbt ls, YAML selectors, etc.). + + This function is maintained for backwards compatibility and API stability. It has identical behavior + to delete_unused_dbt_cache() and deletes all types of Cosmos cache, not just dbt ls cache. + + :param max_age_last_usage: Delete cache for DAGs not run within this timeframe + :param session: SQLAlchemy session for Airflow metadata database (automatically provided by @provide_session) + :return: Number of Airflow Variables deleted + """ + return delete_unused_dbt_cache(max_age_last_usage, session, cache_type="dbt ls") + + +@provide_session +def delete_unused_dbt_ls_remote_cache_files( # pragma: no cover + max_age_last_usage: timedelta = timedelta(days=30), session: Session | None = None +) -> int: + """ + Delete all Cosmos cache stored in remote storage (dbt ls, YAML selectors, etc.). + + This function is maintained for backwards compatibility and API stability. It has identical behavior + to delete_unused_dbt_remote_cache_files() and deletes all types of Cosmos cache, not just dbt ls cache. + + :param max_age_last_usage: Delete cache for DAGs not run within this timeframe + :param session: SQLAlchemy session for Airflow metadata database (automatically provided by @provide_session) + :return: Number of remote cache files deleted + """ + return delete_unused_dbt_remote_cache_files(max_age_last_usage, session, cache_type="dbt ls") + + +@provide_session +def delete_unused_dbt_yaml_selectors_cache( + max_age_last_usage: timedelta = timedelta(days=30), session: Session | None = None +) -> int: + """ + Delete all Cosmos cache stored in Airflow Variables (dbt ls, YAML selectors, etc.). + + This function is maintained for backwards compatibility and API stability. It has identical behavior + to delete_unused_dbt_cache() and deletes all types of Cosmos cache, not just YAML selectors cache. + + :param max_age_last_usage: Delete cache for DAGs not run within this timeframe + :param session: SQLAlchemy session for Airflow metadata database (automatically provided by @provide_session) + :return: Number of Airflow Variables deleted + """ + return delete_unused_dbt_cache(max_age_last_usage, session, cache_type="dbt yaml selectors") + + +@provide_session +def delete_unused_dbt_yaml_selectors_remote_cache_files( # pragma: no cover + max_age_last_usage: timedelta = timedelta(days=30), session: Session | None = None +) -> int: + """ + Delete all Cosmos cache stored in remote storage (dbt ls, YAML selectors, etc.). + + This function is maintained for backwards compatibility and API stability. It has identical behavior + to delete_unused_dbt_remote_cache_files() and deletes all types of Cosmos cache, not just YAML selectors cache. + + :param max_age_last_usage: Delete cache for DAGs not run within this timeframe + :param session: SQLAlchemy session for Airflow metadata database (automatically provided by @provide_session) + :return: Number of remote cache files deleted + """ + return delete_unused_dbt_remote_cache_files(max_age_last_usage, session, cache_type="dbt yaml selectors") + + def is_profile_cache_enabled() -> bool: """Return True if global and profile cache is enable""" return enable_cache and enable_cache_profile diff --git a/cosmos/config.py b/cosmos/config.py index de07e59250..3c655ff4a9 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -74,6 +74,7 @@ class RenderConfig: :param source_rendering_behavior: Determines how source nodes are rendered when using cosmos default source node rendering (ALL, NONE, WITH_TESTS_OR_FRESHNESS). Defaults to "NONE" (since Cosmos 1.6). :param source_pruning: Determines if source nodes without a corresponding downstream task should be removed or not. Default is False :param airflow_vars_to_purge_dbt_ls_cache: Specify Airflow variables that will affect the LoadMode.DBT_LS cache. + :param airflow_vars_to_purge_yaml_selectors_cache: Specify Airflow variables that will affect the parsed manifest YamlSelectors cache. :param normalize_task_id: A callable that takes a dbt node as input and returns the task ID. This allows users to assign a custom node ID separate from the display name. :param normalize_task_display_name: A callable that takes a dbt node as input and returns the task display name. This allows users to assign a custom task display name separate from the node ID. :param should_detach_multiple_parents_tests: A boolean that allows users to decide whether to run tests with multiple parent dependencies in separate tasks. @@ -99,6 +100,7 @@ class RenderConfig: source_rendering_behavior: SourceRenderingBehavior = SourceRenderingBehavior.NONE source_pruning: bool = False airflow_vars_to_purge_dbt_ls_cache: list[str] = field(default_factory=list) + airflow_vars_to_purge_dbt_yaml_selectors_cache: list[str] = field(default_factory=list) normalize_task_id: Callable[..., Any] | None = None normalize_task_display_name: Callable[..., Any] | None = None should_detach_multiple_parents_tests: bool = False diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index d7c23879d8..a15314711c 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -460,6 +460,28 @@ def dbt_ls_cache_key_args(self) -> list[str]: logger.debug(f"Value of `dbt_ls_cache_key_args` for <{self.cache_key}>: {cache_args}") return cache_args + @cached_property + def _yaml_selectors_airflow_vars(self) -> list[str]: + """ + Cached Airflow variable values used for YAML selectors cache invalidation. + These are expensive to retrieve and don't change during a single DAG parse. + """ + cache_args = [] + if self.render_config.airflow_vars_to_purge_dbt_yaml_selectors_cache: + for var_name in self.render_config.airflow_vars_to_purge_dbt_yaml_selectors_cache: + airflow_vars = [var_name, Variable.get(var_name, "")] + cache_args.extend(airflow_vars) + return cache_args + + def get_dbt_yaml_selectors_cache_key_args(self, impl_version: str) -> list[str]: + """ + Values that are used to represent the dbt yaml selectors cache key. If any parts are changed, the manifest selectors + will be reparsed and the new value will be stored. + """ + cache_args = [impl_version] + self._yaml_selectors_airflow_vars + logger.debug(f"Value of `dbt_yaml_selectors_cache_key` for <{self.cache_key}>: {cache_args}") + return cache_args + def save_dbt_ls_cache(self, dbt_ls_output: str) -> None: """ Store compressed dbt ls output into an Airflow Variable. @@ -979,7 +1001,10 @@ def save_yaml_selectors_cache(self, yaml_selectors: YamlSelectors) -> None: cache_dict = { "version": cache._calculate_yaml_selectors_cache_current_version( - self.cache_key, self.project_path, yaml_selectors.raw, yaml_selectors.impl_version + self.cache_key, + self.project_path, + yaml_selectors.raw, + self.get_dbt_yaml_selectors_cache_key_args(yaml_selectors.impl_version), ), "yaml_selectors": selectors_encoded, "last_modified": datetime.datetime.now(datetime.timezone.utc).isoformat(), @@ -1035,7 +1060,10 @@ def load_parsed_selectors(self, selector_definitions: dict[str, Any]) -> YamlSel yaml_selectors: YamlSelectors = cache_dict["yaml_selectors"] current_version = cache._calculate_yaml_selectors_cache_current_version( - self.cache_key, self.project_path, selector_definitions, yaml_selectors.impl_version + self.cache_key, + self.project_path, + selector_definitions, + self.get_dbt_yaml_selectors_cache_key_args(yaml_selectors.impl_version), ) if cache_dict and not cache.were_yaml_selectors_modified(cache_version, current_version): diff --git a/dev/dags/cosmos_manifest_yaml_selectors_example.py b/dev/dags/cosmos_manifest_yaml_selectors_example.py index 490ca450ca..7801a617d5 100644 --- a/dev/dags/cosmos_manifest_yaml_selectors_example.py +++ b/dev/dags/cosmos_manifest_yaml_selectors_example.py @@ -19,7 +19,6 @@ DEFAULT_DBT_ROOT_PATH = Path(__file__).parent / "dbt" DBT_ROOT_PATH = Path(os.getenv("DBT_ROOT_PATH", DEFAULT_DBT_ROOT_PATH)) - execution_config = ExecutionConfig(dbt_project_path=DBT_ROOT_PATH / "jaffle_shop") profile_config = ProfileConfig( @@ -32,7 +31,11 @@ ), ) -render_config = RenderConfig(load_method=LoadMode.DBT_MANIFEST, selector="critical_path") +render_config = RenderConfig( + load_method=LoadMode.DBT_MANIFEST, + selector="critical_path", + airflow_vars_to_purge_dbt_yaml_selectors_cache=["purge"], +) with DAG( diff --git a/dev/dags/example_cosmos_cleanup_dag.py b/dev/dags/example_cosmos_cleanup_dag.py index ab11add14d..2e3d5bd055 100644 --- a/dev/dags/example_cosmos_cleanup_dag.py +++ b/dev/dags/example_cosmos_cleanup_dag.py @@ -9,7 +9,12 @@ from airflow import DAG from airflow.decorators import task -from cosmos.cache import delete_unused_dbt_ls_cache, delete_unused_dbt_ls_remote_cache_files +from cosmos.cache import ( + delete_unused_dbt_ls_cache, + delete_unused_dbt_ls_remote_cache_files, + delete_unused_dbt_yaml_selectors_cache, + delete_unused_dbt_yaml_selectors_remote_cache_files, +) with DAG( dag_id="example_cosmos_cleanup_dag", @@ -20,22 +25,40 @@ ): @task() - def clear_db_ls_cache(session=None): + def clear_dbt_ls_cache(session=None): """ Delete the dbt ls cache that has not been used for the last five days. """ delete_unused_dbt_ls_cache(max_age_last_usage=timedelta(days=5)) - clear_db_ls_cache() + clear_dbt_ls_cache() @task() - def clear_db_ls_remote_cache(session=None): + def clear_dbt_ls_remote_cache(session=None): """ Delete the dbt ls remote cache files that have not been used for the last five days. """ delete_unused_dbt_ls_remote_cache_files(max_age_last_usage=timedelta(days=5)) - clear_db_ls_remote_cache() + clear_dbt_ls_remote_cache() + + @task() + def clear_dbt_yaml_selectors_cache(session=None): + """ + Delete the dbt yaml selectors cache that has not been used for the last five days. + """ + delete_unused_dbt_yaml_selectors_cache(max_age_last_usage=timedelta(days=5)) + + clear_dbt_yaml_selectors_cache() + + @task() + def clear_dbt_yaml_selectors_remote_cache(session=None): + """ + Delete the dbt yaml selectors remote cache files that have not been used for the last five days. + """ + delete_unused_dbt_yaml_selectors_remote_cache_files(max_age_last_usage=timedelta(days=5)) + + clear_dbt_yaml_selectors_remote_cache() # [END cache_example] diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index 325c00c185..900b886c5d 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -2050,6 +2050,20 @@ def test_dbt_ls_cache_key_args_uses_airflow_vars_to_purge_dbt_ls_cache(airflow_v assert graph.dbt_ls_cache_key_args == [key, value] +@pytest.mark.integration +def test_get_dbt_yaml_selectors_cache_key_args_uses_airflow_vars_to_purge_dbt_cache(airflow_variable): + key, value = airflow_variable + graph = DbtGraph( + project=ProjectConfig(), + render_config=RenderConfig( + airflow_vars_to_purge_dbt_yaml_selectors_cache=[key], + source_rendering_behavior=SOURCE_RENDERING_BEHAVIOR, + ), + ) + impl_version = "yamlselectors_hash_v1" + assert graph.get_dbt_yaml_selectors_cache_key_args(impl_version) == [impl_version, key, value] + + @patch("cosmos.dbt.graph.datetime") @patch("cosmos.dbt.graph.Variable.set") def test_save_dbt_ls_cache(mock_variable_set, mock_datetime, tmp_dbt_project_dir): @@ -2094,7 +2108,7 @@ def test_save_yamls_selector_cache(mock_variable_set, mock_datetime, tmp_dbt_pro hash_dir, hash_selectors, hash_impl = version.split(",") assert hash_selectors == "fbfa164dd765f83c2941eeb019a7f7b4" - assert hash_impl == "9a2177baf050abaacec7ca1949ea7959" + assert hash_impl == "ec903f8592b06b28274b8847eb79bb1f" if sys.platform == "darwin": # We faced inconsistent hashing versions depending on the version of MacOS/Linux - the following line aims to address these. diff --git a/tests/test_cache.py b/tests/test_cache.py index 25e67529e7..cdd6735dca 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -35,7 +35,7 @@ _get_sha1_hash, _update_partial_parse_cache, create_cache_profile, - delete_unused_dbt_ls_cache, + delete_unused_dbt_cache, get_cached_profile, is_cache_package_lockfile_enabled, is_profile_cache_enabled, @@ -199,18 +199,18 @@ def vars_session(): @pytest.mark.integration -def test_delete_unused_dbt_ls_cache_deletes_a_week_ago_cache(vars_session): +def test_delete_unused_dbt_cache_deletes_a_week_ago_cache(vars_session): assert vars_session.query(Variable).filter_by(key="cosmos_cache__dag_a").first() - assert delete_unused_dbt_ls_cache(max_age_last_usage=timedelta(days=5), session=vars_session) == 1 + assert delete_unused_dbt_cache(max_age_last_usage=timedelta(days=5), session=vars_session) == 1 assert not vars_session.query(Variable).filter_by(key="cosmos_cache__dag_a").first() @pytest.mark.integration -def test_delete_unused_dbt_ls_cache_deletes_all_cache_five_minutes_ago(vars_session): +def test_delete_unused_dbt_cache_deletes_all_cache_five_minutes_ago(vars_session): assert vars_session.query(Variable).filter_by(key="cosmos_cache__dag_a").first() assert vars_session.query(Variable).filter_by(key="cosmos_cache__dag_b").first() assert vars_session.query(Variable).filter_by(key="cosmos_cache__dag_c__task_group_1").first() - assert delete_unused_dbt_ls_cache(max_age_last_usage=timedelta(minutes=5), session=vars_session) == 3 + assert delete_unused_dbt_cache(max_age_last_usage=timedelta(minutes=5), session=vars_session) == 3 assert not vars_session.query(Variable).filter_by(key="cosmos_cache__dag_a").first() assert not vars_session.query(Variable).filter_by(key="cosmos_cache__dag_b").first() assert not vars_session.query(Variable).filter_by(key="cosmos_cache__dag_c__task_group_1").first() @@ -467,7 +467,7 @@ def test_calculate_yaml_selectors_cache_current_version_equals(): result = _calculate_yaml_selectors_cache_current_version( "cosmos_cache__dag_a", Path("/path/to/project"), selector_definitions, "yamlselectors_hash_v1" ) - assert result == "dbt_project_hash_v1,dcbc007b6ea10b215d0024015f762d93,yamlselectors_hash_v1" + assert result == "dbt_project_hash_v1,dcbc007b6ea10b215d0024015f762d93,fbbaca69581c53891710fed3d53badcb" def test_were_yaml_selectors_modified_true(): From 9b16f0502982232ecbb9ec06fbdc890b47efcc27 Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Fri, 23 Jan 2026 17:12:56 -0500 Subject: [PATCH 18/24] Update docs --- docs/configuration/caching.rst | 92 +++++++++++++++++++++- docs/configuration/render-config.rst | 3 +- docs/configuration/selecting-excluding.rst | 48 ++++++++++- 3 files changed, 138 insertions(+), 5 deletions(-) diff --git a/docs/configuration/caching.rst b/docs/configuration/caching.rst index 879fefe9e8..829a020ac6 100644 --- a/docs/configuration/caching.rst +++ b/docs/configuration/caching.rst @@ -10,10 +10,11 @@ All Cosmos caching mechanisms can be enabled or turned off in the ``airflow.cfg` .. note:: For more information, see `configuring a Cosmos project <./project-config.html>`_. -Depending on the Cosmos version, it creates a cache for two types of data: +Depending on the Cosmos version, it creates a cache for three types of data: - The ``dbt ls`` output - The dbt ``partial_parse.msgpack`` file +- The parsed manifest selectors It is possible to turn off any cache in Cosmos by exporting the environment variable ``AIRFLOW__COSMOS__ENABLE_CACHE=0``. Disabling individual types of cache in Cosmos is also possible, as explained below. @@ -67,13 +68,13 @@ Additionally, if any of the following DAG configurations are changed, we'll auto Finally, if users would like to define specific Airflow variables that, if changed, will cause the recreation of the cache, they can specify those by using: -* ``RenderConfig.airflow_vars_to_purge_cache`` +* ``RenderConfig.airflow_vars_to_purge_dbt_ls_cache`` Example: .. code-block:: python - RenderConfig(airflow_vars_to_purge_cache == ["refresh_cache"]) + RenderConfig(airflow_vars_to_purge_dbt_ls_cache == ["refresh_cache"]) **Cleaning up stale cache** @@ -109,6 +110,91 @@ The cache values contain a few properties: * ``task_group_id`` is the TaskGroup associated to this cache * ``cosmos_type`` is either ``DbtDag`` or ``DbtTaskGroup`` +Caching the YAML selectors +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +(Introduced in Cosmos 1.13) + +While parsing a dbt project using `LoadMode.DBT_MANIFEST <./parsing-methods.html#dbt-manifest>`_, if a ``selector`` argument is provided to the `RenderConfig <./render-config.html>`_ instance passed to the ``DbtDag`` or ``DbtTaskGroup``, +Cosmos will parse the preprocessed YAML selectors found in the manifest. The YAML selectors will be parsed into selection criteria that Cosmos will use to filter the dbt nodes to include in the Airflow DAG. The parsed selectors will be cached to improve performance during DAG parsing. + +This feature is on by default. To turn it off, export the following environment variable: ``AIRFLOW__COSMOS__ENABLE_CACHE_DBT_YAML_SELECTORS=0``. + +Similar to the caching of ``dbt ls`` output, users can also set a remote directory path to store this cache instead of using Airflow +Variables. To do so, you need to configure a remote cache directory. See :ref:`remote_cache_dir` and +:ref:`remote_cache_dir_conn_id` for more information. This is an experimental feature introduced in 1.6.0 to gather +user feedback. The ``remote_cache_dir`` will eventually be merged into the :ref:`cache_dir` setting in upcoming +releases. + +**How the cache is refreshed** + +If using the default Variables cache approach, users can purge or delete the cache via Airflow UI by identifying and +deleting the cache key. In case you're using the alternative approach by setting the ``remote_cache_dir`` introduced +in Cosmos 1.6.0, you can delete the cache by removing the specific files by identifying them using your configured path +in the remote store. + +Cosmos will refresh the cache in a few circumstances: + +* if any files of the dbt project change +* if the YAML selectors in the manifest file change +* if the implementation of the YAML selector parsing logic changes + + * For new definitions of the dbt YAML selector specification. + +To evaluate if the dbt project changed, it calculates the changes using a few of the MD5 of all the files in the directory. + +Finally, if users would like to define specific Airflow variables that, if changed, will cause the recreation of the cache, they can specify those by using: + +* ``RenderConfig.airflow_vars_to_purge_dbt_yaml_selectors_cache`` + +Example: + +.. code-block:: python + + RenderConfig(airflow_vars_to_purge_dbt_yaml_selectors_cache == ["refresh_cache"]) + +**Cleaning up stale cache** + +Not rarely, Cosmos DbtDags and DbtTaskGroups may be renamed or deleted. In those cases, to clean up the Airflow metadata database, it is possible to use the method ``delete_unused_dbt_yaml_selectors_cache``. + +The method deletes the Cosmos cache stored in Airflow Variables based on the last execution of their associated DAGs. + +As an example, the following clean-up DAG will delete any cache associated with Cosmos that has not been used for the last five days: + +.. literalinclude:: ../../dev/dags/example_cosmos_cleanup_dag.py + :language: python + :start-after: [START cache_example] + :end-before: [END cache_example] + +.. note:: + Because the backing Airflow Variable is shared between the dbt ls cache and the YAML selectors cache, delete methods for the non-remote cache will delete the same Airflow variable. + In other words, if you call ``delete_unused_dbt_ls_cache``, it will also delete the YAML selectors cache for the same DAG or TaskGroup, and vice versa. + +**Cache key** + +The Airflow variables that represent the dbt ls cache are prefixed by ``cosmos_cache``. +When using ``DbtDag``, the keys use the DAG name. When using ``DbtTaskGroup``, they contain the ``TaskGroup`` and parent task groups and DAG. + +Examples: + +* The ``DbtDag`` "cosmos_dag" will have the cache represented by "cosmos_cache__basic_cosmos_dag". +* The ``DbtTaskGroup`` "customers" declared inside the DAG "basic_cosmos_task_group" will have the cache key "cosmos_cache__basic_cosmos_task_group__customers". + +**Cache value** + +The cache values contain a few properties: + +* ``last_modified`` timestamp, represented using the ISO 8601 format. +* ``version`` is a hash that represents the version of the dbt project, the raw YAML selectors, and a hash of the YAML selector parser implementation version combined with the keys specified by ``airflow_vars_to_purge_dbt_yaml_selectors_cache`` +* ``yaml_selectors`` is a serialized, compressed, encoded instance of the YamlSelectors class, containing the raw and parsed YAML selectors as well as an implementation version. +* ``dag_id`` is the DAG associated to this cache +* ``task_group_id`` is the TaskGroup associated to this cache +* ``cosmos_type`` is either ``DbtDag`` or ``DbtTaskGroup`` + +**Shared Cache Behavior** + +When using Airflow variables as the backend to store cached cosmos artifacts, both the dbt ls output and the YAML selectors cache will use the same variable. It should not be possible +to have both artifacts occupy the cache at the same time due to their distinct `RenderConfig.load_mode <./render-config.html>`_ and switching from using one cache to the other will invalidate the cache on the next version check. Caching the partial parse file ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/configuration/render-config.rst b/docs/configuration/render-config.rst index 5ac27e03cb..1083abc0db 100644 --- a/docs/configuration/render-config.rst +++ b/docs/configuration/render-config.rst @@ -21,7 +21,8 @@ The ``RenderConfig`` class takes the following arguments: - ``enable_mock_profile``: When using ``LoadMode.DBT_LS`` with a ``ProfileMapping`` class, by default, Cosmos mocks the values of the profile. Defaults to True. In order to leverage partial parsing, this argument should be set to ``False``. Read `Partial parsing <./partial-parsing.html#profile-configuration.html>`_ for more information. - ``env_vars``: (available in v1.2.5, use``ProjectConfig.env_vars`` for v1.3.0 onwards) A dictionary of environment variables for rendering. Only supported when using ``load_method=LoadMode.DBT_LS``. - ``dbt_project_path``: Configures the DBT project location accessible on their airflow controller for DAG rendering - Required when using ``load_method=LoadMode.DBT_LS`` or ``load_method=LoadMode.CUSTOM`` -- ``airflow_vars_to_purge_cache``: (new in v1.5) Specify Airflow variables that will affect the ``LoadMode.DBT_LS`` cache. See `Caching <./caching.html>`_ for more information. +- ``airflow_vars_to_purge_dbt_ls_cache``: (new in v1.5) Specify Airflow variables that will affect the ``LoadMode.DBT_LS`` cache. See `Caching <./caching.html>`_ for more information. +- ``airflow_vars_to_purge_yaml_selectors_cache``: (new in v1.13) Specify Airflow variables that will affect the YAML selectors cache when using selectors with ``LoadMode.DBT_MANIFEST``. See `Caching <./caching.html>`_ for more information. - ``source_rendering_behavior``: Determines how source nodes are rendered when using cosmos default source node rendering (ALL, NONE, WITH_TESTS_OR_FRESHNESS). Defaults to "NONE" (since Cosmos 1.6). See `Source Nodes Rendering <./source-nodes-rendering.html>`_ for more information. - ``source_pruning``: When set to ``True``, automatically removes (or "prunes") any dbt source nodes from your Airflow DAG that do not have any downstream dependencies within the selected portion of the dbt graph. Defaults to ``False``. See `Source Nodes Rendering <./source-nodes-rendering.html>`_ for more information. - ``normalize_task_id``: A callable that takes a dbt node as input and returns the task ID. This function allows users to set a custom task_id independently of the model name, which can be specified as the task's display_name. This way, task_id can be modified using a user-defined function, while the model name remains as the task's display name. The display_name parameter is available in Airflow 2.9 and above. See `Task display name <./task-display-name.html>`_ for more information. diff --git a/docs/configuration/selecting-excluding.rst b/docs/configuration/selecting-excluding.rst index 47fec5ea3d..1e202e6863 100644 --- a/docs/configuration/selecting-excluding.rst +++ b/docs/configuration/selecting-excluding.rst @@ -7,6 +7,8 @@ Cosmos allows you to filter to a subset of your dbt project in each ``DbtDag`` / Since Cosmos 1.3, the ``selector`` parameter is also available in ``RenderConfig`` when using the ``LoadMode.DBT_LS`` to parse the dbt project into Airflow. + Since Cosmos 1.13 , the ``selector`` parameter is also supported when using the ``LoadMode.DBT_MANIFEST`` to parse the dbt project into Airflow. + Using ``select`` and ``exclude`` -------------------------------- @@ -117,7 +119,7 @@ Examples: Using ``selector`` -------------------------------- .. note:: - Only currently supported using the ``dbt_ls`` parsing method since Cosmos 1.3 where the selector is passed directly to the dbt CLI command. \ + Only currently supported using the ``LoadMode.DBT_LS`` (since Cosmos 1.3) or ``LoadMode.DBT_MANIFEST`` (since Cosmos 1.13). If ``select`` and/or ``exclude`` are used with ``selector``, dbt will ignore the ``select`` and ``exclude`` parameters. The ``selector`` parameter is a string that references a `dbt YAML selector `_ already defined in a dbt project. @@ -134,3 +136,47 @@ Examples: load_method=LoadMode.DBT_LS, ) ) + +.. code-block:: python + + from cosmos import DbtDag, RenderConfig, LoadMode + + jaffle_shop = DbtDag( + project_config=ProjectConfig( + manifest_path=DBT_ROOT_PATH / "jaffle_shop" / "target" / "manifest.json", + project_name="jaffle_shop", + ), + render_config=RenderConfig( + selector="nightly_models", # this selector must be defined in your dbt project + load_method=LoadMode.DBT_MANIFEST, + ), + ) + jaffle_shop_remote = DbtDag( + project_config=ProjectConfig( + manifest_path="s3://cosmos-manifest-test/manifest.json", + manifest_conn_id="aws_s3_conn", + project_name="jaffle_shop", + ), + render_config=RenderConfig( + selector="nightly_models", # this selector must be defined in your dbt project + load_method=LoadMode.DBT_MANIFEST, + ), + ) + +Using ``selector`` with ``LoadMode.DBT_MANIFEST`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since Cosmos 1.13, the ``selector`` parameter is also supported when using the ``LoadMode.DBT_MANIFEST`` parsing method. + +When using this combination, Cosmos will read the preprocessed YAML selectors from the manifest file and use them to filter the dbt nodes to include in the Airflow DAG or Task Group. + +The YAML selection parser expects the selectors to be defined in the dbt project and will parse the preprocessed ``selectors`` found in the manifest file. Modifying the selector definitions in the manifest file in any way may lead to undefined behavior. +The parser may or may not catch invalid selector definitions if the selectors in the manifest are altered. + +The YAML selection parsing logic is based off the spec defined in the `dbt documentation `_. +All `graph operators `_ and `set operators `_ are supported. +Parsing of the ``default`` and ``indirect_selection`` keywords is not currently supported. + +In the event the dbt YAML selector specification changes, Cosmos will attempt to keep up to date with the changes, but there may be a lag between dbt releases and Cosmos releases. +Once a new Cosmos version is released with the updated selector parsing logic, users should update their Cosmos version to ensure compatibility with the latest dbt selector specification. +For any subsequent updates to the YAML selector parser, existing YAML selectors caches will be invalidated the next time the DAG is parsed. From 9c2c718780545b9e5bd1cfa3322499f5a8722f96 Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Mon, 26 Jan 2026 12:03:16 -0500 Subject: [PATCH 19/24] Use cached_property for impl_version --- cosmos/dbt/selector.py | 5 ++--- tests/dbt/test_graph.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cosmos/dbt/selector.py b/cosmos/dbt/selector.py index 9e0dcc39e6..616f9b0110 100644 --- a/cosmos/dbt/selector.py +++ b/cosmos/dbt/selector.py @@ -1,12 +1,12 @@ from __future__ import annotations import copy -import functools import hashlib import inspect import re from collections import defaultdict from dataclasses import dataclass +from functools import cached_property from pathlib import Path from typing import TYPE_CHECKING, Any @@ -743,8 +743,7 @@ def parsed(self) -> dict[str, dict[str, Any]]: """ return self._parsed - @property - @functools.lru_cache(maxsize=1) + @cached_property def impl_version(self) -> str: """ Get a hash of the YamlSelectors implementation for version detection. diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index 900b886c5d..56e92cf0df 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -2108,7 +2108,7 @@ def test_save_yamls_selector_cache(mock_variable_set, mock_datetime, tmp_dbt_pro hash_dir, hash_selectors, hash_impl = version.split(",") assert hash_selectors == "fbfa164dd765f83c2941eeb019a7f7b4" - assert hash_impl == "ec903f8592b06b28274b8847eb79bb1f" + assert hash_impl == "3bcd19c20c14f98095f50c11822b46e0" if sys.platform == "darwin": # We faced inconsistent hashing versions depending on the version of MacOS/Linux - the following line aims to address these. From 7de468f98c6e011bd07ba987403b9cbba2763724 Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Mon, 26 Jan 2026 12:07:53 -0500 Subject: [PATCH 20/24] Shorten manifest selectors example dag name --- ...electors_example.py => cosmos_manifest_selectors_example.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename dev/dags/{cosmos_manifest_yaml_selectors_example.py => cosmos_manifest_selectors_example.py} (98%) diff --git a/dev/dags/cosmos_manifest_yaml_selectors_example.py b/dev/dags/cosmos_manifest_selectors_example.py similarity index 98% rename from dev/dags/cosmos_manifest_yaml_selectors_example.py rename to dev/dags/cosmos_manifest_selectors_example.py index 7801a617d5..5bf5bc8c20 100644 --- a/dev/dags/cosmos_manifest_yaml_selectors_example.py +++ b/dev/dags/cosmos_manifest_selectors_example.py @@ -39,7 +39,7 @@ with DAG( - dag_id="cosmos_manifest_yaml_selectors_example", + dag_id="cosmos_manifest_selectors_example", schedule="@daily", start_date=datetime(2023, 1, 1), catchup=False, From 63f55b00002c5b74413d708affb6474ac7f5e6ef Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Mon, 26 Jan 2026 15:32:37 -0500 Subject: [PATCH 21/24] Fix failing unit tests on build agent --- tests/dbt/test_graph.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index 56e92cf0df..9ea0d4c840 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -2079,13 +2079,9 @@ def test_save_dbt_ls_cache(mock_variable_set, mock_datetime, tmp_dbt_project_dir assert hash_args == "d41d8cd98f00b204e9800998ecf8427e" if sys.platform == "darwin": # We faced inconsistent hashing versions depending on the version of MacOS/Linux - the following line aims to address these. - assert hash_dir in ( - "7abb868ed1c22e78de1c00429d950a77", - "85cba4ef17dd7c161938da6980a6ff85", - "7e273d9b7569e959af96f4368b9a036e", - ) + assert hash_dir in ("7e273d9b7569e959af96f4368b9a036e",) else: - assert hash_dir == "85cba4ef17dd7c161938da6980a6ff85" + assert hash_dir == "fbe70f1477c038da4607f9efb7a8a4d8" @patch("cosmos.dbt.graph.datetime") @@ -2112,13 +2108,9 @@ def test_save_yamls_selector_cache(mock_variable_set, mock_datetime, tmp_dbt_pro if sys.platform == "darwin": # We faced inconsistent hashing versions depending on the version of MacOS/Linux - the following line aims to address these. - assert hash_dir in ( - "7abb868ed1c22e78de1c00429d950a77", - "85cba4ef17dd7c161938da6980a6ff85", - "7e273d9b7569e959af96f4368b9a036e", - ) + assert hash_dir in ("7e273d9b7569e959af96f4368b9a036e",) else: - assert hash_dir == "85cba4ef17dd7c161938da6980a6ff85" + assert hash_dir == "fbe70f1477c038da4607f9efb7a8a4d8" @pytest.mark.integration From f5d6b8e7ce56768a4e64ea722add32d331262ff1 Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Mon, 26 Jan 2026 17:00:27 -0500 Subject: [PATCH 22/24] Fix typo in caching docs + Add entry for selector caching to cosmos-conf docs --- docs/configuration/caching.rst | 2 +- docs/configuration/cosmos-conf.rst | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/configuration/caching.rst b/docs/configuration/caching.rst index 829a020ac6..2b08bf1cce 100644 --- a/docs/configuration/caching.rst +++ b/docs/configuration/caching.rst @@ -118,7 +118,7 @@ Caching the YAML selectors While parsing a dbt project using `LoadMode.DBT_MANIFEST <./parsing-methods.html#dbt-manifest>`_, if a ``selector`` argument is provided to the `RenderConfig <./render-config.html>`_ instance passed to the ``DbtDag`` or ``DbtTaskGroup``, Cosmos will parse the preprocessed YAML selectors found in the manifest. The YAML selectors will be parsed into selection criteria that Cosmos will use to filter the dbt nodes to include in the Airflow DAG. The parsed selectors will be cached to improve performance during DAG parsing. -This feature is on by default. To turn it off, export the following environment variable: ``AIRFLOW__COSMOS__ENABLE_CACHE_DBT_YAML_SELECTORS=0``. +This feature is on by default. To turn it off, export the following environment variable: ``AIRFLOW__COSMOS__ENABLE_CACHE_YAML_SELECTORS=0``. Similar to the caching of ``dbt ls`` output, users can also set a remote directory path to store this cache instead of using Airflow Variables. To do so, you need to configure a remote cache directory. See :ref:`remote_cache_dir` and diff --git a/docs/configuration/cosmos-conf.rst b/docs/configuration/cosmos-conf.rst index 8ebdaa8b82..2b6e5a8eec 100644 --- a/docs/configuration/cosmos-conf.rst +++ b/docs/configuration/cosmos-conf.rst @@ -38,6 +38,12 @@ This page lists all available Airflow configurations that affect ``astronomer-co - Default: ``True`` - Environment Variable: ``AIRFLOW__COSMOS__ENABLE_CACHE_DBT_LS`` +`enable_cache_yaml_selectors`_: + Enable or disable caching of the YAML selectors in case using ``LoadMode.DBT_MANIFEST`` with ``RenderConfig.selector`` in an Airflow Variable. + + - Default: ``True`` + - Environment Variable: ``AIRFLOW__COSMOS__ENABLE_CACHE_YAML_SELECTORS`` + .. _enable_cache_partial_parse: `enable_cache_partial_parse`_: From cad91ac0a9e2e3d2a70c6b16f542fd84e4fbc0ba Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Tue, 27 Jan 2026 14:15:57 -0500 Subject: [PATCH 23/24] Apply copilot PR suggestions --- cosmos/cache.py | 11 ++-- cosmos/config.py | 2 +- cosmos/dbt/graph.py | 41 ++++++++------ cosmos/dbt/selector.py | 53 ++++++++++-------- docs/configuration/caching.rst | 4 +- docs/configuration/render-config.rst | 2 +- docs/configuration/selecting-excluding.rst | 4 +- tests/dbt/test_graph.py | 63 +++++++++++++++------- tests/dbt/test_selector.py | 17 +++++- tests/test_cache.py | 11 +++- 10 files changed, 139 insertions(+), 69 deletions(-) diff --git a/cosmos/cache.py b/cosmos/cache.py index 06223a027e..0d08a589a4 100644 --- a/cosmos/cache.py +++ b/cosmos/cache.py @@ -300,8 +300,8 @@ def _calculate_yaml_selectors_cache_current_version( :param cache_identifier: str - Unique identifier of the cache (may include DbtDag or DbtTaskGroup information) :param project_dir: Path - Path to the target dbt project directory :param selector_definitions: dict[str, dict[str, Any]] - Dictionary containing the selectors definitions from the manifest - :param implementation_version: str - The implementation version of the YamlSelectors class - :return: str - Combined hash string of project and selectors + :param cache_key: list[str] - List of strings used as part of the cache key hash calculation + :return: str - Combined hash string of project, selectors, and cache_key (comma-separated) """ start_time = time.perf_counter() @@ -310,8 +310,11 @@ def _calculate_yaml_selectors_cache_current_version( # This is fast (e.g. 0.01s for jaffle shop, 0.135s for a 5k models dbt folder) dbt_project_hash = _create_folder_version_hash(project_dir) - yaml_selector_hash = hashlib.md5(yaml.dump(selector_definitions).encode()).hexdigest() - cache_key_hash = hashlib.md5("".join(cache_key).encode()).hexdigest() + # Use JSON with sorted keys for deterministic hashing, resilient to dict ordering changes + yaml_selector_hash = hashlib.md5( + json.dumps(selector_definitions, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + cache_key_hash = hashlib.md5("".join(sorted(cache_key)).encode()).hexdigest() elapsed_time = time.perf_counter() - start_time logger.info( diff --git a/cosmos/config.py b/cosmos/config.py index 3c655ff4a9..edb300e697 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -74,7 +74,7 @@ class RenderConfig: :param source_rendering_behavior: Determines how source nodes are rendered when using cosmos default source node rendering (ALL, NONE, WITH_TESTS_OR_FRESHNESS). Defaults to "NONE" (since Cosmos 1.6). :param source_pruning: Determines if source nodes without a corresponding downstream task should be removed or not. Default is False :param airflow_vars_to_purge_dbt_ls_cache: Specify Airflow variables that will affect the LoadMode.DBT_LS cache. - :param airflow_vars_to_purge_yaml_selectors_cache: Specify Airflow variables that will affect the parsed manifest YamlSelectors cache. + :param airflow_vars_to_purge_dbt_yaml_selectors_cache: Specify Airflow variables that will affect the parsed manifest YamlSelectors cache. :param normalize_task_id: A callable that takes a dbt node as input and returns the task ID. This allows users to assign a custom node ID separate from the display name. :param normalize_task_display_name: A callable that takes a dbt node as input and returns the task display name. This allows users to assign a custom task display name separate from the node ID. :param should_detach_multiple_parents_tests: A boolean that allows users to decide whether to run tests with multiple parent dependencies in separate tasks. diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index 34dfec5aac..7e0620d65b 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -6,7 +6,6 @@ import itertools import json import os -import pickle import platform import tempfile import warnings @@ -945,12 +944,12 @@ def _get_yaml_selectors_remote_cache(self, remote_cache_dir: Path | ObjectStorag def get_yaml_selectors_cache(self) -> dict[str, Any]: """ - Retrieve previously saved YAML selectors from an Airflow Variable. + Retrieve previously saved YAML selectors from an Airflow Variable, decompressing the selector data. Outputs: { "version": "cache-version", - "yaml_selectors": uncompressed YamlSelectors instance, + "yaml_selectors": YamlSelectors instance reconstructed from cached dictionaries, "last_modified": "Isoformat timestamp" } """ @@ -974,30 +973,41 @@ def get_yaml_selectors_cache(self) -> dict[str, Any]: except tuple(airflow_variable_exceptions): return cache_dict else: - selectors_compressed = cache_dict.pop("yaml_selectors", None) - if selectors_compressed: - encoded_data = base64.b64decode(selectors_compressed.encode("utf-8")) - decompressed_data = zlib.decompress(encoded_data) - cache_dict["yaml_selectors"] = pickle.loads(decompressed_data) + raw_selectors_compressed = cache_dict.pop("raw_selectors_compressed", None) + parsed_selectors_compressed = cache_dict.pop("parsed_selectors_compressed", None) + + if raw_selectors_compressed and parsed_selectors_compressed: + encoded_raw = base64.b64decode(raw_selectors_compressed.encode()) + raw_selectors = json.loads(zlib.decompress(encoded_raw).decode()) + + encoded_parsed = base64.b64decode(parsed_selectors_compressed.encode()) + parsed_selectors = json.loads(zlib.decompress(encoded_parsed).decode()) + + cache_dict["yaml_selectors"] = YamlSelectors(raw_selectors, parsed_selectors) return cache_dict def save_yaml_selectors_cache(self, yaml_selectors: YamlSelectors) -> None: """ - Store parsed YAML selectors into an Airflow Variable. + Store compressed and encoded YAML selectors into an Airflow Variable. Stores: { "version": "cache-version", - "yaml_selectors": compressed YamlSelectors instance, + "raw_selectors_compressed": "compressed raw selector definitions", + "parsed_selectors_compressed": "compressed parsed selector definitions", "last_modified": "Isoformat timestamp" } """ + raw_selectors_json = json.dumps(yaml_selectors.raw, sort_keys=True) + compressed_raw = zlib.compress(raw_selectors_json.encode("utf-8")) + encoded_raw = base64.b64encode(compressed_raw) + raw_selectors_compressed = encoded_raw.decode("utf-8") - serialized_data = pickle.dumps(yaml_selectors) - compressed_data = zlib.compress(serialized_data) - encoded_data = base64.b64encode(compressed_data) - selectors_encoded = encoded_data.decode("utf-8") + parsed_selectors_json = json.dumps(yaml_selectors.parsed, sort_keys=True) + compressed_parsed = zlib.compress(parsed_selectors_json.encode("utf-8")) + encoded_parsed = base64.b64encode(compressed_parsed) + parsed_selectors_compressed = encoded_parsed.decode("utf-8") cache_dict = { "version": cache._calculate_yaml_selectors_cache_current_version( @@ -1006,7 +1016,8 @@ def save_yaml_selectors_cache(self, yaml_selectors: YamlSelectors) -> None: yaml_selectors.raw, self.get_dbt_yaml_selectors_cache_key_args(yaml_selectors.impl_version), ), - "yaml_selectors": selectors_encoded, + "raw_selectors_compressed": raw_selectors_compressed, + "parsed_selectors_compressed": parsed_selectors_compressed, "last_modified": datetime.datetime.now(datetime.timezone.utc).isoformat(), **self.airflow_metadata, } diff --git a/cosmos/dbt/selector.py b/cosmos/dbt/selector.py index 616f9b0110..2f01ee4e78 100644 --- a/cosmos/dbt/selector.py +++ b/cosmos/dbt/selector.py @@ -783,7 +783,7 @@ def _get_list_dicts(dct: dict[str, Any], key: str) -> list[dict[str, Any]]: Extract and validate a list of dictionaries from a given key in a dictionary. This helper method retrieves a value from a dictionary and ensures it's a list containing - either dictionaries with string keys or string values. + only dictionaries with string keys. :param dct: dict[str, Any] - The dictionary to extract from :param key: str - The key to look up in the dictionary @@ -807,7 +807,7 @@ def _get_list_dicts(dct: dict[str, Any], key: str) -> list[dict[str, Any]]: result.append(value) else: raise CosmosValueError( - f'Invalid value type {type(value)} in key "{key}", expected ' f"dict or str (value: {value})." + f'Invalid value type {type(value)} in key "{key}", expected dict (value: {value}).' ) return result @@ -942,7 +942,7 @@ def _parse_selector(cls, dct: dict[str, Any]) -> str: @classmethod def _parse_exclusions( - cls, definition: dict[str, Any], cache: dict[str, tuple[list[str], list[str]]] = {} + cls, definition: dict[str, Any], cache: dict[str, tuple[list[str], list[str]]] | None = None ) -> list[str]: """ Parse exclusion definitions from a selector definition. @@ -954,6 +954,8 @@ def _parse_exclusions( :param cache: dict[str, tuple[list[str], list[str]]] - Cache of previously parsed selectors for reference resolution :return: list[str] - List of exclusion selector strings """ + cache = cache or {} + exclusions = cls._get_list_dicts(definition, "exclude") exclude: list[str] = [] @@ -1004,7 +1006,7 @@ def _parse_include_exclude_subdefs( def _parse_union_definition( cls, definition: dict[str, Any], - cache: dict[str, tuple[list[str], list[str]]] = {}, + cache: dict[str, tuple[list[str], list[str]]] | None = None, ) -> tuple[list[str], list[str]]: """ Parse a union selector definition. @@ -1027,6 +1029,8 @@ def _parse_union_definition( Example Output: (["tag:nightly", "path:models/staging"], []) """ + cache = cache or {} + union_def_parts = cls._get_list_dicts(definition, "union") return cls._parse_include_exclude_subdefs(union_def_parts, cache=cache) @@ -1182,25 +1186,23 @@ def _parse_from_definition( @classmethod def parse(cls, selectors: dict[str, dict[str, Any]]) -> YamlSelectors: """ - Parse a complete dbt selectors YAML file into a YamlSelectors instance. + Parse selector definitions from a dbt manifest into a YamlSelectors instance. - This is the main entry point for parsing YAML selectors. It processes all selector - definitions in the YAML file and converts them to Cosmos format. + This is the main entry point for parsing manifest selector definitions. It processes all + selector definitions from the manifest and converts them to Cosmos format. - :param selectors: dict[str, dict[str, Any]] - The complete selectors YAML content as a dictionary with a 'selectors' key + :param selectors: dict[str, dict[str, Any]] - Dictionary of selector definitions from the manifest, keyed by selector name :return: YamlSelectors - A YamlSelectors instance containing both raw and parsed selector definitions Example Input: { - "selectors": [ - { - "name": "nightly_models", - "definition": { - "method": "tag", - "value": "nightly" - } + "nightly_models": { + "name": "nightly_models", + "definition": { + "method": "tag", + "value": "nightly" } - ] + } } Example Output: @@ -1208,24 +1210,33 @@ def parse(cls, selectors: dict[str, dict[str, Any]]) -> YamlSelectors: { "nightly_models": { "select": ["tag:nightly"], - "exclude": None } + "exclude": None + } } """ result: dict[str, dict[str, Any]] = {} cache: dict[str, tuple[list[str], list[str]]] = {} for name, definition in selectors.items(): - name = definition.get("name", "") - selector_definition = definition.get("definition", {}) + if not isinstance(definition, dict): + raise CosmosValueError( + f"Invalid selector definition for '{name}'. Expected a dict, got {type(definition)}: {definition}" + ) + + if not definition.get("name") or not definition.get("definition"): + raise CosmosValueError(f"Selector definition for '{name}' must contain 'name' and 'definition' keys.") + + selector_name = definition["name"] + selector_definition = definition["definition"] select, exclude = cls._parse_from_definition(selector_definition, cache=cache, rootlevel=True) - result[name] = { + result[selector_name] = { "select": select if select else None, "exclude": exclude if exclude else None, } - cache[name] = (select, exclude) + cache[selector_name] = (select, exclude) return cls(selectors, result) diff --git a/docs/configuration/caching.rst b/docs/configuration/caching.rst index 2b08bf1cce..f708c993d8 100644 --- a/docs/configuration/caching.rst +++ b/docs/configuration/caching.rst @@ -74,7 +74,7 @@ Example: .. code-block:: python - RenderConfig(airflow_vars_to_purge_dbt_ls_cache == ["refresh_cache"]) + RenderConfig(airflow_vars_to_purge_dbt_ls_cache=["refresh_cache"]) **Cleaning up stale cache** @@ -151,7 +151,7 @@ Example: .. code-block:: python - RenderConfig(airflow_vars_to_purge_dbt_yaml_selectors_cache == ["refresh_cache"]) + RenderConfig(airflow_vars_to_purge_dbt_yaml_selectors_cache=["refresh_cache"]) **Cleaning up stale cache** diff --git a/docs/configuration/render-config.rst b/docs/configuration/render-config.rst index 1083abc0db..f153d3c3d1 100644 --- a/docs/configuration/render-config.rst +++ b/docs/configuration/render-config.rst @@ -22,7 +22,7 @@ The ``RenderConfig`` class takes the following arguments: - ``env_vars``: (available in v1.2.5, use``ProjectConfig.env_vars`` for v1.3.0 onwards) A dictionary of environment variables for rendering. Only supported when using ``load_method=LoadMode.DBT_LS``. - ``dbt_project_path``: Configures the DBT project location accessible on their airflow controller for DAG rendering - Required when using ``load_method=LoadMode.DBT_LS`` or ``load_method=LoadMode.CUSTOM`` - ``airflow_vars_to_purge_dbt_ls_cache``: (new in v1.5) Specify Airflow variables that will affect the ``LoadMode.DBT_LS`` cache. See `Caching <./caching.html>`_ for more information. -- ``airflow_vars_to_purge_yaml_selectors_cache``: (new in v1.13) Specify Airflow variables that will affect the YAML selectors cache when using selectors with ``LoadMode.DBT_MANIFEST``. See `Caching <./caching.html>`_ for more information. +- ``airflow_vars_to_purge_dbt_yaml_selectors_cache``: (new in v1.13) Specify Airflow variables that will affect the YAML selectors cache when using selectors with ``LoadMode.DBT_MANIFEST``. See `Caching <./caching.html>`_ for more information. - ``source_rendering_behavior``: Determines how source nodes are rendered when using cosmos default source node rendering (ALL, NONE, WITH_TESTS_OR_FRESHNESS). Defaults to "NONE" (since Cosmos 1.6). See `Source Nodes Rendering <./source-nodes-rendering.html>`_ for more information. - ``source_pruning``: When set to ``True``, automatically removes (or "prunes") any dbt source nodes from your Airflow DAG that do not have any downstream dependencies within the selected portion of the dbt graph. Defaults to ``False``. See `Source Nodes Rendering <./source-nodes-rendering.html>`_ for more information. - ``normalize_task_id``: A callable that takes a dbt node as input and returns the task ID. This function allows users to set a custom task_id independently of the model name, which can be specified as the task's display_name. This way, task_id can be modified using a user-defined function, while the model name remains as the task's display name. The display_name parameter is available in Airflow 2.9 and above. See `Task display name <./task-display-name.html>`_ for more information. diff --git a/docs/configuration/selecting-excluding.rst b/docs/configuration/selecting-excluding.rst index 1e202e6863..8a051f6ad8 100644 --- a/docs/configuration/selecting-excluding.rst +++ b/docs/configuration/selecting-excluding.rst @@ -5,9 +5,9 @@ Selecting & Excluding Cosmos allows you to filter to a subset of your dbt project in each ``DbtDag`` / ``DbtTaskGroup`` using the ``select`` and ``exclude`` parameters in the ``RenderConfig`` class. - Since Cosmos 1.3, the ``selector`` parameter is also available in ``RenderConfig`` when using the ``LoadMode.DBT_LS`` to parse the dbt project into Airflow. +Since Cosmos 1.3, the ``selector`` parameter is also available in ``RenderConfig`` when using the ``LoadMode.DBT_LS`` to parse the dbt project into Airflow. - Since Cosmos 1.13 , the ``selector`` parameter is also supported when using the ``LoadMode.DBT_MANIFEST`` to parse the dbt project into Airflow. +Since Cosmos 1.13, the ``selector`` parameter is also supported when using the ``LoadMode.DBT_MANIFEST`` to parse the dbt project into Airflow. Using ``select`` and ``exclude`` diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index 9ea0d4c840..1ff9149377 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -3,7 +3,6 @@ import json import logging import os -import pickle import shutil import sys import tempfile @@ -395,7 +394,8 @@ def test_load_via_manifest_with_selectors_and_missing_definitions(): ) with pytest.raises(CosmosLoadDbtException) as err_info: dbt_graph.load_from_dbt_manifest() - assert err_info.value.args[0] == f"Selectors not found in the manifest file {project_config.manifest_path}" + + assert err_info.value.args[0] == f"Selectors not found in manifest file `{project_config.manifest_path}`" def test_load_via_manifest_with_selectors_and_missing_selector(): @@ -421,7 +421,8 @@ def test_load_via_manifest_with_selectors_and_missing_selector(): ) with pytest.raises(CosmosLoadDbtException) as err_info: dbt_graph.load_from_dbt_manifest() - assert "Selector 'my_selector' not found in the manifest file" in err_info.value.args[0] + + assert "Selector `my_selector` not found in parsed YAML selectors" in err_info.value.args[0] def test_load_via_manifest_with_selectors(): @@ -1851,7 +1852,7 @@ def test_load_via_dbt_ls_render_config_no_partial_parse( @pytest.mark.parametrize("load_method", [LoadMode.CUSTOM]) def test_load_method_with_unsupported_render_config_selector_arg(load_method): - """Tests that error is raised when RenderConfig.selector is used with LoadMode.DBT_MANIFEST or LoadMode.CUSTOM.""" + """Tests that error is raised when RenderConfig.selector is used with LoadMode.CUSTOM.""" expected_error_msg = ( f"RenderConfig.selector is not yet supported when loading dbt projects using the {load_method} parser." @@ -2079,14 +2080,14 @@ def test_save_dbt_ls_cache(mock_variable_set, mock_datetime, tmp_dbt_project_dir assert hash_args == "d41d8cd98f00b204e9800998ecf8427e" if sys.platform == "darwin": # We faced inconsistent hashing versions depending on the version of MacOS/Linux - the following line aims to address these. - assert hash_dir in ("7e273d9b7569e959af96f4368b9a036e",) + assert hash_dir in ("67c608d42ca8bdeef6c3eb8fa888471b",) else: assert hash_dir == "fbe70f1477c038da4607f9efb7a8a4d8" @patch("cosmos.dbt.graph.datetime") @patch("cosmos.dbt.graph.Variable.set") -def test_save_yamls_selector_cache(mock_variable_set, mock_datetime, tmp_dbt_project_dir): +def test_save_yaml_selectors_cache(mock_variable_set, mock_datetime, tmp_dbt_project_dir): mock_datetime.datetime.now.return_value = datetime(2022, 1, 1, 12, 0, 0) graph = DbtGraph(cache_identifier="something", project=ProjectConfig(dbt_project_path=tmp_dbt_project_dir)) selectors = YamlSelectors( @@ -2095,20 +2096,31 @@ def test_save_yamls_selector_cache(mock_variable_set, mock_datetime, tmp_dbt_pro ) graph.save_yaml_selectors_cache(selectors) assert mock_variable_set.call_args[0][0] == "cosmos_cache__something" - assert ( - mock_variable_set.call_args[0][1]["yaml_selectors"] - == "eJwtjjsKAkEQRAU/i4ImXkKTPYCHMDEykKF3p90dmI/MR00EzTuzvYync9ixkq6iXkM9J5/vqIjWrQvGhVo2sQ6osY3OMy2PYPThHwO/efviB29oIjzcsqNViNAp2wnnJWZiKC0Y5L6ihcSzsioqZ4diZjD2TjKN8xPT9Ao6Yb45CeCUAlXiAj6gLHjZwSemeUZ2BQOq8N7qJJH3KTX1D+xVTPQ=" - ) - assert mock_variable_set.call_args[0][1]["last_modified"] == "2022-01-01T12:00:00" - version = mock_variable_set.call_args[0][1].get("version") + + cache_dict = mock_variable_set.call_args[0][1] + + assert "raw_selectors_compressed" in cache_dict + assert "parsed_selectors_compressed" in cache_dict + + raw_decoded = base64.b64decode(cache_dict["raw_selectors_compressed"].encode()) + raw_decompressed = json.loads(zlib.decompress(raw_decoded).decode()) + assert raw_decompressed == selectors.raw + + parsed_decoded = base64.b64decode(cache_dict["parsed_selectors_compressed"].encode()) + parsed_decompressed = json.loads(zlib.decompress(parsed_decoded).decode()) + assert parsed_decompressed == selectors.parsed + + assert cache_dict["last_modified"] == "2022-01-01T12:00:00" + + version = cache_dict.get("version") hash_dir, hash_selectors, hash_impl = version.split(",") - assert hash_selectors == "fbfa164dd765f83c2941eeb019a7f7b4" - assert hash_impl == "3bcd19c20c14f98095f50c11822b46e0" + assert hash_selectors == "43303af03e84e3b51fbfcf598261fae4" + assert hash_impl == "3ae7ccd90b387308920fa408907de75d" if sys.platform == "darwin": # We faced inconsistent hashing versions depending on the version of MacOS/Linux - the following line aims to address these. - assert hash_dir in ("7e273d9b7569e959af96f4368b9a036e",) + assert hash_dir in ("67c608d42ca8bdeef6c3eb8fa888471b",) else: assert hash_dir == "fbe70f1477c038da4607f9efb7a8a4d8" @@ -2134,7 +2146,11 @@ def test_get_dbt_ls_cache_returns_decoded_and_decompressed_value(mock_variable_g @patch( "cosmos.dbt.graph.Variable.get", return_value={ - "yaml_selectors": "eJwtjjsKAkEQRAU/i4ImXkKTPYCHMDEykKF3p90dmI/MR00EzTuzvYync9ixkq6iXkM9J5/vqIjWrQvGhVo2sQ6osY3OMy2PYPThHwO/efviB29oIjzcsqNViNAp2wnnJWZiKC0Y5L6ihcSzsioqZ4diZjD2TjKN8xPT9Ao6Yb45CeCUAlXiAj6gLHjZwSemeUZ2BQOq8N7qJJH3KTX1D+xVTPQ=" + # Compressed and encoded versions of the selectors + # raw_selectors: {"staging_orders": {"name": "staging_orders", "definition": {"method": "tag", "value": "tag_a"}}} + # parsed_selectors: {"select": ["tag:tag_a"], "exclude": None} + "raw_selectors_compressed": "eJyrViouSUzPzEuPzy9KSS0qVrJSqFZKSU3LzMssyczPA3NzU0sy8lOATCWgUiUdBaWyxJzSVCg/PlGpFiiUl5gLFkEzrbYWAFRnILk=", + "parsed_selectors_compressed": "eJyrVkqtSM4pTUlVslLIK83J0VFQKk7NSU0uAfKjlUoS062AOD5RKbYWADB2DhQ=", }, ) def test_get_yaml_selectors_cache_returns_decoded_and_decompressed_value(mock_variable_get): @@ -2374,13 +2390,20 @@ def test_get_yaml_selectors_remote_cache_dir( {"select": ["tag:tag_a"], "exclude": None}, ) - serialized_data = pickle.dumps(yaml_selectors) - compressed_data = zlib.compress(serialized_data) - encoded_data = base64.b64encode(compressed_data).decode("utf-8") + raw_selectors_json = json.dumps(yaml_selectors.raw, sort_keys=True) + compressed_raw = zlib.compress(raw_selectors_json.encode("utf-8")) + encoded_raw = base64.b64encode(compressed_raw) + raw_selectors_compressed = encoded_raw.decode("utf-8") + + parsed_selectors_json = json.dumps(yaml_selectors.parsed, sort_keys=True) + compressed_parsed = zlib.compress(parsed_selectors_json.encode("utf-8")) + encoded_parsed = base64.b64encode(compressed_parsed) + parsed_selectors_compressed = encoded_parsed.decode("utf-8") cache_dict = { "version": "cache-version", - "yaml_selectors": encoded_data, + "raw_selectors_compressed": raw_selectors_compressed, + "parsed_selectors_compressed": parsed_selectors_compressed, "last_modified": "2024-08-13T12:34:56Z", } diff --git a/tests/dbt/test_selector.py b/tests/dbt/test_selector.py index 94a1a8d403..1dc951f396 100644 --- a/tests/dbt/test_selector.py +++ b/tests/dbt/test_selector.py @@ -1541,6 +1541,21 @@ def test_invalid_cosmos_method_yaml_selectors(selector_name, selector_definition {"name": "unknown_random_method", "definition": {"method": "unknown_random", "value": "test"}}, "Unsupported selector method: 'unknown_random'", ), + ( + "non_dict_definition", + "not_a_dict", + "Invalid selector definition for 'non_dict_definition'. Expected a dict, got : not_a_dict", + ), + ( + "missing_name_key", + {"definition": {"method": "tag", "value": "nightly"}}, + "Selector definition for 'missing_name_key' must contain 'name' and 'definition' keys.", + ), + ( + "missing_definition_key", + {"name": "missing_definition_key"}, + "Selector definition for 'missing_definition_key' must contain 'name' and 'definition' keys.", + ), ], ) def test_invalid_dbt_method_yaml_selectors(selector_name, selector_definition, exception_msg): @@ -1592,7 +1607,7 @@ def test_invalid_items_in_selector_lists(selector_name, definition_key, invalid_ _ = YamlSelectors.parse(selectors) assert ( - f'Invalid value type {item_type_str} in key "{definition_key}", expected dict or str (value: {invalid_list_item}).' + f'Invalid value type {item_type_str} in key "{definition_key}", expected dict (value: {invalid_list_item}).' in err_info.value.args[0] ) diff --git a/tests/test_cache.py b/tests/test_cache.py index cdd6735dca..a725dd9b7d 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -465,9 +465,16 @@ def test_calculate_yaml_selectors_cache_current_version_equals(): with patch("cosmos.cache._create_folder_version_hash", mock_create_dbt_folder_version_hash): result = _calculate_yaml_selectors_cache_current_version( - "cosmos_cache__dag_a", Path("/path/to/project"), selector_definitions, "yamlselectors_hash_v1" + "cosmos_cache__dag_a", Path("/path/to/project"), selector_definitions, ["yamlselectors_hash_v1"] ) - assert result == "dbt_project_hash_v1,dcbc007b6ea10b215d0024015f762d93,fbbaca69581c53891710fed3d53badcb" + + parts = result.split(",") + + assert len(parts) == 3 + + assert parts[0] == "dbt_project_hash_v1" + assert parts[1] == "d424de5f9a889bd3afee43a5012f1c65" + assert parts[2] == "fbbaca69581c53891710fed3d53badcb" def test_were_yaml_selectors_modified_true(): From 2403b42e1e0a68f3080aab6e65272a1acfe0895d Mon Sep 17 00:00:00 2001 From: Jon Billings Date: Tue, 27 Jan 2026 15:29:10 -0500 Subject: [PATCH 24/24] Ensure consistent config naming + Update docs --- .github/workflows/test.yml | 4 ++-- cosmos/dbt/graph.py | 2 +- cosmos/settings.py | 2 +- docs/configuration/caching.rst | 5 +++-- docs/configuration/cosmos-conf.rst | 4 ++-- docs/configuration/selecting-excluding.rst | 6 +++--- tests/dbt/test_graph.py | 4 ++-- 7 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0d90089a5d..082e2a9d71 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -197,7 +197,7 @@ jobs: hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}:test-integration env: AIRFLOW__COSMOS__ENABLE_CACHE_DBT_LS: 0 - AIRFLOW__COSMOS__ENABLE_CACHE_YAML_SELECTORS: 0 + AIRFLOW__COSMOS__ENABLE_CACHE_DBT_YAML_SELECTORS: 0 AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres AIRFLOW_CONN_AWS_S3_CONN: ${{ secrets.AIRFLOW_CONN_AWS_S3_CONN }} @@ -522,7 +522,7 @@ jobs: hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}:test-integration-dbtf env: AIRFLOW__COSMOS__ENABLE_CACHE_DBT_LS: 0 - AIRFLOW__COSMOS__ENABLE_CACHE_YAML_SELECTORS: 0 + AIRFLOW__COSMOS__ENABLE_CACHE_DBT_YAML_SELECTORS: 0 AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index 7e0620d65b..2f76509d11 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -663,7 +663,7 @@ def should_use_dbt_ls_cache(self) -> bool: @functools.lru_cache def should_use_yaml_selectors_cache(self) -> bool: """Identify if Cosmos should use/store YAML selectors cache or not.""" - return settings.enable_cache and settings.enable_cache_yaml_selectors and bool(self.cache_key) + return settings.enable_cache and settings.enable_cache_dbt_yaml_selectors and bool(self.cache_key) def load_via_dbt_ls_cache(self) -> bool: """(Try to) load dbt ls cache from an Airflow Variable""" diff --git a/cosmos/settings.py b/cosmos/settings.py index 243667751b..b8e5ac25bc 100644 --- a/cosmos/settings.py +++ b/cosmos/settings.py @@ -28,7 +28,7 @@ enable_cache_partial_parse = conf.getboolean("cosmos", "enable_cache_partial_parse", fallback=True) enable_cache_package_lockfile = conf.getboolean("cosmos", "enable_cache_package_lockfile", fallback=True) enable_cache_dbt_ls = conf.getboolean("cosmos", "enable_cache_dbt_ls", fallback=True) -enable_cache_yaml_selectors = conf.getboolean("cosmos", "enable_cache_yaml_selectors", fallback=True) +enable_cache_dbt_yaml_selectors = conf.getboolean("cosmos", "enable_cache_dbt_yaml_selectors", fallback=True) rich_logging = conf.getboolean("cosmos", "rich_logging", fallback=False) dbt_docs_dir = conf.get("cosmos", "dbt_docs_dir", fallback=None) dbt_docs_conn_id = conf.get("cosmos", "dbt_docs_conn_id", fallback=None) diff --git a/docs/configuration/caching.rst b/docs/configuration/caching.rst index f708c993d8..58114c46e4 100644 --- a/docs/configuration/caching.rst +++ b/docs/configuration/caching.rst @@ -118,7 +118,7 @@ Caching the YAML selectors While parsing a dbt project using `LoadMode.DBT_MANIFEST <./parsing-methods.html#dbt-manifest>`_, if a ``selector`` argument is provided to the `RenderConfig <./render-config.html>`_ instance passed to the ``DbtDag`` or ``DbtTaskGroup``, Cosmos will parse the preprocessed YAML selectors found in the manifest. The YAML selectors will be parsed into selection criteria that Cosmos will use to filter the dbt nodes to include in the Airflow DAG. The parsed selectors will be cached to improve performance during DAG parsing. -This feature is on by default. To turn it off, export the following environment variable: ``AIRFLOW__COSMOS__ENABLE_CACHE_YAML_SELECTORS=0``. +This feature is on by default. To turn it off, export the following environment variable: ``AIRFLOW__COSMOS__ENABLE_CACHE_DBT_YAML_SELECTORS=0``. Similar to the caching of ``dbt ls`` output, users can also set a remote directory path to store this cache instead of using Airflow Variables. To do so, you need to configure a remote cache directory. See :ref:`remote_cache_dir` and @@ -186,7 +186,8 @@ The cache values contain a few properties: * ``last_modified`` timestamp, represented using the ISO 8601 format. * ``version`` is a hash that represents the version of the dbt project, the raw YAML selectors, and a hash of the YAML selector parser implementation version combined with the keys specified by ``airflow_vars_to_purge_dbt_yaml_selectors_cache`` -* ``yaml_selectors`` is a serialized, compressed, encoded instance of the YamlSelectors class, containing the raw and parsed YAML selectors as well as an implementation version. +* ``raw_selectors_compressed`` represents the raw YAML selector definitions compressed using zlib and encoded to base64 +* ``parsed_selectors_compressed`` represents the parsed YAML selector definitions compressed using zlib and encoded to base64 * ``dag_id`` is the DAG associated to this cache * ``task_group_id`` is the TaskGroup associated to this cache * ``cosmos_type`` is either ``DbtDag`` or ``DbtTaskGroup`` diff --git a/docs/configuration/cosmos-conf.rst b/docs/configuration/cosmos-conf.rst index 2b6e5a8eec..e490e9d6ca 100644 --- a/docs/configuration/cosmos-conf.rst +++ b/docs/configuration/cosmos-conf.rst @@ -38,11 +38,11 @@ This page lists all available Airflow configurations that affect ``astronomer-co - Default: ``True`` - Environment Variable: ``AIRFLOW__COSMOS__ENABLE_CACHE_DBT_LS`` -`enable_cache_yaml_selectors`_: +`enable_cache_dbt_yaml_selectors`_: Enable or disable caching of the YAML selectors in case using ``LoadMode.DBT_MANIFEST`` with ``RenderConfig.selector`` in an Airflow Variable. - Default: ``True`` - - Environment Variable: ``AIRFLOW__COSMOS__ENABLE_CACHE_YAML_SELECTORS`` + - Environment Variable: ``AIRFLOW__COSMOS__ENABLE_CACHE_DBT_YAML_SELECTORS`` .. _enable_cache_partial_parse: diff --git a/docs/configuration/selecting-excluding.rst b/docs/configuration/selecting-excluding.rst index 8a051f6ad8..658f59c48f 100644 --- a/docs/configuration/selecting-excluding.rst +++ b/docs/configuration/selecting-excluding.rst @@ -5,9 +5,9 @@ Selecting & Excluding Cosmos allows you to filter to a subset of your dbt project in each ``DbtDag`` / ``DbtTaskGroup`` using the ``select`` and ``exclude`` parameters in the ``RenderConfig`` class. -Since Cosmos 1.3, the ``selector`` parameter is also available in ``RenderConfig`` when using the ``LoadMode.DBT_LS`` to parse the dbt project into Airflow. + Since Cosmos 1.3, the ``selector`` parameter is available in ``RenderConfig`` when using the ``LoadMode.DBT_LS`` to parse the dbt project into Airflow. -Since Cosmos 1.13, the ``selector`` parameter is also supported when using the ``LoadMode.DBT_MANIFEST`` to parse the dbt project into Airflow. + Since Cosmos 1.13, the ``selector`` parameter is available in ``RenderConfig`` when using the ``LoadMode.DBT_MANIFEST`` to parse the dbt project into Airflow. Using ``select`` and ``exclude`` @@ -179,4 +179,4 @@ Parsing of the ``default`` and ``indirect_selection`` keywords is not currently In the event the dbt YAML selector specification changes, Cosmos will attempt to keep up to date with the changes, but there may be a lag between dbt releases and Cosmos releases. Once a new Cosmos version is released with the updated selector parsing logic, users should update their Cosmos version to ensure compatibility with the latest dbt selector specification. -For any subsequent updates to the YAML selector parser, existing YAML selectors caches will be invalidated the next time the DAG is parsed. +For subsequent updates to the YAML selector parser, existing YAML selector caches will be invalidated the next time the DAG is parsed. diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index 1ff9149377..cd6aa7faad 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -2276,7 +2276,7 @@ def test_should_use_yaml_selectors_cache(enable_cache, enable_cache_yaml_selecto os.environ, { "AIRFLOW__COSMOS__ENABLE_CACHE": str(enable_cache), - "AIRFLOW__COSMOS__ENABLE_CACHE_YAML_SELECTORS": str(enable_cache_yaml_selectors), + "AIRFLOW__COSMOS__ENABLE_CACHE_DBT_YAML_SELECTORS": str(enable_cache_yaml_selectors), }, ): importlib.reload(settings) @@ -2442,7 +2442,7 @@ def test_parse_yaml_selectors_saves_cache(enable_cache, enable_cache_yaml_select os.environ, { "AIRFLOW__COSMOS__ENABLE_CACHE": str(enable_cache), - "AIRFLOW__COSMOS__ENABLE_CACHE_YAML_SELECTORS": str(enable_cache_yaml_selectors), + "AIRFLOW__COSMOS__ENABLE_CACHE_DBT_YAML_SELECTORS": str(enable_cache_yaml_selectors), }, ): importlib.reload(settings)