diff --git a/.gitignore b/.gitignore index 3d11b2aa0d..fd8b3c8199 100644 --- a/.gitignore +++ b/.gitignore @@ -157,6 +157,8 @@ dev/dags/.airflowignore # Local Airflow standalone airflow.cfg airflow.db +airflow.db-shm +airflow.db-wal standalone_admin_password.txt simple_auth_manager_passwords.json.generated webserver_config.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 528cb19255..c4acf2bcc9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -57,7 +57,7 @@ repos: - --py37-plus - --keep-runtime-typing - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.7 + rev: v0.11.10 hooks: - id: ruff args: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bbf48a8162..d4c919fda9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,35 @@ Changelog ========= +1.10.1 (2025-05-21) +--------------------- + +Bug Fixes + +* Fix ``full_refresh`` parameter in ``AIRFLOW_ASYNC`` ``ExecutionConfig`` mode by @tuantran0910 in #1738 +* Fix dbt ls invocation method log message by @tatiana and @dstandish in #1749 +* Ensure remote target directory is created when copying files when using local directory by @tuantran0910 and @corsettigyg in #1740 +* Support custom ``packages-install-path`` by @tatiana in #1768 +* Disable dbt static parser during Airflow task execution using dbt runner by @pankajkoti and @tatiana in #1760 +* Fix ``ExecutionMode.LOCAL`` to leverage ``ProjectConfig.manifest_path`` by @tatiana in #1772 +* Refactor ``AIRFLOW_ASYNC`` so that the path in the remote object store is specific per DAG run by @tuantran0910 in #1741 +* Optimise memory usage with optional explicit imports by @pankajkoti and @tatiana in #1769 + +Documentation + +* Fix documentation rendering for ``use_dataset_airflow3_uri_standard`` by @pankajastro in #1742 +* Correct custom callback example by @walter9388 in #1747 + +Others + +* Re-enable integration tests durations to troubleshoot performance degradation by @tatiana in #1735 +* Run listener tests for Airflow 3 by @pankajastro in #1743 +* Add Airflow 3 db files to ignore from git tracking by @pankajkoti in #1755 +* Log contents of ``packages.yml`` when ``AIRFLOW__LOGGING__LOGGING_LEVEL=DEBUG`` by @tatiana in #1764 +* Fix Airflow dependencies in the CI by @tatiana in #1773 +* Pre-commit updates: #1744, #1765, #1770 + + 1.10.0 (2025-05-01) --------------------- diff --git a/cosmos/__init__.py b/cosmos/__init__.py index 9228a58934..c269dd545f 100644 --- a/cosmos/__init__.py +++ b/cosmos/__init__.py @@ -5,353 +5,320 @@ Contains dags, task groups, and operators. """ +from __future__ import annotations -__version__ = "1.10.0" +from cosmos import settings +__version__ = "1.10.1" -from cosmos.airflow.dag import DbtDag -from cosmos.airflow.task_group import DbtTaskGroup -from cosmos.config import ( - ExecutionConfig, - ProfileConfig, - ProjectConfig, - RenderConfig, -) -from cosmos.constants import ( - DbtResourceType, - ExecutionMode, - InvocationMode, - LoadMode, - SourceRenderingBehavior, - TestBehavior, - TestIndirectSelection, -) -from cosmos.log import get_logger -from cosmos.operators.lazy_load import MissingPackage -from cosmos.operators.local import ( - DbtBuildLocalOperator, - DbtCloneLocalOperator, - DbtDepsLocalOperator, - DbtLSLocalOperator, - DbtRunLocalOperator, - DbtRunOperationLocalOperator, - DbtSeedLocalOperator, - DbtSnapshotLocalOperator, - DbtTestLocalOperator, -) - -logger = get_logger(__name__) - -try: - from cosmos.operators.docker import ( - DbtBuildDockerOperator, - DbtCloneDockerOperator, - DbtLSDockerOperator, - DbtRunDockerOperator, - DbtRunOperationDockerOperator, - DbtSeedDockerOperator, - DbtSnapshotDockerOperator, - DbtTestDockerOperator, - ) -except ImportError: - DbtLSDockerOperator = MissingPackage("cosmos.operators.docker.DbtLSDockerOperator", "docker") - DbtRunDockerOperator = MissingPackage("cosmos.operators.docker.DbtRunDockerOperator", "docker") - DbtRunOperationDockerOperator = MissingPackage( - "cosmos.operators.docker.DbtRunOperationDockerOperator", - "docker", - ) - DbtSeedDockerOperator = MissingPackage("cosmos.operators.docker.DbtSeedDockerOperator", "docker") - DbtSnapshotDockerOperator = MissingPackage("cosmos.operators.docker.DbtSnapshotDockerOperator", "docker") - DbtTestDockerOperator = MissingPackage("cosmos.operators.docker.DbtTestDockerOperator", "docker") - -try: - from cosmos.operators.kubernetes import ( - DbtBuildKubernetesOperator, - DbtCloneKubernetesOperator, - DbtLSKubernetesOperator, - DbtRunKubernetesOperator, - DbtRunOperationKubernetesOperator, - DbtSeedKubernetesOperator, - DbtSnapshotKubernetesOperator, - DbtTestKubernetesOperator, - ) -except ImportError: - logger.debug("To import Kubernetes modules, install astronomer-cosmos[kubernetes].", stack_info=True) - DbtBuildKubernetesOperator = MissingPackage( - "cosmos.operators.kubernetes.DbtBuildKubernetesOperator", - "kubernetes", - ) - DbtLSKubernetesOperator = MissingPackage( - "cosmos.operators.kubernetes.DbtLSKubernetesOperator", - "kubernetes", - ) - DbtRunKubernetesOperator = MissingPackage( - "cosmos.operators.kubernetes.DbtRunKubernetesOperator", - "kubernetes", - ) - DbtRunOperationKubernetesOperator = MissingPackage( - "cosmos.operators.kubernetes.DbtRunOperationKubernetesOperator", - "kubernetes", - ) - DbtSeedKubernetesOperator = MissingPackage( - "cosmos.operators.kubernetes.DbtSeedKubernetesOperator", - "kubernetes", +if not settings.enable_memory_optimised_imports: + from cosmos.airflow.dag import DbtDag + from cosmos.airflow.task_group import DbtTaskGroup + from cosmos.config import ( + ExecutionConfig, + ProfileConfig, + ProjectConfig, + RenderConfig, ) - DbtSnapshotKubernetesOperator = MissingPackage( - "cosmos.operators.kubernetes.DbtSnapshotKubernetesOperator", - "kubernetes", + from cosmos.constants import ( + DbtResourceType, + ExecutionMode, + InvocationMode, + LoadMode, + SourceRenderingBehavior, + TestBehavior, + TestIndirectSelection, ) - DbtTestKubernetesOperator = MissingPackage( - "cosmos.operators.kubernetes.DbtTestKubernetesOperator", - "kubernetes", + from cosmos.log import get_logger + from cosmos.operators.lazy_load import MissingPackage + from cosmos.operators.local import ( + DbtBuildLocalOperator, + DbtCloneLocalOperator, + DbtDepsLocalOperator, + DbtLSLocalOperator, + DbtRunLocalOperator, + DbtRunOperationLocalOperator, + DbtSeedLocalOperator, + DbtSnapshotLocalOperator, + DbtTestLocalOperator, ) -try: - from cosmos.operators.azure_container_instance import ( - DbtBuildAzureContainerInstanceOperator, - DbtCloneAzureContainerInstanceOperator, - DbtLSAzureContainerInstanceOperator, - DbtRunAzureContainerInstanceOperator, - DbtRunOperationAzureContainerInstanceOperator, - DbtSeedAzureContainerInstanceOperator, - DbtSnapshotAzureContainerInstanceOperator, - DbtTestAzureContainerInstanceOperator, - ) -except ImportError: - DbtBuildAzureContainerInstanceOperator = MissingPackage( - "cosmos.operators.azure_container_instance.DbtBuildAzureContainerInstanceOperator", "azure-container-instance" - ) - DbtLSAzureContainerInstanceOperator = MissingPackage( - "cosmos.operators.azure_container_instance.DbtLSAzureContainerInstanceOperator", "azure-container-instance" - ) - DbtRunAzureContainerInstanceOperator = MissingPackage( - "cosmos.operators.azure_container_instance.DbtRunAzureContainerInstanceOperator", "azure-container-instance" - ) - DbtRunOperationAzureContainerInstanceOperator = MissingPackage( - "cosmos.operators.azure_container_instance.DbtRunOperationAzureContainerInstanceOperator", - "azure-container-instance", - ) - DbtSeedAzureContainerInstanceOperator = MissingPackage( - "cosmos.operators.azure_container_instance.DbtSeedAzureContainerInstanceOperator", "azure-container-instance" - ) - DbtSnapshotAzureContainerInstanceOperator = MissingPackage( - "cosmos.operators.azure_container_instance.DbtSnapshotAzureContainerInstanceOperator", - "azure-container-instance", - ) - DbtTestAzureContainerInstanceOperator = MissingPackage( - "cosmos.operators.azure_container_instance.DbtTestAzureContainerInstanceOperator", "azure-container-instance" - ) + logger = get_logger(__name__) + try: + from cosmos.operators.docker import ( + DbtBuildDockerOperator, + DbtCloneDockerOperator, + DbtLSDockerOperator, + DbtRunDockerOperator, + DbtRunOperationDockerOperator, + DbtSeedDockerOperator, + DbtSnapshotDockerOperator, + DbtTestDockerOperator, + ) + except ImportError: # pragma: no cover + DbtLSDockerOperator = MissingPackage("cosmos.operators.docker.DbtLSDockerOperator", "docker") + DbtRunDockerOperator = MissingPackage("cosmos.operators.docker.DbtRunDockerOperator", "docker") + DbtRunOperationDockerOperator = MissingPackage( + "cosmos.operators.docker.DbtRunOperationDockerOperator", + "docker", + ) + DbtSeedDockerOperator = MissingPackage("cosmos.operators.docker.DbtSeedDockerOperator", "docker") + DbtSnapshotDockerOperator = MissingPackage("cosmos.operators.docker.DbtSnapshotDockerOperator", "docker") + DbtTestDockerOperator = MissingPackage("cosmos.operators.docker.DbtTestDockerOperator", "docker") -try: - from cosmos.operators.aws_eks import ( - DbtBuildAwsEksOperator, - DbtCloneAwsEksOperator, - DbtLSAwsEksOperator, - DbtRunAwsEksOperator, - DbtRunOperationAwsEksOperator, - DbtSeedAwsEksOperator, - DbtSnapshotAwsEksOperator, - DbtTestAwsEksOperator, - ) -except ImportError: - DbtBuildAwsEksOperator = MissingPackage( - "cosmos.operators.azure_container_instance.DbtBuildAwsEksOperator", "aws_eks" - ) - DbtLSAwsEksOperator = MissingPackage("cosmos.operators.azure_container_instance.DbtLSAwsEksOperator", "aws_eks") - DbtRunAwsEksOperator = MissingPackage("cosmos.operators.azure_container_instance.DbtRunAwsEksOperator", "aws_eks") - DbtRunOperationAwsEksOperator = MissingPackage( - "cosmos.operators.azure_container_instance.DbtRunOperationAwsEksOperator", - "aws_eks", - ) - DbtSeedAwsEksOperator = MissingPackage("cosmos.operators.azure_container_instance.DbtSeedAwsEksOperator", "aws_eks") - DbtSnapshotAwsEksOperator = MissingPackage( - "cosmos.operators.azure_container_instance.DbtSnapshotAwsEksOperator", - "aws_eks", - ) - DbtTestAwsEksOperator = MissingPackage("cosmos.operators.azure_container_instance.DbtTestAwsEksOperator", "aws_eks") - - -try: - from cosmos.operators.aws_ecs import ( - DbtBuildAwsEcsOperator, - DbtLSAwsEcsOperator, - DbtRunAwsEcsOperator, - DbtRunOperationAwsEcsOperator, - DbtSeedAwsEcsOperator, - DbtSnapshotAwsEcsOperator, - DbtSourceAwsEcsOperator, - DbtTestAwsEcsOperator, - ) -except ImportError: - DbtBuildAwsEcsOperator = MissingPackage( - "cosmos.operators.aws_ecs.DbtBuildAwsEcsOperator", "aws-ecs" - ) # pragma: no cover - DbtLSAwsEcsOperator = MissingPackage("cosmos.operators.aws_ecs.DbtLSAwsEcsOperator", "aws-ecs") # pragma: no cover - DbtRunAwsEcsOperator = MissingPackage( - "cosmos.operators.aws_ecs.DbtRunAwsEcsOperator", "aws-ecs" - ) # pragma: no cover - DbtRunOperationAwsEcsOperator = MissingPackage( - "cosmos.operators.aws_ecs.DbtRunOperationAwsEcsOperator", - "aws-ecs", - ) # pragma: no cover - DbtSeedAwsEcsOperator = MissingPackage( - "cosmos.operators.aws_ecs.DbtSeedAwsEcsOperator", "aws-ecs" - ) # pragma: no cover - DbtSnapshotAwsEcsOperator = MissingPackage( - "cosmos.operators.aws_ecs.DbtSnapshotAwsEcsOperator", - "aws-ecs", - ) # pragma: no cover - DbtTestAwsEcsOperator = MissingPackage( - "cosmos.operators.aws_ecs.DbtTestAwsEcsOperator", "aws-ecs" - ) # pragma: no cover - DbtSourceAwsEcsOperator = MissingPackage( - "cosmos.operators.aws_ecs.DbtSourceAwsEcsOperator", "aws-ecs" - ) # pragma: no cover - - -try: - from cosmos.operators.gcp_cloud_run_job import ( - DbtBuildGcpCloudRunJobOperator, - DbtCloneGcpCloudRunJobOperator, - DbtLSGcpCloudRunJobOperator, - DbtRunGcpCloudRunJobOperator, - DbtRunOperationGcpCloudRunJobOperator, - DbtSeedGcpCloudRunJobOperator, - DbtSnapshotGcpCloudRunJobOperator, - DbtTestGcpCloudRunJobOperator, - ) -except (ImportError, AttributeError): - DbtBuildGcpCloudRunJobOperator = MissingPackage( - "cosmos.operators.gcp_cloud_run_job.DbtBuildGcpCloudRunJobOperator", "gcp-cloud-run-job" - ) - DbtLSGcpCloudRunJobOperator = MissingPackage( - "cosmos.operators.gcp_cloud_run_job.DbtLSGcpCloudRunJobOperator", "gcp-cloud-run-job" - ) - DbtRunGcpCloudRunJobOperator = MissingPackage( - "cosmos.operators.gcp_cloud_run_job.DbtRunGcpCloudRunJobOperator", "gcp-cloud-run-job" - ) - DbtRunOperationGcpCloudRunJobOperator = MissingPackage( - "cosmos.operators.gcp_cloud_run_job.DbtRunOperationGcpCloudRunJobOperator", "gcp-cloud-run-job" - ) - DbtSeedGcpCloudRunJobOperator = MissingPackage( - "cosmos.operators.gcp_cloud_run_job.DbtSeedGcpCloudRunJobOperator", "gcp-cloud-run-job" - ) - DbtSnapshotGcpCloudRunJobOperator = MissingPackage( - "cosmos.operators.gcp_cloud_run_job.DbtSnapshotGcpCloudRunJobOperator", "gcp-cloud-run-job" - ) - DbtTestGcpCloudRunJobOperator = MissingPackage( - "cosmos.operators.gcp_cloud_run_job.DbtTestGcpCloudRunJobOperator", "gcp-cloud-run-job" - ) + try: + from cosmos.operators.kubernetes import ( + DbtBuildKubernetesOperator, + DbtCloneKubernetesOperator, + DbtLSKubernetesOperator, + DbtRunKubernetesOperator, + DbtRunOperationKubernetesOperator, + DbtSeedKubernetesOperator, + DbtSnapshotKubernetesOperator, + DbtTestKubernetesOperator, + ) + except ImportError: # pragma: no cover + logger.debug("To import Kubernetes modules, install astronomer-cosmos[kubernetes].", stack_info=True) + DbtBuildKubernetesOperator = MissingPackage( + "cosmos.operators.kubernetes.DbtBuildKubernetesOperator", + "kubernetes", + ) + DbtLSKubernetesOperator = MissingPackage( + "cosmos.operators.kubernetes.DbtLSKubernetesOperator", + "kubernetes", + ) + DbtRunKubernetesOperator = MissingPackage( + "cosmos.operators.kubernetes.DbtRunKubernetesOperator", + "kubernetes", + ) + DbtRunOperationKubernetesOperator = MissingPackage( + "cosmos.operators.kubernetes.DbtRunOperationKubernetesOperator", + "kubernetes", + ) + DbtSeedKubernetesOperator = MissingPackage( + "cosmos.operators.kubernetes.DbtSeedKubernetesOperator", + "kubernetes", + ) + DbtSnapshotKubernetesOperator = MissingPackage( + "cosmos.operators.kubernetes.DbtSnapshotKubernetesOperator", + "kubernetes", + ) + DbtTestKubernetesOperator = MissingPackage( + "cosmos.operators.kubernetes.DbtTestKubernetesOperator", + "kubernetes", + ) + try: + from cosmos.operators.azure_container_instance import ( + DbtBuildAzureContainerInstanceOperator, + DbtCloneAzureContainerInstanceOperator, + DbtLSAzureContainerInstanceOperator, + DbtRunAzureContainerInstanceOperator, + DbtRunOperationAzureContainerInstanceOperator, + DbtSeedAzureContainerInstanceOperator, + DbtSnapshotAzureContainerInstanceOperator, + DbtTestAzureContainerInstanceOperator, + ) + except ImportError: # pragma: no cover + DbtBuildAzureContainerInstanceOperator = MissingPackage( + "cosmos.operators.azure_container_instance.DbtBuildAzureContainerInstanceOperator", + "azure-container-instance", + ) + DbtLSAzureContainerInstanceOperator = MissingPackage( + "cosmos.operators.azure_container_instance.DbtLSAzureContainerInstanceOperator", "azure-container-instance" + ) + DbtRunAzureContainerInstanceOperator = MissingPackage( + "cosmos.operators.azure_container_instance.DbtRunAzureContainerInstanceOperator", "azure-container-instance" + ) + DbtRunOperationAzureContainerInstanceOperator = MissingPackage( + "cosmos.operators.azure_container_instance.DbtRunOperationAzureContainerInstanceOperator", + "azure-container-instance", + ) + DbtSeedAzureContainerInstanceOperator = MissingPackage( + "cosmos.operators.azure_container_instance.DbtSeedAzureContainerInstanceOperator", + "azure-container-instance", + ) + DbtSnapshotAzureContainerInstanceOperator = MissingPackage( + "cosmos.operators.azure_container_instance.DbtSnapshotAzureContainerInstanceOperator", + "azure-container-instance", + ) + DbtTestAzureContainerInstanceOperator = MissingPackage( + "cosmos.operators.azure_container_instance.DbtTestAzureContainerInstanceOperator", + "azure-container-instance", + ) -__all__ = [ - "ProjectConfig", - "ProfileConfig", - "ExecutionConfig", - "RenderConfig", - "DbtDag", - "DbtTaskGroup", - "ExecutionMode", - "LoadMode", - "TestBehavior", - "InvocationMode", - "TestIndirectSelection", - "SourceRenderingBehavior", - "DbtResourceType", - # Local Execution Mode - "DbtBuildLocalOperator", - "DbtCloneLocalOperator", - "DbtDepsLocalOperator", # deprecated, to be delete in Cosmos 2.x - "DbtLSLocalOperator", - "DbtRunLocalOperator", - "DbtRunOperationLocalOperator", - "DbtSeedLocalOperator", - "DbtSnapshotLocalOperator", - "DbtTestLocalOperator", - # Docker Execution Mode - "DbtBuildDockerOperator", - "DbtCloneDockerOperator", - "DbtLSDockerOperator", - "DbtRunDockerOperator", - "DbtRunOperationDockerOperator", - "DbtSeedDockerOperator", - "DbtSnapshotDockerOperator", - "DbtTestDockerOperator", - # Kubernetes Execution Mode - "DbtBuildKubernetesOperator", - "DbtCloneKubernetesOperator", - "DbtLSKubernetesOperator", - "DbtRunKubernetesOperator", - "DbtRunOperationKubernetesOperator", - "DbtSeedKubernetesOperator", - "DbtSnapshotKubernetesOperator", - "DbtTestKubernetesOperator", - # Azure Container Instance Execution Mode - "DbtBuildAzureContainerInstanceOperator", - "DbtCloneAzureContainerInstanceOperator", - "DbtLSAzureContainerInstanceOperator", - "DbtRunAzureContainerInstanceOperator", - "DbtRunOperationAzureContainerInstanceOperator", - "DbtSeedAzureContainerInstanceOperator", - "DbtSnapshotAzureContainerInstanceOperator", - "DbtTestAzureContainerInstanceOperator", - # AWS EKS Execution Mode - "DbtBuildAwsEksOperator", - "DbtCloneAwsEksOperator", - "DbtLSAwsEksOperator", - "DbtRunAwsEksOperator", - "DbtRunOperationAwsEksOperator", - "DbtSeedAwsEksOperator", - "DbtSnapshotAwsEksOperator", - "DbtTestAwsEksOperator", - # AWS ECS Task Run Execution Mode - "DbtBuildAwsEcsOperator", - "DbtLSAwsEcsOperator", - "DbtRunAwsEcsOperator", - "DbtRunOperationAwsEcsOperator", - "DbtSeedAwsEcsOperator", - "DbtSnapshotAwsEcsOperator", - "DbtTestAwsEcsOperator", - "DbtSourceAwsEcsOperator", - # GCP Cloud Run Job Execution Mode - "DbtBuildGcpCloudRunJobOperator", - "DbtCloneGcpCloudRunJobOperator", - "DbtLSGcpCloudRunJobOperator", - "DbtRunGcpCloudRunJobOperator", - "DbtRunOperationGcpCloudRunJobOperator", - "DbtSeedGcpCloudRunJobOperator", - "DbtSnapshotGcpCloudRunJobOperator", - "DbtTestGcpCloudRunJobOperator", -] + try: + from cosmos.operators.aws_eks import ( + DbtBuildAwsEksOperator, + DbtCloneAwsEksOperator, + DbtLSAwsEksOperator, + DbtRunAwsEksOperator, + DbtRunOperationAwsEksOperator, + DbtSeedAwsEksOperator, + DbtSnapshotAwsEksOperator, + DbtTestAwsEksOperator, + ) + except ImportError: # pragma: no cover + DbtBuildAwsEksOperator = MissingPackage( + "cosmos.operators.azure_container_instance.DbtBuildAwsEksOperator", "aws_eks" + ) + DbtLSAwsEksOperator = MissingPackage("cosmos.operators.azure_container_instance.DbtLSAwsEksOperator", "aws_eks") + DbtRunAwsEksOperator = MissingPackage( + "cosmos.operators.azure_container_instance.DbtRunAwsEksOperator", "aws_eks" + ) + DbtRunOperationAwsEksOperator = MissingPackage( + "cosmos.operators.azure_container_instance.DbtRunOperationAwsEksOperator", + "aws_eks", + ) + DbtSeedAwsEksOperator = MissingPackage( + "cosmos.operators.azure_container_instance.DbtSeedAwsEksOperator", "aws_eks" + ) + DbtSnapshotAwsEksOperator = MissingPackage( + "cosmos.operators.azure_container_instance.DbtSnapshotAwsEksOperator", + "aws_eks", + ) + DbtTestAwsEksOperator = MissingPackage( + "cosmos.operators.azure_container_instance.DbtTestAwsEksOperator", "aws_eks" + ) -""" -Required provider info for using Airflow config for configuration -""" + try: + from cosmos.operators.aws_ecs import ( + DbtBuildAwsEcsOperator, + DbtLSAwsEcsOperator, + DbtRunAwsEcsOperator, + DbtRunOperationAwsEcsOperator, + DbtSeedAwsEcsOperator, + DbtSnapshotAwsEcsOperator, + DbtSourceAwsEcsOperator, + DbtTestAwsEcsOperator, + ) + except ImportError: # pragma: no cover + DbtBuildAwsEcsOperator = MissingPackage("cosmos.operators.aws_ecs.DbtBuildAwsEcsOperator", "aws-ecs") + DbtLSAwsEcsOperator = MissingPackage("cosmos.operators.aws_ecs.DbtLSAwsEcsOperator", "aws-ecs") + DbtRunAwsEcsOperator = MissingPackage("cosmos.operators.aws_ecs.DbtRunAwsEcsOperator", "aws-ecs") + DbtRunOperationAwsEcsOperator = MissingPackage( + "cosmos.operators.aws_ecs.DbtRunOperationAwsEcsOperator", + "aws-ecs", + ) + DbtSeedAwsEcsOperator = MissingPackage("cosmos.operators.aws_ecs.DbtSeedAwsEcsOperator", "aws-ecs") + DbtSnapshotAwsEcsOperator = MissingPackage( + "cosmos.operators.aws_ecs.DbtSnapshotAwsEcsOperator", + "aws-ecs", + ) + DbtTestAwsEcsOperator = MissingPackage("cosmos.operators.aws_ecs.DbtTestAwsEcsOperator", "aws-ecs") + DbtSourceAwsEcsOperator = MissingPackage("cosmos.operators.aws_ecs.DbtSourceAwsEcsOperator", "aws-ecs") + try: + from cosmos.operators.gcp_cloud_run_job import ( + DbtBuildGcpCloudRunJobOperator, + DbtCloneGcpCloudRunJobOperator, + DbtLSGcpCloudRunJobOperator, + DbtRunGcpCloudRunJobOperator, + DbtRunOperationGcpCloudRunJobOperator, + DbtSeedGcpCloudRunJobOperator, + DbtSnapshotGcpCloudRunJobOperator, + DbtTestGcpCloudRunJobOperator, + ) + except (ImportError, AttributeError): + DbtBuildGcpCloudRunJobOperator = MissingPackage( + "cosmos.operators.gcp_cloud_run_job.DbtBuildGcpCloudRunJobOperator", "gcp-cloud-run-job" + ) + DbtLSGcpCloudRunJobOperator = MissingPackage( + "cosmos.operators.gcp_cloud_run_job.DbtLSGcpCloudRunJobOperator", "gcp-cloud-run-job" + ) + DbtRunGcpCloudRunJobOperator = MissingPackage( + "cosmos.operators.gcp_cloud_run_job.DbtRunGcpCloudRunJobOperator", "gcp-cloud-run-job" + ) + DbtRunOperationGcpCloudRunJobOperator = MissingPackage( + "cosmos.operators.gcp_cloud_run_job.DbtRunOperationGcpCloudRunJobOperator", "gcp-cloud-run-job" + ) + DbtSeedGcpCloudRunJobOperator = MissingPackage( + "cosmos.operators.gcp_cloud_run_job.DbtSeedGcpCloudRunJobOperator", "gcp-cloud-run-job" + ) + DbtSnapshotGcpCloudRunJobOperator = MissingPackage( + "cosmos.operators.gcp_cloud_run_job.DbtSnapshotGcpCloudRunJobOperator", "gcp-cloud-run-job" + ) + DbtTestGcpCloudRunJobOperator = MissingPackage( + "cosmos.operators.gcp_cloud_run_job.DbtTestGcpCloudRunJobOperator", "gcp-cloud-run-job" + ) -def get_provider_info(): - return { - "package-name": "astronomer-cosmos", # Required - "name": "Astronomer Cosmos", # Required - "description": "Astronomer Cosmos is a library for rendering dbt workflows in Airflow. Contains dags, task groups, and operators.", # Required - "versions": [__version__], # Required - "config": { - "cosmos": { - "description": None, - "options": { - "propagate_logs": { - "description": "Enable log propagation from Cosmos custom logger\n", - "version_added": "1.3.0a1", - "version_deprecated": "1.6.0a1", - "deprecation_reason": "`propagate_logs` is no longer necessary as of Cosmos 1.6.0" - " because the issue this option was meant to address is no longer an" - " issue with Cosmos's new logging approach.", - "type": "boolean", - "example": None, - "default": "True", - }, - }, - }, - }, - } + __all__ = [ + "ProjectConfig", + "ProfileConfig", + "ExecutionConfig", + "RenderConfig", + "DbtDag", + "DbtTaskGroup", + "ExecutionMode", + "LoadMode", + "TestBehavior", + "InvocationMode", + "TestIndirectSelection", + "SourceRenderingBehavior", + "DbtResourceType", + # Local Execution Mode + "DbtBuildLocalOperator", + "DbtCloneLocalOperator", + "DbtDepsLocalOperator", # deprecated, to be delete in Cosmos 2.x + "DbtLSLocalOperator", + "DbtRunLocalOperator", + "DbtRunOperationLocalOperator", + "DbtSeedLocalOperator", + "DbtSnapshotLocalOperator", + "DbtTestLocalOperator", + # Docker Execution Mode + "DbtBuildDockerOperator", + "DbtCloneDockerOperator", + "DbtLSDockerOperator", + "DbtRunDockerOperator", + "DbtRunOperationDockerOperator", + "DbtSeedDockerOperator", + "DbtSnapshotDockerOperator", + "DbtTestDockerOperator", + # Kubernetes Execution Mode + "DbtBuildKubernetesOperator", + "DbtCloneKubernetesOperator", + "DbtLSKubernetesOperator", + "DbtRunKubernetesOperator", + "DbtRunOperationKubernetesOperator", + "DbtSeedKubernetesOperator", + "DbtSnapshotKubernetesOperator", + "DbtTestKubernetesOperator", + # Azure Container Instance Execution Mode + "DbtBuildAzureContainerInstanceOperator", + "DbtCloneAzureContainerInstanceOperator", + "DbtLSAzureContainerInstanceOperator", + "DbtRunAzureContainerInstanceOperator", + "DbtRunOperationAzureContainerInstanceOperator", + "DbtSeedAzureContainerInstanceOperator", + "DbtSnapshotAzureContainerInstanceOperator", + "DbtTestAzureContainerInstanceOperator", + # AWS EKS Execution Mode + "DbtBuildAwsEksOperator", + "DbtCloneAwsEksOperator", + "DbtLSAwsEksOperator", + "DbtRunAwsEksOperator", + "DbtRunOperationAwsEksOperator", + "DbtSeedAwsEksOperator", + "DbtSnapshotAwsEksOperator", + "DbtTestAwsEksOperator", + # AWS ECS Task Run Execution Mode + "DbtBuildAwsEcsOperator", + "DbtLSAwsEcsOperator", + "DbtRunAwsEcsOperator", + "DbtRunOperationAwsEcsOperator", + "DbtSeedAwsEcsOperator", + "DbtSnapshotAwsEcsOperator", + "DbtTestAwsEcsOperator", + "DbtSourceAwsEcsOperator", + # GCP Cloud Run Job Execution Mode + "DbtBuildGcpCloudRunJobOperator", + "DbtCloneGcpCloudRunJobOperator", + "DbtLSGcpCloudRunJobOperator", + "DbtRunGcpCloudRunJobOperator", + "DbtRunOperationGcpCloudRunJobOperator", + "DbtSeedGcpCloudRunJobOperator", + "DbtSnapshotGcpCloudRunJobOperator", + "DbtTestGcpCloudRunJobOperator", + ] diff --git a/cosmos/constants.py b/cosmos/constants.py index e00f715816..8fcb6588ac 100644 --- a/cosmos/constants.py +++ b/cosmos/constants.py @@ -8,6 +8,7 @@ BIGQUERY_PROFILE_TYPE = "bigquery" DBT_PROFILE_PATH = Path(os.path.expanduser("~")).joinpath(".dbt/profiles.yml") +DBT_PROJECT_FILENAME = "dbt_project.yml" DEFAULT_DBT_PROFILE_NAME = "cosmos_profile" DEFAULT_DBT_TARGET_NAME = "cosmos_target" DEFAULT_COSMOS_CACHE_DIR_NAME = "cosmos" @@ -23,7 +24,7 @@ DBT_BINARY_NAME = "dbt" DEFAULT_PROFILES_FILE_NAME = "profiles.yml" PACKAGE_LOCKFILE_YML = "package-lock.yml" -DBT_PACKAGES_FOLDER = "dbt_packages" +DBT_DEFAULT_PACKAGES_FOLDER = "dbt_packages" DEFAULT_OPENLINEAGE_NAMESPACE = "cosmos" OPENLINEAGE_PRODUCER = "https://github.com/astronomer/astronomer-cosmos/" diff --git a/cosmos/converter.py b/cosmos/converter.py index cb07bdeffa..bf3fa12644 100644 --- a/cosmos/converter.py +++ b/cosmos/converter.py @@ -268,6 +268,7 @@ def __init__( cache_dir = None cache_identifier = None + if settings.enable_cache: cache_identifier = cache._create_cache_identifier(dag, task_group) cache_dir = cache._obtain_cache_dir_path(cache_identifier=cache_identifier) @@ -303,6 +304,7 @@ def __init__( "env": env_vars, "vars": dbt_vars, "cache_dir": cache_dir, + "manifest_filepath": project_config.manifest_path, } validate_arguments( diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index a5b89186e4..84aaac71f5 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -261,12 +261,13 @@ def run_command( ) -> str: """Run a command either with dbtRunner or Python subprocess, returning the stdout.""" - runner = "dbt Runner" if dbt_runner.is_available() else "Python subprocess" + use_dbt_runner = invocation_mode == InvocationMode.DBT_RUNNER and dbt_runner.is_available() + runner = "dbt Runner" if use_dbt_runner else "Python subprocess" command = [str(arg) if arg is not None else "" for arg in command] logger.info("Running command with %s: `%s`", runner, " ".join(command)) logger.debug("Environment variable keys: %s", env_vars.keys()) - if invocation_mode == InvocationMode.DBT_RUNNER and dbt_runner.is_available(): + if use_dbt_runner: stdout = run_command_with_dbt_runner(command, tmp_dir, env_vars) else: stdout = run_command_with_subprocess(command, tmp_dir, env_vars) diff --git a/cosmos/dbt/project.py b/cosmos/dbt/project.py index cf124f28a5..df05915f85 100644 --- a/cosmos/dbt/project.py +++ b/cosmos/dbt/project.py @@ -6,11 +6,15 @@ from pathlib import Path from typing import Generator +import yaml + from cosmos.constants import ( + DBT_DEFAULT_PACKAGES_FOLDER, DBT_DEPENDENCIES_FILE_NAMES, DBT_LOG_DIR_NAME, - DBT_PACKAGES_FOLDER, + DBT_MANIFEST_FILE_NAME, DBT_PARTIAL_PARSE_FILE_NAME, + DBT_PROJECT_FILENAME, DBT_TARGET_DIR_NAME, PACKAGE_LOCKFILE_YML, ) @@ -36,15 +40,44 @@ def has_non_empty_dependencies_file(project_path: Path) -> bool: return False +def get_dbt_packages_subpath(source_folder: Path) -> str: + """ + Return the dbt project's package installation sub path. + + By default, ``dbt deps`` installs packages in the ``dbt_packages`` directory, inside the dbt project folder. + Users can specify a custom directory via the `packages-install-path` in the ``dbt_project.yml`` file. + Example: ``packages-install-path: custom_dbt_packages``. + + More information: + https://docs.getdbt.com/reference/project-configs/packages-install-path + + :param source_folder: The dbt project root directory + :returns: A string containing the dbt_packages subpath within the source folder. + """ + subpath = DBT_DEFAULT_PACKAGES_FOLDER + dbt_project_yml_path = source_folder / DBT_PROJECT_FILENAME + if dbt_project_yml_path.exists(): + with open(dbt_project_yml_path) as fp: + try: + dbt_project_file_content = yaml.safe_load(fp) + except yaml.YAMLError: + logger.info(f"Unable to read the {DBT_PROJECT_FILENAME} file") + else: + subpath = dbt_project_file_content.get("packages-install-path", DBT_DEFAULT_PACKAGES_FOLDER) + return subpath + + def copy_dbt_packages(source_folder: Path, target_folder: Path) -> None: """ Copies the dbt packages related files and directories from source_folder to target_folder. - :param: source_folder: The base directory where paths are sourced from. - :param: target_folder: The directory where paths will be copied to. + :param source_folder: The base directory where paths are sourced from. + :param target_folder: The directory where paths will be copied to. """ logger.info("Copying dbt packages to temporary folder...") - dbt_packages_paths = [DBT_PACKAGES_FOLDER, PACKAGE_LOCKFILE_YML] + + dbt_packages_folder = get_dbt_packages_subpath(source_folder) + dbt_packages_paths = [dbt_packages_folder, PACKAGE_LOCKFILE_YML] for relative_path in dbt_packages_paths: src_path = source_folder / relative_path @@ -60,12 +93,29 @@ def copy_dbt_packages(source_folder: Path, target_folder: Path) -> None: logger.info("Completed copying dbt packages to temporary folder.") +def copy_manifest_file_if_exists(source_manifest: str | Path, dbt_project_folder: str | Path) -> None: + """ + Copies the source manifest.json file, if available, to the given desired dbt project folder. + + :param source_manifest: manifest.json filepath + :param dbt_project_folder: destination dbt project folder (it will be copied to the target folder) + """ + dbt_project_folder = Path(dbt_project_folder) + if source_manifest and Path(source_manifest).exists(): + logger.info(f"Copying the manifest from {source_manifest}...") + target_folder_path = dbt_project_folder / DBT_TARGET_DIR_NAME + tmp_manifest_filepath = target_folder_path / DBT_MANIFEST_FILE_NAME + Path(target_folder_path).mkdir(parents=True, exist_ok=True) + shutil.copy(source_manifest, tmp_manifest_filepath) + + def create_symlinks(project_path: Path, tmp_dir: Path, ignore_dbt_packages: bool) -> None: """Helper function to create symlinks to the dbt project files.""" ignore_paths = [DBT_LOG_DIR_NAME, DBT_TARGET_DIR_NAME, PACKAGE_LOCKFILE_YML, "profiles.yml"] if ignore_dbt_packages: + dbt_packages_subpath = get_dbt_packages_subpath(project_path) # this is linked to dbt deps so if dbt deps is true then ignore existing dbt_packages folder - ignore_paths.append(DBT_PACKAGES_FOLDER) + ignore_paths.append(dbt_packages_subpath) for child_name in os.listdir(project_path): if child_name not in ignore_paths: os.symlink(project_path / child_name, tmp_dir / child_name) diff --git a/cosmos/listeners/dag_run_listener.py b/cosmos/listeners/dag_run_listener.py index 0314c3474d..2abd21e7d1 100644 --- a/cosmos/listeners/dag_run_listener.py +++ b/cosmos/listeners/dag_run_listener.py @@ -1,12 +1,19 @@ from __future__ import annotations +import hashlib + +from airflow import __version__ as airflow_version from airflow.listeners import hookimpl from airflow.models.dag import DAG from airflow.models.dagrun import DagRun +from packaging import version from cosmos import telemetry +from cosmos.constants import _AIRFLOW3_MAJOR_VERSION from cosmos.log import get_logger +AIRFLOW_VERSION_MAJOR = version.parse(airflow_version).major + logger = get_logger(__name__) @@ -50,8 +57,13 @@ def on_dag_run_success(dag_run: DagRun, msg: str) -> None: logger.debug("The DAG does not use Cosmos") return + if AIRFLOW_VERSION_MAJOR < _AIRFLOW3_MAJOR_VERSION: + dag_hash = dag_run.dag_hash + else: + dag_hash = hashlib.md5(dag_run.dag_id.encode("utf-8")).hexdigest() + additional_telemetry_metrics = { - "dag_hash": dag_run.dag_hash, + "dag_hash": dag_hash, "status": EventStatus.SUCCESS, "task_count": len(serialized_dag.task_ids), "cosmos_task_count": total_cosmos_tasks(serialized_dag), @@ -73,8 +85,13 @@ def on_dag_run_failed(dag_run: DagRun, msg: str) -> None: logger.debug("The DAG does not use Cosmos") return + if AIRFLOW_VERSION_MAJOR < _AIRFLOW3_MAJOR_VERSION: + dag_hash = dag_run.dag_hash + else: + dag_hash = hashlib.md5(dag_run.dag_id.encode("utf-8")).hexdigest() + additional_telemetry_metrics = { - "dag_hash": dag_run.dag_hash, + "dag_hash": dag_hash, "status": EventStatus.FAILED, "task_count": len(serialized_dag.task_ids), "cosmos_task_count": total_cosmos_tasks(serialized_dag), diff --git a/cosmos/operators/_asynchronous/__init__.py b/cosmos/operators/_asynchronous/__init__.py index c78cef557d..cbbb51f07e 100644 --- a/cosmos/operators/_asynchronous/__init__.py +++ b/cosmos/operators/_asynchronous/__init__.py @@ -42,7 +42,7 @@ def run_subprocess(self, command: list[str], env: dict[str, str], cwd: str) -> F return super().run_subprocess(command, env, cwd) def execute(self, context: Context, **kwargs: Any) -> None: - async_context = {"profile_type": self.profile_config.get_profile_type()} + async_context = {"profile_type": self.profile_config.get_profile_type(), "run_id": context["run_id"]} self.build_and_run_cmd( context=context, cmd_flags=self.dbt_cmd_flags, run_as_async=True, async_context=async_context ) @@ -74,7 +74,11 @@ def run_subprocess(self, command: list[str], env: dict[str, str], cwd: str) -> F return super().run_subprocess(command, env, cwd) def execute(self, context: Context, **kwargs: Any) -> Any: - async_context = {"profile_type": self.profile_config.get_profile_type(), "teardown_task": True} + async_context = { + "profile_type": self.profile_config.get_profile_type(), + "teardown_task": True, + "run_id": context["run_id"], + } self.build_and_run_cmd( context=context, cmd_flags=self.dbt_cmd_flags, run_as_async=True, async_context=async_context ) diff --git a/cosmos/operators/_asynchronous/bigquery.py b/cosmos/operators/_asynchronous/bigquery.py index 3b1c8d6fbd..8cbc872f37 100644 --- a/cosmos/operators/_asynchronous/bigquery.py +++ b/cosmos/operators/_asynchronous/bigquery.py @@ -61,7 +61,7 @@ def _configure_bigquery_async_op_args(async_op_obj: Any, **kwargs: Any) -> Any: class DbtRunAirflowAsyncBigqueryOperator(BigQueryInsertJobOperator, AbstractDbtLocalBase): # type: ignore[misc] - template_fields: Sequence[str] = ("gcp_project", "dataset", "location", "compiled_sql") + template_fields: Sequence[str] = ("gcp_project", "dataset", "location", "compiled_sql", "full_refresh") template_fields_renderers = { "compiled_sql": "sql", } @@ -81,6 +81,7 @@ def __init__( self.configuration: dict[str, Any] = {} self.dbt_kwargs = dbt_kwargs or {} task_id = self.dbt_kwargs.pop("task_id") + self.full_refresh = self.dbt_kwargs.pop("full_refresh", False) AbstractDbtLocalBase.__init__( self, task_id=task_id, project_dir=project_dir, profile_config=profile_config, **self.dbt_kwargs ) @@ -136,6 +137,7 @@ def get_remote_sql(self) -> str: file_path = self.async_context["dbt_node_config"]["file_path"] # type: ignore dbt_dag_task_group_identifier = self.async_context["dbt_dag_task_group_identifier"] + run_id = self.async_context["run_id"] remote_target_path_str = str(remote_target_path).rstrip("/") @@ -144,7 +146,9 @@ def get_remote_sql(self) -> str: project_dir_parent = str(Path(self.project_dir).parent) relative_file_path = str(file_path).replace(project_dir_parent, "").lstrip("/") - remote_model_path = f"{remote_target_path_str}/{dbt_dag_task_group_identifier}/run/{relative_file_path}" + remote_model_path = ( + f"{remote_target_path_str}/{dbt_dag_task_group_identifier}/{run_id}/run/{relative_file_path}" + ) object_storage_path = ObjectStoragePath(remote_model_path, conn_id=remote_target_path_conn_id) with object_storage_path.open() as fp: # type: ignore @@ -154,6 +158,9 @@ def get_remote_sql(self) -> str: return sql # type: ignore def execute(self, context: Context, **kwargs: Any) -> None: + if self.async_context.get("run_id") is None: + self.async_context["run_id"] = context["run_id"] + if settings.enable_setup_async_task: self.configuration = { "query": { @@ -192,6 +199,9 @@ def execute_complete(self, context: Context, event: dict[str, Any]) -> Any: This returns immediately. It relies on trigger to throw an exception, otherwise it assumes execution was successful. """ + if self.async_context.get("run_id") is None: + self.async_context["run_id"] = context["run_id"] + job_id = super().execute_complete(context=context, event=event) self.log.info("Configuration is %s", str(self.configuration)) self._store_template_fields(context=context) diff --git a/cosmos/operators/airflow_async.py b/cosmos/operators/airflow_async.py index ff874a06ad..c0cb382b60 100644 --- a/cosmos/operators/airflow_async.py +++ b/cosmos/operators/airflow_async.py @@ -63,6 +63,9 @@ def __init__( dbt_kwargs = {} + # Extract full_refresh from kwargs if present + dbt_kwargs["full_refresh"] = kwargs.pop("full_refresh", False) + for arg_key, arg_value in kwargs.items(): if arg_key == "task_id": clean_kwargs[arg_key] = arg_value diff --git a/cosmos/operators/local.py b/cosmos/operators/local.py index 321734c166..5062620417 100644 --- a/cosmos/operators/local.py +++ b/cosmos/operators/local.py @@ -35,9 +35,19 @@ _get_latest_cached_package_lockfile, is_cache_package_lockfile_enabled, ) -from cosmos.constants import _AIRFLOW3_MAJOR_VERSION, FILE_SCHEME_AIRFLOW_DEFAULT_CONN_ID_MAP, InvocationMode +from cosmos.constants import ( + _AIRFLOW3_MAJOR_VERSION, + DBT_DEPENDENCIES_FILE_NAMES, + FILE_SCHEME_AIRFLOW_DEFAULT_CONN_ID_MAP, + InvocationMode, +) from cosmos.dataset import get_dataset_alias_name -from cosmos.dbt.project import copy_dbt_packages, get_partial_parse_path, has_non_empty_dependencies_file +from cosmos.dbt.project import ( + copy_dbt_packages, + copy_manifest_file_if_exists, + get_partial_parse_path, + has_non_empty_dependencies_file, +) from cosmos.exceptions import AirflowCompatibilityError, CosmosDbtRunError, CosmosValueError from cosmos.settings import ( remote_target_path, @@ -143,6 +153,7 @@ class AbstractDbtLocalBase(AbstractDbtBase): :param install_deps (deprecated): If true, install dependencies before running the command :param copy_dbt_packages: If true, copy pre-existing `dbt_packages` (before running dbt deps) :param callback: A callback function called on after a dbt run with a path to the dbt project directory. + :param manifest_filepath: The path to the user-defined Manifest file. It's "" by default. :param target_name: A name to use for the dbt target. If not provided, and no target is found in your project's dbt_project.yml, "cosmos_target" is used. :param should_store_compiled_sql: If true, store the compiled SQL in the compiled_sql rendered template. @@ -166,6 +177,7 @@ def __init__( invocation_mode: InvocationMode | None = None, install_deps: bool = True, copy_dbt_packages: bool = settings.default_copy_dbt_packages, + manifest_filepath: str = "", callback: Callable[[str], None] | list[Callable[[str], None]] | None = None, callback_args: dict[str, Any] | None = None, should_store_compiled_sql: bool = True, @@ -197,6 +209,8 @@ def __init__( self.install_deps = install_deps and has_non_empty_dependencies_file(Path(self.project_dir)) self.copy_dbt_packages = copy_dbt_packages + self.manifest_filepath = manifest_filepath + @cached_property def subprocess_hook(self) -> FullOutputSubprocessHook: """Returns hook for running the bash command.""" @@ -311,8 +325,9 @@ def _construct_dest_file_path( dest_target_dir_str = str(dest_target_dir).rstrip("/") dag_task_group_identifier = self.extra_context["dbt_dag_task_group_identifier"] rel_path = os.path.relpath(file_path, source_compiled_dir).lstrip("/") + run_id = self.extra_context["run_id"] - return f"{dest_target_dir_str}/{dag_task_group_identifier}/{resource_type}/{rel_path}" + return f"{dest_target_dir_str}/{dag_task_group_identifier}/{run_id}/{resource_type}/{rel_path}" def _upload_sql_files(self, tmp_project_dir: str, resource_type: str) -> None: start_time = time.time() @@ -329,23 +344,32 @@ def _upload_sql_files(self, tmp_project_dir: str, resource_type: str) -> None: for file_path in files: dest_file_path = self._construct_dest_file_path(dest_target_dir, file_path, source_run_dir, resource_type) dest_object_storage_path = ObjectStoragePath(dest_file_path, conn_id=dest_conn_id) + dest_object_storage_path.parent.mkdir(parents=True, exist_ok=True) ObjectStoragePath(file_path).copy(dest_object_storage_path) self.log.debug("Copied %s to %s", file_path, dest_object_storage_path) elapsed_time = time.time() - start_time self.log.info("SQL files upload completed in %.2f seconds.", elapsed_time) - def _delete_sql_files(self, tmp_project_dir: Path, resource_type: str) -> None: + def _delete_sql_files(self) -> None: + """Deletes the entire run-specific directory from the remote target.""" dest_target_dir, dest_conn_id = self._configure_remote_target_path() - source_run_dir = Path(tmp_project_dir) / f"target/{resource_type}" - files = [str(file) for file in source_run_dir.rglob("*") if file.is_file()] + if not dest_target_dir or not dest_conn_id: + self.log.warning("Remote target path or connection ID not configured. Skipping deletion.") + return + from airflow.io.path import ObjectStoragePath - for file_path in files: - dest_file_path = self._construct_dest_file_path(dest_target_dir, file_path, source_run_dir, resource_type) # type: ignore - dest_object_storage_path = ObjectStoragePath(dest_file_path, conn_id=dest_conn_id) - dest_object_storage_path.unlink() - self.log.debug("Deleted %s to %s", file_path, dest_object_storage_path) + dag_task_group_identifier = self.extra_context["dbt_dag_task_group_identifier"] + run_id = self.extra_context["run_id"] + run_dir_path_str = f"{str(dest_target_dir).rstrip('/')}/{dag_task_group_identifier}/{run_id}" + run_dir_path = ObjectStoragePath(run_dir_path_str, conn_id=dest_conn_id) + + if run_dir_path.exists(): + run_dir_path.rmdir(recursive=True) + self.log.info("Deleted remote run directory: %s", run_dir_path_str) + else: + self.log.debug("Remote run directory does not exist, skipping deletion: %s", run_dir_path_str) def store_freshness_json(self, tmp_project_dir: str, context: Context) -> None: """ @@ -451,6 +475,8 @@ def _clone_project(self, tmp_dir_path: Path) -> None: copy_dbt_packages(Path(self.project_dir), tmp_dir_path) self.log.info("Completed copying dbt packages to temporary folder.") + copy_manifest_file_if_exists(self.manifest_filepath, Path(tmp_dir_path)) + def _handle_partial_parse(self, tmp_dir_path: Path) -> None: if self.cache_dir is None: return @@ -460,7 +486,7 @@ def _handle_partial_parse(self, tmp_dir_path: Path) -> None: cache._copy_partial_parse_to_project(latest_partial_parse, tmp_dir_path) def _generate_dbt_flags(self, tmp_project_dir: str, profile_path: Path) -> list[str]: - return [ + dbt_flags = [ "--project-dir", str(tmp_project_dir), "--profiles-dir", @@ -470,12 +496,28 @@ def _generate_dbt_flags(self, tmp_project_dir: str, profile_path: Path) -> list[ "--target", self.profile_config.target_name, ] + if self.invocation_mode == InvocationMode.DBT_RUNNER: + # PR #1484 introduced the use of dbtRunner during DAG parsing. As a result, invoking dbtRunner again + # during task execution can lead to task hangs—especially on Airflow 2.x. Investigation revealed that + # the issue stems from how dbtRunner handles static parsing. Cosmos copies the dbt project to temporary + # directories, and the use of different temp paths between parsing and execution appears to interfere + # with dbt's static parsing behavior. As a workaround, passing the --no-static-parser flag avoids these + # hangs and ensures reliable task execution. + dbt_flags.append("--no-static-parser") + return dbt_flags def _install_dependencies( self, tmp_dir_path: Path, flags: list[str], env: dict[str, str | bytes | os.PathLike[Any]] ) -> None: self._cache_package_lockfile(tmp_dir_path) deps_command = [self.dbt_executable_path, "deps"] + flags + + for filename in DBT_DEPENDENCIES_FILE_NAMES: + filepath = tmp_dir_path / filename + if filepath.is_file(): + self.log.debug("Checking for the %s dependencies file.", str(filename)) + self.log.debug("Contents of the <%s> dependencies file:\n %s", str(filepath), str(filepath.read_text())) + self.invoke_dbt(command=deps_command, env=env, cwd=tmp_dir_path) @staticmethod @@ -521,7 +563,7 @@ def _handle_post_execution(self, tmp_project_dir: str, context: Context) -> None def _handle_async_execution(self, tmp_project_dir: str, context: Context, async_context: dict[str, Any]) -> None: if async_context.get("teardown_task") and settings.enable_teardown_async_task: - self._delete_sql_files(Path(tmp_project_dir), "run") + self._delete_sql_files() return if settings.enable_setup_async_task: @@ -535,7 +577,7 @@ def _handle_async_execution(self, tmp_project_dir: str, context: Context, async_ async_op_configurator(self, sql=sql) async_context["async_operator"].execute(self, context) - def run_command( + def run_command( # noqa: C901 self, cmd: list[str], env: dict[str, str | bytes | os.PathLike[Any]], @@ -549,10 +591,13 @@ def run_command( if not self.invocation_mode: self._discover_invocation_mode() - with tempfile.TemporaryDirectory() as tmp_project_dir: + if self.extra_context.get("run_id") is None: + self.extra_context["run_id"] = context["run_id"] + with tempfile.TemporaryDirectory() as tmp_project_dir: tmp_dir_path = Path(tmp_project_dir) env = {k: str(v) for k, v in env.items()} + self._clone_project(tmp_dir_path) if self.partial_parse: @@ -785,6 +830,13 @@ def build_and_run_cmd( run_as_async: bool = False, async_context: dict[str, Any] | None = None, ) -> FullOutputSubprocessResult | dbtRunnerResult: + # If this is an async run and we're using the setup task, make sure to include the full_refresh flag if set + if run_as_async and settings.enable_setup_async_task and getattr(self, "full_refresh", False): + if cmd_flags is None: + cmd_flags = [] + if "--full-refresh" not in cmd_flags: + cmd_flags.append("--full-refresh") + dbt_cmd, env = self.build_cmd(context=context, cmd_flags=cmd_flags) dbt_cmd = dbt_cmd or [] result = self.run_command( @@ -801,7 +853,6 @@ def on_kill(self) -> None: class DbtLocalBaseOperator(AbstractDbtLocalBase, BaseOperator): # type: ignore[misc] - template_fields: Sequence[str] = AbstractDbtLocalBase.template_fields # type: ignore[operator] def __init__(self, *args: Any, **kwargs: Any) -> None: @@ -1269,7 +1320,7 @@ class DbtDepsLocalOperator(DbtLocalBaseOperator): def __init__(self, **kwargs: str) -> None: raise DeprecationWarning( - "The DbtDepsOperator has been deprecated. " "Please use the `install_deps` flag in dbt_args instead." + "The DbtDepsOperator has been deprecated. Please use the `install_deps` flag in dbt_args instead." ) diff --git a/cosmos/provider_info.py b/cosmos/provider_info.py new file mode 100644 index 0000000000..9f1764ff20 --- /dev/null +++ b/cosmos/provider_info.py @@ -0,0 +1,36 @@ +""" +Required provider info for using Airflow config for configuration +""" + +from __future__ import annotations + +from typing import Any + +from cosmos import __version__ # type: ignore[attr-defined] + + +def get_provider_info() -> dict[str, Any]: + return { + "package-name": "astronomer-cosmos", # Required + "name": "Astronomer Cosmos", # Required + "description": "Astronomer Cosmos is a library for rendering dbt workflows in Airflow. Contains dags, task groups, and operators.", # Required + "versions": [__version__], # Required + "config": { + "cosmos": { + "description": None, + "options": { + "propagate_logs": { + "description": "Enable log propagation from Cosmos custom logger\n", + "version_added": "1.3.0a1", + "version_deprecated": "1.6.0a1", + "deprecation_reason": "`propagate_logs` is no longer necessary as of Cosmos 1.6.0" + " because the issue this option was meant to address is no longer an" + " issue with Cosmos's new logging approach.", + "type": "boolean", + "example": None, + "default": "True", + }, + }, + }, + }, + } diff --git a/cosmos/settings.py b/cosmos/settings.py index bdb862a9c7..d7be150437 100644 --- a/cosmos/settings.py +++ b/cosmos/settings.py @@ -39,6 +39,13 @@ remote_target_path = conf.get("cosmos", "remote_target_path", fallback=None) remote_target_path_conn_id = conf.get("cosmos", "remote_target_path_conn_id", fallback=None) +# Eager imports in cosmos/__init__.py expose all Cosmos classes at the top level, +# which can significantly increase memory usage—even when Cosmos is installed but not actively used. +# This option allows disabling those eager imports to reduce memory footprint. +# When enabled, users must access Cosmos classes via their full module paths, +# avoiding the overhead of importing unused modules and classes. +enable_memory_optimised_imports = conf.getboolean("cosmos", "enable_memory_optimised_imports", fallback=False) + # Related to async operators enable_setup_async_task = conf.getboolean("cosmos", "enable_setup_async_task", fallback=True) enable_teardown_async_task = conf.getboolean("cosmos", "enable_teardown_async_task", fallback=True) diff --git a/dev/dags/basic_cosmos_dag.py b/dev/dags/basic_cosmos_dag.py index 613d839acd..2d68ebdec2 100644 --- a/dev/dags/basic_cosmos_dag.py +++ b/dev/dags/basic_cosmos_dag.py @@ -6,7 +6,10 @@ from datetime import datetime from pathlib import Path +# [START cosmos_init_imports] from cosmos import DbtDag, ProfileConfig, ProjectConfig + +# [END cosmos_init_imports] from cosmos.profiles import PostgresUserPasswordProfileMapping DEFAULT_DBT_ROOT_PATH = Path(__file__).parent / "dbt" diff --git a/dev/dags/basic_cosmos_dag_full_module_path_imports.py b/dev/dags/basic_cosmos_dag_full_module_path_imports.py new file mode 100644 index 0000000000..645d36310a --- /dev/null +++ b/dev/dags/basic_cosmos_dag_full_module_path_imports.py @@ -0,0 +1,47 @@ +""" +An example DAG that uses Cosmos by importing Cosmos classes with their full module path. +""" + +import os +from datetime import datetime +from pathlib import Path + +# [START cosmos_explicit_imports] +from cosmos.airflow.dag import DbtDag +from cosmos.config import ProfileConfig, ProjectConfig + +# [END cosmos_explicit_imports] +from cosmos.profiles import PostgresUserPasswordProfileMapping + +DEFAULT_DBT_ROOT_PATH = Path(__file__).parent / "dbt" +DBT_ROOT_PATH = Path(os.getenv("DBT_ROOT_PATH", DEFAULT_DBT_ROOT_PATH)) + +profile_config = ProfileConfig( + profile_name="default", + target_name="dev", + profile_mapping=PostgresUserPasswordProfileMapping( + conn_id="example_conn", + profile_args={"schema": "public"}, + disable_event_tracking=True, + ), +) + +# [START local_example] +basic_cosmos_dag_full_module_path_imports = DbtDag( + # dbt/cosmos-specific parameters + project_config=ProjectConfig( + DBT_ROOT_PATH / "jaffle_shop", + ), + profile_config=profile_config, + operator_args={ + "install_deps": True, # install any necessary dependencies before running any dbt command + "full_refresh": True, # used only in dbt commands that support this flag + }, + # normal dag parameters + schedule="@daily", + start_date=datetime(2023, 1, 1), + catchup=False, + dag_id="basic_cosmos_dag_full_module_path_imports", + default_args={"retries": 2}, +) +# [END local_example] diff --git a/dev/dags/dbt/jaffle_shop/target/manifest.json b/dev/dags/dbt/jaffle_shop/target/manifest.json index c8ba975540..e46748c438 100644 --- a/dev/dags/dbt/jaffle_shop/target/manifest.json +++ b/dev/dags/dbt/jaffle_shop/target/manifest.json @@ -2,7 +2,6 @@ "child_map": { "model.jaffle_shop.customers": [ "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d", - "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2", "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1" ], "model.jaffle_shop.orders": [ @@ -11,10 +10,8 @@ "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49", "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625", "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59", - "test.jaffle_shop.not_null_orders_customer_id.c5f02694af", "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a", "test.jaffle_shop.not_null_orders_order_id.cf6c17daed", - "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2", "test.jaffle_shop.unique_orders_order_id.fed79b3a6e" ], "model.jaffle_shop.stg_customers": [ @@ -53,13 +50,11 @@ "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49": [], "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625": [], "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59": [], - "test.jaffle_shop.not_null_orders_customer_id.c5f02694af": [], "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a": [], "test.jaffle_shop.not_null_orders_order_id.cf6c17daed": [], "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa": [], "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64": [], "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075": [], - "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2": [], "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1": [], "test.jaffle_shop.unique_orders_order_id.fed79b3a6e": [], "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada": [], @@ -102,7 +97,7 @@ "macros": { "macro.dbt._split_part_negative": { "arguments": [], - "created_at": 1719485736.555134, + "created_at": 1747731927.7531478, "depends_on": { "macros": [] }, @@ -124,7 +119,7 @@ }, "macro.dbt.after_commit": { "arguments": [], - "created_at": 1719485736.337471, + "created_at": 1747731927.591688, "depends_on": { "macros": [ "macro.dbt.make_hook_config" @@ -148,7 +143,7 @@ }, "macro.dbt.alter_column_comment": { "arguments": [], - "created_at": 1719485736.583679, + "created_at": 1747731927.778205, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__alter_column_comment" @@ -172,7 +167,7 @@ }, "macro.dbt.alter_column_type": { "arguments": [], - "created_at": 1719485736.599694, + "created_at": 1747731927.791672, "depends_on": { "macros": [ "macro.dbt.default__alter_column_type" @@ -196,7 +191,7 @@ }, "macro.dbt.alter_relation_add_remove_columns": { "arguments": [], - "created_at": 1719485736.600589, + "created_at": 1747731927.792835, "depends_on": { "macros": [ "macro.dbt.default__alter_relation_add_remove_columns" @@ -220,7 +215,7 @@ }, "macro.dbt.alter_relation_comment": { "arguments": [], - "created_at": 1719485736.585442, + "created_at": 1747731927.778694, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__alter_relation_comment" @@ -244,7 +239,7 @@ }, "macro.dbt.any_value": { "arguments": [], - "created_at": 1719485736.542892, + "created_at": 1747731927.7449331, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__any_value" @@ -268,7 +263,7 @@ }, "macro.dbt.apply_grants": { "arguments": [], - "created_at": 1719485736.577766, + "created_at": 1747731927.775132, "depends_on": { "macros": [ "macro.dbt.default__apply_grants" @@ -292,7 +287,7 @@ }, "macro.dbt.array_append": { "arguments": [], - "created_at": 1719485736.5569642, + "created_at": 1747731927.754894, "depends_on": { "macros": [ "macro.dbt.default__array_append" @@ -316,7 +311,7 @@ }, "macro.dbt.array_concat": { "arguments": [], - "created_at": 1719485736.5521111, + "created_at": 1747731927.750371, "depends_on": { "macros": [ "macro.dbt.default__array_concat" @@ -340,7 +335,7 @@ }, "macro.dbt.array_construct": { "arguments": [], - "created_at": 1719485736.556025, + "created_at": 1747731927.754187, "depends_on": { "macros": [ "macro.dbt.default__array_construct" @@ -364,7 +359,7 @@ }, "macro.dbt.assert_columns_equivalent": { "arguments": [], - "created_at": 1719485736.5031989, + "created_at": 1747731927.709482, "depends_on": { "macros": [ "macro.dbt.get_column_schema_from_query", @@ -390,7 +385,7 @@ }, "macro.dbt.before_begin": { "arguments": [], - "created_at": 1719485736.336732, + "created_at": 1747731927.5913181, "depends_on": { "macros": [ "macro.dbt.make_hook_config" @@ -414,7 +409,7 @@ }, "macro.dbt.bool_or": { "arguments": [], - "created_at": 1719485736.5525389, + "created_at": 1747731927.7510011, "depends_on": { "macros": [ "macro.dbt.default__bool_or" @@ -438,7 +433,7 @@ }, "macro.dbt.build_config_dict": { "arguments": [], - "created_at": 1719485736.617571, + "created_at": 1747731927.803214, "depends_on": { "macros": [] }, @@ -460,7 +455,7 @@ }, "macro.dbt.build_ref_function": { "arguments": [], - "created_at": 1719485736.616483, + "created_at": 1747731927.801829, "depends_on": { "macros": [ "macro.dbt.resolve_model_name" @@ -484,7 +479,7 @@ }, "macro.dbt.build_snapshot_staging_table": { "arguments": [], - "created_at": 1719485736.358976, + "created_at": 1747731927.614965, "depends_on": { "macros": [ "macro.dbt.make_temp_relation", @@ -511,7 +506,7 @@ }, "macro.dbt.build_snapshot_table": { "arguments": [], - "created_at": 1719485736.3569238, + "created_at": 1747731927.6137102, "depends_on": { "macros": [ "macro.dbt.default__build_snapshot_table" @@ -535,7 +530,7 @@ }, "macro.dbt.build_source_function": { "arguments": [], - "created_at": 1719485736.616947, + "created_at": 1747731927.80235, "depends_on": { "macros": [ "macro.dbt.resolve_model_name" @@ -559,7 +554,7 @@ }, "macro.dbt.call_dcl_statements": { "arguments": [], - "created_at": 1719485736.576986, + "created_at": 1747731927.774532, "depends_on": { "macros": [ "macro.dbt.default__call_dcl_statements" @@ -583,7 +578,7 @@ }, "macro.dbt.can_clone_table": { "arguments": [], - "created_at": 1719485736.4429, + "created_at": 1747731927.67102, "depends_on": { "macros": [ "macro.dbt.default__can_clone_table" @@ -607,7 +602,7 @@ }, "macro.dbt.cast": { "arguments": [], - "created_at": 1719485736.542182, + "created_at": 1747731927.744382, "depends_on": { "macros": [ "macro.dbt.default__cast" @@ -631,7 +626,7 @@ }, "macro.dbt.cast_bool_to_text": { "arguments": [], - "created_at": 1719485736.541704, + "created_at": 1747731927.743805, "depends_on": { "macros": [ "macro.dbt.default__cast_bool_to_text" @@ -655,7 +650,7 @@ }, "macro.dbt.check_for_schema_changes": { "arguments": [], - "created_at": 1719485736.43984, + "created_at": 1747731927.667893, "depends_on": { "macros": [ "macro.dbt.diff_columns", @@ -680,7 +675,7 @@ }, "macro.dbt.check_schema_exists": { "arguments": [], - "created_at": 1719485736.591485, + "created_at": 1747731927.783475, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__check_schema_exists" @@ -702,9 +697,34 @@ "supported_languages": null, "unique_id": "macro.dbt.check_schema_exists" }, + "macro.dbt.check_time_data_types": { + "arguments": [], + "created_at": 1747731927.616284, + "depends_on": { + "macros": [ + "macro.dbt.get_updated_at_column_data_type", + "macro.dbt.get_snapshot_get_time_data_type" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro check_time_data_types(sql) %}\n {% set dbt_updated_at_data_type = get_updated_at_column_data_type(sql) %}\n {% set snapshot_get_time_data_type = get_snapshot_get_time_data_type() %}\n {% if snapshot_get_time_data_type is not none and dbt_updated_at_data_type is not none and snapshot_get_time_data_type != dbt_updated_at_data_type %}\n {% if exceptions.warn_snapshot_timestamp_data_types %}\n {{ exceptions.warn_snapshot_timestamp_data_types(snapshot_get_time_data_type, dbt_updated_at_data_type) }}\n {% endif %}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "check_time_data_types", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.check_time_data_types" + }, "macro.dbt.collect_freshness": { "arguments": [], - "created_at": 1719485736.569225, + "created_at": 1747731927.766824, "depends_on": { "macros": [ "macro.dbt.default__collect_freshness" @@ -726,9 +746,33 @@ "supported_languages": null, "unique_id": "macro.dbt.collect_freshness" }, + "macro.dbt.collect_freshness_custom_sql": { + "arguments": [], + "created_at": 1747731927.767657, + "depends_on": { + "macros": [ + "macro.dbt.default__collect_freshness_custom_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro collect_freshness_custom_sql(source, loaded_at_query) %}\n {{ return(adapter.dispatch('collect_freshness_custom_sql', 'dbt')(source, loaded_at_query))}}\n{% endmacro %}", + "meta": {}, + "name": "collect_freshness_custom_sql", + "original_file_path": "macros/adapters/freshness.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/freshness.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.collect_freshness_custom_sql" + }, "macro.dbt.concat": { "arguments": [], - "created_at": 1719485736.533403, + "created_at": 1747731927.73525, "depends_on": { "macros": [ "macro.dbt.default__concat" @@ -752,7 +796,7 @@ }, "macro.dbt.convert_datetime": { "arguments": [], - "created_at": 1719485736.522349, + "created_at": 1747731927.727519, "depends_on": { "macros": [] }, @@ -774,7 +818,7 @@ }, "macro.dbt.copy_grants": { "arguments": [], - "created_at": 1719485736.5725539, + "created_at": 1747731927.770463, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__copy_grants" @@ -798,7 +842,7 @@ }, "macro.dbt.create_columns": { "arguments": [], - "created_at": 1719485736.354676, + "created_at": 1747731927.607092, "depends_on": { "macros": [ "macro.dbt.default__create_columns" @@ -822,7 +866,7 @@ }, "macro.dbt.create_csv_table": { "arguments": [], - "created_at": 1719485736.45865, + "created_at": 1747731927.679952, "depends_on": { "macros": [ "macro.dbt.default__create_csv_table" @@ -846,7 +890,7 @@ }, "macro.dbt.create_indexes": { "arguments": [], - "created_at": 1719485736.561525, + "created_at": 1747731927.7599459, "depends_on": { "macros": [ "macro.dbt.default__create_indexes" @@ -870,7 +914,7 @@ }, "macro.dbt.create_or_replace_clone": { "arguments": [], - "created_at": 1719485736.443349, + "created_at": 1747731927.6715572, "depends_on": { "macros": [ "macro.dbt.default__create_or_replace_clone" @@ -894,7 +938,7 @@ }, "macro.dbt.create_or_replace_view": { "arguments": [], - "created_at": 1719485736.513715, + "created_at": 1747731927.718994, "depends_on": { "macros": [ "macro.dbt.run_hooks", @@ -924,7 +968,7 @@ }, "macro.dbt.create_schema": { "arguments": [], - "created_at": 1719485736.558099, + "created_at": 1747731927.7555032, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__create_schema" @@ -948,7 +992,7 @@ }, "macro.dbt.create_table_as": { "arguments": [], - "created_at": 1719485736.508673, + "created_at": 1747731927.713867, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__create_table_as" @@ -972,7 +1016,7 @@ }, "macro.dbt.create_view_as": { "arguments": [], - "created_at": 1719485736.515506, + "created_at": 1747731927.721162, "depends_on": { "macros": [ "macro.dbt.default__create_view_as" @@ -996,7 +1040,7 @@ }, "macro.dbt.current_timestamp": { "arguments": [], - "created_at": 1719485736.55921, + "created_at": 1747731927.7569978, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__current_timestamp" @@ -1020,7 +1064,7 @@ }, "macro.dbt.current_timestamp_backcompat": { "arguments": [], - "created_at": 1719485736.559824, + "created_at": 1747731927.758385, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__current_timestamp_backcompat" @@ -1044,7 +1088,7 @@ }, "macro.dbt.current_timestamp_in_utc_backcompat": { "arguments": [], - "created_at": 1719485736.5600908, + "created_at": 1747731927.758695, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat" @@ -1068,7 +1112,7 @@ }, "macro.dbt.date": { "arguments": [], - "created_at": 1719485736.531044, + "created_at": 1747731927.7339132, "depends_on": { "macros": [ "macro.dbt.default__date" @@ -1092,7 +1136,7 @@ }, "macro.dbt.date_spine": { "arguments": [], - "created_at": 1719485736.529869, + "created_at": 1747731927.7328708, "depends_on": { "macros": [ "macro.dbt.default__date_spine" @@ -1116,7 +1160,7 @@ }, "macro.dbt.date_trunc": { "arguments": [], - "created_at": 1719485736.55547, + "created_at": 1747731927.753514, "depends_on": { "macros": [ "macro.dbt.default__date_trunc" @@ -1140,7 +1184,7 @@ }, "macro.dbt.dateadd": { "arguments": [], - "created_at": 1719485736.5368361, + "created_at": 1747731927.7383602, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__dateadd" @@ -1164,7 +1208,7 @@ }, "macro.dbt.datediff": { "arguments": [], - "created_at": 1719485736.5402482, + "created_at": 1747731927.742094, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__datediff" @@ -1188,7 +1232,7 @@ }, "macro.dbt.dates_in_range": { "arguments": [], - "created_at": 1719485736.524234, + "created_at": 1747731927.7292109, "depends_on": { "macros": [ "macro.dbt.convert_datetime" @@ -1212,7 +1256,7 @@ }, "macro.dbt.default__alter_column_comment": { "arguments": [], - "created_at": 1719485736.5851688, + "created_at": 1747731927.778427, "depends_on": { "macros": [] }, @@ -1234,7 +1278,7 @@ }, "macro.dbt.default__alter_column_type": { "arguments": [], - "created_at": 1719485736.6003149, + "created_at": 1747731927.792504, "depends_on": { "macros": [ "macro.dbt.statement" @@ -1245,7 +1289,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", + "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation.render() }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation.render() }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation.render() }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation.render() }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "meta": {}, "name": "default__alter_column_type", "original_file_path": "macros/adapters/columns.sql", @@ -1258,7 +1302,7 @@ }, "macro.dbt.default__alter_relation_add_remove_columns": { "arguments": [], - "created_at": 1719485736.601542, + "created_at": 1747731927.793906, "depends_on": { "macros": [ "macro.dbt.run_query" @@ -1269,7 +1313,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", + "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation.render() }}\n\n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "meta": {}, "name": "default__alter_relation_add_remove_columns", "original_file_path": "macros/adapters/columns.sql", @@ -1282,7 +1326,7 @@ }, "macro.dbt.default__alter_relation_comment": { "arguments": [], - "created_at": 1719485736.585629, + "created_at": 1747731927.7789161, "depends_on": { "macros": [] }, @@ -1304,7 +1348,7 @@ }, "macro.dbt.default__any_value": { "arguments": [], - "created_at": 1719485736.5430348, + "created_at": 1747731927.745183, "depends_on": { "macros": [] }, @@ -1326,7 +1370,7 @@ }, "macro.dbt.default__apply_grants": { "arguments": [], - "created_at": 1719485736.580575, + "created_at": 1747731927.776596, "depends_on": { "macros": [ "macro.dbt.run_query", @@ -1340,7 +1384,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", + "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation.render() ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "meta": {}, "name": "default__apply_grants", "original_file_path": "macros/adapters/apply_grants.sql", @@ -1353,7 +1397,7 @@ }, "macro.dbt.default__array_append": { "arguments": [], - "created_at": 1719485736.557557, + "created_at": 1747731927.7550652, "depends_on": { "macros": [] }, @@ -1375,7 +1419,7 @@ }, "macro.dbt.default__array_concat": { "arguments": [], - "created_at": 1719485736.552268, + "created_at": 1747731927.7506618, "depends_on": { "macros": [] }, @@ -1397,7 +1441,7 @@ }, "macro.dbt.default__array_construct": { "arguments": [], - "created_at": 1719485736.556629, + "created_at": 1747731927.7545059, "depends_on": { "macros": [] }, @@ -1419,7 +1463,7 @@ }, "macro.dbt.default__bool_or": { "arguments": [], - "created_at": 1719485736.552653, + "created_at": 1747731927.751136, "depends_on": { "macros": [] }, @@ -1441,16 +1485,19 @@ }, "macro.dbt.default__build_snapshot_table": { "arguments": [], - "created_at": 1719485736.3577101, + "created_at": 1747731927.6143968, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.get_snapshot_table_column_names", + "macro.dbt.get_dbt_valid_to_current" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n\n select *,\n {{ strategy.scd_id }} as dbt_scd_id,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", + "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n\n select *,\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ get_dbt_valid_to_current(strategy, columns) }}\n {%- if strategy.hard_deletes == 'new_record' -%}\n , 'False' as {{ columns.dbt_is_deleted }}\n {% endif -%}\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "meta": {}, "name": "default__build_snapshot_table", "original_file_path": "macros/materializations/snapshots/helpers.sql", @@ -1463,7 +1510,7 @@ }, "macro.dbt.default__call_dcl_statements": { "arguments": [], - "created_at": 1719485736.577475, + "created_at": 1747731927.774842, "depends_on": { "macros": [ "macro.dbt.statement" @@ -1487,7 +1534,7 @@ }, "macro.dbt.default__can_clone_table": { "arguments": [], - "created_at": 1719485736.443028, + "created_at": 1747731927.671176, "depends_on": { "macros": [] }, @@ -1509,7 +1556,7 @@ }, "macro.dbt.default__cast": { "arguments": [], - "created_at": 1719485736.5423732, + "created_at": 1747731927.744554, "depends_on": { "macros": [] }, @@ -1531,7 +1578,7 @@ }, "macro.dbt.default__cast_bool_to_text": { "arguments": [], - "created_at": 1719485736.541875, + "created_at": 1747731927.744014, "depends_on": { "macros": [] }, @@ -1553,7 +1600,7 @@ }, "macro.dbt.default__check_schema_exists": { "arguments": [], - "created_at": 1719485736.591805, + "created_at": 1747731927.7838552, "depends_on": { "macros": [ "macro.dbt.replace", @@ -1578,7 +1625,7 @@ }, "macro.dbt.default__collect_freshness": { "arguments": [], - "created_at": 1719485736.569667, + "created_at": 1747731927.767362, "depends_on": { "macros": [ "macro.dbt.statement", @@ -1601,9 +1648,34 @@ "supported_languages": null, "unique_id": "macro.dbt.default__collect_freshness" }, + "macro.dbt.default__collect_freshness_custom_sql": { + "arguments": [], + "created_at": 1747731927.768078, + "depends_on": { + "macros": [ + "macro.dbt.statement", + "macro.dbt.current_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__collect_freshness_custom_sql(source, loaded_at_query) %}\n {% call statement('collect_freshness_custom_sql', fetch_result=True, auto_begin=False) -%}\n with source_query as (\n {{ loaded_at_query }}\n )\n select\n (select * from source_query) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n {% endcall %}\n {{ return(load_result('collect_freshness_custom_sql')) }}\n{% endmacro %}", + "meta": {}, + "name": "default__collect_freshness_custom_sql", + "original_file_path": "macros/adapters/freshness.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/freshness.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__collect_freshness_custom_sql" + }, "macro.dbt.default__concat": { "arguments": [], - "created_at": 1719485736.5335412, + "created_at": 1747731927.735413, "depends_on": { "macros": [] }, @@ -1625,7 +1697,7 @@ }, "macro.dbt.default__copy_grants": { "arguments": [], - "created_at": 1719485736.5727968, + "created_at": 1747731927.770632, "depends_on": { "macros": [] }, @@ -1647,7 +1719,7 @@ }, "macro.dbt.default__create_columns": { "arguments": [], - "created_at": 1719485736.354986, + "created_at": 1747731927.6075032, "depends_on": { "macros": [ "macro.dbt.statement" @@ -1658,7 +1730,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", + "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation.render() }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "meta": {}, "name": "default__create_columns", "original_file_path": "macros/materializations/snapshots/helpers.sql", @@ -1671,7 +1743,7 @@ }, "macro.dbt.default__create_csv_table": { "arguments": [], - "created_at": 1719485736.4600089, + "created_at": 1747731927.6811981, "depends_on": { "macros": [ "macro.dbt.statement" @@ -1695,7 +1767,7 @@ }, "macro.dbt.default__create_indexes": { "arguments": [], - "created_at": 1719485736.562071, + "created_at": 1747731927.760452, "depends_on": { "macros": [ "macro.dbt.get_create_index_sql", @@ -1720,7 +1792,7 @@ }, "macro.dbt.default__create_or_replace_clone": { "arguments": [], - "created_at": 1719485736.4434948, + "created_at": 1747731927.671776, "depends_on": { "macros": [] }, @@ -1729,7 +1801,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation }} clone {{ defer_relation }}\n{% endmacro %}", + "macro_sql": "{% macro default__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation.render() }} clone {{ defer_relation.render() }}\n{% endmacro %}", "meta": {}, "name": "default__create_or_replace_clone", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", @@ -1742,7 +1814,7 @@ }, "macro.dbt.default__create_schema": { "arguments": [], - "created_at": 1719485736.5583, + "created_at": 1747731927.755875, "depends_on": { "macros": [ "macro.dbt.statement" @@ -1766,7 +1838,7 @@ }, "macro.dbt.default__create_table_as": { "arguments": [], - "created_at": 1719485736.509567, + "created_at": 1747731927.714765, "depends_on": { "macros": [ "macro.dbt.get_assert_columns_equivalent", @@ -1792,7 +1864,7 @@ }, "macro.dbt.default__create_view_as": { "arguments": [], - "created_at": 1719485736.515955, + "created_at": 1747731927.721749, "depends_on": { "macros": [ "macro.dbt.get_assert_columns_equivalent" @@ -1803,7 +1875,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", + "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation.render() }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "meta": {}, "name": "default__create_view_as", "original_file_path": "macros/relations/view/create.sql", @@ -1816,7 +1888,7 @@ }, "macro.dbt.default__current_timestamp": { "arguments": [], - "created_at": 1719485736.5593758, + "created_at": 1747731927.757301, "depends_on": { "macros": [] }, @@ -1838,7 +1910,7 @@ }, "macro.dbt.default__current_timestamp_backcompat": { "arguments": [], - "created_at": 1719485736.559911, + "created_at": 1747731927.758484, "depends_on": { "macros": [] }, @@ -1860,7 +1932,7 @@ }, "macro.dbt.default__current_timestamp_in_utc_backcompat": { "arguments": [], - "created_at": 1719485736.560272, + "created_at": 1747731927.7589061, "depends_on": { "macros": [ "macro.dbt.current_timestamp_backcompat", @@ -1885,7 +1957,7 @@ }, "macro.dbt.default__date": { "arguments": [], - "created_at": 1719485736.531544, + "created_at": 1747731927.734285, "depends_on": { "macros": [] }, @@ -1907,7 +1979,7 @@ }, "macro.dbt.default__date_spine": { "arguments": [], - "created_at": 1719485736.53054, + "created_at": 1747731927.733346, "depends_on": { "macros": [ "macro.dbt.generate_series", @@ -1933,7 +2005,7 @@ }, "macro.dbt.default__date_trunc": { "arguments": [], - "created_at": 1719485736.555618, + "created_at": 1747731927.753688, "depends_on": { "macros": [] }, @@ -1955,7 +2027,7 @@ }, "macro.dbt.default__dateadd": { "arguments": [], - "created_at": 1719485736.537018, + "created_at": 1747731927.738684, "depends_on": { "macros": [] }, @@ -1977,7 +2049,7 @@ }, "macro.dbt.default__datediff": { "arguments": [], - "created_at": 1719485736.540429, + "created_at": 1747731927.742321, "depends_on": { "macros": [] }, @@ -1999,7 +2071,7 @@ }, "macro.dbt.default__drop_materialized_view": { "arguments": [], - "created_at": 1719485736.4865391, + "created_at": 1747731927.701109, "depends_on": { "macros": [] }, @@ -2008,7 +2080,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation }} cascade\n{%- endmacro %}", + "macro_sql": "{% macro default__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "meta": {}, "name": "default__drop_materialized_view", "original_file_path": "macros/relations/materialized_view/drop.sql", @@ -2021,7 +2093,7 @@ }, "macro.dbt.default__drop_relation": { "arguments": [], - "created_at": 1719485736.4714549, + "created_at": 1747731927.690887, "depends_on": { "macros": [ "macro.dbt.statement", @@ -2046,7 +2118,7 @@ }, "macro.dbt.default__drop_schema": { "arguments": [], - "created_at": 1719485736.558672, + "created_at": 1747731927.756342, "depends_on": { "macros": [ "macro.dbt.statement" @@ -2070,7 +2142,7 @@ }, "macro.dbt.default__drop_schema_named": { "arguments": [], - "created_at": 1719485736.476362, + "created_at": 1747731927.695164, "depends_on": { "macros": [] }, @@ -2092,7 +2164,7 @@ }, "macro.dbt.default__drop_table": { "arguments": [], - "created_at": 1719485736.505599, + "created_at": 1747731927.711045, "depends_on": { "macros": [] }, @@ -2101,7 +2173,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__drop_table(relation) -%}\n drop table if exists {{ relation }} cascade\n{%- endmacro %}", + "macro_sql": "{% macro default__drop_table(relation) -%}\n drop table if exists {{ relation.render() }} cascade\n{%- endmacro %}", "meta": {}, "name": "default__drop_table", "original_file_path": "macros/relations/table/drop.sql", @@ -2114,7 +2186,7 @@ }, "macro.dbt.default__drop_view": { "arguments": [], - "created_at": 1719485736.511198, + "created_at": 1747731927.716324, "depends_on": { "macros": [] }, @@ -2123,7 +2195,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__drop_view(relation) -%}\n drop view if exists {{ relation }} cascade\n{%- endmacro %}", + "macro_sql": "{% macro default__drop_view(relation) -%}\n drop view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "meta": {}, "name": "default__drop_view", "original_file_path": "macros/relations/view/drop.sql", @@ -2136,7 +2208,7 @@ }, "macro.dbt.default__escape_single_quotes": { "arguments": [], - "created_at": 1719485736.538025, + "created_at": 1747731927.739678, "depends_on": { "macros": [] }, @@ -2158,7 +2230,7 @@ }, "macro.dbt.default__except": { "arguments": [], - "created_at": 1719485736.526438, + "created_at": 1747731927.730975, "depends_on": { "macros": [] }, @@ -2180,7 +2252,7 @@ }, "macro.dbt.default__format_column": { "arguments": [], - "created_at": 1719485736.504833, + "created_at": 1747731927.710492, "depends_on": { "macros": [] }, @@ -2202,7 +2274,7 @@ }, "macro.dbt.default__generate_alias_name": { "arguments": [], - "created_at": 1719485736.466625, + "created_at": 1747731927.68687, "depends_on": { "macros": [] }, @@ -2224,7 +2296,7 @@ }, "macro.dbt.default__generate_database_name": { "arguments": [], - "created_at": 1719485736.469049, + "created_at": 1747731927.689066, "depends_on": { "macros": [] }, @@ -2246,7 +2318,7 @@ }, "macro.dbt.default__generate_schema_name": { "arguments": [], - "created_at": 1719485736.4677172, + "created_at": 1747731927.687853, "depends_on": { "macros": [] }, @@ -2268,7 +2340,7 @@ }, "macro.dbt.default__generate_series": { "arguments": [], - "created_at": 1719485736.535907, + "created_at": 1747731927.7374568, "depends_on": { "macros": [ "macro.dbt.get_powers_of_two" @@ -2292,7 +2364,7 @@ }, "macro.dbt.default__get_alter_materialized_view_as_sql": { "arguments": [], - "created_at": 1719485736.4936142, + "created_at": 1747731927.7040021, "depends_on": { "macros": [] }, @@ -2314,7 +2386,7 @@ }, "macro.dbt.default__get_assert_columns_equivalent": { "arguments": [], - "created_at": 1719485736.501285, + "created_at": 1747731927.7077289, "depends_on": { "macros": [ "macro.dbt.assert_columns_equivalent" @@ -2338,7 +2410,7 @@ }, "macro.dbt.default__get_batch_size": { "arguments": [], - "created_at": 1719485736.4618182, + "created_at": 1747731927.6832871, "depends_on": { "macros": [] }, @@ -2360,7 +2432,7 @@ }, "macro.dbt.default__get_binding_char": { "arguments": [], - "created_at": 1719485736.461474, + "created_at": 1747731927.682925, "depends_on": { "macros": [] }, @@ -2382,7 +2454,7 @@ }, "macro.dbt.default__get_catalog": { "arguments": [], - "created_at": 1719485736.590454, + "created_at": 1747731927.7822511, "depends_on": { "macros": [] }, @@ -2402,9 +2474,31 @@ "supported_languages": null, "unique_id": "macro.dbt.default__get_catalog" }, + "macro.dbt.default__get_catalog_for_single_relation": { + "arguments": [], + "created_at": 1747731927.784731, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_catalog_for_single_relation(relation) %}\n {{ exceptions.raise_not_implemented(\n 'get_catalog_for_single_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_catalog_for_single_relation", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_catalog_for_single_relation" + }, "macro.dbt.default__get_catalog_relations": { "arguments": [], - "created_at": 1719485736.589964, + "created_at": 1747731927.781642, "depends_on": { "macros": [] }, @@ -2426,7 +2520,7 @@ }, "macro.dbt.default__get_column_names": { "arguments": [], - "created_at": 1719485736.5100741, + "created_at": 1747731927.715374, "depends_on": { "macros": [] }, @@ -2448,7 +2542,7 @@ }, "macro.dbt.default__get_columns_in_query": { "arguments": [], - "created_at": 1719485736.599448, + "created_at": 1747731927.79138, "depends_on": { "macros": [ "macro.dbt.statement", @@ -2473,7 +2567,7 @@ }, "macro.dbt.default__get_columns_in_relation": { "arguments": [], - "created_at": 1719485736.59628, + "created_at": 1747731927.7875772, "depends_on": { "macros": [] }, @@ -2495,7 +2589,7 @@ }, "macro.dbt.default__get_create_backup_sql": { "arguments": [], - "created_at": 1719485736.47981, + "created_at": 1747731927.698654, "depends_on": { "macros": [ "macro.dbt.make_backup_relation", @@ -2521,7 +2615,7 @@ }, "macro.dbt.default__get_create_index_sql": { "arguments": [], - "created_at": 1719485736.561354, + "created_at": 1747731927.7597451, "depends_on": { "macros": [] }, @@ -2543,7 +2637,7 @@ }, "macro.dbt.default__get_create_intermediate_sql": { "arguments": [], - "created_at": 1719485736.475838, + "created_at": 1747731927.694537, "depends_on": { "macros": [ "macro.dbt.make_intermediate_relation", @@ -2569,7 +2663,7 @@ }, "macro.dbt.default__get_create_materialized_view_as_sql": { "arguments": [], - "created_at": 1719485736.494792, + "created_at": 1747731927.7051551, "depends_on": { "macros": [] }, @@ -2591,7 +2685,7 @@ }, "macro.dbt.default__get_create_sql": { "arguments": [], - "created_at": 1719485736.483476, + "created_at": 1747731927.699806, "depends_on": { "macros": [ "macro.dbt.get_create_view_as_sql", @@ -2617,7 +2711,7 @@ }, "macro.dbt.default__get_create_table_as_sql": { "arguments": [], - "created_at": 1719485736.508211, + "created_at": 1747731927.713313, "depends_on": { "macros": [ "macro.dbt.create_table_as" @@ -2641,7 +2735,7 @@ }, "macro.dbt.default__get_create_view_as_sql": { "arguments": [], - "created_at": 1719485736.515317, + "created_at": 1747731927.720923, "depends_on": { "macros": [ "macro.dbt.create_view_as" @@ -2665,7 +2759,7 @@ }, "macro.dbt.default__get_csv_sql": { "arguments": [], - "created_at": 1719485736.461133, + "created_at": 1747731927.682581, "depends_on": { "macros": [] }, @@ -2687,7 +2781,7 @@ }, "macro.dbt.default__get_dcl_statement_list": { "arguments": [], - "created_at": 1719485736.5756311, + "created_at": 1747731927.774283, "depends_on": { "macros": [ "macro.dbt.support_multiple_grantees_per_dcl_statement" @@ -2711,7 +2805,7 @@ }, "macro.dbt.default__get_delete_insert_merge_sql": { "arguments": [], - "created_at": 1719485736.415282, + "created_at": 1747731927.652731, "depends_on": { "macros": [ "macro.dbt.get_quoted_csv" @@ -2735,7 +2829,7 @@ }, "macro.dbt.default__get_drop_backup_sql": { "arguments": [], - "created_at": 1719485736.476908, + "created_at": 1747731927.695827, "depends_on": { "macros": [ "macro.dbt.make_backup_relation", @@ -2760,7 +2854,7 @@ }, "macro.dbt.default__get_drop_index_sql": { "arguments": [], - "created_at": 1719485736.562772, + "created_at": 1747731927.760863, "depends_on": { "macros": [] }, @@ -2782,7 +2876,7 @@ }, "macro.dbt.default__get_drop_sql": { "arguments": [], - "created_at": 1719485736.471034, + "created_at": 1747731927.6902988, "depends_on": { "macros": [ "macro.dbt.drop_view", @@ -2795,7 +2889,7 @@ "node_color": null, "show": true }, - "macro_sql": "{%- macro default__get_drop_sql(relation) -%}\n\n {%- if relation.is_view -%}\n {{ drop_view(relation) }}\n\n {%- elif relation.is_table -%}\n {{ drop_table(relation) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n\n {%- else -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", + "macro_sql": "{%- macro default__get_drop_sql(relation) -%}\n\n {%- if relation.is_view -%}\n {{ drop_view(relation) }}\n\n {%- elif relation.is_table -%}\n {{ drop_table(relation) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n\n {%- else -%}\n drop {{ relation.type }} if exists {{ relation.render() }} cascade\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "meta": {}, "name": "default__get_drop_sql", "original_file_path": "macros/relations/drop.sql", @@ -2808,7 +2902,7 @@ }, "macro.dbt.default__get_empty_schema_sql": { "arguments": [], - "created_at": 1719485736.5985012, + "created_at": 1747731927.790267, "depends_on": { "macros": [ "macro.dbt.cast" @@ -2832,7 +2926,7 @@ }, "macro.dbt.default__get_empty_subquery_sql": { "arguments": [], - "created_at": 1719485736.597066, + "created_at": 1747731927.788509, "depends_on": { "macros": [] }, @@ -2854,7 +2948,7 @@ }, "macro.dbt.default__get_grant_sql": { "arguments": [], - "created_at": 1719485736.5742211, + "created_at": 1747731927.772546, "depends_on": { "macros": [] }, @@ -2863,7 +2957,7 @@ "node_color": null, "show": true }, - "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", + "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation.render() }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "meta": {}, "name": "default__get_grant_sql", "original_file_path": "macros/adapters/apply_grants.sql", @@ -2876,7 +2970,7 @@ }, "macro.dbt.default__get_incremental_append_sql": { "arguments": [], - "created_at": 1719485736.418623, + "created_at": 1747731927.656011, "depends_on": { "macros": [ "macro.dbt.get_insert_into_sql" @@ -2900,7 +2994,7 @@ }, "macro.dbt.default__get_incremental_default_sql": { "arguments": [], - "created_at": 1719485736.4215739, + "created_at": 1747731927.65827, "depends_on": { "macros": [ "macro.dbt.get_incremental_append_sql" @@ -2924,7 +3018,7 @@ }, "macro.dbt.default__get_incremental_delete_insert_sql": { "arguments": [], - "created_at": 1719485736.419171, + "created_at": 1747731927.656642, "depends_on": { "macros": [ "macro.dbt.get_delete_insert_merge_sql" @@ -2948,7 +3042,7 @@ }, "macro.dbt.default__get_incremental_insert_overwrite_sql": { "arguments": [], - "created_at": 1719485736.421212, + "created_at": 1747731927.6578429, "depends_on": { "macros": [ "macro.dbt.get_insert_overwrite_merge_sql" @@ -2972,7 +3066,7 @@ }, "macro.dbt.default__get_incremental_merge_sql": { "arguments": [], - "created_at": 1719485736.420272, + "created_at": 1747731927.657262, "depends_on": { "macros": [ "macro.dbt.get_merge_sql" @@ -2994,9 +3088,31 @@ "supported_languages": null, "unique_id": "macro.dbt.default__get_incremental_merge_sql" }, + "macro.dbt.default__get_incremental_microbatch_sql": { + "arguments": [], + "created_at": 1747731927.658707, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_incremental_microbatch_sql(arg_dict) %}\n\n {{ exceptions.raise_not_implemented('microbatch materialization strategy not implemented for adapter ' + adapter.type()) }}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_incremental_microbatch_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_incremental_microbatch_sql" + }, "macro.dbt.default__get_insert_overwrite_merge_sql": { "arguments": [], - "created_at": 1719485736.416304, + "created_at": 1747731927.6539059, "depends_on": { "macros": [ "macro.dbt.get_quoted_csv" @@ -3020,7 +3136,7 @@ }, "macro.dbt.default__get_intervals_between": { "arguments": [], - "created_at": 1719485736.5290911, + "created_at": 1747731927.732573, "depends_on": { "macros": [ "macro.dbt.statement", @@ -3043,9 +3159,9 @@ "supported_languages": null, "unique_id": "macro.dbt.default__get_intervals_between" }, - "macro.dbt.default__get_limit_subquery_sql": { + "macro.dbt.default__get_limit_sql": { "arguments": [], - "created_at": 1719485736.581942, + "created_at": 1747731927.777602, "depends_on": { "macros": [] }, @@ -3054,20 +3170,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__get_limit_subquery_sql(sql, limit) %}\n select *\n from (\n {{ sql }}\n ) as model_limit_subq\n limit {{ limit }}\n{% endmacro %}", + "macro_sql": "{% macro default__get_limit_sql(sql, limit) %}\n {{ sql }}\n {% if limit is not none %}\n limit {{ limit }}\n {%- endif -%}\n{% endmacro %}", "meta": {}, - "name": "default__get_limit_subquery_sql", + "name": "default__get_limit_sql", "original_file_path": "macros/adapters/show.sql", "package_name": "dbt", "patch_path": null, "path": "macros/adapters/show.sql", "resource_type": "macro", "supported_languages": null, - "unique_id": "macro.dbt.default__get_limit_subquery_sql" + "unique_id": "macro.dbt.default__get_limit_sql" }, "macro.dbt.default__get_materialized_view_configuration_changes": { "arguments": [], - "created_at": 1719485736.4943428, + "created_at": 1747731927.704607, "depends_on": { "macros": [] }, @@ -3089,7 +3205,7 @@ }, "macro.dbt.default__get_merge_sql": { "arguments": [], - "created_at": 1719485736.413476, + "created_at": 1747731927.6510909, "depends_on": { "macros": [ "macro.dbt.get_quoted_csv", @@ -3114,7 +3230,7 @@ }, "macro.dbt.default__get_merge_update_columns": { "arguments": [], - "created_at": 1719485736.399054, + "created_at": 1747731927.646931, "depends_on": { "macros": [] }, @@ -3136,7 +3252,7 @@ }, "macro.dbt.default__get_or_create_relation": { "arguments": [], - "created_at": 1719485736.568331, + "created_at": 1747731927.7655392, "depends_on": { "macros": [] }, @@ -3158,7 +3274,7 @@ }, "macro.dbt.default__get_powers_of_two": { "arguments": [], - "created_at": 1719485736.534994, + "created_at": 1747731927.736556, "depends_on": { "macros": [] }, @@ -3180,7 +3296,7 @@ }, "macro.dbt.default__get_relation_last_modified": { "arguments": [], - "created_at": 1719485736.5935519, + "created_at": 1747731927.785716, "depends_on": { "macros": [] }, @@ -3202,7 +3318,7 @@ }, "macro.dbt.default__get_relations": { "arguments": [], - "created_at": 1719485736.592618, + "created_at": 1747731927.785134, "depends_on": { "macros": [] }, @@ -3224,7 +3340,7 @@ }, "macro.dbt.default__get_rename_intermediate_sql": { "arguments": [], - "created_at": 1719485736.485399, + "created_at": 1747731927.700591, "depends_on": { "macros": [ "macro.dbt.make_intermediate_relation", @@ -3249,7 +3365,7 @@ }, "macro.dbt.default__get_rename_materialized_view_sql": { "arguments": [], - "created_at": 1719485736.491203, + "created_at": 1747731927.702991, "depends_on": { "macros": [] }, @@ -3271,7 +3387,7 @@ }, "macro.dbt.default__get_rename_sql": { "arguments": [], - "created_at": 1719485736.4781098, + "created_at": 1747731927.6970391, "depends_on": { "macros": [ "macro.dbt.get_rename_view_sql", @@ -3297,7 +3413,7 @@ }, "macro.dbt.default__get_rename_table_sql": { "arguments": [], - "created_at": 1719485736.507071, + "created_at": 1747731927.712139, "depends_on": { "macros": [] }, @@ -3319,7 +3435,7 @@ }, "macro.dbt.default__get_rename_view_sql": { "arguments": [], - "created_at": 1719485736.514697, + "created_at": 1747731927.72008, "depends_on": { "macros": [] }, @@ -3341,7 +3457,7 @@ }, "macro.dbt.default__get_replace_materialized_view_sql": { "arguments": [], - "created_at": 1719485736.4885828, + "created_at": 1747731927.7017732, "depends_on": { "macros": [] }, @@ -3363,7 +3479,7 @@ }, "macro.dbt.default__get_replace_sql": { "arguments": [], - "created_at": 1719485736.475151, + "created_at": 1747731927.693742, "depends_on": { "macros": [ "macro.dbt.get_replace_view_sql", @@ -3382,7 +3498,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__get_replace_sql(existing_relation, target_relation, sql) %}\n\n {# /* use a create or replace statement if possible */ #}\n\n {% set is_replaceable = existing_relation.type == target_relation_type and existing_relation.can_be_replaced %}\n\n {% if is_replaceable and existing_relation.is_view %}\n {{ get_replace_view_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_table %}\n {{ get_replace_table_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_materialized_view %}\n {{ get_replace_materialized_view_sql(target_relation, sql) }}\n\n {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}\n {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}\n {%- elif target_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_drop_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }}\n\n {# /* create target_relation in place by first backing up the existing relation */ #}\n {%- elif existing_relation.can_be_renamed -%}\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* no renaming is allowed, so just drop and create */ #}\n {%- else -%}\n {{ get_drop_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }}\n\n {%- endif -%}\n\n{% endmacro %}", + "macro_sql": "{% macro default__get_replace_sql(existing_relation, target_relation, sql) %}\n\n {# /* use a create or replace statement if possible */ #}\n\n {% set is_replaceable = existing_relation.type == target_relation.type and existing_relation.can_be_replaced %}\n\n {% if is_replaceable and existing_relation.is_view %}\n {{ get_replace_view_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_table %}\n {{ get_replace_table_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_materialized_view %}\n {{ get_replace_materialized_view_sql(target_relation, sql) }}\n\n {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}\n {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}\n {%- elif target_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_drop_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }}\n\n {# /* create target_relation in place by first backing up the existing relation */ #}\n {%- elif existing_relation.can_be_renamed -%}\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* no renaming is allowed, so just drop and create */ #}\n {%- else -%}\n {{ get_drop_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }}\n\n {%- endif -%}\n\n{% endmacro %}", "meta": {}, "name": "default__get_replace_sql", "original_file_path": "macros/relations/replace.sql", @@ -3395,7 +3511,7 @@ }, "macro.dbt.default__get_replace_table_sql": { "arguments": [], - "created_at": 1719485736.506342, + "created_at": 1747731927.7115898, "depends_on": { "macros": [] }, @@ -3417,7 +3533,7 @@ }, "macro.dbt.default__get_replace_view_sql": { "arguments": [], - "created_at": 1719485736.512308, + "created_at": 1747731927.71724, "depends_on": { "macros": [] }, @@ -3439,7 +3555,7 @@ }, "macro.dbt.default__get_revoke_sql": { "arguments": [], - "created_at": 1719485736.574667, + "created_at": 1747731927.773101, "depends_on": { "macros": [] }, @@ -3448,7 +3564,7 @@ "node_color": null, "show": true }, - "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", + "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation.render() }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "meta": {}, "name": "default__get_revoke_sql", "original_file_path": "macros/adapters/apply_grants.sql", @@ -3461,7 +3577,7 @@ }, "macro.dbt.default__get_select_subquery": { "arguments": [], - "created_at": 1719485736.510467, + "created_at": 1747731927.715831, "depends_on": { "macros": [ "macro.dbt.default__get_column_names" @@ -3485,7 +3601,7 @@ }, "macro.dbt.default__get_show_grant_sql": { "arguments": [], - "created_at": 1719485736.573777, + "created_at": 1747731927.771996, "depends_on": { "macros": [] }, @@ -3494,7 +3610,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation }}\n{% endmacro %}", + "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation.render() }}\n{% endmacro %}", "meta": {}, "name": "default__get_show_grant_sql", "original_file_path": "macros/adapters/apply_grants.sql", @@ -3507,7 +3623,7 @@ }, "macro.dbt.default__get_show_indexes_sql": { "arguments": [], - "created_at": 1719485736.563092, + "created_at": 1747731927.761222, "depends_on": { "macros": [] }, @@ -3529,7 +3645,7 @@ }, "macro.dbt.default__get_table_columns_and_constraints": { "arguments": [], - "created_at": 1719485736.497637, + "created_at": 1747731927.706486, "depends_on": { "macros": [ "macro.dbt.table_columns_and_constraints" @@ -3553,7 +3669,7 @@ }, "macro.dbt.default__get_test_sql": { "arguments": [], - "created_at": 1719485736.372639, + "created_at": 1747731927.627583, "depends_on": { "macros": [] }, @@ -3575,7 +3691,7 @@ }, "macro.dbt.default__get_true_sql": { "arguments": [], - "created_at": 1719485736.3555431, + "created_at": 1747731927.608193, "depends_on": { "macros": [] }, @@ -3597,7 +3713,7 @@ }, "macro.dbt.default__get_unit_test_sql": { "arguments": [], - "created_at": 1719485736.3736708, + "created_at": 1747731927.6286259, "depends_on": { "macros": [ "macro.dbt.string_literal" @@ -3621,7 +3737,7 @@ }, "macro.dbt.default__get_where_subquery": { "arguments": [], - "created_at": 1719485736.374502, + "created_at": 1747731927.629546, "depends_on": { "macros": [] }, @@ -3643,7 +3759,7 @@ }, "macro.dbt.default__handle_existing_table": { "arguments": [], - "created_at": 1719485736.514176, + "created_at": 1747731927.719549, "depends_on": { "macros": [] }, @@ -3652,7 +3768,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", + "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation.render() ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "meta": {}, "name": "default__handle_existing_table", "original_file_path": "macros/relations/view/replace.sql", @@ -3665,7 +3781,7 @@ }, "macro.dbt.default__hash": { "arguments": [], - "created_at": 1719485736.541434, + "created_at": 1747731927.743469, "depends_on": { "macros": [] }, @@ -3687,7 +3803,7 @@ }, "macro.dbt.default__information_schema_name": { "arguments": [], - "created_at": 1719485736.590823, + "created_at": 1747731927.782689, "depends_on": { "macros": [] }, @@ -3709,7 +3825,7 @@ }, "macro.dbt.default__intersect": { "arguments": [], - "created_at": 1719485736.537451, + "created_at": 1747731927.7391071, "depends_on": { "macros": [] }, @@ -3731,7 +3847,7 @@ }, "macro.dbt.default__last_day": { "arguments": [], - "created_at": 1719485736.55349, + "created_at": 1747731927.752128, "depends_on": { "macros": [ "macro.dbt.default_last_day" @@ -3755,7 +3871,7 @@ }, "macro.dbt.default__length": { "arguments": [], - "created_at": 1719485736.536426, + "created_at": 1747731927.737946, "depends_on": { "macros": [] }, @@ -3777,7 +3893,7 @@ }, "macro.dbt.default__list_relations_without_caching": { "arguments": [], - "created_at": 1719485736.592287, + "created_at": 1747731927.784299, "depends_on": { "macros": [] }, @@ -3799,7 +3915,7 @@ }, "macro.dbt.default__list_schemas": { "arguments": [], - "created_at": 1719485736.591269, + "created_at": 1747731927.783221, "depends_on": { "macros": [ "macro.dbt.information_schema_name", @@ -3824,7 +3940,7 @@ }, "macro.dbt.default__listagg": { "arguments": [], - "created_at": 1719485736.539655, + "created_at": 1747731927.7416582, "depends_on": { "macros": [] }, @@ -3846,7 +3962,7 @@ }, "macro.dbt.default__load_csv_rows": { "arguments": [], - "created_at": 1719485736.465408, + "created_at": 1747731927.6858292, "depends_on": { "macros": [ "macro.dbt.get_batch_size", @@ -3872,7 +3988,7 @@ }, "macro.dbt.default__make_backup_relation": { "arguments": [], - "created_at": 1719485736.5670989, + "created_at": 1747731927.7640681, "depends_on": { "macros": [] }, @@ -3894,7 +4010,7 @@ }, "macro.dbt.default__make_intermediate_relation": { "arguments": [], - "created_at": 1719485736.56591, + "created_at": 1747731927.762398, "depends_on": { "macros": [ "macro.dbt.default__make_temp_relation" @@ -3918,7 +4034,7 @@ }, "macro.dbt.default__make_temp_relation": { "arguments": [], - "created_at": 1719485736.566462, + "created_at": 1747731927.763351, "depends_on": { "macros": [] }, @@ -3940,7 +4056,7 @@ }, "macro.dbt.default__persist_docs": { "arguments": [], - "created_at": 1719485736.586432, + "created_at": 1747731927.779957, "depends_on": { "macros": [ "macro.dbt.run_query", @@ -3966,7 +4082,7 @@ }, "macro.dbt.default__position": { "arguments": [], - "created_at": 1719485736.543534, + "created_at": 1747731927.745738, "depends_on": { "macros": [] }, @@ -3988,7 +4104,7 @@ }, "macro.dbt.default__post_snapshot": { "arguments": [], - "created_at": 1719485736.3552608, + "created_at": 1747731927.6078432, "depends_on": { "macros": [] }, @@ -4010,7 +4126,7 @@ }, "macro.dbt.default__refresh_materialized_view": { "arguments": [], - "created_at": 1719485736.49013, + "created_at": 1747731927.7024262, "depends_on": { "macros": [] }, @@ -4032,7 +4148,7 @@ }, "macro.dbt.default__rename_relation": { "arguments": [], - "created_at": 1719485736.4786189, + "created_at": 1747731927.6977248, "depends_on": { "macros": [ "macro.dbt.statement" @@ -4043,7 +4159,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", + "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation.render() }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "meta": {}, "name": "default__rename_relation", "original_file_path": "macros/relations/rename.sql", @@ -4056,7 +4172,7 @@ }, "macro.dbt.default__replace": { "arguments": [], - "created_at": 1719485736.533091, + "created_at": 1747731927.734913, "depends_on": { "macros": [] }, @@ -4078,7 +4194,7 @@ }, "macro.dbt.default__reset_csv_table": { "arguments": [], - "created_at": 1719485736.460783, + "created_at": 1747731927.682154, "depends_on": { "macros": [ "macro.dbt.create_csv_table" @@ -4089,7 +4205,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", + "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation.render() %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "meta": {}, "name": "default__reset_csv_table", "original_file_path": "macros/materializations/seeds/helpers.sql", @@ -4102,7 +4218,7 @@ }, "macro.dbt.default__resolve_model_name": { "arguments": [], - "created_at": 1719485736.615371, + "created_at": 1747731927.8007932, "depends_on": { "macros": [] }, @@ -4124,7 +4240,7 @@ }, "macro.dbt.default__right": { "arguments": [], - "created_at": 1719485736.538501, + "created_at": 1747731927.740396, "depends_on": { "macros": [] }, @@ -4146,7 +4262,7 @@ }, "macro.dbt.default__safe_cast": { "arguments": [], - "created_at": 1719485736.540907, + "created_at": 1747731927.742908, "depends_on": { "macros": [] }, @@ -4168,7 +4284,7 @@ }, "macro.dbt.default__snapshot_get_time": { "arguments": [], - "created_at": 1719485736.559646, + "created_at": 1747731927.757626, "depends_on": { "macros": [ "macro.dbt.current_timestamp" @@ -4192,7 +4308,7 @@ }, "macro.dbt.default__snapshot_hash_arguments": { "arguments": [], - "created_at": 1719485736.345648, + "created_at": 1747731927.5983949, "depends_on": { "macros": [] }, @@ -4214,16 +4330,18 @@ }, "macro.dbt.default__snapshot_merge_sql": { "arguments": [], - "created_at": 1719485736.340331, + "created_at": 1747731927.5943701, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.get_snapshot_table_column_names" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", + "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n {%- set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() -%}\n\n merge into {{ target.render() }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}\n\n when matched\n {% if config.get(\"dbt_valid_to_current\") %}\n and (DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or\n DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null)\n {% else %}\n and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null\n {% endif %}\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "meta": {}, "name": "default__snapshot_merge_sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", @@ -4236,10 +4354,18 @@ }, "macro.dbt.default__snapshot_staging_table": { "arguments": [], - "created_at": 1719485736.356719, + "created_at": 1747731927.613434, "depends_on": { "macros": [ - "macro.dbt.snapshot_get_time" + "macro.dbt.get_snapshot_table_column_names", + "macro.dbt.snapshot_hash_arguments", + "macro.dbt.snapshot_get_time", + "macro.dbt.unique_key_fields", + "macro.dbt.get_dbt_valid_to_current", + "macro.dbt.unique_key_join_on", + "macro.dbt.unique_key_is_null", + "macro.dbt.unique_key_is_not_null", + "macro.dbt.get_column_schema_from_query" ] }, "description": "", @@ -4247,7 +4373,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n\n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", + "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n {% if strategy.hard_deletes == 'new_record' %}\n {% set new_scd_id = snapshot_hash_arguments([columns.dbt_scd_id, snapshot_get_time()]) %}\n {% endif %}\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }}\n from {{ target_relation }}\n where\n {% if config.get('dbt_valid_to_current') %}\n {# Check for either dbt_valid_to_current OR null, in order to correctly update records with nulls #}\n ( {{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or {{ columns.dbt_valid_to }} is null)\n {% else %}\n {{ columns.dbt_valid_to }} is null\n {% endif %}\n\n ),\n\n insertions_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ get_dbt_valid_to_current(strategy, columns) }},\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_to }}\n\n from snapshot_query\n ),\n\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n\n deletes_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }}\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n {%- if strategy.hard_deletes == 'new_record' -%}\n ,'False' as {{ columns.dbt_is_deleted }}\n {%- endif %}\n\n from insertions_source_data as source_data\n left outer join snapshotted_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"snapshotted_data\") }}\n or ({{ unique_key_is_not_null(strategy.unique_key, \"snapshotted_data\") }} and ({{ strategy.row_changed }})\n\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.{{ columns.dbt_scd_id }}\n {%- if strategy.hard_deletes == 'new_record' -%}\n , snapshotted_data.{{ columns.dbt_is_deleted }}\n {%- endif %}\n\n from updates_source_data as source_data\n join snapshotted_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n ,\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},\n {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_to }},\n snapshotted_data.{{ columns.dbt_scd_id }}\n {%- if strategy.hard_deletes == 'new_record' -%}\n , snapshotted_data.{{ columns.dbt_is_deleted }}\n {%- endif %}\n from snapshotted_data\n left join deletes_source_data as source_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"source_data\") }}\n )\n {%- endif %}\n\n {%- if strategy.hard_deletes == 'new_record' %}\n {% set source_sql_cols = get_column_schema_from_query(source_sql) %}\n ,\n deletion_records as (\n\n select\n 'insert' as dbt_change_type,\n {%- for col in source_sql_cols -%}\n snapshotted_data.{{ adapter.quote(col.column) }},\n {% endfor -%}\n {%- if strategy.unique_key | is_list -%}\n {%- for key in strategy.unique_key -%}\n snapshotted_data.{{ key }} as dbt_unique_key_{{ loop.index }},\n {% endfor -%}\n {%- else -%}\n snapshotted_data.dbt_unique_key as dbt_unique_key,\n {% endif -%}\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},\n {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},\n snapshotted_data.{{ columns.dbt_valid_to }} as {{ columns.dbt_valid_to }},\n {{ new_scd_id }} as {{ columns.dbt_scd_id }},\n 'True' as {{ columns.dbt_is_deleted }}\n from snapshotted_data\n left join deletes_source_data as source_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"source_data\") }}\n\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n union all\n select * from deletes\n {%- endif %}\n {%- if strategy.hard_deletes == 'new_record' %}\n union all\n select * from deletion_records\n {%- endif %}\n\n\n{%- endmacro %}", "meta": {}, "name": "default__snapshot_staging_table", "original_file_path": "macros/materializations/snapshots/helpers.sql", @@ -4260,7 +4386,7 @@ }, "macro.dbt.default__snapshot_string_as_time": { "arguments": [], - "created_at": 1719485736.346766, + "created_at": 1747731927.60026, "depends_on": { "macros": [] }, @@ -4282,7 +4408,7 @@ }, "macro.dbt.default__split_part": { "arguments": [], - "created_at": 1719485736.554402, + "created_at": 1747731927.752853, "depends_on": { "macros": [] }, @@ -4304,7 +4430,7 @@ }, "macro.dbt.default__string_literal": { "arguments": [], - "created_at": 1719485736.544776, + "created_at": 1747731927.7462022, "depends_on": { "macros": [] }, @@ -4326,7 +4452,7 @@ }, "macro.dbt.default__support_multiple_grantees_per_dcl_statement": { "arguments": [], - "created_at": 1719485736.573099, + "created_at": 1747731927.7710059, "depends_on": { "macros": [] }, @@ -4348,7 +4474,7 @@ }, "macro.dbt.default__test_accepted_values": { "arguments": [], - "created_at": 1719485736.517527, + "created_at": 1747731927.72354, "depends_on": { "macros": [] }, @@ -4370,7 +4496,7 @@ }, "macro.dbt.default__test_not_null": { "arguments": [], - "created_at": 1719485736.516648, + "created_at": 1747731927.7225542, "depends_on": { "macros": [ "macro.dbt.should_store_failures" @@ -4394,7 +4520,7 @@ }, "macro.dbt.default__test_relationships": { "arguments": [], - "created_at": 1719485736.516331, + "created_at": 1747731927.7221591, "depends_on": { "macros": [] }, @@ -4416,7 +4542,7 @@ }, "macro.dbt.default__test_unique": { "arguments": [], - "created_at": 1719485736.5169122, + "created_at": 1747731927.72286, "depends_on": { "macros": [] }, @@ -4438,7 +4564,7 @@ }, "macro.dbt.default__truncate_relation": { "arguments": [], - "created_at": 1719485736.5674748, + "created_at": 1747731927.7645252, "depends_on": { "macros": [ "macro.dbt.statement" @@ -4449,7 +4575,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", + "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation.render() }}\n {%- endcall %}\n{% endmacro %}", "meta": {}, "name": "default__truncate_relation", "original_file_path": "macros/adapters/relation.sql", @@ -4462,7 +4588,7 @@ }, "macro.dbt.default__type_bigint": { "arguments": [], - "created_at": 1719485736.5486028, + "created_at": 1747731927.749114, "depends_on": { "macros": [] }, @@ -4484,7 +4610,7 @@ }, "macro.dbt.default__type_boolean": { "arguments": [], - "created_at": 1719485736.550734, + "created_at": 1747731927.7499251, "depends_on": { "macros": [] }, @@ -4506,7 +4632,7 @@ }, "macro.dbt.default__type_float": { "arguments": [], - "created_at": 1719485736.5472598, + "created_at": 1747731927.74826, "depends_on": { "macros": [] }, @@ -4528,7 +4654,7 @@ }, "macro.dbt.default__type_int": { "arguments": [], - "created_at": 1719485736.549174, + "created_at": 1747731927.749523, "depends_on": { "macros": [] }, @@ -4550,7 +4676,7 @@ }, "macro.dbt.default__type_numeric": { "arguments": [], - "created_at": 1719485736.547792, + "created_at": 1747731927.748706, "depends_on": { "macros": [] }, @@ -4572,7 +4698,7 @@ }, "macro.dbt.default__type_string": { "arguments": [], - "created_at": 1719485736.5461628, + "created_at": 1747731927.747373, "depends_on": { "macros": [] }, @@ -4594,7 +4720,7 @@ }, "macro.dbt.default__type_timestamp": { "arguments": [], - "created_at": 1719485736.5468001, + "created_at": 1747731927.74783, "depends_on": { "macros": [] }, @@ -4614,9 +4740,31 @@ "supported_languages": null, "unique_id": "macro.dbt.default__type_timestamp" }, + "macro.dbt.default__validate_fixture_rows": { + "arguments": [], + "created_at": 1747731927.799627, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro default__validate_fixture_rows(rows, row_number) -%}\n {# This is an abstract method for adapter overrides as needed #}\n{%- endmacro -%}", + "meta": {}, + "name": "default__validate_fixture_rows", + "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/unit_test_sql/get_fixture_sql.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__validate_fixture_rows" + }, "macro.dbt.default__validate_sql": { "arguments": [], - "created_at": 1719485736.570561, + "created_at": 1747731927.768754, "depends_on": { "macros": [ "macro.dbt.statement" @@ -4640,7 +4788,7 @@ }, "macro.dbt.default_last_day": { "arguments": [], - "created_at": 1719485736.5533261, + "created_at": 1747731927.751937, "depends_on": { "macros": [ "macro.dbt.dateadd", @@ -4665,7 +4813,7 @@ }, "macro.dbt.diff_column_data_types": { "arguments": [], - "created_at": 1719485736.397441, + "created_at": 1747731927.6457212, "depends_on": { "macros": [] }, @@ -4687,7 +4835,7 @@ }, "macro.dbt.diff_columns": { "arguments": [], - "created_at": 1719485736.396736, + "created_at": 1747731927.6448839, "depends_on": { "macros": [] }, @@ -4709,7 +4857,7 @@ }, "macro.dbt.drop_materialized_view": { "arguments": [], - "created_at": 1719485736.486141, + "created_at": 1747731927.700943, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__drop_materialized_view" @@ -4733,7 +4881,7 @@ }, "macro.dbt.drop_relation": { "arguments": [], - "created_at": 1719485736.471232, + "created_at": 1747731927.690571, "depends_on": { "macros": [ "macro.dbt.default__drop_relation" @@ -4757,7 +4905,7 @@ }, "macro.dbt.drop_relation_if_exists": { "arguments": [], - "created_at": 1719485736.471669, + "created_at": 1747731927.691197, "depends_on": { "macros": [] }, @@ -4779,7 +4927,7 @@ }, "macro.dbt.drop_schema": { "arguments": [], - "created_at": 1719485736.558478, + "created_at": 1747731927.756108, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__drop_schema" @@ -4803,7 +4951,7 @@ }, "macro.dbt.drop_schema_named": { "arguments": [], - "created_at": 1719485736.4761379, + "created_at": 1747731927.69489, "depends_on": { "macros": [ "macro.dbt.default__drop_schema_named" @@ -4827,7 +4975,7 @@ }, "macro.dbt.drop_table": { "arguments": [], - "created_at": 1719485736.505447, + "created_at": 1747731927.710874, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__drop_table" @@ -4851,7 +4999,7 @@ }, "macro.dbt.drop_view": { "arguments": [], - "created_at": 1719485736.511067, + "created_at": 1747731927.7161689, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__drop_view" @@ -4875,7 +5023,7 @@ }, "macro.dbt.escape_single_quotes": { "arguments": [], - "created_at": 1719485736.537864, + "created_at": 1747731927.739479, "depends_on": { "macros": [ "macro.dbt.default__escape_single_quotes" @@ -4899,7 +5047,7 @@ }, "macro.dbt.except": { "arguments": [], - "created_at": 1719485736.5260952, + "created_at": 1747731927.730844, "depends_on": { "macros": [ "macro.dbt.default__except" @@ -4923,7 +5071,7 @@ }, "macro.dbt.format_columns": { "arguments": [], - "created_at": 1719485736.504094, + "created_at": 1747731927.7099679, "depends_on": { "macros": [ "macro.dbt.default__format_column" @@ -4947,7 +5095,7 @@ }, "macro.dbt.format_row": { "arguments": [], - "created_at": 1719485736.611443, + "created_at": 1747731927.799233, "depends_on": { "macros": [ "macro.dbt.string_literal", @@ -4960,7 +5108,7 @@ "node_color": null, "show": true }, - "macro_sql": "\n\n{%- macro format_row(row, column_name_to_data_types) -%}\n {#-- generate case-insensitive formatted row --#}\n {% set formatted_row = {} %}\n {%- for column_name, column_value in row.items() -%}\n {% set column_name = column_name|lower %}\n\n {%- if column_name not in column_name_to_data_types %}\n {#-- if user-provided row contains column name that relation does not contain, raise an error --#}\n {% set fixture_name = \"expected output\" if model.resource_type == 'unit_test' else (\"'\" ~ model.name ~ \"'\") %}\n {{ exceptions.raise_compiler_error(\n \"Invalid column name: '\" ~ column_name ~ \"' in unit test fixture for \" ~ fixture_name ~ \".\"\n \"\\nAccepted columns for \" ~ fixture_name ~ \" are: \" ~ (column_name_to_data_types.keys()|list)\n ) }}\n {%- endif -%}\n\n {%- set column_type = column_name_to_data_types[column_name] %}\n\n {#-- sanitize column_value: wrap yaml strings in quotes, apply cast --#}\n {%- set column_value_clean = column_value -%}\n {%- if column_value is string -%}\n {%- set column_value_clean = dbt.string_literal(dbt.escape_single_quotes(column_value)) -%}\n {%- elif column_value is none -%}\n {%- set column_value_clean = 'null' -%}\n {%- endif -%}\n\n {%- set row_update = {column_name: safe_cast(column_value_clean, column_type) } -%}\n {%- do formatted_row.update(row_update) -%}\n {%- endfor -%}\n {{ return(formatted_row) }}\n{%- endmacro -%}", + "macro_sql": "\n\n{%- macro format_row(row, column_name_to_data_types) -%}\n {#-- generate case-insensitive formatted row --#}\n {% set formatted_row = {} %}\n {%- for column_name, column_value in row.items() -%}\n {% set column_name = column_name|lower %}\n\n {%- if column_name not in column_name_to_data_types %}\n {#-- if user-provided row contains column name that relation does not contain, raise an error --#}\n {% set fixture_name = \"expected output\" if model.resource_type == 'unit_test' else (\"'\" ~ model.name ~ \"'\") %}\n {{ exceptions.raise_compiler_error(\n \"Invalid column name: '\" ~ column_name ~ \"' in unit test fixture for \" ~ fixture_name ~ \".\"\n \"\\nAccepted columns for \" ~ fixture_name ~ \" are: \" ~ (column_name_to_data_types.keys()|list)\n ) }}\n {%- endif -%}\n\n {%- set column_type = column_name_to_data_types[column_name] %}\n\n {#-- sanitize column_value: wrap yaml strings in quotes, apply cast --#}\n {%- set column_value_clean = column_value -%}\n {%- if column_value is string -%}\n {%- set column_value_clean = dbt.string_literal(dbt.escape_single_quotes(column_value)) -%}\n {%- elif column_value is none -%}\n {%- set column_value_clean = 'null' -%}\n {%- endif -%}\n\n {%- set row_update = {column_name: safe_cast(column_value_clean, column_type) } -%}\n {%- do formatted_row.update(row_update) -%}\n {%- endfor -%}\n {{ return(formatted_row) }}\n{%- endmacro -%}\n\n", "meta": {}, "name": "format_row", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", @@ -4973,7 +5121,7 @@ }, "macro.dbt.generate_alias_name": { "arguments": [], - "created_at": 1719485736.466082, + "created_at": 1747731927.6863801, "depends_on": { "macros": [ "macro.dbt.default__generate_alias_name" @@ -4997,7 +5145,7 @@ }, "macro.dbt.generate_database_name": { "arguments": [], - "created_at": 1719485736.468485, + "created_at": 1747731927.6887062, "depends_on": { "macros": [ "macro.dbt.default__generate_database_name" @@ -5021,7 +5169,7 @@ }, "macro.dbt.generate_schema_name": { "arguments": [], - "created_at": 1719485736.467374, + "created_at": 1747731927.68751, "depends_on": { "macros": [ "macro.dbt.default__generate_schema_name" @@ -5045,7 +5193,7 @@ }, "macro.dbt.generate_schema_name_for_env": { "arguments": [], - "created_at": 1719485736.46803, + "created_at": 1747731927.688221, "depends_on": { "macros": [] }, @@ -5067,7 +5215,7 @@ }, "macro.dbt.generate_series": { "arguments": [], - "created_at": 1719485736.5352, + "created_at": 1747731927.7367969, "depends_on": { "macros": [ "macro.dbt.default__generate_series" @@ -5091,7 +5239,7 @@ }, "macro.dbt.get_alter_materialized_view_as_sql": { "arguments": [], - "created_at": 1719485736.492841, + "created_at": 1747731927.703752, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql" @@ -5115,7 +5263,7 @@ }, "macro.dbt.get_assert_columns_equivalent": { "arguments": [], - "created_at": 1719485736.5009642, + "created_at": 1747731927.707553, "depends_on": { "macros": [ "macro.dbt.default__get_assert_columns_equivalent" @@ -5139,7 +5287,7 @@ }, "macro.dbt.get_batch_size": { "arguments": [], - "created_at": 1719485736.461688, + "created_at": 1747731927.6831338, "depends_on": { "macros": [ "macro.dbt.default__get_batch_size" @@ -5163,7 +5311,7 @@ }, "macro.dbt.get_binding_char": { "arguments": [], - "created_at": 1719485736.461306, + "created_at": 1747731927.682775, "depends_on": { "macros": [ "macro.dbt.default__get_binding_char" @@ -5187,7 +5335,7 @@ }, "macro.dbt.get_catalog": { "arguments": [], - "created_at": 1719485736.590185, + "created_at": 1747731927.781922, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_catalog" @@ -5209,9 +5357,33 @@ "supported_languages": null, "unique_id": "macro.dbt.get_catalog" }, + "macro.dbt.get_catalog_for_single_relation": { + "arguments": [], + "created_at": 1747731927.7845309, + "depends_on": { + "macros": [ + "macro.dbt.default__get_catalog_for_single_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_catalog_for_single_relation(relation) %}\n {{ return(adapter.dispatch('get_catalog_for_single_relation', 'dbt')(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_catalog_for_single_relation", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_catalog_for_single_relation" + }, "macro.dbt.get_catalog_relations": { "arguments": [], - "created_at": 1719485736.5896802, + "created_at": 1747731927.781291, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_catalog_relations" @@ -5235,7 +5407,7 @@ }, "macro.dbt.get_column_schema_from_query": { "arguments": [], - "created_at": 1719485736.598866, + "created_at": 1747731927.79071, "depends_on": { "macros": [ "macro.dbt.get_empty_subquery_sql" @@ -5259,7 +5431,7 @@ }, "macro.dbt.get_columns_in_query": { "arguments": [], - "created_at": 1719485736.5990841, + "created_at": 1747731927.790944, "depends_on": { "macros": [ "macro.dbt.default__get_columns_in_query" @@ -5283,7 +5455,7 @@ }, "macro.dbt.get_columns_in_relation": { "arguments": [], - "created_at": 1719485736.596093, + "created_at": 1747731927.7873778, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_columns_in_relation" @@ -5307,7 +5479,7 @@ }, "macro.dbt.get_create_backup_sql": { "arguments": [], - "created_at": 1719485736.478982, + "created_at": 1747731927.6981509, "depends_on": { "macros": [ "macro.dbt.default__get_create_backup_sql" @@ -5331,7 +5503,7 @@ }, "macro.dbt.get_create_index_sql": { "arguments": [], - "created_at": 1719485736.561203, + "created_at": 1747731927.759569, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_create_index_sql" @@ -5355,7 +5527,7 @@ }, "macro.dbt.get_create_intermediate_sql": { "arguments": [], - "created_at": 1719485736.4755762, + "created_at": 1747731927.6942022, "depends_on": { "macros": [ "macro.dbt.default__get_create_intermediate_sql" @@ -5379,7 +5551,7 @@ }, "macro.dbt.get_create_materialized_view_as_sql": { "arguments": [], - "created_at": 1719485736.4946408, + "created_at": 1747731927.704972, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_create_materialized_view_as_sql" @@ -5403,7 +5575,7 @@ }, "macro.dbt.get_create_sql": { "arguments": [], - "created_at": 1719485736.481704, + "created_at": 1747731927.699233, "depends_on": { "macros": [ "macro.dbt.default__get_create_sql" @@ -5427,7 +5599,7 @@ }, "macro.dbt.get_create_table_as_sql": { "arguments": [], - "created_at": 1719485736.50801, + "created_at": 1747731927.7130592, "depends_on": { "macros": [ "macro.dbt.default__get_create_table_as_sql" @@ -5451,7 +5623,7 @@ }, "macro.dbt.get_create_view_as_sql": { "arguments": [], - "created_at": 1719485736.515145, + "created_at": 1747731927.7206929, "depends_on": { "macros": [ "macro.dbt.default__get_create_view_as_sql" @@ -5475,7 +5647,7 @@ }, "macro.dbt.get_csv_sql": { "arguments": [], - "created_at": 1719485736.460989, + "created_at": 1747731927.682409, "depends_on": { "macros": [ "macro.dbt.default__get_csv_sql" @@ -5497,9 +5669,31 @@ "supported_languages": null, "unique_id": "macro.dbt.get_csv_sql" }, + "macro.dbt.get_dbt_valid_to_current": { + "arguments": [], + "created_at": 1747731927.616639, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_dbt_valid_to_current(strategy, columns) %}\n {% set dbt_valid_to_current = config.get('dbt_valid_to_current') or \"null\" %}\n coalesce(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}), {{dbt_valid_to_current}})\n as {{ columns.dbt_valid_to }}\n{% endmacro %}", + "meta": {}, + "name": "get_dbt_valid_to_current", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_dbt_valid_to_current" + }, "macro.dbt.get_dcl_statement_list": { "arguments": [], - "created_at": 1719485736.574908, + "created_at": 1747731927.773398, "depends_on": { "macros": [ "macro.dbt.default__get_dcl_statement_list" @@ -5523,7 +5717,7 @@ }, "macro.dbt.get_delete_insert_merge_sql": { "arguments": [], - "created_at": 1719485736.414207, + "created_at": 1747731927.651454, "depends_on": { "macros": [ "macro.dbt.default__get_delete_insert_merge_sql" @@ -5547,7 +5741,7 @@ }, "macro.dbt.get_drop_backup_sql": { "arguments": [], - "created_at": 1719485736.476695, + "created_at": 1747731927.695561, "depends_on": { "macros": [ "macro.dbt.default__get_drop_backup_sql" @@ -5571,7 +5765,7 @@ }, "macro.dbt.get_drop_index_sql": { "arguments": [], - "created_at": 1719485736.562388, + "created_at": 1747731927.7606862, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_drop_index_sql" @@ -5595,7 +5789,7 @@ }, "macro.dbt.get_drop_sql": { "arguments": [], - "created_at": 1719485736.470603, + "created_at": 1747731927.6896992, "depends_on": { "macros": [ "macro.dbt.default__get_drop_sql" @@ -5619,7 +5813,7 @@ }, "macro.dbt.get_empty_schema_sql": { "arguments": [], - "created_at": 1719485736.597257, + "created_at": 1747731927.788762, "depends_on": { "macros": [ "macro.dbt.default__get_empty_schema_sql" @@ -5643,7 +5837,7 @@ }, "macro.dbt.get_empty_subquery_sql": { "arguments": [], - "created_at": 1719485736.596842, + "created_at": 1747731927.788246, "depends_on": { "macros": [ "macro.dbt.default__get_empty_subquery_sql" @@ -5667,7 +5861,7 @@ }, "macro.dbt.get_expected_sql": { "arguments": [], - "created_at": 1719485736.608841, + "created_at": 1747731927.797819, "depends_on": { "macros": [ "macro.dbt.format_row" @@ -5691,11 +5885,12 @@ }, "macro.dbt.get_fixture_sql": { "arguments": [], - "created_at": 1719485736.607797, + "created_at": 1747731927.797125, "depends_on": { "macros": [ "macro.dbt.load_relation", "macro.dbt.safe_cast", + "macro.dbt.validate_fixture_rows", "macro.dbt.format_row" ] }, @@ -5704,7 +5899,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_fixture_sql(rows, column_name_to_data_types) %}\n-- Fixture for {{ model.name }}\n{% set default_row = {} %}\n\n{%- if not column_name_to_data_types -%}\n{#-- Use defer_relation IFF it is available in the manifest and 'this' is missing from the database --#}\n{%- set this_or_defer_relation = defer_relation if (defer_relation and not load_relation(this)) else this -%}\n{%- set columns_in_relation = adapter.get_columns_in_relation(this_or_defer_relation) -%}\n\n{%- set column_name_to_data_types = {} -%}\n{%- for column in columns_in_relation -%}\n{#-- This needs to be a case-insensitive comparison --#}\n{%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n{%- endfor -%}\n{%- endif -%}\n\n{%- if not column_name_to_data_types -%}\n {{ exceptions.raise_compiler_error(\"Not able to get columns for unit test '\" ~ model.name ~ \"' from relation \" ~ this ~ \" because the relation doesn't exist\") }}\n{%- endif -%}\n\n{%- for column_name, column_type in column_name_to_data_types.items() -%}\n {%- do default_row.update({column_name: (safe_cast(\"null\", column_type) | trim )}) -%}\n{%- endfor -%}\n\n\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\n{%- set default_row_copy = default_row.copy() -%}\n{%- do default_row_copy.update(formatted_row) -%}\nselect\n{%- for column_name, column_value in default_row_copy.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n\n{%- if (rows | length) == 0 -%}\n select\n {%- for column_name, column_value in default_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%},{%- endif %}\n {%- endfor %}\n limit 0\n{%- endif -%}\n{% endmacro %}", + "macro_sql": "{% macro get_fixture_sql(rows, column_name_to_data_types) %}\n-- Fixture for {{ model.name }}\n{% set default_row = {} %}\n\n{%- if not column_name_to_data_types -%}\n{#-- Use defer_relation IFF it is available in the manifest and 'this' is missing from the database --#}\n{%- set this_or_defer_relation = defer_relation if (defer_relation and not load_relation(this)) else this -%}\n{%- set columns_in_relation = adapter.get_columns_in_relation(this_or_defer_relation) -%}\n\n{%- set column_name_to_data_types = {} -%}\n{%- for column in columns_in_relation -%}\n{#-- This needs to be a case-insensitive comparison --#}\n{%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n{%- endfor -%}\n{%- endif -%}\n\n{%- if not column_name_to_data_types -%}\n {{ exceptions.raise_compiler_error(\"Not able to get columns for unit test '\" ~ model.name ~ \"' from relation \" ~ this ~ \" because the relation doesn't exist\") }}\n{%- endif -%}\n\n{%- for column_name, column_type in column_name_to_data_types.items() -%}\n {%- do default_row.update({column_name: (safe_cast(\"null\", column_type) | trim )}) -%}\n{%- endfor -%}\n\n{{ validate_fixture_rows(rows, row_number) }}\n\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\n{%- set default_row_copy = default_row.copy() -%}\n{%- do default_row_copy.update(formatted_row) -%}\nselect\n{%- for column_name, column_value in default_row_copy.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n\n{%- if (rows | length) == 0 -%}\n select\n {%- for column_name, column_value in default_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%},{%- endif %}\n {%- endfor %}\n limit 0\n{%- endif -%}\n{% endmacro %}", "meta": {}, "name": "get_fixture_sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", @@ -5717,7 +5912,7 @@ }, "macro.dbt.get_grant_sql": { "arguments": [], - "created_at": 1719485736.574021, + "created_at": 1747731927.7722838, "depends_on": { "macros": [ "macro.dbt.default__get_grant_sql" @@ -5741,7 +5936,7 @@ }, "macro.dbt.get_incremental_append_sql": { "arguments": [], - "created_at": 1719485736.418218, + "created_at": 1747731927.6556969, "depends_on": { "macros": [ "macro.dbt.default__get_incremental_append_sql" @@ -5765,7 +5960,7 @@ }, "macro.dbt.get_incremental_default_sql": { "arguments": [], - "created_at": 1719485736.421412, + "created_at": 1747731927.658079, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_incremental_default_sql" @@ -5789,7 +5984,7 @@ }, "macro.dbt.get_incremental_delete_insert_sql": { "arguments": [], - "created_at": 1719485736.4188569, + "created_at": 1747731927.656255, "depends_on": { "macros": [ "macro.dbt.default__get_incremental_delete_insert_sql" @@ -5813,7 +6008,7 @@ }, "macro.dbt.get_incremental_insert_overwrite_sql": { "arguments": [], - "created_at": 1719485736.420912, + "created_at": 1747731927.657504, "depends_on": { "macros": [ "macro.dbt.default__get_incremental_insert_overwrite_sql" @@ -5837,7 +6032,7 @@ }, "macro.dbt.get_incremental_merge_sql": { "arguments": [], - "created_at": 1719485736.419648, + "created_at": 1747731927.656883, "depends_on": { "macros": [ "macro.dbt.default__get_incremental_merge_sql" @@ -5859,9 +6054,33 @@ "supported_languages": null, "unique_id": "macro.dbt.get_incremental_merge_sql" }, + "macro.dbt.get_incremental_microbatch_sql": { + "arguments": [], + "created_at": 1747731927.6585019, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_incremental_microbatch_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_incremental_microbatch_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_microbatch_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_incremental_microbatch_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_incremental_microbatch_sql" + }, "macro.dbt.get_insert_into_sql": { "arguments": [], - "created_at": 1719485736.4218738, + "created_at": 1747731927.65906, "depends_on": { "macros": [ "macro.dbt.get_quoted_csv" @@ -5885,7 +6104,7 @@ }, "macro.dbt.get_insert_overwrite_merge_sql": { "arguments": [], - "created_at": 1719485736.415607, + "created_at": 1747731927.6530812, "depends_on": { "macros": [ "macro.dbt.default__get_insert_overwrite_merge_sql" @@ -5909,7 +6128,7 @@ }, "macro.dbt.get_intervals_between": { "arguments": [], - "created_at": 1719485736.527589, + "created_at": 1747731927.7318048, "depends_on": { "macros": [ "macro.dbt.default__get_intervals_between" @@ -5933,10 +6152,10 @@ }, "macro.dbt.get_limit_subquery_sql": { "arguments": [], - "created_at": 1719485736.58177, + "created_at": 1747731927.777363, "depends_on": { "macros": [ - "macro.dbt.default__get_limit_subquery_sql" + "macro.dbt.default__get_limit_sql" ] }, "description": "", @@ -5944,7 +6163,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_limit_subquery_sql(sql, limit) %}\n {{ adapter.dispatch('get_limit_subquery_sql', 'dbt')(sql, limit) }}\n{% endmacro %}", + "macro_sql": "\n{%- macro get_limit_subquery_sql(sql, limit) -%}\n {{ adapter.dispatch('get_limit_sql', 'dbt')(sql, limit) }}\n{%- endmacro -%}\n\n", "meta": {}, "name": "get_limit_subquery_sql", "original_file_path": "macros/adapters/show.sql", @@ -5957,7 +6176,7 @@ }, "macro.dbt.get_materialized_view_configuration_changes": { "arguments": [], - "created_at": 1719485736.494153, + "created_at": 1747731927.704397, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_materialized_view_configuration_changes" @@ -5981,7 +6200,7 @@ }, "macro.dbt.get_merge_sql": { "arguments": [], - "created_at": 1719485736.410901, + "created_at": 1747731927.648832, "depends_on": { "macros": [ "macro.dbt.default__get_merge_sql" @@ -6005,7 +6224,7 @@ }, "macro.dbt.get_merge_update_columns": { "arguments": [], - "created_at": 1719485736.397727, + "created_at": 1747731927.646025, "depends_on": { "macros": [ "macro.dbt.default__get_merge_update_columns" @@ -6029,7 +6248,7 @@ }, "macro.dbt.get_or_create_relation": { "arguments": [], - "created_at": 1719485736.567741, + "created_at": 1747731927.764838, "depends_on": { "macros": [ "macro.dbt.default__get_or_create_relation" @@ -6053,7 +6272,7 @@ }, "macro.dbt.get_powers_of_two": { "arguments": [], - "created_at": 1719485736.534465, + "created_at": 1747731927.736021, "depends_on": { "macros": [ "macro.dbt.default__get_powers_of_two" @@ -6077,7 +6296,7 @@ }, "macro.dbt.get_quoted_csv": { "arguments": [], - "created_at": 1719485736.395989, + "created_at": 1747731927.644193, "depends_on": { "macros": [] }, @@ -6099,7 +6318,7 @@ }, "macro.dbt.get_relation_last_modified": { "arguments": [], - "created_at": 1719485736.593024, + "created_at": 1747731927.7853959, "depends_on": { "macros": [ "macro.dbt.default__get_relation_last_modified" @@ -6123,7 +6342,7 @@ }, "macro.dbt.get_relations": { "arguments": [], - "created_at": 1719485736.592459, + "created_at": 1747731927.784939, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_relations" @@ -6147,7 +6366,7 @@ }, "macro.dbt.get_rename_intermediate_sql": { "arguments": [], - "created_at": 1719485736.484606, + "created_at": 1747731927.700288, "depends_on": { "macros": [ "macro.dbt.default__get_rename_intermediate_sql" @@ -6171,7 +6390,7 @@ }, "macro.dbt.get_rename_materialized_view_sql": { "arguments": [], - "created_at": 1719485736.4909282, + "created_at": 1747731927.702796, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_rename_materialized_view_sql" @@ -6195,7 +6414,7 @@ }, "macro.dbt.get_rename_sql": { "arguments": [], - "created_at": 1719485736.47766, + "created_at": 1747731927.696455, "depends_on": { "macros": [ "macro.dbt.default__get_rename_sql" @@ -6219,7 +6438,7 @@ }, "macro.dbt.get_rename_table_sql": { "arguments": [], - "created_at": 1719485736.5068932, + "created_at": 1747731927.711943, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_rename_table_sql" @@ -6243,7 +6462,7 @@ }, "macro.dbt.get_rename_view_sql": { "arguments": [], - "created_at": 1719485736.5145411, + "created_at": 1747731927.719897, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_rename_view_sql" @@ -6267,7 +6486,7 @@ }, "macro.dbt.get_replace_materialized_view_sql": { "arguments": [], - "created_at": 1719485736.487545, + "created_at": 1747731927.70146, "depends_on": { "macros": [ "macro.dbt.default__get_replace_materialized_view_sql" @@ -6291,7 +6510,7 @@ }, "macro.dbt.get_replace_sql": { "arguments": [], - "created_at": 1719485736.4724941, + "created_at": 1747731927.6920698, "depends_on": { "macros": [ "macro.dbt.default__get_replace_sql" @@ -6315,7 +6534,7 @@ }, "macro.dbt.get_replace_table_sql": { "arguments": [], - "created_at": 1719485736.506175, + "created_at": 1747731927.711402, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_replace_table_sql" @@ -6339,7 +6558,7 @@ }, "macro.dbt.get_replace_view_sql": { "arguments": [], - "created_at": 1719485736.5121448, + "created_at": 1747731927.7170482, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_replace_view_sql" @@ -6363,7 +6582,7 @@ }, "macro.dbt.get_revoke_sql": { "arguments": [], - "created_at": 1719485736.574466, + "created_at": 1747731927.772835, "depends_on": { "macros": [ "macro.dbt.default__get_revoke_sql" @@ -6387,7 +6606,7 @@ }, "macro.dbt.get_seed_column_quoted_csv": { "arguments": [], - "created_at": 1719485736.462852, + "created_at": 1747731927.683903, "depends_on": { "macros": [] }, @@ -6409,7 +6628,7 @@ }, "macro.dbt.get_select_subquery": { "arguments": [], - "created_at": 1719485736.510276, + "created_at": 1747731927.715607, "depends_on": { "macros": [ "macro.dbt.default__get_select_subquery" @@ -6433,7 +6652,7 @@ }, "macro.dbt.get_show_grant_sql": { "arguments": [], - "created_at": 1719485736.573664, + "created_at": 1747731927.771843, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_show_grant_sql" @@ -6457,7 +6676,7 @@ }, "macro.dbt.get_show_indexes_sql": { "arguments": [], - "created_at": 1719485736.562955, + "created_at": 1747731927.761064, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_show_indexes_sql" @@ -6481,7 +6700,7 @@ }, "macro.dbt.get_show_sql": { "arguments": [], - "created_at": 1719485736.581328, + "created_at": 1747731927.7771242, "depends_on": { "macros": [ "macro.dbt.get_limit_subquery_sql" @@ -6492,7 +6711,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_show_sql(compiled_code, sql_header, limit) -%}\n {%- if sql_header -%}\n {{ sql_header }}\n {%- endif -%}\n {%- if limit is not none -%}\n {{ get_limit_subquery_sql(compiled_code, limit) }}\n {%- else -%}\n {{ compiled_code }}\n {%- endif -%}\n{% endmacro %}", + "macro_sql": "{% macro get_show_sql(compiled_code, sql_header, limit) -%}\n {%- if sql_header is not none -%}\n {{ sql_header }}\n {%- endif %}\n {{ get_limit_subquery_sql(compiled_code, limit) }}\n{% endmacro %}", "meta": {}, "name": "get_show_sql", "original_file_path": "macros/adapters/show.sql", @@ -6503,9 +6722,57 @@ "supported_languages": null, "unique_id": "macro.dbt.get_show_sql" }, + "macro.dbt.get_snapshot_get_time_data_type": { + "arguments": [], + "created_at": 1747731927.758155, + "depends_on": { + "macros": [ + "macro.dbt.snapshot_get_time", + "macro.dbt_postgres.postgres__snapshot_get_time", + "macro.dbt.get_column_schema_from_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_snapshot_get_time_data_type() %}\n {% set snapshot_time = adapter.dispatch('snapshot_get_time', 'dbt')() %}\n {% set time_data_type_sql = 'select ' ~ snapshot_time ~ ' as dbt_snapshot_time' %}\n {% set snapshot_time_column_schema = get_column_schema_from_query(time_data_type_sql) %}\n {% set time_data_type = snapshot_time_column_schema[0].dtype %}\n {{ return(time_data_type or none) }}\n{% endmacro %}", + "meta": {}, + "name": "get_snapshot_get_time_data_type", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_snapshot_get_time_data_type" + }, + "macro.dbt.get_snapshot_table_column_names": { + "arguments": [], + "created_at": 1747731927.608804, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_snapshot_table_column_names() %}\n {{ return({'dbt_valid_to': 'dbt_valid_to', 'dbt_valid_from': 'dbt_valid_from', 'dbt_scd_id': 'dbt_scd_id', 'dbt_updated_at': 'dbt_updated_at', 'dbt_is_deleted': 'dbt_is_deleted'}) }}\n{% endmacro %}", + "meta": {}, + "name": "get_snapshot_table_column_names", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_snapshot_table_column_names" + }, "macro.dbt.get_table_columns_and_constraints": { "arguments": [], - "created_at": 1719485736.497471, + "created_at": 1747731927.706323, "depends_on": { "macros": [ "macro.dbt.default__get_table_columns_and_constraints" @@ -6529,7 +6796,7 @@ }, "macro.dbt.get_test_sql": { "arguments": [], - "created_at": 1719485736.37231, + "created_at": 1747731927.6271858, "depends_on": { "macros": [ "macro.dbt.default__get_test_sql" @@ -6553,7 +6820,7 @@ }, "macro.dbt.get_true_sql": { "arguments": [], - "created_at": 1719485736.355415, + "created_at": 1747731927.608038, "depends_on": { "macros": [ "macro.dbt.default__get_true_sql" @@ -6577,7 +6844,7 @@ }, "macro.dbt.get_unit_test_sql": { "arguments": [], - "created_at": 1719485736.3730109, + "created_at": 1747731927.6278489, "depends_on": { "macros": [ "macro.dbt.default__get_unit_test_sql" @@ -6599,9 +6866,33 @@ "supported_languages": null, "unique_id": "macro.dbt.get_unit_test_sql" }, + "macro.dbt.get_updated_at_column_data_type": { + "arguments": [], + "created_at": 1747731927.615725, + "depends_on": { + "macros": [ + "macro.dbt.get_column_schema_from_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_updated_at_column_data_type(snapshot_sql) %}\n {% set snapshot_sql_column_schema = get_column_schema_from_query(snapshot_sql) %}\n {% set dbt_updated_at_data_type = null %}\n {% set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {% set ns.dbt_updated_at_data_type = null -%}\n {% for column in snapshot_sql_column_schema %}\n {% if ((column.column == 'dbt_updated_at') or (column.column == 'DBT_UPDATED_AT')) %}\n {% set ns.dbt_updated_at_data_type = column.dtype %}\n {% endif %}\n {% endfor %}\n {{ return(ns.dbt_updated_at_data_type or none) }}\n{% endmacro %}", + "meta": {}, + "name": "get_updated_at_column_data_type", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_updated_at_column_data_type" + }, "macro.dbt.get_where_subquery": { "arguments": [], - "created_at": 1719485736.3740978, + "created_at": 1747731927.629067, "depends_on": { "macros": [ "macro.dbt.default__get_where_subquery" @@ -6625,7 +6916,7 @@ }, "macro.dbt.handle_existing_table": { "arguments": [], - "created_at": 1719485736.513942, + "created_at": 1747731927.71925, "depends_on": { "macros": [ "macro.dbt.default__handle_existing_table" @@ -6649,7 +6940,7 @@ }, "macro.dbt.hash": { "arguments": [], - "created_at": 1719485736.5412428, + "created_at": 1747731927.7432551, "depends_on": { "macros": [ "macro.dbt.default__hash" @@ -6673,7 +6964,7 @@ }, "macro.dbt.in_transaction": { "arguments": [], - "created_at": 1719485736.336884, + "created_at": 1747731927.5915048, "depends_on": { "macros": [ "macro.dbt.make_hook_config" @@ -6697,7 +6988,7 @@ }, "macro.dbt.incremental_validate_on_schema_change": { "arguments": [], - "created_at": 1719485736.437581, + "created_at": 1747731927.666322, "depends_on": { "macros": [] }, @@ -6719,7 +7010,7 @@ }, "macro.dbt.information_schema_name": { "arguments": [], - "created_at": 1719485736.590652, + "created_at": 1747731927.782491, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__information_schema_name" @@ -6743,7 +7034,7 @@ }, "macro.dbt.intersect": { "arguments": [], - "created_at": 1719485736.537264, + "created_at": 1747731927.739006, "depends_on": { "macros": [ "macro.dbt.default__intersect" @@ -6767,7 +7058,7 @@ }, "macro.dbt.is_incremental": { "arguments": [], - "created_at": 1719485736.417088, + "created_at": 1747731927.654745, "depends_on": { "macros": [ "macro.dbt.should_full_refresh" @@ -6791,7 +7082,7 @@ }, "macro.dbt.last_day": { "arguments": [], - "created_at": 1719485736.553028, + "created_at": 1747731927.7515619, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__last_day" @@ -6815,7 +7106,7 @@ }, "macro.dbt.length": { "arguments": [], - "created_at": 1719485736.536302, + "created_at": 1747731927.737795, "depends_on": { "macros": [ "macro.dbt.default__length" @@ -6839,7 +7130,7 @@ }, "macro.dbt.list_relations_without_caching": { "arguments": [], - "created_at": 1719485736.592111, + "created_at": 1747731927.784093, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__list_relations_without_caching" @@ -6863,7 +7154,7 @@ }, "macro.dbt.list_schemas": { "arguments": [], - "created_at": 1719485736.591011, + "created_at": 1747731927.782917, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__list_schemas" @@ -6887,7 +7178,7 @@ }, "macro.dbt.listagg": { "arguments": [], - "created_at": 1719485736.5392041, + "created_at": 1747731927.741078, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__listagg" @@ -6911,7 +7202,7 @@ }, "macro.dbt.load_cached_relation": { "arguments": [], - "created_at": 1719485736.568574, + "created_at": 1747731927.7659461, "depends_on": { "macros": [] }, @@ -6933,7 +7224,7 @@ }, "macro.dbt.load_csv_rows": { "arguments": [], - "created_at": 1719485736.4631271, + "created_at": 1747731927.684154, "depends_on": { "macros": [ "macro.dbt.default__load_csv_rows" @@ -6957,7 +7248,7 @@ }, "macro.dbt.load_relation": { "arguments": [], - "created_at": 1719485736.568728, + "created_at": 1747731927.766169, "depends_on": { "macros": [ "macro.dbt.load_cached_relation" @@ -6981,7 +7272,7 @@ }, "macro.dbt.make_backup_relation": { "arguments": [], - "created_at": 1719485736.566729, + "created_at": 1747731927.7636638, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__make_backup_relation" @@ -7005,7 +7296,7 @@ }, "macro.dbt.make_hook_config": { "arguments": [], - "created_at": 1719485736.336577, + "created_at": 1747731927.5911272, "depends_on": { "macros": [] }, @@ -7027,7 +7318,7 @@ }, "macro.dbt.make_intermediate_relation": { "arguments": [], - "created_at": 1719485736.565727, + "created_at": 1747731927.762184, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__make_intermediate_relation" @@ -7051,7 +7342,7 @@ }, "macro.dbt.make_temp_relation": { "arguments": [], - "created_at": 1719485736.5661461, + "created_at": 1747731927.762978, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__make_temp_relation" @@ -7062,7 +7353,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", + "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {#-- This ensures microbatch batches get unique temp relations to avoid clobbering --#}\n {% if suffix == '__dbt_tmp' and model.batch %}\n {% set suffix = suffix ~ '_' ~ model.batch.id %}\n {% endif %}\n\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "meta": {}, "name": "make_temp_relation", "original_file_path": "macros/adapters/relation.sql", @@ -7075,7 +7366,7 @@ }, "macro.dbt.materialization_clone_default": { "arguments": [], - "created_at": 1719485736.448061, + "created_at": 1747731927.675092, "depends_on": { "macros": [ "macro.dbt.load_cached_relation", @@ -7093,7 +7384,7 @@ "node_color": null, "show": true }, - "macro_sql": "{%- materialization clone, default -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {% if target_relation and defer_relation and target_relation == defer_relation %}\n {{ log(\"Target relation and defer relation are the same, skipping clone for relation: \" ~ target_relation) }}\n {% else %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endif %}\n\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n\n {%- endif -%}\n\n{%- endmaterialization -%}", + "macro_sql": "{%- materialization clone, default -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation.render() ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {% if target_relation and defer_relation and target_relation == defer_relation %}\n {{ log(\"Target relation and defer relation are the same, skipping clone for relation: \" ~ target_relation.render()) }}\n {% else %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endif %}\n\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n\n {%- endif -%}\n\n{%- endmaterialization -%}", "meta": {}, "name": "materialization_clone_default", "original_file_path": "macros/materializations/models/clone/clone.sql", @@ -7108,7 +7399,7 @@ }, "macro.dbt.materialization_incremental_default": { "arguments": [], - "created_at": 1719485736.4306219, + "created_at": 1747731927.664597, "depends_on": { "macros": [ "macro.dbt.load_cached_relation", @@ -7134,7 +7425,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", + "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% set contract_config = config.get('contract') %}\n {% if not contract_config or not contract_config.enforced %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {% endif %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "meta": {}, "name": "materialization_incremental_default", "original_file_path": "macros/materializations/models/incremental/incremental.sql", @@ -7149,7 +7440,7 @@ }, "macro.dbt.materialization_materialized_view_default": { "arguments": [], - "created_at": 1719485736.383056, + "created_at": 1747731927.633707, "depends_on": { "macros": [ "macro.dbt.load_cached_relation", @@ -7182,7 +7473,7 @@ }, "macro.dbt.materialization_seed_default": { "arguments": [], - "created_at": 1719485736.4520829, + "created_at": 1747731927.67847, "depends_on": { "macros": [ "macro.dbt.should_full_refresh", @@ -7203,7 +7494,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparison later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", + "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparison later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation.render())) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "meta": {}, "name": "materialization_seed_default", "original_file_path": "macros/materializations/seeds/seed.sql", @@ -7218,7 +7509,7 @@ }, "macro.dbt.materialization_snapshot_default": { "arguments": [], - "created_at": 1719485736.3679821, + "created_at": 1747731927.6235409, "depends_on": { "macros": [ "macro.dbt.get_or_create_relation", @@ -7226,9 +7517,12 @@ "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", + "macro.dbt.get_snapshot_table_column_names", + "macro.dbt.snapshot_staging_table", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", + "macro.dbt.check_time_data_types", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", @@ -7242,7 +7536,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparision later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", + "macro_sql": "{% materialization snapshot, default %}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparision later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {# The model['config'] parameter below is no longer used, but passing anyway for compatibility #}\n {# It was a dictionary of config, instead of the config object from the context #}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", model['config'], target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set build_or_select_sql = build_sql %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {{ adapter.assert_valid_snapshot_target_given_strategy(target_relation, columns, strategy) }}\n\n {% set build_or_select_sql = snapshot_staging_table(strategy, sql, target_relation) %}\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set remove_columns = ['dbt_change_type', 'DBT_CHANGE_TYPE', 'dbt_unique_key', 'DBT_UNIQUE_KEY'] %}\n {% if unique_key | is_list %}\n {% for key in strategy.unique_key %}\n {{ remove_columns.append('dbt_unique_key_' + loop.index|string) }}\n {{ remove_columns.append('DBT_UNIQUE_KEY_' + loop.index|string) }}\n {% endfor %}\n {% endif %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'in', remove_columns)\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'in', remove_columns)\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n\n {{ check_time_data_types(build_or_select_sql) }}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "meta": {}, "name": "materialization_snapshot_default", "original_file_path": "macros/materializations/snapshots/snapshot.sql", @@ -7257,7 +7551,7 @@ }, "macro.dbt.materialization_table_default": { "arguments": [], - "created_at": 1719485736.3933172, + "created_at": 1747731927.643024, "depends_on": { "macros": [ "macro.dbt.load_cached_relation", @@ -7293,7 +7587,7 @@ }, "macro.dbt.materialization_test_default": { "arguments": [], - "created_at": 1719485736.370986, + "created_at": 1747731927.6263418, "depends_on": { "macros": [ "macro.dbt.should_store_failures", @@ -7322,7 +7616,7 @@ }, "macro.dbt.materialization_unit_default": { "arguments": [], - "created_at": 1719485736.3763871, + "created_at": 1747731927.6315389, "depends_on": { "macros": [ "macro.dbt.get_columns_in_query", @@ -7355,7 +7649,7 @@ }, "macro.dbt.materialization_view_default": { "arguments": [], - "created_at": 1719485736.390083, + "created_at": 1747731927.640254, "depends_on": { "macros": [ "macro.dbt.load_cached_relation", @@ -7390,7 +7684,7 @@ }, "macro.dbt.materialized_view_execute_build_sql": { "arguments": [], - "created_at": 1719485736.386875, + "created_at": 1747731927.6374998, "depends_on": { "macros": [ "macro.dbt.run_hooks", @@ -7418,7 +7712,7 @@ }, "macro.dbt.materialized_view_execute_no_op": { "arguments": [], - "created_at": 1719485736.385866, + "created_at": 1747731927.636649, "depends_on": { "macros": [] }, @@ -7440,7 +7734,7 @@ }, "macro.dbt.materialized_view_get_build_sql": { "arguments": [], - "created_at": 1719485736.385609, + "created_at": 1747731927.6363468, "depends_on": { "macros": [ "macro.dbt.should_full_refresh", @@ -7456,7 +7750,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = '' %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", + "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = '' %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation.render() ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation.render() ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "meta": {}, "name": "materialized_view_get_build_sql", "original_file_path": "macros/materializations/models/materialized_view.sql", @@ -7469,7 +7763,7 @@ }, "macro.dbt.materialized_view_setup": { "arguments": [], - "created_at": 1719485736.3838332, + "created_at": 1747731927.63421, "depends_on": { "macros": [ "macro.dbt.load_cached_relation", @@ -7495,7 +7789,7 @@ }, "macro.dbt.materialized_view_teardown": { "arguments": [], - "created_at": 1719485736.384107, + "created_at": 1747731927.634565, "depends_on": { "macros": [ "macro.dbt.drop_relation_if_exists", @@ -7520,7 +7814,7 @@ }, "macro.dbt.noop_statement": { "arguments": [], - "created_at": 1719485736.5199249, + "created_at": 1747731927.726103, "depends_on": { "macros": [] }, @@ -7542,7 +7836,7 @@ }, "macro.dbt.partition_range": { "arguments": [], - "created_at": 1719485736.5252612, + "created_at": 1747731927.7302258, "depends_on": { "macros": [ "macro.dbt.dates_in_range" @@ -7566,7 +7860,7 @@ }, "macro.dbt.persist_docs": { "arguments": [], - "created_at": 1719485736.585922, + "created_at": 1747731927.779349, "depends_on": { "macros": [ "macro.dbt.default__persist_docs" @@ -7590,7 +7884,7 @@ }, "macro.dbt.position": { "arguments": [], - "created_at": 1719485736.543356, + "created_at": 1747731927.7455611, "depends_on": { "macros": [ "macro.dbt.default__position" @@ -7614,7 +7908,7 @@ }, "macro.dbt.post_snapshot": { "arguments": [], - "created_at": 1719485736.355165, + "created_at": 1747731927.607726, "depends_on": { "macros": [ "macro.dbt.default__post_snapshot" @@ -7638,7 +7932,7 @@ }, "macro.dbt.process_schema_changes": { "arguments": [], - "created_at": 1719485736.44261, + "created_at": 1747731927.670649, "depends_on": { "macros": [ "macro.dbt.check_for_schema_changes", @@ -7663,7 +7957,7 @@ }, "macro.dbt.py_current_timestring": { "arguments": [], - "created_at": 1719485736.5255191, + "created_at": 1747731927.7305338, "depends_on": { "macros": [] }, @@ -7685,7 +7979,7 @@ }, "macro.dbt.py_script_comment": { "arguments": [], - "created_at": 1719485736.6181471, + "created_at": 1747731927.80398, "depends_on": { "macros": [] }, @@ -7707,7 +8001,7 @@ }, "macro.dbt.py_script_postfix": { "arguments": [], - "created_at": 1719485736.618067, + "created_at": 1747731927.803885, "depends_on": { "macros": [ "macro.dbt.build_ref_function", @@ -7736,7 +8030,7 @@ }, "macro.dbt.refresh_materialized_view": { "arguments": [], - "created_at": 1719485736.489407, + "created_at": 1747731927.702241, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__refresh_materialized_view" @@ -7760,7 +8054,7 @@ }, "macro.dbt.rename_relation": { "arguments": [], - "created_at": 1719485736.478327, + "created_at": 1747731927.69734, "depends_on": { "macros": [ "macro.dbt.default__rename_relation" @@ -7784,7 +8078,7 @@ }, "macro.dbt.replace": { "arguments": [], - "created_at": 1719485736.532748, + "created_at": 1747731927.734698, "depends_on": { "macros": [ "macro.dbt.default__replace" @@ -7808,7 +8102,7 @@ }, "macro.dbt.reset_csv_table": { "arguments": [], - "created_at": 1719485736.460277, + "created_at": 1747731927.681515, "depends_on": { "macros": [ "macro.dbt.default__reset_csv_table" @@ -7832,7 +8126,7 @@ }, "macro.dbt.resolve_model_name": { "arguments": [], - "created_at": 1719485736.614536, + "created_at": 1747731927.800597, "depends_on": { "macros": [ "macro.dbt.default__resolve_model_name" @@ -7856,7 +8150,7 @@ }, "macro.dbt.right": { "arguments": [], - "created_at": 1719485736.538353, + "created_at": 1747731927.74013, "depends_on": { "macros": [ "macro.dbt.default__right" @@ -7880,7 +8174,7 @@ }, "macro.dbt.run_hooks": { "arguments": [], - "created_at": 1719485736.336357, + "created_at": 1747731927.590857, "depends_on": { "macros": [ "macro.dbt.statement" @@ -7904,7 +8198,7 @@ }, "macro.dbt.run_query": { "arguments": [], - "created_at": 1719485736.5202482, + "created_at": 1747731927.72649, "depends_on": { "macros": [ "macro.dbt.statement" @@ -7928,7 +8222,7 @@ }, "macro.dbt.safe_cast": { "arguments": [], - "created_at": 1719485736.540742, + "created_at": 1747731927.7427158, "depends_on": { "macros": [ "macro.dbt.default__safe_cast" @@ -7952,7 +8246,7 @@ }, "macro.dbt.set_sql_header": { "arguments": [], - "created_at": 1719485736.338798, + "created_at": 1747731927.592107, "depends_on": { "macros": [] }, @@ -7974,7 +8268,7 @@ }, "macro.dbt.should_full_refresh": { "arguments": [], - "created_at": 1719485736.339242, + "created_at": 1747731927.5924969, "depends_on": { "macros": [] }, @@ -7996,7 +8290,7 @@ }, "macro.dbt.should_revoke": { "arguments": [], - "created_at": 1719485736.573465, + "created_at": 1747731927.7715871, "depends_on": { "macros": [ "macro.dbt.copy_grants" @@ -8020,7 +8314,7 @@ }, "macro.dbt.should_store_failures": { "arguments": [], - "created_at": 1719485736.339565, + "created_at": 1747731927.592878, "depends_on": { "macros": [] }, @@ -8042,7 +8336,7 @@ }, "macro.dbt.snapshot_check_all_get_existing_columns": { "arguments": [], - "created_at": 1719485736.3482602, + "created_at": 1747731927.6020842, "depends_on": { "macros": [ "macro.dbt.get_columns_in_query" @@ -8066,7 +8360,7 @@ }, "macro.dbt.snapshot_check_strategy": { "arguments": [], - "created_at": 1719485736.3500328, + "created_at": 1747731927.6040199, "depends_on": { "macros": [ "macro.dbt.snapshot_get_time", @@ -8080,7 +8374,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n {% set updated_at = config.get('updated_at', snapshot_get_time()) %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", + "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set check_cols_config = config.get('check_cols') %}\n {% set primary_key = config.get('unique_key') %}\n {% set hard_deletes = adapter.get_hard_deletes_behavior(config) %}\n {% set invalidate_hard_deletes = hard_deletes == 'invalidate' %}\n {% set updated_at = config.get('updated_at') or snapshot_get_time() %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_args = api.Relation.scd_args(primary_key, updated_at) %}\n {% set scd_id_expr = snapshot_hash_arguments(scd_args) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes,\n \"hard_deletes\": hard_deletes\n }) %}\n{% endmacro %}", "meta": {}, "name": "snapshot_check_strategy", "original_file_path": "macros/materializations/snapshots/strategies.sql", @@ -8093,7 +8387,7 @@ }, "macro.dbt.snapshot_get_time": { "arguments": [], - "created_at": 1719485736.55953, + "created_at": 1747731927.75749, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__snapshot_get_time" @@ -8117,7 +8411,7 @@ }, "macro.dbt.snapshot_hash_arguments": { "arguments": [], - "created_at": 1719485736.3454158, + "created_at": 1747731927.598088, "depends_on": { "macros": [ "macro.dbt.default__snapshot_hash_arguments" @@ -8141,7 +8435,7 @@ }, "macro.dbt.snapshot_merge_sql": { "arguments": [], - "created_at": 1719485736.3400362, + "created_at": 1747731927.5934432, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__snapshot_merge_sql" @@ -8165,7 +8459,7 @@ }, "macro.dbt.snapshot_staging_table": { "arguments": [], - "created_at": 1719485736.355765, + "created_at": 1747731927.60846, "depends_on": { "macros": [ "macro.dbt.default__snapshot_staging_table" @@ -8189,7 +8483,7 @@ }, "macro.dbt.snapshot_string_as_time": { "arguments": [], - "created_at": 1719485736.3465831, + "created_at": 1747731927.600032, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__snapshot_string_as_time" @@ -8213,9 +8507,10 @@ }, "macro.dbt.snapshot_timestamp_strategy": { "arguments": [], - "created_at": 1719485736.346405, + "created_at": 1747731927.599787, "depends_on": { "macros": [ + "macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_hash_arguments" ] }, @@ -8224,7 +8519,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", + "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set primary_key = config.get('unique_key') %}\n {% set updated_at = config.get('updated_at') %}\n {% set hard_deletes = adapter.get_hard_deletes_behavior(config) %}\n {% set invalidate_hard_deletes = hard_deletes == 'invalidate' %}\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.{{ columns.dbt_valid_from }} < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_args = api.Relation.scd_args(primary_key, updated_at) %}\n {% set scd_id_expr = snapshot_hash_arguments(scd_args) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes,\n \"hard_deletes\": hard_deletes\n }) %}\n{% endmacro %}", "meta": {}, "name": "snapshot_timestamp_strategy", "original_file_path": "macros/materializations/snapshots/strategies.sql", @@ -8237,7 +8532,7 @@ }, "macro.dbt.split_part": { "arguments": [], - "created_at": 1719485736.55408, + "created_at": 1747731927.75264, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__split_part" @@ -8261,7 +8556,7 @@ }, "macro.dbt.sql_convert_columns_in_relation": { "arguments": [], - "created_at": 1719485736.59661, + "created_at": 1747731927.787963, "depends_on": { "macros": [] }, @@ -8283,7 +8578,7 @@ }, "macro.dbt.statement": { "arguments": [], - "created_at": 1719485736.51912, + "created_at": 1747731927.725317, "depends_on": { "macros": [] }, @@ -8305,7 +8600,7 @@ }, "macro.dbt.strategy_dispatch": { "arguments": [], - "created_at": 1719485736.345131, + "created_at": 1747731927.597855, "depends_on": { "macros": [] }, @@ -8327,7 +8622,7 @@ }, "macro.dbt.string_literal": { "arguments": [], - "created_at": 1719485736.543961, + "created_at": 1747731927.746068, "depends_on": { "macros": [ "macro.dbt.default__string_literal" @@ -8351,7 +8646,7 @@ }, "macro.dbt.support_multiple_grantees_per_dcl_statement": { "arguments": [], - "created_at": 1719485736.572978, + "created_at": 1747731927.770852, "depends_on": { "macros": [ "macro.dbt.default__support_multiple_grantees_per_dcl_statement" @@ -8375,7 +8670,7 @@ }, "macro.dbt.sync_column_schemas": { "arguments": [], - "created_at": 1719485736.4416761, + "created_at": 1747731927.6694121, "depends_on": { "macros": [ "macro.dbt.alter_relation_add_remove_columns", @@ -8400,7 +8695,7 @@ }, "macro.dbt.table_columns_and_constraints": { "arguments": [], - "created_at": 1719485736.500673, + "created_at": 1747731927.707312, "depends_on": { "macros": [] }, @@ -8422,7 +8717,7 @@ }, "macro.dbt.test_accepted_values": { "arguments": [], - "created_at": 1719485736.6202009, + "created_at": 1747731927.805286, "depends_on": { "macros": [ "macro.dbt.default__test_accepted_values" @@ -8446,7 +8741,7 @@ }, "macro.dbt.test_not_null": { "arguments": [], - "created_at": 1719485736.619859, + "created_at": 1747731927.804893, "depends_on": { "macros": [ "macro.dbt.default__test_not_null" @@ -8470,7 +8765,7 @@ }, "macro.dbt.test_relationships": { "arguments": [], - "created_at": 1719485736.620512, + "created_at": 1747731927.805658, "depends_on": { "macros": [ "macro.dbt.default__test_relationships" @@ -8494,7 +8789,7 @@ }, "macro.dbt.test_unique": { "arguments": [], - "created_at": 1719485736.619491, + "created_at": 1747731927.804576, "depends_on": { "macros": [ "macro.dbt.default__test_unique" @@ -8518,7 +8813,7 @@ }, "macro.dbt.truncate_relation": { "arguments": [], - "created_at": 1719485736.567298, + "created_at": 1747731927.764297, "depends_on": { "macros": [ "macro.dbt.default__truncate_relation" @@ -8542,7 +8837,7 @@ }, "macro.dbt.type_bigint": { "arguments": [], - "created_at": 1719485736.548424, + "created_at": 1747731927.748919, "depends_on": { "macros": [ "macro.dbt.default__type_bigint" @@ -8566,7 +8861,7 @@ }, "macro.dbt.type_boolean": { "arguments": [], - "created_at": 1719485736.55002, + "created_at": 1747731927.749732, "depends_on": { "macros": [ "macro.dbt.default__type_boolean" @@ -8590,7 +8885,7 @@ }, "macro.dbt.type_float": { "arguments": [], - "created_at": 1719485736.547037, + "created_at": 1747731927.748051, "depends_on": { "macros": [ "macro.dbt.default__type_float" @@ -8614,7 +8909,7 @@ }, "macro.dbt.type_int": { "arguments": [], - "created_at": 1719485736.54878, + "created_at": 1747731927.749332, "depends_on": { "macros": [ "macro.dbt.default__type_int" @@ -8638,7 +8933,7 @@ }, "macro.dbt.type_numeric": { "arguments": [], - "created_at": 1719485736.547442, + "created_at": 1747731927.748472, "depends_on": { "macros": [ "macro.dbt.default__type_numeric" @@ -8662,7 +8957,7 @@ }, "macro.dbt.type_string": { "arguments": [], - "created_at": 1719485736.545998, + "created_at": 1747731927.747178, "depends_on": { "macros": [ "macro.dbt.default__type_string" @@ -8686,7 +8981,7 @@ }, "macro.dbt.type_timestamp": { "arguments": [], - "created_at": 1719485736.546634, + "created_at": 1747731927.7476008, "depends_on": { "macros": [ "macro.dbt.default__type_timestamp" @@ -8708,72 +9003,184 @@ "supported_languages": null, "unique_id": "macro.dbt.type_timestamp" }, - "macro.dbt.validate_sql": { + "macro.dbt.unique_key_fields": { "arguments": [], - "created_at": 1719485736.5702772, + "created_at": 1747731927.6170702, "depends_on": { - "macros": [ - "macro.dbt.default__validate_sql" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro validate_sql(sql) -%}\n {{ return(adapter.dispatch('validate_sql', 'dbt')(sql)) }}\n{% endmacro %}", + "macro_sql": "{% macro unique_key_fields(unique_key) %}\n {% if unique_key | is_list %}\n {% for key in unique_key %}\n {{ key }} as dbt_unique_key_{{ loop.index }}\n {%- if not loop.last %} , {%- endif %}\n {% endfor %}\n {% else %}\n {{ unique_key }} as dbt_unique_key\n {% endif %}\n{% endmacro %}", "meta": {}, - "name": "validate_sql", - "original_file_path": "macros/adapters/validate_sql.sql", + "name": "unique_key_fields", + "original_file_path": "macros/materializations/snapshots/helpers.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/validate_sql.sql", + "path": "macros/materializations/snapshots/helpers.sql", "resource_type": "macro", "supported_languages": null, - "unique_id": "macro.dbt.validate_sql" + "unique_id": "macro.dbt.unique_key_fields" }, - "macro.dbt_postgres.postgres__alter_column_comment": { + "macro.dbt.unique_key_is_not_null": { "arguments": [], - "created_at": 1719485736.314291, + "created_at": 1747731927.618146, "depends_on": { - "macros": [ - "macro.dbt_postgres.postgres_escape_comment" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% for column_name in column_dict if (column_name in existing_columns) %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", + "macro_sql": "{% macro unique_key_is_not_null(unique_key, identifier) %}\n {% if unique_key | is_list %}\n {{ identifier }}.dbt_unique_key_1 is not null\n {% else %}\n {{ identifier }}.dbt_unique_key is not null\n {% endif %}\n{% endmacro %}", "meta": {}, - "name": "postgres__alter_column_comment", - "original_file_path": "macros/adapters.sql", - "package_name": "dbt_postgres", + "name": "unique_key_is_not_null", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", "patch_path": null, - "path": "macros/adapters.sql", + "path": "macros/materializations/snapshots/helpers.sql", "resource_type": "macro", "supported_languages": null, - "unique_id": "macro.dbt_postgres.postgres__alter_column_comment" + "unique_id": "macro.dbt.unique_key_is_not_null" }, - "macro.dbt_postgres.postgres__alter_relation_comment": { + "macro.dbt.unique_key_is_null": { "arguments": [], - "created_at": 1719485736.312385, + "created_at": 1747731927.617883, "depends_on": { - "macros": [ - "macro.dbt_postgres.postgres_escape_comment" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", + "macro_sql": "{% macro unique_key_is_null(unique_key, identifier) %}\n {% if unique_key | is_list %}\n {{ identifier }}.dbt_unique_key_1 is null\n {% else %}\n {{ identifier }}.dbt_unique_key is null\n {% endif %}\n{% endmacro %}", "meta": {}, - "name": "postgres__alter_relation_comment", - "original_file_path": "macros/adapters.sql", - "package_name": "dbt_postgres", + "name": "unique_key_is_null", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.unique_key_is_null" + }, + "macro.dbt.unique_key_join_on": { + "arguments": [], + "created_at": 1747731927.617616, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro unique_key_join_on(unique_key, identifier, from_identifier) %}\n {% if unique_key | is_list %}\n {% for key in unique_key %}\n {{ identifier }}.dbt_unique_key_{{ loop.index }} = {{ from_identifier }}.dbt_unique_key_{{ loop.index }}\n {%- if not loop.last %} and {%- endif %}\n {% endfor %}\n {% else %}\n {{ identifier }}.dbt_unique_key = {{ from_identifier }}.dbt_unique_key\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "unique_key_join_on", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.unique_key_join_on" + }, + "macro.dbt.validate_fixture_rows": { + "arguments": [], + "created_at": 1747731927.7995038, + "depends_on": { + "macros": [ + "macro.dbt.default__validate_fixture_rows" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro validate_fixture_rows(rows, row_number) -%}\n {{ return(adapter.dispatch('validate_fixture_rows', 'dbt')(rows, row_number)) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "validate_fixture_rows", + "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/unit_test_sql/get_fixture_sql.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.validate_fixture_rows" + }, + "macro.dbt.validate_sql": { + "arguments": [], + "created_at": 1747731927.7684648, + "depends_on": { + "macros": [ + "macro.dbt.default__validate_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro validate_sql(sql) -%}\n {{ return(adapter.dispatch('validate_sql', 'dbt')(sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "validate_sql", + "original_file_path": "macros/adapters/validate_sql.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/validate_sql.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.validate_sql" + }, + "macro.dbt_postgres.postgres__alter_column_comment": { + "arguments": [], + "created_at": 1747731927.573863, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres_escape_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% for column_name in column_dict if (column_name in existing_columns) %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", + "meta": {}, + "name": "postgres__alter_column_comment", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__alter_column_comment" + }, + "macro.dbt_postgres.postgres__alter_relation_comment": { + "arguments": [], + "created_at": 1747731927.573079, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres_escape_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n {% if relation.type == 'materialized_view' -%}\n {% set relation_type = \"materialized view\" %}\n {%- else -%}\n {%- set relation_type = relation.type -%}\n {%- endif -%}\n comment on {{ relation_type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", + "meta": {}, + "name": "postgres__alter_relation_comment", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", "patch_path": null, "path": "macros/adapters.sql", "resource_type": "macro", @@ -8782,7 +9189,7 @@ }, "macro.dbt_postgres.postgres__any_value": { "arguments": [], - "created_at": 1719485736.332639, + "created_at": 1747731927.588298, "depends_on": { "macros": [] }, @@ -8804,7 +9211,7 @@ }, "macro.dbt_postgres.postgres__check_schema_exists": { "arguments": [], - "created_at": 1719485736.309761, + "created_at": 1747731927.567126, "depends_on": { "macros": [ "macro.dbt.statement" @@ -8828,7 +9235,7 @@ }, "macro.dbt_postgres.postgres__copy_grants": { "arguments": [], - "created_at": 1719485736.3146439, + "created_at": 1747731927.57429, "depends_on": { "macros": [] }, @@ -8850,7 +9257,7 @@ }, "macro.dbt_postgres.postgres__create_schema": { "arguments": [], - "created_at": 1719485736.306344, + "created_at": 1747731927.562762, "depends_on": { "macros": [ "macro.dbt.statement" @@ -8874,7 +9281,7 @@ }, "macro.dbt_postgres.postgres__create_table_as": { "arguments": [], - "created_at": 1719485736.3053398, + "created_at": 1747731927.5616121, "depends_on": { "macros": [ "macro.dbt.get_assert_columns_equivalent", @@ -8901,7 +9308,7 @@ }, "macro.dbt_postgres.postgres__current_timestamp": { "arguments": [], - "created_at": 1719485736.2888992, + "created_at": 1747731927.5544891, "depends_on": { "macros": [] }, @@ -8923,7 +9330,7 @@ }, "macro.dbt_postgres.postgres__current_timestamp_backcompat": { "arguments": [], - "created_at": 1719485736.289357, + "created_at": 1747731927.5550542, "depends_on": { "macros": [ "macro.dbt.type_timestamp" @@ -8947,7 +9354,7 @@ }, "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat": { "arguments": [], - "created_at": 1719485736.289477, + "created_at": 1747731927.555201, "depends_on": { "macros": [ "macro.dbt.type_timestamp" @@ -8971,7 +9378,7 @@ }, "macro.dbt_postgres.postgres__dateadd": { "arguments": [], - "created_at": 1719485736.328303, + "created_at": 1747731927.5842838, "depends_on": { "macros": [] }, @@ -8993,7 +9400,7 @@ }, "macro.dbt_postgres.postgres__datediff": { "arguments": [], - "created_at": 1719485736.332466, + "created_at": 1747731927.588048, "depends_on": { "macros": [ "macro.dbt.datediff" @@ -9017,7 +9424,7 @@ }, "macro.dbt_postgres.postgres__describe_materialized_view": { "arguments": [], - "created_at": 1719485736.320213, + "created_at": 1747731927.578354, "depends_on": { "macros": [ "macro.dbt.run_query", @@ -9042,7 +9449,7 @@ }, "macro.dbt_postgres.postgres__drop_materialized_view": { "arguments": [], - "created_at": 1719485736.319324, + "created_at": 1747731927.577953, "depends_on": { "macros": [] }, @@ -9064,7 +9471,7 @@ }, "macro.dbt_postgres.postgres__drop_schema": { "arguments": [], - "created_at": 1719485736.307206, + "created_at": 1747731927.5632598, "depends_on": { "macros": [ "macro.dbt.statement" @@ -9088,7 +9495,7 @@ }, "macro.dbt_postgres.postgres__drop_table": { "arguments": [], - "created_at": 1719485736.3256729, + "created_at": 1747731927.581559, "depends_on": { "macros": [] }, @@ -9110,7 +9517,7 @@ }, "macro.dbt_postgres.postgres__drop_view": { "arguments": [], - "created_at": 1719485736.327039, + "created_at": 1747731927.583054, "depends_on": { "macros": [] }, @@ -9132,7 +9539,7 @@ }, "macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql": { "arguments": [], - "created_at": 1719485736.3214989, + "created_at": 1747731927.5796201, "depends_on": { "macros": [ "macro.dbt.get_replace_sql", @@ -9157,7 +9564,7 @@ }, "macro.dbt_postgres.postgres__get_catalog": { "arguments": [], - "created_at": 1719485736.292309, + "created_at": 1747731927.557344, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_catalog_relations" @@ -9181,7 +9588,7 @@ }, "macro.dbt_postgres.postgres__get_catalog_relations": { "arguments": [], - "created_at": 1719485736.2918808, + "created_at": 1747731927.55684, "depends_on": { "macros": [ "macro.dbt.statement" @@ -9205,7 +9612,7 @@ }, "macro.dbt_postgres.postgres__get_columns_in_relation": { "arguments": [], - "created_at": 1719485736.308002, + "created_at": 1747731927.563999, "depends_on": { "macros": [ "macro.dbt.statement", @@ -9230,7 +9637,7 @@ }, "macro.dbt_postgres.postgres__get_create_index_sql": { "arguments": [], - "created_at": 1719485736.30595, + "created_at": 1747731927.562346, "depends_on": { "macros": [] }, @@ -9252,7 +9659,7 @@ }, "macro.dbt_postgres.postgres__get_create_materialized_view_as_sql": { "arguments": [], - "created_at": 1719485736.3255181, + "created_at": 1747731927.581357, "depends_on": { "macros": [ "macro.dbt.get_create_index_sql" @@ -9263,7 +9670,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__get_create_materialized_view_as_sql(relation, sql) %}\n create materialized view if not exists {{ relation }} as {{ sql }};\n\n {% for _index_dict in config.get('indexes', []) -%}\n {{- get_create_index_sql(relation, _index_dict) -}}\n {%- endfor -%}\n\n{% endmacro %}", + "macro_sql": "{% macro postgres__get_create_materialized_view_as_sql(relation, sql) %}\n create materialized view if not exists {{ relation }} as {{ sql }};\n\n {% for _index_dict in config.get('indexes', []) -%}\n {{- get_create_index_sql(relation, _index_dict) -}}{{ ';' if not loop.last else \"\" }}\n {%- endfor -%}\n\n{% endmacro %}", "meta": {}, "name": "postgres__get_create_materialized_view_as_sql", "original_file_path": "macros/relations/materialized_view/create.sql", @@ -9276,7 +9683,7 @@ }, "macro.dbt_postgres.postgres__get_drop_index_sql": { "arguments": [], - "created_at": 1719485736.3150148, + "created_at": 1747731927.57473, "depends_on": { "macros": [] }, @@ -9298,7 +9705,7 @@ }, "macro.dbt_postgres.postgres__get_incremental_default_sql": { "arguments": [], - "created_at": 1719485736.316301, + "created_at": 1747731927.575518, "depends_on": { "macros": [ "macro.dbt.get_incremental_delete_insert_sql", @@ -9321,9 +9728,34 @@ "supported_languages": null, "unique_id": "macro.dbt_postgres.postgres__get_incremental_default_sql" }, + "macro.dbt_postgres.postgres__get_incremental_microbatch_sql": { + "arguments": [], + "created_at": 1747731927.576177, + "depends_on": { + "macros": [ + "macro.dbt.get_incremental_merge_sql", + "macro.dbt.default__get_incremental_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_incremental_microbatch_sql(arg_dict) %}\n\n {% if arg_dict[\"unique_key\"] %}\n {% do return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"dbt-postgres 'microbatch' requires a `unique_key` config\") }}\n {% endif %}\n\n{% endmacro %}", + "meta": {}, + "name": "postgres__get_incremental_microbatch_sql", + "original_file_path": "macros/materializations/incremental_strategies.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/materializations/incremental_strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_incremental_microbatch_sql" + }, "macro.dbt_postgres.postgres__get_materialized_view_configuration_changes": { "arguments": [], - "created_at": 1719485736.324945, + "created_at": 1747731927.580746, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__describe_materialized_view" @@ -9347,7 +9779,7 @@ }, "macro.dbt_postgres.postgres__get_relations": { "arguments": [], - "created_at": 1719485736.293338, + "created_at": 1747731927.558175, "depends_on": { "macros": [ "macro.dbt.statement" @@ -9371,7 +9803,7 @@ }, "macro.dbt_postgres.postgres__get_rename_materialized_view_sql": { "arguments": [], - "created_at": 1719485736.3205602, + "created_at": 1747731927.5788019, "depends_on": { "macros": [] }, @@ -9393,7 +9825,7 @@ }, "macro.dbt_postgres.postgres__get_rename_table_sql": { "arguments": [], - "created_at": 1719485736.326884, + "created_at": 1747731927.582858, "depends_on": { "macros": [] }, @@ -9415,7 +9847,7 @@ }, "macro.dbt_postgres.postgres__get_rename_view_sql": { "arguments": [], - "created_at": 1719485736.32807, + "created_at": 1747731927.5839942, "depends_on": { "macros": [] }, @@ -9437,7 +9869,7 @@ }, "macro.dbt_postgres.postgres__get_replace_table_sql": { "arguments": [], - "created_at": 1719485736.326431, + "created_at": 1747731927.582569, "depends_on": { "macros": [ "macro.dbt.get_assert_columns_equivalent", @@ -9463,7 +9895,7 @@ }, "macro.dbt_postgres.postgres__get_replace_view_sql": { "arguments": [], - "created_at": 1719485736.327884, + "created_at": 1747731927.583754, "depends_on": { "macros": [ "macro.dbt.get_assert_columns_equivalent" @@ -9487,7 +9919,7 @@ }, "macro.dbt_postgres.postgres__get_show_grant_sql": { "arguments": [], - "created_at": 1719485736.314513, + "created_at": 1747731927.574135, "depends_on": { "macros": [] }, @@ -9509,7 +9941,7 @@ }, "macro.dbt_postgres.postgres__get_show_indexes_sql": { "arguments": [], - "created_at": 1719485736.31486, + "created_at": 1747731927.5745301, "depends_on": { "macros": [] }, @@ -9531,7 +9963,7 @@ }, "macro.dbt_postgres.postgres__information_schema_name": { "arguments": [], - "created_at": 1719485736.308933, + "created_at": 1747731927.564801, "depends_on": { "macros": [] }, @@ -9553,7 +9985,7 @@ }, "macro.dbt_postgres.postgres__last_day": { "arguments": [], - "created_at": 1719485736.333154, + "created_at": 1747731927.588948, "depends_on": { "macros": [ "macro.dbt.dateadd", @@ -9579,7 +10011,7 @@ }, "macro.dbt_postgres.postgres__list_relations_without_caching": { "arguments": [], - "created_at": 1719485736.308728, + "created_at": 1747731927.564577, "depends_on": { "macros": [ "macro.dbt.statement" @@ -9603,7 +10035,7 @@ }, "macro.dbt_postgres.postgres__list_schemas": { "arguments": [], - "created_at": 1719485736.309331, + "created_at": 1747731927.566249, "depends_on": { "macros": [ "macro.dbt.statement" @@ -9627,7 +10059,7 @@ }, "macro.dbt_postgres.postgres__listagg": { "arguments": [], - "created_at": 1719485736.329083, + "created_at": 1747731927.584971, "depends_on": { "macros": [] }, @@ -9649,7 +10081,7 @@ }, "macro.dbt_postgres.postgres__make_backup_relation": { "arguments": [], - "created_at": 1719485736.311583, + "created_at": 1747731927.572017, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__make_relation_with_suffix" @@ -9673,7 +10105,7 @@ }, "macro.dbt_postgres.postgres__make_intermediate_relation": { "arguments": [], - "created_at": 1719485736.3109372, + "created_at": 1747731927.5711749, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__make_relation_with_suffix" @@ -9697,7 +10129,7 @@ }, "macro.dbt_postgres.postgres__make_relation_with_suffix": { "arguments": [], - "created_at": 1719485736.31072, + "created_at": 1747731927.570091, "depends_on": { "macros": [] }, @@ -9719,7 +10151,7 @@ }, "macro.dbt_postgres.postgres__make_temp_relation": { "arguments": [], - "created_at": 1719485736.31129, + "created_at": 1747731927.57165, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__make_relation_with_suffix" @@ -9743,7 +10175,7 @@ }, "macro.dbt_postgres.postgres__refresh_materialized_view": { "arguments": [], - "created_at": 1719485736.32038, + "created_at": 1747731927.578568, "depends_on": { "macros": [] }, @@ -9765,7 +10197,7 @@ }, "macro.dbt_postgres.postgres__snapshot_get_time": { "arguments": [], - "created_at": 1719485736.289238, + "created_at": 1747731927.5549068, "depends_on": { "macros": [ "macro.dbt.current_timestamp" @@ -9789,16 +10221,18 @@ }, "macro.dbt_postgres.postgres__snapshot_merge_sql": { "arguments": [], - "created_at": 1719485736.318985, + "created_at": 1747731927.577738, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.get_snapshot_table_column_names" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n update {{ target }}\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text\n and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)\n and {{ target }}.dbt_valid_to is null;\n\n insert into {{ target }} ({{ insert_cols_csv }})\n select {% for column in insert_cols -%}\n DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;\n{% endmacro %}", + "macro_sql": "{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n {%- set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() -%}\n\n update {{ target }}\n set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }}::text = {{ target }}.{{ columns.dbt_scd_id }}::text\n and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)\n {% if config.get(\"dbt_valid_to_current\") %}\n and ({{ target }}.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or {{ target }}.{{ columns.dbt_valid_to }} is null);\n {% else %}\n and {{ target }}.{{ columns.dbt_valid_to }} is null;\n {% endif %}\n\n\n insert into {{ target }} ({{ insert_cols_csv }})\n select {% for column in insert_cols -%}\n DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;\n{% endmacro %}", "meta": {}, "name": "postgres__snapshot_merge_sql", "original_file_path": "macros/materializations/snapshot_merge.sql", @@ -9811,7 +10245,7 @@ }, "macro.dbt_postgres.postgres__snapshot_string_as_time": { "arguments": [], - "created_at": 1719485736.289119, + "created_at": 1747731927.55475, "depends_on": { "macros": [] }, @@ -9833,7 +10267,7 @@ }, "macro.dbt_postgres.postgres__split_part": { "arguments": [], - "created_at": 1719485736.334007, + "created_at": 1747731927.589598, "depends_on": { "macros": [ "macro.dbt.default__split_part", @@ -9858,7 +10292,7 @@ }, "macro.dbt_postgres.postgres__update_indexes_on_materialized_view": { "arguments": [], - "created_at": 1719485736.3239639, + "created_at": 1747731927.580349, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_drop_index_sql", @@ -9870,7 +10304,7 @@ "node_color": null, "show": true }, - "macro_sql": "\n\n\n{%- macro postgres__update_indexes_on_materialized_view(relation, index_changes) -%}\n {{- log(\"Applying UPDATE INDEXES to: \" ~ relation) -}}\n\n {%- for _index_change in index_changes -%}\n {%- set _index = _index_change.context -%}\n\n {%- if _index_change.action == \"drop\" -%}\n\n {{ postgres__get_drop_index_sql(relation, _index.name) }};\n\n {%- elif _index_change.action == \"create\" -%}\n\n {{ postgres__get_create_index_sql(relation, _index.as_node_config) }}\n\n {%- endif -%}\n\n {%- endfor -%}\n\n{%- endmacro -%}\n\n\n", + "macro_sql": "\n\n\n{%- macro postgres__update_indexes_on_materialized_view(relation, index_changes) -%}\n {{- log(\"Applying UPDATE INDEXES to: \" ~ relation) -}}\n\n {%- for _index_change in index_changes -%}\n {%- set _index = _index_change.context -%}\n\n {%- if _index_change.action == \"drop\" -%}\n\n {{ postgres__get_drop_index_sql(relation, _index.name) }}\n\n {%- elif _index_change.action == \"create\" -%}\n\n {{ postgres__get_create_index_sql(relation, _index.as_node_config) }}\n\n {%- endif -%}\n\t{{ ';' if not loop.last else \"\" }}\n\n {%- endfor -%}\n\n{%- endmacro -%}\n\n\n", "meta": {}, "name": "postgres__update_indexes_on_materialized_view", "original_file_path": "macros/relations/materialized_view/alter.sql", @@ -9883,7 +10317,7 @@ }, "macro.dbt_postgres.postgres_escape_comment": { "arguments": [], - "created_at": 1719485736.3121421, + "created_at": 1747731927.57259, "depends_on": { "macros": [] }, @@ -9905,7 +10339,7 @@ }, "macro.dbt_postgres.postgres_get_relations": { "arguments": [], - "created_at": 1719485736.2934961, + "created_at": 1747731927.55835, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_relations" @@ -9929,7 +10363,7 @@ }, "macro.dbt_utils._bigquery__get_matching_schemata": { "arguments": [], - "created_at": 1719485736.7140338, + "created_at": 1747731927.872, "depends_on": { "macros": [ "macro.dbt.run_query" @@ -9953,7 +10387,7 @@ }, "macro.dbt_utils._is_ephemeral": { "arguments": [], - "created_at": 1719485736.667329, + "created_at": 1747731927.838956, "depends_on": { "macros": [] }, @@ -9975,7 +10409,7 @@ }, "macro.dbt_utils._is_relation": { "arguments": [], - "created_at": 1719485736.6636631, + "created_at": 1747731927.836006, "depends_on": { "macros": [] }, @@ -9997,7 +10431,7 @@ }, "macro.dbt_utils.bigquery__deduplicate": { "arguments": [], - "created_at": 1719485736.703784, + "created_at": 1747731927.865907, "depends_on": { "macros": [] }, @@ -10019,7 +10453,7 @@ }, "macro.dbt_utils.bigquery__get_tables_by_pattern_sql": { "arguments": [], - "created_at": 1719485736.713356, + "created_at": 1747731927.8713422, "depends_on": { "macros": [ "macro.dbt_utils._bigquery__get_matching_schemata", @@ -10044,7 +10478,7 @@ }, "macro.dbt_utils.bigquery__haversine_distance": { "arguments": [], - "created_at": 1719485736.73806, + "created_at": 1747731927.888902, "depends_on": { "macros": [ "macro.dbt_utils.degrees_to_radians" @@ -10068,7 +10502,7 @@ }, "macro.dbt_utils.databricks__get_table_types_sql": { "arguments": [], - "created_at": 1719485736.73151, + "created_at": 1747731927.8842628, "depends_on": { "macros": [] }, @@ -10090,7 +10524,7 @@ }, "macro.dbt_utils.date_spine": { "arguments": [], - "created_at": 1719485736.669255, + "created_at": 1747731927.840684, "depends_on": { "macros": [ "macro.dbt_utils.default__date_spine" @@ -10114,7 +10548,7 @@ }, "macro.dbt_utils.deduplicate": { "arguments": [], - "created_at": 1719485736.7012029, + "created_at": 1747731927.8645592, "depends_on": { "macros": [ "macro.dbt_utils.postgres__deduplicate" @@ -10138,7 +10572,7 @@ }, "macro.dbt_utils.default__date_spine": { "arguments": [], - "created_at": 1719485736.669651, + "created_at": 1747731927.8411489, "depends_on": { "macros": [ "macro.dbt_utils.generate_series", @@ -10164,7 +10598,7 @@ }, "macro.dbt_utils.default__deduplicate": { "arguments": [], - "created_at": 1719485736.701886, + "created_at": 1747731927.864841, "depends_on": { "macros": [] }, @@ -10186,7 +10620,7 @@ }, "macro.dbt_utils.default__generate_series": { "arguments": [], - "created_at": 1719485736.678598, + "created_at": 1747731927.8471801, "depends_on": { "macros": [ "macro.dbt_utils.get_powers_of_two" @@ -10210,7 +10644,7 @@ }, "macro.dbt_utils.default__generate_surrogate_key": { "arguments": [], - "created_at": 1719485736.730099, + "created_at": 1747731927.883199, "depends_on": { "macros": [ "macro.dbt.type_string", @@ -10236,7 +10670,7 @@ }, "macro.dbt_utils.default__get_column_values": { "arguments": [], - "created_at": 1719485736.717495, + "created_at": 1747731927.87493, "depends_on": { "macros": [ "macro.dbt_utils._is_ephemeral", @@ -10262,7 +10696,7 @@ }, "macro.dbt_utils.default__get_filtered_columns_in_relation": { "arguments": [], - "created_at": 1719485736.722564, + "created_at": 1747731927.878995, "depends_on": { "macros": [ "macro.dbt_utils._is_relation", @@ -10287,7 +10721,7 @@ }, "macro.dbt_utils.default__get_intervals_between": { "arguments": [], - "created_at": 1719485736.668746, + "created_at": 1747731927.84039, "depends_on": { "macros": [ "macro.dbt.statement", @@ -10312,7 +10746,7 @@ }, "macro.dbt_utils.default__get_powers_of_two": { "arguments": [], - "created_at": 1719485736.677862, + "created_at": 1747731927.846244, "depends_on": { "macros": [] }, @@ -10334,7 +10768,7 @@ }, "macro.dbt_utils.default__get_query_results_as_dict": { "arguments": [], - "created_at": 1719485736.727696, + "created_at": 1747731927.881738, "depends_on": { "macros": [ "macro.dbt.statement" @@ -10358,7 +10792,7 @@ }, "macro.dbt_utils.default__get_relations_by_pattern": { "arguments": [], - "created_at": 1719485736.674046, + "created_at": 1747731927.845014, "depends_on": { "macros": [ "macro.dbt.statement", @@ -10383,7 +10817,7 @@ }, "macro.dbt_utils.default__get_relations_by_prefix": { "arguments": [], - "created_at": 1719485736.680293, + "created_at": 1747731927.8490582, "depends_on": { "macros": [ "macro.dbt.statement", @@ -10408,7 +10842,7 @@ }, "macro.dbt_utils.default__get_single_value": { "arguments": [], - "created_at": 1719485736.7332969, + "created_at": 1747731927.886017, "depends_on": { "macros": [ "macro.dbt.statement" @@ -10432,7 +10866,7 @@ }, "macro.dbt_utils.default__get_table_types_sql": { "arguments": [], - "created_at": 1719485736.73105, + "created_at": 1747731927.883857, "depends_on": { "macros": [] }, @@ -10454,7 +10888,7 @@ }, "macro.dbt_utils.default__get_tables_by_pattern_sql": { "arguments": [], - "created_at": 1719485736.7122319, + "created_at": 1747731927.870323, "depends_on": { "macros": [ "macro.dbt_utils.get_table_types_sql" @@ -10478,7 +10912,7 @@ }, "macro.dbt_utils.default__get_tables_by_prefix_sql": { "arguments": [], - "created_at": 1719485736.681414, + "created_at": 1747731927.849935, "depends_on": { "macros": [ "macro.dbt_utils.get_tables_by_pattern_sql" @@ -10502,7 +10936,7 @@ }, "macro.dbt_utils.default__get_url_host": { "arguments": [], - "created_at": 1719485736.621399, + "created_at": 1747731927.806769, "depends_on": { "macros": [ "macro.dbt.split_part", @@ -10529,7 +10963,7 @@ }, "macro.dbt_utils.default__get_url_parameter": { "arguments": [], - "created_at": 1719485736.624197, + "created_at": 1747731927.808855, "depends_on": { "macros": [ "macro.dbt.split_part" @@ -10553,7 +10987,7 @@ }, "macro.dbt_utils.default__get_url_path": { "arguments": [], - "created_at": 1719485736.6228778, + "created_at": 1747731927.808048, "depends_on": { "macros": [ "macro.dbt.replace", @@ -10583,7 +11017,7 @@ }, "macro.dbt_utils.default__group_by": { "arguments": [], - "created_at": 1719485736.700402, + "created_at": 1747731927.86381, "depends_on": { "macros": [] }, @@ -10605,7 +11039,7 @@ }, "macro.dbt_utils.default__haversine_distance": { "arguments": [], - "created_at": 1719485736.736367, + "created_at": 1747731927.887831, "depends_on": { "macros": [] }, @@ -10627,7 +11061,7 @@ }, "macro.dbt_utils.default__log_info": { "arguments": [], - "created_at": 1719485736.665316, + "created_at": 1747731927.837184, "depends_on": { "macros": [ "macro.dbt_utils.pretty_log_format" @@ -10651,7 +11085,7 @@ }, "macro.dbt_utils.default__nullcheck": { "arguments": [], - "created_at": 1719485736.7089212, + "created_at": 1747731927.868929, "depends_on": { "macros": [] }, @@ -10673,7 +11107,7 @@ }, "macro.dbt_utils.default__nullcheck_table": { "arguments": [], - "created_at": 1719485736.6715178, + "created_at": 1747731927.843101, "depends_on": { "macros": [ "macro.dbt_utils._is_relation", @@ -10699,7 +11133,7 @@ }, "macro.dbt_utils.default__pivot": { "arguments": [], - "created_at": 1719485736.720366, + "created_at": 1747731927.877446, "depends_on": { "macros": [ "macro.dbt.escape_single_quotes", @@ -10724,7 +11158,7 @@ }, "macro.dbt_utils.default__pretty_log_format": { "arguments": [], - "created_at": 1719485736.66297, + "created_at": 1747731927.835427, "depends_on": { "macros": [ "macro.dbt_utils.pretty_time" @@ -10748,7 +11182,7 @@ }, "macro.dbt_utils.default__pretty_time": { "arguments": [], - "created_at": 1719485736.66434, + "created_at": 1747731927.836617, "depends_on": { "macros": [] }, @@ -10770,7 +11204,7 @@ }, "macro.dbt_utils.default__safe_add": { "arguments": [], - "created_at": 1719485736.707396, + "created_at": 1747731927.867966, "depends_on": { "macros": [] }, @@ -10792,7 +11226,7 @@ }, "macro.dbt_utils.default__safe_divide": { "arguments": [], - "created_at": 1719485736.689852, + "created_at": 1747731927.856703, "depends_on": { "macros": [] }, @@ -10814,7 +11248,7 @@ }, "macro.dbt_utils.default__safe_subtract": { "arguments": [], - "created_at": 1719485736.670645, + "created_at": 1747731927.842276, "depends_on": { "macros": [] }, @@ -10836,7 +11270,7 @@ }, "macro.dbt_utils.default__star": { "arguments": [], - "created_at": 1719485736.685639, + "created_at": 1747731927.85287, "depends_on": { "macros": [ "macro.dbt_utils._is_relation", @@ -10862,7 +11296,7 @@ }, "macro.dbt_utils.default__surrogate_key": { "arguments": [], - "created_at": 1719485736.7051, + "created_at": 1747731927.866734, "depends_on": { "macros": [] }, @@ -10884,7 +11318,7 @@ }, "macro.dbt_utils.default__test_accepted_range": { "arguments": [], - "created_at": 1719485736.6358259, + "created_at": 1747731927.8190022, "depends_on": { "macros": [] }, @@ -10906,7 +11340,7 @@ }, "macro.dbt_utils.default__test_at_least_one": { "arguments": [], - "created_at": 1719485736.638953, + "created_at": 1747731927.8220842, "depends_on": { "macros": [] }, @@ -10928,7 +11362,7 @@ }, "macro.dbt_utils.default__test_cardinality_equality": { "arguments": [], - "created_at": 1719485736.646685, + "created_at": 1747731927.824446, "depends_on": { "macros": [ "macro.dbt.except" @@ -10952,7 +11386,7 @@ }, "macro.dbt_utils.default__test_equal_rowcount": { "arguments": [], - "created_at": 1719485736.629295, + "created_at": 1747731927.813327, "depends_on": { "macros": [] }, @@ -10974,7 +11408,7 @@ }, "macro.dbt_utils.default__test_equality": { "arguments": [], - "created_at": 1719485736.654316, + "created_at": 1747731927.830786, "depends_on": { "macros": [ "macro.dbt_utils._is_relation", @@ -11000,7 +11434,7 @@ }, "macro.dbt_utils.default__test_expression_is_true": { "arguments": [], - "created_at": 1719485736.6476219, + "created_at": 1747731927.825386, "depends_on": { "macros": [ "macro.dbt.should_store_failures" @@ -11024,7 +11458,7 @@ }, "macro.dbt_utils.default__test_fewer_rows_than": { "arguments": [], - "created_at": 1719485736.626648, + "created_at": 1747731927.810981, "depends_on": { "macros": [] }, @@ -11046,7 +11480,7 @@ }, "macro.dbt_utils.default__test_mutually_exclusive_ranges": { "arguments": [], - "created_at": 1719485736.662267, + "created_at": 1747731927.834852, "depends_on": { "macros": [] }, @@ -11068,7 +11502,7 @@ }, "macro.dbt_utils.default__test_not_accepted_values": { "arguments": [], - "created_at": 1719485736.636965, + "created_at": 1747731927.820447, "depends_on": { "macros": [] }, @@ -11090,7 +11524,7 @@ }, "macro.dbt_utils.default__test_not_constant": { "arguments": [], - "created_at": 1719485736.634388, + "created_at": 1747731927.8177211, "depends_on": { "macros": [] }, @@ -11112,7 +11546,7 @@ }, "macro.dbt_utils.default__test_not_empty_string": { "arguments": [], - "created_at": 1719485736.655891, + "created_at": 1747731927.83168, "depends_on": { "macros": [] }, @@ -11134,7 +11568,7 @@ }, "macro.dbt_utils.default__test_not_null_proportion": { "arguments": [], - "created_at": 1719485736.6493912, + "created_at": 1747731927.827005, "depends_on": { "macros": [] }, @@ -11156,7 +11590,7 @@ }, "macro.dbt_utils.default__test_recency": { "arguments": [], - "created_at": 1719485736.633423, + "created_at": 1747731927.816565, "depends_on": { "macros": [ "macro.dbt.dateadd", @@ -11182,7 +11616,7 @@ }, "macro.dbt_utils.default__test_relationships_where": { "arguments": [], - "created_at": 1719485736.630768, + "created_at": 1747731927.814611, "depends_on": { "macros": [] }, @@ -11204,7 +11638,7 @@ }, "macro.dbt_utils.default__test_sequential_values": { "arguments": [], - "created_at": 1719485736.652041, + "created_at": 1747731927.828941, "depends_on": { "macros": [ "macro.dbt_utils.slugify", @@ -11230,7 +11664,7 @@ }, "macro.dbt_utils.default__test_unique_combination_of_columns": { "arguments": [], - "created_at": 1719485736.643641, + "created_at": 1747731927.823483, "depends_on": { "macros": [] }, @@ -11252,7 +11686,7 @@ }, "macro.dbt_utils.default__union_relations": { "arguments": [], - "created_at": 1719485736.6993861, + "created_at": 1747731927.8630772, "depends_on": { "macros": [ "macro.dbt_utils._is_relation", @@ -11279,7 +11713,7 @@ }, "macro.dbt_utils.default__unpivot": { "arguments": [], - "created_at": 1719485736.689354, + "created_at": 1747731927.8560798, "depends_on": { "macros": [ "macro.dbt_utils._is_relation", @@ -11306,7 +11740,7 @@ }, "macro.dbt_utils.default__width_bucket": { "arguments": [], - "created_at": 1719485736.7257888, + "created_at": 1747731927.880204, "depends_on": { "macros": [ "macro.dbt.safe_cast", @@ -11331,7 +11765,7 @@ }, "macro.dbt_utils.degrees_to_radians": { "arguments": [], - "created_at": 1719485736.735244, + "created_at": 1747731927.88672, "depends_on": { "macros": [] }, @@ -11353,7 +11787,7 @@ }, "macro.dbt_utils.generate_series": { "arguments": [], - "created_at": 1719485736.678059, + "created_at": 1747731927.8464909, "depends_on": { "macros": [ "macro.dbt_utils.default__generate_series" @@ -11377,7 +11811,7 @@ }, "macro.dbt_utils.generate_surrogate_key": { "arguments": [], - "created_at": 1719485736.7287118, + "created_at": 1747731927.882237, "depends_on": { "macros": [ "macro.dbt_utils.default__generate_surrogate_key" @@ -11401,7 +11835,7 @@ }, "macro.dbt_utils.get_column_values": { "arguments": [], - "created_at": 1719485736.7158039, + "created_at": 1747731927.87301, "depends_on": { "macros": [ "macro.dbt_utils.default__get_column_values" @@ -11425,7 +11859,7 @@ }, "macro.dbt_utils.get_filtered_columns_in_relation": { "arguments": [], - "created_at": 1719485736.7211049, + "created_at": 1747731927.8780348, "depends_on": { "macros": [ "macro.dbt_utils.default__get_filtered_columns_in_relation" @@ -11449,7 +11883,7 @@ }, "macro.dbt_utils.get_intervals_between": { "arguments": [], - "created_at": 1719485736.668108, + "created_at": 1747731927.839635, "depends_on": { "macros": [ "macro.dbt_utils.default__get_intervals_between" @@ -11473,7 +11907,7 @@ }, "macro.dbt_utils.get_powers_of_two": { "arguments": [], - "created_at": 1719485736.677409, + "created_at": 1747731927.845665, "depends_on": { "macros": [ "macro.dbt_utils.default__get_powers_of_two" @@ -11497,7 +11931,7 @@ }, "macro.dbt_utils.get_query_results_as_dict": { "arguments": [], - "created_at": 1719485736.726866, + "created_at": 1747731927.88096, "depends_on": { "macros": [ "macro.dbt_utils.default__get_query_results_as_dict" @@ -11521,7 +11955,7 @@ }, "macro.dbt_utils.get_relations_by_pattern": { "arguments": [], - "created_at": 1719485736.6725519, + "created_at": 1747731927.8437781, "depends_on": { "macros": [ "macro.dbt_utils.default__get_relations_by_pattern" @@ -11545,7 +11979,7 @@ }, "macro.dbt_utils.get_relations_by_prefix": { "arguments": [], - "created_at": 1719485736.67938, + "created_at": 1747731927.847949, "depends_on": { "macros": [ "macro.dbt_utils.default__get_relations_by_prefix" @@ -11569,7 +12003,7 @@ }, "macro.dbt_utils.get_single_value": { "arguments": [], - "created_at": 1719485736.732079, + "created_at": 1747731927.884848, "depends_on": { "macros": [ "macro.dbt_utils.default__get_single_value" @@ -11593,7 +12027,7 @@ }, "macro.dbt_utils.get_table_types_sql": { "arguments": [], - "created_at": 1719485736.730877, + "created_at": 1747731927.8836749, "depends_on": { "macros": [ "macro.dbt_utils.postgres__get_table_types_sql" @@ -11617,7 +12051,7 @@ }, "macro.dbt_utils.get_tables_by_pattern_sql": { "arguments": [], - "created_at": 1719485736.7118368, + "created_at": 1747731927.869857, "depends_on": { "macros": [ "macro.dbt_utils.default__get_tables_by_pattern_sql" @@ -11641,7 +12075,7 @@ }, "macro.dbt_utils.get_tables_by_prefix_sql": { "arguments": [], - "created_at": 1719485736.680733, + "created_at": 1747731927.8495731, "depends_on": { "macros": [ "macro.dbt_utils.default__get_tables_by_prefix_sql" @@ -11665,7 +12099,7 @@ }, "macro.dbt_utils.get_url_host": { "arguments": [], - "created_at": 1719485736.6209002, + "created_at": 1747731927.806067, "depends_on": { "macros": [ "macro.dbt_utils.default__get_url_host" @@ -11689,7 +12123,7 @@ }, "macro.dbt_utils.get_url_parameter": { "arguments": [], - "created_at": 1719485736.6233978, + "created_at": 1747731927.808449, "depends_on": { "macros": [ "macro.dbt_utils.default__get_url_parameter" @@ -11713,7 +12147,7 @@ }, "macro.dbt_utils.get_url_path": { "arguments": [], - "created_at": 1719485736.62192, + "created_at": 1747731927.8072262, "depends_on": { "macros": [ "macro.dbt_utils.default__get_url_path" @@ -11737,7 +12171,7 @@ }, "macro.dbt_utils.group_by": { "arguments": [], - "created_at": 1719485736.700075, + "created_at": 1747731927.863491, "depends_on": { "macros": [ "macro.dbt_utils.default__group_by" @@ -11761,7 +12195,7 @@ }, "macro.dbt_utils.haversine_distance": { "arguments": [], - "created_at": 1719485736.7356942, + "created_at": 1747731927.8871, "depends_on": { "macros": [ "macro.dbt_utils.default__haversine_distance" @@ -11785,7 +12219,7 @@ }, "macro.dbt_utils.log_info": { "arguments": [], - "created_at": 1719485736.664978, + "created_at": 1747731927.836946, "depends_on": { "macros": [ "macro.dbt_utils.default__log_info" @@ -11809,7 +12243,7 @@ }, "macro.dbt_utils.nullcheck": { "arguments": [], - "created_at": 1719485736.708089, + "created_at": 1747731927.868504, "depends_on": { "macros": [ "macro.dbt_utils.default__nullcheck" @@ -11833,7 +12267,7 @@ }, "macro.dbt_utils.nullcheck_table": { "arguments": [], - "created_at": 1719485736.671076, + "created_at": 1747731927.8426628, "depends_on": { "macros": [ "macro.dbt_utils.default__nullcheck_table" @@ -11857,7 +12291,7 @@ }, "macro.dbt_utils.pivot": { "arguments": [], - "created_at": 1719485736.719448, + "created_at": 1747731927.876304, "depends_on": { "macros": [ "macro.dbt_utils.default__pivot" @@ -11881,7 +12315,7 @@ }, "macro.dbt_utils.postgres__deduplicate": { "arguments": [], - "created_at": 1719485736.703091, + "created_at": 1747731927.8654132, "depends_on": { "macros": [] }, @@ -11903,7 +12337,7 @@ }, "macro.dbt_utils.postgres__get_table_types_sql": { "arguments": [], - "created_at": 1719485736.731321, + "created_at": 1747731927.884053, "depends_on": { "macros": [] }, @@ -11925,7 +12359,7 @@ }, "macro.dbt_utils.pretty_log_format": { "arguments": [], - "created_at": 1719485736.6625931, + "created_at": 1747731927.8352141, "depends_on": { "macros": [ "macro.dbt_utils.default__pretty_log_format" @@ -11949,7 +12383,7 @@ }, "macro.dbt_utils.pretty_time": { "arguments": [], - "created_at": 1719485736.664063, + "created_at": 1747731927.83637, "depends_on": { "macros": [ "macro.dbt_utils.default__pretty_time" @@ -11973,7 +12407,7 @@ }, "macro.dbt_utils.redshift__deduplicate": { "arguments": [], - "created_at": 1719485736.7025979, + "created_at": 1747731927.865097, "depends_on": { "macros": [ "macro.dbt_utils.default__deduplicate" @@ -11997,7 +12431,7 @@ }, "macro.dbt_utils.safe_add": { "arguments": [], - "created_at": 1719485736.7061272, + "created_at": 1747731927.867192, "depends_on": { "macros": [ "macro.dbt_utils.default__safe_add" @@ -12021,7 +12455,7 @@ }, "macro.dbt_utils.safe_divide": { "arguments": [], - "created_at": 1719485736.689705, + "created_at": 1747731927.8565161, "depends_on": { "macros": [ "macro.dbt_utils.default__safe_divide" @@ -12045,7 +12479,7 @@ }, "macro.dbt_utils.safe_subtract": { "arguments": [], - "created_at": 1719485736.670065, + "created_at": 1747731927.841585, "depends_on": { "macros": [ "macro.dbt_utils.default__safe_subtract" @@ -12069,7 +12503,7 @@ }, "macro.dbt_utils.slugify": { "arguments": [], - "created_at": 1719485736.6662788, + "created_at": 1747731927.838023, "depends_on": { "macros": [] }, @@ -12091,7 +12525,7 @@ }, "macro.dbt_utils.snowflake__deduplicate": { "arguments": [], - "created_at": 1719485736.703385, + "created_at": 1747731927.8656719, "depends_on": { "macros": [] }, @@ -12113,7 +12547,7 @@ }, "macro.dbt_utils.snowflake__width_bucket": { "arguments": [], - "created_at": 1719485736.72608, + "created_at": 1747731927.88046, "depends_on": { "macros": [] }, @@ -12135,7 +12569,7 @@ }, "macro.dbt_utils.star": { "arguments": [], - "created_at": 1719485736.6838112, + "created_at": 1747731927.85095, "depends_on": { "macros": [ "macro.dbt_utils.default__star" @@ -12159,7 +12593,7 @@ }, "macro.dbt_utils.surrogate_key": { "arguments": [], - "created_at": 1719485736.704803, + "created_at": 1747731927.866403, "depends_on": { "macros": [ "macro.dbt_utils.default__surrogate_key" @@ -12183,7 +12617,7 @@ }, "macro.dbt_utils.test_accepted_range": { "arguments": [], - "created_at": 1719485736.6351202, + "created_at": 1747731927.818371, "depends_on": { "macros": [ "macro.dbt_utils.default__test_accepted_range" @@ -12207,7 +12641,7 @@ }, "macro.dbt_utils.test_at_least_one": { "arguments": [], - "created_at": 1719485736.637673, + "created_at": 1747731927.821079, "depends_on": { "macros": [ "macro.dbt_utils.default__test_at_least_one" @@ -12231,7 +12665,7 @@ }, "macro.dbt_utils.test_cardinality_equality": { "arguments": [], - "created_at": 1719485736.64627, + "created_at": 1747731927.8240068, "depends_on": { "macros": [ "macro.dbt_utils.default__test_cardinality_equality" @@ -12255,7 +12689,7 @@ }, "macro.dbt_utils.test_equal_rowcount": { "arguments": [], - "created_at": 1719485736.6276648, + "created_at": 1747731927.811861, "depends_on": { "macros": [ "macro.dbt_utils.default__test_equal_rowcount" @@ -12279,7 +12713,7 @@ }, "macro.dbt_utils.test_equality": { "arguments": [], - "created_at": 1719485736.6529899, + "created_at": 1747731927.8296452, "depends_on": { "macros": [ "macro.dbt_utils.default__test_equality" @@ -12303,7 +12737,7 @@ }, "macro.dbt_utils.test_expression_is_true": { "arguments": [], - "created_at": 1719485736.64721, + "created_at": 1747731927.8249311, "depends_on": { "macros": [ "macro.dbt_utils.default__test_expression_is_true" @@ -12327,7 +12761,7 @@ }, "macro.dbt_utils.test_fewer_rows_than": { "arguments": [], - "created_at": 1719485736.625401, + "created_at": 1747731927.809653, "depends_on": { "macros": [ "macro.dbt_utils.default__test_fewer_rows_than" @@ -12351,7 +12785,7 @@ }, "macro.dbt_utils.test_mutually_exclusive_ranges": { "arguments": [], - "created_at": 1719485736.6603332, + "created_at": 1747731927.832834, "depends_on": { "macros": [ "macro.dbt_utils.default__test_mutually_exclusive_ranges" @@ -12375,7 +12809,7 @@ }, "macro.dbt_utils.test_not_accepted_values": { "arguments": [], - "created_at": 1719485736.6365619, + "created_at": 1747731927.819565, "depends_on": { "macros": [ "macro.dbt_utils.default__test_not_accepted_values" @@ -12399,7 +12833,7 @@ }, "macro.dbt_utils.test_not_constant": { "arguments": [], - "created_at": 1719485736.6339169, + "created_at": 1747731927.8171258, "depends_on": { "macros": [ "macro.dbt_utils.default__test_not_constant" @@ -12423,7 +12857,7 @@ }, "macro.dbt_utils.test_not_empty_string": { "arguments": [], - "created_at": 1719485736.655252, + "created_at": 1747731927.831296, "depends_on": { "macros": [ "macro.dbt_utils.default__test_not_empty_string" @@ -12447,7 +12881,7 @@ }, "macro.dbt_utils.test_not_null_proportion": { "arguments": [], - "created_at": 1719485736.64849, + "created_at": 1747731927.825979, "depends_on": { "macros": [ "macro.dbt_utils.default__test_not_null_proportion" @@ -12471,7 +12905,7 @@ }, "macro.dbt_utils.test_recency": { "arguments": [], - "created_at": 1719485736.632009, + "created_at": 1747731927.8154519, "depends_on": { "macros": [ "macro.dbt_utils.default__test_recency" @@ -12495,7 +12929,7 @@ }, "macro.dbt_utils.test_relationships_where": { "arguments": [], - "created_at": 1719485736.6303911, + "created_at": 1747731927.8140779, "depends_on": { "macros": [ "macro.dbt_utils.default__test_relationships_where" @@ -12519,7 +12953,7 @@ }, "macro.dbt_utils.test_sequential_values": { "arguments": [], - "created_at": 1719485736.650635, + "created_at": 1747731927.827792, "depends_on": { "macros": [ "macro.dbt_utils.default__test_sequential_values" @@ -12543,7 +12977,7 @@ }, "macro.dbt_utils.test_unique_combination_of_columns": { "arguments": [], - "created_at": 1719485736.642268, + "created_at": 1747731927.822668, "depends_on": { "macros": [ "macro.dbt_utils.default__test_unique_combination_of_columns" @@ -12567,7 +13001,7 @@ }, "macro.dbt_utils.union_relations": { "arguments": [], - "created_at": 1719485736.6939468, + "created_at": 1747731927.858424, "depends_on": { "macros": [ "macro.dbt_utils.default__union_relations" @@ -12591,7 +13025,7 @@ }, "macro.dbt_utils.unpivot": { "arguments": [], - "created_at": 1719485736.68751, + "created_at": 1747731927.853896, "depends_on": { "macros": [ "macro.dbt_utils.default__unpivot" @@ -12615,7 +13049,7 @@ }, "macro.dbt_utils.width_bucket": { "arguments": [], - "created_at": 1719485736.724451, + "created_at": 1747731927.8796082, "depends_on": { "macros": [ "macro.dbt_utils.default__width_bucket" @@ -12639,7 +13073,7 @@ }, "macro.jaffle_shop.drop_table_by_name": { "arguments": [], - "created_at": 1719485736.288547, + "created_at": 1747731927.554156, "depends_on": { "macros": [ "macro.dbt.run_query" @@ -12665,14 +13099,14 @@ "metadata": { "adapter_type": "postgres", "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v12.json", - "dbt_version": "1.8.0", + "dbt_version": "1.9.1", "env": {}, - "generated_at": "2024-06-27T10:55:36.063508Z", - "invocation_id": "31cdaee6-885b-4bdf-b794-4065f9530edb", + "generated_at": "2025-05-20T09:05:27.377511Z", + "invocation_id": "f85e599d-1d87-481e-95d6-020a8ea84a16", "project_id": "06e5b98c2db46f8a72cc4f66410e9b3b", "project_name": "jaffle_shop", "send_anonymous_usage_stats": true, - "user_id": "f5b1bc43-6cc6-4fd4-849c-18b31ffa1e2d" + "user_id": "09ff61b8-69e5-4614-88b7-1528cb41dfe4" }, "metrics": {}, "nodes": { @@ -12681,7 +13115,7 @@ "alias": "customers", "build_path": null, "checksum": { - "checksum": "60bd72e33da43fff3a7e7609135c17cd4468bd22afec0735dd36018bfb5af30a", + "checksum": "9f2eb13e3f280ace0666a0e18bbf66245f1c189e1362d6f3b9e276d0b3133749", "name": "sha256" }, "columns": { @@ -12689,6 +13123,7 @@ "constraints": [], "data_type": null, "description": "This is a unique identifier for a customer", + "granularity": null, "meta": {}, "name": "customer_id", "quote": null, @@ -12698,6 +13133,7 @@ "constraints": [], "data_type": null, "description": "Customer's first name. PII.", + "granularity": null, "meta": {}, "name": "first_name", "quote": null, @@ -12707,6 +13143,7 @@ "constraints": [], "data_type": null, "description": "Date (UTC) of a customer's first order", + "granularity": null, "meta": {}, "name": "first_order", "quote": null, @@ -12716,6 +13153,7 @@ "constraints": [], "data_type": null, "description": "Customer's last name. PII.", + "granularity": null, "meta": {}, "name": "last_name", "quote": null, @@ -12725,6 +13163,7 @@ "constraints": [], "data_type": null, "description": "Date (UTC) of a customer's most recent order", + "granularity": null, "meta": {}, "name": "most_recent_order", "quote": null, @@ -12734,6 +13173,7 @@ "constraints": [], "data_type": null, "description": "Count of the number of orders a customer has placed", + "granularity": null, "meta": {}, "name": "number_of_orders", "quote": null, @@ -12743,6 +13183,7 @@ "constraints": [], "data_type": null, "description": "Total value (AUD) of a customer's orders", + "granularity": null, "meta": {}, "name": "total_order_amount", "quote": null, @@ -12753,7 +13194,10 @@ "config": { "access": "protected", "alias": null, + "batch_size": null, + "begin": null, "column_types": {}, + "concurrent_batches": null, "contract": { "alias_types": true, "enforced": false @@ -12764,10 +13208,12 @@ "show": true }, "enabled": true, + "event_time": null, "full_refresh": null, "grants": {}, "group": null, "incremental_strategy": null, + "lookback": 1, "materialized": "table", "meta": {}, "on_configuration_change": "apply", @@ -12778,7 +13224,9 @@ "pre-hook": [], "quoting": {}, "schema": null, - "tags": [], + "tags": [ + "customers" + ], "unique_key": null }, "constraints": [], @@ -12787,7 +13235,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.5078778, + "created_at": 1747731928.553196, "database": "postgres", "depends_on": { "macros": [], @@ -12817,7 +13265,10 @@ "package_name": "jaffle_shop", "patch_path": "jaffle_shop://models/schema.yml", "path": "customers.sql", - "raw_code": "with customers as (\n\n select * from {{ ref('stg_customers') }}\n\n),\n\norders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\ncustomer_orders as (\n\n select\n customer_id,\n\n min(order_date) as first_order,\n max(order_date) as most_recent_order,\n count(order_id) as number_of_orders\n from orders\n\n group by customer_id\n\n),\n\ncustomer_payments as (\n\n select\n orders.customer_id,\n sum(amount) as total_amount\n\n from payments\n\n left join orders on\n payments.order_id = orders.order_id\n\n group by orders.customer_id\n\n),\n\nfinal as (\n\n select\n customers.customer_id,\n customers.first_name,\n customers.last_name,\n customer_orders.first_order,\n customer_orders.most_recent_order,\n customer_orders.number_of_orders,\n customer_payments.total_amount as customer_lifetime_value\n\n from customers\n\n left join customer_orders\n on customers.customer_id = customer_orders.customer_id\n\n left join customer_payments\n on customers.customer_id = customer_payments.customer_id\n\n)\n\nselect * from final", + "primary_key": [ + "customer_id" + ], + "raw_code": "{{ config(tags=[\"customers\"]) }}\n\nwith customers as (\n\n select * from {{ ref('stg_customers') }}\n\n),\n\norders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\ncustomer_orders as (\n\n select\n customer_id,\n\n min(order_date) as first_order,\n max(order_date) as most_recent_order,\n count(order_id) as number_of_orders\n from orders\n\n group by customer_id\n\n),\n\ncustomer_payments as (\n\n select\n orders.customer_id,\n sum(amount) as total_amount\n\n from payments\n\n left join orders on\n payments.order_id = orders.order_id\n\n group by orders.customer_id\n\n),\n\nfinal as (\n\n select\n customers.customer_id,\n customers.first_name,\n customers.last_name,\n customer_orders.first_order,\n customer_orders.most_recent_order,\n customer_orders.number_of_orders,\n customer_payments.total_amount as customer_lifetime_value\n\n from customers\n\n left join customer_orders\n on customers.customer_id = customer_orders.customer_id\n\n left join customer_payments\n on customers.customer_id = customer_payments.customer_id\n\n)\n\nselect * from final", "refs": [ { "name": "stg_customers", @@ -12839,10 +13290,16 @@ "resource_type": "model", "schema": "public", "sources": [], - "tags": [], + "tags": [ + "customers" + ], + "time_spine": null, "unique_id": "model.jaffle_shop.customers", "unrendered_config": { - "materialized": "table" + "materialized": "table", + "tags": [ + "customers" + ] }, "version": null }, @@ -12859,6 +13316,7 @@ "constraints": [], "data_type": null, "description": "Total amount (AUD) of the order", + "granularity": null, "meta": {}, "name": "amount", "quote": null, @@ -12868,6 +13326,7 @@ "constraints": [], "data_type": null, "description": "Amount of the order (AUD) paid for by bank transfer", + "granularity": null, "meta": {}, "name": "bank_transfer_amount", "quote": null, @@ -12877,6 +13336,7 @@ "constraints": [], "data_type": null, "description": "Amount of the order (AUD) paid for by coupon", + "granularity": null, "meta": {}, "name": "coupon_amount", "quote": null, @@ -12886,24 +13346,17 @@ "constraints": [], "data_type": null, "description": "Amount of the order (AUD) paid for by credit card", + "granularity": null, "meta": {}, "name": "credit_card_amount", "quote": null, "tags": [] }, - "customer_id": { - "constraints": [], - "data_type": null, - "description": "Foreign key to the customers table", - "meta": {}, - "name": "customer_id", - "quote": null, - "tags": [] - }, "gift_card_amount": { "constraints": [], "data_type": null, "description": "Amount of the order (AUD) paid for by gift card", + "granularity": null, "meta": {}, "name": "gift_card_amount", "quote": null, @@ -12913,6 +13366,7 @@ "constraints": [], "data_type": null, "description": "Date (UTC) that the order was placed", + "granularity": null, "meta": {}, "name": "order_date", "quote": null, @@ -12922,6 +13376,7 @@ "constraints": [], "data_type": null, "description": "This is a unique identifier for an order", + "granularity": null, "meta": {}, "name": "order_id", "quote": null, @@ -12931,6 +13386,7 @@ "constraints": [], "data_type": null, "description": "Orders can be one of the following statuses:\n\n| status | description |\n|----------------|------------------------------------------------------------------------------------------------------------------------|\n| placed | The order has been placed but has not yet left the warehouse |\n| shipped | The order has ben shipped to the customer and is currently in transit |\n| completed | The order has been received by the customer |\n| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse |\n| returned | The order has been returned by the customer and received at the warehouse |", + "granularity": null, "meta": {}, "name": "status", "quote": null, @@ -12941,7 +13397,10 @@ "config": { "access": "protected", "alias": null, + "batch_size": null, + "begin": null, "column_types": {}, + "concurrent_batches": null, "contract": { "alias_types": true, "enforced": false @@ -12952,10 +13411,12 @@ "show": true }, "enabled": true, + "event_time": null, "full_refresh": null, "grants": {}, "group": null, "incremental_strategy": null, + "lookback": 1, "materialized": "table", "meta": {}, "on_configuration_change": "apply", @@ -12975,7 +13436,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.511349, + "created_at": 1747731928.5560288, "database": "postgres", "depends_on": { "macros": [], @@ -13004,6 +13465,9 @@ "package_name": "jaffle_shop", "patch_path": "jaffle_shop://models/schema.yml", "path": "orders.sql", + "primary_key": [ + "order_id" + ], "raw_code": "{% set payment_methods = ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] %}\n\nwith orders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\norder_payments as (\n\n select\n order_id,\n\n {% for payment_method in payment_methods -%}\n sum(case when payment_method = '{{ payment_method }}' then amount else 0 end) as {{ payment_method }}_amount,\n {% endfor -%}\n\n sum(amount) as total_amount\n\n from payments\n\n group by order_id\n\n),\n\nfinal as (\n\n select\n orders.order_id,\n orders.customer_id,\n orders.order_date,\n orders.status,\n\n {% for payment_method in payment_methods -%}\n\n order_payments.{{ payment_method }}_amount,\n\n {% endfor -%}\n\n order_payments.total_amount as amount\n\n from orders\n\n\n left join order_payments\n on orders.order_id = order_payments.order_id\n\n)\n\nselect * from final", "refs": [ { @@ -13022,6 +13486,7 @@ "schema": "public", "sources": [], "tags": [], + "time_spine": null, "unique_id": "model.jaffle_shop.orders", "unrendered_config": { "materialized": "table" @@ -13041,6 +13506,7 @@ "constraints": [], "data_type": null, "description": "", + "granularity": null, "meta": {}, "name": "customer_id", "quote": null, @@ -13051,7 +13517,10 @@ "config": { "access": "protected", "alias": null, + "batch_size": null, + "begin": null, "column_types": {}, + "concurrent_batches": null, "contract": { "alias_types": true, "enforced": false @@ -13062,10 +13531,12 @@ "show": true }, "enabled": true, + "event_time": null, "full_refresh": null, "grants": {}, "group": null, "incremental_strategy": null, + "lookback": 1, "materialized": "view", "meta": {}, "on_configuration_change": "apply", @@ -13085,7 +13556,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.629529, + "created_at": 1747731928.6452749, "database": "postgres", "depends_on": { "macros": [], @@ -13114,6 +13585,9 @@ "package_name": "jaffle_shop", "patch_path": "jaffle_shop://models/staging/schema.yml", "path": "staging/stg_customers.sql", + "primary_key": [ + "customer_id" + ], "raw_code": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_customers') }}\n\n),\n\nrenamed as (\n\n select\n id as customer_id,\n first_name,\n last_name\n\n from source\n\n)\n\nselect * from renamed", "refs": [ { @@ -13127,6 +13601,7 @@ "schema": "public", "sources": [], "tags": [], + "time_spine": null, "unique_id": "model.jaffle_shop.stg_customers", "unrendered_config": { "materialized": "view" @@ -13146,6 +13621,7 @@ "constraints": [], "data_type": null, "description": "", + "granularity": null, "meta": {}, "name": "order_id", "quote": null, @@ -13155,6 +13631,7 @@ "constraints": [], "data_type": null, "description": "", + "granularity": null, "meta": {}, "name": "status", "quote": null, @@ -13165,7 +13642,10 @@ "config": { "access": "protected", "alias": null, + "batch_size": null, + "begin": null, "column_types": {}, + "concurrent_batches": null, "contract": { "alias_types": true, "enforced": false @@ -13176,10 +13656,12 @@ "show": true }, "enabled": true, + "event_time": null, "full_refresh": null, "grants": {}, "group": null, "incremental_strategy": null, + "lookback": 1, "materialized": "view", "meta": {}, "on_configuration_change": "apply", @@ -13199,7 +13681,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.630356, + "created_at": 1747731928.6457899, "database": "postgres", "depends_on": { "macros": [], @@ -13228,6 +13710,9 @@ "package_name": "jaffle_shop", "patch_path": "jaffle_shop://models/staging/schema.yml", "path": "staging/stg_orders.sql", + "primary_key": [ + "order_id" + ], "raw_code": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_orders') }}\n\n),\n\nrenamed as (\n\n select\n id as order_id,\n user_id as customer_id,\n order_date,\n status\n\n from source\n\n)\n\nselect * from renamed", "refs": [ { @@ -13241,6 +13726,7 @@ "schema": "public", "sources": [], "tags": [], + "time_spine": null, "unique_id": "model.jaffle_shop.stg_orders", "unrendered_config": { "materialized": "view" @@ -13260,6 +13746,7 @@ "constraints": [], "data_type": null, "description": "", + "granularity": null, "meta": {}, "name": "payment_id", "quote": null, @@ -13269,6 +13756,7 @@ "constraints": [], "data_type": null, "description": "", + "granularity": null, "meta": {}, "name": "payment_method", "quote": null, @@ -13279,7 +13767,10 @@ "config": { "access": "protected", "alias": null, + "batch_size": null, + "begin": null, "column_types": {}, + "concurrent_batches": null, "contract": { "alias_types": true, "enforced": false @@ -13290,10 +13781,12 @@ "show": true }, "enabled": true, + "event_time": null, "full_refresh": null, "grants": {}, "group": null, "incremental_strategy": null, + "lookback": 1, "materialized": "view", "meta": {}, "on_configuration_change": "apply", @@ -13313,7 +13806,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.6316102, + "created_at": 1747731928.64695, "database": "postgres", "depends_on": { "macros": [], @@ -13342,6 +13835,9 @@ "package_name": "jaffle_shop", "patch_path": "jaffle_shop://models/staging/schema.yml", "path": "staging/stg_payments.sql", + "primary_key": [ + "payment_id" + ], "raw_code": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_payments') }}\n\n),\n\nrenamed as (\n\n select\n id as payment_id,\n order_id,\n payment_method,\n\n -- `amount` is currently stored in cents, so we convert it to dollars\n amount / 100 as amount\n\n from source\n\n)\n\nselect * from renamed", "refs": [ { @@ -13355,6 +13851,7 @@ "schema": "public", "sources": [], "tags": [], + "time_spine": null, "unique_id": "model.jaffle_shop.stg_payments", "unrendered_config": { "materialized": "view" @@ -13371,7 +13868,10 @@ "columns": {}, "config": { "alias": null, + "batch_size": null, + "begin": null, "column_types": {}, + "concurrent_batches": null, "contract": { "alias_types": true, "enforced": false @@ -13383,12 +13883,23 @@ "show": true }, "enabled": true, + "event_time": null, "full_refresh": null, "grants": {}, "group": null, "incremental_strategy": null, + "lookback": 1, "materialized": "seed", - "meta": {}, + "meta": { + "cosmos": { + "profile_config": { + "profile_mapping": { + "threads": 2 + }, + "profile_name": "postgres_profile" + } + } + }, "on_configuration_change": "apply", "on_schema_change": "ignore", "packages": [], @@ -13401,7 +13912,7 @@ "tags": [], "unique_key": null }, - "created_at": 1719485737.409044, + "created_at": 1747731928.439897, "database": "postgres", "depends_on": { "macros": [] @@ -13416,7 +13927,16 @@ "raw_customers" ], "group": null, - "meta": {}, + "meta": { + "cosmos": { + "profile_config": { + "profile_mapping": { + "threads": 2 + }, + "profile_name": "postgres_profile" + } + } + }, "name": "raw_customers", "original_file_path": "seeds/raw_customers.csv", "package_name": "jaffle_shop", @@ -13425,11 +13945,22 @@ "raw_code": "", "relation_name": "\"postgres\".\"public\".\"raw_customers\"", "resource_type": "seed", - "root_path": "/Users/tati/Code/cosmos-clean/astronomer-cosmos/dev/dags/dbt/jaffle_shop", + "root_path": "/Users/tati/Code/cosmos-fresh/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public", "tags": [], "unique_id": "seed.jaffle_shop.raw_customers", - "unrendered_config": {} + "unrendered_config": { + "meta": { + "cosmos": { + "profile_config": { + "profile_mapping": { + "threads": 2 + }, + "profile_name": "postgres_profile" + } + } + } + } }, "seed.jaffle_shop.raw_orders": { "alias": "raw_orders", @@ -13441,7 +13972,10 @@ "columns": {}, "config": { "alias": null, + "batch_size": null, + "begin": null, "column_types": {}, + "concurrent_batches": null, "contract": { "alias_types": true, "enforced": false @@ -13453,12 +13987,23 @@ "show": true }, "enabled": true, + "event_time": null, "full_refresh": null, "grants": {}, "group": null, "incremental_strategy": null, + "lookback": 1, "materialized": "seed", - "meta": {}, + "meta": { + "cosmos": { + "profile_config": { + "profile_mapping": { + "threads": 2 + }, + "profile_name": "postgres_profile" + } + } + }, "on_configuration_change": "apply", "on_schema_change": "ignore", "packages": [], @@ -13471,7 +14016,7 @@ "tags": [], "unique_key": null }, - "created_at": 1719485737.411613, + "created_at": 1747731928.441768, "database": "postgres", "depends_on": { "macros": [] @@ -13486,7 +14031,16 @@ "raw_orders" ], "group": null, - "meta": {}, + "meta": { + "cosmos": { + "profile_config": { + "profile_mapping": { + "threads": 2 + }, + "profile_name": "postgres_profile" + } + } + }, "name": "raw_orders", "original_file_path": "seeds/raw_orders.csv", "package_name": "jaffle_shop", @@ -13495,11 +14049,22 @@ "raw_code": "", "relation_name": "\"postgres\".\"public\".\"raw_orders\"", "resource_type": "seed", - "root_path": "/Users/tati/Code/cosmos-clean/astronomer-cosmos/dev/dags/dbt/jaffle_shop", + "root_path": "/Users/tati/Code/cosmos-fresh/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public", "tags": [], "unique_id": "seed.jaffle_shop.raw_orders", - "unrendered_config": {} + "unrendered_config": { + "meta": { + "cosmos": { + "profile_config": { + "profile_mapping": { + "threads": 2 + }, + "profile_name": "postgres_profile" + } + } + } + } }, "seed.jaffle_shop.raw_payments": { "alias": "raw_payments", @@ -13511,7 +14076,10 @@ "columns": {}, "config": { "alias": null, + "batch_size": null, + "begin": null, "column_types": {}, + "concurrent_batches": null, "contract": { "alias_types": true, "enforced": false @@ -13523,12 +14091,23 @@ "show": true }, "enabled": true, + "event_time": null, "full_refresh": null, "grants": {}, "group": null, "incremental_strategy": null, + "lookback": 1, "materialized": "seed", - "meta": {}, + "meta": { + "cosmos": { + "profile_config": { + "profile_mapping": { + "threads": 2 + }, + "profile_name": "postgres_profile" + } + } + }, "on_configuration_change": "apply", "on_schema_change": "ignore", "packages": [], @@ -13541,7 +14120,7 @@ "tags": [], "unique_key": null }, - "created_at": 1719485737.4141219, + "created_at": 1747731928.443347, "database": "postgres", "depends_on": { "macros": [] @@ -13556,7 +14135,16 @@ "raw_payments" ], "group": null, - "meta": {}, + "meta": { + "cosmos": { + "profile_config": { + "profile_mapping": { + "threads": 2 + }, + "profile_name": "postgres_profile" + } + } + }, "name": "raw_payments", "original_file_path": "seeds/raw_payments.csv", "package_name": "jaffle_shop", @@ -13565,11 +14153,22 @@ "raw_code": "", "relation_name": "\"postgres\".\"public\".\"raw_payments\"", "resource_type": "seed", - "root_path": "/Users/tati/Code/cosmos-clean/astronomer-cosmos/dev/dags/dbt/jaffle_shop", + "root_path": "/Users/tati/Code/cosmos-fresh/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public", "tags": [], "unique_id": "seed.jaffle_shop.raw_payments", - "unrendered_config": {} + "unrendered_config": { + "meta": { + "cosmos": { + "profile_config": { + "profile_mapping": { + "threads": 2 + }, + "profile_name": "postgres_profile" + } + } + } + } }, "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3": { "alias": "accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758", @@ -13605,7 +14204,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.607407, + "created_at": 1747731928.621603, "database": "postgres", "depends_on": { "macros": [ @@ -13702,7 +14301,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.637758, + "created_at": 1747731928.652761, "database": "postgres", "depends_on": { "macros": [ @@ -13800,7 +14399,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.6424909, + "created_at": 1747731928.658643, "database": "postgres", "depends_on": { "macros": [ @@ -13897,7 +14496,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.5844128, + "created_at": 1747731928.61806, "database": "postgres", "depends_on": { "macros": [ @@ -13984,7 +14583,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.6199622, + "created_at": 1747731928.6392431, "database": "postgres", "depends_on": { "macros": [ @@ -14071,7 +14670,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.6258051, + "created_at": 1747731928.643073, "database": "postgres", "depends_on": { "macros": [ @@ -14158,7 +14757,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.624228, + "created_at": 1747731928.641845, "database": "postgres", "depends_on": { "macros": [ @@ -14245,7 +14844,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.622118, + "created_at": 1747731928.6405818, "database": "postgres", "depends_on": { "macros": [ @@ -14298,93 +14897,6 @@ "unique_id": "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59", "unrendered_config": {} }, - "test.jaffle_shop.not_null_orders_customer_id.c5f02694af": { - "alias": "not_null_orders_customer_id", - "attached_node": "model.jaffle_shop.orders", - "build_path": null, - "checksum": { - "checksum": "", - "name": "none" - }, - "column_name": "customer_id", - "columns": {}, - "compiled_path": null, - "config": { - "alias": null, - "database": null, - "enabled": true, - "error_if": "!= 0", - "fail_calc": "count(*)", - "group": null, - "limit": null, - "materialized": "test", - "meta": {}, - "schema": "dbt_test__audit", - "severity": "ERROR", - "store_failures": null, - "store_failures_as": null, - "tags": [], - "warn_if": "!= 0", - "where": null - }, - "contract": { - "alias_types": true, - "checksum": null, - "enforced": false - }, - "created_at": 1719485737.587272, - "database": "postgres", - "depends_on": { - "macros": [ - "macro.dbt.test_not_null" - ], - "nodes": [ - "model.jaffle_shop.orders" - ] - }, - "description": "", - "docs": { - "node_color": null, - "show": true - }, - "file_key_name": "models.orders", - "fqn": [ - "jaffle_shop", - "not_null_orders_customer_id" - ], - "group": null, - "language": "sql", - "meta": {}, - "metrics": [], - "name": "not_null_orders_customer_id", - "original_file_path": "models/schema.yml", - "package_name": "jaffle_shop", - "patch_path": null, - "path": "not_null_orders_customer_id.sql", - "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", - "refs": [ - { - "name": "orders", - "package": null, - "version": null - } - ], - "relation_name": null, - "resource_type": "test", - "schema": "public_dbt_test__audit", - "sources": [], - "tags": [], - "test_metadata": { - "kwargs": { - "column_name": "customer_id", - "model": "{{ get_where_subquery(ref('orders')) }}" - }, - "name": "not_null", - "namespace": null - }, - "unique_id": "test.jaffle_shop.not_null_orders_customer_id.c5f02694af", - "unrendered_config": {} - }, "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a": { "alias": "not_null_orders_gift_card_amount", "attached_node": "model.jaffle_shop.orders", @@ -14419,7 +14931,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.6277661, + "created_at": 1747731928.644136, "database": "postgres", "depends_on": { "macros": [ @@ -14506,7 +15018,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.586345, + "created_at": 1747731928.62046, "database": "postgres", "depends_on": { "macros": [ @@ -14593,7 +15105,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.633585, + "created_at": 1747731928.649, "database": "postgres", "depends_on": { "macros": [ @@ -14681,7 +15193,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.6366348, + "created_at": 1747731928.651482, "database": "postgres", "depends_on": { "macros": [ @@ -14769,7 +15281,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.641508, + "created_at": 1747731928.657465, "database": "postgres", "depends_on": { "macros": [ @@ -14823,102 +15335,6 @@ "unique_id": "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075", "unrendered_config": {} }, - "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2": { - "alias": "relationships_orders_customer_id__customer_id__ref_customers_", - "attached_node": "model.jaffle_shop.orders", - "build_path": null, - "checksum": { - "checksum": "", - "name": "none" - }, - "column_name": "customer_id", - "columns": {}, - "compiled_path": null, - "config": { - "alias": null, - "database": null, - "enabled": true, - "error_if": "!= 0", - "fail_calc": "count(*)", - "group": null, - "limit": null, - "materialized": "test", - "meta": {}, - "schema": "dbt_test__audit", - "severity": "ERROR", - "store_failures": null, - "store_failures_as": null, - "tags": [], - "warn_if": "!= 0", - "where": null - }, - "contract": { - "alias_types": true, - "checksum": null, - "enforced": false - }, - "created_at": 1719485737.590184, - "database": "postgres", - "depends_on": { - "macros": [ - "macro.dbt.test_relationships", - "macro.dbt.get_where_subquery" - ], - "nodes": [ - "model.jaffle_shop.customers", - "model.jaffle_shop.orders" - ] - }, - "description": "", - "docs": { - "node_color": null, - "show": true - }, - "file_key_name": "models.orders", - "fqn": [ - "jaffle_shop", - "relationships_orders_customer_id__customer_id__ref_customers_" - ], - "group": null, - "language": "sql", - "meta": {}, - "metrics": [], - "name": "relationships_orders_customer_id__customer_id__ref_customers_", - "original_file_path": "models/schema.yml", - "package_name": "jaffle_shop", - "patch_path": null, - "path": "relationships_orders_customer_id__customer_id__ref_customers_.sql", - "raw_code": "{{ test_relationships(**_dbt_generic_test_kwargs) }}", - "refs": [ - { - "name": "customers", - "package": null, - "version": null - }, - { - "name": "orders", - "package": null, - "version": null - } - ], - "relation_name": null, - "resource_type": "test", - "schema": "public_dbt_test__audit", - "sources": [], - "tags": [], - "test_metadata": { - "kwargs": { - "column_name": "customer_id", - "field": "customer_id", - "model": "{{ get_where_subquery(ref('orders')) }}", - "to": "ref('customers')" - }, - "name": "relationships", - "namespace": null - }, - "unique_id": "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2", - "unrendered_config": {} - }, "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1": { "alias": "unique_customers_customer_id", "attached_node": "model.jaffle_shop.customers", @@ -14953,7 +15369,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.583136, + "created_at": 1747731928.6167219, "database": "postgres", "depends_on": { "macros": [ @@ -15040,7 +15456,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.5853949, + "created_at": 1747731928.619267, "database": "postgres", "depends_on": { "macros": [ @@ -15127,7 +15543,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.6324039, + "created_at": 1747731928.647725, "database": "postgres", "depends_on": { "macros": [ @@ -15215,7 +15631,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.635364, + "created_at": 1747731928.650167, "database": "postgres", "depends_on": { "macros": [ @@ -15303,7 +15719,7 @@ "checksum": null, "enforced": false }, - "created_at": 1719485737.640539, + "created_at": 1747731928.656186, "database": "postgres", "depends_on": { "macros": [ @@ -15404,9 +15820,6 @@ "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59": [ "model.jaffle_shop.orders" ], - "test.jaffle_shop.not_null_orders_customer_id.c5f02694af": [ - "model.jaffle_shop.orders" - ], "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a": [ "model.jaffle_shop.orders" ], @@ -15422,10 +15835,6 @@ "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075": [ "model.jaffle_shop.stg_payments" ], - "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2": [ - "model.jaffle_shop.customers", - "model.jaffle_shop.orders" - ], "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1": [ "model.jaffle_shop.customers" ], diff --git a/dev/dags/simple_dag_async.py b/dev/dags/simple_dag_async.py index 79803126c2..237cf0c2ff 100644 --- a/dev/dags/simple_dag_async.py +++ b/dev/dags/simple_dag_async.py @@ -38,6 +38,10 @@ catchup=False, dag_id="simple_dag_async", tags=["simple"], - operator_args={"location": "US", "install_deps": True}, + operator_args={ + "location": "US", + "install_deps": True, + "full_refresh": True, + }, ) # [END airflow_async_execution_mode_example] diff --git a/docs/configuration/callbacks.rst b/docs/configuration/callbacks.rst index 6c637b9d59..387bf1014d 100644 --- a/docs/configuration/callbacks.rst +++ b/docs/configuration/callbacks.rst @@ -89,16 +89,18 @@ Below, find an example of a callback method that raises an exception if the quer slow_query_threshold = 10 - run_results_path = Path(project_dir, "run_results.json") + target_path = f"{project_dir}/target" + run_results_path = Path(target_path, "run_results.json") if run_results_path.exists(): with open(run_results_path) as fp: run_results = json.load(fp) - node_name = run_results["unique_id"] - execution_time = run_results["execution_time"] - if execution_time > slow_query_threshold: - raise TimeoutError( - f"The query for the node {node_name} took too long: {execution_time}" - ) + for result in run_results["results"]: + node_name = result["unique_id"] + execution_time = result["execution_time"] + if execution_time > slow_query_threshold: + raise TimeoutError( + f"The query for the node {node_name} took too long: {execution_time}" + ) Users can use the same approach to call the data observability platform `montecarlo `_ or other services. diff --git a/docs/configuration/cosmos-conf.rst b/docs/configuration/cosmos-conf.rst index a3dcff0e02..7ff0ddfa3a 100644 --- a/docs/configuration/cosmos-conf.rst +++ b/docs/configuration/cosmos-conf.rst @@ -178,11 +178,41 @@ This page lists all available Airflow configurations that affect ``astronomer-co - Default: ``True`` - Environment Variable: ``AIRFLOW__COSMOS__ENABLE_TEARDOWN_ASYNC_TASK`` +.. _use_dataset_airflow3_uri_standard: + `use_dataset_airflow3_uri_standard`_: (Introduced in Cosmos 1.10.0): Changes Cosmos Dataset (Asset) URIs to be Airflow 3 compliant. Since this would be a breaking change, it is False by default in Cosmos 1.x. - Default: ``False`` - Environment Variable: ``AIRFLOW__COSMOS__USE_DATASET_AIRFLOW3_URI_STANDARD`` +.. _enable_memory_optimised_imports: + +`enable_memory_optimised_imports`_: + (Introduced in Cosmos 1.10.1): Eager imports in cosmos/__init__.py expose all Cosmos classes at the top level, + which can significantly increase memory usage—even when Cosmos is just installed but not actively used. This option allows + disabling those eager imports to reduce memory footprint. When enabled, users must access Cosmos classes via their full + module paths, avoiding the overhead of importing unused modules and classes. + + - Default: ``False`` + - Environment Variable: ``AIRFLOW__COSMOS__ENABLE_MEMORY_OPTIMISED_IMPORTS`` + + .. note:: + This option will become the default behavior in Cosmos 2.0.0, where all eager imports will be removed from ``cosmos/__init__.py``. + + As an example, when this option is enabled, the following is an example of specifying the imports with full module paths: + + .. literalinclude:: ../../dev/dags/basic_cosmos_dag_full_module_path_imports.py + :language: python + :start-after: [START cosmos_explicit_imports] + :end-before: [END cosmos_explicit_imports] + + as opposed to the following approach you might be have when this option is disabled (default): + + .. literalinclude:: ../../dev/dags/basic_cosmos_dag.py + :language: python + :start-after: [START cosmos_init_imports] + :end-before: [END cosmos_init_imports] + [openlineage] ~~~~~~~~~~~~~ diff --git a/docs/configuration/operator-args.rst b/docs/configuration/operator-args.rst index bca4d48999..69d3ce846b 100644 --- a/docs/configuration/operator-args.rst +++ b/docs/configuration/operator-args.rst @@ -97,6 +97,7 @@ dbt-related - ``full_refresh``: If True, then full refresh the node. This only applies to model and seed nodes. - ``copy_dbt_packages``: (new in v1.10) When using ``ExecutionMode.LOCAL`` or ``ExecutionMode.VIRTUALENV``, copy the dbt project ``dbt_packages`` instead of creating symbolic links, so Cosmos can run ``dbt deps`` incrementally. - ``install_deps``: (deprecated in v1.9, use ``ProjectConfig.install_dbt_deps`` onwards) When using ``ExecutionMode.LOCAL`` or ``ExecutionMode.VIRTUALENV``, run ``dbt deps`` every time a task is executed. +- ``manifest_filepath`` (new in v1.10.1): When using ``ExecutionMode.LOCAL`` or ``ExecutionMode.VIRTUALENV``, use the user-defined ``manifest.json`` file. Airflow-related diff --git a/pyproject.toml b/pyproject.toml index 9f648ecaa6..ba4efae51c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,7 +116,7 @@ gcp-cloud-run-job = [ ] [project.entry-points.apache_airflow_provider] -provider_info = "cosmos:get_provider_info" +provider_info = "cosmos.provider_info:get_provider_info" [project.entry-points."airflow.plugins"] cosmos = "cosmos.plugin:CosmosPlugin" diff --git a/scripts/test/integration.sh b/scripts/test/integration.sh index bf19b4248f..60bb57e057 100644 --- a/scripts/test/integration.sh +++ b/scripts/test/integration.sh @@ -17,6 +17,7 @@ pytest -vv \ --cov=cosmos \ --cov-report=term-missing \ --cov-report=xml \ + --durations=0 \ -m 'integration' \ --ignore=tests/perf \ --ignore=tests/test_async_example_dag.py \ diff --git a/scripts/test/pre-install-airflow.sh b/scripts/test/pre-install-airflow.sh index edc87c9dc0..e56cd79903 100755 --- a/scripts/test/pre-install-airflow.sh +++ b/scripts/test/pre-install-airflow.sh @@ -26,9 +26,7 @@ mv /tmp/constraint.txt.tmp /tmp/constraint.txt pip install uv uv pip install pip --upgrade -uv pip install "apache-airflow==$AIRFLOW_VERSION" --constraint /tmp/constraint.txt -uv pip install apache-airflow-providers-docker --constraint /tmp/constraint.txt -uv pip install apache-airflow-providers-postgres --constraint /tmp/constraint.txt +uv pip install "apache-airflow==$AIRFLOW_VERSION" apache-airflow-providers-docker apache-airflow-providers-postgres --constraint /tmp/constraint.txt # Due to issue https://github.com/fsspec/gcsfs/issues/664 uv pip install "gcsfs<2025.3.0" @@ -44,10 +42,21 @@ elif [ "$AIRFLOW_VERSION" = "2.7" ] ; then uv pip install "apache-airflow-providers-cncf-kubernetes" --constraint /tmp/constraint.txt uv pip install "apache-airflow-providers-google>10.11" "apache-airflow==$AIRFLOW_VERSION" uv pip install apache-airflow-providers-microsoft-azure --constraint /tmp/constraint.txt -else +elif [ "$AIRFLOW_VERSION" = "2.8" ] ; then uv pip install "apache-airflow-providers-amazon[s3fs]" --constraint /tmp/constraint.txt uv pip install "apache-airflow-providers-cncf-kubernetes" --constraint /tmp/constraint.txt - + uv pip install "apache-airflow-providers-google<=10.26" "apache-airflow==$AIRFLOW_VERSION" + # The Airflow 2.8 constraints file at + # https://raw.githubusercontent.com/apache/airflow/constraints-2.8.0/constraints-3.11.txt + # specifies apache-airflow-providers-microsoft-azure==8.4.0. However, our Azure connection setup in the CI, + # previously led to authentication issues with this version. This issue got resolved in + # apache-airflow-providers-microsoft-azure==8.5.0. Hence, we are using apache-airflow-providers-microsoft-azure>=8.5.0 + # and skipping installation with constraints, as the specified version does not meet our requirements. + uv pip install "apache-airflow-providers-microsoft-azure>=8.5.0" "apache-airflow==$AIRFLOW_VERSION" +elif [ "$AIRFLOW_VERSION" = "2.9" ] ; then + uv pip install "apache-airflow-providers-amazon[s3fs]" --constraint /tmp/constraint.txt + uv pip install "apache-airflow-providers-cncf-kubernetes" --constraint /tmp/constraint.txt + uv pip install "apache-airflow-providers-microsoft-azure" --constraint /tmp/constraint.txt # The Airflow 2.9 constraints file at # https://raw.githubusercontent.com/apache/airflow/constraints-2.9.0/constraints-3.11.txt # specifies apache-airflow-providers-google==10.16.0. However, our CI setup uses a Google connection without a token, @@ -56,14 +65,11 @@ else # we are using apache-airflow-providers-google>=10.17.0 and skipping constraints installation, as the specified # version does not meet our requirements. uv pip install "apache-airflow-providers-google>=10.17.0" "apache-airflow==$AIRFLOW_VERSION" - - # The Airflow 2.8 constraints file at - # https://raw.githubusercontent.com/apache/airflow/constraints-2.8.0/constraints-3.11.txt - # specifies apache-airflow-providers-microsoft-azure==8.4.0. However, our Azure connection setup in the CI, - # previously led to authentication issues with this version. This issue got resolved in - # apache-airflow-providers-microsoft-azure==8.5.0. Hence, we are using apache-airflow-providers-microsoft-azure>=8.5.0 - # and skipping installation with constraints, as the specified version does not meet our requirements. - uv pip install "apache-airflow-providers-microsoft-azure>=8.5.0" "apache-airflow==$AIRFLOW_VERSION" +else + uv pip install "apache-airflow-providers-amazon[s3fs]" --constraint /tmp/constraint.txt + uv pip install "apache-airflow-providers-cncf-kubernetes" --constraint /tmp/constraint.txt + uv pip install "apache-airflow-providers-google" --constraint /tmp/constraint.txt + uv pip install "apache-airflow-providers-microsoft-azure" --constraint /tmp/constraint.txt fi rm /tmp/constraint.txt diff --git a/tests/dbt/test_project.py b/tests/dbt/test_project.py index 85ecfb80b3..7038d3b75d 100644 --- a/tests/dbt/test_project.py +++ b/tests/dbt/test_project.py @@ -3,19 +3,81 @@ from unittest.mock import patch import pytest +import yaml -from cosmos.constants import DBT_PACKAGES_FOLDER, PACKAGE_LOCKFILE_YML +from cosmos.constants import DBT_DEFAULT_PACKAGES_FOLDER, DBT_PROJECT_FILENAME, PACKAGE_LOCKFILE_YML from cosmos.dbt.project import ( change_working_directory, copy_dbt_packages, + copy_manifest_file_if_exists, create_symlinks, environ, + get_dbt_packages_subpath, has_non_empty_dependencies_file, ) DBT_PROJECTS_ROOT_DIR = Path(__file__).parent.parent.parent / "dev/dags/dbt" +def test_copy_manifest_file_if_exists(tmpdir): + source_manifest = tmpdir / "manifest.json" + source_manifest.write_text('{"mock_key": "mock_value"}', encoding="utf-8") + + dbt_project_folder = tmpdir / "dbt_project" + + target_manifest_path = dbt_project_folder / "target/manifest.json" + assert not target_manifest_path.exists() + + copy_manifest_file_if_exists(source_manifest, dbt_project_folder) + + assert target_manifest_path.exists() + assert target_manifest_path.read_text(encoding="utf-8") == '{"mock_key": "mock_value"}' + + +def test_does_nothing_if_manifest_missing(tmpdir): + source_manifest = tmpdir / "nonexistent_manifest.json" + dbt_project_folder = tmpdir / "dbt_project" + + copy_manifest_file_if_exists(source_manifest, dbt_project_folder) + + target_folder = dbt_project_folder / "target" + assert not target_folder.exists() + + +def write_dbt_project_yml(path: Path, content: dict): + with open(path / "dbt_project.yml", "w") as fp: + yaml.dump(content, fp) + return path + + +def test_returns_default_when_file_missing(tmpdir): + result = get_dbt_packages_subpath(tmpdir) + assert result == "dbt_packages" + + +def test_returns_default_when_key_missing(tmpdir): + write_dbt_project_yml(tmpdir, {"name": "test_project"}) + result = get_dbt_packages_subpath(tmpdir) + assert result == "dbt_packages" + + +def test_returns_default_on_invalid_yaml(tmpdir, caplog): + # Write malformed YAML + bad_content = ":\nbad_yaml: [::" + path = tmpdir / DBT_PROJECT_FILENAME + path.write_text(bad_content, "utf-8") + + result = get_dbt_packages_subpath(tmpdir) + assert result == DBT_DEFAULT_PACKAGES_FOLDER + assert "Unable to read" in caplog.text + + +def test_returns_custom_path_when_defined(tmpdir): + write_dbt_project_yml(tmpdir, {"packages-install-path": "custom_dbt_packages"}) + result = get_dbt_packages_subpath(tmpdir) + assert result == "custom_dbt_packages" + + def test_create_symlinks(tmp_path): """Tests that symlinks are created for expected files in the dbt project directory.""" tmp_dir = tmp_path / "dbt-project" @@ -92,7 +154,7 @@ def test_copy_dbt_packages_all_cases(mock_logger, mock_dir, mock_makedirs, mock_ assert mock_makedirs.call_count == 2 mock_copytree.assert_called_once_with( - source_folder / DBT_PACKAGES_FOLDER, target_folder / DBT_PACKAGES_FOLDER, dirs_exist_ok=True + source_folder / DBT_DEFAULT_PACKAGES_FOLDER, target_folder / DBT_DEFAULT_PACKAGES_FOLDER, dirs_exist_ok=True ) mock_copy2.assert_called_once_with(source_folder / PACKAGE_LOCKFILE_YML, target_folder / PACKAGE_LOCKFILE_YML) diff --git a/tests/listeners/test_dag_run_listener.py b/tests/listeners/test_dag_run_listener.py index 77e30f8466..456dea9c1a 100644 --- a/tests/listeners/test_dag_run_listener.py +++ b/tests/listeners/test_dag_run_listener.py @@ -1,6 +1,6 @@ import logging import uuid -from datetime import datetime +from datetime import datetime, timedelta, timezone from pathlib import Path from unittest.mock import patch @@ -13,12 +13,15 @@ from cosmos import DbtRunLocalOperator, ProfileConfig, ProjectConfig from cosmos.airflow.dag import DbtDag from cosmos.airflow.task_group import DbtTaskGroup +from cosmos.constants import _AIRFLOW3_MAJOR_VERSION from cosmos.listeners.dag_run_listener import on_dag_run_failed, on_dag_run_success, total_cosmos_tasks from cosmos.profiles import PostgresUserPasswordProfileMapping DBT_ROOT_PATH = Path(__file__).parent.parent.parent / "dev/dags/dbt" DBT_PROJECT_NAME = "jaffle_shop" +AIRFLOW_VERSION_MAJOR = version.parse(airflow_version).major + profile_config = ProfileConfig( profile_name="default", target_name="dev", @@ -79,8 +82,6 @@ def test_not_cosmos_dag(): assert total_cosmos_tasks(dag) == 0 -# TODO: Make test compatible with Airflow 3.0. Issue:https://github.com/astronomer/astronomer-cosmos/issues/1703 -@pytest.mark.skipif(version.parse(airflow_version).major == 3, reason="Test need to be updated for Airflow 3.0") @pytest.mark.integration @patch("cosmos.listeners.dag_run_listener.telemetry.emit_usage_metrics_if_enabled") def test_on_dag_run_success(mock_emit_usage_metrics_if_enabled, caplog): @@ -95,10 +96,25 @@ def test_on_dag_run_success(mock_emit_usage_metrics_if_enabled, caplog): dag_id="basic_cosmos_dag", ) run_id = str(uuid.uuid1()) - dag_run = dag.create_dagrun( - state=State.NONE, - run_id=run_id, - ) + + run_after = datetime.now(timezone.utc) - timedelta(seconds=1) + if AIRFLOW_VERSION_MAJOR < _AIRFLOW3_MAJOR_VERSION: + # Airflow 2 + dag_run = dag.create_dagrun( + state=State.NONE, + run_id=run_id, + ) + else: + # Airflow 3 + from airflow.utils.types import DagRunTriggeredByType, DagRunType + + dag_run = dag.create_dagrun( + state=State.NONE, + run_id=run_id, + run_after=run_after, + run_type=DagRunType.MANUAL, + triggered_by=DagRunTriggeredByType.TIMETABLE, + ) on_dag_run_success(dag_run, msg="test success") assert "Running on_dag_run_success" in caplog.text @@ -106,8 +122,6 @@ def test_on_dag_run_success(mock_emit_usage_metrics_if_enabled, caplog): assert mock_emit_usage_metrics_if_enabled.call_count == 1 -# TODO: Make test compatible with Airflow 3.0. Issue:https://github.com/astronomer/astronomer-cosmos/issues/1703 -@pytest.mark.skipif(version.parse(airflow_version).major == 3, reason="Test need to be updated for Airflow 3.0") @pytest.mark.integration @patch("cosmos.listeners.dag_run_listener.telemetry.emit_usage_metrics_if_enabled") def test_on_dag_run_failed(mock_emit_usage_metrics_if_enabled, caplog): @@ -122,10 +136,24 @@ def test_on_dag_run_failed(mock_emit_usage_metrics_if_enabled, caplog): dag_id="basic_cosmos_dag", ) run_id = str(uuid.uuid1()) - dag_run = dag.create_dagrun( - state=State.FAILED, - run_id=run_id, - ) + run_after = datetime.now(timezone.utc) - timedelta(seconds=1) + if AIRFLOW_VERSION_MAJOR < _AIRFLOW3_MAJOR_VERSION: + # Airflow 2 + dag_run = dag.create_dagrun( + state=State.NONE, + run_id=run_id, + ) + else: + # Airflow 3 + from airflow.utils.types import DagRunTriggeredByType, DagRunType + + dag_run = dag.create_dagrun( + state=State.NONE, + run_id=run_id, + run_after=run_after, + run_type=DagRunType.MANUAL, + triggered_by=DagRunTriggeredByType.TIMETABLE, + ) on_dag_run_failed(dag_run, msg="test failed") assert "Running on_dag_run_failed" in caplog.text diff --git a/tests/operators/_asynchronous/test_base.py b/tests/operators/_asynchronous/test_base.py index 4a1e953bbb..e55737f3c4 100644 --- a/tests/operators/_asynchronous/test_base.py +++ b/tests/operators/_asynchronous/test_base.py @@ -78,7 +78,7 @@ def test_teardown_execute(mock_build_and_run_cmd): profile_config=Mock(), project_dir="fake-dir", ) - operator.execute({}) + operator.execute({"run_id": "test_run_id"}) mock_build_and_run_cmd.assert_called_once() @@ -150,10 +150,13 @@ def test_setup_execute(mock_operator_params): op = SetupAsyncOperator(**mock_operator_params) with patch.object(op, "build_and_run_cmd") as mock_build_and_run: - op.execute(context={}) + op.execute(context={"run_id": "test_run_id"}) mock_build_and_run.assert_called_once_with( - context={}, cmd_flags=op.dbt_cmd_flags, run_as_async=True, async_context={"profile_type": "bigquery"} + context={"run_id": "test_run_id"}, + cmd_flags=op.dbt_cmd_flags, + run_as_async=True, + async_context={"profile_type": "bigquery", "run_id": "test_run_id"}, ) diff --git a/tests/operators/_asynchronous/test_bigquery.py b/tests/operators/_asynchronous/test_bigquery.py index ab50e4fe9d..9f7a28093a 100644 --- a/tests/operators/_asynchronous/test_bigquery.py +++ b/tests/operators/_asynchronous/test_bigquery.py @@ -38,6 +38,7 @@ def test_dbt_run_airflow_async_bigquery_operator_init(profile_config_mock): assert operator.project_dir == "/path/to/project" assert operator.profile_config == profile_config_mock assert operator.gcp_conn_id == "google_cloud_default" + assert operator.full_refresh is False # Default value should be False def test_dbt_run_airflow_async_bigquery_operator_base_cmd(profile_config_mock): @@ -62,17 +63,20 @@ def test_dbt_run_airflow_async_bigquery_operator_execute(mock_build_and_run_cmd, dbt_kwargs={"task_id": "test_task"}, ) + # Mock context with run_id mock_context = MagicMock() + mock_context.__getitem__.return_value = "test_run_id" # For context["run_id"] + operator.execute(mock_context) - mock_build_and_run_cmd.assert_called_once_with( - context=mock_context, - run_as_async=True, - async_context={ - "profile_type": "bigquery", - "async_operator": BigQueryInsertJobOperator, - }, - ) + # Check that build_and_run_cmd was called with the correct parameters + # but ignore the run_id in async_context for the assertion + assert mock_build_and_run_cmd.call_count == 1 + args, kwargs = mock_build_and_run_cmd.call_args + assert kwargs["context"] == mock_context + assert kwargs["run_as_async"] is True + assert kwargs["async_context"]["profile_type"] == "bigquery" + assert kwargs["async_context"]["async_operator"] == BigQueryInsertJobOperator @pytest.fixture @@ -140,7 +144,10 @@ def test_store_compiled_sql(mock_override_rtif, mock_get_remote_sql, profile_con @patch("cosmos.operators._asynchronous.bigquery.DbtRunAirflowAsyncBigqueryOperator._store_template_fields") def test_execute_complete(mock_store_sql, profile_config_mock): - mock_context = Mock() + # Create a mock context with run_id + mock_context = MagicMock() + mock_context.__getitem__.return_value = "test_run_id" # For context["run_id"] + mock_event = {"job_id": "test_job"} operator = DbtRunAirflowAsyncBigqueryOperator( @@ -156,3 +163,15 @@ def test_execute_complete(mock_store_sql, profile_config_mock): assert result == "test_job" mock_super_execute.assert_called_once_with(context=mock_context, event=mock_event) mock_store_sql.assert_called_once_with(context=mock_context) + + +def test_dbt_run_airflow_async_bigquery_operator_with_full_refresh(profile_config_mock): + """Test DbtRunAirflowAsyncBigqueryOperator initializes with full_refresh=True.""" + operator = DbtRunAirflowAsyncBigqueryOperator( + task_id="test_task", + project_dir="/path/to/project", + profile_config=profile_config_mock, + dbt_kwargs={"task_id": "test_task", "full_refresh": True}, + ) + + assert operator.full_refresh is True diff --git a/tests/operators/test_local.py b/tests/operators/test_local.py index c133edd6db..9ba037631d 100644 --- a/tests/operators/test_local.py +++ b/tests/operators/test_local.py @@ -30,6 +30,7 @@ ) from cosmos.exceptions import CosmosDbtRunError, CosmosValueError from cosmos.hooks.subprocess import FullOutputSubprocessResult +from cosmos.io import _construct_dest_file_path from cosmos.operators.local import ( AbstractDbtLocalBase, DbtBuildLocalOperator, @@ -531,7 +532,6 @@ def test_run_operator_dataset_inlets_and_outlets_airflow_210(caplog): ) @pytest.mark.integration def test_run_operator_dataset_inlets_and_outlets_airflow_3_onwards(caplog): - with DAG("test_id_1", start_date=datetime(2022, 1, 1)) as dag: seed_operator = DbtSeedLocalOperator( profile_config=real_profile_config, @@ -626,7 +626,6 @@ def test_run_operator_dataset_inlets_and_outlets_airflow_210_onwards_disabled_vi ) @pytest.mark.integration def test_run_operator_dataset_emission_is_skipped(caplog): - with DAG("test-id-1", start_date=datetime(2022, 1, 1)) as dag: seed_operator = DbtSeedLocalOperator( profile_config=real_profile_config, @@ -760,8 +759,32 @@ def test_run_operator_caches_partial_parsing(caplog, tmp_path): assert not "Unable to do partial parsing" in caplog.text -def test_dbt_base_operator_no_partial_parse() -> None: +@pytest.mark.integration +def test_run_operator_copies_manifest_file(caplog, tmp_path): + manifest_filepath = DBT_PROJ_DIR / "target/manifest.json" + assert manifest_filepath.exists() + caplog.clear() + caplog.set_level(logging.DEBUG) + with DAG("test-partial-parsing", start_date=datetime(2022, 1, 1)) as dag: + seed_operator = DbtSeedLocalOperator( + profile_config=real_profile_config, + project_dir=DBT_PROJ_DIR, + task_id="seed", + dbt_cmd_flags=["--select", "raw_customers"], + install_deps=True, + append_env=True, + cache_dir=cache._obtain_cache_dir_path("test-partial-parsing", tmp_path), + invocation_mode=InvocationMode.SUBPROCESS, + manifest_filepath=manifest_filepath, + ) + seed_operator + + run_test_dag(dag) + + assert caplog.text.count("Copying the manifest from") == 1 + +def test_dbt_base_operator_no_partial_parse() -> None: dbt_base_operator = ConcreteDbtLocalBaseOperator( profile_config=profile_config, task_id="my-task", @@ -1204,7 +1227,7 @@ def test_operator_execute_deps_parameters( ) mock_ensure_profile.return_value.__enter__.return_value = (Path("/path/to/profile"), {"ENV_VAR": "value"}) mock_temporary_directory.return_value.__enter__.return_value = project_dir.as_posix() - task.execute(context={"task_instance": MagicMock()}) + task.execute(context={"task_instance": MagicMock(), "run_id": "test_run_id"}) if invocation_mode == InvocationMode.SUBPROCESS: assert mock_subprocess.call_args_list[0].kwargs["command"] == expected_call_kwargs elif invocation_mode == InvocationMode.DBT_RUNNER: @@ -1239,7 +1262,6 @@ class CustomDbtDocsLocalOperator(DbtDocsLocalOperator): @patch("cosmos.hooks.subprocess.FullOutputSubprocessHook.send_sigint") def test_dbt_local_operator_on_kill_sigint(mock_send_sigint) -> None: - dbt_base_operator = ConcreteDbtLocalBaseOperator( profile_config=profile_config, task_id="my-task", @@ -1255,7 +1277,6 @@ def test_dbt_local_operator_on_kill_sigint(mock_send_sigint) -> None: @patch("cosmos.hooks.subprocess.FullOutputSubprocessHook.send_sigterm") def test_dbt_local_operator_on_kill_sigterm(mock_send_sigterm) -> None: - dbt_base_operator = ConcreteDbtLocalBaseOperator( profile_config=profile_config, task_id="my-task", @@ -1534,12 +1555,13 @@ def test_upload_compiled_sql_should_upload(mock_configure_remote, mock_object_st files = [file1, file2] + operator.extra_context["run_id"] = "test_run_id" with patch.object(Path, "rglob", return_value=files): operator._upload_sql_files(tmp_project_dir, "compiled") for file_path in files: rel_path = os.path.relpath(str(file_path), str(source_compiled_dir)) - expected_dest_path = f"mock_remote_path/test_dag/compiled/{rel_path.lstrip('/')}" + expected_dest_path = f"mock_remote_path/test_dag/test_run_id/compiled/{rel_path.lstrip('/')}" mock_object_storage_path.assert_any_call(expected_dest_path, conn_id="mock_conn_id") mock_object_storage_path.return_value.copy.assert_any_call(mock_object_storage_path.return_value) @@ -1624,24 +1646,50 @@ def test_async_execution_without_start_task(mock_read_sql, mock_bq_execute): mock_bq_execute.assert_called_once() +def test_build_and_run_cmd_with_full_refresh_in_async_mode(): + """Test that build_and_run_cmd adds --full-refresh flag when full_refresh is True in async mode.""" + AbstractDbtLocalBase.__abstractmethods__ = set() + + with patch("cosmos.operators.local.settings.enable_setup_async_task", True): + operator = AbstractDbtLocalBase( + task_id="test_task", + project_dir="/tmp", + profile_config=profile_config, + ) + operator.full_refresh = True + + with patch.object(operator, "build_cmd") as mock_build_cmd: + mock_build_cmd.return_value = (["dbt", "run"], {}) + + with patch.object(operator, "run_command"): + operator.build_and_run_cmd(context={}, run_as_async=True) + cmd_flags_arg = mock_build_cmd.call_args[1].get("cmd_flags", []) + assert "--full-refresh" in cmd_flags_arg + + @pytest.mark.integration @pytest.mark.skipif(not AIRFLOW_IO_AVAILABLE, reason="Airflow did not have Object Storage until the 2.8 release") @patch("pathlib.Path.rglob") @patch("cosmos.operators.local.AbstractDbtLocalBase._construct_dest_file_path") -@patch("airflow.io.path.ObjectStoragePath.unlink") -def test_async_execution_teardown_delete_files(mock_unlink, mock_construct_dest_file_path, mock_rglob): - mock_file = MagicMock() - mock_file.is_file.return_value = True - mock_file.__str__.return_value = "/altered_jaffle_shop/target/run/file1.sql" - mock_rglob.return_value = [mock_file] +@patch("cosmos.operators.local.AbstractDbtLocalBase._configure_remote_target_path") +@patch("airflow.io.path.ObjectStoragePath") +def test_async_execution_teardown_delete_files( + mock_object_storage_path, mock_configure_remote, mock_construct_dest_file_path, mock_rglob +): + mock_path = MagicMock() + mock_path.exists.return_value = True + mock_object_storage_path.return_value = mock_path + mock_configure_remote.return_value = (Path("/mock/path"), "mock_conn_id") + project_dir = Path(__file__).parent.parent.parent / "dev/dags/dbt/altered_jaffle_shop" operator = DbtRunLocalOperator( task_id="test", project_dir=project_dir, profile_config=profile_config, + extra_context={"dbt_dag_task_group_identifier": "test_dag_task_group", "run_id": "test_run_id"}, ) operator._handle_async_execution(project_dir, {}, {"profile_type": "bigquery", "teardown_task": True}) - mock_unlink.assert_called() + mock_path.rmdir.assert_called_once_with(recursive=True) def test_read_run_sql_from_target_dir(): @@ -1688,7 +1736,6 @@ def test_test_clone_project(create_symlinks_mock, copy_dbt_packages_mock, caplog def test_handle_post_execution_with_multiple_callbacks( mock_override_rtif, mock_store_compiled_sql, mock_store_freshness_json ): - multiple_callbacks = [MagicMock(), MagicMock(), MagicMock()] operator = ConcreteDbtLocalBaseOperator( profile_config=profile_config, @@ -1703,3 +1750,173 @@ def test_handle_post_execution_with_multiple_callbacks( for callback_fn in multiple_callbacks: callback_fn.assert_called_once_with("/tmp/project_dir", arg1="value1", context=context) + + +def test_construct_dest_file_path_with_run_id(): + """Test _construct_dest_file_path uses run_id correctly.""" + dest_target_dir = Path("/dest") + source_target_dir = Path("/project_dir/target") + file_path = "/project_dir/target/subdir/file.txt" + source_subpath = "target" + + expected_path = "/dest/test_dag/test_run_id/test_task/1/target/subdir/file.txt" + context = { + "dag": MagicMock(dag_id="test_dag"), + "run_id": "test_run_id", + "task_instance": MagicMock(task_id="test_task", _try_number=1), + } + result = _construct_dest_file_path(dest_target_dir, file_path, source_target_dir, source_subpath, context=context) + + assert result == expected_path + assert "test_run_id" in result + + +def test_operator_construct_dest_file_path_with_run_id(): + """Test that the operator's _construct_dest_file_path method uses run_id correctly.""" + operator = ConcreteDbtLocalBaseOperator( + task_id="test_task", profile_config=profile_config, project_dir="/project_dir" + ) + + operator.extra_context = {"run_id": "test_run_id", "dbt_dag_task_group_identifier": "test_task_group"} + + dest_target_dir = Path("/dest") + source_compiled_dir = Path("/project_dir/target/compiled") + file_path = "/project_dir/target/compiled/models/my_model.sql" + resource_type = "compiled" + + expected_path = "/dest/test_task_group/test_run_id/compiled/models/my_model.sql" + result = operator._construct_dest_file_path(dest_target_dir, file_path, source_compiled_dir, resource_type) + + assert result == expected_path + assert "test_run_id" in result + + +def test_construct_dest_file_path_in_operator(): + """Test that the operator's _construct_dest_file_path method uses run_id correctly.""" + operator = ConcreteDbtLocalBaseOperator( + task_id="test_task", profile_config=profile_config, project_dir="/project_dir" + ) + + operator.extra_context = {"run_id": "test_run_id", "dbt_dag_task_group_identifier": "test_task_group"} + + dest_target_dir = Path("/dest") + source_compiled_dir = Path("/project_dir/target/compiled") + file_path = "/project_dir/target/compiled/models/my_model.sql" + resource_type = "compiled" + + expected_path = "/dest/test_task_group/test_run_id/compiled/models/my_model.sql" + + with patch.object( + operator, "_construct_dest_file_path", wraps=operator._construct_dest_file_path + ) as mock_construct: + result = operator._construct_dest_file_path(dest_target_dir, file_path, source_compiled_dir, resource_type) + + assert result == expected_path + assert "test_run_id" in result + + mock_construct.assert_called_once_with(dest_target_dir, file_path, source_compiled_dir, resource_type) + + +@pytest.mark.skipif(not AIRFLOW_IO_AVAILABLE, reason="Airflow did not have Object Storage until the 2.8 release") +@patch("airflow.io.path.ObjectStoragePath") +def test_upload_sql_files_creates_parent_directories(mock_object_storage_path): + """Test that parent directories are created during file uploads.""" + + operator = ConcreteDbtLocalBaseOperator( + profile_config=profile_config, + task_id="test-task", + project_dir="test/dir", + ) + + with patch.object( + operator, "_configure_remote_target_path", return_value=("dest/dir", "mock_conn_id") + ), patch.object(operator, "_construct_dest_file_path", return_value="dest/path/file.sql"), patch( + "pathlib.Path.rglob", return_value=[Path("file.sql")] + ), patch( + "pathlib.Path.is_file", return_value=True + ): + mock_dest_path = MagicMock() + mock_dest_path.parent = MagicMock() + mock_object_storage_path.return_value = mock_dest_path + + operator._upload_sql_files("tmp_dir", "compiled") + + mock_dest_path.parent.mkdir.assert_called_with(parents=True, exist_ok=True) + + +@pytest.mark.integration +@pytest.mark.skipif(not AIRFLOW_IO_AVAILABLE, reason="Airflow did not have Object Storage until the 2.8 release") +@patch("cosmos.operators.local.AbstractDbtLocalBase._configure_remote_target_path") +@patch("airflow.io.path.ObjectStoragePath") +def test_delete_sql_files_directory_not_exists(mock_object_storage_path, mock_configure_remote): + """Test the _delete_sql_files method when the remote directory doesn't exist.""" + mock_path = MagicMock() + mock_path.exists.return_value = False + mock_object_storage_path.return_value = mock_path + mock_configure_remote.return_value = (Path("/mock/path"), "mock_conn_id") + + operator = DbtRunLocalOperator( + task_id="test", + project_dir="/project/dir", + profile_config=profile_config, + extra_context={"dbt_dag_task_group_identifier": "test_dag_task_group", "run_id": "test_run_id"}, + ) + + with patch.object(operator.log, "debug") as mock_log_debug: + operator._delete_sql_files() + mock_log_debug.assert_called_once() + log_format, log_path = mock_log_debug.call_args[0] + assert "Remote run directory does not exist, skipping deletion: %s" == log_format + assert "/mock/path/test_dag_task_group/test_run_id" == log_path + + mock_path.rmdir.assert_not_called() + + +def test_generate_dbt_flags_appends_no_static_parser(tmp_path): + operator = ConcreteDbtLocalBaseOperator( + profile_config=profile_config, + task_id="test-task", + project_dir=tmp_path, + ) + operator.invocation_mode = InvocationMode.DBT_RUNNER + tmp_project_dir = str(tmp_path) + profile_path = tmp_path / "profiles.yml" + flags = operator._generate_dbt_flags(tmp_project_dir, profile_path) + assert "--no-static-parser" in flags + + +def test_generate_dbt_flags_does_not_append_no_static_parser_in_subprocess(tmp_path): + operator = ConcreteDbtLocalBaseOperator( + profile_config=profile_config, + task_id="test-task", + project_dir=tmp_path, + ) + operator.invocation_mode = InvocationMode.SUBPROCESS + tmp_project_dir = str(tmp_path) + profile_path = tmp_path / "profiles.yml" + flags = operator._generate_dbt_flags(tmp_project_dir, profile_path) + assert "--no-static-parser" not in flags + + +@pytest.mark.integration +@pytest.mark.skipif(not AIRFLOW_IO_AVAILABLE, reason="Airflow did not have Object Storage until the 2.8 release") +@patch("cosmos.operators.local.AbstractDbtLocalBase._configure_remote_target_path") +def test_delete_sql_files_no_remote_target_configured(mock_configure_remote): + """Test that _delete_sql_files exits early with a warning when remote path is not configured.""" + mock_configure_remote.return_value = (None, None) + operator = DbtRunLocalOperator( + task_id="test", + project_dir="/project/dir", + profile_config=profile_config, + extra_context={"dbt_dag_task_group_identifier": "test_dag_task_group", "run_id": "test_run_id"}, + ) + + with patch.object(operator.log, "warning") as mock_log_warning: + operator._delete_sql_files() + expected_log_message = "Remote target path or connection ID not configured. Skipping deletion." + mock_log_warning.assert_called_once_with(expected_log_message) + + mock_configure_remote.return_value = (Path("/mock/path"), None) + with patch.object(operator.log, "warning") as mock_log_warning: + operator._delete_sql_files() + mock_log_warning.assert_called_once_with(expected_log_message) diff --git a/tests/operators/test_virtualenv.py b/tests/operators/test_virtualenv.py index 0e2f0c7363..329a399b34 100644 --- a/tests/operators/test_virtualenv.py +++ b/tests/operators/test_virtualenv.py @@ -97,7 +97,9 @@ def test_run_command_without_virtualenv_dir( invocation_mode=InvocationMode.SUBPROCESS, ) assert venv_operator.virtualenv_dir == None - venv_operator.run_command(cmd=["fake-dbt", "do-something"], env={}, context={"task_instance": MagicMock()}) + venv_operator.run_command( + cmd=["fake-dbt", "do-something"], env={}, context={"task_instance": MagicMock(), "run_id": "test_run_id"} + ) run_command_args = mock_subprocess_hook.run_command.call_args_list assert len(run_command_args) == 2 dbt_deps = run_command_args[0].kwargs @@ -174,7 +176,9 @@ def test_run_command_with_virtualenv_dir( invocation_mode=InvocationMode.SUBPROCESS, virtualenv_dir=Path("mock-venv"), ) - venv_operator.run_command(cmd=["fake-dbt", "do-something"], env={}, context={"task_instance": MagicMock()}) + venv_operator.run_command( + cmd=["fake-dbt", "do-something"], env={}, context={"task_instance": MagicMock(), "run_id": "test_run_id"} + ) assert str(venv_operator.virtualenv_dir) == "mock-venv" run_command_args = mock_subprocess_hook.run_command.call_args_list assert len(run_command_args) == 2 diff --git a/tests/test_log.py b/tests/test_log.py index 9f75634ad8..194799e2b6 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -1,8 +1,8 @@ import pytest import cosmos.log -from cosmos import get_provider_info from cosmos.log import CosmosRichLogger, get_logger +from cosmos.provider_info import get_provider_info def test_get_logger(monkeypatch): diff --git a/tests/test_settings.py b/tests/test_settings.py index d9f5e0f6e7..5bb0ad37d5 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1,4 +1,6 @@ import os +import subprocess +import textwrap from importlib import reload from unittest.mock import patch @@ -9,3 +11,33 @@ def test_enable_cache_env_var(): reload(settings) assert settings.enable_cache is False + + +def test_enable_memory_optimised_imports_true(monkeypatch): + script = textwrap.dedent( + """ + import os + os.environ["AIRFLOW__COSMOS__ENABLE_MEMORY_OPTIMISED_IMPORTS"] = "True" + import cosmos + assert cosmos.settings.enable_memory_optimised_imports is True + assert not hasattr(cosmos, "DbtDag") + """ + ) + + result = subprocess.run(["python", "-c", script], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + + +def test_enable_memory_optimised_imports_false(monkeypatch): + script = textwrap.dedent( + """ + import os + os.environ["AIRFLOW__COSMOS__ENABLE_MEMORY_OPTIMISED_IMPORTS"] = "False" + import cosmos + assert cosmos.settings.enable_memory_optimised_imports is False + assert hasattr(cosmos, "DbtDag") + """ + ) + + result = subprocess.run(["python", "-c", script], capture_output=True, text=True) + assert result.returncode == 0, result.stderr