diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d9b484b545..b37f167892 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @astronomer/astro-cosmos-admins @astronomer/astro-devex +@astronomer/astro-cosmos-admins @astronomer/astro-cosmos-codeowners diff --git a/.github/ISSUE_TEMPLATE/01-bug.yml b/.github/ISSUE_TEMPLATE/01-bug.yml new file mode 100644 index 0000000000..4a5517338b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/01-bug.yml @@ -0,0 +1,183 @@ +--- +name: Bug Report +description: File a bug report. +title: "[Bug] " +labels: ["bug", "triage-needed"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + - type: dropdown + id: cosmos-version + attributes: + label: Astronomer Cosmos Version + description: What version of Astronomer Cosmos are you running? If you do not see your version in the list, please (ideally) test on + the latest release or main to see if the issue is fixed before reporting it. + options: + - "1.4.1" + - "main (development)" + - "Other Astronomer Cosmos version (please specify below)" + multiple: false + validations: + required: true + - type: input + attributes: + label: If "Other Astronomer Cosmos version" selected, which one? + # yamllint disable rule:line-length + description: > + On what version of Astronomer Cosmos are you currently experiencing the issue? Remember, you are encouraged to + test with the latest release or on the main branch to verify your issue still exists. + - type: input + id: dbt-core-version + attributes: + label: dbt-core version + description: What version of dbt-core are you running? + placeholder: ex. 1.8.0 + validations: + required: true + - type: textarea + attributes: + label: Versions of dbt adapters + description: What dbt adapter versions are you using? + placeholder: You can use `pip freeze | grep dbt` (you can leave only relevant ones) + - type: dropdown + id: load-mode + attributes: + label: LoadMode + description: Which LoadMode are you using? + options: + - "AUTOMATIC" + - "CUSTOM" + - "DBT_LS" + - "DBT_LS_FILE" + - "DBT_LS_MANIFEST" + multiple: false + validations: + required: true + - type: dropdown + id: execution-mode + attributes: + label: ExecutionMode + description: Which ExecutionMode are you using? + options: + - "AWS_EKS" + - "AZURE_CONTAINER_INSTANCE" + - "DOCKER" + - "KUBERNETES" + - "LOCAL" + - "VIRTUALENV" + multiple: false + validations: + required: true + - type: dropdown + id: invocation-mode + attributes: + label: InvocationMode + description: Which InvocationMode are you using? + options: + - "DBT_RUNNER" + - "SUBPROCESS" + multiple: false + - type: input + id: airflow-version + attributes: + label: airflow version + description: What version of Apache Airflow are you running? + placeholder: ex. 2.9.0 + validations: + required: true + - type: input + attributes: + label: Operating System + description: What Operating System are you using? + placeholder: "You can get it via `cat /etc/os-release` for example" + validations: + required: true + - type: dropdown + id: browsers + attributes: + label: If a you think it's an UI issue, what browsers are you seeing the problem on? + multiple: true + options: + - Firefox + - Chrome + - Safari + - Microsoft Edge + - type: dropdown + attributes: + label: Deployment + description: > + What kind of deployment do you have? + multiple: false + options: + - "Official Apache Airflow Helm Chart" + - "Other 3rd-party Helm chart" + - "Docker-Compose" + - "Other Docker-based deployment" + - "Virtualenv installation" + - "Astronomer" + - "Google Cloud Composer" + - "Amazon (AWS) MWAA" + - "Microsoft ADF Managed Airflow" + - "Other" + validations: + required: true + - type: textarea + attributes: + label: Deployment details + description: Additional description of your deployment. + placeholder: > + Enter any relevant details of your deployment. Especially version of your tools, + software (docker-compose, helm, k8s, etc.), any customisation and configuration you added. + - type: textarea + id: what-happened + attributes: + label: What happened? + description: Also tell us, what did you expect to happen? + placeholder: Tell us what you see! + value: "A bug happened!" + validations: + required: true + - type: textarea + id: logs + attributes: + label: Relevant log output + description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + render: shell + - type: textarea + attributes: + label: How to reproduce + description: What should we do to reproduce the problem? + placeholder: > + Please make sure you provide a reproducible step-by-step case of how to reproduce the problem + as minimally and precisely as possible. Keep in mind we do not have access to your cluster or DAGs. + Remember that non-reproducible issues make it hard for us to help you or resolve the issue! + validations: + required: true + - type: textarea + attributes: + label: Anything else :)? + description: Anything else we need to know? + placeholder: > + How often does this problem occur? (Once? Every time? Only when certain conditions are met?) + - type: checkboxes + attributes: + label: Are you willing to submit PR? + description: > + This is absolutely not required, but we are happy to guide you in the contribution process + especially if you already have a good understanding of how to implement the fix. We love to bring new + contributors in. + options: + - label: Yes I am willing to submit a PR! + - type: input + id: contact + attributes: + label: Contact Details + description: (Optional) How can we get in touch with you if we need more info? + placeholder: ex. email@example.com + validations: + required: false + - type: markdown + attributes: + value: "Thanks for completing our form!" diff --git a/.github/ISSUE_TEMPLATE/02-feature.yml b/.github/ISSUE_TEMPLATE/02-feature.yml new file mode 100644 index 0000000000..f8cd9e24da --- /dev/null +++ b/.github/ISSUE_TEMPLATE/02-feature.yml @@ -0,0 +1,34 @@ +--- +name: Feature request +description: Suggest an idea for this project +title: "[Feature] " +labels: ["enhancement", "triage-needed"] +body: + - type: markdown + attributes: + # yamllint disable rule:line-length + value: " + Thank you for finding the time to propose new feature! + + We really appreciate the community efforts to improve Cosmos." + # yamllint enable rule:line-length + - type: textarea + attributes: + label: Description + description: A short description of your feature + - type: textarea + attributes: + label: Use case/motivation + description: What would you like to happen? + - type: textarea + attributes: + label: Related issues + description: Is there currently another issue associated with this? + - type: checkboxes + attributes: + label: Are you willing to submit a PR? + options: + - label: Yes, I am willing to submit a PR! + - type: markdown + attributes: + value: "Thanks for completing our form!" diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f6ed27a6ce..be13f34bea 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install build dependencies run: python -m pip install --upgrade build @@ -37,7 +37,7 @@ jobs: path: dist - name: Push build artifacts to PyPi - uses: pypa/gh-action-pypi-publish@v1.6.4 + uses: pypa/gh-action-pypi-publish@v1.8.14 with: user: __token__ password: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index af71efc1c3..96f0d5564b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,10 +11,8 @@ concurrency: cancel-in-progress: true jobs: - Authorize: - environment: - ${{ github.event_name == 'pull_request_target' && + environment: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository && 'external' || 'internal' }} runs-on: ubuntu-latest @@ -30,8 +28,8 @@ jobs: - uses: actions/setup-python@v3 with: - python-version: '3.9' - architecture: 'x64' + python-version: "3.9" + architecture: "x64" - run: pip3 install hatch - run: hatch run tests.py3.9-2.7:type-check @@ -40,8 +38,25 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] - airflow-version: ["2.3", "2.4", "2.5", "2.6", "2.7"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + airflow-version: ["2.4", "2.5", "2.6", "2.7", "2.8", "2.9"] + exclude: + - python-version: "3.11" + airflow-version: "2.4" + # Apache Airflow versions prior to 2.9.0 have not been tested with Python 3.12. + # Official support for Python 3.12 and the corresponding constraints.txt are available only for Apache Airflow >= 2.9.0. + # See: https://github.com/apache/airflow/tree/2.9.0?tab=readme-ov-file#requirements + # See: https://github.com/apache/airflow/tree/2.8.4?tab=readme-ov-file#requirements + - python-version: "3.12" + airflow-version: "2.4" + - python-version: "3.12" + airflow-version: "2.5" + - python-version: "3.12" + airflow-version: "2.6" + - python-version: "3.12" + airflow-version: "2.7" + - python-version: "3.12" + airflow-version: "2.8" steps: - uses: actions/checkout@v3 with: @@ -51,7 +66,7 @@ jobs: with: path: | ~/.cache/pip - .nox + .local/share/hatch/ key: unit-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} - name: Set up Python ${{ matrix.python-version }} @@ -61,7 +76,8 @@ jobs: - name: Install packages and dependencies run: | - python -m pip install hatch + python -m pip install uv + uv pip install --system hatch hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} @@ -79,8 +95,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] - airflow-version: ["2.3", "2.4", "2.5", "2.6", "2.7"] + python-version: ["3.8", "3.9", "3.10", "3.11"] + airflow-version: ["2.4", "2.5", "2.6", "2.7", "2.8", "2.9"] + exclude: + - python-version: "3.11" + airflow-version: "2.4" services: postgres: image: postgres @@ -97,11 +116,12 @@ jobs: - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha || github.ref }} + - uses: actions/cache@v3 with: path: | ~/.cache/pip - .nox + .local/share/hatch/ key: integration-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} - name: Set up Python ${{ matrix.python-version }} @@ -111,23 +131,24 @@ jobs: - name: Install packages and dependencies run: | - python -m pip install hatch + python -m pip install uv + uv pip install --system hatch hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} run: | hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-setup - DATABRICKS_UNIQUE_ID="${{github.run_id}}_${{matrix.python-version}}_${{ matrix.airflow-version }}" hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration + hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration env: + AIRFLOW__COSMOS__ENABLE_CACHE_DBT_LS: 0 AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ - AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres - AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} + AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres + DATABRICKS_HOST: mock + DATABRICKS_WAREHOUSE_ID: mock + DATABRICKS_TOKEN: mock + DATABRICKS_CLUSTER_ID: mock AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH - DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} - DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} - DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} - DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} POSTGRES_HOST: localhost POSTGRES_USER: postgres @@ -147,7 +168,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10"] + python-version: ["3.11"] airflow-version: ["2.6"] services: @@ -171,7 +192,7 @@ jobs: with: path: | ~/.cache/pip - .nox + .local/share/hatch/ key: integration-expensive-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} - name: Set up Python ${{ matrix.python-version }} @@ -181,7 +202,8 @@ jobs: - name: Install packages and dependencies run: | - python -m pip install hatch + python -m pip install uv + uv pip install --system hatch hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} @@ -190,14 +212,11 @@ jobs: DATABRICKS_UNIQUE_ID="${{github.run_id}}" hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-expensive env: AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ - AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres + AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} - AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 - DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} - DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} - DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} + AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} POSTGRES_HOST: localhost POSTGRES_USER: postgres @@ -214,12 +233,9 @@ jobs: env: AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ - AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres + AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} - DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} - DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} - DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} Run-Integration-Tests-Sqlite: @@ -227,7 +243,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10"] + python-version: ["3.11"] airflow-version: ["2.7"] steps: @@ -238,7 +254,7 @@ jobs: with: path: | ~/.cache/pip - .nox + .local/share/hatch/ key: integration-sqlite-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} - name: Set up Python ${{ matrix.python-version }} @@ -248,7 +264,8 @@ jobs: - name: Install packages and dependencies run: | - python -m pip install hatch + python -m pip install uv + uv pip install --system hatch hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} @@ -257,15 +274,14 @@ jobs: hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-sqlite env: AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ - AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres + AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH - AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} - DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} - DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} - DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} - DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} + DATABRICKS_CLUSTER_ID: mock + DATABRICKS_HOST: mock + DATABRICKS_WAREHOUSE_ID: mock + DATABRICKS_TOKEN: mock POSTGRES_HOST: localhost POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -281,9 +297,150 @@ jobs: env: AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ - AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres + AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + + Run-Integration-Tests-DBT-1-5-4: + needs: Authorize + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.11" ] + airflow-version: [ "2.7" ] + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + .local/share/hatch/ + key: integration-dbt-1-5-4-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install packages and dependencies + run: | + python -m pip install uv + uv pip install --system hatch + hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze + + - name: Test Cosmos against Airflow ${{ matrix.airflow-version }}, Python ${{ matrix.python-version }} and dbt 1.5.4 + run: | + hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-dbt-1-5-4 + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres + AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} + DATABRICKS_CLUSTER_ID: mock + DATABRICKS_HOST: mock + DATABRICKS_WAREHOUSE_ID: mock + DATABRICKS_TOKEN: mock + POSTGRES_HOST: localhost + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + POSTGRES_SCHEMA: public + POSTGRES_PORT: 5432 + + - name: Upload coverage to Github + uses: actions/upload-artifact@v2 + with: + name: coverage-integration-dbt-1-5-4-test-${{ matrix.python-version }}-${{ matrix.airflow-version }} + path: .coverage + + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + Run-Performance-Tests: + needs: Authorize + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11"] + airflow-version: ["2.7"] + num-models: [1, 10, 50, 100] + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + .local/share/hatch/ + key: perf-test-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install packages and dependencies + run: | + python -m pip install uv + uv pip install --system hatch + hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze + + - name: Run performance tests against against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} + id: run-performance-tests + run: | + hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-performance-setup + hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-performance + + # read the performance results and set them as an env var for the next step + # format: NUM_MODELS={num_models}\nTIME={end - start}\n + cat /tmp/performance_results.txt > $GITHUB_STEP_SUMMARY + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres + AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} + POSTGRES_HOST: localhost + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + POSTGRES_SCHEMA: public + POSTGRES_PORT: 5432 + MODEL_COUNT: ${{ matrix.num-models }} + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH Code-Coverage: if: github.event.action != 'labeled' @@ -296,10 +453,10 @@ jobs: - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha || github.ref }} - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v3 with: - python-version: '3.10' + python-version: "3.11" - name: Install coverage run: | pip3 install coverage diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 53f80df2f7..fcd19b42a7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: types: [file] pass_filenames: false - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-added-large-files - id: check-merge-conflict @@ -22,10 +22,10 @@ repos: - id: end-of-file-fixer - id: mixed-line-ending - id: pretty-format-json - args: ['--autofix'] + args: ["--autofix"] - id: trailing-whitespace - repo: https://github.com/codespell-project/codespell - rev: v2.2.6 + rev: v2.3.0 hooks: - id: codespell name: Run codespell to check for common misspellings in files @@ -33,59 +33,65 @@ repos: types: [text] args: - --exclude-file=tests/sample/manifest_model_version.json - - --skip=**/manifest.json + - --skip=**/manifest.json,**.min.js + - -L connexion,aci - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 hooks: - id: rst-backticks - id: python-check-mock-methods - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.5.4 + rev: v1.5.5 hooks: - id: remove-crlf - id: remove-tabs exclude: ^docs/make.bat$|^docs/Makefile$|^dev/dags/dbt/jaffle_shop/seeds/raw_orders.csv$ - repo: https://github.com/asottile/pyupgrade - rev: v3.15.0 + rev: v3.16.0 hooks: - id: pyupgrade args: - --py37-plus - --keep-runtime-typing - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.3 + rev: v0.5.1 hooks: - id: ruff args: - --fix - repo: https://github.com/psf/black - rev: 23.10.1 + rev: 24.4.2 hooks: - id: black - args: [ "--config", "./pyproject.toml" ] + args: ["--config", "./pyproject.toml"] - repo: https://github.com/asottile/blacken-docs - rev: 1.16.0 + rev: 1.18.0 hooks: - id: blacken-docs alias: black additional_dependencies: [black>=22.10.0] - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v1.6.1' + rev: "v1.10.1" + hooks: - id: mypy name: mypy-python - additional_dependencies: [types-PyYAML, types-attrs, attrs, types-requests, types-python-dateutil, apache-airflow] + args: [--config-file, "./pyproject.toml"] + additional_dependencies: + [ + types-PyYAML, + types-attrs, + attrs, + types-pytz, + types-requests, + types-python-dateutil, + apache-airflow, + ] files: ^cosmos - - repo: https://github.com/pycqa/flake8 - rev: 6.1.0 - hooks: - - id: flake8 - entry: pflake8 - additional_dependencies: [pyproject-flake8] ci: autofix_commit_msg: 🎨 [pre-commit.ci] Auto format from pre-commit.com hooks autoupdate_commit_msg: ⬆ [pre-commit.ci] pre-commit autoupdate skip: - - mypy # build of https://github.com/pre-commit/mirrors-mypy:types-PyYAML,types-attrs,attrs,types-requests, + - mypy # build of https://github.com/pre-commit/mirrors-mypy:types-PyYAML,types-attrs,attrs,types-requests, #types-python-dateutil,apache-airflow@v1.5.0 for python@python3 exceeds tier max size 250MiB: 262.6MiB diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d8b2bd8e15..66e9af5eaa 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,14 +1,308 @@ Changelog ========= -1.3.0a1 (2023-10-26) +1.5.0 (2024-06-27) +------------------ + +New Features + +* Speed up ``LoadMode.DBT_LS`` by caching dbt ls output in Airflow Variable by @tatiana in #1014 +* Support to cache profiles created via ``ProfileMapping`` by @pankajastro in #1046 +* Support for running dbt tasks in AWS EKS in #944 by @VolkerSchiewe +* Add Clickhouse profile mapping by @roadan and @pankajastro in #353 and #1016 +* Add node config to TaskInstance Context by @linchun3 in #1044 + +Bug fixes + +* Support partial parsing when cache is disabled by @tatiana in #1070 +* Fix disk permission error in restricted env by @pankajastro in #1051 +* Add CSP header to iframe contents by @dwreeves in #1055 +* Stop attaching log adaptors to root logger to reduce logging costs by @glebkrapivin in #1047 + +Enhancements + +* Support ``static_index.html`` docs by @dwreeves in #999 +* Support deep linking dbt docs via Airflow UI by @dwreeves in #1038 +* Add ability to specify host/port for Snowflake connection by @whummer in #1063 + +Docs + +* Fix rendering for env ``enable_cache_dbt_ls`` by @pankajastro in #1069 + +Others + +* Update documentation for DbtDocs generator by @arjunanan6 in #1043 +* Use uv in CI by @dwreeves in #1013 +* Cache hatch folder in the CI by @tatiana in #1056 +* Change example DAGs to use ``example_conn`` as opposed to ``airflow_db`` by @tatiana in #1054 +* Mark plugin integration tests as integration by @tatiana in #1057 +* Ensure compliance with linting rule D300 by using triple quotes for docstrings by @pankajastro in #1049 +* Pre-commit hook updates in #1039, #1050, #1064 +* Remove duplicates in changelog by @jedcunningham in #1068 + + +1.4.3 (2024-06-07) +------------------ + +Bug fixes + +* Bring back ``dataset`` as a required field for BigQuery profile by @pankajkoti in #1033 + +Enhancements + +* Only run ``dbt deps`` when there are dependencies by @tatiana and @AlgirdasDubickas in #1030 + +Docs + +* Fix docs so it does not reference non-existing ``get_dbt_dataset`` by @tatiana in #1034 + + +1.4.2 (2024-06-06) +------------------ + +Bug fixes + +* Fix the invocation mode for ``ExecutionMode.VIRTUALENV`` by @marco9663 in #1023 +* Fix Cosmos ``enable_cache`` setting by @tatiana in #1025 +* Make ``GoogleCloudServiceAccountDictProfileMapping`` dataset profile arg optional by @oliverrmaa and @pankajastro in #839 and #1017 +* Athena profile mapping set ``aws_session_token`` in profile only if it exists by @pankajastro in #1022 + +Others + +* Update dbt and Airflow conflicts matrix by @tatiana in #1026 +* Enable Python 3.12 unittest by @pankajastro in #1018 +* Improve error logging in ``DbtLocalBaseOperator`` by @davidsteinar in #1004 +* Add GitHub issue templates for bug reports and feature request by @pankajkoti in #1009 +* Add more fields in bug template to reduce turnaround in issue triaging by @pankajkoti in #1027 +* Fix ``dev/Dockerfile`` + Add ``uv pip install`` for faster build time by @dwreeves in #997 +* Drop support for Airflow 2.3 by @pankajkoti in #994 +* Update Astro Runtime image by @RNHTTR in #988 and #989 +* Enable ruff F linting by @pankajastro in #985 +* Move Cosmos Airflow configuration to settings.py by @pankajastro in #975 +* Fix CI Issues by @tatiana in #1005 +* Pre-commit hook updates in #1000, #1019 + + +1.4.1 (2024-05-17) +------------------ + +Bug fixes + +* Fix manifest testing behavior by @chris-okorodudu in #955 +* Handle ValueError when unpacking partial_parse.msgpack by @tatiana in #972 + +Others + +* Enable pre-commit run and fix type-check job by @pankajastro in #957 +* Clean databricks credentials in test/CI by @tatiana in #969 +* Update CODEOWNERS by @tatiana in #969 x +* Update emeritus contributors list by @tatiana in #961 +* Promote @dwreeves to committer by @tatiana in #960 +* Pre-commit hook updates in #956 + + +1.4.0 (2024-05-13) -------------------- Features -* Add ``ProfileMapping`` for Vertica by @perttus in #540 -* Add ``ProfileMapping`` for Snowflake encrypted private key path by @ivanstillfront in #608 -* Add ``DbtDocsGCSOperator`` for uploading dbt docs to GCS by @jbandoro in #616 +* Add dbt docs natively in Airflow via plugin by @dwreeves in #737 +* Add support for ``InvocationMode.DBT_RUNNER`` for local execution mode by @jbandoro in #850 +* Support partial parsing to render DAGs faster when using ``ExecutionMode.LOCAL``, ``ExecutionMode.VIRTUALENV`` and ``LoadMode.DBT_LS`` by @dwreeves in #800 +* Improve performance by 22-35% or more by caching partial parse artefact by @tatiana in #904 +* Add Azure Container Instance as Execution Mode by @danielvdende in #771 +* Add dbt build operators by @dylanharper-qz in #795 +* Add dbt profile config variables to mapped profile by @ykuc in #794 +* Add more template fields to ``DbtBaseOperator`` by @dwreeves in #786 +* Add ``pip_install_options`` argument to operators by @octiva in #808 + +Bug fixes + +* Make ``PostgresUserPasswordProfileMapping`` schema argument optional by @FouziaTariq in #683 +* Fix ``folder_dir`` not showing on logs for ``DbtDocsS3LocalOperator`` by @PrimOox in #856 +* Improve ``dbt ls`` parsing resilience to missing tags/config by @tatiana in #859 +* Fix ``operator_args`` modified in place in Airflow converter by @jbandoro in #835 +* Fix Docker and Kubernetes operators execute method resolution by @jbandoro in #849 +* Fix ``TrinoBaseProfileMapping`` required parameter for non method authentication by @AlexandrKhabarov in #921 +* Fix global flags for lists by @ms32035 in #863 +* Fix ``GoogleCloudServiceAccountDictProfileMapping`` when getting values from the Airflow connection ``extra__`` keys by @glebkrapivin in #923 +* Fix using the dag as a keyword argument as ``specific_args_keys`` in DbtTaskGroup by @tboutaour in #916 +* Fix ACI integration (``DbtAzureContainerInstanceBaseOperator``) by @danielvdende in #872 +* Fix setting dbt project dir to the tmp dir by @dwreeves in #873 +* Fix dbt docs operator to not use ``graph.gpickle`` file when ``--no-write-json`` is passed by @dwreeves in #883 +* Make Pydantic a required dependency by @pankajkoti in #939 +* Gracefully error if users try to ``emit_datasets`` with ``Airflow 2.9.0`` or ``2.9.1`` by @tatiana in #948 +* Fix parsing tests that have no parents in #933 by @jlaneve +* Correct ``root_path`` in partial parse cache by @pankajkoti in #950 + +Docs + +* Fix docs homepage link by @jlaneve in #860 +* Fix docs ``ExecutionConfig.dbt_project_path`` by @jbandoro in #847 +* Fix typo in MWAA getting started guide by @jlaneve in #846 +* Fix typo related to exporting docs to GCS by @tboutaour in #922 +* Improve partial parsing docs by @tatiana in #898 +* Improve docs for datasets for airflow >= 2.4 by @SiddiqueAhmad in #879 +* Improve test behaviour docs to highlight ``warning`` feature in the ``virtualenv`` mode by @mc51 in #910 +* Fix docs typo by @SiddiqueAhmad in #917 +* Improve Astro docs by @RNHTTR in #951 + +Others + +* Add performance integration tests by @jlaneve in #827 +* Enable ``append_env`` in ``operator_args`` by default by @tatiana in #899 +* Change default ``append_env`` behaviour depending on Cosmos ``ExecutionMode`` by @pankajkoti and @pankajastro in #954 +* Expose the ``dbt`` graph in the ``DbtToAirflowConverter`` class by @tommyjxl in #886 +* Improve dbt docs plugin rendering padding by @dwreeves in #876 +* Add ``connect_retries`` to databricks profile to fix expensive integration failures by @jbandoro in #826 +* Add import sorting (isort) to Cosmos by @jbandoro in #866 +* Add Python 3.11 to CI/tests by @tatiana and @jbandoro in #821, #824 and #825 +* Fix failing ``test_created_pod`` for ``apache-airflow-providers-cncf-kubernetes`` after v8.0.0 update by @jbandoro in #854 +* Extend ``DatabricksTokenProfileMapping`` test to include session properties by @tatiana in #858 +* Fix broken integration test uncovered from Pytest 8.0 update by @jbandoro in #845 +* Add Apache Airflow 2.9 to the test matrix by @tatiana in #940 +* Replace deprecated ``DummyOperator`` by ``EmptyOperator`` if Airflow >=2.4.0 by @tatiana in #900 +* Improve logs to troubleshoot issue in 1.4.0a2 with astro-cli by @tatiana in #947 +* Fix issue when publishing a new release to PyPI by @tatiana in #946 +* Pre-commit hook updates in #820, #834, #843 and #852, #890, #896, #901, #905, #908, #919, #931, #941 + + +1.3.2 (2024-01-26) +------------------ + +Bug fixes + +* Fix: ensure ``DbtGraph.update_node_dependency`` is called for all load methods by @jbandoro in #803 +* Fix: ensure operator ``execute`` method is consistent across all execution base subclasses by @jbandoro in #805 +* Fix custom selector when ``test`` node has no ``depends_on`` values by @tatiana in #814 +* Fix forwarding selectors to test task when using ``TestBehavior.AFTER_ALL`` by @tatiana in #816 + +Others + +* Docs: Remove incorrect docstring from ``DbtLocalBaseOperator`` by @jakob-hvitnov-telia in #797 +* Add more logs to troubleshoot custom selector by @tatiana in #809 +* Fix OpenLineage integration documentation by @tatiana in #810 +* Fix test dependencies after Airflow 2.8 release by @jbandoro and @tatiana in #806 +* Use Airflow constraint file for test environment setup by @jbandoro in #812 +* pre-commit updates in #799, #807 + + +1.3.1 (2023-01-10) +------------------ + +Bug fixes + +* Fix disable event tracking throwing error by @jbandoro in #784 +* Fix support for string path for ``LoadMode.DBT_LS_FILE`` and docs by @flinz in #788 +* Remove stack trace to disable unnecessary K8s error by @tatiana in #790 + +Others + +* Update examples to use the astro-runtime 10.0.0 by @RNHTTR in #777 +* Docs: add missing imports for mwaa getting started by @Benjamin0313 in #792 +* Refactor common executor constructors with test coverage by @jbandoro in #774 +* pre-commit updates in #789 + + +1.3.0 (2023-01-04) +------------------ + +Features + +* Add new parsing method ``LoadMode.DBT_LS_FILE`` by @woogakoki in #733 (`documentation `_). +* Add support to select using (some) graph operators when using ``LoadMode.CUSTOM`` and ``LoadMode.DBT_MANIFEST`` by @tatiana in #728 (`documentation `_) +* Add support for dbt ``selector`` arg for DAG parsing by @jbandoro in #755 (`documentation `_). +* Add ``ProfileMapping`` for Vertica by @perttus in #540, #688 and #741 (`documentation `_). +* Add ``ProfileMapping`` for Snowflake encrypted private key path by @ivanstillfront in #608 (`documentation `_). +* Add support for Snowflake encrypted private key environment variable by @DanMawdsleyBA in #649 +* Add ``DbtDocsGCSOperator`` for uploading dbt docs to GCS by @jbandoro in #616, (`documentation `_). +* Add cosmos/propagate_logs Airflow config support for disabling log propagation by @agreenburg in #648 (`documentation `_). +* Add operator_args ``full_refresh`` as a templated field by @joppevos in #623 +* Expose environment variables and dbt variables in ``ProjectConfig`` by @jbandoro in #735 (`documentation `_). +* Support disabling event tracking when using Cosmos profile mapping by @jbandoro in #768 (`documentation `_). + +Enhancements + +* Make Pydantic an optional dependency by @pixie79 in #736 +* Create a symbolic link to ``dbt_packages`` when ``dbt_deps`` is False when using ``LoadMode.DBT_LS`` by @DanMawdsleyBA in #730 +* Add ``aws_session_token`` for Athena mapping by @benjamin-awd in #663 +* Retrieve temporary credentials from ``conn_id`` for Athena by @octiva in #758 +* Extend ``DbtDocsLocalOperator`` with static flag by @joppevos in #759 + +Bug fixes + +* Remove Pydantic upper version restriction so Cosmos can be used with Airflow 2.8 by @jlaneve in #772 + +Others + +* Replace flake8 for Ruff by @joppevos in #743 +* Reduce code complexity to 8 by @joppevos in #738 +* Speed up integration tests by @jbandoro in #732 +* Fix README quickstart link in by @RNHTTR in #776 +* Add package location to work with hatchling 1.19.0 by @jbandoro in #761 +* Fix type check error in ``DbtKubernetesBaseOperator.build_env_args`` by @jbandoro in #766 +* Improve ``DBT_MANIFEST`` documentation by @dwreeves in #757 +* Update conflict matrix between Airflow and dbt versions by @tatiana in #731 and #779 +* pre-commit updates in #775, #770, #762 + + +1.2.5 (2023-11-23) +------------------ + +Bug fixes + +* Fix running models that use alias while supporting dbt versions by @binhnq94 in #662 +* Make ``profiles_yml_path`` optional for ``ExecutionMode.DOCKER`` and ``KUBERNETES`` by @MrBones757 in #681 +* Prevent overriding dbt profile fields with profile args of "type" or "method" by @jbandoro in #702 +* Fix ``LoadMode.DBT_LS`` fail when dbt outputs ``WarnErrorOptions`` by @adammarples in #692 +* Add support for env vars in ``RenderConfig`` for dbt ls parsing by @jbandoro in #690 +* Add support for Kubernetes ``on_warning_callback`` by @david-mag in #673 +* Fix ``ExecutionConfig.dbt_executable_path`` to use ``default_factory`` by @jbandoro in #678 + +Others + +* Docs fix: example DAG in the README and docs/index by @tatiana in #705 +* Docs improvement: highlight DAG examples in README by @iancmoritz and @jlaneve in #695 + + +1.2.4 (2023-11-14) +------------------ + +Bug fixes + +* Store ``compiled_sql`` even when task fails by @agreenburg in #671 +* Refactor ``LoadMethod.LOCAL`` to use symlinks instead of copying directory by @jbandoro in #660 +* Fix 'Unable to find the dbt executable: dbt' error by @tatiana in #666 +* Fix installing deps when using ``profile_mapping`` & ``ExecutionMode.LOCAL`` by @joppevos in #659 + +Others + +* Docs: add execution config to MWAA code example by @ugmuka in #674 +* Docs: highlight DAG examples in docs by @iancmoritz and @jlaneve in #695 + + +1.2.3 (2023-11-09) +------------------ + +Bug fix + +* Fix reusing config across TaskGroups/DAGs by @tatiana in #664 + + +1.2.2 (2023-11-06) +------------------ + +Bug fixes + +* Support ``ProjectConfig.dbt_project_path = None`` & different paths for Rendering and Execution by @MrBones757 in #634 +* Fix adding test nodes to DAGs built using ``LoadMethod.DBT_MANIFEST`` and ``LoadMethod.CUSTOM`` by @edgga in #615 + +Others + +* Add pre-commit hook for McCabe max complexity check and fix errors by @jbandoro in #629 +* Update contributing docs for running integration tests by @jbandoro in #638 +* Fix CI issue running integration tests by @tatiana in #640 and #644 +* pre-commit updates in #637 1.2.1 (2023-10-25) diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000000..2ae1cb59dc --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +@astronomer/astro-cosmos-admins @astronomer/astro-cosmos-codeowners @jbandoro @dwreeves diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index aad2b30712..b3378c60a2 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -5,7 +5,7 @@ We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender -identity and expression, level of experience, education, socio-economic status, +identity and expression, level of experience, education, socioeconomic status, nationality, personal appearance, race, religion, or sexual identity and orientation. diff --git a/README.rst b/README.rst index e4f69af639..0310114d70 100644 --- a/README.rst +++ b/README.rst @@ -31,57 +31,29 @@ Run your dbt Core projects as `Apache Airflow `_ DA Quickstart __________ -Check out the Quickstart guide on our `docs `_. +Check out the Getting Started guide on our `docs `_. See more examples at `/dev/dags `_ and at the `cosmos-demo repo `_. Example Usage ___________________ -You can render an Airflow Task Group using the ``DbtTaskGroup`` class. Here's an example with the `jaffle_shop project `_: +You can render a Cosmos Airflow DAG using the ``DbtDag`` class. Here's an example with the `jaffle_shop project `_: +.. + This renders on Github but not Sphinx: -.. code-block:: python +https://github.com/astronomer/astronomer-cosmos/blob/24aa38e528e299ef51ca6baf32f5a6185887d432/dev/dags/basic_cosmos_dag.py#L1-L42 - from pendulum import datetime +This will generate an Airflow DAG that looks like this: - from airflow import DAG - from airflow.operators.empty import EmptyOperator - from cosmos import DbtTaskGroup, ProfileConfig, ProjectConfig - from cosmos.profiles import PostgresUserPasswordProfileMapping +.. figure:: /docs/_static/jaffle_shop_dag.png - profile_config = ProfileConfig( - profile_name="default", - target_name="dev", - profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="airflow_db", - profile_args={"schema": "public"}, - ), - ) - - with DAG( - dag_id="extract_dag", - start_date=datetime(2022, 11, 27), - schedule_interval="@daily", - ): - e1 = EmptyOperator(task_id="pre_dbt") - - dbt_tg = DbtTaskGroup( - project_config=ProjectConfig("jaffle_shop"), - profile_config=profile_config, - ) - - e2 = EmptyOperator(task_id="post_dbt") - - e1 >> dbt_tg >> e2 - -This will generate an Airflow Task Group that looks like this: - -.. figure:: /docs/_static/jaffle_shop_task_group.png Community _________ - Join us on the Airflow `Slack `_ at #airflow-dbt + Changelog _________ @@ -89,6 +61,7 @@ We follow `Semantic Versioning `_ for releases. Check `CHANGELOG.rst `_ for the latest changes. + Contributing Guide __________________ diff --git a/cosmos/__init__.py b/cosmos/__init__.py index 10fea5a1bd..cfaa5def3e 100644 --- a/cosmos/__init__.py +++ b/cosmos/__init__.py @@ -5,20 +5,22 @@ Contains dags, task groups, and operators. """ -__version__ = "1.3.0a1" +__version__ = "1.5.0" + from cosmos.airflow.dag import DbtDag from cosmos.airflow.task_group import DbtTaskGroup from cosmos.config import ( - ProjectConfig, - ProfileConfig, ExecutionConfig, + ProfileConfig, + ProjectConfig, RenderConfig, ) -from cosmos.constants import LoadMode, TestBehavior, ExecutionMode +from cosmos.constants import ExecutionMode, LoadMode, TestBehavior from cosmos.log import get_logger from cosmos.operators.lazy_load import MissingPackage from cosmos.operators.local import ( + DbtBuildLocalOperator, DbtDepsLocalOperator, DbtLSLocalOperator, DbtRunLocalOperator, @@ -28,7 +30,7 @@ DbtTestLocalOperator, ) -logger = get_logger() +logger = get_logger(__name__) try: from cosmos.operators.docker import ( @@ -86,6 +88,37 @@ "kubernetes", ) +try: + from cosmos.operators.azure_container_instance import ( + DbtLSAzureContainerInstanceOperator, + DbtRunAzureContainerInstanceOperator, + DbtRunOperationAzureContainerInstanceOperator, + DbtSeedAzureContainerInstanceOperator, + DbtSnapshotAzureContainerInstanceOperator, + DbtTestAzureContainerInstanceOperator, + ) +except ImportError: + DbtLSAzureContainerInstanceOperator = MissingPackage( + "cosmos.operators.azure_container_instance.DbtLSAzureContainerInstanceOperator", "azure-container-instance" + ) + DbtRunAzureContainerInstanceOperator = MissingPackage( + "cosmos.operators.azure_container_instance.DbtRunAzureContainerInstanceOperator", "azure-container-instance" + ) + DbtRunOperationAzureContainerInstanceOperator = MissingPackage( + "cosmos.operators.azure_container_instance.DbtRunOperationAzureContainerInstanceOperator", + "azure-container-instance", + ) + DbtSeedAzureContainerInstanceOperator = MissingPackage( + "cosmos.operators.azure_container_instance.DbtSeedAzureContainerInstanceOperator", "azure-container-instance" + ) + DbtSnapshotAzureContainerInstanceOperator = MissingPackage( + "cosmos.operators.azure_container_instance.DbtSnapshotAzureContainerInstanceOperator", + "azure-container-instance", + ) + DbtTestAzureContainerInstanceOperator = MissingPackage( + "cosmos.operators.azure_container_instance.DbtTestAzureContainerInstanceOperator", "azure-container-instance" + ) + __all__ = [ "ProjectConfig", "ProfileConfig", @@ -96,6 +129,7 @@ "DbtRunLocalOperator", "DbtSeedLocalOperator", "DbtTestLocalOperator", + "DbtBuildLocalOperator", "DbtDepsLocalOperator", "DbtSnapshotLocalOperator", "DbtDag", @@ -105,14 +139,49 @@ "DbtRunDockerOperator", "DbtSeedDockerOperator", "DbtTestDockerOperator", + "DbtBuildDockerOperator", "DbtSnapshotDockerOperator", "DbtLSKubernetesOperator", "DbtRunOperationKubernetesOperator", "DbtRunKubernetesOperator", "DbtSeedKubernetesOperator", "DbtTestKubernetesOperator", + "DbtBuildKubernetesOperator", "DbtSnapshotKubernetesOperator", + "DbtLSAzureContainerInstanceOperator", + "DbtRunOperationAzureContainerInstanceOperator", + "DbtRunAzureContainerInstanceOperator", + "DbtSeedAzureContainerInstanceOperator", + "DbtTestAzureContainerInstanceOperator", + "DbtSnapshotAzureContainerInstanceOperator", "ExecutionMode", "LoadMode", "TestBehavior", ] + +""" +Required provider info for using Airflow config for configuration +""" + + +def get_provider_info(): + return { + "package-name": "astronomer-cosmos", # Required + "name": "Astronomer Cosmos", # Required + "description": "Astronomer Cosmos is a library for rendering dbt workflows in Airflow. Contains dags, task groups, and operators.", # Required + "versions": [__version__], # Required + "config": { + "cosmos": { + "description": None, + "options": { + "propagate_logs": { + "description": "Enable log propagation from Cosmos custom logger\n", + "version_added": "1.3.0a1", + "type": "boolean", + "example": None, + "default": "True", + }, + }, + }, + }, + } diff --git a/cosmos/airflow/dag.py b/cosmos/airflow/dag.py index d5465ac81c..de958f118f 100644 --- a/cosmos/airflow/dag.py +++ b/cosmos/airflow/dag.py @@ -1,13 +1,14 @@ """ This module contains a function to render a dbt project as an Airflow DAG. """ + from __future__ import annotations from typing import Any from airflow.models.dag import DAG -from cosmos.converter import airflow_kwargs, specific_kwargs, DbtToAirflowConverter +from cosmos.converter import DbtToAirflowConverter, airflow_kwargs, specific_kwargs class DbtDag(DAG, DbtToAirflowConverter): diff --git a/cosmos/airflow/graph.py b/cosmos/airflow/graph.py index 3a140a2357..0c81351787 100644 --- a/cosmos/airflow/graph.py +++ b/cosmos/airflow/graph.py @@ -1,28 +1,39 @@ from __future__ import annotations -from typing import Any, Callable +from typing import Any, Callable, Union from airflow.models import BaseOperator from airflow.models.dag import DAG from airflow.utils.task_group import TaskGroup +from cosmos.config import RenderConfig from cosmos.constants import ( + DEFAULT_DBT_RESOURCES, + TESTABLE_DBT_RESOURCES, DbtResourceType, + ExecutionMode, TestBehavior, TestIndirectSelection, - ExecutionMode, - TESTABLE_DBT_RESOURCES, - DEFAULT_DBT_RESOURCES, ) from cosmos.core.airflow import get_airflow_task as create_airflow_task from cosmos.core.graph.entities import Task as TaskMetadata from cosmos.dbt.graph import DbtNode from cosmos.log import get_logger - logger = get_logger(__name__) +def _snake_case_to_camelcase(value: str) -> str: + """Convert snake_case to CamelCase + + Example: foo_bar_baz -> FooBarBaz + + :param value: Value to convert to CamelCase + :return: Converted value + """ + return "".join(x.capitalize() for x in value.lower().split("_")) + + def calculate_operator_class( execution_mode: ExecutionMode, dbt_class: str, @@ -35,7 +46,9 @@ def calculate_operator_class( :returns: path string to the correspondent Cosmos Airflow operator (e.g. cosmos.operators.localDbtSnapshotLocalOperator) """ - return f"cosmos.operators.{execution_mode.value}.{dbt_class}{execution_mode.value.capitalize()}Operator" + return ( + f"cosmos.operators.{execution_mode.value}.{dbt_class}{_snake_case_to_camelcase(execution_mode.value)}Operator" + ) def calculate_leaves(tasks_ids: list[str], nodes: dict[str, DbtNode]) -> list[str]: @@ -65,6 +78,7 @@ def create_test_task_metadata( task_args: dict[str, Any], on_warning_callback: Callable[..., Any] | None = None, node: DbtNode | None = None, + render_config: RenderConfig | None = None, ) -> TaskMetadata: """ Create the metadata that will be used to instantiate the Airflow Task that will be used to run the Dbt test node. @@ -79,15 +93,25 @@ def create_test_task_metadata( """ task_args = dict(task_args) task_args["on_warning_callback"] = on_warning_callback + extra_context = {} + if test_indirect_selection != TestIndirectSelection.EAGER: task_args["indirect_selection"] = test_indirect_selection.value if node is not None: if node.resource_type == DbtResourceType.MODEL: - task_args["models"] = node.name + task_args["models"] = node.resource_name elif node.resource_type == DbtResourceType.SOURCE: - task_args["select"] = f"source:{node.unique_id[len('source.'):]}" + task_args["select"] = f"source:{node.resource_name}" else: # tested with node.resource_type == DbtResourceType.SEED or DbtResourceType.SNAPSHOT - task_args["select"] = node.name + task_args["select"] = node.resource_name + + extra_context = {"dbt_node_config": node.context_dict} + + elif render_config is not None: # TestBehavior.AFTER_ALL + task_args["select"] = render_config.select + task_args["selector"] = render_config.selector + task_args["exclude"] = render_config.exclude + return TaskMetadata( id=test_task_name, operator_class=calculate_operator_class( @@ -95,6 +119,7 @@ def create_test_task_metadata( dbt_class="DbtTest", ), arguments=task_args, + extra_context=extra_context, ) @@ -108,8 +133,8 @@ def create_task_metadata( :param execution_mode: Where Cosmos should run each dbt task (e.g. ExecutionMode.LOCAL, ExecutionMode.KUBERNETES). Default is ExecutionMode.LOCAL. :param args: Arguments to be used to instantiate an Airflow Task - :param use_name_as_task_id_prefix: If resource_type is DbtResourceType.MODEL, it determines whether - using name as task id prefix or not. If it is True task_id = _run, else task_id=run. + :param use_task_group: It determines whether to use the name as a prefix for the task id or not. + If it is False, then use the name as a prefix for the task id, otherwise do not. :returns: The metadata necessary to instantiate the source dbt node as an Airflow task. """ dbt_resource_to_class = { @@ -117,14 +142,28 @@ def create_task_metadata( DbtResourceType.SNAPSHOT: "DbtSnapshot", DbtResourceType.SEED: "DbtSeed", DbtResourceType.TEST: "DbtTest", + DbtResourceType.SOURCE: "DbtSource", } - args = {**args, **{"models": node.name}} + args = {**args, **{"models": node.resource_name}} if DbtResourceType(node.resource_type) in DEFAULT_DBT_RESOURCES and node.resource_type in dbt_resource_to_class: + extra_context = {"dbt_node_config": node.context_dict} if node.resource_type == DbtResourceType.MODEL: task_id = f"{node.name}_run" if use_task_group is True: task_id = "run" + if node.resource_type == DbtResourceType.SOURCE: + task_full_name = node.unique_id[len("source.") :] + task_id = f"{task_full_name}_source" + args["select"] = f"source:{node.unique_id[len('source.'):]}" + args["models"] = None + if use_task_group is True: + task_id = node.resource_type.value + if node.has_freshness is False: + return TaskMetadata( + id=task_id, + # arguments=args, + ) else: task_id = f"{node.name}_{node.resource_type.value}" if use_task_group is True: @@ -136,6 +175,7 @@ def create_task_metadata( execution_mode=execution_mode, dbt_class=dbt_resource_to_class[node.resource_type] ), arguments=args, + extra_context=extra_context, ) return task_metadata else: @@ -198,12 +238,11 @@ def build_airflow_graph( dag: DAG, # Airflow-specific - parent DAG where to associate tasks and (optional) task groups execution_mode: ExecutionMode, # Cosmos-specific - decide what which class to use task_args: dict[str, Any], # Cosmos/DBT - used to instantiate tasks - test_behavior: TestBehavior, # Cosmos-specific: how to inject tests to Airflow DAG test_indirect_selection: TestIndirectSelection, # Cosmos/DBT - used to set test indirect selection mode dbt_project_name: str, # DBT / Cosmos - used to name test task if mode is after_all, + render_config: RenderConfig, task_group: TaskGroup | None = None, on_warning_callback: Callable[..., Any] | None = None, # argument specific to the DBT test command - node_converters: dict[DbtResourceType, Callable[..., Any]] | None = None, ) -> None: """ Instantiate dbt `nodes` as Airflow tasks within the given `task_group` (optional) or `dag` (mandatory). @@ -223,13 +262,13 @@ def build_airflow_graph( :param execution_mode: Where Cosmos should run each dbt task (e.g. ExecutionMode.LOCAL, ExecutionMode.KUBERNETES). Default is ExecutionMode.LOCAL. :param task_args: Arguments to be used to instantiate an Airflow Task - :param test_behavior: When to run `dbt` tests. Default is TestBehavior.AFTER_EACH, that runs tests after each model. :param dbt_project_name: Name of the dbt pipeline of interest :param task_group: Airflow Task Group instance :param on_warning_callback: A callback function called on warnings with additional Context variables “test_names” and “test_results” of type List. """ - node_converters = node_converters or {} + node_converters = render_config.node_converters or {} + test_behavior = render_config.test_behavior tasks_map = {} task_or_group: TaskGroup | BaseOperator @@ -265,13 +304,24 @@ def build_airflow_graph( test_indirect_selection, task_args=task_args, on_warning_callback=on_warning_callback, + render_config=render_config, ) test_task = create_airflow_task(test_meta, dag, task_group=task_group) leaves_ids = calculate_leaves(tasks_ids=list(tasks_map.keys()), nodes=nodes) for leaf_node_id in leaves_ids: tasks_map[leaf_node_id] >> test_task - # Create the Airflow task dependencies between non-test nodes + create_airflow_task_dependencies(nodes, tasks_map) + + +def create_airflow_task_dependencies( + nodes: dict[str, DbtNode], tasks_map: dict[str, Union[TaskGroup, BaseOperator]] +) -> None: + """ + Create the Airflow task dependencies between non-test nodes. + :param nodes: Dictionary mapping dbt nodes (node.unique_id to node) + :param tasks_map: Dictionary mapping dbt nodes (node.unique_id to Airflow task) + """ for node_id, node in nodes.items(): for parent_node_id in node.depends_on: # depending on the node type, it will not have mapped 1:1 to tasks_map diff --git a/cosmos/airflow/task_group.py b/cosmos/airflow/task_group.py index dcdedb685e..64fcb298aa 100644 --- a/cosmos/airflow/task_group.py +++ b/cosmos/airflow/task_group.py @@ -1,12 +1,14 @@ """ This module contains a function to render a dbt project as an Airflow Task Group. """ + from __future__ import annotations + from typing import Any from airflow.utils.task_group import TaskGroup -from cosmos.converter import airflow_kwargs, specific_kwargs, DbtToAirflowConverter +from cosmos.converter import DbtToAirflowConverter, airflow_kwargs, specific_kwargs class DbtTaskGroup(TaskGroup, DbtToAirflowConverter): diff --git a/cosmos/cache.py b/cosmos/cache.py new file mode 100644 index 0000000000..d519eaca47 --- /dev/null +++ b/cosmos/cache.py @@ -0,0 +1,399 @@ +from __future__ import annotations + +import functools +import hashlib +import json +import os +import shutil +import time +from collections import defaultdict +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import msgpack +from airflow.models import DagRun, Variable +from airflow.models.dag import DAG +from airflow.utils.session import provide_session +from airflow.utils.task_group import TaskGroup +from sqlalchemy import select +from sqlalchemy.orm import Session + +from cosmos import settings +from cosmos.constants import DBT_MANIFEST_FILE_NAME, DBT_TARGET_DIR_NAME, DEFAULT_PROFILES_FILE_NAME +from cosmos.dbt.project import get_partial_parse_path +from cosmos.log import get_logger +from cosmos.settings import cache_dir, dbt_profile_cache_dir_name, enable_cache, enable_cache_profile + +logger = get_logger(__name__) +VAR_KEY_CACHE_PREFIX = "cosmos_cache__" + + +def _get_airflow_metadata(dag: DAG, task_group: TaskGroup | None) -> dict[str, str | None]: + dag_id = None + task_group_id = None + cosmos_type = "DbtDag" + + if task_group: + if task_group.dag_id is not None: + dag_id = task_group.dag_id + if task_group.group_id is not None: + task_group_id = task_group.group_id + cosmos_type = "DbtTaskGroup" + else: + dag_id = dag.dag_id + + return {"cosmos_type": cosmos_type, "dag_id": dag_id, "task_group_id": task_group_id} + + +# It was considered to create a cache identifier based on the dbt project path, as opposed +# to where it is used in Airflow. However, we could have concurrency issues if the same +# dbt cached directory was being used by different dbt task groups or DAGs within the same +# node. For this reason, as a starting point, the cache is identified by where it is used. +# This can be reviewed in the future. +def _create_cache_identifier(dag: DAG, task_group: TaskGroup | None) -> str: + """ + Given a DAG name and a (optional) task_group_name, create the identifier for caching. + + :param dag_name: Name of the Cosmos DbtDag being cached + :param task_group_name: (optional) Name of the Cosmos DbtTaskGroup being cached + :return: Unique identifier representing the cache + """ + metadata = _get_airflow_metadata(dag, task_group) + cache_identifiers_list = [] + dag_id = metadata.get("dag_id") + task_group_id = metadata.get("task_group_id") + + if dag_id: + cache_identifiers_list.append(dag_id) + if task_group_id: + cache_identifiers_list.append(task_group_id.replace(".", "__")) + + return "__".join(cache_identifiers_list) + + +def create_cache_key(cache_identifier: str) -> str: + return f"{VAR_KEY_CACHE_PREFIX}{cache_identifier}" + + +def _obtain_cache_dir_path(cache_identifier: str, base_dir: Path = settings.cache_dir) -> Path: + """ + Return a directory used to cache a specific Cosmos DbtDag or DbtTaskGroup. If the directory + does not exist, create it. + + :param cache_identifier: Unique key used as a cache identifier + :param base_dir: Root directory where cache will be stored + :return: Path to directory used to cache this specific Cosmos DbtDag or DbtTaskGroup + """ + cache_dir_path = base_dir / cache_identifier + tmp_target_dir = cache_dir_path / DBT_TARGET_DIR_NAME + tmp_target_dir.mkdir(parents=True, exist_ok=True) + return cache_dir_path + + +def _get_timestamp(path: Path) -> float: + """ + Return the timestamp of a path or 0, if it does not exist. + + :param path: Path to the file or directory of interest + :return: File or directory timestamp + """ + try: + timestamp = path.stat().st_mtime + except FileNotFoundError: + timestamp = 0 + return timestamp + + +def _get_latest_partial_parse(dbt_project_path: Path, cache_dir: Path) -> Path | None: + """ + Return the path to the latest partial parse file, if defined. + + :param dbt_project_path: Original dbt project path + :param cache_dir: Path to the Cosmos project cache directory + :return: Either return the Path to the latest partial parse file, or None. + """ + project_partial_parse_path = get_partial_parse_path(dbt_project_path) + cosmos_cached_partial_parse_filepath = get_partial_parse_path(cache_dir) + + age_project_partial_parse = _get_timestamp(project_partial_parse_path) + age_cosmos_cached_partial_parse_filepath = _get_timestamp(cosmos_cached_partial_parse_filepath) + + if age_project_partial_parse and age_cosmos_cached_partial_parse_filepath: + if age_project_partial_parse > age_cosmos_cached_partial_parse_filepath: + return project_partial_parse_path + else: + return cosmos_cached_partial_parse_filepath + elif age_project_partial_parse: + return project_partial_parse_path + elif age_cosmos_cached_partial_parse_filepath: + return cosmos_cached_partial_parse_filepath + + return None + + +def _update_partial_parse_cache(latest_partial_parse_filepath: Path, cache_dir: Path) -> None: + """ + Update the cache to have the latest partial parse file contents. + + :param latest_partial_parse_filepath: Path to the most up-to-date partial parse file + :param cache_dir: Path to the Cosmos project cache directory + """ + cache_path = get_partial_parse_path(cache_dir) + cache_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path = get_partial_parse_path(cache_dir).parent / DBT_MANIFEST_FILE_NAME + latest_manifest_filepath = latest_partial_parse_filepath.parent / DBT_MANIFEST_FILE_NAME + + shutil.copyfile(str(latest_partial_parse_filepath), str(cache_path)) + shutil.copyfile(str(latest_manifest_filepath), str(manifest_path)) + + +def patch_partial_parse_content(partial_parse_filepath: Path, project_path: Path) -> bool: + """ + Update, if needed, the root_path references in partial_parse.msgpack to an existing project directory. + This is necessary because an issue is observed where on specific earlier versions of dbt-core like 1.5.4 and 1.6.5, + the commands fail to locate project files as they are pointed to a stale directory by the root_path in the partial + parse file. + + This issue was not observed on recent versions of dbt-core 1.5.8, 1.6.6, 1.7.0 and 1.8.0 as tested on. + It is suspected that PR dbt-labs/dbt-core#8762 is likely the fix and the fix appears to be backported to later + version releases of 1.5.x and 1.6.x. However, the below modification is applied to ensure that the root_path is + correctly set to the needed project directory and the feature is compatible across all dbt-core versions. + + :param partial_parse_filepath: Path to the most up-to-date partial parse file + :param project_path: Path to the target dbt project directory + """ + should_patch_partial_parse_content = False + + try: + with partial_parse_filepath.open("rb") as f: + # Issue reported: https://github.com/astronomer/astronomer-cosmos/issues/971 + # it may be due a race condition of multiple processes trying to read/write this file + data = msgpack.unpack(f) + except ValueError as e: + logger.info("Unable to patch the partial_parse.msgpack file due to %s" % repr(e)) + else: + for node in data["nodes"].values(): + expected_filepath = node.get("root_path") + if expected_filepath is None: + continue + elif expected_filepath and not Path(expected_filepath).exists(): + node["root_path"] = str(project_path) + should_patch_partial_parse_content = True + else: + break + if should_patch_partial_parse_content: + with partial_parse_filepath.open("wb") as f: + packed = msgpack.packb(data) + f.write(packed) + return should_patch_partial_parse_content + + +def _copy_partial_parse_to_project(partial_parse_filepath: Path, project_path: Path) -> None: + """ + Update target dbt project directory to have the latest partial parse file contents. + + :param partial_parse_filepath: Path to the most up-to-date partial parse file + :param project_path: Path to the target dbt project directory + """ + target_partial_parse_file = get_partial_parse_path(project_path) + tmp_target_dir = project_path / DBT_TARGET_DIR_NAME + tmp_target_dir.mkdir(exist_ok=True) + + source_manifest_filepath = partial_parse_filepath.parent / DBT_MANIFEST_FILE_NAME + target_manifest_filepath = target_partial_parse_file.parent / DBT_MANIFEST_FILE_NAME + shutil.copy(str(partial_parse_filepath), str(target_partial_parse_file)) + + patch_partial_parse_content(target_partial_parse_file, project_path) + + if source_manifest_filepath.exists(): + shutil.copy(str(source_manifest_filepath), str(target_manifest_filepath)) + + +def _create_folder_version_hash(dir_path: Path) -> str: + """ + Given a directory, iterate through its content and create a hash that will change in case the + contents of the directory change. The value should not change if the values of the directory do not change, even if + the command is run from different Airflow instances. + + This method output must be concise and it currently changes based on operating system. + """ + # This approach is less efficient than using modified time + # sum([path.stat().st_mtime for path in dir_path.glob("**/*")]) + # unfortunately, the modified time approach does not work well for dag-only deployments + # where DAGs are constantly synced to the deployed Airflow + # for 5k files, this seems to take 0.14 + hasher = hashlib.md5() + filepaths = [] + + for root_dir, dirs, files in os.walk(dir_path): + paths = [os.path.join(root_dir, filepath) for filepath in files] + filepaths.extend(paths) + + for filepath in sorted(filepaths): + with open(str(filepath), "rb") as fp: + buf = fp.read() + hasher.update(buf) + + return hasher.hexdigest() + + +def _calculate_dbt_ls_cache_current_version(cache_identifier: str, project_dir: Path, cmd_args: list[str]) -> str: + """ + Taking into account the project directory contents and the command arguments, calculate the + hash that represents the "dbt ls" command version - to be used to decide if the cache should be refreshed or not. + + :param cache_identifier: Unique identifier of the cache (may include DbtDag or DbtTaskGroup information) + :param project_path: Path to the target dbt project directory + :param cmd_args: List containing the arguments passed to the dbt ls command that would affect its output + """ + start_time = time.perf_counter() + + # Combined value for when the dbt project directory files were last modified + # This is fast (e.g. 0.01s for jaffle shop, 0.135s for a 5k models dbt folder) + dbt_project_hash = _create_folder_version_hash(project_dir) + + # The performance for the following will depend on the user's configuration + hash_args = hashlib.md5("".join(cmd_args).encode()).hexdigest() + + elapsed_time = time.perf_counter() - start_time + logger.info( + f"Cosmos performance: time to calculate cache identifier {cache_identifier} for current version: {elapsed_time}" + ) + return f"{dbt_project_hash},{hash_args}" + + +@functools.lru_cache +def was_project_modified(previous_version: str, current_version: str) -> bool: + """ + Given the cache version of a project and the latest version of the project, + decides if the project was modified or not. + """ + return previous_version != current_version + + +@provide_session +def delete_unused_dbt_ls_cache( + max_age_last_usage: timedelta = timedelta(days=30), session: Session | None = None +) -> int: + """ + Delete Cosmos cache stored in Airflow Variables based on the last execution of their associated DAGs. + + Example usage: + + There are three Cosmos cache Airflow Variables: + 1. ``cache cosmos_cache__basic_cosmos_dag`` + 2. ``cosmos_cache__basic_cosmos_task_group__orders`` + 3. ``cosmos_cache__basic_cosmos_task_group__customers`` + + The first relates to the ``DbtDag`` ``basic_cosmos_dag`` and the two last ones relate to the DAG + ``basic_cosmos_task_group`` that has two ``DbtTaskGroups``: ``orders`` and ``customers``. + + Let's assume the last DAG run of ``basic_cosmos_dag`` was a week ago and the last DAG run of + ``basic_cosmos_task_group`` was an hour ago. + + To delete the cache related to ``DbtDags`` and ``DbtTaskGroup`` that were run more than 5 days ago: + + ..code: python + >>> delete_unused_dbt_ls_cache(max_age_last_usage=timedelta(days=5)) + INFO - Removing the dbt ls cache cosmos_cache__basic_cosmos_dag + + To delete the cache related to ``DbtDags`` and ``DbtTaskGroup`` that were run more than 10 minutes ago: + + ..code: python + >>> delete_unused_dbt_ls_cache(max_age_last_usage=timedelta(minutes=10)) + INFO - Removing the dbt ls cache cosmos_cache__basic_cosmos_dag + INFO - Removing the dbt ls cache cosmos_cache__basic_cosmos_task_group__orders + INFO - Removing the dbt ls cache cosmos_cache__basic_cosmos_task_group__orders + + To delete the cache related to ``DbtDags`` and ``DbtTaskGroup`` that were run more than 10 days ago + + ..code: python + >>> delete_unused_dbt_ls_cache(max_age_last_usage=timedelta(days=10)) + + In this last example, nothing is deleted. + """ + if session is None: + return 0 + + logger.info(f"Delete the Cosmos cache stored in Airflow Variables that hasn't been used for {max_age_last_usage}") + cosmos_dags_ids = defaultdict(list) + all_variables = session.scalars(select(Variable)).all() + total_cosmos_variables = 0 + deleted_cosmos_variables = 0 + + # Identify Cosmos-related cache in Airflow variables + for var in all_variables: + if var.key.startswith(VAR_KEY_CACHE_PREFIX): + var_value = json.loads(var.val) + cosmos_dags_ids[var_value["dag_id"]].append(var.key) + total_cosmos_variables += 1 + + # Delete DAGs that have not been run in the last X time + for dag_id, vars_keys in cosmos_dags_ids.items(): + last_dag_run = ( + session.query(DagRun) + .filter( + DagRun.dag_id == dag_id, + ) + .order_by(DagRun.execution_date.desc()) + .first() + ) + if last_dag_run and last_dag_run.execution_date < (datetime.now(timezone.utc) - max_age_last_usage): + for var_key in vars_keys: + logger.info(f"Removing the dbt ls cache {var_key}") + Variable.delete(var_key) + deleted_cosmos_variables += 1 + + logger.info( + f"Deleted {deleted_cosmos_variables}/{total_cosmos_variables} Airflow Variables used to store Cosmos cache. " + ) + return deleted_cosmos_variables + + +def is_profile_cache_enabled() -> bool: + """Return True if global and profile cache is enable""" + return enable_cache and enable_cache_profile + + +def _get_or_create_profile_cache_dir() -> Path: + """ + Get or create the directory path for caching DBT profiles. + + - Constructs the profile cache directory path based on cache_dir and dbt_profile_cache_dir. + - Checks if the directory exists; if not, creates it + - Return profile cache directory + """ + profile_cache_dir = cache_dir / dbt_profile_cache_dir_name + if not profile_cache_dir.exists(): + profile_cache_dir.mkdir(parents=True, exist_ok=True) + return profile_cache_dir + + +def get_cached_profile(version: str) -> Path | None: + """ + Retrieve the path to a cached DBT profile YML file if it exists for the given version. + + - Constructs the DBT profile YML Path based on version and profile cache directory + - Checks if the profile YML exists + - Return the profile YML Path + """ + profile_yml_path = _get_or_create_profile_cache_dir() / version / DEFAULT_PROFILES_FILE_NAME + if profile_yml_path.exists() and profile_yml_path.is_file(): + return profile_yml_path + return None + + +def create_cache_profile(version: str, profile_content: str) -> Path: + """ + Create a cached DBT profile YAML file with the provided content for the given version. + + - Constructs the path for profile YML based on the version in the profile cache directory + - Creates the profile directory if it does not exist + - Writes the profile content to the profile YML file + - Return the profile YML Path + """ + profile_yml_dir = _get_or_create_profile_cache_dir() / version + profile_yml_dir.mkdir(parents=True, exist_ok=True) + profile_yml_path = profile_yml_dir / DEFAULT_PROFILES_FILE_NAME + profile_yml_path.write_text(profile_content) + return profile_yml_path diff --git a/cosmos/config.py b/cosmos/config.py index 87baba8645..e1e5d56f9a 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -3,12 +3,23 @@ from __future__ import annotations import contextlib +import shutil import tempfile +import warnings from dataclasses import InitVar, dataclass, field from pathlib import Path -from typing import Any, Iterator, Callable - -from cosmos.constants import DbtResourceType, TestBehavior, ExecutionMode, LoadMode, TestIndirectSelection +from typing import Any, Callable, Iterator + +from cosmos.cache import create_cache_profile, get_cached_profile, is_profile_cache_enabled +from cosmos.constants import ( + DEFAULT_PROFILES_FILE_NAME, + DbtResourceType, + ExecutionMode, + InvocationMode, + LoadMode, + TestBehavior, + TestIndirectSelection, +) from cosmos.dbt.executable import get_system_dbt from cosmos.exceptions import CosmosValueError from cosmos.log import get_logger @@ -16,7 +27,13 @@ logger = get_logger(__name__) -DEFAULT_PROFILES_FILE_NAME = "profiles.yml" + +class CosmosConfigException(Exception): + """ + Exceptions related to user misconfiguration. + """ + + pass @dataclass @@ -30,10 +47,14 @@ class RenderConfig: :param load_method: The parsing method for loading the dbt model. Defaults to AUTOMATIC :param select: A list of dbt select arguments (e.g. 'config.materialized:incremental') :param exclude: A list of dbt exclude arguments (e.g. 'tag:nightly') + :param selector: Name of a dbt YAML selector to use for parsing. Only supported when using ``load_method=LoadMode.DBT_LS``. :param dbt_deps: Configure to run dbt deps when using dbt ls for dag parsing :param node_converters: a dictionary mapping a ``DbtResourceType`` into a callable. Users can control how to render dbt nodes in Airflow. Only supported when using ``load_method=LoadMode.DBT_MANIFEST`` or ``LoadMode.DBT_LS``. - :param dbt_executable_path: The path to the dbt executable for dag generation. Defaults to dbt if available on the path. Mutually Exclusive with ProjectConfig.dbt_project_path - :param dbt_project_path Configures the DBT project location accessible on the airflow controller for DAG rendering - Required when using ``load_method=LoadMode.DBT_LS`` or ``load_method=LoadMode.CUSTOM`` + :param dbt_executable_path: The path to the dbt executable for dag generation. Defaults to dbt if available on the path. + :param env_vars: (Deprecated since Cosmos 1.3 use ProjectConfig.env_vars) A dictionary of environment variables for rendering. Only supported when using ``LoadMode.DBT_LS``. + :param dbt_project_path: Configures the DBT project location accessible on the airflow controller for DAG rendering. Mutually Exclusive with ProjectConfig.dbt_project_path. Required when using ``load_method=LoadMode.DBT_LS`` or ``load_method=LoadMode.CUSTOM``. + :param dbt_ls_path: Configures the location of an output of ``dbt ls``. Required when using ``load_method=LoadMode.DBT_LS_FILE``. + :param enable_mock_profile: Allows to enable/disable mocking profile. Enabled by default. Mock profiles are useful for parsing Cosmos DAGs in the CI, but should be disabled to benefit from partial parsing (since Cosmos 1.4). """ emit_datasets: bool = True @@ -41,15 +62,56 @@ class RenderConfig: load_method: LoadMode = LoadMode.AUTOMATIC select: list[str] = field(default_factory=list) exclude: list[str] = field(default_factory=list) + selector: str | None = None dbt_deps: bool = True node_converters: dict[DbtResourceType, Callable[..., Any]] | None = None dbt_executable_path: str | Path = get_system_dbt() + env_vars: dict[str, str] | None = None dbt_project_path: InitVar[str | Path | None] = None - + dbt_ls_path: Path | None = None project_path: Path | None = field(init=False) + enable_mock_profile: bool = True + airflow_vars_to_purge_dbt_ls_cache: list[str] = field(default_factory=list) def __post_init__(self, dbt_project_path: str | Path | None) -> None: + if self.env_vars: + warnings.warn( + "RenderConfig.env_vars is deprecated since Cosmos 1.3 and will be removed in Cosmos 2.0. Use ProjectConfig.env_vars instead.", + DeprecationWarning, + ) self.project_path = Path(dbt_project_path) if dbt_project_path else None + # allows us to initiate this attribute from Path objects and str + self.dbt_ls_path = Path(self.dbt_ls_path) if self.dbt_ls_path else None + + def validate_dbt_command(self, fallback_cmd: str | Path = "") -> None: + """ + When using LoadMode.DBT_LS, the dbt executable path is necessary for rendering. + + Validates that the original dbt command works, if not, attempt to use the fallback_dbt_cmd. + If neither works, raise an exception. + + The fallback behaviour is necessary for Cosmos < 1.2.2 backwards compatibility. + """ + if not shutil.which(self.dbt_executable_path): + if isinstance(fallback_cmd, Path): + fallback_cmd = fallback_cmd.as_posix() + + if fallback_cmd and shutil.which(fallback_cmd): + self.dbt_executable_path = fallback_cmd + else: + raise CosmosConfigException( + "Unable to find the dbt executable, attempted: " + f"<{self.dbt_executable_path}>" + (f" and <{fallback_cmd}>." if fallback_cmd else ".") + ) + + def is_dbt_ls_file_available(self) -> bool: + """ + Check if the `dbt ls` output is set and if the file exists. + """ + if not self.dbt_ls_path: + return False + + return self.dbt_ls_path.exists() class ProjectConfig: @@ -64,6 +126,14 @@ class ProjectConfig: :param manifest_path: The absolute path to the dbt manifest file. Defaults to None :param project_name: Allows the user to define the project name. Required if dbt_project_path is not defined. Defaults to the folder name of dbt_project_path. + :param env_vars: Dictionary of environment variables that are used for both rendering and execution. Rendering with + env vars is only supported when using ``RenderConfig.LoadMode.DBT_LS`` load mode. + :param dbt_vars: Dictionary of dbt variables for the project. This argument overrides variables defined in your dbt_project.yml + file. The dictionary is dumped to a yaml string and passed to dbt commands as the --vars argument. Variables are only + supported for rendering when using ``RenderConfig.LoadMode.DBT_LS`` and ``RenderConfig.LoadMode.CUSTOM`` load mode. + :param partial_parse: If True, then attempt to use the ``partial_parse.msgpack`` if it exists. This is only used + for the ``LoadMode.DBT_LS`` load mode, and for the ``ExecutionMode.LOCAL`` and ``ExecutionMode.VIRTUALENV`` + execution modes. """ dbt_project_path: Path | None = None @@ -81,6 +151,9 @@ def __init__( snapshots_relative_path: str | Path = "snapshots", manifest_path: str | Path | None = None, project_name: str | None = None, + env_vars: dict[str, str] | None = None, + dbt_vars: dict[str, str] | None = None, + partial_parse: bool = True, ): # Since we allow dbt_project_path to be defined in ExecutionConfig and RenderConfig # dbt_project_path may not always be defined here. @@ -104,6 +177,10 @@ def __init__( if manifest_path: self.manifest_path = Path(manifest_path) + self.env_vars = env_vars + self.dbt_vars = dbt_vars + self.partial_parse = partial_parse + def validate_project(self) -> None: """ Validates necessary context is present for a project. @@ -165,54 +242,83 @@ class ProfileConfig: profile_mapping: BaseProfileMapping | None = None def __post_init__(self) -> None: - "Validates that we have enough information to render a profile." - # if using a user-supplied profiles.yml, validate that it exists - if self.profiles_yml_filepath and not Path(self.profiles_yml_filepath).exists(): - raise CosmosValueError(f"The file {self.profiles_yml_filepath} does not exist.") + self.validate_profile() def validate_profile(self) -> None: - "Validates that we have enough information to render a profile." + """Validates that we have enough information to render a profile.""" if not self.profiles_yml_filepath and not self.profile_mapping: raise CosmosValueError("Either profiles_yml_filepath or profile_mapping must be set to render a profile") + if self.profiles_yml_filepath and self.profile_mapping: + raise CosmosValueError( + "Both profiles_yml_filepath and profile_mapping are defined and are mutually exclusive. Ensure only one of these is defined." + ) + + def validate_profiles_yml(self) -> None: + """Validates a user-supplied profiles.yml is present""" + if self.profiles_yml_filepath and not Path(self.profiles_yml_filepath).exists(): + raise CosmosValueError(f"The file {self.profiles_yml_filepath} does not exist.") + + def _get_profile_path(self, use_mock_values: bool = False) -> Path: + """ + Handle the profile caching mechanism. + + Check if profile object version is exist then reuse it + Otherwise, create profile yml for requested object and return the profile path + """ + assert self.profile_mapping # To satisfy MyPy + current_profile_version = self.profile_mapping.version(self.profile_name, self.target_name, use_mock_values) + cached_profile_path = get_cached_profile(current_profile_version) + if cached_profile_path: + logger.info("Profile found in cache using profile: %s.", cached_profile_path) + return cached_profile_path + else: + profile_contents = self.profile_mapping.get_profile_file_contents( + profile_name=self.profile_name, target_name=self.target_name, use_mock_values=use_mock_values + ) + profile_path = create_cache_profile(current_profile_version, profile_contents) + logger.info("Profile not found in cache storing and using profile: %s.", profile_path) + return profile_path @contextlib.contextmanager def ensure_profile( self, desired_profile_path: Path | None = None, use_mock_values: bool = False ) -> Iterator[tuple[Path, dict[str, str]]]: - "Context manager to ensure that there is a profile. If not, create one." + """Context manager to ensure that there is a profile. If not, create one.""" if self.profiles_yml_filepath: logger.info("Using user-supplied profiles.yml at %s", self.profiles_yml_filepath) yield Path(self.profiles_yml_filepath), {} - elif self.profile_mapping: - profile_contents = self.profile_mapping.get_profile_file_contents( - profile_name=self.profile_name, target_name=self.target_name, use_mock_values=use_mock_values - ) - - if use_mock_values: - env_vars = {} + if is_profile_cache_enabled(): + logger.info("Profile caching is enable.") + cached_profile_path = self._get_profile_path(use_mock_values) + env_vars = {} if use_mock_values else self.profile_mapping.env_vars + yield cached_profile_path, env_vars else: - env_vars = self.profile_mapping.env_vars - - if desired_profile_path: - logger.info( - "Writing profile to %s with the following contents:\n%s", - desired_profile_path, - profile_contents, + profile_contents = self.profile_mapping.get_profile_file_contents( + profile_name=self.profile_name, target_name=self.target_name, use_mock_values=use_mock_values ) - # write profile_contents to desired_profile_path using yaml library - desired_profile_path.write_text(profile_contents) - yield desired_profile_path, env_vars - else: - with tempfile.TemporaryDirectory() as temp_dir: - temp_file = Path(temp_dir) / DEFAULT_PROFILES_FILE_NAME + env_vars = {} if use_mock_values else self.profile_mapping.env_vars + + if desired_profile_path: logger.info( - "Creating temporary profiles.yml at %s with the following contents:\n%s", - temp_file, + "Writing profile to %s with the following contents:\n%s", + desired_profile_path, profile_contents, ) - temp_file.write_text(profile_contents) - yield temp_file, env_vars + # write profile_contents to desired_profile_path using yaml library + desired_profile_path.write_text(profile_contents) + yield desired_profile_path, env_vars + else: + with tempfile.TemporaryDirectory() as temp_dir: + temp_file = Path(temp_dir) / DEFAULT_PROFILES_FILE_NAME + logger.info( + "Creating temporary profiles.yml with use_mock_values=%s at %s with the following contents:\n%s", + use_mock_values, + temp_file, + profile_contents, + ) + temp_file.write_text(profile_contents) + yield temp_file, env_vars @dataclass @@ -221,18 +327,33 @@ class ExecutionConfig: Contains configuration about how to execute dbt. :param execution_mode: The execution mode for dbt. Defaults to local + :param invocation_mode: The invocation mode for the dbt command. This is only configurable for ExecutionMode.LOCAL. :param test_indirect_selection: The mode to configure the test behavior when performing indirect selection. :param dbt_executable_path: The path to the dbt executable for runtime execution. Defaults to dbt if available on the path. - :param dbt_project_path Configures the DBT project location accessible at runtime for dag execution. This is the project path in a docker container for ExecutionMode.DOCKER or ExecutionMode.KUBERNETES. Mutually Exclusive with ProjectConfig.dbt_project_path + :param dbt_project_path: Configures the DBT project location accessible at runtime for dag execution. This is the project path in a docker container for ExecutionMode.DOCKER or ExecutionMode.KUBERNETES. Mutually Exclusive with ProjectConfig.dbt_project_path """ execution_mode: ExecutionMode = ExecutionMode.LOCAL + invocation_mode: InvocationMode | None = None test_indirect_selection: TestIndirectSelection = TestIndirectSelection.EAGER - dbt_executable_path: str | Path = get_system_dbt() + dbt_executable_path: str | Path = field(default_factory=get_system_dbt) dbt_project_path: InitVar[str | Path | None] = None - project_path: Path | None = field(init=False) def __post_init__(self, dbt_project_path: str | Path | None) -> None: + if self.invocation_mode and self.execution_mode not in (ExecutionMode.LOCAL, ExecutionMode.VIRTUALENV): + raise CosmosValueError( + "ExecutionConfig.invocation_mode is only configurable for ExecutionMode.LOCAL and ExecutionMode.VIRTUALENV." + ) + if self.execution_mode == ExecutionMode.VIRTUALENV: + if self.invocation_mode == InvocationMode.DBT_RUNNER: + raise CosmosValueError( + "InvocationMode.DBT_RUNNER has not been implemented for ExecutionMode.VIRTUALENV" + ) + elif self.invocation_mode is None: + logger.debug( + "Defaulting to InvocationMode.SUBPROCESS as it is the only supported invocation mode for ExecutionMode.VIRTUALENV" + ) + self.invocation_mode = InvocationMode.SUBPROCESS self.project_path = Path(dbt_project_path) if dbt_project_path else None diff --git a/cosmos/constants.py b/cosmos/constants.py index 9aa38c34e6..956660e016 100644 --- a/cosmos/constants.py +++ b/cosmos/constants.py @@ -3,21 +3,30 @@ from pathlib import Path import aenum - +from packaging.version import Version DBT_PROFILE_PATH = Path(os.path.expanduser("~")).joinpath(".dbt/profiles.yml") DEFAULT_DBT_PROFILE_NAME = "cosmos_profile" DEFAULT_DBT_TARGET_NAME = "cosmos_target" +DEFAULT_COSMOS_CACHE_DIR_NAME = "cosmos" DBT_LOG_PATH_ENVVAR = "DBT_LOG_PATH" DBT_LOG_DIR_NAME = "logs" DBT_TARGET_PATH_ENVVAR = "DBT_TARGET_PATH" DBT_TARGET_DIR_NAME = "target" +DBT_PARTIAL_PARSE_FILE_NAME = "partial_parse.msgpack" +DBT_MANIFEST_FILE_NAME = "manifest.json" +DBT_DEPENDENCIES_FILE_NAMES = {"packages.yml", "dependencies.yml"} DBT_LOG_FILENAME = "dbt.log" DBT_BINARY_NAME = "dbt" +DEFAULT_PROFILES_FILE_NAME = "profiles.yml" DEFAULT_OPENLINEAGE_NAMESPACE = "cosmos" OPENLINEAGE_PRODUCER = "https://github.com/astronomer/astronomer-cosmos/" +# Cosmos will not emit datasets for the following Airflow versions, due to a breaking change that's fixed in later Airflow 2.x versions +# https://github.com/apache/airflow/issues/39486 +PARTIALLY_SUPPORTED_AIRFLOW_VERSIONS = [Version("2.9.0"), Version("2.9.1")] + class LoadMode(Enum): """ @@ -27,6 +36,8 @@ class LoadMode(Enum): AUTOMATIC = "automatic" CUSTOM = "custom" DBT_LS = "dbt_ls" + DBT_LS_FILE = "dbt_ls_file" + DBT_LS_CACHE = "dbt_ls_cache" DBT_MANIFEST = "dbt_manifest" @@ -48,7 +59,18 @@ class ExecutionMode(Enum): LOCAL = "local" DOCKER = "docker" KUBERNETES = "kubernetes" + AWS_EKS = "aws_eks" VIRTUALENV = "virtualenv" + AZURE_CONTAINER_INSTANCE = "azure_container_instance" + + +class InvocationMode(Enum): + """ + How the dbt command should be invoked. + """ + + SUBPROCESS = "subprocess" + DBT_RUNNER = "dbt_runner" class TestIndirectSelection(Enum): diff --git a/cosmos/converter.py b/cosmos/converter.py index dbc290271e..40929ef559 100644 --- a/cosmos/converter.py +++ b/cosmos/converter.py @@ -3,22 +3,41 @@ from __future__ import annotations +import copy import inspect +import os +import platform +import time from typing import Any, Callable +from warnings import warn from airflow.models.dag import DAG from airflow.utils.task_group import TaskGroup +from cosmos import cache, settings from cosmos.airflow.graph import build_airflow_graph +from cosmos.config import ExecutionConfig, ProfileConfig, ProjectConfig, RenderConfig +from cosmos.constants import ExecutionMode from cosmos.dbt.graph import DbtGraph from cosmos.dbt.selector import retrieve_by_label -from cosmos.config import ProjectConfig, ExecutionConfig, RenderConfig, ProfileConfig from cosmos.exceptions import CosmosValueError from cosmos.log import get_logger logger = get_logger(__name__) +def migrate_to_new_interface( + execution_config: ExecutionConfig, project_config: ProjectConfig, render_config: RenderConfig +): + # We copy the configuration so the change does not affect other DAGs or TaskGroups + # that may reuse the same original configuration + render_config = copy.deepcopy(render_config) + execution_config = copy.deepcopy(execution_config) + render_config.project_path = project_config.dbt_project_path + execution_config.project_path = project_config.dbt_project_path + return execution_config, render_config + + def specific_kwargs(**kwargs: dict[str, Any]) -> dict[str, Any]: """ Extract kwargs specific to the cosmos.converter.DbtToAirflowConverter class initialization method. @@ -42,13 +61,17 @@ def airflow_kwargs(**kwargs: dict[str, Any]) -> dict[str, Any]: new_kwargs = {} non_airflow_kwargs = specific_kwargs(**kwargs) for arg_key, arg_value in kwargs.items(): - if arg_key not in non_airflow_kwargs: + if arg_key not in non_airflow_kwargs or arg_key == "dag": new_kwargs[arg_key] = arg_value return new_kwargs def validate_arguments( - select: list[str], exclude: list[str], profile_args: dict[str, Any], task_args: dict[str, Any] + select: list[str], + exclude: list[str], + profile_config: ProfileConfig, + task_args: dict[str, Any], + execution_mode: ExecutionMode, ) -> None: """ Validate that mutually exclusive selectors filters have not been given. @@ -56,8 +79,9 @@ def validate_arguments( :param select: A list of dbt select arguments (e.g. 'config.materialized:incremental') :param exclude: A list of dbt exclude arguments (e.g. 'tag:nightly') - :param profile_args: Arguments to pass to the dbt profile + :param profile_config: ProfileConfig Object :param task_args: Arguments to be used to instantiate an Airflow Task + :param execution_mode: the current execution mode """ for field in ("tags", "paths"): select_items = retrieve_by_label(select, field) @@ -68,8 +92,95 @@ def validate_arguments( # if task_args has a schema, add it to the profile args and add a deprecated warning if "schema" in task_args: - profile_args["schema"] = task_args["schema"] logger.warning("Specifying a schema in the `task_args` is deprecated. Please use the `profile_args` instead.") + if profile_config.profile_mapping: + profile_config.profile_mapping.profile_args["schema"] = task_args["schema"] + + if execution_mode in [ExecutionMode.LOCAL, ExecutionMode.VIRTUALENV]: + profile_config.validate_profiles_yml() + + +def validate_initial_user_config( + execution_config: ExecutionConfig, + profile_config: ProfileConfig | None, + project_config: ProjectConfig, + render_config: RenderConfig, + operator_args: dict[str, Any], +): + """ + Validates if the user set the fields as expected. + + :param execution_config: Configuration related to how to run dbt in Airflow tasks + :param profile_config: Configuration related to dbt database configuration (profile) + :param project_config: Configuration related to the overall dbt project + :param render_config: Configuration related to how to convert the dbt workflow into an Airflow DAG + :param operator_args: Arguments to pass to the underlying operators. + """ + if profile_config is None and execution_config.execution_mode not in ( + ExecutionMode.KUBERNETES, + ExecutionMode.AWS_EKS, + ExecutionMode.DOCKER, + ): + raise CosmosValueError(f"The profile_config is mandatory when using {execution_config.execution_mode}") + + # Since we now support both project_config.dbt_project_path, render_config.project_path and execution_config.project_path + # We need to ensure that only one interface is being used. + if project_config.dbt_project_path and (render_config.project_path or execution_config.project_path): + raise CosmosValueError( + "ProjectConfig.dbt_project_path is mutually exclusive with RenderConfig.dbt_project_path and ExecutionConfig.dbt_project_path." + + "If using RenderConfig.dbt_project_path or ExecutionConfig.dbt_project_path, ProjectConfig.dbt_project_path should be None" + ) + + # Cosmos 2.0 will remove the ability to pass in operator_args with 'env' and 'vars' in place of ProjectConfig.env_vars and + # ProjectConfig.dbt_vars. + if "env" in operator_args: + warn( + "operator_args with 'env' is deprecated since Cosmos 1.3 and will be removed in Cosmos 2.0. Use ProjectConfig.env_vars instead.", + DeprecationWarning, + ) + if project_config.env_vars: + raise CosmosValueError( + "ProjectConfig.env_vars and operator_args with 'env' are mutually exclusive and only one can be used." + ) + if "vars" in operator_args: + warn( + "operator_args with 'vars' is deprecated since Cosmos 1.3 and will be removed in Cosmos 2.0. Use ProjectConfig.vars instead.", + DeprecationWarning, + ) + if project_config.dbt_vars: + raise CosmosValueError( + "ProjectConfig.dbt_vars and operator_args with 'vars' are mutually exclusive and only one can be used." + ) + # Cosmos 2.0 will remove the ability to pass RenderConfig.env_vars in place of ProjectConfig.env_vars, check that both are not set. + if project_config.env_vars and render_config.env_vars: + raise CosmosValueError( + "Both ProjectConfig.env_vars and RenderConfig.env_vars were provided. RenderConfig.env_vars is deprecated since Cosmos 1.3, " + "please use ProjectConfig.env_vars instead." + ) + + +def validate_adapted_user_config( + execution_config: ExecutionConfig | None, project_config: ProjectConfig, render_config: RenderConfig | None +): + """ + Validates if all the necessary fields required by Cosmos to render the DAG are set. + + :param execution_config: Configuration related to how to run dbt in Airflow tasks + :param project_config: Configuration related to the overall dbt project + :param render_config: Configuration related to how to convert the dbt workflow into an Airflow DAG + """ + # At this point, execution_config.project_path should always be non-null + if not execution_config.project_path: + raise CosmosValueError( + "ExecutionConfig.dbt_project_path is required for the execution of dbt tasks in all execution modes." + ) + + # We now have a guaranteed execution_config.project_path, but still need to process render_config.project_path + # We require render_config.project_path when we dont have a manifest + if not project_config.manifest_path and not render_config.project_path: + raise CosmosValueError( + "RenderConfig.dbt_project_path is required for rendering an airflow DAG from a DBT Graph if no manifest is provided." + ) class DbtToAirflowConverter: @@ -90,7 +201,7 @@ class DbtToAirflowConverter: def __init__( self, project_config: ProjectConfig, - profile_config: ProfileConfig, + profile_config: ProfileConfig | None = None, execution_config: ExecutionConfig | None = None, render_config: RenderConfig | None = None, dag: DAG | None = None, @@ -102,85 +213,83 @@ def __init__( ) -> None: project_config.validate_project() - if not execution_config: - execution_config = ExecutionConfig() - if not render_config: - render_config = RenderConfig() + execution_config = execution_config or ExecutionConfig() + render_config = render_config or RenderConfig() + operator_args = operator_args or {} - # Since we now support both project_config.dbt_project_path, render_config.project_path and execution_config.project_path - # We need to ensure that only one interface is being used. - if project_config.dbt_project_path and (render_config.project_path or execution_config.project_path): - raise CosmosValueError( - "ProjectConfig.dbt_project_path is mutually exclusive with RenderConfig.dbt_project_path and ExecutionConfig.dbt_project_path." - + "If using RenderConfig.dbt_project_path or ExecutionConfig.dbt_project_path, ProjectConfig.dbt_project_path should be None" - ) + validate_initial_user_config(execution_config, profile_config, project_config, render_config, operator_args) - # If we are using the old interface, we should migrate it to the new interface - # This is safe to do now since we have validated which config interface we're using if project_config.dbt_project_path: - render_config.project_path = project_config.dbt_project_path - execution_config.project_path = project_config.dbt_project_path + execution_config, render_config = migrate_to_new_interface(execution_config, project_config, render_config) - # At this point, execution_config.project_path should always be non-null - if not execution_config.project_path: - raise CosmosValueError( - "ExecutionConfig.dbt_project_path is required for the execution of dbt tasks in all execution modes." - ) + validate_adapted_user_config(execution_config, project_config, render_config) - # We now have a guaranteed execution_config.project_path, but still need to process render_config.project_path - # We require render_config.project_path when we dont have a manifest - if not project_config.manifest_path and not render_config.project_path: - raise CosmosValueError( - "RenderConfig.dbt_project_path is required for rendering an airflow DAG from a DBT Graph if no manifest is provided." - ) + env_vars = project_config.env_vars or operator_args.get("env") + dbt_vars = project_config.dbt_vars or operator_args.get("vars") - profile_args = {} - if profile_config.profile_mapping: - profile_args = profile_config.profile_mapping.profile_args - - if not operator_args: - operator_args = {} - - # Previously, we were creating a cosmos.dbt.project.DbtProject - # DbtProject has now been replaced with ProjectConfig directly - # since the interface of the two classes were effectively the same - # Under this previous implementation, we were passing: - # - name, root dir, models dir, snapshots dir and manifest path - # Internally in the dbtProject class, we were defaulting the profile_path - # To be root dir/profiles.yml - # To keep this logic working, if converter is given no ProfileConfig, - # we can create a default retaining this value to preserve this functionality. - # We may want to consider defaulting this value in our actual ProjceConfig class? - dbt_graph = DbtGraph( + cache_dir = None + cache_identifier = None + if settings.enable_cache: + cache_identifier = cache._create_cache_identifier(dag, task_group) + cache_dir = cache._obtain_cache_dir_path(cache_identifier=cache_identifier) + + previous_time = time.perf_counter() + self.dbt_graph = DbtGraph( project=project_config, render_config=render_config, execution_config=execution_config, - dbt_cmd=render_config.dbt_executable_path, profile_config=profile_config, - operator_args=operator_args, + cache_dir=cache_dir, + cache_identifier=cache_identifier, + dbt_vars=dbt_vars, + airflow_metadata=cache._get_airflow_metadata(dag, task_group), + ) + self.dbt_graph.load(method=render_config.load_method, execution_mode=execution_config.execution_mode) + + current_time = time.perf_counter() + elapsed_time = current_time - previous_time + logger.info( + f"Cosmos performance ({cache_identifier}) - [{platform.node()}|{os.getpid()}]: It took {elapsed_time:.3}s to parse the dbt project for DAG using {self.dbt_graph.load_method}" ) - dbt_graph.load(method=render_config.load_method, execution_mode=execution_config.execution_mode) + previous_time = current_time task_args = { **operator_args, "project_dir": execution_config.project_path, + "partial_parse": project_config.partial_parse, "profile_config": profile_config, "emit_datasets": render_config.emit_datasets, + "env": env_vars, + "vars": dbt_vars, + "cache_dir": cache_dir, } if execution_config.dbt_executable_path: task_args["dbt_executable_path"] = execution_config.dbt_executable_path + if execution_config.invocation_mode: + task_args["invocation_mode"] = execution_config.invocation_mode - validate_arguments(render_config.select, render_config.exclude, profile_args, task_args) + validate_arguments( + render_config.select, + render_config.exclude, + profile_config, + task_args, + execution_mode=execution_config.execution_mode, + ) build_airflow_graph( - nodes=dbt_graph.filtered_nodes, + nodes=self.dbt_graph.filtered_nodes, dag=dag or (task_group and task_group.dag), task_group=task_group, execution_mode=execution_config.execution_mode, task_args=task_args, - test_behavior=render_config.test_behavior, test_indirect_selection=execution_config.test_indirect_selection, dbt_project_name=project_config.project_name, on_warning_callback=on_warning_callback, - node_converters=render_config.node_converters, + render_config=render_config, + ) + + current_time = time.perf_counter() + elapsed_time = current_time - previous_time + logger.info( + f"Cosmos performance ({cache_identifier}) - [{platform.node()}|{os.getpid()}]: It took {elapsed_time:.3}s to build the Airflow DAG." ) diff --git a/cosmos/core/airflow.py b/cosmos/core/airflow.py index 7c5dee3281..d4a9624832 100644 --- a/cosmos/core/airflow.py +++ b/cosmos/core/airflow.py @@ -7,7 +7,6 @@ from cosmos.core.graph.entities import Task from cosmos.log import get_logger - logger = get_logger(__name__) @@ -30,6 +29,7 @@ def get_airflow_task(task: Task, dag: DAG, task_group: "TaskGroup | None" = None task_id=task.id, dag=dag, task_group=task_group, + extra_context=task.extra_context, **task.arguments, ) diff --git a/cosmos/core/graph/entities.py b/cosmos/core/graph/entities.py index f88c3d6b23..3c3ee58d03 100644 --- a/cosmos/core/graph/entities.py +++ b/cosmos/core/graph/entities.py @@ -59,3 +59,4 @@ class Task(CosmosEntity): operator_class: str = "airflow.operators.empty.EmptyOperator" arguments: Dict[str, Any] = field(default_factory=dict) + extra_context: Dict[str, Any] = field(default_factory=dict) diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index 0322c8ac4a..a072014dd5 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -1,15 +1,23 @@ from __future__ import annotations +import base64 +import datetime +import functools import itertools import json import os -import shutil +import platform import tempfile +import zlib from dataclasses import dataclass, field +from functools import cached_property from pathlib import Path from subprocess import PIPE, Popen from typing import Any +from airflow.models import Variable + +from cosmos import cache, settings from cosmos.config import ExecutionConfig, ProfileConfig, ProjectConfig, RenderConfig from cosmos.constants import ( DBT_LOG_DIR_NAME, @@ -21,8 +29,8 @@ ExecutionMode, LoadMode, ) -from cosmos.dbt.executable import get_system_dbt from cosmos.dbt.parser.project import LegacyDbtProject +from cosmos.dbt.project import create_symlinks, environ, get_partial_parse_path, has_non_empty_dependencies_file from cosmos.dbt.selector import select_nodes from cosmos.log import get_logger @@ -40,31 +48,82 @@ class CosmosLoadDbtException(Exception): @dataclass class DbtNode: """ - Metadata related to a dbt node (e.g. model, seed, snapshot). + Metadata related to a dbt node (e.g. model, seed, snapshot, source). """ - name: str unique_id: str resource_type: DbtResourceType depends_on: list[str] file_path: Path tags: list[str] = field(default_factory=lambda: []) config: dict[str, Any] = field(default_factory=lambda: {}) + has_freshness: bool = False has_test: bool = False + @property + def resource_name(self) -> str: + """ + Use this property to retrieve the resource name for command generation, for instance: ["dbt", "run", "--models", f"{resource_name}"]. + The unique_id format is defined as [..](https://docs.getdbt.com/reference/artifacts/manifest-json#resource-details). + For a special case like a versioned model, the unique_id follows this pattern: [model...](https://github.com/dbt-labs/dbt-core/blob/main/core/dbt/contracts/graph/node_args.py#L26C3-L31) + """ + return self.unique_id.split(".", 2)[2] + + @property + def name(self) -> str: + """ + Use this property as the task name or task group name. + Replace period (.) with underscore (_) due to versioned models. + """ + return self.resource_name.replace(".", "_") + + @property + def context_dict(self) -> dict[str, Any]: + """ + Returns a dictionary containing all the attributes of the DbtNode object, + ensuring that the output is JSON serializable so it can be stored in Airflow's db + """ + return { + "unique_id": self.unique_id, + "resource_type": self.resource_type.value, # convert enum to value + "depends_on": self.depends_on, + "file_path": str(self.file_path), # convert path to string + "tags": self.tags, + "config": self.config, + "has_test": self.has_test, + "resource_name": self.resource_name, + "name": self.name, + } + -def create_symlinks(project_path: Path, tmp_dir: Path) -> None: - """Helper function to create symlinks to the dbt project files.""" - ignore_paths = (DBT_LOG_DIR_NAME, DBT_TARGET_DIR_NAME, "dbt_packages", "profiles.yml") - for child_name in os.listdir(project_path): - if child_name not in ignore_paths: - os.symlink(project_path / child_name, tmp_dir / child_name) +def is_freshness_effective(freshness: dict[str, Any]) -> bool: + """Function to find if a source has null freshness. Scenarios where freshness + looks like: + "freshness": { + "warn_after": { + "count": null, + "period": null + }, + "error_after": { + "count": null, + "period": null + }, + "filter": null + } + should be considered as null, this function ensures that.""" + for key, value in freshness.items(): + if isinstance(value, dict): + if any(subvalue is not None for subvalue in value.values()): + return True + elif value is not None: + return True + return False def run_command(command: list[str], tmp_dir: Path, env_vars: dict[str, str]) -> str: """Run a command in a subprocess, returning the stdout.""" logger.info("Running command: `%s`", " ".join(command)) - logger.info("Environment variable keys: %s", env_vars.keys()) + logger.debug("Environment variable keys: %s", env_vars.keys()) process = Popen( command, stdout=PIPE, @@ -81,14 +140,14 @@ def run_command(command: list[str], tmp_dir: Path, env_vars: dict[str, str]) -> "Unable to run dbt ls command due to missing dbt_packages. Set RenderConfig.dbt_deps=True." ) - if returncode or "Error" in stdout: + if returncode or "Error" in stdout.replace("WarnErrorOptions", ""): details = stderr or stdout raise CosmosLoadDbtException(f"Unable to run {command} due to the error:\n{details}") return stdout -def parse_dbt_ls_output(project_path: Path, ls_stdout: str) -> dict[str, DbtNode]: +def parse_dbt_ls_output(project_path: Path | None, ls_stdout: str) -> dict[str, DbtNode]: """Parses the output of `dbt ls` into a dictionary of `DbtNode` instances.""" nodes = {} for line in ls_stdout.split("\n"): @@ -98,13 +157,15 @@ def parse_dbt_ls_output(project_path: Path, ls_stdout: str) -> dict[str, DbtNode logger.debug("Skipped dbt ls line: %s", line) else: node = DbtNode( - name=node_dict.get("alias", node_dict["name"]), unique_id=node_dict["unique_id"], resource_type=DbtResourceType(node_dict["resource_type"]), depends_on=node_dict.get("depends_on", {}).get("nodes", []), file_path=project_path / node_dict["original_file_path"], - tags=node_dict["tags"], - config=node_dict["config"], + tags=node_dict.get("tags", []), + config=node_dict.get("config", {}), + has_freshness=is_freshness_effective(node_dict.get("freshness"), False) + if node_dict["resource_type"] == "source" + else False, ) nodes[node.unique_id] = node logger.debug("Parsed dbt resource `%s` of type `%s`", node.unique_id, node.resource_type) @@ -117,19 +178,12 @@ class DbtGraph: Supports different ways of loading the `dbt` project into this representation. Different loading methods can result in different `nodes` and `filtered_nodes`. - - Example of how to use: - - dbt_graph = DbtGraph( - project=ProjectConfig(dbt_project_path=DBT_PROJECT_PATH), - render_config=RenderConfig(exclude=["*orders*"], select=[]), - dbt_cmd="/usr/local/bin/dbt" - ) - dbt_graph.load(method=LoadMode.DBT_LS, execution_mode=ExecutionMode.LOCAL) """ nodes: dict[str, DbtNode] = dict() filtered_nodes: dict[str, DbtNode] = dict() + load_method: LoadMode = LoadMode.AUTOMATIC + current_version: str = "" def __init__( self, @@ -137,15 +191,135 @@ def __init__( render_config: RenderConfig = RenderConfig(), execution_config: ExecutionConfig = ExecutionConfig(), profile_config: ProfileConfig | None = None, - dbt_cmd: str = get_system_dbt(), - operator_args: dict[str, Any] | None = None, + cache_dir: Path | None = None, + cache_identifier: str = "", + dbt_vars: dict[str, str] | None = None, + airflow_metadata: dict[str, str] | None = None, ): self.project = project self.render_config = render_config self.profile_config = profile_config self.execution_config = execution_config - self.operator_args = operator_args or {} - self.dbt_cmd = dbt_cmd + self.cache_dir = cache_dir + self.airflow_metadata = airflow_metadata or {} + if cache_identifier: + self.dbt_ls_cache_key = cache.create_cache_key(cache_identifier) + else: + self.dbt_ls_cache_key = "" + self.dbt_vars = dbt_vars or {} + + @cached_property + def env_vars(self) -> dict[str, str]: + """ + User-defined environment variables, relevant to running dbt ls. + """ + return self.render_config.env_vars or self.project.env_vars or {} + + @cached_property + def project_path(self) -> Path: + """ + Return the user-defined path to their dbt project. Tries to retrieve the configuration from render_config and + (legacy support) ExecutionConfig, where it was originally defined. + """ + # we're considering the execution_config only due to backwards compatibility + path = self.render_config.project_path or self.project.dbt_project_path or self.execution_config.project_path + if not path: + raise CosmosLoadDbtException( + "Unable to load project via dbt ls without RenderConfig.dbt_project_path, ProjectConfig.dbt_project_path or ExecutionConfig.dbt_project_path" + ) + return path.absolute() + + @cached_property + def dbt_ls_args(self) -> list[str]: + """ + Flags set while running dbt ls. This information is also used to define the dbt ls cache key. + """ + ls_args = [] + if self.render_config.exclude: + ls_args.extend(["--exclude", *self.render_config.exclude]) + + if self.render_config.select: + ls_args.extend(["--select", *self.render_config.select]) + + if self.project.dbt_vars: + ls_args.extend(["--vars", json.dumps(self.project.dbt_vars, sort_keys=True)]) + + if self.render_config.selector: + ls_args.extend(["--selector", self.render_config.selector]) + + if not self.project.partial_parse: + ls_args.append("--no-partial-parse") + + return ls_args + + @cached_property + def dbt_ls_cache_key_args(self) -> list[str]: + """ + Values that are used to represent the dbt ls cache key. If any parts are changed, the dbt ls command will be + executed and the new value will be stored. + """ + # if dbt deps, we can consider the md5 of the packages or deps file + cache_args = list(self.dbt_ls_args) + env_vars = self.env_vars + if env_vars: + envvars_str = json.dumps(env_vars, sort_keys=True) + cache_args.append(envvars_str) + if self.render_config.airflow_vars_to_purge_dbt_ls_cache: + for var_name in self.render_config.airflow_vars_to_purge_dbt_ls_cache: + airflow_vars = [var_name, Variable.get(var_name, "")] + cache_args.extend(airflow_vars) + + logger.debug(f"Value of `dbt_ls_cache_key_args` for <{self.dbt_ls_cache_key}>: {cache_args}") + return cache_args + + def save_dbt_ls_cache(self, dbt_ls_output: str) -> None: + """ + Store compressed dbt ls output into an Airflow Variable. + + Stores: + { + "version": "cache-version", + "dbt_ls_compressed": "compressed dbt ls output", + "last_modified": "Isoformat timestamp" + } + """ + # This compression reduces the dbt ls output to 10% of the original size + compressed_data = zlib.compress(dbt_ls_output.encode("utf-8")) + encoded_data = base64.b64encode(compressed_data) + dbt_ls_compressed = encoded_data.decode("utf-8") + cache_dict = { + "version": cache._calculate_dbt_ls_cache_current_version( + self.dbt_ls_cache_key, self.project_path, self.dbt_ls_cache_key_args + ), + "dbt_ls_compressed": dbt_ls_compressed, + "last_modified": datetime.datetime.now(datetime.timezone.utc).isoformat(), + **self.airflow_metadata, + } + Variable.set(self.dbt_ls_cache_key, cache_dict, serialize_json=True) + + def get_dbt_ls_cache(self) -> dict[str, str]: + """ + Retrieve previously saved dbt ls cache from an Airflow Variable, decompressing the dbt ls output. + + Outputs: + { + "version": "cache-version", + "dbt_ls": "uncompressed dbt ls output", + "last_modified": "Isoformat timestamp" + } + """ + cache_dict: dict[str, str] = {} + try: + cache_dict = Variable.get(self.dbt_ls_cache_key, deserialize_json=True) + except (json.decoder.JSONDecodeError, KeyError): + return cache_dict + else: + dbt_ls_compressed = cache_dict.pop("dbt_ls_compressed", None) + if dbt_ls_compressed: + encoded_data = base64.b64decode(dbt_ls_compressed.encode()) + cache_dict["dbt_ls"] = zlib.decompress(encoded_data).decode() + + return cache_dict def load( self, @@ -162,10 +336,11 @@ def load( Fundamentally, there are two different execution paths There is automatic, and manual. """ - load_method = { LoadMode.CUSTOM: self.load_via_custom_parser, LoadMode.DBT_LS: self.load_via_dbt_ls, + LoadMode.DBT_LS_FILE: self.load_via_dbt_ls_file, + LoadMode.DBT_LS_CACHE: self.load_via_dbt_ls_cache, LoadMode.DBT_MANIFEST: self.load_from_dbt_manifest, } @@ -183,17 +358,27 @@ def load( else: load_method[method]() - def run_dbt_ls(self, project_path: Path, tmp_dir: Path, env_vars: dict[str, str]) -> dict[str, DbtNode]: - """Runs dbt ls command and returns the parsed nodes.""" - ls_command = [self.dbt_cmd, "ls", "--output", "json"] - - if self.render_config.exclude: - ls_command.extend(["--exclude", *self.render_config.exclude]) + self.update_node_dependency() - if self.render_config.select: - ls_command.extend(["--select", *self.render_config.select]) + logger.info("Total nodes: %i", len(self.nodes)) + logger.info("Total filtered nodes: %i", len(self.nodes)) + def run_dbt_ls( + self, dbt_cmd: str, project_path: Path, tmp_dir: Path, env_vars: dict[str, str] + ) -> dict[str, DbtNode]: + """Runs dbt ls command and returns the parsed nodes.""" + ls_command = [ + dbt_cmd, + "ls", + "--output", + "json", + "--output-keys", + "name alias unique_id resource_type depends_on original_file_path tags config freshness", + ] + + ls_args = self.dbt_ls_args ls_command.extend(self.local_flags) + ls_command.extend(ls_args) stdout = run_command(ls_command, tmp_dir, env_vars) @@ -205,41 +390,98 @@ def run_dbt_ls(self, project_path: Path, tmp_dir: Path, env_vars: dict[str, str] for line in logfile: logger.debug(line.strip()) + if self.should_use_dbt_ls_cache(): + self.save_dbt_ls_cache(stdout) + nodes = parse_dbt_ls_output(project_path, stdout) return nodes def load_via_dbt_ls(self) -> None: + """Retrieve the dbt ls cache if enabled and available or run dbt ls""" + if not self.load_via_dbt_ls_cache(): + self.load_via_dbt_ls_without_cache() + + @functools.lru_cache + def should_use_dbt_ls_cache(self) -> bool: + """Identify if Cosmos should use/store dbt ls cache or not.""" + return settings.enable_cache and settings.enable_cache_dbt_ls and bool(self.dbt_ls_cache_key) + + def load_via_dbt_ls_cache(self) -> bool: + """(Try to) load dbt ls cache from an Airflow Variable""" + + logger.info(f"Trying to parse the dbt project using dbt ls cache {self.dbt_ls_cache_key}...") + if self.should_use_dbt_ls_cache(): + project_path = self.project_path + + cache_dict = self.get_dbt_ls_cache() + if not cache_dict: + logger.info(f"Cosmos performance: Cache miss for {self.dbt_ls_cache_key}") + return False + + cache_version = cache_dict.get("version") + dbt_ls_cache = cache_dict.get("dbt_ls") + + current_version = cache._calculate_dbt_ls_cache_current_version( + self.dbt_ls_cache_key, project_path, self.dbt_ls_cache_key_args + ) + + if dbt_ls_cache and not cache.was_project_modified(cache_version, current_version): + logger.info( + f"Cosmos performance [{platform.node()}|{os.getpid()}]: The cache size for {self.dbt_ls_cache_key} is {len(dbt_ls_cache)}" + ) + self.load_method = LoadMode.DBT_LS_CACHE + + nodes = parse_dbt_ls_output(project_path=project_path, ls_stdout=dbt_ls_cache) + self.nodes = nodes + self.filtered_nodes = nodes + logger.info(f"Cosmos performance: Cache hit for {self.dbt_ls_cache_key} - {current_version}") + return True + logger.info(f"Cosmos performance: Cache miss for {self.dbt_ls_cache_key} - skipped") + return False + + def should_use_partial_parse_cache(self) -> bool: + """Identify if Cosmos should use/store dbt partial parse cache or not.""" + return settings.enable_cache_partial_parse and settings.enable_cache and bool(self.cache_dir) + + def load_via_dbt_ls_without_cache(self) -> None: """ This is the most accurate way of loading `dbt` projects and filtering them out, since it uses the `dbt` command line for both parsing and filtering the nodes. - Noted that if dbt project contains versioned models, need to use dbt>=1.6.0 instead. Because, as dbt<1.6.0, - dbt cli doesn't support select a specific versioned models as stg_customers_v1, customers_v1, ... - Updates in-place: * self.nodes * self.filtered_nodes """ - logger.info(f"Trying to parse the dbt project in `{self.render_config.project_path}` using dbt ls...") - if not self.render_config.project_path or not self.execution_config.project_path: - raise CosmosLoadDbtException( - "Unable to load project via dbt ls without RenderConfig.dbt_project_path and ExecutionConfig.dbt_project_path" - ) + self.load_method = LoadMode.DBT_LS + self.render_config.validate_dbt_command(fallback_cmd=self.execution_config.dbt_executable_path) + dbt_cmd = self.render_config.dbt_executable_path + dbt_cmd = dbt_cmd.as_posix() if isinstance(dbt_cmd, Path) else dbt_cmd + logger.info(f"Trying to parse the dbt project in `{self.render_config.project_path}` using dbt ls...") + project_path = self.project_path if not self.profile_config: raise CosmosLoadDbtException("Unable to load project via dbt ls without a profile config.") - if not shutil.which(self.dbt_cmd): - raise CosmosLoadDbtException(f"Unable to find the dbt executable: {self.dbt_cmd}") - with tempfile.TemporaryDirectory() as tmpdir: - logger.info( - f"Content of the dbt project dir {self.render_config.project_path}: `{os.listdir(self.render_config.project_path)}`" - ) + logger.debug(f"Content of the dbt project dir {project_path}: `{os.listdir(project_path)}`") tmpdir_path = Path(tmpdir) - create_symlinks(self.render_config.project_path, tmpdir_path) - with self.profile_config.ensure_profile(use_mock_values=True) as profile_values: + create_symlinks(project_path, tmpdir_path, self.render_config.dbt_deps) + + latest_partial_parse = None + if self.project.partial_parse: + if self.should_use_partial_parse_cache() and self.cache_dir: + latest_partial_parse = cache._get_latest_partial_parse(project_path, self.cache_dir) + else: + latest_partial_parse = get_partial_parse_path(project_path) + + if latest_partial_parse is not None and latest_partial_parse.exists(): + logger.info("Partial parse is enabled and the latest partial parse file is %s", latest_partial_parse) + cache._copy_partial_parse_to_project(latest_partial_parse, tmpdir_path) + + with self.profile_config.ensure_profile( + use_mock_values=self.render_config.enable_mock_profile + ) as profile_values, environ(self.env_vars): (profile_path, env_vars) = profile_values env = os.environ.copy() env.update(env_vars) @@ -259,21 +501,46 @@ def load_via_dbt_ls(self) -> None: env[DBT_LOG_PATH_ENVVAR] = str(self.log_dir) env[DBT_TARGET_PATH_ENVVAR] = str(self.target_dir) - if self.render_config.dbt_deps: - deps_command = [self.dbt_cmd, "deps"] + if self.render_config.dbt_deps and has_non_empty_dependencies_file(self.project_path): + deps_command = [dbt_cmd, "deps"] deps_command.extend(self.local_flags) stdout = run_command(deps_command, tmpdir_path, env) logger.debug("dbt deps output: %s", stdout) - nodes = self.run_dbt_ls(self.execution_config.project_path, tmpdir_path, env) + nodes = self.run_dbt_ls(dbt_cmd, self.project_path, tmpdir_path, env) self.nodes = nodes self.filtered_nodes = nodes - self.update_node_dependency() + if self.should_use_partial_parse_cache(): + partial_parse_file = get_partial_parse_path(tmpdir_path) + if partial_parse_file.exists() and self.cache_dir: + cache._update_partial_parse_cache(partial_parse_file, self.cache_dir) - logger.info("Total nodes: %i", len(self.nodes)) - logger.info("Total filtered nodes: %i", len(self.nodes)) + def load_via_dbt_ls_file(self) -> None: + """ + This is between dbt ls and full manifest. It allows to use the output (needs to be json output) of the dbt ls as a + file stored in the image you run Cosmos on. The advantage is that you can use the parser from LoadMode.DBT_LS without + actually running dbt ls every time. BUT you will need one dbt ls file for each separate group. + + This technically should increase performance and also removes the necessity to have your whole dbt project copied + to the airflow image. + """ + self.load_method = LoadMode.DBT_LS_FILE + logger.info("Trying to parse the dbt project `%s` using a dbt ls output file...", self.project.project_name) + + if not self.render_config.is_dbt_ls_file_available(): + raise CosmosLoadDbtException(f"Unable to load dbt ls file using {self.render_config.dbt_ls_path}") + + project_path = self.render_config.project_path + if not project_path: + raise CosmosLoadDbtException("Unable to load dbt ls file without RenderConfig.project_path") + with open(self.render_config.dbt_ls_path) as fp: # type: ignore[arg-type] + dbt_ls_output = fp.read() + nodes = parse_dbt_ls_output(project_path=project_path, ls_stdout=dbt_ls_output) + + self.nodes = nodes + self.filtered_nodes = nodes def load_via_custom_parser(self) -> None: """ @@ -287,8 +554,14 @@ def load_via_custom_parser(self) -> None: * self.nodes * self.filtered_nodes """ + self.load_method = LoadMode.CUSTOM logger.info("Trying to parse the dbt project `%s` using a custom Cosmos method...", self.project.project_name) + if self.render_config.selector: + raise CosmosLoadDbtException( + "RenderConfig.selector is not yet supported when loading dbt projects using the LoadMode.CUSTOM parser." + ) + if not self.render_config.project_path or not self.execution_config.project_path: raise CosmosLoadDbtException( "Unable to load dbt project without RenderConfig.dbt_project_path and ExecutionConfig.dbt_project_path" @@ -299,7 +572,7 @@ def load_via_custom_parser(self) -> None: dbt_root_path=self.render_config.project_path.parent.as_posix(), dbt_models_dir=self.project.models_path.stem if self.project.models_path else "models", dbt_seeds_dir=self.project.seeds_path.stem if self.project.seeds_path else "seeds", - operator_args=self.operator_args, + dbt_vars=self.dbt_vars, ) nodes = {} models = itertools.chain( @@ -308,8 +581,7 @@ def load_via_custom_parser(self) -> None: for model_name, model in models: config = {item.split(":")[0]: item.split(":")[-1] for item in model.config.config_selectors} node = DbtNode( - name=model_name, - unique_id=model_name, + unique_id=f"{model.type.value}.{self.project.project_name}.{model_name}", resource_type=DbtResourceType(model.type.value), depends_on=list(model.config.upstream_models), file_path=Path( @@ -330,11 +602,6 @@ def load_via_custom_parser(self) -> None: exclude=self.render_config.exclude, ) - self.update_node_dependency() - - logger.info("Total nodes: %i", len(self.nodes)) - logger.info("Total filtered nodes: %i", len(self.nodes)) - def load_from_dbt_manifest(self) -> None: """ This approach accurately loads `dbt` projects using the `manifest.yml` file. @@ -342,15 +609,18 @@ def load_from_dbt_manifest(self) -> None: However, since the Manifest does not represent filters, it relies on the Custom Cosmos implementation to filter out the nodes relevant to the user (based on self.exclude and self.select). - Noted that if dbt project contains versioned models, need to use dbt>=1.6.0 instead. Because, as dbt<1.6.0, - dbt cli doesn't support select a specific versioned models as stg_customers_v1, customers_v1, ... - Updates in-place: * self.nodes * self.filtered_nodes """ + self.load_method = LoadMode.DBT_MANIFEST logger.info("Trying to parse the dbt project `%s` using a dbt manifest...", self.project.project_name) + if self.render_config.selector: + raise CosmosLoadDbtException( + "RenderConfig.selector is not yet supported when loading dbt projects using the LoadMode.DBT_MANIFEST parser." + ) + if not self.project.is_manifest_available(): raise CosmosLoadDbtException(f"Unable to load manifest using {self.project.manifest_path}") @@ -364,13 +634,15 @@ def load_from_dbt_manifest(self) -> None: resources = {**manifest.get("nodes", {}), **manifest.get("sources", {}), **manifest.get("exposures", {})} for unique_id, node_dict in resources.items(): node = DbtNode( - name=node_dict.get("alias", node_dict["name"]), unique_id=unique_id, resource_type=DbtResourceType(node_dict["resource_type"]), depends_on=node_dict.get("depends_on", {}).get("nodes", []), file_path=self.execution_config.project_path / Path(node_dict["original_file_path"]), tags=node_dict["tags"], config=node_dict["config"], + has_freshness=node_dict["freshness"] is not None + if node_dict["resource_type"] == "source" and "freshness" in node_dict + else False, ) nodes[node.unique_id] = node @@ -383,20 +655,16 @@ def load_from_dbt_manifest(self) -> None: exclude=self.render_config.exclude, ) - self.update_node_dependency() - - logger.info("Total nodes: %i", len(self.nodes)) - logger.info("Total filtered nodes: %i", len(self.nodes)) - def update_node_dependency(self) -> None: """ - This will update the property `has_text` if node has `dbt` test + This will update the property `has_test` if node has `dbt` test Updates in-place: * self.filtered_nodes """ - for _, node in self.filtered_nodes.items(): + for _, node in list(self.nodes.items()): if node.resource_type == DbtResourceType.TEST: for node_id in node.depends_on: if node_id in self.filtered_nodes: self.filtered_nodes[node_id].has_test = True + self.filtered_nodes[node.unique_id] = node diff --git a/cosmos/dbt/parser/output.py b/cosmos/dbt/parser/output.py index 791c4b6057..3ff377941e 100644 --- a/cosmos/dbt/parser/output.py +++ b/cosmos/dbt/parser/output.py @@ -1,33 +1,52 @@ +from __future__ import annotations + import logging import re -from typing import List, Tuple +from typing import TYPE_CHECKING, List, Tuple + +if TYPE_CHECKING: + from dbt.cli.main import dbtRunnerResult from cosmos.hooks.subprocess import FullOutputSubprocessResult +DBT_NO_TESTS_MSG = "Nothing to do" +DBT_WARN_MSG = "WARN" -def parse_output(result: FullOutputSubprocessResult, keyword: str) -> int: + +def parse_number_of_warnings_subprocess(result: FullOutputSubprocessResult) -> int: """ - Parses the dbt test output message and returns the number of errors or warnings. + Parses the dbt test output message and returns the number of warnings. :param result: String containing the output to be parsed. - :param keyword: String representing the keyword to search for in the output (WARN, ERROR). :return: An integer value associated with the keyword, or 0 if parsing fails. Usage: ----- output_str = "Done. PASS=15 WARN=1 ERROR=0 SKIP=0 TOTAL=16" - keyword = "WARN" - num_warns = parse_output(output_str, keyword) + num_warns = parse_output(output_str) print(num_warns) # Output: 1 """ output = result.output - try: - num = int(output.split(f"{keyword}=")[1].split()[0]) - except ValueError: - logging.error( - f"Could not parse number of {keyword}s. Check your dbt/airflow version or if --quiet is not being used" - ) + num = 0 + if DBT_NO_TESTS_MSG not in result.output and DBT_WARN_MSG in result.output: + try: + num = int(output.split(f"{DBT_WARN_MSG}=")[1].split()[0]) + except ValueError: + logging.error( + f"Could not parse number of {DBT_WARN_MSG}s. Check your dbt/airflow version or if --quiet is not being used" + ) + return num + + +def parse_number_of_warnings_dbt_runner(result: dbtRunnerResult) -> int: + """Parses a dbt runner result and returns the number of warnings found. This only works for dbtRunnerResult + from invoking dbt build, compile, run, seed, snapshot, test, or run-operation. + """ + num = 0 + for run_result in result.result.results: # type: ignore + if run_result.status == "warn": + num += 1 return num @@ -67,3 +86,28 @@ def clean_line(line: str) -> str: test_results.append(test_result) return test_names, test_results + + +def extract_dbt_runner_issues( + result: dbtRunnerResult, status_levels: list[str] = ["warn"] +) -> Tuple[List[str], List[str]]: + """ + Extracts messages from the dbt runner result and returns them as a formatted string. + + This function iterates over dbtRunnerResult messages in dbt run. It extracts results that match the + status levels provided and appends them to a list of issues. + + :param result: dbtRunnerResult object containing the output to be parsed. + :param status_levels: List of strings, where each string is a result status level. Default is ["warn"]. + :return: two lists of strings, the first one containing the node names and the second one + containing the node result message. + """ + node_names = [] + node_results = [] + + for node_result in result.result.results: # type: ignore + if node_result.status in status_levels: + node_names.append(str(node_result.node.name)) + node_results.append(str(node_result.message)) + + return node_names, node_results diff --git a/cosmos/dbt/parser/project.py b/cosmos/dbt/parser/project.py index 278b1a0f73..8876d68786 100644 --- a/cosmos/dbt/parser/project.py +++ b/cosmos/dbt/parser/project.py @@ -1,10 +1,11 @@ """ Used to parse and extract information from dbt projects. """ + from __future__ import annotations -import os import ast +import os from dataclasses import dataclass, field from enum import Enum from pathlib import Path @@ -130,7 +131,7 @@ class DbtModel: name: str type: DbtModelType path: Path - operator_args: Dict[str, Any] = field(default_factory=dict) + dbt_vars: Dict[str, str] = field(default_factory=dict) config: DbtModelConfig = field(default_factory=DbtModelConfig) def __post_init__(self) -> None: @@ -141,7 +142,6 @@ def __post_init__(self) -> None: return config = DbtModelConfig() - self.var_args: Dict[str, Any] = self.operator_args.get("vars", {}) code = self.path.read_text() if self.type == DbtModelType.DBT_SNAPSHOT: @@ -203,7 +203,7 @@ def _parse_jinja_ref_node(self, base_node: jinja2.nodes.Call) -> str | None: and isinstance(node.args[0], jinja2.nodes.Const) and node.node.name == "var" ): - value += self.var_args[node.args[0].value] + value += self.dbt_vars[node.args[0].value] # type: ignore elif isinstance(first_arg, jinja2.nodes.Const): # and add it to the config value = first_arg.value @@ -272,20 +272,16 @@ class LegacyDbtProject: snapshots_dir: Path = field(init=False) seeds_dir: Path = field(init=False) - operator_args: Dict[str, Any] = field(default_factory=dict) + dbt_vars: Dict[str, str] = field(default_factory=dict) def __post_init__(self) -> None: """ Initializes the parser. """ - if self.dbt_root_path is None: - self.dbt_root_path = "/usr/local/airflow/dags/dbt" - if self.dbt_models_dir is None: - self.dbt_models_dir = "models" - if self.dbt_snapshots_dir is None: - self.dbt_snapshots_dir = "snapshots" - if self.dbt_seeds_dir is None: - self.dbt_seeds_dir = "seeds" + self.dbt_root_path = self.dbt_root_path or "/usr/local/airflow/dags/dbt" + self.dbt_models_dir = self.dbt_models_dir or "models" + self.dbt_snapshots_dir = self.dbt_snapshots_dir or "snapshots" + self.dbt_seeds_dir = self.dbt_seeds_dir or "seeds" # set the project and model dirs self.project_dir = Path(os.path.join(self.dbt_root_path, self.project_name)) @@ -325,7 +321,7 @@ def _handle_csv_file(self, path: Path) -> None: name=model_name, type=DbtModelType.DBT_SEED, path=path, - operator_args=self.operator_args, + dbt_vars=self.dbt_vars, ) # add the model to the project self.seeds[model_name] = model @@ -343,7 +339,7 @@ def _handle_sql_file(self, path: Path) -> None: name=model_name, type=DbtModelType.DBT_MODEL, path=path, - operator_args=self.operator_args, + dbt_vars=self.dbt_vars, ) # add the model to the project self.models[model.name] = model @@ -353,7 +349,7 @@ def _handle_sql_file(self, path: Path) -> None: name=model_name, type=DbtModelType.DBT_SNAPSHOT, path=path, - operator_args=self.operator_args, + dbt_vars=self.dbt_vars, ) # add the snapshot to the project self.snapshots[model.name] = model @@ -414,7 +410,7 @@ def _extract_model_tests( name=f"{test}_{column['name']}_{model_name}", type=DbtModelType.DBT_TEST, path=path, - operator_args=self.operator_args, + dbt_vars=self.dbt_vars, config=DbtModelConfig(upstream_models=set({model_name})), ) tests[test_model.name] = test_model diff --git a/cosmos/dbt/project.py b/cosmos/dbt/project.py new file mode 100644 index 0000000000..987d10f3a6 --- /dev/null +++ b/cosmos/dbt/project.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import os +from contextlib import contextmanager +from pathlib import Path +from typing import Generator + +from cosmos.constants import ( + DBT_DEPENDENCIES_FILE_NAMES, + DBT_LOG_DIR_NAME, + DBT_PARTIAL_PARSE_FILE_NAME, + DBT_TARGET_DIR_NAME, +) +from cosmos.log import get_logger + +logger = get_logger(__name__) + + +def has_non_empty_dependencies_file(project_path: Path) -> bool: + """ + Check if the dbt project has dependencies.yml or packages.yml. + + :param project_path: Path to the project + :returns: True or False + """ + project_dir = Path(project_path) + has_deps = False + for filename in DBT_DEPENDENCIES_FILE_NAMES: + filepath = project_dir / filename + if filepath.exists() and filepath.stat().st_size > 0: + has_deps = True + break + + if not has_deps: + logger.info(f"Project {project_path} does not have {DBT_DEPENDENCIES_FILE_NAMES}") + return has_deps + + +def create_symlinks(project_path: Path, tmp_dir: Path, ignore_dbt_packages: bool) -> None: + """Helper function to create symlinks to the dbt project files.""" + ignore_paths = [DBT_LOG_DIR_NAME, DBT_TARGET_DIR_NAME, "profiles.yml"] + if ignore_dbt_packages: + # this is linked to dbt deps so if dbt deps is true then ignore existing dbt_packages folder + ignore_paths.append("dbt_packages") + for child_name in os.listdir(project_path): + if child_name not in ignore_paths: + os.symlink(project_path / child_name, tmp_dir / child_name) + + +def get_partial_parse_path(project_dir_path: Path) -> Path: + """ + Return the partial parse (partial_parse.msgpack) path for a given dbt project directory. + """ + return project_dir_path / DBT_TARGET_DIR_NAME / DBT_PARTIAL_PARSE_FILE_NAME + + +@contextmanager +def environ(env_vars: dict[str, str]) -> Generator[None, None, None]: + """Temporarily set environment variables inside the context manager and restore + when exiting. + """ + original_env = {key: os.getenv(key) for key in env_vars} + os.environ.update(env_vars) + try: + yield + finally: + for key, value in original_env.items(): + if value is None: + del os.environ[key] + else: + os.environ[key] = value + + +@contextmanager +def change_working_directory(path: str) -> Generator[None, None, None]: + """Temporarily changes the working directory to the given path, and then restores + back to the previous value on exit. + """ + previous_cwd = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(previous_cwd) diff --git a/cosmos/dbt/selector.py b/cosmos/dbt/selector.py index c7316dc75e..257d60721a 100644 --- a/cosmos/dbt/selector.py +++ b/cosmos/dbt/selector.py @@ -1,7 +1,10 @@ from __future__ import annotations -from pathlib import Path -import copy +import copy +import re +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path from typing import TYPE_CHECKING, Any from cosmos.constants import DbtResourceType @@ -16,11 +19,154 @@ PATH_SELECTOR = "path:" TAG_SELECTOR = "tag:" CONFIG_SELECTOR = "config." - +PLUS_SELECTOR = "+" +GRAPH_SELECTOR_REGEX = r"^([0-9]*\+)?([^\+]+)(\+[0-9]*)?$|" logger = get_logger(__name__) +@dataclass +class GraphSelector: + """ + Implements dbt graph operator selectors: + model_a + +model_b + model_c+ + +model_d+ + 2+model_e + model_f+3 + + https://docs.getdbt.com/reference/node-selection/graph-operators + """ + + node_name: str + precursors: str | None + descendants: str | None + + @property + def precursors_depth(self) -> int: + """ + Calculates the depth/degrees/generations of precursors (parents). + Return: + -1: if it should return all the generations of precursors + 0: if it shouldn't return any precursors + >0: upperbound number of parent generations + """ + if not self.precursors: + return 0 + if self.precursors == "+": + return -1 + else: + return int(self.precursors[:-1]) + + @property + def descendants_depth(self) -> int: + """ + Calculates the depth/degrees/generations of descendants (children). + Return: + -1: if it should return all the generations of children + 0: if it shouldn't return any children + >0: upperbound of children generations + """ + if not self.descendants: + return 0 + if self.descendants == "+": + return -1 + else: + return int(self.descendants[1:]) + + @staticmethod + def parse(text: str) -> GraphSelector | None: + """ + Parse a string and identify if there are graph selectors, including the desired node name, descendants and + precursors. Return a GraphSelector instance if the pattern matches. + """ + regex_match = re.search(GRAPH_SELECTOR_REGEX, text) + if regex_match: + precursors, node_name, descendants = regex_match.groups() + return GraphSelector(node_name, precursors, descendants) + return None + + def select_node_precursors(self, nodes: dict[str, DbtNode], root_id: str, selected_nodes: set[str]) -> None: + """ + Parse original nodes and add the precursor nodes related to this config to the selected_nodes set. + + :param nodes: Original dbt nodes list + :param root_id: Unique identifier of self.node_name + :param selected_nodes: Set where precursor nodes will be added to. + """ + if self.precursors: + depth = self.precursors_depth + previous_generation = {root_id} + processed_nodes = set() + while depth and previous_generation: + new_generation: set[str] = set() + for node_id in previous_generation: + if node_id not in processed_nodes: + new_generation.update(set(nodes[node_id].depends_on)) + processed_nodes.add(node_id) + selected_nodes.update(new_generation) + previous_generation = new_generation + depth -= 1 + + def select_node_descendants(self, nodes: dict[str, DbtNode], root_id: str, selected_nodes: set[str]) -> None: + """ + Parse original nodes and add the descendant nodes related to this config to the selected_nodes set. + + :param nodes: Original dbt nodes list + :param root_id: Unique identifier of self.node_name + :param selected_nodes: Set where descendant nodes will be added to. + """ + if self.descendants: + children_by_node = defaultdict(set) + # Index nodes by parent id + # We could optimize by doing this only once for the dbt project and giving it + # as a parameter to the GraphSelector + for node_id, node in nodes.items(): + for parent_id in node.depends_on: + children_by_node[parent_id].add(node_id) + + depth = self.descendants_depth + previous_generation = {root_id} + processed_nodes = set() + while depth and previous_generation: + new_generation: set[str] = set() + for node_id in previous_generation: + if node_id not in processed_nodes: + new_generation.update(children_by_node[node_id]) + processed_nodes.add(node_id) + selected_nodes.update(new_generation) + previous_generation = new_generation + depth -= 1 + + def filter_nodes(self, nodes: dict[str, DbtNode]) -> set[str]: + """ + Given a dictionary with the original dbt project nodes, applies the current graph selector to + identify the subset of nodes that matches the selection criteria. + + :param nodes: dbt project nodes + :return: set of node ids that matches current graph selector + """ + selected_nodes: set[str] = set() + + # Index nodes by name, we can improve performance by doing this once + # for multiple GraphSelectors + node_by_name = {} + for node_id, node in nodes.items(): + node_by_name[node.name] = node_id + + if self.node_name in node_by_name: + root_id = node_by_name[self.node_name] + else: + logger.warn(f"Selector {self.node_name} not found.") + return selected_nodes + + selected_nodes.add(root_id) + self.select_node_precursors(nodes, root_id, selected_nodes) + self.select_node_descendants(nodes, root_id, selected_nodes) + return selected_nodes + + class SelectorConfig: """ Represents a select/exclude statement. @@ -43,11 +189,12 @@ def __init__(self, project_dir: Path | None, statement: str): self.tags: list[str] = [] self.config: dict[str, str] = {} self.other: list[str] = [] + self.graph_selectors: list[GraphSelector] = [] self.load_from_statement(statement) @property def is_empty(self) -> bool: - return not (self.paths or self.tags or self.config or self.other) + return not (self.paths or self.tags or self.config or self.graph_selectors or self.other) def load_from_statement(self, statement: str) -> None: """ @@ -61,27 +208,45 @@ def load_from_statement(self, statement: str) -> None: https://docs.getdbt.com/reference/node-selection/yaml-selectors """ items = statement.split(",") + for item in items: if item.startswith(PATH_SELECTOR): - index = len(PATH_SELECTOR) - if self.project_dir: - self.paths.append(self.project_dir / Path(item[index:])) - else: - self.paths.append(Path(item[index:])) + self._parse_path_selector(item) elif item.startswith(TAG_SELECTOR): - index = len(TAG_SELECTOR) - self.tags.append(item[index:]) + self._parse_tag_selector(item) elif item.startswith(CONFIG_SELECTOR): - index = len(CONFIG_SELECTOR) - key, value = item[index:].split(":") - if key in SUPPORTED_CONFIG: - self.config[key] = value + self._parse_config_selector(item) + else: + self._parse_unknown_selector(item) + + def _parse_unknown_selector(self, item: str) -> None: + if item: + graph_selector = GraphSelector.parse(item) + if graph_selector is not None: + self.graph_selectors.append(graph_selector) else: self.other.append(item) logger.warning("Unsupported select statement: %s", item) + def _parse_config_selector(self, item: str) -> None: + index = len(CONFIG_SELECTOR) + key, value = item[index:].split(":") + if key in SUPPORTED_CONFIG: + self.config[key] = value + + def _parse_tag_selector(self, item: str) -> None: + index = len(TAG_SELECTOR) + self.tags.append(item[index:]) + + def _parse_path_selector(self, item: str) -> None: + index = len(PATH_SELECTOR) + if self.project_dir: + self.paths.append(self.project_dir / Path(item[index:])) + else: + self.paths.append(Path(item[index:])) + def __repr__(self) -> str: - return f"SelectorConfig(paths={self.paths}, tags={self.tags}, config={self.config}, other={self.other})" + return f"SelectorConfig(paths={self.paths}, tags={self.tags}, config={self.config}, other={self.other}, graph_selectors={self.graph_selectors})" class NodeSelector: @@ -95,7 +260,9 @@ class NodeSelector: def __init__(self, nodes: dict[str, DbtNode], config: SelectorConfig) -> None: self.nodes = nodes self.config = config + self.selected_nodes: set[str] = set() + @property def select_nodes_ids_by_intersection(self) -> set[str]: """ Return a list of node ids which matches the configuration defined in config. @@ -107,26 +274,39 @@ def select_nodes_ids_by_intersection(self) -> set[str]: if self.config.is_empty: return set(self.nodes.keys()) - self.selected_nodes: set[str] = set() + selected_nodes: set[str] = set() self.visited_nodes: set[str] = set() for node_id, node in self.nodes.items(): if self._should_include_node(node_id, node): - self.selected_nodes.add(node_id) + selected_nodes.add(node_id) + + if self.config.graph_selectors: + nodes_by_graph_selector = self.select_by_graph_operator() + selected_nodes = selected_nodes.intersection(nodes_by_graph_selector) - return self.selected_nodes + self.selected_nodes = selected_nodes + return selected_nodes def _should_include_node(self, node_id: str, node: DbtNode) -> bool: - "Checks if a single node should be included. Only runs once per node with caching." + """Checks if a single node should be included. Only runs once per node with caching.""" + logger.debug("Inspecting if the node <%s> should be included.", node_id) if node_id in self.visited_nodes: return node_id in self.selected_nodes self.visited_nodes.add(node_id) - if node.resource_type == DbtResourceType.TEST: + if node.resource_type == DbtResourceType.TEST and node.depends_on and len(node.depends_on) > 0: node.tags = getattr(self.nodes.get(node.depends_on[0]), "tags", []) + logger.debug( + "The test node <%s> inherited these tags from the parent node <%s>: %s", + node_id, + node.depends_on[0], + node.tags, + ) if not self._is_tags_subset(node): + logger.debug("Excluding node <%s>", node_id) return False node_config = {key: value for key, value in node.config.items() if key in SUPPORTED_CONFIG} @@ -175,6 +355,22 @@ def _is_path_matching(self, node: DbtNode) -> bool: return self._should_include_node(node.depends_on[0], model_node) return False + def select_by_graph_operator(self) -> set[str]: + """ + Return a list of node ids which match the configuration defined in the config. + + Return all nodes that are parents (or parents from parents) of the root defined in the configuration. + + References: + https://docs.getdbt.com/reference/node-selection/syntax + https://docs.getdbt.com/reference/node-selection/yaml-selectors + """ + selected_nodes_by_selector: list[set[str]] = [] + + for graph_selector in self.config.graph_selectors: + selected_nodes_by_selector.append(graph_selector.filter_nodes(self.nodes)) + return set.intersection(*selected_nodes_by_selector) + def retrieve_by_label(statement_list: list[str], label: str) -> set[str]: """ @@ -189,7 +385,7 @@ def retrieve_by_label(statement_list: list[str], label: str) -> set[str]: for statement in statement_list: config = SelectorConfig(Path(), statement) item_values = getattr(config, label) - label_values = label_values.union(item_values) + label_values.update(item_values) return label_values @@ -213,35 +409,53 @@ def select_nodes( if not select and not exclude: return nodes - # validates select and exclude filters - filters = [["select", select], ["exclude", exclude]] - for filter_type, filter in filters: - for filter_parameter in filter: - if filter_parameter.startswith(PATH_SELECTOR) or filter_parameter.startswith(TAG_SELECTOR): - continue - elif any([filter_parameter.startswith(CONFIG_SELECTOR + config + ":") for config in SUPPORTED_CONFIG]): - continue - else: - raise CosmosValueError(f"Invalid {filter_type} filter: {filter_parameter}") + validate_filters(exclude, select) + subset_ids = apply_select_filter(nodes, project_dir, select) + if select: + nodes = get_nodes_from_subset(nodes, subset_ids) + exclude_ids = apply_exclude_filter(nodes, project_dir, exclude) + subset_ids = set(nodes.keys()) - exclude_ids - subset_ids: set[str] = set() + return get_nodes_from_subset(nodes, subset_ids) - for statement in select: - config = SelectorConfig(project_dir, statement) - node_selector = NodeSelector(nodes, config) - select_ids = node_selector.select_nodes_ids_by_intersection() - subset_ids = subset_ids.union(set(select_ids)) - if select: - nodes = {id_: nodes[id_] for id_ in subset_ids} +def get_nodes_from_subset(nodes: dict[str, DbtNode], subset_ids: set[str]) -> dict[str, DbtNode]: + nodes = {id_: nodes[id_] for id_ in subset_ids} + return nodes - nodes_ids = set(nodes.keys()) +def apply_exclude_filter(nodes: dict[str, DbtNode], project_dir: Path | None, exclude: list[str]) -> set[str]: exclude_ids: set[str] = set() for statement in exclude: config = SelectorConfig(project_dir, statement) node_selector = NodeSelector(nodes, config) - exclude_ids = exclude_ids.union(set(node_selector.select_nodes_ids_by_intersection())) - subset_ids = set(nodes_ids) - set(exclude_ids) + exclude_ids.update(node_selector.select_nodes_ids_by_intersection) + return exclude_ids - return {id_: nodes[id_] for id_ in subset_ids} + +def apply_select_filter(nodes: dict[str, DbtNode], project_dir: Path | None, select: list[str]) -> set[str]: + subset_ids: set[str] = set() + for statement in select: + config = SelectorConfig(project_dir, statement) + node_selector = NodeSelector(nodes, config) + select_ids = node_selector.select_nodes_ids_by_intersection + subset_ids.update(select_ids) + return subset_ids + + +def validate_filters(exclude: list[str], select: list[str]) -> None: + """ + Validate select and exclude filters. + """ + filters = [["select", select], ["exclude", exclude]] + for filter_type, filter in filters: + for filter_parameter in filter: + if ( + filter_parameter.startswith(PATH_SELECTOR) + or filter_parameter.startswith(TAG_SELECTOR) + or PLUS_SELECTOR in filter_parameter + or any([filter_parameter.startswith(CONFIG_SELECTOR + config + ":") for config in SUPPORTED_CONFIG]) + ): + continue + elif ":" in filter_parameter: + raise CosmosValueError(f"Invalid {filter_type} filter: {filter_parameter}") diff --git a/cosmos/exceptions.py b/cosmos/exceptions.py index 74091f4a13..308214475a 100644 --- a/cosmos/exceptions.py +++ b/cosmos/exceptions.py @@ -1,5 +1,9 @@ -"Contains exceptions that Cosmos uses" +"""Contains exceptions that Cosmos uses""" class CosmosValueError(ValueError): """Raised when a Cosmos config value is invalid.""" + + +class AirflowCompatibilityError(Exception): + """Raised when Cosmos features are limited for Airflow version being used.""" diff --git a/cosmos/hooks/subprocess.py b/cosmos/hooks/subprocess.py index cf6b489e8b..19c88540b7 100644 --- a/cosmos/hooks/subprocess.py +++ b/cosmos/hooks/subprocess.py @@ -7,9 +7,9 @@ import contextlib import os import signal -from typing import NamedTuple from subprocess import PIPE, STDOUT, Popen from tempfile import TemporaryDirectory, gettempdir +from typing import NamedTuple from airflow.hooks.base import BaseHook @@ -105,3 +105,9 @@ def send_sigterm(self) -> None: logger.info("Sending SIGTERM signal to process group") if self.sub_process and hasattr(self.sub_process, "pid"): os.killpg(os.getpgid(self.sub_process.pid), signal.SIGTERM) + + def send_sigint(self) -> None: + """Sends SIGINT signal to ``self.sub_process`` if one exists.""" + logger.info("Sending SIGINT signal to process group") + if self.sub_process and hasattr(self.sub_process, "pid"): + os.killpg(os.getpgid(self.sub_process.pid), signal.SIGINT) diff --git a/cosmos/log.py b/cosmos/log.py index 0527621532..3522b09dd0 100644 --- a/cosmos/log.py +++ b/cosmos/log.py @@ -1,8 +1,10 @@ from __future__ import annotations + import logging from airflow.utils.log.colored_log import CustomTTYColoredFormatter +from cosmos.settings import propagate_logs LOG_FORMAT: str = ( "[%(blue)s%(asctime)s%(reset)s] " @@ -12,8 +14,10 @@ "%(log_color)s%(message)s%(reset)s" ) +LOGGER_NAME_TEMPLATE = "astronomer-cosmos-{}" + -def get_logger(name: str | None = None) -> logging.Logger: +def get_logger(name: str) -> logging.Logger: """ Get custom Astronomer cosmos logger. @@ -23,9 +27,10 @@ def get_logger(name: str | None = None) -> logging.Logger: By using this logger, we introduce a (yellow) astronomer-cosmos string into the project's log messages: [2023-08-09T14:20:55.532+0100] {subprocess.py:94} INFO - (astronomer-cosmos) - 13:20:55 Completed successfully """ - logger = logging.getLogger(name) + logger = logging.getLogger(LOGGER_NAME_TEMPLATE.format(name)) formatter: logging.Formatter = CustomTTYColoredFormatter(fmt=LOG_FORMAT) # type: ignore handler = logging.StreamHandler() handler.setFormatter(formatter) logger.addHandler(handler) + logger.propagate = propagate_logs return logger diff --git a/cosmos/operators/__init__.py b/cosmos/operators/__init__.py index b7e36abff5..c546da0199 100644 --- a/cosmos/operators/__init__.py +++ b/cosmos/operators/__init__.py @@ -1,8 +1,9 @@ +from .local import DbtBuildLocalOperator as DbtBuildOperator from .local import DbtDepsLocalOperator as DbtDepsOperator from .local import DbtDocsAzureStorageLocalOperator as DbtDocsAzureStorageOperator +from .local import DbtDocsGCSLocalOperator as DbtDocsGCSOperator from .local import DbtDocsLocalOperator as DbtDocsOperator from .local import DbtDocsS3LocalOperator as DbtDocsS3Operator -from .local import DbtDocsGCSLocalOperator as DbtDocsGCSOperator from .local import DbtLSLocalOperator as DbtLSOperator from .local import DbtRunLocalOperator as DbtRunOperator from .local import DbtRunOperationLocalOperator as DbtRunOperationOperator @@ -16,6 +17,7 @@ "DbtSnapshotOperator", "DbtRunOperator", "DbtTestOperator", + "DbtBuildOperator", "DbtRunOperationOperator", "DbtDepsOperator", "DbtDocsOperator", diff --git a/cosmos/operators/aws_eks.py b/cosmos/operators/aws_eks.py new file mode 100644 index 0000000000..1800283783 --- /dev/null +++ b/cosmos/operators/aws_eks.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +from typing import Any, Sequence + +from airflow.exceptions import AirflowException +from airflow.providers.amazon.aws.hooks.eks import EksHook +from airflow.utils.context import Context + +from cosmos.operators.kubernetes import ( + DbtBuildKubernetesOperator, + DbtKubernetesBaseOperator, + DbtLSKubernetesOperator, + DbtRunKubernetesOperator, + DbtRunOperationKubernetesOperator, + DbtSeedKubernetesOperator, + DbtSnapshotKubernetesOperator, + DbtTestKubernetesOperator, +) + +DEFAULT_CONN_ID = "aws_default" +DEFAULT_NAMESPACE = "default" + + +class DbtAwsEksBaseOperator(DbtKubernetesBaseOperator): + template_fields: Sequence[str] = tuple( + { + "cluster_name", + "in_cluster", + "namespace", + "pod_name", + "aws_conn_id", + "region", + } + | set(DbtKubernetesBaseOperator.template_fields) + ) + + def __init__( + self, + cluster_name: str, + pod_name: str | None = None, + namespace: str | None = DEFAULT_NAMESPACE, + aws_conn_id: str = DEFAULT_CONN_ID, + region: str | None = None, + **kwargs: Any, + ) -> None: + self.cluster_name = cluster_name + self.pod_name = pod_name + self.namespace = namespace + self.aws_conn_id = aws_conn_id + self.region = region + super().__init__( + name=self.pod_name, + namespace=self.namespace, + **kwargs, + ) + # There is no need to manage the kube_config file, as it will be generated automatically. + # All Kubernetes parameters (except config_file) are also valid for the EksPodOperator. + if self.config_file: + raise AirflowException("The config_file is not an allowed parameter for the EksPodOperator.") + + def execute(self, context: Context) -> Any | None: # type: ignore + eks_hook = EksHook( + aws_conn_id=self.aws_conn_id, + region_name=self.region, + ) + with eks_hook.generate_config_file( + eks_cluster_name=self.cluster_name, pod_namespace=self.namespace + ) as self.config_file: + return super().execute(context) + + +class DbtBuildAwsEksOperator(DbtAwsEksBaseOperator, DbtBuildKubernetesOperator): + """ + Executes a dbt core build command. + """ + + template_fields: Sequence[str] = ( + DbtAwsEksBaseOperator.template_fields + DbtBuildKubernetesOperator.template_fields # type: ignore[operator] + ) + + +class DbtLSAwsEksOperator(DbtAwsEksBaseOperator, DbtLSKubernetesOperator): + """ + Executes a dbt core ls command. + """ + + +class DbtSeedAwsEksOperator(DbtAwsEksBaseOperator, DbtSeedKubernetesOperator): + """ + Executes a dbt core seed command. + """ + + template_fields: Sequence[str] = ( + DbtAwsEksBaseOperator.template_fields + DbtSeedKubernetesOperator.template_fields # type: ignore[operator] + ) + + +class DbtSnapshotAwsEksOperator(DbtAwsEksBaseOperator, DbtSnapshotKubernetesOperator): + """ + Executes a dbt core snapshot command. + """ + + +class DbtRunAwsEksOperator(DbtAwsEksBaseOperator, DbtRunKubernetesOperator): + """ + Executes a dbt core run command. + """ + + template_fields: Sequence[str] = ( + DbtAwsEksBaseOperator.template_fields + DbtRunKubernetesOperator.template_fields # type: ignore[operator] + ) + + +class DbtTestAwsEksOperator(DbtAwsEksBaseOperator, DbtTestKubernetesOperator): + """ + Executes a dbt core test command. + """ + + template_fields: Sequence[str] = ( + DbtAwsEksBaseOperator.template_fields + DbtTestKubernetesOperator.template_fields # type: ignore[operator] + ) + + +class DbtRunOperationAwsEksOperator(DbtAwsEksBaseOperator, DbtRunOperationKubernetesOperator): + """ + Executes a dbt core run-operation command. + """ + + template_fields: Sequence[str] = ( + DbtAwsEksBaseOperator.template_fields + DbtRunOperationKubernetesOperator.template_fields # type: ignore[operator] + ) diff --git a/cosmos/operators/azure_container_instance.py b/cosmos/operators/azure_container_instance.py new file mode 100644 index 0000000000..d8427b2fbb --- /dev/null +++ b/cosmos/operators/azure_container_instance.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +from typing import Any, Callable, Sequence + +from airflow.utils.context import Context + +from cosmos.config import ProfileConfig +from cosmos.log import get_logger +from cosmos.operators.base import ( + AbstractDbtBaseOperator, + DbtLSMixin, + DbtRunMixin, + DbtRunOperationMixin, + DbtSeedMixin, + DbtSnapshotMixin, + DbtTestMixin, +) + +logger = get_logger(__name__) + +# ACI is an optional dependency, so we need to check if it's installed +try: + from airflow.providers.microsoft.azure.operators.container_instances import AzureContainerInstancesOperator +except ImportError: + raise ImportError( + "Could not import AzureContainerInstancesOperator. Ensure you've installed the Microsoft Azure provider " + "separately or with `pip install astronomer-cosmos[...,azure-container-instance]`." + ) + + +class DbtAzureContainerInstanceBaseOperator(AbstractDbtBaseOperator, AzureContainerInstancesOperator): # type: ignore + """ + Executes a dbt core cli command in an Azure Container Instance + """ + + template_fields: Sequence[str] = tuple( + list(AbstractDbtBaseOperator.template_fields) + list(AzureContainerInstancesOperator.template_fields) + ) + + def __init__( + self, + ci_conn_id: str, + resource_group: str, + name: str, + image: str, + region: str, + profile_config: ProfileConfig | None = None, + remove_on_error: bool = False, + fail_if_exists: bool = False, + registry_conn_id: str | None = None, # need to add a default for Airflow 2.3 support + **kwargs: Any, + ) -> None: + self.profile_config = profile_config + super().__init__( + ci_conn_id=ci_conn_id, + resource_group=resource_group, + name=name, + image=image, + region=region, + remove_on_error=remove_on_error, + fail_if_exists=fail_if_exists, + registry_conn_id=registry_conn_id, + **kwargs, + ) + + def build_and_run_cmd(self, context: Context, cmd_flags: list[str] | None = None) -> None: + self.build_command(context, cmd_flags) + self.log.info(f"Running command: {self.command}") + result = AzureContainerInstancesOperator.execute(self, context) + logger.info(result) + + def build_command(self, context: Context, cmd_flags: list[str] | None = None) -> None: + # For the first round, we're going to assume that the command is dbt + # This means that we don't have openlineage support, but we will create a ticket + # to add that in the future + self.dbt_executable_path = "dbt" + dbt_cmd, env_vars = self.build_cmd(context=context, cmd_flags=cmd_flags) + self.environment_variables: dict[str, Any] = {**env_vars, **self.environment_variables} + self.command: list[str] = dbt_cmd + + +class DbtLSAzureContainerInstanceOperator(DbtLSMixin, DbtAzureContainerInstanceBaseOperator): # type: ignore + """ + Executes a dbt core ls command. + """ + + +class DbtSeedAzureContainerInstanceOperator(DbtSeedMixin, DbtAzureContainerInstanceBaseOperator): # type: ignore + """ + Executes a dbt core seed command. + + :param full_refresh: dbt optional arg - dbt will treat incremental models as table models + """ + + template_fields: Sequence[str] = DbtAzureContainerInstanceBaseOperator.template_fields + DbtRunMixin.template_fields # type: ignore[operator] + + +class DbtSnapshotAzureContainerInstanceOperator(DbtSnapshotMixin, DbtAzureContainerInstanceBaseOperator): # type: ignore + """ + Executes a dbt core snapshot command. + + """ + + +class DbtRunAzureContainerInstanceOperator(DbtRunMixin, DbtAzureContainerInstanceBaseOperator): # type: ignore + """ + Executes a dbt core run command. + """ + + template_fields: Sequence[str] = DbtAzureContainerInstanceBaseOperator.template_fields + DbtRunMixin.template_fields # type: ignore[operator] + + +class DbtTestAzureContainerInstanceOperator(DbtTestMixin, DbtAzureContainerInstanceBaseOperator): # type: ignore + """ + Executes a dbt core test command. + """ + + def __init__(self, on_warning_callback: Callable[..., Any] | None = None, **kwargs: Any) -> None: + super().__init__(**kwargs) + # as of now, on_warning_callback in azure container instance executor does nothing + self.on_warning_callback = on_warning_callback + + +class DbtRunOperationAzureContainerInstanceOperator(DbtRunOperationMixin, DbtAzureContainerInstanceBaseOperator): # type: ignore + """ + Executes a dbt core run-operation command. + + :param macro_name: name of macro to execute + :param args: Supply arguments to the macro. This dictionary will be mapped to the keyword arguments defined in the + selected macro. + """ + + template_fields: Sequence[str] = DbtAzureContainerInstanceBaseOperator.template_fields + DbtRunOperationMixin.template_fields # type: ignore[operator] diff --git a/cosmos/operators/base.py b/cosmos/operators/base.py index 6d276013d5..d0cbdd282a 100644 --- a/cosmos/operators/base.py +++ b/cosmos/operators/base.py @@ -1,28 +1,29 @@ from __future__ import annotations import os +from abc import ABCMeta, abstractmethod +from pathlib import Path from typing import Any, Sequence, Tuple import yaml from airflow.models.baseoperator import BaseOperator -from airflow.utils.context import Context +from airflow.utils.context import Context, context_merge from airflow.utils.operator_helpers import context_to_airflow_vars +from airflow.utils.strings import to_boolean from cosmos.dbt.executable import get_system_dbt from cosmos.log import get_logger - logger = get_logger(__name__) -class DbtBaseOperator(BaseOperator): +class AbstractDbtBaseOperator(BaseOperator, metaclass=ABCMeta): """ Executes a dbt core cli command. :param project_dir: Which directory to look in for the dbt_project.yml file. Default is the current working directory and its parents. :param conn_id: The airflow connection to use as the target - :param base_cmd: dbt sub-command to run (i.e ls, seed, run, test, etc.) :param select: dbt optional argument that specifies which nodes to include. :param exclude: dbt optional argument that specifies which models to exclude. :param selector: dbt optional argument - the selector name to use, as defined in selectors.yml @@ -43,23 +44,29 @@ class DbtBaseOperator(BaseOperator): environment variables for the new process; these are used instead of inheriting the current process environment, which is the default behavior. (templated) - :param append_env: If False(default) uses the environment variables passed in env params - and does not inherit the current process environment. If True, inherits the environment variables - from current passes and then environment variable passed by the user will either update the existing - inherited environment variables or the new variables gets appended to it + :param append_env: If True, inherits the environment variables + from current process and then environment variable passed by the user will either update the existing + inherited environment variables or the new variables gets appended to it. + If False (default), only uses the environment variables passed in env params + and does not inherit the current process environment. :param output_encoding: Output encoding of bash command :param skip_exit_code: If task exits with this exit code, leave the task in ``skipped`` state (default: 99). If set to ``None``, any non-zero exit code will be treated as a failure. + :param partial_parse: If True (default), then the operator will use the + ``partial_parse.msgpack`` during execution if it exists. If False, then + a flag will be explicitly set to turn off partial parsing. :param cancel_query_on_kill: If true, then cancel any running queries when the task's on_kill() is executed. Otherwise, the query will keep running when the task is killed. :param dbt_executable_path: Path to dbt executable can be used with venv (i.e. /home/astro/.pyenv/versions/dbt_venv/bin/dbt) :param dbt_cmd_flags: List of flags to pass to dbt command :param dbt_cmd_global_flags: List of dbt global flags to be passed to the dbt command + :param cache_dir: Directory used to cache Cosmos/dbt artifacts in Airflow worker nodes + :param extra_context: A dictionary of values to add to the TaskInstance's Context """ - template_fields: Sequence[str] = ("env", "vars") + template_fields: Sequence[str] = ("env", "select", "exclude", "selector", "vars", "models") global_flags = ( "project_dir", "select", @@ -68,21 +75,19 @@ class DbtBaseOperator(BaseOperator): "vars", "models", ) - global_boolean_flags = ( - "no_version_check", - "cache_selected_only", - "fail_fast", - "quiet", - "warn_error", - ) + global_boolean_flags = ("no_version_check", "cache_selected_only", "fail_fast", "quiet", "warn_error") intercept_flag = True + @property + @abstractmethod + def base_cmd(self) -> list[str]: + """Override this property to set the dbt sub-command (i.e ls, seed, run, test, etc.) for the operator""" + def __init__( self, project_dir: str, conn_id: str | None = None, - base_cmd: list[str] | None = None, select: str | None = None, exclude: str | None = None, selector: str | None = None, @@ -101,15 +106,17 @@ def __init__( append_env: bool = False, output_encoding: str = "utf-8", skip_exit_code: int = 99, + partial_parse: bool = True, cancel_query_on_kill: bool = True, dbt_executable_path: str = get_system_dbt(), dbt_cmd_flags: list[str] | None = None, dbt_cmd_global_flags: list[str] | None = None, + cache_dir: Path | None = None, + extra_context: dict[str, Any] | None = None, **kwargs: Any, ) -> None: self.project_dir = project_dir self.conn_id = conn_id - self.base_cmd = base_cmd self.select = select self.exclude = exclude self.selector = selector @@ -128,10 +135,13 @@ def __init__( self.append_env = append_env self.output_encoding = output_encoding self.skip_exit_code = skip_exit_code + self.partial_parse = partial_parse self.cancel_query_on_kill = cancel_query_on_kill self.dbt_executable_path = dbt_executable_path self.dbt_cmd_flags = dbt_cmd_flags self.dbt_cmd_global_flags = dbt_cmd_global_flags or [] + self.cache_dir = cache_dir + self.extra_context = extra_context or {} super().__init__(**kwargs) def get_env(self, context: Context) -> dict[str, str | bytes | os.PathLike[Any]]: @@ -192,17 +202,32 @@ def add_global_flags(self) -> list[str]: dbt_name = f"--{global_flag.replace('_', '-')}" global_flag_value = self.__getattribute__(global_flag) - if global_flag_value is not None: - if isinstance(global_flag_value, dict): - yaml_string = yaml.dump(global_flag_value) - flags.extend([dbt_name, yaml_string]) - else: - flags.extend([dbt_name, str(global_flag_value)]) + flags.extend(self._process_global_flag(dbt_name, global_flag_value)) + for global_boolean_flag in self.global_boolean_flags: if self.__getattribute__(global_boolean_flag): flags.append(f"--{global_boolean_flag.replace('_', '-')}") return flags + @staticmethod + def _process_global_flag(flag_name: str, flag_value: Any) -> list[str]: + """Helper method to process global flags and reduce complexity.""" + if flag_value is None: + return [] + elif isinstance(flag_value, dict): + yaml_string = yaml.dump(flag_value) + return [flag_name, yaml_string] + elif isinstance(flag_value, list) and flag_value: + return [flag_name, " ".join(flag_value)] + elif isinstance(flag_value, list): + return [] + else: + return [flag_name, str(flag_value)] + + def add_cmd_flags(self) -> list[str]: + """Allows subclasses to override to add flags for their dbt command""" + return [] + def build_cmd( self, context: Context, @@ -212,8 +237,10 @@ def build_cmd( dbt_cmd.extend(self.dbt_cmd_global_flags) - if self.base_cmd: - dbt_cmd.extend(self.base_cmd) + if not self.partial_parse: + dbt_cmd.append("--no-partial-parse") + + dbt_cmd.extend(self.base_cmd) if self.indirect_selection: dbt_cmd += ["--indirect-selection", self.indirect_selection] @@ -231,3 +258,167 @@ def build_cmd( env = self.get_env(context) return dbt_cmd, env + + @abstractmethod + def build_and_run_cmd(self, context: Context, cmd_flags: list[str]) -> Any: + """Override this method for the operator to execute the dbt command""" + + def execute(self, context: Context) -> Any | None: # type: ignore + if self.extra_context: + context_merge(context, self.extra_context) + + self.build_and_run_cmd(context=context, cmd_flags=self.add_cmd_flags()) + + +class DbtBuildMixin: + """Mixin for dbt build command.""" + + base_cmd = ["build"] + ui_color = "#8194E0" + + template_fields: Sequence[str] = ("full_refresh",) + + def __init__(self, full_refresh: bool | str = False, **kwargs: Any) -> None: + self.full_refresh = full_refresh + super().__init__(**kwargs) + + def add_cmd_flags(self) -> list[str]: + flags = [] + + if isinstance(self.full_refresh, str): + # Handle template fields when render_template_as_native_obj=False + full_refresh = to_boolean(self.full_refresh) + else: + full_refresh = self.full_refresh + + if full_refresh is True: + flags.append("--full-refresh") + + return flags + + +class DbtLSMixin: + """ + Executes a dbt core ls command. + """ + + base_cmd = ["ls"] + ui_color = "#DBCDF6" + + +class DbtSeedMixin: + """ + Mixin for dbt seed operation command. + + :param full_refresh: whether to add the flag --full-refresh to the dbt seed command + """ + + base_cmd = ["seed"] + ui_color = "#F58D7E" + + template_fields: Sequence[str] = ("full_refresh",) + + def __init__(self, full_refresh: bool | str = False, **kwargs: Any) -> None: + self.full_refresh = full_refresh + super().__init__(**kwargs) + + def add_cmd_flags(self) -> list[str]: + flags = [] + + if isinstance(self.full_refresh, str): + # Handle template fields when render_template_as_native_obj=False + full_refresh = to_boolean(self.full_refresh) + else: + full_refresh = self.full_refresh + + if full_refresh is True: + flags.append("--full-refresh") + + return flags + + +class DbtSnapshotMixin: + """Mixin for a dbt snapshot command.""" + + base_cmd = ["snapshot"] + ui_color = "#964B00" + + +class DbtRunMixin: + """ + Mixin for dbt run command. + + :param full_refresh: whether to add the flag --full-refresh to the dbt seed command + """ + + base_cmd = ["run"] + ui_color = "#7352BA" + ui_fgcolor = "#F4F2FC" + + template_fields: Sequence[str] = ("full_refresh",) + + def __init__(self, full_refresh: bool | str = False, **kwargs: Any) -> None: + self.full_refresh = full_refresh + super().__init__(**kwargs) + + def add_cmd_flags(self) -> list[str]: + flags = [] + + if isinstance(self.full_refresh, str): + # Handle template fields when render_template_as_native_obj=False + full_refresh = to_boolean(self.full_refresh) + else: + full_refresh = self.full_refresh + + if full_refresh is True: + flags.append("--full-refresh") + + return flags + + +class DbtTestMixin: + """Mixin for dbt test command.""" + + base_cmd = ["test"] + ui_color = "#8194E0" + + def __init__( + self, + exclude: str | None = None, + select: str | None = None, + selector: str | None = None, + **kwargs: Any, + ) -> None: + self.select = select + self.exclude = exclude + self.selector = selector + super().__init__(exclude=exclude, select=select, selector=selector, **kwargs) # type: ignore + + +class DbtRunOperationMixin: + """ + Mixin for dbt run operation command. + + :param macro_name: name of macro to execute + :param args: Supply arguments to the macro. This dictionary will be mapped to the keyword arguments defined in the + selected macro. + """ + + ui_color = "#8194E0" + template_fields: Sequence[str] = ("args",) + + def __init__(self, macro_name: str, args: dict[str, Any] | None = None, **kwargs: Any) -> None: + self.macro_name = macro_name + self.args = args + super().__init__(**kwargs) + + @property + def base_cmd(self) -> list[str]: + return ["run-operation", self.macro_name] + + def add_cmd_flags(self) -> list[str]: + flags = [] + if self.args is not None: + flags.append("--args") + flags.append(yaml.dump(self.args)) + return flags diff --git a/cosmos/operators/docker.py b/cosmos/operators/docker.py index fb2e1c90c7..532de380e7 100644 --- a/cosmos/operators/docker.py +++ b/cosmos/operators/docker.py @@ -2,11 +2,19 @@ from typing import Any, Callable, Sequence -import yaml from airflow.utils.context import Context from cosmos.log import get_logger -from cosmos.operators.base import DbtBaseOperator +from cosmos.operators.base import ( + AbstractDbtBaseOperator, + DbtBuildMixin, + DbtLSMixin, + DbtRunMixin, + DbtRunOperationMixin, + DbtSeedMixin, + DbtSnapshotMixin, + DbtTestMixin, +) logger = get_logger(__name__) @@ -20,13 +28,15 @@ ) -class DbtDockerBaseOperator(DockerOperator, DbtBaseOperator): # type: ignore +class DbtDockerBaseOperator(AbstractDbtBaseOperator, DockerOperator): # type: ignore """ Executes a dbt core cli command in a Docker container. """ - template_fields: Sequence[str] = tuple(list(DbtBaseOperator.template_fields) + list(DockerOperator.template_fields)) + template_fields: Sequence[str] = tuple( + list(AbstractDbtBaseOperator.template_fields) + list(DockerOperator.template_fields) + ) intercept_flag = False @@ -40,7 +50,7 @@ def __init__( def build_and_run_cmd(self, context: Context, cmd_flags: list[str] | None = None) -> Any: self.build_command(context, cmd_flags) self.log.info(f"Running command: {self.command}") - result = super().execute(context) + result = DockerOperator.execute(self, context) logger.info(result) def build_command(self, context: Context, cmd_flags: list[str] | None = None) -> None: @@ -53,89 +63,57 @@ def build_command(self, context: Context, cmd_flags: list[str] | None = None) -> self.environment: dict[str, Any] = {**env_vars, **self.environment} self.command: list[str] = dbt_cmd - def execute(self, context: Context) -> None: - self.build_and_run_cmd(context=context) - -class DbtLSDockerOperator(DbtDockerBaseOperator): +class DbtBuildDockerOperator(DbtBuildMixin, DbtDockerBaseOperator): """ - Executes a dbt core ls command. + Executes a dbt core build command. """ - ui_color = "#DBCDF6" + template_fields: Sequence[str] = DbtDockerBaseOperator.template_fields + DbtBuildMixin.template_fields # type: ignore[operator] - def __init__(self, **kwargs: str) -> None: - super().__init__(**kwargs) - self.base_cmd = ["ls"] + +class DbtLSDockerOperator(DbtLSMixin, DbtDockerBaseOperator): + """ + Executes a dbt core ls command. + """ -class DbtSeedDockerOperator(DbtDockerBaseOperator): +class DbtSeedDockerOperator(DbtSeedMixin, DbtDockerBaseOperator): """ Executes a dbt core seed command. :param full_refresh: dbt optional arg - dbt will treat incremental models as table models """ - ui_color = "#F58D7E" - - def __init__(self, full_refresh: bool = False, **kwargs: str) -> None: - self.full_refresh = full_refresh - super().__init__(**kwargs) - self.base_cmd = ["seed"] - - def add_cmd_flags(self) -> list[str]: - flags = [] - if self.full_refresh is True: - flags.append("--full-refresh") + template_fields: Sequence[str] = DbtDockerBaseOperator.template_fields + DbtSeedMixin.template_fields # type: ignore[operator] - return flags - def execute(self, context: Context) -> None: - cmd_flags = self.add_cmd_flags() - self.build_and_run_cmd(context=context, cmd_flags=cmd_flags) - - -class DbtSnapshotDockerOperator(DbtDockerBaseOperator): +class DbtSnapshotDockerOperator(DbtSnapshotMixin, DbtDockerBaseOperator): """ Executes a dbt core snapshot command. - """ - ui_color = "#964B00" - - def __init__(self, **kwargs: str) -> None: - super().__init__(**kwargs) - self.base_cmd = ["snapshot"] - -class DbtRunDockerOperator(DbtDockerBaseOperator): +class DbtRunDockerOperator(DbtRunMixin, DbtDockerBaseOperator): """ Executes a dbt core run command. """ - ui_color = "#7352BA" - ui_fgcolor = "#F4F2FC" - - def __init__(self, **kwargs: str) -> None: - super().__init__(**kwargs) - self.base_cmd = ["run"] + template_fields: Sequence[str] = DbtDockerBaseOperator.template_fields + DbtRunMixin.template_fields # type: ignore[operator] -class DbtTestDockerOperator(DbtDockerBaseOperator): +class DbtTestDockerOperator(DbtTestMixin, DbtDockerBaseOperator): """ Executes a dbt core test command. """ - ui_color = "#8194E0" - def __init__(self, on_warning_callback: Callable[..., Any] | None = None, **kwargs: str) -> None: super().__init__(**kwargs) - self.base_cmd = ["test"] # as of now, on_warning_callback in docker executor does nothing self.on_warning_callback = on_warning_callback -class DbtRunOperationDockerOperator(DbtDockerBaseOperator): +class DbtRunOperationDockerOperator(DbtRunOperationMixin, DbtDockerBaseOperator): """ Executes a dbt core run-operation command. @@ -144,22 +122,4 @@ class DbtRunOperationDockerOperator(DbtDockerBaseOperator): selected macro. """ - ui_color = "#8194E0" - template_fields: Sequence[str] = ("args",) - - def __init__(self, macro_name: str, args: dict[str, Any] | None = None, **kwargs: str) -> None: - self.macro_name = macro_name - self.args = args - super().__init__(**kwargs) - self.base_cmd = ["run-operation", macro_name] - - def add_cmd_flags(self) -> list[str]: - flags = [] - if self.args is not None: - flags.append("--args") - flags.append(yaml.dump(self.args)) - return flags - - def execute(self, context: Context) -> None: - cmd_flags = self.add_cmd_flags() - self.build_and_run_cmd(context=context, cmd_flags=cmd_flags) + template_fields: Sequence[str] = DbtDockerBaseOperator.template_fields + DbtRunOperationMixin.template_fields # type: ignore[operator] diff --git a/cosmos/operators/kubernetes.py b/cosmos/operators/kubernetes.py index 996bbc9dda..f842191998 100644 --- a/cosmos/operators/kubernetes.py +++ b/cosmos/operators/kubernetes.py @@ -3,13 +3,25 @@ from os import PathLike from typing import Any, Callable, Sequence -import yaml -from airflow.utils.context import Context +from airflow.models import TaskInstance +from airflow.utils.context import Context, context_merge -from cosmos.log import get_logger from cosmos.config import ProfileConfig -from cosmos.operators.base import DbtBaseOperator - +from cosmos.dbt.parser.output import extract_log_issues +from cosmos.log import get_logger +from cosmos.operators.base import ( + AbstractDbtBaseOperator, + DbtBuildMixin, + DbtLSMixin, + DbtRunMixin, + DbtRunOperationMixin, + DbtSeedMixin, + DbtSnapshotMixin, + DbtTestMixin, +) + +DBT_NO_TESTS_MSG = "Nothing to do" +DBT_WARN_MSG = "WARN" logger = get_logger(__name__) @@ -19,26 +31,26 @@ convert_env_vars, ) from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator + from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction except ImportError: try: # apache-airflow-providers-cncf-kubernetes < 7.4.0 from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator - except ImportError as error: - logger.exception(error) + except ImportError: raise ImportError( "Could not import KubernetesPodOperator. Ensure you've installed the Kubernetes provider " "separately or with with `pip install astronomer-cosmos[...,kubernetes]`." ) -class DbtKubernetesBaseOperator(KubernetesPodOperator, DbtBaseOperator): # type: ignore +class DbtKubernetesBaseOperator(AbstractDbtBaseOperator, KubernetesPodOperator): # type: ignore """ Executes a dbt core cli command in a Kubernetes Pod. """ template_fields: Sequence[str] = tuple( - list(DbtBaseOperator.template_fields) + list(KubernetesPodOperator.template_fields) + list(AbstractDbtBaseOperator.template_fields) + list(KubernetesPodOperator.template_fields) ) intercept_flag = False @@ -50,15 +62,17 @@ def __init__(self, profile_config: ProfileConfig | None = None, **kwargs: Any) - def build_env_args(self, env: dict[str, str | bytes | PathLike[Any]]) -> None: env_vars_dict: dict[str, str] = dict() + for env_var_key, env_var_value in env.items(): + env_vars_dict[env_var_key] = str(env_var_value) for env_var in self.env_vars: env_vars_dict[env_var.name] = env_var.value - self.env_vars: list[Any] = convert_env_vars({**env, **env_vars_dict}) + self.env_vars: list[Any] = convert_env_vars(env_vars_dict) def build_and_run_cmd(self, context: Context, cmd_flags: list[str] | None = None) -> Any: self.build_kube_args(context, cmd_flags) self.log.info(f"Running command: {self.arguments}") - result = super().execute(context) + result = KubernetesPodOperator.execute(self, context) logger.info(result) def build_kube_args(self, context: Context, cmd_flags: list[str] | None = None) -> None: @@ -82,113 +96,142 @@ def build_kube_args(self, context: Context, cmd_flags: list[str] | None = None) self.build_env_args(env_vars) self.arguments = dbt_cmd - def execute(self, context: Context) -> None: - self.build_and_run_cmd(context=context) - -class DbtLSKubernetesOperator(DbtKubernetesBaseOperator): +class DbtBuildKubernetesOperator(DbtBuildMixin, DbtKubernetesBaseOperator): """ - Executes a dbt core ls command. + Executes a dbt core build command. """ - ui_color = "#DBCDF6" - - def __init__(self, **kwargs: Any) -> None: - super().__init__(**kwargs) - self.base_cmd = ["ls"] + template_fields: Sequence[str] = DbtKubernetesBaseOperator.template_fields + DbtBuildMixin.template_fields # type: ignore[operator] -class DbtSeedKubernetesOperator(DbtKubernetesBaseOperator): +class DbtLSKubernetesOperator(DbtLSMixin, DbtKubernetesBaseOperator): """ - Executes a dbt core seed command. - - :param full_refresh: dbt optional arg - dbt will treat incremental models as table models + Executes a dbt core ls command. """ - ui_color = "#F58D7E" - - def __init__(self, full_refresh: bool = False, **kwargs: Any) -> None: - self.full_refresh = full_refresh - super().__init__(**kwargs) - self.base_cmd = ["seed"] - def add_cmd_flags(self) -> list[str]: - flags = [] - if self.full_refresh is True: - flags.append("--full-refresh") - - return flags +class DbtSeedKubernetesOperator(DbtSeedMixin, DbtKubernetesBaseOperator): + """ + Executes a dbt core seed command. + """ - def execute(self, context: Context) -> None: - cmd_flags = self.add_cmd_flags() - self.build_and_run_cmd(context=context, cmd_flags=cmd_flags) + template_fields: Sequence[str] = DbtKubernetesBaseOperator.template_fields + DbtSeedMixin.template_fields # type: ignore[operator] -class DbtSnapshotKubernetesOperator(DbtKubernetesBaseOperator): +class DbtSnapshotKubernetesOperator(DbtSnapshotMixin, DbtKubernetesBaseOperator): """ Executes a dbt core snapshot command. - """ - ui_color = "#964B00" - - def __init__(self, **kwargs: Any) -> None: - super().__init__(**kwargs) - self.base_cmd = ["snapshot"] - -class DbtRunKubernetesOperator(DbtKubernetesBaseOperator): +class DbtRunKubernetesOperator(DbtRunMixin, DbtKubernetesBaseOperator): """ Executes a dbt core run command. """ - ui_color = "#7352BA" - ui_fgcolor = "#F4F2FC" - - def __init__(self, **kwargs: Any) -> None: - super().__init__(**kwargs) - self.base_cmd = ["run"] + template_fields: Sequence[str] = DbtKubernetesBaseOperator.template_fields + DbtRunMixin.template_fields # type: ignore[operator] -class DbtTestKubernetesOperator(DbtKubernetesBaseOperator): +class DbtTestKubernetesOperator(DbtTestMixin, DbtKubernetesBaseOperator): """ Executes a dbt core test command. """ - ui_color = "#8194E0" - def __init__(self, on_warning_callback: Callable[..., Any] | None = None, **kwargs: Any) -> None: - super().__init__(**kwargs) - self.base_cmd = ["test"] - # as of now, on_warning_callback in kubernetes executor does nothing - self.on_warning_callback = on_warning_callback - + if not on_warning_callback: + super().__init__(**kwargs) + else: + self.on_warning_callback = on_warning_callback + self.is_delete_operator_pod_original = kwargs.get("is_delete_operator_pod", None) + if self.is_delete_operator_pod_original is not None: + self.on_finish_action_original = ( + OnFinishAction.DELETE_POD if self.is_delete_operator_pod_original else OnFinishAction.KEEP_POD + ) + else: + self.on_finish_action_original = OnFinishAction(kwargs.get("on_finish_action", "delete_pod")) + self.is_delete_operator_pod_original = self.on_finish_action_original == OnFinishAction.DELETE_POD + # In order to read the pod logs, we need to keep the pod around. + # Depending on the on_finish_action & is_delete_operator_pod settings, + # we will clean up the pod later in the _handle_warnings method, which + # is called in on_success_callback. + kwargs["is_delete_operator_pod"] = False + kwargs["on_finish_action"] = OnFinishAction.KEEP_POD + + # Add an additional callback to both success and failure callbacks. + # In case of success, check for a warning in the logs and clean up the pod. + self.on_success_callback = kwargs.get("on_success_callback", None) or [] + if isinstance(self.on_success_callback, list): + self.on_success_callback += [self._handle_warnings] + else: + self.on_success_callback = [self.on_success_callback, self._handle_warnings] + kwargs["on_success_callback"] = self.on_success_callback + # In case of failure, clean up the pod. + self.on_failure_callback = kwargs.get("on_failure_callback", None) or [] + if isinstance(self.on_failure_callback, list): + self.on_failure_callback += [self._cleanup_pod] + else: + self.on_failure_callback = [self.on_failure_callback, self._cleanup_pod] + kwargs["on_failure_callback"] = self.on_failure_callback + + super().__init__(**kwargs) + + def _handle_warnings(self, context: Context) -> None: + """ + Handles warnings by extracting log issues, creating additional context, and calling the + on_warning_callback with the updated context. + + :param context: The original airflow context in which the build and run command was executed. + """ + if not ( + isinstance(context["task_instance"], TaskInstance) + and isinstance(context["task_instance"].task, DbtTestKubernetesOperator) + ): + return + task = context["task_instance"].task + logs = [ + log.decode("utf-8") for log in task.pod_manager.read_pod_logs(task.pod, "base") if log.decode("utf-8") != "" + ] + + should_trigger_callback = all( + [ + logs, + self.on_warning_callback, + DBT_NO_TESTS_MSG not in logs[-1], + DBT_WARN_MSG in logs[-1], + ] + ) -class DbtRunOperationKubernetesOperator(DbtKubernetesBaseOperator): + if should_trigger_callback: + warnings = int(logs[-1].split(f"{DBT_WARN_MSG}=")[1].split()[0]) + if warnings > 0: + test_names, test_results = extract_log_issues(logs) + context_merge(context, test_names=test_names, test_results=test_results) + self.on_warning_callback(context) + + self._cleanup_pod(context) + + def _cleanup_pod(self, context: Context) -> None: + """ + Handles the cleaning up of the pod after success or failure, if + there is a on_warning_callback function defined. + + :param context: The original airflow context in which the build and run command was executed. + """ + if not ( + isinstance(context["task_instance"], TaskInstance) + and isinstance(context["task_instance"].task, DbtTestKubernetesOperator) + ): + return + task = context["task_instance"].task + if task.pod: + task.on_finish_action = self.on_finish_action_original + task.cleanup(pod=task.pod, remote_pod=task.remote_pod) + + +class DbtRunOperationKubernetesOperator(DbtRunOperationMixin, DbtKubernetesBaseOperator): """ Executes a dbt core run-operation command. - - :param macro_name: name of macro to execute - :param args: Supply arguments to the macro. This dictionary will be mapped to the keyword arguments defined in the - selected macro. """ - ui_color = "#8194E0" - template_fields: Sequence[str] = ("args",) - - def __init__(self, macro_name: str, args: dict[str, Any] | None = None, **kwargs: Any) -> None: - self.macro_name = macro_name - self.args = args - super().__init__(**kwargs) - self.base_cmd = ["run-operation", macro_name] - - def add_cmd_flags(self) -> list[str]: - flags = [] - if self.args is not None: - flags.append("--args") - flags.append(yaml.dump(self.args)) - return flags - - def execute(self, context: Context) -> None: - cmd_flags = self.add_cmd_flags() - self.build_and_run_cmd(context=context, cmd_flags=cmd_flags) + template_fields: Sequence[str] = DbtKubernetesBaseOperator.template_fields + DbtRunOperationMixin.template_fields # type: ignore[operator] diff --git a/cosmos/operators/local.py b/cosmos/operators/local.py index 1c00f476c8..83c0dfa74b 100644 --- a/cosmos/operators/local.py +++ b/cosmos/operators/local.py @@ -1,29 +1,30 @@ from __future__ import annotations import os -import shutil -import signal import tempfile -from attr import define -from pathlib import Path -from typing import Any, Callable, Literal, Sequence, TYPE_CHECKING -from abc import ABC, abstractmethod import warnings +from abc import ABC, abstractmethod +from functools import cached_property +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable, Literal, Sequence -import airflow import jinja2 -import yaml from airflow import DAG -from airflow.compat.functools import cached_property -from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowSkipException from airflow.models.taskinstance import TaskInstance from airflow.utils.context import Context from airflow.utils.session import NEW_SESSION, create_session, provide_session +from attr import define + +from cosmos import cache +from cosmos.constants import InvocationMode +from cosmos.dbt.project import get_partial_parse_path, has_non_empty_dependencies_file +from cosmos.exceptions import AirflowCompatibilityError +from cosmos.settings import LINEAGE_NAMESPACE try: - from openlineage.common.provider.dbt.local import DbtLocalArtifactProcessor from airflow.datasets import Dataset + from openlineage.common.provider.dbt.local import DbtLocalArtifactProcessor except ModuleNotFoundError: is_openlineage_available = False DbtLocalArtifactProcessor = None @@ -32,22 +33,37 @@ if TYPE_CHECKING: from airflow.datasets import Dataset # noqa: F811 + from dbt.cli.main import dbtRunner, dbtRunnerResult from openlineage.client.run import RunEvent from sqlalchemy.orm import Session -from cosmos.constants import DEFAULT_OPENLINEAGE_NAMESPACE, OPENLINEAGE_PRODUCER from cosmos.config import ProfileConfig -from cosmos.log import get_logger -from cosmos.operators.base import DbtBaseOperator +from cosmos.constants import ( + OPENLINEAGE_PRODUCER, +) +from cosmos.dbt.parser.output import ( + extract_dbt_runner_issues, + extract_log_issues, + parse_number_of_warnings_dbt_runner, + parse_number_of_warnings_subprocess, +) +from cosmos.dbt.project import change_working_directory, create_symlinks, environ from cosmos.hooks.subprocess import ( FullOutputSubprocessHook, FullOutputSubprocessResult, ) -from cosmos.dbt.parser.output import extract_log_issues, parse_output - -DBT_NO_TESTS_MSG = "Nothing to do" -DBT_WARN_MSG = "WARN" +from cosmos.log import get_logger +from cosmos.operators.base import ( + AbstractDbtBaseOperator, + DbtBuildMixin, + DbtLSMixin, + DbtRunMixin, + DbtRunOperationMixin, + DbtSeedMixin, + DbtSnapshotMixin, + DbtTestMixin, +) logger = get_logger(__name__) @@ -74,13 +90,7 @@ class OperatorLineage: # type: ignore job_facets: dict[str, str] = dict() -try: - LINEAGE_NAMESPACE = conf.get("openlineage", "namespace") -except airflow.exceptions.AirflowConfigException: - LINEAGE_NAMESPACE = os.getenv("OPENLINEAGE_NAMESPACE", DEFAULT_OPENLINEAGE_NAMESPACE) - - -class DbtLocalBaseOperator(DbtBaseOperator): +class DbtLocalBaseOperator(AbstractDbtBaseOperator): """ Executes a dbt core cli command locally. @@ -89,14 +99,18 @@ class DbtLocalBaseOperator(DbtBaseOperator): :param profile_name: A name to use for the dbt profile. If not provided, and no profile target is found in your project's dbt_project.yml, "cosmos_profile" is used. :param install_deps: If true, install dependencies before running the command - :param install_deps: If true, the operator will set inlets and outlets :param callback: A callback function called on after a dbt run with a path to the dbt project directory. :param target_name: A name to use for the dbt target. If not provided, and no target is found in your project's dbt_project.yml, "cosmos_target" is used. :param should_store_compiled_sql: If true, store the compiled SQL in the compiled_sql rendered template. + :param append_env: If True(default), inherits the environment variables + from current process and then environment variable passed by the user will either update the existing + inherited environment variables or the new variables gets appended to it. + If False, only uses the environment variables passed in env params + and does not inherit the current process environment. """ - template_fields: Sequence[str] = DbtBaseOperator.template_fields + ("compiled_sql",) # type: ignore[operator] + template_fields: Sequence[str] = AbstractDbtBaseOperator.template_fields + ("compiled_sql",) # type: ignore[operator] template_fields_renderers = { "compiled_sql": "sql", } @@ -104,33 +118,81 @@ class DbtLocalBaseOperator(DbtBaseOperator): def __init__( self, profile_config: ProfileConfig, + invocation_mode: InvocationMode | None = None, install_deps: bool = False, callback: Callable[[str], None] | None = None, should_store_compiled_sql: bool = True, + append_env: bool = True, **kwargs: Any, ) -> None: self.profile_config = profile_config - self.install_deps = install_deps self.callback = callback self.compiled_sql = "" self.should_store_compiled_sql = should_store_compiled_sql self.openlineage_events_completes: list[RunEvent] = [] + self.invocation_mode = invocation_mode + self.invoke_dbt: Callable[..., FullOutputSubprocessResult | dbtRunnerResult] + self.handle_exception: Callable[..., None] + self._dbt_runner: dbtRunner | None = None + if self.invocation_mode: + self._set_invocation_methods() kwargs.pop("full_refresh", None) # usage of this param should be implemented in child classes super().__init__(**kwargs) + # For local execution mode, we're consistent with the LoadMode.DBT_LS command in forwarding the environment + # variables to the subprocess by default. Although this behavior is designed for ExecuteMode.LOCAL and + # ExecuteMode.VIRTUALENV, it is not desired for the other execution modes to forward the environment variables + # as it can break existing DAGs. + self.append_env = append_env + + # We should not spend time trying to install deps if the project doesn't have any dependencies + self.install_deps = install_deps and has_non_empty_dependencies_file(Path(self.project_dir)) + @cached_property def subprocess_hook(self) -> FullOutputSubprocessHook: """Returns hook for running the bash command.""" return FullOutputSubprocessHook() - def exception_handling(self, result: FullOutputSubprocessResult) -> None: + def _set_invocation_methods(self) -> None: + """Sets the associated run and exception handling methods based on the invocation mode.""" + if self.invocation_mode == InvocationMode.SUBPROCESS: + self.invoke_dbt = self.run_subprocess + self.handle_exception = self.handle_exception_subprocess + elif self.invocation_mode == InvocationMode.DBT_RUNNER: + self.invoke_dbt = self.run_dbt_runner + self.handle_exception = self.handle_exception_dbt_runner + + def _discover_invocation_mode(self) -> None: + """Discovers the invocation mode based on the availability of dbtRunner for import. If dbtRunner is available, it will + be used since it is faster than subprocess. If dbtRunner is not available, it will fall back to subprocess. + This method is called at runtime to work in the environment where the operator is running. + """ + try: + from dbt.cli.main import dbtRunner # noqa + except ImportError: + self.invocation_mode = InvocationMode.SUBPROCESS + logger.info("Could not import dbtRunner. Falling back to subprocess for invoking dbt.") + else: + self.invocation_mode = InvocationMode.DBT_RUNNER + logger.info("dbtRunner is available. Using dbtRunner for invoking dbt.") + self._set_invocation_methods() + + def handle_exception_subprocess(self, result: FullOutputSubprocessResult) -> None: if self.skip_exit_code is not None and result.exit_code == self.skip_exit_code: raise AirflowSkipException(f"dbt command returned exit code {self.skip_exit_code}. Skipping.") elif result.exit_code != 0: - raise AirflowException( - f"dbt command failed. The command returned a non-zero exit code {result.exit_code}. Details: ", - *result.full_output, - ) + logger.error("\n".join(result.full_output)) + raise AirflowException(f"dbt command failed. The command returned a non-zero exit code {result.exit_code}.") + + def handle_exception_dbt_runner(self, result: dbtRunnerResult) -> None: + """dbtRunnerResult has an attribute `success` that is False if the command failed.""" + if not result.success: + if result.exception: + raise AirflowException(f"dbt invocation did not complete with unhandled error: {result.exception}") + else: + node_names, node_results = extract_dbt_runner_issues(result, ["error", "fail", "runtime error"]) + error_message = "\n".join([f"{name}: {result}" for name, result in zip(node_names, node_results)]) + raise AirflowException(f"dbt invocation completed with errors: {error_message}") @provide_session def store_compiled_sql(self, tmp_project_dir: str, context: Context, session: Session = NEW_SESSION) -> None: @@ -166,6 +228,8 @@ def store_compiled_sql(self, tmp_project_dir: str, context: Context, session: Se ti = context["ti"] if isinstance(ti, TaskInstance): # verifies ti is a TaskInstance in order to access and use the "task" field + if TYPE_CHECKING: + assert ti.task is not None ti.task.template_fields = self.template_fields rtif = RenderedTaskInstanceFields(ti, render_templates=False) @@ -179,44 +243,74 @@ def store_compiled_sql(self, tmp_project_dir: str, context: Context, session: Se else: logger.info("Warning: ti is of type TaskInstancePydantic. Cannot update template_fields.") - def run_subprocess(self, *args: Any, **kwargs: Any) -> FullOutputSubprocessResult: - subprocess_result: FullOutputSubprocessResult = self.subprocess_hook.run_command(*args, **kwargs) + def run_subprocess(self, command: list[str], env: dict[str, str], cwd: str) -> FullOutputSubprocessResult: + logger.info("Trying to run the command:\n %s\nFrom %s", command, cwd) + subprocess_result: FullOutputSubprocessResult = self.subprocess_hook.run_command( + command=command, + env=env, + cwd=cwd, + output_encoding=self.output_encoding, + ) + logger.info(subprocess_result.output) return subprocess_result + def run_dbt_runner(self, command: list[str], env: dict[str, str], cwd: str) -> dbtRunnerResult: + """Invokes the dbt command programmatically.""" + try: + from dbt.cli.main import dbtRunner + except ImportError: + raise ImportError( + "Could not import dbt core. Ensure that dbt-core >= v1.5 is installed and available in the environment where the operator is running." + ) + + if self._dbt_runner is None: + self._dbt_runner = dbtRunner() + + # Exclude the dbt executable path from the command + cli_args = command[1:] + logger.info("Trying to run dbtRunner with:\n %s\n in %s", cli_args, cwd) + + with change_working_directory(cwd), environ(env): + result = self._dbt_runner.invoke(cli_args) + + return result + def run_command( self, cmd: list[str], env: dict[str, str | bytes | os.PathLike[Any]], context: Context, - ) -> FullOutputSubprocessResult: + ) -> FullOutputSubprocessResult | dbtRunnerResult: """ Copies the dbt project to a temporary directory and runs the command. """ - with tempfile.TemporaryDirectory() as tmp_dir: + if not self.invocation_mode: + self._discover_invocation_mode() + + with tempfile.TemporaryDirectory() as tmp_project_dir: + logger.info( "Cloning project to writable temp directory %s from %s", - tmp_dir, - self.project_dir, - ) - - # need a subfolder because shutil.copytree will fail if the destination dir already exists - tmp_project_dir = os.path.join(tmp_dir, "dbt_project") - shutil.copytree( - self.project_dir, tmp_project_dir, + self.project_dir, ) - - # if we need to install deps, do so - if self.install_deps: - self.run_subprocess( - command=[self.dbt_executable_path, "deps"], - env=env, - output_encoding=self.output_encoding, - cwd=tmp_project_dir, - ) - with self.profile_config.ensure_profile() as (profile_path, env_vars): + tmp_dir_path = Path(tmp_project_dir) + env = {k: str(v) for k, v in env.items()} + create_symlinks(Path(self.project_dir), tmp_dir_path, self.install_deps) + + if self.partial_parse and self.cache_dir is not None: + latest_partial_parse = cache._get_latest_partial_parse(Path(self.project_dir), self.cache_dir) + logger.info("Partial parse is enabled and the latest partial parse file is %s", latest_partial_parse) + if latest_partial_parse is not None: + cache._copy_partial_parse_to_project(latest_partial_parse, tmp_dir_path) + + with self.profile_config.ensure_profile() as profile_values: + (profile_path, env_vars) = profile_values env.update(env_vars) - full_cmd = cmd + [ + + flags = [ + "--project-dir", + str(tmp_project_dir), "--profiles-dir", str(profile_path.parent), "--profile", @@ -225,16 +319,26 @@ def run_command( self.profile_config.target_name, ] - logger.info("Trying to run the command:\n %s\nFrom %s", full_cmd, tmp_project_dir) - logger.info("Using environment variables keys: %s", env.keys()) - result = self.run_subprocess( + if self.install_deps: + deps_command = [self.dbt_executable_path, "deps"] + deps_command.extend(flags) + self.invoke_dbt( + command=deps_command, + env=env, + cwd=tmp_project_dir, + ) + + full_cmd = cmd + flags + + logger.debug("Using environment variables keys: %s", env.keys()) + + result = self.invoke_dbt( command=full_cmd, env=env, - output_encoding=self.output_encoding, cwd=tmp_project_dir, ) if is_openlineage_available: - self.calculate_openlineage_events_completes(env, Path(tmp_project_dir)) + self.calculate_openlineage_events_completes(env, tmp_dir_path) context[ "task_instance" ].openlineage_events_completes = self.openlineage_events_completes # type: ignore @@ -246,8 +350,13 @@ def run_command( logger.info("Outlets: %s", outlets) self.register_dataset(inlets, outlets) - self.exception_handling(result) + if self.partial_parse and self.cache_dir: + partial_parse_file = get_partial_parse_path(tmp_dir_path) + if partial_parse_file.exists(): + cache._update_partial_parse_cache(partial_parse_file, self.cache_dir) + self.store_compiled_sql(tmp_project_dir, context) + self.handle_exception(result) if self.callback: self.callback(tmp_project_dir) @@ -302,7 +411,22 @@ def get_datasets(self, source: Literal["inputs", "outputs"]) -> list[Dataset]: for output in getattr(completed, source): dataset_uri = output.namespace + "/" + output.name uris.append(dataset_uri) - return [Dataset(uri) for uri in uris] + logger.debug("URIs to be converted to Dataset: %s", uris) + + datasets = [] + try: + datasets = [Dataset(uri) for uri in uris] + except ValueError: + raise AirflowCompatibilityError( + """ + Apache Airflow 2.9.0 & 2.9.1 introduced a breaking change in Dataset URIs, to be fixed in newer versions: + https://github.com/apache/airflow/issues/39486 + + If you want to use Cosmos with one of these Airflow versions, you will have to disable emission of Datasets: + By setting ``emit_datasets=False`` in ``RenderConfig``. For more information, see https://astronomer.github.io/astronomer-cosmos/configuration/render-config.html. + """ + ) + return datasets def register_dataset(self, new_inlets: list[Dataset], new_outlets: list[Dataset]) -> None: """ @@ -357,119 +481,76 @@ def get_openlineage_facets_on_complete(self, task_instance: TaskInstance) -> Ope job_facets=job_facets, ) - def build_and_run_cmd(self, context: Context, cmd_flags: list[str] | None = None) -> FullOutputSubprocessResult: + def build_and_run_cmd( + self, context: Context, cmd_flags: list[str] | None = None + ) -> FullOutputSubprocessResult | dbtRunnerResult: dbt_cmd, env = self.build_cmd(context=context, cmd_flags=cmd_flags) dbt_cmd = dbt_cmd or [] result = self.run_command(cmd=dbt_cmd, env=env, context=context) - logger.info(result.output) return result - def execute(self, context: Context) -> None: - self.build_and_run_cmd(context=context) - def on_kill(self) -> None: - if self.cancel_query_on_kill: - self.subprocess_hook.log.info("Sending SIGINT signal to process group") - if self.subprocess_hook.sub_process and hasattr(self.subprocess_hook.sub_process, "pid"): - os.killpg(os.getpgid(self.subprocess_hook.sub_process.pid), signal.SIGINT) - else: - self.subprocess_hook.send_sigterm() + if self.invocation_mode == InvocationMode.SUBPROCESS: + if self.cancel_query_on_kill: + self.subprocess_hook.send_sigint() + else: + self.subprocess_hook.send_sigterm() -class DbtLSLocalOperator(DbtLocalBaseOperator): +class DbtBuildLocalOperator(DbtBuildMixin, DbtLocalBaseOperator): """ - Executes a dbt core ls command. + Executes a dbt core build command. """ - ui_color = "#DBCDF6" - - def __init__(self, **kwargs: Any) -> None: - super().__init__(**kwargs) - self.base_cmd = ["ls"] + template_fields: Sequence[str] = DbtLocalBaseOperator.template_fields + DbtBuildMixin.template_fields # type: ignore[operator] -class DbtSeedLocalOperator(DbtLocalBaseOperator): +class DbtLSLocalOperator(DbtLSMixin, DbtLocalBaseOperator): """ - Executes a dbt core seed command. - - :param full_refresh: dbt optional arg - dbt will treat incremental models as table models + Executes a dbt core ls command. """ - ui_color = "#F58D7E" - def __init__(self, full_refresh: bool = False, **kwargs: Any) -> None: - self.full_refresh = full_refresh - super().__init__(**kwargs) - self.base_cmd = ["seed"] - - def add_cmd_flags(self) -> list[str]: - flags = [] - if self.full_refresh is True: - flags.append("--full-refresh") - - return flags +class DbtSeedLocalOperator(DbtSeedMixin, DbtLocalBaseOperator): + """ + Executes a dbt core seed command. + """ - def execute(self, context: Context) -> None: - cmd_flags = self.add_cmd_flags() - self.build_and_run_cmd(context=context, cmd_flags=cmd_flags) + template_fields: Sequence[str] = DbtLocalBaseOperator.template_fields + DbtSeedMixin.template_fields # type: ignore[operator] -class DbtSnapshotLocalOperator(DbtLocalBaseOperator): +class DbtSnapshotLocalOperator(DbtSnapshotMixin, DbtLocalBaseOperator): """ Executes a dbt core snapshot command. - """ - ui_color = "#964B00" - - def __init__(self, **kwargs: Any) -> None: - super().__init__(**kwargs) - self.base_cmd = ["snapshot"] - -class DbtRunLocalOperator(DbtLocalBaseOperator): +class DbtRunLocalOperator(DbtRunMixin, DbtLocalBaseOperator): """ Executes a dbt core run command. """ - ui_color = "#7352BA" - ui_fgcolor = "#F4F2FC" - - def __init__(self, full_refresh: bool = False, **kwargs: Any) -> None: - self.full_refresh = full_refresh - super().__init__(**kwargs) - self.base_cmd = ["run"] - - def add_cmd_flags(self) -> list[str]: - flags = [] - if self.full_refresh is True: - flags.append("--full-refresh") - return flags - - def execute(self, context: Context) -> None: - cmd_flags = self.add_cmd_flags() - self.build_and_run_cmd(context=context, cmd_flags=cmd_flags) + template_fields: Sequence[str] = DbtLocalBaseOperator.template_fields + DbtRunMixin.template_fields # type: ignore[operator] -class DbtTestLocalOperator(DbtLocalBaseOperator): +class DbtTestLocalOperator(DbtTestMixin, DbtLocalBaseOperator): """ Executes a dbt core test command. :param on_warning_callback: A callback function called on warnings with additional Context variables "test_names" and "test_results" of type `List`. Each index in "test_names" corresponds to the same index in "test_results". """ - ui_color = "#8194E0" - def __init__( self, on_warning_callback: Callable[..., Any] | None = None, **kwargs: Any, ) -> None: super().__init__(**kwargs) - self.base_cmd = ["test"] self.on_warning_callback = on_warning_callback + self.extract_issues: Callable[..., tuple[list[str], list[str]]] + self.parse_number_of_warnings: Callable[..., int] - def _handle_warnings(self, result: FullOutputSubprocessResult, context: Context) -> None: + def _handle_warnings(self, result: FullOutputSubprocessResult | dbtRunnerResult, context: Context) -> None: """ Handles warnings by extracting log issues, creating additional context, and calling the on_warning_callback with the updated context. @@ -477,7 +558,7 @@ def _handle_warnings(self, result: FullOutputSubprocessResult, context: Context) :param result: The result object from the build and run command. :param context: The original airflow context in which the build and run command was executed. """ - test_names, test_results = extract_log_issues(result.full_output) + test_names, test_results = self.extract_issues(result) warning_context = dict(context) warning_context["test_names"] = test_names @@ -485,22 +566,24 @@ def _handle_warnings(self, result: FullOutputSubprocessResult, context: Context) self.on_warning_callback and self.on_warning_callback(warning_context) + def _set_test_result_parsing_methods(self) -> None: + """Sets the extract_issues and parse_number_of_warnings methods based on the invocation mode.""" + if self.invocation_mode == InvocationMode.SUBPROCESS: + self.extract_issues = lambda result: extract_log_issues(result.full_output) + self.parse_number_of_warnings = parse_number_of_warnings_subprocess + elif self.invocation_mode == InvocationMode.DBT_RUNNER: + self.extract_issues = extract_dbt_runner_issues + self.parse_number_of_warnings = parse_number_of_warnings_dbt_runner + def execute(self, context: Context) -> None: - result = self.build_and_run_cmd(context=context) - should_trigger_callback = all( - [ - self.on_warning_callback, - DBT_NO_TESTS_MSG not in result.output, - DBT_WARN_MSG in result.output, - ] - ) - if should_trigger_callback: - warnings = parse_output(result, "WARN") - if warnings > 0: - self._handle_warnings(result, context) + result = self.build_and_run_cmd(context=context, cmd_flags=self.add_cmd_flags()) + self._set_test_result_parsing_methods() + number_of_warnings = self.parse_number_of_warnings(result) # type: ignore + if self.on_warning_callback and number_of_warnings > 0: + self._handle_warnings(result, context) -class DbtRunOperationLocalOperator(DbtLocalBaseOperator): +class DbtRunOperationLocalOperator(DbtRunOperationMixin, DbtLocalBaseOperator): """ Executes a dbt core run-operation command. @@ -509,25 +592,7 @@ class DbtRunOperationLocalOperator(DbtLocalBaseOperator): selected macro. """ - ui_color = "#8194E0" - template_fields: Sequence[str] = ("args",) - - def __init__(self, macro_name: str, args: dict[str, Any] | None = None, **kwargs: Any) -> None: - self.macro_name = macro_name - self.args = args - super().__init__(**kwargs) - self.base_cmd = ["run-operation", macro_name] - - def add_cmd_flags(self) -> list[str]: - flags = [] - if self.args is not None: - flags.append("--args") - flags.append(yaml.dump(self.args)) - return flags - - def execute(self, context: Context) -> None: - cmd_flags = self.add_cmd_flags() - self.build_and_run_cmd(context=context, cmd_flags=cmd_flags) + template_fields: Sequence[str] = DbtLocalBaseOperator.template_fields + DbtRunOperationMixin.template_fields # type: ignore[operator] class DbtDocsLocalOperator(DbtLocalBaseOperator): @@ -537,12 +602,21 @@ class DbtDocsLocalOperator(DbtLocalBaseOperator): """ ui_color = "#8194E0" - - required_files = ["index.html", "manifest.json", "graph.gpickle", "catalog.json"] + required_files = ["index.html", "manifest.json", "catalog.json"] + base_cmd = ["docs", "generate"] def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) - self.base_cmd = ["docs", "generate"] + self.check_static_flag() + + def check_static_flag(self) -> None: + if self.dbt_cmd_flags: + if "--static" in self.dbt_cmd_flags: + # For the --static flag we only upload the generated static_index.html file + self.required_files = ["static_index.html"] + if self.dbt_cmd_global_flags: + if "--no-write-json" in self.dbt_cmd_global_flags and "graph.gpickle" in self.required_files: + self.required_files.remove("graph.gpickle") class DbtDocsCloudLocalOperator(DbtDocsLocalOperator, ABC): @@ -557,7 +631,7 @@ def __init__( folder_dir: str | None = None, **kwargs: Any, ) -> None: - "Initializes the operator." + """Initializes the operator.""" self.connection_id = connection_id self.bucket_name = bucket_name self.folder_dir = folder_dir @@ -574,7 +648,7 @@ def upload_to_cloud_storage(self, project_dir: str) -> None: class DbtDocsS3LocalOperator(DbtDocsCloudLocalOperator): """ - Executes `dbt docs generate` command and upload to S3 storage. Returns the S3 path to the generated documentation. + Executes `dbt docs generate` command and upload to S3 storage. :param connection_id: S3's Airflow connection ID :param bucket_name: S3's bucket name @@ -600,7 +674,7 @@ def __init__( super().__init__(*args, **kwargs) def upload_to_cloud_storage(self, project_dir: str) -> None: - "Uploads the generated documentation to S3." + """Uploads the generated documentation to S3.""" logger.info( 'Attempting to upload generated docs to S3 using S3Hook("%s")', self.connection_id, @@ -618,9 +692,9 @@ def upload_to_cloud_storage(self, project_dir: str) -> None: ) for filename in self.required_files: - logger.info("Uploading %s to %s", filename, f"s3://{self.bucket_name}/{filename}") - key = f"{self.folder_dir}/{filename}" if self.folder_dir else filename + s3_path = f"s3://{self.bucket_name}/{key}" + logger.info("Uploading %s to %s", filename, s3_path) hook.load_file( filename=f"{target_dir}/{filename}", @@ -666,7 +740,7 @@ def __init__( super().__init__(*args, **kwargs) def upload_to_cloud_storage(self, project_dir: str) -> None: - "Uploads the generated documentation to Azure Blob Storage." + """Uploads the generated documentation to Azure Blob Storage.""" logger.info( 'Attempting to upload generated docs to Azure Blob Storage using WasbHook(conn_id="%s")', self.connection_id, @@ -710,7 +784,7 @@ class DbtDocsGCSLocalOperator(DbtDocsCloudLocalOperator): ui_color = "#4772d5" def upload_to_cloud_storage(self, project_dir: str) -> None: - "Uploads the generated documentation to Google Cloud Storage" + """Uploads the generated documentation to Google Cloud Storage""" logger.info( 'Attempting to upload generated docs to Storage using GCSHook(conn_id="%s")', self.connection_id, @@ -742,3 +816,15 @@ def __init__(self, **kwargs: str) -> None: raise DeprecationWarning( "The DbtDepsOperator has been deprecated. " "Please use the `install_deps` flag in dbt_args instead." ) + + +class DbtSourceLocalOperator(DbtLocalBaseOperator): + """ + Executes a dbt source freshness command. + """ + + ui_color = "#34CCEB" + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self.base_cmd = ["source", "freshness"] diff --git a/cosmos/operators/virtualenv.py b/cosmos/operators/virtualenv.py index 4d6338e095..3a58956324 100644 --- a/cosmos/operators/virtualenv.py +++ b/cosmos/operators/virtualenv.py @@ -1,15 +1,16 @@ from __future__ import annotations +from functools import cached_property from pathlib import Path from tempfile import TemporaryDirectory from typing import TYPE_CHECKING, Any -from airflow.compat.functools import cached_property from airflow.utils.python_virtualenv import prepare_virtualenv -from cosmos.hooks.subprocess import FullOutputSubprocessResult +from cosmos.hooks.subprocess import FullOutputSubprocessResult from cosmos.log import get_logger from cosmos.operators.local import ( + DbtBuildLocalOperator, DbtDocsLocalOperator, DbtLocalBaseOperator, DbtLSLocalOperator, @@ -18,6 +19,7 @@ DbtSeedLocalOperator, DbtSnapshotLocalOperator, DbtTestLocalOperator, + DbtSourceLocalOperator, ) if TYPE_CHECKING: @@ -36,6 +38,7 @@ class DbtVirtualenvBaseOperator(DbtLocalBaseOperator): :param py_requirements: If defined, creates a virtual environment with the specified dependencies. Example: ["dbt-postgres==1.5.0"] + :param pip_install_options: Pip options to use when installing Python dependencies. Example: ["--upgrade", "--no-cache-dir"] :param py_system_site_packages: Whether or not all the Python packages from the Airflow instance will be accessible within the virtual environment (if py_requirements argument is specified). Avoid using unless the dbt job requires it. @@ -44,10 +47,12 @@ class DbtVirtualenvBaseOperator(DbtLocalBaseOperator): def __init__( self, py_requirements: list[str] | None = None, + pip_install_options: list[str] | None = None, py_system_site_packages: bool = False, **kwargs: Any, ) -> None: self.py_requirements = py_requirements or [] + self.pip_install_options = pip_install_options or [] self.py_system_site_packages = py_system_site_packages super().__init__(**kwargs) self._venv_tmp_dir: None | TemporaryDirectory[str] = None @@ -60,7 +65,7 @@ def venv_dbt_path( Path to the dbt binary within a Python virtualenv. The first time this property is called, it creates a virtualenv and installs the dependencies based on the - self.py_requirements and self.py_system_site_packages. This value is cached for future calls. + self.py_requirements, self.pip_install_options, and self.py_system_site_packages. This value is cached for future calls. """ # We are reusing the virtualenv directory for all subprocess calls within this task/operator. # For this reason, we are not using contexts at this point. @@ -71,6 +76,7 @@ def venv_dbt_path( python_bin=PY_INTERPRETER, system_site_packages=self.py_system_site_packages, requirements=self.py_requirements, + pip_install_options=self.pip_install_options, ) dbt_binary = Path(py_interpreter).parent / "dbt" cmd_output = self.subprocess_hook.run_command( @@ -84,11 +90,16 @@ def venv_dbt_path( self.log.info("Using dbt version %s available at %s", dbt_version, dbt_binary) return str(dbt_binary) - def run_subprocess(self, *args: Any, command: list[str], **kwargs: Any) -> FullOutputSubprocessResult: + def run_subprocess(self, command: list[str], env: dict[str, str], cwd: str) -> FullOutputSubprocessResult: if self.py_requirements: command[0] = self.venv_dbt_path - subprocess_result: FullOutputSubprocessResult = self.subprocess_hook.run_command(command, *args, **kwargs) + subprocess_result: FullOutputSubprocessResult = self.subprocess_hook.run_command( + command=command, + env=env, + cwd=cwd, + output_encoding=self.output_encoding, + ) return subprocess_result def execute(self, context: Context) -> None: @@ -98,6 +109,13 @@ def execute(self, context: Context) -> None: logger.info(output) +class DbtBuildVirtualenvOperator(DbtVirtualenvBaseOperator, DbtBuildLocalOperator): # type: ignore[misc] + """ + Executes a dbt core build command within a Python Virtual Environment, that is created before running the dbt command + and deleted just after. + """ + + class DbtLSVirtualenvOperator(DbtVirtualenvBaseOperator, DbtLSLocalOperator): """ Executes a dbt core ls command within a Python Virtual Environment, that is created before running the dbt command @@ -105,7 +123,7 @@ class DbtLSVirtualenvOperator(DbtVirtualenvBaseOperator, DbtLSLocalOperator): """ -class DbtSeedVirtualenvOperator(DbtVirtualenvBaseOperator, DbtSeedLocalOperator): +class DbtSeedVirtualenvOperator(DbtVirtualenvBaseOperator, DbtSeedLocalOperator): # type: ignore[misc] """ Executes a dbt core seed command within a Python Virtual Environment, that is created before running the dbt command and deleted just after. @@ -119,7 +137,7 @@ class DbtSnapshotVirtualenvOperator(DbtVirtualenvBaseOperator, DbtSnapshotLocalO """ -class DbtRunVirtualenvOperator(DbtVirtualenvBaseOperator, DbtRunLocalOperator): +class DbtRunVirtualenvOperator(DbtVirtualenvBaseOperator, DbtRunLocalOperator): # type: ignore[misc] """ Executes a dbt core run command within a Python Virtual Environment, that is created before running the dbt command and deleted just after. @@ -133,7 +151,7 @@ class DbtTestVirtualenvOperator(DbtVirtualenvBaseOperator, DbtTestLocalOperator) """ -class DbtRunOperationVirtualenvOperator(DbtVirtualenvBaseOperator, DbtRunOperationLocalOperator): +class DbtRunOperationVirtualenvOperator(DbtVirtualenvBaseOperator, DbtRunOperationLocalOperator): # type: ignore[misc] """ Executes a dbt core run-operation command within a Python Virtual Environment, that is created before running the dbt command and deleted just after. @@ -145,3 +163,10 @@ class DbtDocsVirtualenvOperator(DbtVirtualenvBaseOperator, DbtDocsLocalOperator) Executes `dbt docs generate` command within a Python Virtual Environment, that is created before running the dbt command and deleted just after. """ + + +class DbtSourceVirtualenvOperator(DbtVirtualenvBaseOperator, DbtSourceLocalOperator): + """ + Executes `dbt source freshness` command within a Python Virtual Environment, that is created before running the dbt + command and deleted just after. + """ diff --git a/cosmos/plugin/__init__.py b/cosmos/plugin/__init__.py new file mode 100644 index 0000000000..ff4d8e8175 --- /dev/null +++ b/cosmos/plugin/__init__.py @@ -0,0 +1,254 @@ +import os.path as op +from typing import Any, Dict, Optional, Tuple +from urllib.parse import urlsplit + +from airflow.plugins_manager import AirflowPlugin +from airflow.security import permissions +from airflow.www.auth import has_access +from airflow.www.views import AirflowBaseView +from flask import abort, url_for +from flask_appbuilder import AppBuilder, expose + +from cosmos.settings import dbt_docs_conn_id, dbt_docs_dir, dbt_docs_index_file_name + + +def bucket_and_key(path: str) -> Tuple[str, str]: + parsed_url = urlsplit(path) + bucket = parsed_url.netloc + key = parsed_url.path.lstrip("/") + return bucket, key + + +def open_s3_file(path: str, conn_id: Optional[str]) -> str: + from airflow.providers.amazon.aws.hooks.s3 import S3Hook + from botocore.exceptions import ClientError + + if conn_id is None: + conn_id = S3Hook.default_conn_name + + hook = S3Hook(aws_conn_id=conn_id) + bucket, key = bucket_and_key(path) + try: + content = hook.read_key(key=key, bucket_name=bucket) + except ClientError as e: + if e.response.get("Error", {}).get("Code", "") == "NoSuchKey": + raise FileNotFoundError(f"{path} does not exist") + raise e + return content # type: ignore[no-any-return] + + +def open_gcs_file(path: str, conn_id: Optional[str]) -> str: + from airflow.providers.google.cloud.hooks.gcs import GCSHook + from google.cloud.exceptions import NotFound + + if conn_id is None: + conn_id = GCSHook.default_conn_name + + hook = GCSHook(gcp_conn_id=conn_id) + bucket, blob = bucket_and_key(path) + try: + content = hook.download(bucket_name=bucket, object_name=blob) + except NotFound: + raise FileNotFoundError(f"{path} does not exist") + return content.decode("utf-8") # type: ignore[no-any-return] + + +def open_azure_file(path: str, conn_id: Optional[str]) -> str: + from airflow.providers.microsoft.azure.hooks.wasb import WasbHook + from azure.core.exceptions import ResourceNotFoundError + + if conn_id is None: + conn_id = WasbHook.default_conn_name + + hook = WasbHook(wasb_conn_id=conn_id) + + container, blob = bucket_and_key(path) + try: + content = hook.read_file(container_name=container, blob_name=blob) + except ResourceNotFoundError: + raise FileNotFoundError(f"{path} does not exist") + return content # type: ignore[no-any-return] + + +def open_http_file(path: str, conn_id: Optional[str]) -> str: + from airflow.providers.http.hooks.http import HttpHook + from requests.exceptions import HTTPError + + if conn_id is None: + conn_id = "" + + hook = HttpHook(method="GET", http_conn_id=conn_id) + try: + res = hook.run(endpoint=path) + hook.check_response(res) + except HTTPError as e: + if str(e).startswith("404"): + raise FileNotFoundError(f"{path} does not exist") + raise e + return res.text # type: ignore[no-any-return] + + +def open_file(path: str, conn_id: Optional[str] = None) -> str: + """ + Retrieve a file from http, https, gs, s3, or wasb. + + Raise a (base Python) FileNotFoundError if the file is not found. + """ + if path.strip().startswith("s3://"): + return open_s3_file(path, conn_id=conn_id) + elif path.strip().startswith("gs://"): + return open_gcs_file(path, conn_id=conn_id) + elif path.strip().startswith("wasb://"): + return open_azure_file(path, conn_id=conn_id) + elif path.strip().startswith("http://") or path.strip().startswith("https://"): + return open_http_file(path, conn_id=conn_id) + else: + with open(path) as f: + content = f.read() + return content # type: ignore[no-any-return] + + +iframe_script = """ + +""" + + +class DbtDocsView(AirflowBaseView): + default_view = "dbt_docs" + route_base = "/cosmos" + template_folder = op.join(op.dirname(__file__), "templates") + static_folder = op.join(op.dirname(__file__), "static") + + def create_blueprint( + self, appbuilder: AppBuilder, endpoint: Optional[str] = None, static_folder: Optional[str] = None + ) -> None: + # Make sure the static folder is not overwritten, as we want to use it. + return super().create_blueprint(appbuilder, endpoint=endpoint, static_folder=self.static_folder) # type: ignore[no-any-return] + + @expose("/dbt_docs") # type: ignore[misc] + @has_access([(permissions.ACTION_CAN_READ, permissions.RESOURCE_WEBSITE)]) + def dbt_docs(self) -> str: + if dbt_docs_dir is None: + return self.render_template("dbt_docs_not_set_up.html") # type: ignore[no-any-return,no-untyped-call] + return self.render_template("dbt_docs.html") # type: ignore[no-any-return,no-untyped-call] + + @expose("/dbt_docs_index.html") # type: ignore[misc] + @has_access([(permissions.ACTION_CAN_READ, permissions.RESOURCE_WEBSITE)]) + def dbt_docs_index(self) -> Tuple[str, int, Dict[str, Any]]: + if dbt_docs_dir is None: + abort(404) + try: + html = open_file(op.join(dbt_docs_dir, dbt_docs_index_file_name), conn_id=dbt_docs_conn_id) + except FileNotFoundError: + abort(404) + else: + # Hack the dbt docs to render properly in an iframe + iframe_resizer_url = url_for(".static", filename="iframeResizer.contentWindow.min.js") + html = html.replace("", f'{iframe_script}', 1) + return html, 200, {"Content-Security-Policy": "frame-ancestors 'self'"} + + @expose("/catalog.json") # type: ignore[misc] + @has_access([(permissions.ACTION_CAN_READ, permissions.RESOURCE_WEBSITE)]) + def catalog(self) -> Tuple[str, int, Dict[str, Any]]: + if dbt_docs_dir is None: + abort(404) + try: + data = open_file(op.join(dbt_docs_dir, "catalog.json"), conn_id=dbt_docs_conn_id) + except FileNotFoundError: + abort(404) + else: + return data, 200, {"Content-Type": "application/json"} + + @expose("/manifest.json") # type: ignore[misc] + @has_access([(permissions.ACTION_CAN_READ, permissions.RESOURCE_WEBSITE)]) + def manifest(self) -> Tuple[str, int, Dict[str, Any]]: + if dbt_docs_dir is None: + abort(404) + try: + data = open_file(op.join(dbt_docs_dir, "manifest.json"), conn_id=dbt_docs_conn_id) + except FileNotFoundError: + abort(404) + else: + return data, 200, {"Content-Type": "application/json"} + + +dbt_docs_view = DbtDocsView() + + +class CosmosPlugin(AirflowPlugin): + name = "cosmos" + appbuilder_views = [{"name": "dbt Docs", "category": "Browse", "view": dbt_docs_view}] diff --git a/cosmos/plugin/static/iframeResizer.contentWindow.min.js b/cosmos/plugin/static/iframeResizer.contentWindow.min.js new file mode 100644 index 0000000000..914161c09f --- /dev/null +++ b/cosmos/plugin/static/iframeResizer.contentWindow.min.js @@ -0,0 +1,9 @@ +/*! iFrame Resizer (iframeSizer.contentWindow.min.js) - v4.3.5 - 2023-03-08 + * Desc: Include this file in any page being loaded into an iframe + * to force the iframe to resize to the content size. + * Requires: iframeResizer.min.js on host page. + * Copyright: (c) 2023 David J. Bradshaw - dave@bradshaw.net + * License: MIT + */ +!function(a){if("undefined"!=typeof window){var r=!0,P="",u=0,c="",s=null,D="",d=!1,j={resize:1,click:1},l=128,q=!0,f=1,n="bodyOffset",m=n,H=!0,W="",h={},g=32,B=null,p=!1,v=!1,y="[iFrameSizer]",J=y.length,w="",U={max:1,min:1,bodyScroll:1,documentElementScroll:1},b="child",V=!0,X=window.parent,T="*",E=0,i=!1,Y=null,O=16,S=1,K="scroll",M=K,Q=window,G=function(){x("onMessage function not defined")},Z=function(){},$=function(){},_={height:function(){return x("Custom height calculation function not defined"),document.documentElement.offsetHeight},width:function(){return x("Custom width calculation function not defined"),document.body.scrollWidth}},ee={},te=!1;try{var ne=Object.create({},{passive:{get:function(){te=!0}}});window.addEventListener("test",ae,ne),window.removeEventListener("test",ae,ne)}catch(e){}var oe,o,I,ie,N,A,C={bodyOffset:function(){return document.body.offsetHeight+ye("marginTop")+ye("marginBottom")},offset:function(){return C.bodyOffset()},bodyScroll:function(){return document.body.scrollHeight},custom:function(){return _.height()},documentElementOffset:function(){return document.documentElement.offsetHeight},documentElementScroll:function(){return document.documentElement.scrollHeight},max:function(){return Math.max.apply(null,e(C))},min:function(){return Math.min.apply(null,e(C))},grow:function(){return C.max()},lowestElement:function(){return Math.max(C.bodyOffset()||C.documentElementOffset(),we("bottom",Te()))},taggedElement:function(){return be("bottom","data-iframe-height")}},z={bodyScroll:function(){return document.body.scrollWidth},bodyOffset:function(){return document.body.offsetWidth},custom:function(){return _.width()},documentElementScroll:function(){return document.documentElement.scrollWidth},documentElementOffset:function(){return document.documentElement.offsetWidth},scroll:function(){return Math.max(z.bodyScroll(),z.documentElementScroll())},max:function(){return Math.max.apply(null,e(z))},min:function(){return Math.min.apply(null,e(z))},rightMostElement:function(){return we("right",Te())},taggedElement:function(){return be("right","data-iframe-width")}},re=(oe=Ee,N=null,A=0,function(){var e=Date.now(),t=O-(e-(A=A||e));return o=this,I=arguments,t<=0||Ok[r]["max"+e])throw new Error("Value for min"+e+" can not be greater than max"+e)}}function h(e,n){null===i&&(i=setTimeout(function(){i=null,e()},n))}function e(){"hidden"!==document.visibilityState&&(O("document","Trigger event: Visibility change"),h(function(){b("Tab Visible","resize")},16))}function b(i,t){Object.keys(k).forEach(function(e){var n;k[n=e]&&"parent"===k[n].resizeFrom&&k[n].autoResize&&!k[n].firstRun&&A(i,t,k[e].iframe,e)})}function y(){F(window,"message",w),F(window,"resize",function(){var e;O("window","Trigger event: "+(e="resize")),h(function(){b("Window "+e,"resize")},16)}),F(document,"visibilitychange",e),F(document,"-webkit-visibilitychange",e)}function n(){function t(e,n){if(n){if(!n.tagName)throw new TypeError("Object is not a valid DOM element");if("IFRAME"!==n.tagName.toUpperCase())throw new TypeError("Expected + +{% endblock %} diff --git a/cosmos/plugin/templates/dbt_docs_not_set_up.html b/cosmos/plugin/templates/dbt_docs_not_set_up.html new file mode 100644 index 0000000000..1fcc6ef7f3 --- /dev/null +++ b/cosmos/plugin/templates/dbt_docs_not_set_up.html @@ -0,0 +1,9 @@ +{% extends base_template %} +{% block content %} +

⚠️ Your dbt docs are not set up yet! ⚠️

+ +

+ Read the Astronomer Cosmos docs for information on how to set up dbt docs. +

+ +{% endblock %} diff --git a/cosmos/profiles/__init__.py b/cosmos/profiles/__init__.py index 47c7309abc..4877e44505 100644 --- a/cosmos/profiles/__init__.py +++ b/cosmos/profiles/__init__.py @@ -1,23 +1,27 @@ -"Contains a function to get the profile mapping based on the connection ID." +"""Contains a function to get the profile mapping based on the connection ID.""" from __future__ import annotations from typing import Any, Type - from .athena import AthenaAccessKeyProfileMapping -from .base import BaseProfileMapping +from .base import BaseProfileMapping, DbtProfileConfigVars +from .bigquery.oauth import GoogleCloudOauthProfileMapping from .bigquery.service_account_file import GoogleCloudServiceAccountFileProfileMapping from .bigquery.service_account_keyfile_dict import GoogleCloudServiceAccountDictProfileMapping -from .bigquery.oauth import GoogleCloudOauthProfileMapping +from .clickhouse.user_pass import ClickhouseUserPasswordProfileMapping from .databricks.token import DatabricksTokenProfileMapping from .exasol.user_pass import ExasolUserPasswordProfileMapping from .postgres.user_pass import PostgresUserPasswordProfileMapping from .redshift.user_pass import RedshiftUserPasswordProfileMapping +from .snowflake.user_encrypted_privatekey_env_variable import SnowflakeEncryptedPrivateKeyPemProfileMapping +from .snowflake.user_encrypted_privatekey_file import SnowflakeEncryptedPrivateKeyFilePemProfileMapping from .snowflake.user_pass import SnowflakeUserPasswordProfileMapping from .snowflake.user_privatekey import SnowflakePrivateKeyPemProfileMapping -from .snowflake.user_encrypted_privatekey import SnowflakeEncryptedPrivateKeyPemProfileMapping +from .snowflake.user_encrypted_privatekey_file import SnowflakeEncryptedPrivateKeyFilePemProfileMapping +from .snowflake.user_encrypted_privatekey_env_variable import SnowflakeEncryptedPrivateKeyPemProfileMapping from .spark.thrift import SparkThriftProfileMapping +from .teradata.user_pass import TeradataUserPasswordProfileMapping from .trino.certificate import TrinoCertificateProfileMapping from .trino.jwt import TrinoJWTProfileMapping from .trino.ldap import TrinoLDAPProfileMapping @@ -25,6 +29,7 @@ profile_mappings: list[Type[BaseProfileMapping]] = [ AthenaAccessKeyProfileMapping, + ClickhouseUserPasswordProfileMapping, GoogleCloudServiceAccountFileProfileMapping, GoogleCloudServiceAccountDictProfileMapping, GoogleCloudOauthProfileMapping, @@ -32,10 +37,12 @@ PostgresUserPasswordProfileMapping, RedshiftUserPasswordProfileMapping, SnowflakeUserPasswordProfileMapping, + SnowflakeEncryptedPrivateKeyFilePemProfileMapping, SnowflakeEncryptedPrivateKeyPemProfileMapping, SnowflakePrivateKeyPemProfileMapping, SparkThriftProfileMapping, ExasolUserPasswordProfileMapping, + TeradataUserPasswordProfileMapping, TrinoLDAPProfileMapping, TrinoCertificateProfileMapping, TrinoJWTProfileMapping, @@ -62,17 +69,21 @@ def get_automatic_profile_mapping( __all__ = [ + "AthenaAccessKeyProfileMapping", "BaseProfileMapping", "GoogleCloudServiceAccountFileProfileMapping", "GoogleCloudServiceAccountDictProfileMapping", "GoogleCloudOauthProfileMapping", "DatabricksTokenProfileMapping", + "DbtProfileConfigVars", "PostgresUserPasswordProfileMapping", "RedshiftUserPasswordProfileMapping", "SnowflakeUserPasswordProfileMapping", "SnowflakePrivateKeyPemProfileMapping", + "SnowflakeEncryptedPrivateKeyFilePemProfileMapping", "SparkThriftProfileMapping", "ExasolUserPasswordProfileMapping", + "TeradataUserPasswordProfileMapping", "TrinoLDAPProfileMapping", "TrinoCertificateProfileMapping", "TrinoJWTProfileMapping", diff --git a/cosmos/profiles/athena/__init__.py b/cosmos/profiles/athena/__init__.py index 0cbb09a7c3..3be305ace6 100644 --- a/cosmos/profiles/athena/__init__.py +++ b/cosmos/profiles/athena/__init__.py @@ -1,4 +1,4 @@ -"Athena Airflow connection -> dbt profile mappings" +"""Athena Airflow connection -> dbt profile mappings""" from .access_key import AthenaAccessKeyProfileMapping diff --git a/cosmos/profiles/athena/access_key.py b/cosmos/profiles/athena/access_key.py index b79bb793ab..05c2ec0998 100644 --- a/cosmos/profiles/athena/access_key.py +++ b/cosmos/profiles/athena/access_key.py @@ -1,22 +1,36 @@ -"Maps Airflow AWS connections to a dbt Athena profile using an access key id and secret access key." +"""Maps Airflow AWS connections to a dbt Athena profile using an access key id and secret access key.""" + from __future__ import annotations from typing import Any +from cosmos.exceptions import CosmosValueError + from ..base import BaseProfileMapping class AthenaAccessKeyProfileMapping(BaseProfileMapping): """ - Maps Airflow AWS connections to a dbt Athena profile using an access key id and secret access key. + Uses the Airflow AWS Connection provided to get_credentials() to generate the profile for dbt. - https://docs.getdbt.com/docs/core/connect-data-platform/athena-setup https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/connections/aws.html + + + This behaves similarly to other provider operators such as the AWS Athena Operator. + Where you pass the aws_conn_id and the operator will generate the credentials for you. + + https://registry.astronomer.io/providers/amazon/versions/latest/modules/athenaoperator + + Information about the dbt Athena profile that is generated can be found here: + + https://github.com/dbt-athena/dbt-athena?tab=readme-ov-file#configuring-your-profile + https://docs.getdbt.com/docs/core/connect-data-platform/athena-setup """ airflow_connection_type: str = "aws" dbt_profile_type: str = "athena" is_community: bool = True + temporary_credentials = None required_fields = [ "aws_access_key_id", @@ -26,12 +40,7 @@ class AthenaAccessKeyProfileMapping(BaseProfileMapping): "s3_staging_dir", "schema", ] - secret_fields = [ - "aws_secret_access_key", - ] airflow_param_mapping = { - "aws_access_key_id": "login", - "aws_secret_access_key": "password", "aws_profile_name": "extra.aws_profile_name", "database": "extra.database", "debug_query_state": "extra.debug_query_state", @@ -49,11 +58,46 @@ class AthenaAccessKeyProfileMapping(BaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Gets profile. The password is stored in an environment variable." + """Gets profile. The password is stored in an environment variable.""" + + self.temporary_credentials = self._get_temporary_credentials() # type: ignore + profile = { **self.mapped_params, **self.profile_args, - # aws_secret_access_key should always get set as env var + "aws_access_key_id": self.temporary_credentials.access_key, "aws_secret_access_key": self.get_env_var_format("aws_secret_access_key"), } + + if self.temporary_credentials.token: + profile["aws_session_token"] = self.get_env_var_format("aws_session_token") + return self.filter_null(profile) + + @property + def env_vars(self) -> dict[str, str]: + """Overwrites the env_vars for athena, Returns a dictionary of environment variables that should be set based on the self.temporary_credentials.""" + + if self.temporary_credentials is None: + raise CosmosValueError(f"Could not find the athena credentials.") + + env_vars = {} + + env_secret_key_name = self.get_env_var_name("aws_secret_access_key") + env_session_token_name = self.get_env_var_name("aws_session_token") + + env_vars[env_secret_key_name] = str(self.temporary_credentials.secret_key) + env_vars[env_session_token_name] = str(self.temporary_credentials.token) + + return env_vars + + def _get_temporary_credentials(self): # type: ignore + """ + Helper function to retrieve temporary short lived credentials + Returns an object including access_key, secret_key and token + """ + from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook + + hook = AwsGenericHook(self.conn_id) # type: ignore + credentials = hook.get_credentials() + return credentials diff --git a/cosmos/profiles/base.py b/cosmos/profiles/base.py old mode 100644 new mode 100755 index 36f2ccbc54..a81512dbb0 --- a/cosmos/profiles/base.py +++ b/cosmos/profiles/base.py @@ -2,15 +2,18 @@ This module contains a base class that other profile mappings should inherit from to ensure consistency. """ + from __future__ import annotations +import hashlib +import json +import warnings from abc import ABC, abstractmethod -from typing import Any +from typing import TYPE_CHECKING, Any, Dict, Literal, Optional -from typing import TYPE_CHECKING import yaml - from airflow.hooks.base import BaseHook +from pydantic import dataclasses from cosmos.exceptions import CosmosValueError from cosmos.log import get_logger @@ -18,9 +21,38 @@ if TYPE_CHECKING: from airflow.models import Connection +DBT_PROFILE_TYPE_FIELD = "type" +DBT_PROFILE_METHOD_FIELD = "method" + logger = get_logger(__name__) +@dataclasses.dataclass +class DbtProfileConfigVars: + send_anonymous_usage_stats: Optional[bool] = False + partial_parse: Optional[bool] = None + use_experimental_parser: Optional[bool] = None + static_parser: Optional[bool] = None + printer_width: Optional[bool] = None + write_json: Optional[bool] = None + warn_error: Optional[bool] = None + warn_error_options: Optional[Dict[Literal["include", "exclude"], Any]] = None + log_format: Optional[Literal["text", "json", "default"]] = None + debug: Optional[bool] = None + version_check: Optional[bool] = None + + def as_dict(self) -> dict[str, Any] | None: + result = { + field.name: getattr(self, field.name) + # Look like the __dataclass_fields__ attribute is not recognized by mypy + for field in self.__dataclass_fields__.values() # type: ignore[attr-defined] + if getattr(self, field.name) is not None + } + if result != {}: + return result + return None + + class BaseProfileMapping(ABC): """ A base class that other profile mappings should inherit from to ensure consistency. @@ -38,13 +70,78 @@ class BaseProfileMapping(ABC): _conn: Connection | None = None - def __init__(self, conn_id: str, profile_args: dict[str, Any] | None = None): + def __init__( + self, + conn_id: str, + profile_args: dict[str, Any] | None = None, + disable_event_tracking: bool | None = None, + dbt_config_vars: DbtProfileConfigVars | None = None, + ): self.conn_id = conn_id self.profile_args = profile_args or {} + self._validate_profile_args() + self.disable_event_tracking = disable_event_tracking + self.dbt_config_vars = dbt_config_vars + self._validate_disable_event_tracking() + + def version(self, profile_name: str, target_name: str, mock_profile: bool = False) -> str: + """ + Generate SHA-256 hash digest based on the provided profile, profile and target names. + + :param profile_name: Name of the DBT profile. + :param target_name: Name of the DBT target + :param mock_profile: If True, use a mock profile. + """ + if mock_profile: + profile = self.mock_profile + else: + profile = self.profile + profile["profile_name"] = profile_name + profile["target_name"] = target_name + hash_object = hashlib.sha256(json.dumps(profile, sort_keys=True).encode()) + return hash_object.hexdigest() + + def _validate_profile_args(self) -> None: + """ + Check if profile_args contains keys that should not be overridden from the + class variables when creating the profile. + """ + for profile_field in [DBT_PROFILE_TYPE_FIELD, DBT_PROFILE_METHOD_FIELD]: + if profile_field in self.profile_args and self.profile_args.get(profile_field) != getattr( + self, f"dbt_profile_{profile_field}" + ): + raise CosmosValueError( + "`profile_args` for {0} has {1}='{2}' that will override the dbt profile required value of '{3}'. " + "To fix this, remove {1} from `profile_args`.".format( + self.__class__.__name__, + profile_field, + self.profile_args.get(profile_field), + getattr(self, f"dbt_profile_{profile_field}"), + ) + ) + + def _validate_disable_event_tracking(self) -> None: + """ + Check if disable_event_tracking is set and warn that it is deprecated. + """ + if self.disable_event_tracking: + warnings.warn( + "Disabling dbt event tracking is deprecated since Cosmos 1.3 and will be removed in Cosmos 2.0. " + "Use dbt_config_vars=DbtProfileConfigVars(send_anonymous_usage_stats=False) instead.", + DeprecationWarning, + ) + if ( + isinstance(self.dbt_config_vars, DbtProfileConfigVars) + and self.dbt_config_vars.send_anonymous_usage_stats is not None + ): + raise CosmosValueError( + "Cannot set both disable_event_tracking and " + "dbt_config_vars=DbtProfileConfigVars(send_anonymous_usage_stats ..." + ) @property def conn(self) -> Connection: - "Returns the Airflow connection." + """Returns the Airflow connection.""" if not self._conn: conn = BaseHook.get_connection(self.conn_id) if not conn: @@ -100,11 +197,11 @@ def mock_profile(self) -> dict[str, Any]: where live connection values don't matter. """ mock_profile = { - "type": self.dbt_profile_type, + DBT_PROFILE_TYPE_FIELD: self.dbt_profile_type, } if self.dbt_profile_method: - mock_profile["method"] = self.dbt_profile_method + mock_profile[DBT_PROFILE_METHOD_FIELD] = self.dbt_profile_method for field in self.required_fields: # if someone has passed in a value for this field, use it @@ -119,7 +216,7 @@ def mock_profile(self) -> dict[str, Any]: @property def env_vars(self) -> dict[str, str]: - "Returns a dictionary of environment variables that should be set based on self.secret_fields." + """Returns a dictionary of environment variables that should be set based on self.secret_fields.""" env_vars = {} for field in self.secret_fields: @@ -149,14 +246,33 @@ def get_profile_file_contents( # filter out any null values profile_vars = {k: v for k, v in profile_vars.items() if v is not None} - profile_contents = { + profile_contents: dict[str, Any] = { profile_name: { "target": target_name, "outputs": {target_name: profile_vars}, } } + + if self.dbt_config_vars: + profile_contents["config"] = self.dbt_config_vars.as_dict() + + if self.disable_event_tracking: + profile_contents["config"] = {"send_anonymous_usage_stats": False} + return str(yaml.dump(profile_contents, indent=4)) + def _get_airflow_conn_field(self, airflow_field: str) -> Any: + # make sure there's no "extra." prefix + if airflow_field.startswith("extra."): + airflow_field = airflow_field.replace("extra.", "", 1) + value = self.conn.extra_dejson.get(airflow_field) + elif airflow_field.startswith("extra__"): + value = self.conn.extra_dejson.get(airflow_field) + else: + value = getattr(self.conn, airflow_field, None) + + return value + def get_dbt_value(self, name: str) -> Any: """ Gets values for the dbt profile based on the required_by_dbt and required_in_profile_args lists. @@ -176,16 +292,9 @@ def get_dbt_value(self, name: str) -> Any: airflow_fields = [airflow_fields] for airflow_field in airflow_fields: - # make sure there's no "extra." prefix - if airflow_field.startswith("extra."): - airflow_field = airflow_field.replace("extra.", "", 1) - value = self.conn.extra_dejson.get(airflow_field) - else: - value = getattr(self.conn, airflow_field, None) - + value = self._get_airflow_conn_field(airflow_field) if not value: continue - # if there's a transform method, use it if hasattr(self, f"transform_{name}"): return getattr(self, f"transform_{name}")(value) @@ -197,13 +306,13 @@ def get_dbt_value(self, name: str) -> Any: @property def mapped_params(self) -> dict[str, Any]: - "Turns the self.airflow_param_mapping into a dictionary of dbt fields and their values." + """Turns the self.airflow_param_mapping into a dictionary of dbt fields and their values.""" mapped_params = { - "type": self.dbt_profile_type, + DBT_PROFILE_TYPE_FIELD: self.dbt_profile_type, } if self.dbt_profile_method: - mapped_params["method"] = self.dbt_profile_method + mapped_params[DBT_PROFILE_METHOD_FIELD] = self.dbt_profile_method for dbt_field in self.airflow_param_mapping: mapped_params[dbt_field] = self.get_dbt_value(dbt_field) diff --git a/cosmos/profiles/bigquery/__init__.py b/cosmos/profiles/bigquery/__init__.py index c03f9a0ab0..97217ea9d9 100644 --- a/cosmos/profiles/bigquery/__init__.py +++ b/cosmos/profiles/bigquery/__init__.py @@ -1,8 +1,8 @@ -"BigQuery Airflow connection -> dbt profile mappings" +"""BigQuery Airflow connection -> dbt profile mappings""" +from .oauth import GoogleCloudOauthProfileMapping from .service_account_file import GoogleCloudServiceAccountFileProfileMapping from .service_account_keyfile_dict import GoogleCloudServiceAccountDictProfileMapping -from .oauth import GoogleCloudOauthProfileMapping __all__ = [ "GoogleCloudServiceAccountFileProfileMapping", diff --git a/cosmos/profiles/bigquery/oauth.py b/cosmos/profiles/bigquery/oauth.py index 6ea02cfa4a..f619a28fff 100644 --- a/cosmos/profiles/bigquery/oauth.py +++ b/cosmos/profiles/bigquery/oauth.py @@ -1,4 +1,5 @@ -"Maps Airflow GCP connections to dbt BigQuery profiles that uses oauth via gcloud, if they don't use key file or JSON." +"""Maps Airflow GCP connections to dbt BigQuery profiles that uses oauth via gcloud, if they don't use key file or JSON.""" + from __future__ import annotations from typing import Any @@ -31,7 +32,7 @@ class GoogleCloudOauthProfileMapping(BaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Generates profile. Defaults `threads` to 1." + """Generates profile. Defaults `threads` to 1.""" return { **self.mapped_params, "method": "oauth", @@ -41,7 +42,7 @@ def profile(self) -> dict[str, Any | None]: @property def mock_profile(self) -> dict[str, Any | None]: - "Generates mock profile. Defaults `threads` to 1." + """Generates mock profile. Defaults `threads` to 1.""" parent_mock_profile = super().mock_profile return { diff --git a/cosmos/profiles/bigquery/service_account_file.py b/cosmos/profiles/bigquery/service_account_file.py index 02d00ffae9..fb294a167d 100644 --- a/cosmos/profiles/bigquery/service_account_file.py +++ b/cosmos/profiles/bigquery/service_account_file.py @@ -1,4 +1,5 @@ -"Maps Airflow GCP connections to dbt BigQuery profiles if they use a service account file." +"""Maps Airflow GCP connections to dbt BigQuery profiles if they use a service account file.""" + from __future__ import annotations from typing import Any @@ -32,7 +33,7 @@ class GoogleCloudServiceAccountFileProfileMapping(BaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Generates profile. Defaults `threads` to 1." + """Generates profile. Defaults `threads` to 1.""" return { **self.mapped_params, "threads": 1, @@ -41,7 +42,7 @@ def profile(self) -> dict[str, Any | None]: @property def mock_profile(self) -> dict[str, Any | None]: - "Generates mock profile. Defaults `threads` to 1." + """Generates mock profile. Defaults `threads` to 1.""" parent_mock_profile = super().mock_profile return { diff --git a/cosmos/profiles/bigquery/service_account_keyfile_dict.py b/cosmos/profiles/bigquery/service_account_keyfile_dict.py index e84587faf4..cfa57056b7 100644 --- a/cosmos/profiles/bigquery/service_account_keyfile_dict.py +++ b/cosmos/profiles/bigquery/service_account_keyfile_dict.py @@ -1,10 +1,11 @@ -"Maps Airflow GCP connections to dbt BigQuery profiles if they use a service account keyfile dict/json." +"""Maps Airflow GCP connections to dbt BigQuery profiles if they use a service account keyfile dict/json.""" + from __future__ import annotations -from typing import Any import json -from cosmos.exceptions import CosmosValueError +from typing import Any +from cosmos.exceptions import CosmosValueError from cosmos.profiles.base import BaseProfileMapping @@ -20,6 +21,8 @@ class GoogleCloudServiceAccountDictProfileMapping(BaseProfileMapping): dbt_profile_type: str = "bigquery" dbt_profile_method: str = "service-account-json" + # Do not remove dataset as a required field form the below list. Although it's observed that it's not a required + # field for some databases like Postgres, it's required for BigQuery. required_fields = [ "project", "dataset", @@ -45,15 +48,17 @@ def profile(self) -> dict[str, Any | None]: Even though the Airflow connection contains hard-coded Service account credentials, we generate a temporary file and the DBT profile uses it. """ - return { + profile_dict = { **self.mapped_params, "threads": 1, **self.profile_args, } + return self.filter_null(profile_dict) + @property def mock_profile(self) -> dict[str, Any | None]: - "Generates mock profile. Defaults `threads` to 1." + """Generates mock profile. Defaults `threads` to 1.""" parent_mock_profile = super().mock_profile return {**parent_mock_profile, "threads": 1, "keyfile_json": None} @@ -82,5 +87,5 @@ def transform_keyfile_json(self, keyfile_json: str | dict[str, str]) -> dict[str @property def env_vars(self) -> dict[str, str]: - "Returns a dictionary of environment variables that should be set based on self.secret_fields." + """Returns a dictionary of environment variables that should be set based on self.secret_fields.""" return self._env_vars diff --git a/cosmos/profiles/clickhouse/__init__.py b/cosmos/profiles/clickhouse/__init__.py new file mode 100644 index 0000000000..bd94af5fec --- /dev/null +++ b/cosmos/profiles/clickhouse/__init__.py @@ -0,0 +1,5 @@ +"""Generic Airflow connection -> dbt profile mappings""" + +from .user_pass import ClickhouseUserPasswordProfileMapping + +__all__ = ["ClickhouseUserPasswordProfileMapping"] diff --git a/cosmos/profiles/clickhouse/user_pass.py b/cosmos/profiles/clickhouse/user_pass.py new file mode 100644 index 0000000000..7d168895a2 --- /dev/null +++ b/cosmos/profiles/clickhouse/user_pass.py @@ -0,0 +1,70 @@ +"""Maps Airflow Postgres connections using user + password authentication to dbt profiles.""" + +from __future__ import annotations + +from typing import Any + +from ..base import BaseProfileMapping + + +class ClickhouseUserPasswordProfileMapping(BaseProfileMapping): + """ + Maps Airflow generic connections using user + password authentication to dbt Clickhouse profiles. + https://docs.getdbt.com/docs/core/connect-data-platform/clickhouse-setup + """ + + airflow_connection_type: str = "generic" + dbt_profile_type: str = "clickhouse" + default_port = 9000 + is_community = True + + required_fields = [ + "host", + "login", + "schema", + "clickhouse", + ] + secret_fields = [ + "password", + ] + airflow_param_mapping = { + "host": "host", + "login": "login", + "password": "password", + "port": "port", + "schema": "schema", + "clickhouse": "extra.clickhouse", + } + + def _set_default_param(self, profile_dict: dict[str, Any]) -> dict[str, Any]: + if not profile_dict.get("driver"): + profile_dict["driver"] = "native" + + if not profile_dict.get("port"): + profile_dict["port"] = self.default_port + + if not profile_dict.get("secure"): + profile_dict["secure"] = False + return profile_dict + + @property + def profile(self) -> dict[str, Any | None]: + """Gets profile. The password is stored in an environment variable.""" + profile_dict = { + **self.mapped_params, + **self.profile_args, + # password should always get set as env var + "password": self.get_env_var_format("password"), + } + + return self.filter_null(self._set_default_param(profile_dict)) + + @property + def mock_profile(self) -> dict[str, Any | None]: + """Gets mock profile.""" + + profile_dict = { + **super().mock_profile, + } + + return self._set_default_param(profile_dict) diff --git a/cosmos/profiles/databricks/__init__.py b/cosmos/profiles/databricks/__init__.py index dda537ab1e..2e3a9d1143 100644 --- a/cosmos/profiles/databricks/__init__.py +++ b/cosmos/profiles/databricks/__init__.py @@ -1,4 +1,4 @@ -"Databricks Airflow connection -> dbt profile mappings" +"""Databricks Airflow connection -> dbt profile mappings""" from .token import DatabricksTokenProfileMapping diff --git a/cosmos/profiles/databricks/token.py b/cosmos/profiles/databricks/token.py index cf73713027..78d97eb77c 100644 --- a/cosmos/profiles/databricks/token.py +++ b/cosmos/profiles/databricks/token.py @@ -1,4 +1,5 @@ -"Maps Airflow Databricks connections with a token to dbt profiles." +"""Maps Airflow Databricks connections with a token to dbt profiles.""" + from __future__ import annotations from typing import Any @@ -37,7 +38,7 @@ class DatabricksTokenProfileMapping(BaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Generates profile. The token is stored in an environment variable." + """Generates profile. The token is stored in an environment variable.""" return { **self.mapped_params, **self.profile_args, @@ -46,5 +47,5 @@ def profile(self) -> dict[str, Any | None]: } def transform_host(self, host: str) -> str: - "Removes the https:// prefix." + """Removes the https:// prefix.""" return host.replace("https://", "") diff --git a/cosmos/profiles/exasol/__init__.py b/cosmos/profiles/exasol/__init__.py index 48585d7fc5..17062569db 100644 --- a/cosmos/profiles/exasol/__init__.py +++ b/cosmos/profiles/exasol/__init__.py @@ -1,4 +1,4 @@ -"Exasol Airflow connection -> dbt profile mappings" +"""Exasol Airflow connection -> dbt profile mappings""" from .user_pass import ExasolUserPasswordProfileMapping diff --git a/cosmos/profiles/exasol/user_pass.py b/cosmos/profiles/exasol/user_pass.py index 62311221d8..aa5d66ebf3 100644 --- a/cosmos/profiles/exasol/user_pass.py +++ b/cosmos/profiles/exasol/user_pass.py @@ -1,4 +1,5 @@ -"Maps Airflow Exasol connections with a username and password to dbt profiles." +"""Maps Airflow Exasol connections with a username and password to dbt profiles.""" + from __future__ import annotations from typing import Any @@ -45,7 +46,7 @@ class ExasolUserPasswordProfileMapping(BaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Gets profile. The password is stored in an environment variable." + """Gets profile. The password is stored in an environment variable.""" profile_vars = { **self.mapped_params, **self.profile_args, @@ -57,7 +58,7 @@ def profile(self) -> dict[str, Any | None]: return self.filter_null(profile_vars) def transform_dsn(self, host: str) -> str: - "Adds the port if it's not already there." + """Adds the port if it's not already there.""" if ":" not in host: port = self.conn.port or self.default_port return f"{host}:{port}" diff --git a/cosmos/profiles/postgres/__init__.py b/cosmos/profiles/postgres/__init__.py index 3b9a6c895a..2c07cfd109 100644 --- a/cosmos/profiles/postgres/__init__.py +++ b/cosmos/profiles/postgres/__init__.py @@ -1,4 +1,4 @@ -"Postgres Airflow connection -> dbt profile mappings" +"""Postgres Airflow connection -> dbt profile mappings""" from .user_pass import PostgresUserPasswordProfileMapping diff --git a/cosmos/profiles/postgres/user_pass.py b/cosmos/profiles/postgres/user_pass.py index 731d600794..c204ff8d4b 100644 --- a/cosmos/profiles/postgres/user_pass.py +++ b/cosmos/profiles/postgres/user_pass.py @@ -1,4 +1,5 @@ -"Maps Airflow Postgres connections using user + password authentication to dbt profiles." +"""Maps Airflow Postgres connections using user + password authentication to dbt profiles.""" + from __future__ import annotations from typing import Any @@ -21,7 +22,6 @@ class PostgresUserPasswordProfileMapping(BaseProfileMapping): "user", "password", "dbname", - "schema", ] secret_fields = [ "password", @@ -38,7 +38,7 @@ class PostgresUserPasswordProfileMapping(BaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Gets profile. The password is stored in an environment variable." + """Gets profile. The password is stored in an environment variable.""" profile = { "port": 5432, **self.mapped_params, @@ -47,14 +47,19 @@ def profile(self) -> dict[str, Any | None]: "password": self.get_env_var_format("password"), } + if "schema" in self.profile_args: + profile["schema"] = self.profile_args["schema"] + return self.filter_null(profile) @property def mock_profile(self) -> dict[str, Any | None]: - "Gets mock profile. Defaults port to 5432." - parent_mock = super().mock_profile - - return { + """Gets mock profile. Defaults port to 5432.""" + profile_dict = { "port": 5432, - **parent_mock, + **super().mock_profile, } + user_defined_schema = self.profile_args.get("schema") + if user_defined_schema: + profile_dict["schema"] = user_defined_schema + return profile_dict diff --git a/cosmos/profiles/redshift/__init__.py b/cosmos/profiles/redshift/__init__.py index 3528e058dd..c434850dc7 100644 --- a/cosmos/profiles/redshift/__init__.py +++ b/cosmos/profiles/redshift/__init__.py @@ -1,4 +1,4 @@ -"Redshift Airflow connection -> dbt profile mappings" +"""Redshift Airflow connection -> dbt profile mappings""" from .user_pass import RedshiftUserPasswordProfileMapping diff --git a/cosmos/profiles/redshift/user_pass.py b/cosmos/profiles/redshift/user_pass.py index 0dd3e115f4..adf89c2c6e 100644 --- a/cosmos/profiles/redshift/user_pass.py +++ b/cosmos/profiles/redshift/user_pass.py @@ -1,4 +1,5 @@ -"Maps Airflow Redshift connections to dbt Redshift profiles if they use a username and password." +"""Maps Airflow Redshift connections to dbt Redshift profiles if they use a username and password.""" + from __future__ import annotations from typing import Any @@ -39,7 +40,7 @@ class RedshiftUserPasswordProfileMapping(BaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Gets profile." + """Gets profile.""" profile = { "port": 5439, **self.mapped_params, @@ -52,7 +53,7 @@ def profile(self) -> dict[str, Any | None]: @property def mock_profile(self) -> dict[str, Any | None]: - "Gets mock profile. Defaults port to 5439." + """Gets mock profile. Defaults port to 5439.""" parent_mock = super().mock_profile return { diff --git a/cosmos/profiles/snowflake/__init__.py b/cosmos/profiles/snowflake/__init__.py index 26c3fb5954..ac71f27ac5 100644 --- a/cosmos/profiles/snowflake/__init__.py +++ b/cosmos/profiles/snowflake/__init__.py @@ -1,11 +1,15 @@ -"Snowflake Airflow connection -> dbt profile mapping." +"""Snowflake Airflow connection -> dbt profile mapping.""" +from .user_encrypted_privatekey_env_variable import SnowflakeEncryptedPrivateKeyPemProfileMapping +from .user_encrypted_privatekey_file import SnowflakeEncryptedPrivateKeyFilePemProfileMapping from .user_pass import SnowflakeUserPasswordProfileMapping from .user_privatekey import SnowflakePrivateKeyPemProfileMapping -from .user_encrypted_privatekey import SnowflakeEncryptedPrivateKeyPemProfileMapping +from .user_encrypted_privatekey_file import SnowflakeEncryptedPrivateKeyFilePemProfileMapping +from .user_encrypted_privatekey_env_variable import SnowflakeEncryptedPrivateKeyPemProfileMapping __all__ = [ "SnowflakeUserPasswordProfileMapping", "SnowflakePrivateKeyPemProfileMapping", + "SnowflakeEncryptedPrivateKeyFilePemProfileMapping", "SnowflakeEncryptedPrivateKeyPemProfileMapping", ] diff --git a/cosmos/profiles/snowflake/user_encrypted_privatekey_env_variable.py b/cosmos/profiles/snowflake/user_encrypted_privatekey_env_variable.py new file mode 100644 index 0000000000..70722dd59a --- /dev/null +++ b/cosmos/profiles/snowflake/user_encrypted_privatekey_env_variable.py @@ -0,0 +1,94 @@ +"""Maps Airflow Snowflake connections to dbt profiles if they use a user/private key.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any + +from ..base import BaseProfileMapping + +if TYPE_CHECKING: + from airflow.models import Connection + + +class SnowflakeEncryptedPrivateKeyPemProfileMapping(BaseProfileMapping): + """ + Maps Airflow Snowflake connections to dbt profiles if they use a user/private key. + https://docs.getdbt.com/docs/core/connect-data-platform/snowflake-setup#key-pair-authentication + https://airflow.apache.org/docs/apache-airflow-providers-snowflake/stable/connections/snowflake.html + """ + + airflow_connection_type: str = "snowflake" + dbt_profile_type: str = "snowflake" + is_community: bool = True + + required_fields = [ + "account", + "user", + "database", + "warehouse", + "schema", + "private_key", + "private_key_passphrase", + ] + secret_fields = [ + "private_key", + "private_key_passphrase", + ] + airflow_param_mapping = { + "account": "extra.account", + "user": "login", + "database": "extra.database", + "warehouse": "extra.warehouse", + "schema": "schema", + "role": "extra.role", + "private_key": "extra.private_key_content", + "private_key_passphrase": "password", + } + + def can_claim_connection(self) -> bool: + # Make sure this isn't a private key path credential + result = super().can_claim_connection() + if result and self.conn.extra_dejson.get("private_key_file") is not None: + return False + return result + + @property + def conn(self) -> Connection: + """ + Snowflake can be odd because the fields used to be stored with keys in the format + 'extra__snowflake__account', but now are stored as 'account'. + + This standardizes the keys to be 'account', 'database', etc. + """ + conn = super().conn + + conn_dejson = conn.extra_dejson + + if conn_dejson.get("extra__snowflake__account"): + conn_dejson = {key.replace("extra__snowflake__", ""): value for key, value in conn_dejson.items()} + + conn.extra = json.dumps(conn_dejson) + + return conn + + @property + def profile(self) -> dict[str, Any | None]: + """Gets profile.""" + profile_vars = { + **self.mapped_params, + **self.profile_args, + "private_key": self.get_env_var_format("private_key"), + "private_key_passphrase": self.get_env_var_format("private_key_passphrase"), + } + + # remove any null values + return self.filter_null(profile_vars) + + def transform_account(self, account: str) -> str: + """Transform the account to the format . if it's not already.""" + region = self.conn.extra_dejson.get("region") + if region and region not in account: + account = f"{account}.{region}" + + return str(account) diff --git a/cosmos/profiles/snowflake/user_encrypted_privatekey.py b/cosmos/profiles/snowflake/user_encrypted_privatekey_file.py similarity index 79% rename from cosmos/profiles/snowflake/user_encrypted_privatekey.py rename to cosmos/profiles/snowflake/user_encrypted_privatekey_file.py index 0623598be6..e217a6c22e 100644 --- a/cosmos/profiles/snowflake/user_encrypted_privatekey.py +++ b/cosmos/profiles/snowflake/user_encrypted_privatekey_file.py @@ -1,4 +1,5 @@ -"Maps Airflow Snowflake connections to dbt profiles if they use a user/private key." +"""Maps Airflow Snowflake connections to dbt profiles if they use a user/private key path.""" + from __future__ import annotations import json @@ -10,9 +11,9 @@ from airflow.models import Connection -class SnowflakeEncryptedPrivateKeyPemProfileMapping(BaseProfileMapping): +class SnowflakeEncryptedPrivateKeyFilePemProfileMapping(BaseProfileMapping): """ - Maps Airflow Snowflake connections to dbt profiles if they use a user/private key. + Maps Airflow Snowflake connections to dbt profiles if they use a user/private key path. https://docs.getdbt.com/docs/core/connect-data-platform/snowflake-setup#key-pair-authentication https://airflow.apache.org/docs/apache-airflow-providers-snowflake/stable/connections/snowflake.html """ @@ -44,6 +45,13 @@ class SnowflakeEncryptedPrivateKeyPemProfileMapping(BaseProfileMapping): "private_key_path": "extra.private_key_file", } + def can_claim_connection(self) -> bool: + # Make sure this isn't a private key environmentvariable + result = super().can_claim_connection() + if result and self.conn.extra_dejson.get("private_key_content") is not None: + return False + return result + @property def conn(self) -> Connection: """ @@ -65,7 +73,7 @@ def conn(self) -> Connection: @property def profile(self) -> dict[str, Any | None]: - "Gets profile." + """Gets profile.""" profile_vars = { **self.mapped_params, **self.profile_args, @@ -77,7 +85,7 @@ def profile(self) -> dict[str, Any | None]: return self.filter_null(profile_vars) def transform_account(self, account: str) -> str: - "Transform the account to the format . if it's not already." + """Transform the account to the format . if it's not already.""" region = self.conn.extra_dejson.get("region") if region and region not in account: account = f"{account}.{region}" diff --git a/cosmos/profiles/snowflake/user_pass.py b/cosmos/profiles/snowflake/user_pass.py index 2e1025a2c6..a0c6ade678 100644 --- a/cosmos/profiles/snowflake/user_pass.py +++ b/cosmos/profiles/snowflake/user_pass.py @@ -1,4 +1,5 @@ -"Maps Airflow Snowflake connections to dbt profiles if they use a user/password." +"""Maps Airflow Snowflake connections to dbt profiles if they use a user/password.""" + from __future__ import annotations import json @@ -39,12 +40,17 @@ class SnowflakeUserPasswordProfileMapping(BaseProfileMapping): "warehouse": "extra.warehouse", "schema": "schema", "role": "extra.role", + "host": "extra.host", + "port": "extra.port", } def can_claim_connection(self) -> bool: # Make sure this isn't a private key path credential result = super().can_claim_connection() - if result and self.conn.extra_dejson.get("private_key_file") is not None: + if result and ( + self.conn.extra_dejson.get("private_key_file") is not None + or self.conn.extra_dejson.get("private_key_content") is not None + ): return False return result @@ -69,7 +75,7 @@ def conn(self) -> Connection: @property def profile(self) -> dict[str, Any | None]: - "Gets profile." + """Gets profile.""" profile_vars = { **self.mapped_params, **self.profile_args, @@ -81,9 +87,9 @@ def profile(self) -> dict[str, Any | None]: return self.filter_null(profile_vars) def transform_account(self, account: str) -> str: - "Transform the account to the format . if it's not already." + """Transform the account to the format . if it's not already.""" region = self.conn.extra_dejson.get("region") if region and region not in account: account = f"{account}.{region}" - return str(account) + return str(account) \ No newline at end of file diff --git a/cosmos/profiles/snowflake/user_privatekey.py b/cosmos/profiles/snowflake/user_privatekey.py index 3aa0454ec6..c74194b7ae 100644 --- a/cosmos/profiles/snowflake/user_privatekey.py +++ b/cosmos/profiles/snowflake/user_privatekey.py @@ -1,4 +1,5 @@ -"Maps Airflow Snowflake connections to dbt profiles if they use a user/private key." +"""Maps Airflow Snowflake connections to dbt profiles if they use a user/private key.""" + from __future__ import annotations import json @@ -63,7 +64,7 @@ def conn(self) -> Connection: @property def profile(self) -> dict[str, Any | None]: - "Gets profile." + """Gets profile.""" profile_vars = { **self.mapped_params, **self.profile_args, @@ -75,7 +76,7 @@ def profile(self) -> dict[str, Any | None]: return self.filter_null(profile_vars) def transform_account(self, account: str) -> str: - "Transform the account to the format . if it's not already." + """Transform the account to the format . if it's not already.""" region = self.conn.extra_dejson.get("region") if region and region not in account: account = f"{account}.{region}" diff --git a/cosmos/profiles/spark/__init__.py b/cosmos/profiles/spark/__init__.py index d90b71dcc9..9bbfd775a9 100644 --- a/cosmos/profiles/spark/__init__.py +++ b/cosmos/profiles/spark/__init__.py @@ -1,4 +1,4 @@ -"Spark Airflow connection -> dbt profile mappings" +"""Spark Airflow connection -> dbt profile mappings""" from .thrift import SparkThriftProfileMapping diff --git a/cosmos/profiles/spark/thrift.py b/cosmos/profiles/spark/thrift.py index 3f9b44c041..e81851a747 100644 --- a/cosmos/profiles/spark/thrift.py +++ b/cosmos/profiles/spark/thrift.py @@ -1,4 +1,5 @@ -"Maps Airflow Spark connections to dbt profiles if they use a thrift connection." +"""Maps Airflow Spark connections to dbt profiles if they use a thrift connection.""" + from __future__ import annotations from typing import Any diff --git a/cosmos/profiles/teradata/__init__.py b/cosmos/profiles/teradata/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cosmos/profiles/teradata/user_pass.py b/cosmos/profiles/teradata/user_pass.py new file mode 100644 index 0000000000..535a7c717c --- /dev/null +++ b/cosmos/profiles/teradata/user_pass.py @@ -0,0 +1,58 @@ +"""Maps Airflow Snowflake connections to dbt profiles if they use a user/password.""" + +from __future__ import annotations + +from typing import Any + +from ..base import BaseProfileMapping + + +class TeradataUserPasswordProfileMapping(BaseProfileMapping): + """ + Maps Airflow Teradata connections using user + password authentication to dbt profiles. + https://docs.getdbt.com/docs/core/connect-data-platform/teradata-setup + https://airflow.apache.org/docs/apache-airflow-providers-teradata/stable/connections/teradata.html + """ + + airflow_connection_type: str = "teradata" + dbt_profile_type: str = "teradata" + is_community = True + + required_fields = [ + "host", + "user", + "password", + ] + secret_fields = [ + "password", + ] + airflow_param_mapping = { + "host": "host", + "user": "login", + "password": "password", + "schema": "schema", + "tmode": "extra.tmode", + } + + @property + def profile(self) -> dict[str, Any]: + """Gets profile. The password is stored in an environment variable.""" + profile = { + **self.mapped_params, + **self.profile_args, + # password should always get set as env var + "password": self.get_env_var_format("password"), + } + # schema is not mandatory in teradata. In teradata user itself a database so if schema is not mentioned + # in both airflow connection and profile_args then treating user as schema. + if "schema" not in self.profile_args and self.mapped_params.get("schema") is None: + profile["schema"] = profile["user"] + + return self.filter_null(profile) + + @property + def mock_profile(self) -> dict[str, Any | None]: + """Gets mock profile. Assigning user to schema as default""" + mock_profile = super().mock_profile + mock_profile["schema"] = mock_profile["user"] + return mock_profile diff --git a/cosmos/profiles/trino/__init__.py b/cosmos/profiles/trino/__init__.py index adb633208b..97431e4508 100644 --- a/cosmos/profiles/trino/__init__.py +++ b/cosmos/profiles/trino/__init__.py @@ -1,4 +1,4 @@ -"Trino Airflow connection -> dbt profile mappings" +"""Trino Airflow connection -> dbt profile mappings""" from .base import TrinoBaseProfileMapping from .certificate import TrinoCertificateProfileMapping diff --git a/cosmos/profiles/trino/base.py b/cosmos/profiles/trino/base.py index 1e7e841246..a3c242d8f9 100644 --- a/cosmos/profiles/trino/base.py +++ b/cosmos/profiles/trino/base.py @@ -1,4 +1,5 @@ -"Maps common fields for Airflow Trino connections to dbt profiles." +"""Maps common fields for Airflow Trino connections to dbt profiles.""" + from __future__ import annotations from typing import Any @@ -7,28 +8,31 @@ class TrinoBaseProfileMapping(BaseProfileMapping): - "Maps common fields for Airflow Trino connections to dbt profiles." + """Maps common fields for Airflow Trino connections to dbt profiles.""" airflow_connection_type: str = "trino" dbt_profile_type: str = "trino" is_community: bool = True - required_fields = [ + base_fields = [ "host", "database", "schema", "port", ] + required_fields = base_fields + ["user"] + airflow_param_mapping = { "host": "host", "port": "port", + "user": "login", "session_properties": "extra.session_properties", } @property def profile(self) -> dict[str, Any]: - "Gets profile." + """Gets profile.""" profile_vars = { **self.mapped_params, **self.profile_args, @@ -38,5 +42,5 @@ def profile(self) -> dict[str, Any]: return self.filter_null(profile_vars) def transform_host(self, host: str) -> str: - "Replaces http:// or https:// with nothing." + """Replaces http:// or https:// with nothing.""" return host.replace("http://", "").replace("https://", "") diff --git a/cosmos/profiles/trino/certificate.py b/cosmos/profiles/trino/certificate.py index b792664a43..1ee5687383 100644 --- a/cosmos/profiles/trino/certificate.py +++ b/cosmos/profiles/trino/certificate.py @@ -1,4 +1,5 @@ -"Maps Airflow Trino connections to Certificate Trino dbt profiles." +"""Maps Airflow Trino connections to Certificate Trino dbt profiles.""" + from __future__ import annotations from typing import Any @@ -15,7 +16,7 @@ class TrinoCertificateProfileMapping(TrinoBaseProfileMapping): dbt_profile_method: str = "certificate" - required_fields = TrinoBaseProfileMapping.required_fields + [ + required_fields = TrinoBaseProfileMapping.base_fields + [ "client_certificate", "client_private_key", ] @@ -28,7 +29,7 @@ class TrinoCertificateProfileMapping(TrinoBaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Gets profile." + """Gets profile.""" common_profile_vars = super().profile profile_vars = { **self.mapped_params, diff --git a/cosmos/profiles/trino/jwt.py b/cosmos/profiles/trino/jwt.py index 5d6d989081..f9c0fd60d9 100644 --- a/cosmos/profiles/trino/jwt.py +++ b/cosmos/profiles/trino/jwt.py @@ -1,4 +1,5 @@ -"Maps Airflow Trino connections to JWT Trino dbt profiles." +"""Maps Airflow Trino connections to JWT Trino dbt profiles.""" + from __future__ import annotations from typing import Any @@ -16,7 +17,7 @@ class TrinoJWTProfileMapping(TrinoBaseProfileMapping): dbt_profile_method: str = "jwt" - required_fields = TrinoBaseProfileMapping.required_fields + [ + required_fields = TrinoBaseProfileMapping.base_fields + [ "jwt_token", ] secret_fields = [ @@ -29,7 +30,7 @@ class TrinoJWTProfileMapping(TrinoBaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Gets profile." + """Gets profile.""" common_profile_vars: dict[str, Any] = super().profile # need to remove jwt from profile_args because it will be set as an environment variable diff --git a/cosmos/profiles/trino/ldap.py b/cosmos/profiles/trino/ldap.py index d456f7c68d..7b1e7579b4 100644 --- a/cosmos/profiles/trino/ldap.py +++ b/cosmos/profiles/trino/ldap.py @@ -1,4 +1,5 @@ -"Maps Airflow Trino connections to LDAP Trino dbt profiles." +"""Maps Airflow Trino connections to LDAP Trino dbt profiles.""" + from __future__ import annotations from typing import Any @@ -16,7 +17,7 @@ class TrinoLDAPProfileMapping(TrinoBaseProfileMapping): dbt_profile_method: str = "ldap" - required_fields = TrinoBaseProfileMapping.required_fields + [ + required_fields = TrinoBaseProfileMapping.base_fields + [ "user", "password", ] diff --git a/cosmos/profiles/vertica/__init__.py b/cosmos/profiles/vertica/__init__.py index 4a88f2edd7..7a8a8acfd2 100644 --- a/cosmos/profiles/vertica/__init__.py +++ b/cosmos/profiles/vertica/__init__.py @@ -1,4 +1,4 @@ -"Vertica Airflow connection -> dbt profile mappings" +"""Vertica Airflow connection -> dbt profile mappings""" from .user_pass import VerticaUserPasswordProfileMapping diff --git a/cosmos/profiles/vertica/user_pass.py b/cosmos/profiles/vertica/user_pass.py index 494185e051..214dbc3570 100644 --- a/cosmos/profiles/vertica/user_pass.py +++ b/cosmos/profiles/vertica/user_pass.py @@ -1,4 +1,5 @@ -"Maps Airflow Vertica connections using user + password authentication to dbt profiles." +"""Maps Airflow Vertica connections using username + password authentication to dbt profiles.""" + from __future__ import annotations from typing import Any @@ -8,9 +9,15 @@ class VerticaUserPasswordProfileMapping(BaseProfileMapping): """ - Maps Airflow Vertica connections using user + password authentication to dbt profiles. - https://docs.getdbt.com/reference/warehouse-setups/vertica-setup - https://airflow.apache.org/docs/apache-airflow-providers-vertica/stable/connections/vertica.html + Maps Airflow Vertica connections using username + password authentication to dbt profiles. + .. note:: + Use Airflow connection ``schema`` for vertica ``database`` to keep it consistent with other connection types and profiles. \ + The Vertica Airflow provider hook `assumes this `_. + This seems to be a common approach also for `Postgres `_, \ + Redshift and Exasol since there is no ``database`` field in Airflow connection and ``schema`` is not required for the database connection. + .. seealso:: + https://docs.getdbt.com/reference/warehouse-setups/vertica-setup + https://airflow.apache.org/docs/apache-airflow-providers-vertica/stable/connections/vertica.html """ airflow_connection_type: str = "vertica" @@ -18,7 +25,7 @@ class VerticaUserPasswordProfileMapping(BaseProfileMapping): required_fields = [ "host", - "user", + "username", "password", "database", "schema", @@ -28,11 +35,10 @@ class VerticaUserPasswordProfileMapping(BaseProfileMapping): ] airflow_param_mapping = { "host": "host", - "user": "login", + "username": "login", "password": "password", "port": "port", - "schema": "schema", - "database": "extra.database", + "database": "schema", "autocommit": "extra.autocommit", "backup_server_node": "extra.backup_server_node", "binary_transfer": "extra.binary_transfer", @@ -54,7 +60,7 @@ class VerticaUserPasswordProfileMapping(BaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Gets profile. The password is stored in an environment variable." + """Gets profile. The password is stored in an environment variable.""" profile = { "port": 5433, **self.mapped_params, @@ -67,7 +73,7 @@ def profile(self) -> dict[str, Any | None]: @property def mock_profile(self) -> dict[str, Any | None]: - "Gets mock profile. Defaults port to 5433." + """Gets mock profile. Defaults port to 5433.""" parent_mock = super().mock_profile return { diff --git a/cosmos/settings.py b/cosmos/settings.py new file mode 100644 index 0000000000..62d4ee5bdf --- /dev/null +++ b/cosmos/settings.py @@ -0,0 +1,26 @@ +import os +import tempfile +from pathlib import Path + +import airflow +from airflow.configuration import conf + +from cosmos.constants import DEFAULT_COSMOS_CACHE_DIR_NAME, DEFAULT_OPENLINEAGE_NAMESPACE + +# In MacOS users may want to set the envvar `TMPDIR` if they do not want the value of the temp directory to change +DEFAULT_CACHE_DIR = Path(tempfile.gettempdir(), DEFAULT_COSMOS_CACHE_DIR_NAME) +cache_dir = Path(conf.get("cosmos", "cache_dir", fallback=DEFAULT_CACHE_DIR) or DEFAULT_CACHE_DIR) +enable_cache = conf.getboolean("cosmos", "enable_cache", fallback=True) +enable_cache_partial_parse = conf.getboolean("cosmos", "enable_cache_partial_parse", fallback=True) +enable_cache_dbt_ls = conf.getboolean("cosmos", "enable_cache_dbt_ls", fallback=True) +propagate_logs = conf.getboolean("cosmos", "propagate_logs", fallback=True) +dbt_docs_dir = conf.get("cosmos", "dbt_docs_dir", fallback=None) +dbt_docs_conn_id = conf.get("cosmos", "dbt_docs_conn_id", fallback=None) +dbt_docs_index_file_name = conf.get("cosmos", "dbt_docs_index_file_name", fallback="index.html") +enable_cache_profile = conf.getboolean("cosmos", "enable_cache_profile", fallback=True) +dbt_profile_cache_dir_name = conf.get("cosmos", "profile_cache_dir_name", fallback="profile") + +try: + LINEAGE_NAMESPACE = conf.get("openlineage", "namespace") +except airflow.exceptions.AirflowConfigException: + LINEAGE_NAMESPACE = os.getenv("OPENLINEAGE_NAMESPACE", DEFAULT_OPENLINEAGE_NAMESPACE) diff --git a/dev/Dockerfile b/dev/Dockerfile index 90c49ed6ca..cc6307f36a 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -1,13 +1,23 @@ -FROM quay.io/astronomer/astro-runtime:7.3.0-base +FROM quay.io/astronomer/astro-runtime:11.3.0-base USER root + +# dbt-postgres 1.8.0 requires building psycopg2 from source +RUN /bin/sh -c set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends build-essential libpq-dev; \ + apt-get clean; \ + rm -rf /var/lib/apt/lists/* + +RUN pip install -U uv + COPY ./pyproject.toml ${AIRFLOW_HOME}/astronomer_cosmos/ COPY ./README.rst ${AIRFLOW_HOME}/astronomer_cosmos/ COPY ./cosmos/ ${AIRFLOW_HOME}/astronomer_cosmos/cosmos/ # install the package in editable mode -RUN pip install -e "${AIRFLOW_HOME}/astronomer_cosmos"[dbt-postgres,dbt-databricks] +RUN uv pip install --system -e "${AIRFLOW_HOME}/astronomer_cosmos"[dbt-postgres,dbt-databricks] # make sure astro user owns the package RUN chown -R astro:astro ${AIRFLOW_HOME}/astronomer_cosmos diff --git a/dev/dags/basic_cosmos_dag.py b/dev/dags/basic_cosmos_dag.py index 8bd49b0b39..f71f351f0a 100644 --- a/dev/dags/basic_cosmos_dag.py +++ b/dev/dags/basic_cosmos_dag.py @@ -1,12 +1,12 @@ """ -An example DAG that uses Cosmos to render a dbt project. +An example DAG that uses Cosmos to render a dbt project into an Airflow DAG. """ import os from datetime import datetime from pathlib import Path -from cosmos import DbtDag, ProjectConfig, ProfileConfig +from cosmos import DbtDag, ProfileConfig, ProjectConfig from cosmos.profiles import PostgresUserPasswordProfileMapping DEFAULT_DBT_ROOT_PATH = Path(__file__).parent / "dbt" @@ -16,8 +16,9 @@ profile_name="default", target_name="dev", profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="airflow_db", + conn_id="example_conn", profile_args={"schema": "public"}, + disable_event_tracking=True, ), ) diff --git a/dev/dags/basic_cosmos_task_group.py b/dev/dags/basic_cosmos_task_group.py index 50cb6ed09e..d63cf2c923 100644 --- a/dev/dags/basic_cosmos_task_group.py +++ b/dev/dags/basic_cosmos_task_group.py @@ -1,6 +1,7 @@ """ -An example DAG that uses Cosmos to render a dbt project as a TaskGroup. +An example DAG that uses Cosmos to render a dbt project as an Airflow TaskGroup. """ + import os from datetime import datetime from pathlib import Path @@ -8,7 +9,8 @@ from airflow.decorators import dag from airflow.operators.empty import EmptyOperator -from cosmos import DbtTaskGroup, ProjectConfig, ProfileConfig +from cosmos import DbtTaskGroup, ExecutionConfig, ProfileConfig, ProjectConfig, RenderConfig +from cosmos.constants import InvocationMode from cosmos.profiles import PostgresUserPasswordProfileMapping DEFAULT_DBT_ROOT_PATH = Path(__file__).parent / "dbt" @@ -18,11 +20,15 @@ profile_name="default", target_name="dev", profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="airflow_db", + conn_id="example_conn", profile_args={"schema": "public"}, ), ) +shared_execution_config = ExecutionConfig( + invocation_mode=InvocationMode.SUBPROCESS, +) + @dag( schedule_interval="@daily", @@ -35,11 +41,31 @@ def basic_cosmos_task_group() -> None: """ pre_dbt = EmptyOperator(task_id="pre_dbt") - jaffle_shop = DbtTaskGroup( - group_id="test_123", + customers = DbtTaskGroup( + group_id="customers", + project_config=ProjectConfig((DBT_ROOT_PATH / "jaffle_shop").as_posix(), dbt_vars={"var": "2"}), + render_config=RenderConfig( + select=["path:seeds/raw_customers.csv"], + enable_mock_profile=False, + env_vars={"PURGE": os.getenv("PURGE", "0")}, + airflow_vars_to_purge_dbt_ls_cache=["purge"], + ), + execution_config=shared_execution_config, + operator_args={"install_deps": True}, + profile_config=profile_config, + default_args={"retries": 2}, + ) + + orders = DbtTaskGroup( + group_id="orders", project_config=ProjectConfig( (DBT_ROOT_PATH / "jaffle_shop").as_posix(), ), + render_config=RenderConfig( + select=["path:seeds/raw_orders.csv"], + enable_mock_profile=False, # This is necessary to benefit from partial parsing when using ProfileMapping + ), + execution_config=shared_execution_config, operator_args={"install_deps": True}, profile_config=profile_config, default_args={"retries": 2}, @@ -47,7 +73,8 @@ def basic_cosmos_task_group() -> None: post_dbt = EmptyOperator(task_id="post_dbt") - pre_dbt >> jaffle_shop >> post_dbt + pre_dbt >> customers >> post_dbt + pre_dbt >> orders >> post_dbt basic_cosmos_task_group() diff --git a/dev/dags/cosmos_manifest_example.py b/dev/dags/cosmos_manifest_example.py index a00f9e75b7..225b38ddc1 100644 --- a/dev/dags/cosmos_manifest_example.py +++ b/dev/dags/cosmos_manifest_example.py @@ -1,13 +1,13 @@ """ -An example DAG that uses Cosmos to render a dbt project. +An example DAG that uses Cosmos to render a dbt project into Airflow using a dbt manifest file. """ import os from datetime import datetime from pathlib import Path -from cosmos import DbtDag, ProjectConfig, ProfileConfig, RenderConfig, LoadMode -from cosmos.profiles import PostgresUserPasswordProfileMapping +from cosmos import DbtDag, ExecutionConfig, LoadMode, ProfileConfig, ProjectConfig, RenderConfig +from cosmos.profiles import DbtProfileConfigVars, PostgresUserPasswordProfileMapping DEFAULT_DBT_ROOT_PATH = Path(__file__).parent / "dbt" DBT_ROOT_PATH = Path(os.getenv("DBT_ROOT_PATH", DEFAULT_DBT_ROOT_PATH)) @@ -16,8 +16,9 @@ profile_name="default", target_name="dev", profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="airflow_db", + conn_id="example_conn", profile_args={"schema": "public"}, + dbt_config_vars=DbtProfileConfigVars(send_anonymous_usage_stats=True), ), ) @@ -25,12 +26,12 @@ cosmos_manifest_example = DbtDag( # dbt/cosmos-specific parameters project_config=ProjectConfig( - dbt_project_path=DBT_ROOT_PATH / "jaffle_shop", manifest_path=DBT_ROOT_PATH / "jaffle_shop" / "target" / "manifest.json", project_name="jaffle_shop", ), profile_config=profile_config, render_config=RenderConfig(load_method=LoadMode.DBT_MANIFEST, select=["path:seeds/raw_customers.csv"]), + execution_config=ExecutionConfig(dbt_project_path=DBT_ROOT_PATH / "jaffle_shop"), operator_args={"install_deps": True}, # normal dag parameters schedule_interval="@daily", diff --git a/dev/dags/cosmos_profile_mapping.py b/dev/dags/cosmos_profile_mapping.py index 33619a39db..36f344446e 100644 --- a/dev/dags/cosmos_profile_mapping.py +++ b/dev/dags/cosmos_profile_mapping.py @@ -3,6 +3,7 @@ It uses the automatic profile rendering from an Airflow connection. """ + import os from datetime import datetime from pathlib import Path @@ -10,12 +11,15 @@ from airflow.decorators import dag from airflow.operators.empty import EmptyOperator -from cosmos import DbtTaskGroup, ProjectConfig, ProfileConfig +from cosmos import DbtTaskGroup, ExecutionConfig, ProfileConfig, ProjectConfig +from cosmos.constants import InvocationMode from cosmos.profiles import get_automatic_profile_mapping DEFAULT_DBT_ROOT_PATH = Path(__file__).parent / "dbt" DBT_ROOT_PATH = Path(os.getenv("DBT_ROOT_PATH", DEFAULT_DBT_ROOT_PATH)) +execution_config = ExecutionConfig(invocation_mode=InvocationMode.DBT_RUNNER) + @dag( schedule_interval="@daily", @@ -29,6 +33,7 @@ def cosmos_profile_mapping() -> None: pre_dbt = EmptyOperator(task_id="pre_dbt") jaffle_shop = DbtTaskGroup( + execution_config=execution_config, project_config=ProjectConfig( DBT_ROOT_PATH / "jaffle_shop", ), @@ -36,7 +41,7 @@ def cosmos_profile_mapping() -> None: profile_name="default", target_name="dev", profile_mapping=get_automatic_profile_mapping( - conn_id="airflow_db", + conn_id="example_conn", profile_args={"schema": "public"}, ), ), diff --git a/dev/dags/cosmos_seed_dag.py b/dev/dags/cosmos_seed_dag.py index cef84dd66f..d0b4f41034 100644 --- a/dev/dags/cosmos_seed_dag.py +++ b/dev/dags/cosmos_seed_dag.py @@ -9,6 +9,7 @@ would be ingesting data from various sources (i.e. sftp, blob like s3 or gcs, http endpoint, database, etc.) """ + import os from pathlib import Path @@ -28,7 +29,7 @@ profile_name="default", target_name="dev", profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="airflow_db", + conn_id="example_conn", profile_args={"schema": "public"}, ), ) diff --git a/dev/dags/dbt/simple/data/imdb.db b/dev/dags/dbt/data/imdb.db similarity index 71% rename from dev/dags/dbt/simple/data/imdb.db rename to dev/dags/dbt/data/imdb.db index 605f6526e9..be0c1fe773 100644 Binary files a/dev/dags/dbt/simple/data/imdb.db and b/dev/dags/dbt/data/imdb.db differ diff --git a/dev/dags/dbt/jaffle_shop/.gitignore b/dev/dags/dbt/jaffle_shop/.gitignore index 49f147cb98..45d294b9af 100644 --- a/dev/dags/dbt/jaffle_shop/.gitignore +++ b/dev/dags/dbt/jaffle_shop/.gitignore @@ -2,3 +2,4 @@ target/ dbt_packages/ logs/ +!target/manifest.json diff --git a/dev/dags/dbt/jaffle_shop/dbt_ls_models_staging.txt b/dev/dags/dbt/jaffle_shop/dbt_ls_models_staging.txt new file mode 100644 index 0000000000..b8cc902ec0 --- /dev/null +++ b/dev/dags/dbt/jaffle_shop/dbt_ls_models_staging.txt @@ -0,0 +1,9 @@ +14:26:04 Running with dbt=1.6.9 +14:26:04 Registered adapter: exasol=1.6.2 +14:26:04 Found 5 models, 3 seeds, 20 tests, 0 sources, 0 exposures, 0 metrics, 366 macros, 0 groups, 0 semantic models +{"name": "stg_customers", "resource_type": "model", "package_name": "jaffle_shop", "original_file_path": "models/staging/stg_customers.sql", "unique_id": "model.jaffle_shop.stg_customers", "alias": "stg_customers", "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "post-hook": [], "pre-hook": []}, "tags": [], "depends_on": {"macros": [], "nodes": ["seed.jaffle_shop.raw_customers"]}} +{"name": "stg_orders", "resource_type": "model", "package_name": "jaffle_shop", "original_file_path": "models/staging/stg_orders.sql", "unique_id": "model.jaffle_shop.stg_orders", "alias": "stg_orders", "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "post-hook": [], "pre-hook": []}, "tags": [], "depends_on": {"macros": [], "nodes": ["seed.jaffle_shop.raw_orders"]}} +{"name": "stg_payments", "resource_type": "model", "package_name": "jaffle_shop", "original_file_path": "models/staging/stg_payments.sql", "unique_id": "model.jaffle_shop.stg_payments", "alias": "stg_payments", "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "post-hook": [], "pre-hook": []}, "tags": [], "depends_on": {"macros": [], "nodes": ["seed.jaffle_shop.raw_payments"]}} +{"name": "raw_customers", "resource_type": "seed", "package_name": "jaffle_shop", "original_file_path": "seeds/raw_customers.csv", "unique_id": "seed.jaffle_shop.raw_customers", "alias": "raw_customers", "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "quote_columns": null, "post-hook": [], "pre-hook": []}, "tags": [], "depends_on": {"macros": []}} +{"name": "raw_orders", "resource_type": "seed", "package_name": "jaffle_shop", "original_file_path": "seeds/raw_orders.csv", "unique_id": "seed.jaffle_shop.raw_orders", "alias": "raw_orders", "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "quote_columns": null, "post-hook": [], "pre-hook": []}, "tags": [], "depends_on": {"macros": []}} +{"name": "raw_payments", "resource_type": "seed", "package_name": "jaffle_shop", "original_file_path": "seeds/raw_payments.csv", "unique_id": "seed.jaffle_shop.raw_payments", "alias": "raw_payments", "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "quote_columns": null, "post-hook": [], "pre-hook": []}, "tags": [], "depends_on": {"macros": []}} diff --git a/dev/dags/dbt/jaffle_shop/target/manifest.json b/dev/dags/dbt/jaffle_shop/target/manifest.json index 9446d80c3a..c8ba975540 100644 --- a/dev/dags/dbt/jaffle_shop/target/manifest.json +++ b/dev/dags/dbt/jaffle_shop/target/manifest.json @@ -68,39 +68,41 @@ }, "disabled": {}, "docs": { - "dbt.__overview__": { + "doc.dbt.__overview__": { "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion", "name": "__overview__", "original_file_path": "docs/overview.md", "package_name": "dbt", "path": "overview.md", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", - "unique_id": "dbt.__overview__" + "resource_type": "doc", + "unique_id": "doc.dbt.__overview__" }, - "jaffle_shop.__overview__": { + "doc.jaffle_shop.__overview__": { "block_contents": "## Data Documentation for Jaffle Shop\n\n`jaffle_shop` is a fictional ecommerce store.\n\nThis [dbt](https://www.getdbt.com/) project is for testing out code.\n\nThe source code can be found [here](https://github.com/clrcrl/jaffle_shop).", "name": "__overview__", "original_file_path": "models/overview.md", "package_name": "jaffle_shop", "path": "overview.md", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", - "unique_id": "jaffle_shop.__overview__" + "resource_type": "doc", + "unique_id": "doc.jaffle_shop.__overview__" }, - "jaffle_shop.orders_status": { + "doc.jaffle_shop.orders_status": { "block_contents": "Orders can be one of the following statuses:\n\n| status | description |\n|----------------|------------------------------------------------------------------------------------------------------------------------|\n| placed | The order has been placed but has not yet left the warehouse |\n| shipped | The order has ben shipped to the customer and is currently in transit |\n| completed | The order has been received by the customer |\n| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse |\n| returned | The order has been returned by the customer and received at the warehouse |", "name": "orders_status", "original_file_path": "models/docs.md", "package_name": "jaffle_shop", "path": "docs.md", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", - "unique_id": "jaffle_shop.orders_status" + "resource_type": "doc", + "unique_id": "doc.jaffle_shop.orders_status" } }, "exposures": {}, + "group_map": {}, + "groups": {}, "macros": { "macro.dbt._split_part_negative": { "arguments": [], - "created_at": 1696458269.796739, + "created_at": 1719485736.555134, "depends_on": { "macros": [] }, @@ -109,7 +111,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 {{ part_number }}\n )\n\n{% endmacro %}", + "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 + {{ part_number }}\n )\n\n{% endmacro %}", "meta": {}, "name": "_split_part_negative", "original_file_path": "macros/utils/split_part.sql", @@ -117,14 +119,12 @@ "patch_path": null, "path": "macros/utils/split_part.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt._split_part_negative" }, "macro.dbt.after_commit": { "arguments": [], - "created_at": 1696458269.591864, + "created_at": 1719485736.337471, "depends_on": { "macros": [ "macro.dbt.make_hook_config" @@ -143,14 +143,12 @@ "patch_path": null, "path": "macros/materializations/hooks.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.after_commit" }, "macro.dbt.alter_column_comment": { "arguments": [], - "created_at": 1696458269.832114, + "created_at": 1719485736.583679, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__alter_column_comment" @@ -169,14 +167,12 @@ "patch_path": null, "path": "macros/adapters/persist_docs.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.alter_column_comment" }, "macro.dbt.alter_column_type": { "arguments": [], - "created_at": 1696458269.845595, + "created_at": 1719485736.599694, "depends_on": { "macros": [ "macro.dbt.default__alter_column_type" @@ -195,14 +191,12 @@ "patch_path": null, "path": "macros/adapters/columns.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.alter_column_type" }, "macro.dbt.alter_relation_add_remove_columns": { "arguments": [], - "created_at": 1696458269.847001, + "created_at": 1719485736.600589, "depends_on": { "macros": [ "macro.dbt.default__alter_relation_add_remove_columns" @@ -221,14 +215,12 @@ "patch_path": null, "path": "macros/adapters/columns.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.alter_relation_add_remove_columns" }, "macro.dbt.alter_relation_comment": { "arguments": [], - "created_at": 1696458269.832729, + "created_at": 1719485736.585442, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__alter_relation_comment" @@ -247,14 +239,12 @@ "patch_path": null, "path": "macros/adapters/persist_docs.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.alter_relation_comment" }, "macro.dbt.any_value": { "arguments": [], - "created_at": 1696458269.7846951, + "created_at": 1719485736.542892, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__any_value" @@ -273,14 +263,12 @@ "patch_path": null, "path": "macros/utils/any_value.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.any_value" }, "macro.dbt.apply_grants": { "arguments": [], - "created_at": 1696458269.828692, + "created_at": 1719485736.577766, "depends_on": { "macros": [ "macro.dbt.default__apply_grants" @@ -299,14 +287,12 @@ "patch_path": null, "path": "macros/adapters/apply_grants.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.apply_grants" }, "macro.dbt.array_append": { "arguments": [], - "created_at": 1696458269.7995489, + "created_at": 1719485736.5569642, "depends_on": { "macros": [ "macro.dbt.default__array_append" @@ -325,14 +311,12 @@ "patch_path": null, "path": "macros/utils/array_append.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.array_append" }, "macro.dbt.array_concat": { "arguments": [], - "created_at": 1696458269.792314, + "created_at": 1719485736.5521111, "depends_on": { "macros": [ "macro.dbt.default__array_concat" @@ -351,14 +335,12 @@ "patch_path": null, "path": "macros/utils/array_concat.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.array_concat" }, "macro.dbt.array_construct": { "arguments": [], - "created_at": 1696458269.798442, + "created_at": 1719485736.556025, "depends_on": { "macros": [ "macro.dbt.default__array_construct" @@ -377,14 +359,38 @@ "patch_path": null, "path": "macros/utils/array_construct.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.array_construct" }, + "macro.dbt.assert_columns_equivalent": { + "arguments": [], + "created_at": 1719485736.5031989, + "depends_on": { + "macros": [ + "macro.dbt.get_column_schema_from_query", + "macro.dbt.get_empty_schema_sql", + "macro.dbt.format_columns" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro assert_columns_equivalent(sql) %}\n\n {#-- First ensure the user has defined 'columns' in yaml specification --#}\n {%- set user_defined_columns = model['columns'] -%}\n {%- if not user_defined_columns -%}\n {{ exceptions.raise_contract_error([], []) }}\n {%- endif -%}\n\n {#-- Obtain the column schema provided by sql file. #}\n {%- set sql_file_provided_columns = get_column_schema_from_query(sql, config.get('sql_header', none)) -%}\n {#--Obtain the column schema provided by the schema file by generating an 'empty schema' query from the model's columns. #}\n {%- set schema_file_provided_columns = get_column_schema_from_query(get_empty_schema_sql(user_defined_columns)) -%}\n\n {#-- create dictionaries with name and formatted data type and strings for exception #}\n {%- set sql_columns = format_columns(sql_file_provided_columns) -%}\n {%- set yaml_columns = format_columns(schema_file_provided_columns) -%}\n\n {%- if sql_columns|length != yaml_columns|length -%}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n\n {%- for sql_col in sql_columns -%}\n {%- set yaml_col = [] -%}\n {%- for this_col in yaml_columns -%}\n {%- if this_col['name'] == sql_col['name'] -%}\n {%- do yaml_col.append(this_col) -%}\n {%- break -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if not yaml_col -%}\n {#-- Column with name not found in yaml #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- if sql_col['formatted'] != yaml_col[0]['formatted'] -%}\n {#-- Column data types don't match #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- endfor -%}\n\n{% endmacro %}", + "meta": {}, + "name": "assert_columns_equivalent", + "original_file_path": "macros/relations/column/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/column/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.assert_columns_equivalent" + }, "macro.dbt.before_begin": { "arguments": [], - "created_at": 1696458269.591393, + "created_at": 1719485736.336732, "depends_on": { "macros": [ "macro.dbt.make_hook_config" @@ -403,14 +409,12 @@ "patch_path": null, "path": "macros/materializations/hooks.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.before_begin" }, "macro.dbt.bool_or": { "arguments": [], - "created_at": 1696458269.7931142, + "created_at": 1719485736.5525389, "depends_on": { "macros": [ "macro.dbt.default__bool_or" @@ -429,14 +433,12 @@ "patch_path": null, "path": "macros/utils/bool_or.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.bool_or" }, "macro.dbt.build_config_dict": { "arguments": [], - "created_at": 1696458269.852396, + "created_at": 1719485736.617571, "depends_on": { "macros": [] }, @@ -445,7 +447,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro build_config_dict(model) %}\n {%- set config_dict = {} -%}\n {%- for key in model.config.config_keys_used -%}\n {# weird type testing with enum, would be much easier to write this logic in Python! #}\n {%- if key == 'language' -%}\n {%- set value = 'python' -%}\n {%- endif -%}\n {%- set value = model.config[key] -%}\n {%- do config_dict.update({key: value}) -%}\n {%- endfor -%}\nconfig_dict = {{ config_dict }}\n{% endmacro %}", + "macro_sql": "{% macro build_config_dict(model) %}\n {%- set config_dict = {} -%}\n {% set config_dbt_used = zip(model.config.config_keys_used, model.config.config_keys_defaults) | list %}\n {%- for key, default in config_dbt_used -%}\n {# weird type testing with enum, would be much easier to write this logic in Python! #}\n {%- if key == \"language\" -%}\n {%- set value = \"python\" -%}\n {%- endif -%}\n {%- set value = model.config.get(key, default) -%}\n {%- do config_dict.update({key: value}) -%}\n {%- endfor -%}\nconfig_dict = {{ config_dict }}\n{% endmacro %}", "meta": {}, "name": "build_config_dict", "original_file_path": "macros/python_model/python.sql", @@ -453,23 +455,23 @@ "patch_path": null, "path": "macros/python_model/python.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.build_config_dict" }, "macro.dbt.build_ref_function": { "arguments": [], - "created_at": 1696458269.850849, + "created_at": 1719485736.616483, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.resolve_model_name" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro build_ref_function(model) %}\n\n {%- set ref_dict = {} -%}\n {%- for _ref in model.refs -%}\n {%- set resolved = ref(*_ref) -%}\n {%- do ref_dict.update({_ref | join(\".\"): resolved.quote(database=False, schema=False, identifier=False) | string}) -%}\n {%- endfor -%}\n\ndef ref(*args,dbt_load_df_function):\n refs = {{ ref_dict | tojson }}\n key = \".\".join(args)\n return dbt_load_df_function(refs[key])\n\n{% endmacro %}", + "macro_sql": "{% macro build_ref_function(model) %}\n\n {%- set ref_dict = {} -%}\n {%- for _ref in model.refs -%}\n {% set _ref_args = [_ref.get('package'), _ref['name']] if _ref.get('package') else [_ref['name'],] %}\n {%- set resolved = ref(*_ref_args, v=_ref.get('version')) -%}\n {%- if _ref.get('version') -%}\n {% do _ref_args.extend([\"v\" ~ _ref['version']]) %}\n {%- endif -%}\n {%- do ref_dict.update({_ref_args | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef ref(*args, **kwargs):\n refs = {{ ref_dict | tojson }}\n key = '.'.join(args)\n version = kwargs.get(\"v\") or kwargs.get(\"version\")\n if version:\n key += f\".v{version}\"\n dbt_load_df_function = kwargs.get(\"dbt_load_df_function\")\n return dbt_load_df_function(refs[key])\n\n{% endmacro %}", "meta": {}, "name": "build_ref_function", "original_file_path": "macros/python_model/python.sql", @@ -477,14 +479,12 @@ "patch_path": null, "path": "macros/python_model/python.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.build_ref_function" }, "macro.dbt.build_snapshot_staging_table": { "arguments": [], - "created_at": 1696458269.620571, + "created_at": 1719485736.358976, "depends_on": { "macros": [ "macro.dbt.make_temp_relation", @@ -506,14 +506,12 @@ "patch_path": null, "path": "macros/materializations/snapshots/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.build_snapshot_staging_table" }, "macro.dbt.build_snapshot_table": { "arguments": [], - "created_at": 1696458269.619425, + "created_at": 1719485736.3569238, "depends_on": { "macros": [ "macro.dbt.default__build_snapshot_table" @@ -532,23 +530,23 @@ "patch_path": null, "path": "macros/materializations/snapshots/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.build_snapshot_table" }, "macro.dbt.build_source_function": { "arguments": [], - "created_at": 1696458269.851639, + "created_at": 1719485736.616947, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.resolve_model_name" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro build_source_function(model) %}\n\n {%- set source_dict = {} -%}\n {%- for _source in model.sources -%}\n {%- set resolved = source(*_source) -%}\n {%- do source_dict.update({_source | join(\".\"): resolved.quote(database=False, schema=False, identifier=False) | string}) -%}\n {%- endfor -%}\n\ndef source(*args, dbt_load_df_function):\n sources = {{ source_dict | tojson }}\n key = \".\".join(args)\n return dbt_load_df_function(sources[key])\n\n{% endmacro %}", + "macro_sql": "{% macro build_source_function(model) %}\n\n {%- set source_dict = {} -%}\n {%- for _source in model.sources -%}\n {%- set resolved = source(*_source) -%}\n {%- do source_dict.update({_source | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef source(*args, dbt_load_df_function):\n sources = {{ source_dict | tojson }}\n key = '.'.join(args)\n return dbt_load_df_function(sources[key])\n\n{% endmacro %}", "meta": {}, "name": "build_source_function", "original_file_path": "macros/python_model/python.sql", @@ -556,14 +554,12 @@ "patch_path": null, "path": "macros/python_model/python.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.build_source_function" }, "macro.dbt.call_dcl_statements": { "arguments": [], - "created_at": 1696458269.827903, + "created_at": 1719485736.576986, "depends_on": { "macros": [ "macro.dbt.default__call_dcl_statements" @@ -582,14 +578,60 @@ "patch_path": null, "path": "macros/adapters/apply_grants.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.call_dcl_statements" }, + "macro.dbt.can_clone_table": { + "arguments": [], + "created_at": 1719485736.4429, + "depends_on": { + "macros": [ + "macro.dbt.default__can_clone_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro can_clone_table() %}\n {{ return(adapter.dispatch('can_clone_table', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "can_clone_table", + "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/clone/can_clone_table.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.can_clone_table" + }, + "macro.dbt.cast": { + "arguments": [], + "created_at": 1719485736.542182, + "depends_on": { + "macros": [ + "macro.dbt.default__cast" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro cast(field, type) %}\n {{ return(adapter.dispatch('cast', 'dbt') (field, type)) }}\n{% endmacro %}", + "meta": {}, + "name": "cast", + "original_file_path": "macros/utils/cast.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/cast.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.cast" + }, "macro.dbt.cast_bool_to_text": { "arguments": [], - "created_at": 1696458269.783756, + "created_at": 1719485736.541704, "depends_on": { "macros": [ "macro.dbt.default__cast_bool_to_text" @@ -608,14 +650,12 @@ "patch_path": null, "path": "macros/utils/cast_bool_to_text.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.cast_bool_to_text" }, "macro.dbt.check_for_schema_changes": { "arguments": [], - "created_at": 1696458269.71107, + "created_at": 1719485736.43984, "depends_on": { "macros": [ "macro.dbt.diff_columns", @@ -635,14 +675,12 @@ "patch_path": null, "path": "macros/materializations/models/incremental/on_schema_change.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.check_for_schema_changes" }, "macro.dbt.check_schema_exists": { "arguments": [], - "created_at": 1696458269.8388941, + "created_at": 1719485736.591485, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__check_schema_exists" @@ -661,14 +699,12 @@ "patch_path": null, "path": "macros/adapters/metadata.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.check_schema_exists" }, "macro.dbt.collect_freshness": { "arguments": [], - "created_at": 1696458269.819403, + "created_at": 1719485736.569225, "depends_on": { "macros": [ "macro.dbt.default__collect_freshness" @@ -687,14 +723,12 @@ "patch_path": null, "path": "macros/adapters/freshness.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.collect_freshness" }, "macro.dbt.concat": { "arguments": [], - "created_at": 1696458269.774415, + "created_at": 1719485736.533403, "depends_on": { "macros": [ "macro.dbt.default__concat" @@ -713,14 +747,12 @@ "patch_path": null, "path": "macros/utils/concat.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.concat" }, "macro.dbt.convert_datetime": { "arguments": [], - "created_at": 1696458269.768708, + "created_at": 1719485736.522349, "depends_on": { "macros": [] }, @@ -737,14 +769,12 @@ "patch_path": null, "path": "macros/etc/datetime.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.convert_datetime" }, "macro.dbt.copy_grants": { "arguments": [], - "created_at": 1696458269.822963, + "created_at": 1719485736.5725539, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__copy_grants" @@ -763,14 +793,12 @@ "patch_path": null, "path": "macros/adapters/apply_grants.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.copy_grants" }, "macro.dbt.create_columns": { "arguments": [], - "created_at": 1696458269.6159968, + "created_at": 1719485736.354676, "depends_on": { "macros": [ "macro.dbt.default__create_columns" @@ -789,14 +817,12 @@ "patch_path": null, "path": "macros/materializations/snapshots/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.create_columns" }, "macro.dbt.create_csv_table": { "arguments": [], - "created_at": 1696458269.7461991, + "created_at": 1719485736.45865, "depends_on": { "macros": [ "macro.dbt.default__create_csv_table" @@ -815,14 +841,12 @@ "patch_path": null, "path": "macros/materializations/seeds/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.create_csv_table" }, "macro.dbt.create_indexes": { "arguments": [], - "created_at": 1696458269.8056178, + "created_at": 1719485736.561525, "depends_on": { "macros": [ "macro.dbt.default__create_indexes" @@ -841,14 +865,36 @@ "patch_path": null, "path": "macros/adapters/indexes.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.create_indexes" }, + "macro.dbt.create_or_replace_clone": { + "arguments": [], + "created_at": 1719485736.443349, + "depends_on": { + "macros": [ + "macro.dbt.default__create_or_replace_clone" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_or_replace_clone(this_relation, defer_relation) %}\n {{ return(adapter.dispatch('create_or_replace_clone', 'dbt')(this_relation, defer_relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "create_or_replace_clone", + "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/clone/create_or_replace_clone.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_or_replace_clone" + }, "macro.dbt.create_or_replace_view": { "arguments": [], - "created_at": 1696458269.730096, + "created_at": 1719485736.513715, "depends_on": { "macros": [ "macro.dbt.run_hooks", @@ -865,22 +911,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=True) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", + "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "meta": {}, "name": "create_or_replace_view", - "original_file_path": "macros/materializations/models/view/create_or_replace_view.sql", + "original_file_path": "macros/relations/view/replace.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/view/create_or_replace_view.sql", + "path": "macros/relations/view/replace.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.create_or_replace_view" }, "macro.dbt.create_schema": { "arguments": [], - "created_at": 1696458269.800509, + "created_at": 1719485736.558099, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__create_schema" @@ -899,14 +943,12 @@ "patch_path": null, "path": "macros/adapters/schema.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.create_schema" }, "macro.dbt.create_table_as": { "arguments": [], - "created_at": 1696458269.720823, + "created_at": 1719485736.508673, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__create_table_as" @@ -920,19 +962,17 @@ "macro_sql": "{% macro create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {# backward compatibility for create_table_as that does not support language #}\n {% if language == \"sql\" %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code)}}\n {% else %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code, language) }}\n {% endif %}\n\n{%- endmacro %}", "meta": {}, "name": "create_table_as", - "original_file_path": "macros/materializations/models/table/create_table_as.sql", + "original_file_path": "macros/relations/table/create.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/table/create_table_as.sql", + "path": "macros/relations/table/create.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.create_table_as" }, "macro.dbt.create_view_as": { "arguments": [], - "created_at": 1696458269.731384, + "created_at": 1719485736.515506, "depends_on": { "macros": [ "macro.dbt.default__create_view_as" @@ -946,19 +986,17 @@ "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "meta": {}, "name": "create_view_as", - "original_file_path": "macros/materializations/models/view/create_view_as.sql", + "original_file_path": "macros/relations/view/create.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/view/create_view_as.sql", + "path": "macros/relations/view/create.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.create_view_as" }, "macro.dbt.current_timestamp": { "arguments": [], - "created_at": 1696458269.802294, + "created_at": 1719485736.55921, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__current_timestamp" @@ -977,14 +1015,12 @@ "patch_path": null, "path": "macros/adapters/timestamps.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.current_timestamp" }, "macro.dbt.current_timestamp_backcompat": { "arguments": [], - "created_at": 1696458269.8032, + "created_at": 1719485736.559824, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__current_timestamp_backcompat" @@ -1003,14 +1039,12 @@ "patch_path": null, "path": "macros/adapters/timestamps.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.current_timestamp_backcompat" }, "macro.dbt.current_timestamp_in_utc_backcompat": { "arguments": [], - "created_at": 1696458269.803582, + "created_at": 1719485736.5600908, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat" @@ -1029,14 +1063,60 @@ "patch_path": null, "path": "macros/adapters/timestamps.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.current_timestamp_in_utc_backcompat" }, + "macro.dbt.date": { + "arguments": [], + "created_at": 1719485736.531044, + "depends_on": { + "macros": [ + "macro.dbt.default__date" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro date(year, month, day) %}\n {{ return(adapter.dispatch('date', 'dbt') (year, month, day)) }}\n{% endmacro %}", + "meta": {}, + "name": "date", + "original_file_path": "macros/utils/date.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/date.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.date" + }, + "macro.dbt.date_spine": { + "arguments": [], + "created_at": 1719485736.529869, + "depends_on": { + "macros": [ + "macro.dbt.default__date_spine" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro date_spine(datepart, start_date, end_date) %}\n {{ return(adapter.dispatch('date_spine', 'dbt')(datepart, start_date, end_date)) }}\n{%- endmacro %}", + "meta": {}, + "name": "date_spine", + "original_file_path": "macros/utils/date_spine.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/date_spine.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.date_spine" + }, "macro.dbt.date_trunc": { "arguments": [], - "created_at": 1696458269.797339, + "created_at": 1719485736.55547, "depends_on": { "macros": [ "macro.dbt.default__date_trunc" @@ -1055,14 +1135,12 @@ "patch_path": null, "path": "macros/utils/date_trunc.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.date_trunc" }, "macro.dbt.dateadd": { "arguments": [], - "created_at": 1696458269.776004, + "created_at": 1719485736.5368361, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__dateadd" @@ -1081,14 +1159,12 @@ "patch_path": null, "path": "macros/utils/dateadd.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.dateadd" }, "macro.dbt.datediff": { "arguments": [], - "created_at": 1696458269.781129, + "created_at": 1719485736.5402482, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__datediff" @@ -1107,14 +1183,12 @@ "patch_path": null, "path": "macros/utils/datediff.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.datediff" }, "macro.dbt.dates_in_range": { "arguments": [], - "created_at": 1696458269.7706811, + "created_at": 1719485736.524234, "depends_on": { "macros": [ "macro.dbt.convert_datetime" @@ -1125,7 +1199,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partiton start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", + "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "meta": {}, "name": "dates_in_range", "original_file_path": "macros/etc/datetime.sql", @@ -1133,14 +1207,12 @@ "patch_path": null, "path": "macros/etc/datetime.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.dates_in_range" }, "macro.dbt.default__alter_column_comment": { "arguments": [], - "created_at": 1696458269.832395, + "created_at": 1719485736.5851688, "depends_on": { "macros": [] }, @@ -1157,14 +1229,12 @@ "patch_path": null, "path": "macros/adapters/persist_docs.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__alter_column_comment" }, "macro.dbt.default__alter_column_type": { "arguments": [], - "created_at": 1696458269.846581, + "created_at": 1719485736.6003149, "depends_on": { "macros": [ "macro.dbt.statement" @@ -1183,14 +1253,12 @@ "patch_path": null, "path": "macros/adapters/columns.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__alter_column_type" }, "macro.dbt.default__alter_relation_add_remove_columns": { "arguments": [], - "created_at": 1696458269.848277, + "created_at": 1719485736.601542, "depends_on": { "macros": [ "macro.dbt.run_query" @@ -1209,14 +1277,12 @@ "patch_path": null, "path": "macros/adapters/columns.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__alter_relation_add_remove_columns" }, "macro.dbt.default__alter_relation_comment": { "arguments": [], - "created_at": 1696458269.83301, + "created_at": 1719485736.585629, "depends_on": { "macros": [] }, @@ -1233,14 +1299,12 @@ "patch_path": null, "path": "macros/adapters/persist_docs.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__alter_relation_comment" }, "macro.dbt.default__any_value": { "arguments": [], - "created_at": 1696458269.7848792, + "created_at": 1719485736.5430348, "depends_on": { "macros": [] }, @@ -1257,14 +1321,12 @@ "patch_path": null, "path": "macros/utils/any_value.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__any_value" }, "macro.dbt.default__apply_grants": { "arguments": [], - "created_at": 1696458269.830645, + "created_at": 1719485736.580575, "depends_on": { "macros": [ "macro.dbt.run_query", @@ -1286,14 +1348,12 @@ "patch_path": null, "path": "macros/adapters/apply_grants.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__apply_grants" }, "macro.dbt.default__array_append": { "arguments": [], - "created_at": 1696458269.7997708, + "created_at": 1719485736.557557, "depends_on": { "macros": [] }, @@ -1310,14 +1370,12 @@ "patch_path": null, "path": "macros/utils/array_append.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__array_append" }, "macro.dbt.default__array_concat": { "arguments": [], - "created_at": 1696458269.792536, + "created_at": 1719485736.552268, "depends_on": { "macros": [] }, @@ -1334,14 +1392,12 @@ "patch_path": null, "path": "macros/utils/array_concat.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__array_concat" }, "macro.dbt.default__array_construct": { "arguments": [], - "created_at": 1696458269.7988842, + "created_at": 1719485736.556629, "depends_on": { "macros": [] }, @@ -1358,14 +1414,12 @@ "patch_path": null, "path": "macros/utils/array_construct.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__array_construct" }, "macro.dbt.default__bool_or": { "arguments": [], - "created_at": 1696458269.793303, + "created_at": 1719485736.552653, "depends_on": { "macros": [] }, @@ -1382,14 +1436,12 @@ "patch_path": null, "path": "macros/utils/bool_or.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__bool_or" }, "macro.dbt.default__build_snapshot_table": { "arguments": [], - "created_at": 1696458269.619839, + "created_at": 1719485736.3577101, "depends_on": { "macros": [] }, @@ -1406,14 +1458,12 @@ "patch_path": null, "path": "macros/materializations/snapshots/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__build_snapshot_table" }, "macro.dbt.default__call_dcl_statements": { "arguments": [], - "created_at": 1696458269.828314, + "created_at": 1719485736.577475, "depends_on": { "macros": [ "macro.dbt.statement" @@ -1432,14 +1482,56 @@ "patch_path": null, "path": "macros/adapters/apply_grants.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__call_dcl_statements" }, + "macro.dbt.default__can_clone_table": { + "arguments": [], + "created_at": 1719485736.443028, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__can_clone_table() %}\n {{ return(False) }}\n{% endmacro %}", + "meta": {}, + "name": "default__can_clone_table", + "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/clone/can_clone_table.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__can_clone_table" + }, + "macro.dbt.default__cast": { + "arguments": [], + "created_at": 1719485736.5423732, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__cast(field, type) %}\n cast({{field}} as {{type}})\n{% endmacro %}", + "meta": {}, + "name": "default__cast", + "original_file_path": "macros/utils/cast.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/cast.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__cast" + }, "macro.dbt.default__cast_bool_to_text": { "arguments": [], - "created_at": 1696458269.784034, + "created_at": 1719485736.541875, "depends_on": { "macros": [] }, @@ -1456,14 +1548,12 @@ "patch_path": null, "path": "macros/utils/cast_bool_to_text.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__cast_bool_to_text" }, "macro.dbt.default__check_schema_exists": { "arguments": [], - "created_at": 1696458269.8393972, + "created_at": 1719485736.591805, "depends_on": { "macros": [ "macro.dbt.replace", @@ -1483,14 +1573,12 @@ "patch_path": null, "path": "macros/adapters/metadata.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__check_schema_exists" }, "macro.dbt.default__collect_freshness": { "arguments": [], - "created_at": 1696458269.820089, + "created_at": 1719485736.569667, "depends_on": { "macros": [ "macro.dbt.statement", @@ -1502,7 +1590,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness').table) }}\n{% endmacro %}", + "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness')) }}\n{% endmacro %}", "meta": {}, "name": "default__collect_freshness", "original_file_path": "macros/adapters/freshness.sql", @@ -1510,14 +1598,12 @@ "patch_path": null, "path": "macros/adapters/freshness.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__collect_freshness" }, "macro.dbt.default__concat": { "arguments": [], - "created_at": 1696458269.774613, + "created_at": 1719485736.5335412, "depends_on": { "macros": [] }, @@ -1534,14 +1620,12 @@ "patch_path": null, "path": "macros/utils/concat.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__concat" }, "macro.dbt.default__copy_grants": { "arguments": [], - "created_at": 1696458269.823154, + "created_at": 1719485736.5727968, "depends_on": { "macros": [] }, @@ -1558,14 +1642,12 @@ "patch_path": null, "path": "macros/adapters/apply_grants.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__copy_grants" }, "macro.dbt.default__create_columns": { "arguments": [], - "created_at": 1696458269.616479, + "created_at": 1719485736.354986, "depends_on": { "macros": [ "macro.dbt.statement" @@ -1584,14 +1666,12 @@ "patch_path": null, "path": "macros/materializations/snapshots/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__create_columns" }, "macro.dbt.default__create_csv_table": { "arguments": [], - "created_at": 1696458269.747761, + "created_at": 1719485736.4600089, "depends_on": { "macros": [ "macro.dbt.statement" @@ -1610,14 +1690,12 @@ "patch_path": null, "path": "macros/materializations/seeds/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__create_csv_table" }, "macro.dbt.default__create_indexes": { "arguments": [], - "created_at": 1696458269.806336, + "created_at": 1719485736.562071, "depends_on": { "macros": [ "macro.dbt.get_create_index_sql", @@ -1637,14 +1715,34 @@ "patch_path": null, "path": "macros/adapters/indexes.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__create_indexes" }, + "macro.dbt.default__create_or_replace_clone": { + "arguments": [], + "created_at": 1719485736.4434948, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation }} clone {{ defer_relation }}\n{% endmacro %}", + "meta": {}, + "name": "default__create_or_replace_clone", + "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/clone/create_or_replace_clone.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_or_replace_clone" + }, "macro.dbt.default__create_schema": { "arguments": [], - "created_at": 1696458269.800829, + "created_at": 1719485736.5583, "depends_on": { "macros": [ "macro.dbt.statement" @@ -1663,62 +1761,62 @@ "patch_path": null, "path": "macros/adapters/schema.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__create_schema" }, "macro.dbt.default__create_table_as": { "arguments": [], - "created_at": 1696458269.721521, + "created_at": 1719485736.509567, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.get_assert_columns_equivalent", + "macro.dbt.get_table_columns_and_constraints", + "macro.dbt.get_select_subquery" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n as (\n {{ sql }}\n );\n{%- endmacro %}", + "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced and (not temporary) %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "meta": {}, "name": "default__create_table_as", - "original_file_path": "macros/materializations/models/table/create_table_as.sql", + "original_file_path": "macros/relations/table/create.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/table/create_table_as.sql", + "path": "macros/relations/table/create.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__create_table_as" }, "macro.dbt.default__create_view_as": { "arguments": [], - "created_at": 1696458269.7317991, + "created_at": 1719485736.515955, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.get_assert_columns_equivalent" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }} as (\n {{ sql }}\n );\n{%- endmacro %}", + "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "meta": {}, "name": "default__create_view_as", - "original_file_path": "macros/materializations/models/view/create_view_as.sql", + "original_file_path": "macros/relations/view/create.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/view/create_view_as.sql", + "path": "macros/relations/view/create.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__create_view_as" }, "macro.dbt.default__current_timestamp": { "arguments": [], - "created_at": 1696458269.802537, + "created_at": 1719485736.5593758, "depends_on": { "macros": [] }, @@ -1735,14 +1833,12 @@ "patch_path": null, "path": "macros/adapters/timestamps.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__current_timestamp" }, "macro.dbt.default__current_timestamp_backcompat": { "arguments": [], - "created_at": 1696458269.8033202, + "created_at": 1719485736.559911, "depends_on": { "macros": [] }, @@ -1759,14 +1855,12 @@ "patch_path": null, "path": "macros/adapters/timestamps.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__current_timestamp_backcompat" }, "macro.dbt.default__current_timestamp_in_utc_backcompat": { "arguments": [], - "created_at": 1696458269.803849, + "created_at": 1719485736.560272, "depends_on": { "macros": [ "macro.dbt.current_timestamp_backcompat", @@ -1786,14 +1880,60 @@ "patch_path": null, "path": "macros/adapters/timestamps.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__current_timestamp_in_utc_backcompat" }, + "macro.dbt.default__date": { + "arguments": [], + "created_at": 1719485736.531544, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__date(year, month, day) -%}\n {%- set dt = modules.datetime.date(year, month, day) -%}\n {%- set iso_8601_formatted_date = dt.strftime('%Y-%m-%d') -%}\n to_date('{{ iso_8601_formatted_date }}', 'YYYY-MM-DD')\n{%- endmacro %}", + "meta": {}, + "name": "default__date", + "original_file_path": "macros/utils/date.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/date.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__date" + }, + "macro.dbt.default__date_spine": { + "arguments": [], + "created_at": 1719485736.53054, + "depends_on": { + "macros": [ + "macro.dbt.generate_series", + "macro.dbt.get_intervals_between", + "macro.dbt.dateadd" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__date_spine(datepart, start_date, end_date) %}\n\n\n {# call as follows:\n\n date_spine(\n \"day\",\n \"to_date('01/01/2016', 'mm/dd/yyyy')\",\n \"dbt.dateadd(week, 1, current_date)\"\n ) #}\n\n\n with rawdata as (\n\n {{dbt.generate_series(\n dbt.get_intervals_between(start_date, end_date, datepart)\n )}}\n\n ),\n\n all_periods as (\n\n select (\n {{\n dbt.dateadd(\n datepart,\n \"row_number() over (order by 1) - 1\",\n start_date\n )\n }}\n ) as date_{{datepart}}\n from rawdata\n\n ),\n\n filtered as (\n\n select *\n from all_periods\n where date_{{datepart}} <= {{ end_date }}\n\n )\n\n select * from filtered\n\n{% endmacro %}", + "meta": {}, + "name": "default__date_spine", + "original_file_path": "macros/utils/date_spine.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/date_spine.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__date_spine" + }, "macro.dbt.default__date_trunc": { "arguments": [], - "created_at": 1696458269.7975512, + "created_at": 1719485736.555618, "depends_on": { "macros": [] }, @@ -1810,14 +1950,12 @@ "patch_path": null, "path": "macros/utils/date_trunc.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__date_trunc" }, "macro.dbt.default__dateadd": { "arguments": [], - "created_at": 1696458269.7762759, + "created_at": 1719485736.537018, "depends_on": { "macros": [] }, @@ -1834,14 +1972,12 @@ "patch_path": null, "path": "macros/utils/dateadd.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__dateadd" }, "macro.dbt.default__datediff": { "arguments": [], - "created_at": 1696458269.7814112, + "created_at": 1719485736.540429, "depends_on": { "macros": [] }, @@ -1858,17 +1994,38 @@ "patch_path": null, "path": "macros/utils/datediff.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__datediff" }, + "macro.dbt.default__drop_materialized_view": { + "arguments": [], + "created_at": 1719485736.4865391, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation }} cascade\n{%- endmacro %}", + "meta": {}, + "name": "default__drop_materialized_view", + "original_file_path": "macros/relations/materialized_view/drop.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/materialized_view/drop.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__drop_materialized_view" + }, "macro.dbt.default__drop_relation": { "arguments": [], - "created_at": 1696458269.8147898, + "created_at": 1719485736.4714549, "depends_on": { "macros": [ - "macro.dbt.statement" + "macro.dbt.statement", + "macro.dbt.get_drop_sql" ] }, "description": "", @@ -1876,22 +2033,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n {%- endcall %}\n{% endmacro %}", + "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {{ get_drop_sql(relation) }}\n {%- endcall %}\n{% endmacro %}", "meta": {}, "name": "default__drop_relation", - "original_file_path": "macros/adapters/relation.sql", + "original_file_path": "macros/relations/drop.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/relation.sql", + "path": "macros/relations/drop.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__drop_relation" }, "macro.dbt.default__drop_schema": { "arguments": [], - "created_at": 1696458269.8013818, + "created_at": 1719485736.558672, "depends_on": { "macros": [ "macro.dbt.statement" @@ -1910,14 +2065,78 @@ "patch_path": null, "path": "macros/adapters/schema.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__drop_schema" }, + "macro.dbt.default__drop_schema_named": { + "arguments": [], + "created_at": 1719485736.476362, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__drop_schema_named(schema_name) %}\n {% set schema_relation = api.Relation.create(schema=schema_name) %}\n {{ adapter.drop_schema(schema_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "default__drop_schema_named", + "original_file_path": "macros/relations/schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__drop_schema_named" + }, + "macro.dbt.default__drop_table": { + "arguments": [], + "created_at": 1719485736.505599, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__drop_table(relation) -%}\n drop table if exists {{ relation }} cascade\n{%- endmacro %}", + "meta": {}, + "name": "default__drop_table", + "original_file_path": "macros/relations/table/drop.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/table/drop.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__drop_table" + }, + "macro.dbt.default__drop_view": { + "arguments": [], + "created_at": 1719485736.511198, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__drop_view(relation) -%}\n drop view if exists {{ relation }} cascade\n{%- endmacro %}", + "meta": {}, + "name": "default__drop_view", + "original_file_path": "macros/relations/view/drop.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/view/drop.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__drop_view" + }, "macro.dbt.default__escape_single_quotes": { "arguments": [], - "created_at": 1696458269.777675, + "created_at": 1719485736.538025, "depends_on": { "macros": [] }, @@ -1934,14 +2153,12 @@ "patch_path": null, "path": "macros/utils/escape_single_quotes.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__escape_single_quotes" }, "macro.dbt.default__except": { "arguments": [], - "created_at": 1696458269.7729568, + "created_at": 1719485736.526438, "depends_on": { "macros": [] }, @@ -1958,14 +2175,34 @@ "patch_path": null, "path": "macros/utils/except.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__except" }, + "macro.dbt.default__format_column": { + "arguments": [], + "created_at": 1719485736.504833, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__format_column(column) -%}\n {% set data_type = column.dtype %}\n {% set formatted = column.column.lower() ~ \" \" ~ data_type %}\n {{ return({'name': column.name, 'data_type': data_type, 'formatted': formatted}) }}\n{%- endmacro -%}", + "meta": {}, + "name": "default__format_column", + "original_file_path": "macros/relations/column/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/column/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__format_column" + }, "macro.dbt.default__generate_alias_name": { "arguments": [], - "created_at": 1696458269.7549078, + "created_at": 1719485736.466625, "depends_on": { "macros": [] }, @@ -1974,7 +2211,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name is none -%}\n\n {{ node.name }}\n\n {%- else -%}\n\n {{ custom_alias_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", + "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name -%}\n\n {{ custom_alias_name | trim }}\n\n {%- elif node.version -%}\n\n {{ return(node.name ~ \"_v\" ~ (node.version | replace(\".\", \"_\"))) }}\n\n {%- else -%}\n\n {{ node.name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "meta": {}, "name": "default__generate_alias_name", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", @@ -1982,14 +2219,12 @@ "patch_path": null, "path": "macros/get_custom_name/get_custom_alias.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__generate_alias_name" }, "macro.dbt.default__generate_database_name": { "arguments": [], - "created_at": 1696458269.758259, + "created_at": 1719485736.469049, "depends_on": { "macros": [] }, @@ -2006,14 +2241,12 @@ "patch_path": null, "path": "macros/get_custom_name/get_custom_database.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__generate_database_name" }, "macro.dbt.default__generate_schema_name": { "arguments": [], - "created_at": 1696458269.756475, + "created_at": 1719485736.4677172, "depends_on": { "macros": [] }, @@ -2030,14 +2263,82 @@ "patch_path": null, "path": "macros/get_custom_name/get_custom_schema.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__generate_schema_name" }, + "macro.dbt.default__generate_series": { + "arguments": [], + "created_at": 1719485736.535907, + "depends_on": { + "macros": [ + "macro.dbt.get_powers_of_two" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__generate_series(upper_bound) %}\n\n {% set n = dbt.get_powers_of_two(upper_bound) %}\n\n with p as (\n select 0 as generated_number union all select 1\n ), unioned as (\n\n select\n\n {% for i in range(n) %}\n p{{i}}.generated_number * power(2, {{i}})\n {% if not loop.last %} + {% endif %}\n {% endfor %}\n + 1\n as generated_number\n\n from\n\n {% for i in range(n) %}\n p as p{{i}}\n {% if not loop.last %} cross join {% endif %}\n {% endfor %}\n\n )\n\n select *\n from unioned\n where generated_number <= {{upper_bound}}\n order by generated_number\n\n{% endmacro %}", + "meta": {}, + "name": "default__generate_series", + "original_file_path": "macros/utils/generate_series.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/generate_series.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__generate_series" + }, + "macro.dbt.default__get_alter_materialized_view_as_sql": { + "arguments": [], + "created_at": 1719485736.4936142, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_alter_materialized_view_as_sql", + "original_file_path": "macros/relations/materialized_view/alter.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/materialized_view/alter.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_alter_materialized_view_as_sql" + }, + "macro.dbt.default__get_assert_columns_equivalent": { + "arguments": [], + "created_at": 1719485736.501285, + "depends_on": { + "macros": [ + "macro.dbt.assert_columns_equivalent" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_assert_columns_equivalent(sql) -%}\n {{ return(assert_columns_equivalent(sql)) }}\n{%- endmacro %}", + "meta": {}, + "name": "default__get_assert_columns_equivalent", + "original_file_path": "macros/relations/column/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/column/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_assert_columns_equivalent" + }, "macro.dbt.default__get_batch_size": { "arguments": [], - "created_at": 1696458269.750355, + "created_at": 1719485736.4618182, "depends_on": { "macros": [] }, @@ -2054,14 +2355,12 @@ "patch_path": null, "path": "macros/materializations/seeds/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_batch_size" }, "macro.dbt.default__get_binding_char": { "arguments": [], - "created_at": 1696458269.749901, + "created_at": 1719485736.461474, "depends_on": { "macros": [] }, @@ -2078,14 +2377,12 @@ "patch_path": null, "path": "macros/materializations/seeds/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_binding_char" }, "macro.dbt.default__get_catalog": { "arguments": [], - "created_at": 1696458269.8372722, + "created_at": 1719485736.590454, "depends_on": { "macros": [] }, @@ -2102,17 +2399,60 @@ "patch_path": null, "path": "macros/adapters/metadata.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_catalog" }, + "macro.dbt.default__get_catalog_relations": { + "arguments": [], + "created_at": 1719485736.589964, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_catalog_relations(information_schema, relations) -%}\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog_relations not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{%- endmacro %}", + "meta": {}, + "name": "default__get_catalog_relations", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_catalog_relations" + }, + "macro.dbt.default__get_column_names": { + "arguments": [], + "created_at": 1719485736.5100741, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_column_names() %}\n {#- loop through user_provided_columns to get column names -#}\n {%- set user_provided_columns = model['columns'] -%}\n {%- for i in user_provided_columns %}\n {%- set col = user_provided_columns[i] -%}\n {%- set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] -%}\n {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n{% endmacro %}", + "meta": {}, + "name": "default__get_column_names", + "original_file_path": "macros/relations/table/create.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/table/create.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_column_names" + }, "macro.dbt.default__get_columns_in_query": { "arguments": [], - "created_at": 1696458269.8452191, + "created_at": 1719485736.599448, "depends_on": { "macros": [ - "macro.dbt.statement" + "macro.dbt.statement", + "macro.dbt.get_empty_subquery_sql" ] }, "description": "", @@ -2120,7 +2460,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n {% endcall %}\n\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", + "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n {{ get_empty_subquery_sql(select_sql) }}\n {% endcall %}\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "meta": {}, "name": "default__get_columns_in_query", "original_file_path": "macros/adapters/columns.sql", @@ -2128,14 +2468,12 @@ "patch_path": null, "path": "macros/adapters/columns.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_columns_in_query" }, "macro.dbt.default__get_columns_in_relation": { "arguments": [], - "created_at": 1696458269.843777, + "created_at": 1719485736.59628, "depends_on": { "macros": [] }, @@ -2152,14 +2490,38 @@ "patch_path": null, "path": "macros/adapters/columns.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_columns_in_relation" }, + "macro.dbt.default__get_create_backup_sql": { + "arguments": [], + "created_at": 1719485736.47981, + "depends_on": { + "macros": [ + "macro.dbt.make_backup_relation", + "macro.dbt.get_drop_sql", + "macro.dbt.get_rename_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro default__get_create_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n -- drop any pre-existing backup\n {{ get_drop_sql(backup_relation) }};\n\n {{ get_rename_sql(relation, backup_relation.identifier) }}\n\n{%- endmacro -%}", + "meta": {}, + "name": "default__get_create_backup_sql", + "original_file_path": "macros/relations/create_backup.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/create_backup.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_create_backup_sql" + }, "macro.dbt.default__get_create_index_sql": { "arguments": [], - "created_at": 1696458269.805347, + "created_at": 1719485736.561354, "depends_on": { "macros": [] }, @@ -2176,14 +2538,86 @@ "patch_path": null, "path": "macros/adapters/indexes.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_create_index_sql" }, + "macro.dbt.default__get_create_intermediate_sql": { + "arguments": [], + "created_at": 1719485736.475838, + "depends_on": { + "macros": [ + "macro.dbt.make_intermediate_relation", + "macro.dbt.get_drop_sql", + "macro.dbt.get_create_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro default__get_create_intermediate_sql(relation, sql) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n -- drop any pre-existing intermediate\n {{ get_drop_sql(intermediate_relation) }};\n\n {{ get_create_sql(intermediate_relation, sql) }}\n\n{%- endmacro -%}", + "meta": {}, + "name": "default__get_create_intermediate_sql", + "original_file_path": "macros/relations/create_intermediate.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/create_intermediate.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_create_intermediate_sql" + }, + "macro.dbt.default__get_create_materialized_view_as_sql": { + "arguments": [], + "created_at": 1719485736.494792, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_create_materialized_view_as_sql(relation, sql) -%}\n {{ exceptions.raise_compiler_error(\n \"`get_create_materialized_view_as_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_create_materialized_view_as_sql", + "original_file_path": "macros/relations/materialized_view/create.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/materialized_view/create.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_create_materialized_view_as_sql" + }, + "macro.dbt.default__get_create_sql": { + "arguments": [], + "created_at": 1719485736.483476, + "depends_on": { + "macros": [ + "macro.dbt.get_create_view_as_sql", + "macro.dbt.get_create_table_as_sql", + "macro.dbt.get_create_materialized_view_as_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro default__get_create_sql(relation, sql) -%}\n\n {%- if relation.is_view -%}\n {{ get_create_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_table -%}\n {{ get_create_table_as_sql(False, relation, sql) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_create_materialized_view_as_sql(relation, sql) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_create_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}", + "meta": {}, + "name": "default__get_create_sql", + "original_file_path": "macros/relations/create.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/create.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_create_sql" + }, "macro.dbt.default__get_create_table_as_sql": { "arguments": [], - "created_at": 1696458269.720119, + "created_at": 1719485736.508211, "depends_on": { "macros": [ "macro.dbt.create_table_as" @@ -2197,19 +2631,17 @@ "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", "meta": {}, "name": "default__get_create_table_as_sql", - "original_file_path": "macros/materializations/models/table/create_table_as.sql", + "original_file_path": "macros/relations/table/create.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/table/create_table_as.sql", + "path": "macros/relations/table/create.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_create_table_as_sql" }, "macro.dbt.default__get_create_view_as_sql": { "arguments": [], - "created_at": 1696458269.7310889, + "created_at": 1719485736.515317, "depends_on": { "macros": [ "macro.dbt.create_view_as" @@ -2223,19 +2655,17 @@ "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", "meta": {}, "name": "default__get_create_view_as_sql", - "original_file_path": "macros/materializations/models/view/create_view_as.sql", + "original_file_path": "macros/relations/view/create.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/view/create_view_as.sql", + "path": "macros/relations/view/create.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_create_view_as_sql" }, "macro.dbt.default__get_csv_sql": { "arguments": [], - "created_at": 1696458269.749481, + "created_at": 1719485736.461133, "depends_on": { "macros": [] }, @@ -2252,14 +2682,12 @@ "patch_path": null, "path": "macros/materializations/seeds/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_csv_sql" }, "macro.dbt.default__get_dcl_statement_list": { "arguments": [], - "created_at": 1696458269.827488, + "created_at": 1719485736.5756311, "depends_on": { "macros": [ "macro.dbt.support_multiple_grantees_per_dcl_statement" @@ -2278,14 +2706,12 @@ "patch_path": null, "path": "macros/adapters/apply_grants.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_dcl_statement_list" }, "macro.dbt.default__get_delete_insert_merge_sql": { "arguments": [], - "created_at": 1696458269.6825862, + "created_at": 1719485736.415282, "depends_on": { "macros": [ "macro.dbt.get_quoted_csv" @@ -2296,7 +2722,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not string %}\n delete from {{target }}\n using {{ source }}\n where (\n {% for key in unique_key %}\n {{ source }}.{{ key }} = {{ target }}.{{ key }}\n {{ \"and \" if not loop.last }}\n {% endfor %}\n );\n {% else %}\n delete from {{ target }}\n where (\n {{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n );\n\n {% endif %}\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", + "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not string %}\n delete from {{target }}\n using {{ source }}\n where (\n {% for key in unique_key %}\n {{ source }}.{{ key }} = {{ target }}.{{ key }}\n {{ \"and \" if not loop.last}}\n {% endfor %}\n {% if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {% endif %}\n );\n {% else %}\n delete from {{ target }}\n where (\n {{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n )\n {%- if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {%- endif -%};\n\n {% endif %}\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "meta": {}, "name": "default__get_delete_insert_merge_sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", @@ -2304,38 +2730,153 @@ "patch_path": null, "path": "macros/materializations/models/incremental/merge.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_delete_insert_merge_sql" }, - "macro.dbt.default__get_grant_sql": { + "macro.dbt.default__get_drop_backup_sql": { "arguments": [], - "created_at": 1696458269.825314, + "created_at": 1719485736.476908, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.make_backup_relation", + "macro.dbt.get_drop_sql" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", + "macro_sql": "{%- macro default__get_drop_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n {{ get_drop_sql(backup_relation) }}\n\n{%- endmacro -%}", "meta": {}, - "name": "default__get_grant_sql", - "original_file_path": "macros/adapters/apply_grants.sql", + "name": "default__get_drop_backup_sql", + "original_file_path": "macros/relations/drop_backup.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/apply_grants.sql", + "path": "macros/relations/drop_backup.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_drop_backup_sql" + }, + "macro.dbt.default__get_drop_index_sql": { + "arguments": [], + "created_at": 1719485736.562772, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_drop_index_sql(relation, index_name) -%}\n {{ exceptions.raise_compiler_error(\"`get_drop_index_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", + "meta": {}, + "name": "default__get_drop_index_sql", + "original_file_path": "macros/adapters/indexes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/indexes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_drop_index_sql" + }, + "macro.dbt.default__get_drop_sql": { + "arguments": [], + "created_at": 1719485736.471034, + "depends_on": { + "macros": [ + "macro.dbt.drop_view", + "macro.dbt.drop_table", + "macro.dbt.drop_materialized_view" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro default__get_drop_sql(relation) -%}\n\n {%- if relation.is_view -%}\n {{ drop_view(relation) }}\n\n {%- elif relation.is_table -%}\n {{ drop_table(relation) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n\n {%- else -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "default__get_drop_sql", + "original_file_path": "macros/relations/drop.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/drop.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_drop_sql" + }, + "macro.dbt.default__get_empty_schema_sql": { + "arguments": [], + "created_at": 1719485736.5985012, + "depends_on": { + "macros": [ + "macro.dbt.cast" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_empty_schema_sql(columns) %}\n {%- set col_err = [] -%}\n {%- set col_naked_numeric = [] -%}\n select\n {% for i in columns %}\n {%- set col = columns[i] -%}\n {%- if col['data_type'] is not defined -%}\n {%- do col_err.append(col['name']) -%}\n {#-- If this column's type is just 'numeric' then it is missing precision/scale, raise a warning --#}\n {%- elif col['data_type'].strip().lower() in ('numeric', 'decimal', 'number') -%}\n {%- do col_naked_numeric.append(col['name']) -%}\n {%- endif -%}\n {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %}\n {{ cast('null', col['data_type']) }} as {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n {%- if (col_err | length) > 0 -%}\n {{ exceptions.column_type_missing(column_names=col_err) }}\n {%- elif (col_naked_numeric | length) > 0 -%}\n {{ exceptions.warn(\"Detected columns with numeric type and unspecified precision/scale, this can lead to unintended rounding: \" ~ col_naked_numeric ~ \"`\") }}\n {%- endif -%}\n{% endmacro %}", + "meta": {}, + "name": "default__get_empty_schema_sql", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_empty_schema_sql" + }, + "macro.dbt.default__get_empty_subquery_sql": { + "arguments": [], + "created_at": 1719485736.597066, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_empty_subquery_sql(select_sql, select_sql_header=none) %}\n {%- if select_sql_header is not none -%}\n {{ select_sql_header }}\n {%- endif -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n{% endmacro %}", + "meta": {}, + "name": "default__get_empty_subquery_sql", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_empty_subquery_sql" + }, + "macro.dbt.default__get_grant_sql": { + "arguments": [], + "created_at": 1719485736.5742211, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "default__get_grant_sql", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_grant_sql" }, "macro.dbt.default__get_incremental_append_sql": { "arguments": [], - "created_at": 1696458269.687379, + "created_at": 1719485736.418623, "depends_on": { "macros": [ "macro.dbt.get_insert_into_sql" @@ -2354,14 +2895,12 @@ "patch_path": null, "path": "macros/materializations/models/incremental/strategies.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_incremental_append_sql" }, "macro.dbt.default__get_incremental_default_sql": { "arguments": [], - "created_at": 1696458269.690251, + "created_at": 1719485736.4215739, "depends_on": { "macros": [ "macro.dbt.get_incremental_append_sql" @@ -2380,14 +2919,12 @@ "patch_path": null, "path": "macros/materializations/models/incremental/strategies.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_incremental_default_sql" }, "macro.dbt.default__get_incremental_delete_insert_sql": { "arguments": [], - "created_at": 1696458269.688123, + "created_at": 1719485736.419171, "depends_on": { "macros": [ "macro.dbt.get_delete_insert_merge_sql" @@ -2398,7 +2935,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__get_incremental_delete_insert_sql(arg_dict) %}\n\n {% do return(get_delete_insert_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", + "macro_sql": "{% macro default__get_incremental_delete_insert_sql(arg_dict) %}\n\n {% do return(get_delete_insert_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "meta": {}, "name": "default__get_incremental_delete_insert_sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", @@ -2406,14 +2943,12 @@ "patch_path": null, "path": "macros/materializations/models/incremental/strategies.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_incremental_delete_insert_sql" }, "macro.dbt.default__get_incremental_insert_overwrite_sql": { "arguments": [], - "created_at": 1696458269.689686, + "created_at": 1719485736.421212, "depends_on": { "macros": [ "macro.dbt.get_insert_overwrite_merge_sql" @@ -2424,7 +2959,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {% do return(get_insert_overwrite_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"], arg_dict[\"predicates\"])) %}\n\n{% endmacro %}", + "macro_sql": "{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {% do return(get_insert_overwrite_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "meta": {}, "name": "default__get_incremental_insert_overwrite_sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", @@ -2432,14 +2967,12 @@ "patch_path": null, "path": "macros/materializations/models/incremental/strategies.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_incremental_insert_overwrite_sql" }, "macro.dbt.default__get_incremental_merge_sql": { "arguments": [], - "created_at": 1696458269.6889029, + "created_at": 1719485736.420272, "depends_on": { "macros": [ "macro.dbt.get_merge_sql" @@ -2450,7 +2983,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__get_incremental_merge_sql(arg_dict) %}\n\n {% do return(get_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", + "macro_sql": "{% macro default__get_incremental_merge_sql(arg_dict) %}\n\n {% do return(get_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "meta": {}, "name": "default__get_incremental_merge_sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", @@ -2458,14 +2991,12 @@ "patch_path": null, "path": "macros/materializations/models/incremental/strategies.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_incremental_merge_sql" }, "macro.dbt.default__get_insert_overwrite_merge_sql": { "arguments": [], - "created_at": 1696458269.684117, + "created_at": 1719485736.416304, "depends_on": { "macros": [ "macro.dbt.get_quoted_csv" @@ -2484,14 +3015,81 @@ "patch_path": null, "path": "macros/materializations/models/incremental/merge.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql" }, + "macro.dbt.default__get_intervals_between": { + "arguments": [], + "created_at": 1719485736.5290911, + "depends_on": { + "macros": [ + "macro.dbt.statement", + "macro.dbt.datediff" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_intervals_between(start_date, end_date, datepart) -%}\n {%- call statement('get_intervals_between', fetch_result=True) %}\n\n select {{ dbt.datediff(start_date, end_date, datepart) }}\n\n {%- endcall -%}\n\n {%- set value_list = load_result('get_intervals_between') -%}\n\n {%- if value_list and value_list['data'] -%}\n {%- set values = value_list['data'] | map(attribute=0) | list %}\n {{ return(values[0]) }}\n {%- else -%}\n {{ return(1) }}\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "default__get_intervals_between", + "original_file_path": "macros/utils/date_spine.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/date_spine.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_intervals_between" + }, + "macro.dbt.default__get_limit_subquery_sql": { + "arguments": [], + "created_at": 1719485736.581942, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_limit_subquery_sql(sql, limit) %}\n select *\n from (\n {{ sql }}\n ) as model_limit_subq\n limit {{ limit }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_limit_subquery_sql", + "original_file_path": "macros/adapters/show.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/show.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_limit_subquery_sql" + }, + "macro.dbt.default__get_materialized_view_configuration_changes": { + "arguments": [], + "created_at": 1719485736.4943428, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_materialized_view_configuration_changes(existing_relation, new_config) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_materialized_view_configuration_changes", + "original_file_path": "macros/relations/materialized_view/alter.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/materialized_view/alter.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_materialized_view_configuration_changes" + }, "macro.dbt.default__get_merge_sql": { "arguments": [], - "created_at": 1696458269.680794, + "created_at": 1719485736.413476, "depends_on": { "macros": [ "macro.dbt.get_quoted_csv", @@ -2503,7 +3101,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, predicates) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", + "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{\"(\" ~ predicates | join(\") and (\") ~ \")\"}}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "meta": {}, "name": "default__get_merge_sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", @@ -2511,14 +3109,12 @@ "patch_path": null, "path": "macros/materializations/models/incremental/merge.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_merge_sql" }, "macro.dbt.default__get_merge_update_columns": { "arguments": [], - "created_at": 1696458269.67032, + "created_at": 1719485736.399054, "depends_on": { "macros": [] }, @@ -2535,14 +3131,12 @@ "patch_path": null, "path": "macros/materializations/models/incremental/column_helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_merge_update_columns" }, "macro.dbt.default__get_or_create_relation": { "arguments": [], - "created_at": 1696458269.817579, + "created_at": 1719485736.568331, "depends_on": { "macros": [] }, @@ -2559,14 +3153,12 @@ "patch_path": null, "path": "macros/adapters/relation.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt.default__get_or_create_relation" }, - "macro.dbt.default__get_revoke_sql": { + "macro.dbt.default__get_powers_of_two": { "arguments": [], - "created_at": 1696458269.8259919, + "created_at": 1719485736.534994, "depends_on": { "macros": [] }, @@ -2575,22 +3167,20 @@ "node_color": null, "show": true }, - "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", + "macro_sql": "{% macro default__get_powers_of_two(upper_bound) %}\n\n {% if upper_bound <= 0 %}\n {{ exceptions.raise_compiler_error(\"upper bound must be positive\") }}\n {% endif %}\n\n {% for _ in range(1, 100) %}\n {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %}\n {% endfor %}\n\n{% endmacro %}", "meta": {}, - "name": "default__get_revoke_sql", - "original_file_path": "macros/adapters/apply_grants.sql", + "name": "default__get_powers_of_two", + "original_file_path": "macros/utils/generate_series.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/apply_grants.sql", + "path": "macros/utils/generate_series.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__get_revoke_sql" + "unique_id": "macro.dbt.default__get_powers_of_two" }, - "macro.dbt.default__get_show_grant_sql": { + "macro.dbt.default__get_relation_last_modified": { "arguments": [], - "created_at": 1696458269.824637, + "created_at": 1719485736.5935519, "depends_on": { "macros": [] }, @@ -2599,22 +3189,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation }}\n{% endmacro %}", + "macro_sql": "{% macro default__get_relation_last_modified(information_schema, relations) %}\n {{ exceptions.raise_not_implemented(\n 'get_relation_last_modified macro not implemented for adapter ' + adapter.type()) }}\n{% endmacro %}", "meta": {}, - "name": "default__get_show_grant_sql", - "original_file_path": "macros/adapters/apply_grants.sql", + "name": "default__get_relation_last_modified", + "original_file_path": "macros/adapters/metadata.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/apply_grants.sql", + "path": "macros/adapters/metadata.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__get_show_grant_sql" + "unique_id": "macro.dbt.default__get_relation_last_modified" }, - "macro.dbt.default__get_test_sql": { + "macro.dbt.default__get_relations": { "arguments": [], - "created_at": 1696458269.662534, + "created_at": 1719485736.592618, "depends_on": { "macros": [] }, @@ -2623,46 +3211,45 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", + "macro_sql": "{% macro default__get_relations() %}\n {{ exceptions.raise_not_implemented(\n 'get_relations macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "meta": {}, - "name": "default__get_test_sql", - "original_file_path": "macros/materializations/tests/helpers.sql", + "name": "default__get_relations", + "original_file_path": "macros/adapters/metadata.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/tests/helpers.sql", + "path": "macros/adapters/metadata.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__get_test_sql" + "unique_id": "macro.dbt.default__get_relations" }, - "macro.dbt.default__get_true_sql": { + "macro.dbt.default__get_rename_intermediate_sql": { "arguments": [], - "created_at": 1696458269.6173341, + "created_at": 1719485736.485399, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.make_intermediate_relation", + "macro.dbt.get_rename_sql" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", + "macro_sql": "{%- macro default__get_rename_intermediate_sql(relation) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n {{ get_rename_sql(intermediate_relation, relation.identifier) }}\n\n{%- endmacro -%}", "meta": {}, - "name": "default__get_true_sql", - "original_file_path": "macros/materializations/snapshots/helpers.sql", + "name": "default__get_rename_intermediate_sql", + "original_file_path": "macros/relations/rename_intermediate.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/snapshots/helpers.sql", + "path": "macros/relations/rename_intermediate.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__get_true_sql" + "unique_id": "macro.dbt.default__get_rename_intermediate_sql" }, - "macro.dbt.default__get_where_subquery": { + "macro.dbt.default__get_rename_materialized_view_sql": { "arguments": [], - "created_at": 1696458269.6639218, + "created_at": 1719485736.491203, "depends_on": { "macros": [] }, @@ -2671,46 +3258,46 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", + "macro_sql": "{% macro default__get_rename_materialized_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "meta": {}, - "name": "default__get_where_subquery", - "original_file_path": "macros/materializations/tests/where_subquery.sql", + "name": "default__get_rename_materialized_view_sql", + "original_file_path": "macros/relations/materialized_view/rename.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/tests/where_subquery.sql", + "path": "macros/relations/materialized_view/rename.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__get_where_subquery" + "unique_id": "macro.dbt.default__get_rename_materialized_view_sql" }, - "macro.dbt.default__handle_existing_table": { + "macro.dbt.default__get_rename_sql": { "arguments": [], - "created_at": 1696458269.727338, + "created_at": 1719485736.4781098, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.get_rename_view_sql", + "macro.dbt.get_rename_table_sql", + "macro.dbt.get_rename_materialized_view_sql" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", + "macro_sql": "{%- macro default__get_rename_sql(relation, new_name) -%}\n\n {%- if relation.is_view -%}\n {{ get_rename_view_sql(relation, new_name) }}\n\n {%- elif relation.is_table -%}\n {{ get_rename_table_sql(relation, new_name) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_rename_materialized_view_sql(relation, new_name) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_rename_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "meta": {}, - "name": "default__handle_existing_table", - "original_file_path": "macros/materializations/models/view/helpers.sql", + "name": "default__get_rename_sql", + "original_file_path": "macros/relations/rename.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/view/helpers.sql", + "path": "macros/relations/rename.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__handle_existing_table" + "unique_id": "macro.dbt.default__get_rename_sql" }, - "macro.dbt.default__hash": { + "macro.dbt.default__get_rename_table_sql": { "arguments": [], - "created_at": 1696458269.783129, + "created_at": 1719485736.507071, "depends_on": { "macros": [] }, @@ -2719,22 +3306,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", + "macro_sql": "{% macro default__get_rename_table_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "meta": {}, - "name": "default__hash", - "original_file_path": "macros/utils/hash.sql", + "name": "default__get_rename_table_sql", + "original_file_path": "macros/relations/table/rename.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/hash.sql", + "path": "macros/relations/table/rename.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__hash" + "unique_id": "macro.dbt.default__get_rename_table_sql" }, - "macro.dbt.default__information_schema_name": { + "macro.dbt.default__get_rename_view_sql": { "arguments": [], - "created_at": 1696458269.8378282, + "created_at": 1719485736.514697, "depends_on": { "macros": [] }, @@ -2743,22 +3328,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", + "macro_sql": "{% macro default__get_rename_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "meta": {}, - "name": "default__information_schema_name", - "original_file_path": "macros/adapters/metadata.sql", + "name": "default__get_rename_view_sql", + "original_file_path": "macros/relations/view/rename.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/metadata.sql", + "path": "macros/relations/view/rename.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__information_schema_name" + "unique_id": "macro.dbt.default__get_rename_view_sql" }, - "macro.dbt.default__intersect": { + "macro.dbt.default__get_replace_materialized_view_sql": { "arguments": [], - "created_at": 1696458269.776896, + "created_at": 1719485736.4885828, "depends_on": { "macros": [] }, @@ -2767,25 +3350,31 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", + "macro_sql": "{% macro default__get_replace_materialized_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "meta": {}, - "name": "default__intersect", - "original_file_path": "macros/utils/intersect.sql", + "name": "default__get_replace_materialized_view_sql", + "original_file_path": "macros/relations/materialized_view/replace.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/intersect.sql", + "path": "macros/relations/materialized_view/replace.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__intersect" + "unique_id": "macro.dbt.default__get_replace_materialized_view_sql" }, - "macro.dbt.default__last_day": { + "macro.dbt.default__get_replace_sql": { "arguments": [], - "created_at": 1696458269.794904, + "created_at": 1719485736.475151, "depends_on": { "macros": [ - "macro.dbt.default_last_day" + "macro.dbt.get_replace_view_sql", + "macro.dbt.get_replace_table_sql", + "macro.dbt.get_replace_materialized_view_sql", + "macro.dbt.get_create_intermediate_sql", + "macro.dbt.get_create_backup_sql", + "macro.dbt.get_rename_intermediate_sql", + "macro.dbt.get_drop_backup_sql", + "macro.dbt.get_drop_sql", + "macro.dbt.get_create_sql" ] }, "description": "", @@ -2793,22 +3382,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", + "macro_sql": "{% macro default__get_replace_sql(existing_relation, target_relation, sql) %}\n\n {# /* use a create or replace statement if possible */ #}\n\n {% set is_replaceable = existing_relation.type == target_relation_type and existing_relation.can_be_replaced %}\n\n {% if is_replaceable and existing_relation.is_view %}\n {{ get_replace_view_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_table %}\n {{ get_replace_table_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_materialized_view %}\n {{ get_replace_materialized_view_sql(target_relation, sql) }}\n\n {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}\n {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}\n {%- elif target_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_drop_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }}\n\n {# /* create target_relation in place by first backing up the existing relation */ #}\n {%- elif existing_relation.can_be_renamed -%}\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* no renaming is allowed, so just drop and create */ #}\n {%- else -%}\n {{ get_drop_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }}\n\n {%- endif -%}\n\n{% endmacro %}", "meta": {}, - "name": "default__last_day", - "original_file_path": "macros/utils/last_day.sql", + "name": "default__get_replace_sql", + "original_file_path": "macros/relations/replace.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/last_day.sql", + "path": "macros/relations/replace.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__last_day" + "unique_id": "macro.dbt.default__get_replace_sql" }, - "macro.dbt.default__length": { + "macro.dbt.default__get_replace_table_sql": { "arguments": [], - "created_at": 1696458269.7753239, + "created_at": 1719485736.506342, "depends_on": { "macros": [] }, @@ -2817,22 +3404,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", + "macro_sql": "{% macro default__get_replace_table_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "meta": {}, - "name": "default__length", - "original_file_path": "macros/utils/length.sql", + "name": "default__get_replace_table_sql", + "original_file_path": "macros/relations/table/replace.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/length.sql", + "path": "macros/relations/table/replace.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__length" + "unique_id": "macro.dbt.default__get_replace_table_sql" }, - "macro.dbt.default__list_relations_without_caching": { + "macro.dbt.default__get_replace_view_sql": { "arguments": [], - "created_at": 1696458269.839978, + "created_at": 1719485736.512308, "depends_on": { "macros": [] }, @@ -2841,101 +3426,88 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", + "macro_sql": "{% macro default__get_replace_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "meta": {}, - "name": "default__list_relations_without_caching", - "original_file_path": "macros/adapters/metadata.sql", + "name": "default__get_replace_view_sql", + "original_file_path": "macros/relations/view/replace.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/metadata.sql", + "path": "macros/relations/view/replace.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__list_relations_without_caching" + "unique_id": "macro.dbt.default__get_replace_view_sql" }, - "macro.dbt.default__list_schemas": { + "macro.dbt.default__get_revoke_sql": { "arguments": [], - "created_at": 1696458269.83854, + "created_at": 1719485736.574667, "depends_on": { - "macros": [ - "macro.dbt.information_schema_name", - "macro.dbt.run_query" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", + "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "meta": {}, - "name": "default__list_schemas", - "original_file_path": "macros/adapters/metadata.sql", + "name": "default__get_revoke_sql", + "original_file_path": "macros/adapters/apply_grants.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/metadata.sql", + "path": "macros/adapters/apply_grants.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__list_schemas" + "unique_id": "macro.dbt.default__get_revoke_sql" }, - "macro.dbt.default__listagg": { + "macro.dbt.default__get_select_subquery": { "arguments": [], - "created_at": 1696458269.780426, + "created_at": 1719485736.510467, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.default__get_column_names" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", + "macro_sql": "{% macro default__get_select_subquery(sql) %}\n select {{ adapter.dispatch('get_column_names', 'dbt')() }}\n from (\n {{ sql }}\n ) as model_subq\n{%- endmacro %}", "meta": {}, - "name": "default__listagg", - "original_file_path": "macros/utils/listagg.sql", + "name": "default__get_select_subquery", + "original_file_path": "macros/relations/table/create.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/listagg.sql", + "path": "macros/relations/table/create.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__listagg" + "unique_id": "macro.dbt.default__get_select_subquery" }, - "macro.dbt.default__load_csv_rows": { + "macro.dbt.default__get_show_grant_sql": { "arguments": [], - "created_at": 1696458269.753568, + "created_at": 1719485736.573777, "depends_on": { - "macros": [ - "macro.dbt.get_batch_size", - "macro.dbt.get_seed_column_quoted_csv", - "macro.dbt.get_binding_char" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", + "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation }}\n{% endmacro %}", "meta": {}, - "name": "default__load_csv_rows", - "original_file_path": "macros/materializations/seeds/helpers.sql", + "name": "default__get_show_grant_sql", + "original_file_path": "macros/adapters/apply_grants.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/seeds/helpers.sql", + "path": "macros/adapters/apply_grants.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__load_csv_rows" + "unique_id": "macro.dbt.default__get_show_grant_sql" }, - "macro.dbt.default__make_backup_relation": { + "macro.dbt.default__get_show_indexes_sql": { "arguments": [], - "created_at": 1696458269.814139, + "created_at": 1719485736.563092, "depends_on": { "macros": [] }, @@ -2944,25 +3516,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", + "macro_sql": "{% macro default__get_show_indexes_sql(relation) -%}\n {{ exceptions.raise_compiler_error(\"`get_show_indexes_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "meta": {}, - "name": "default__make_backup_relation", - "original_file_path": "macros/adapters/relation.sql", + "name": "default__get_show_indexes_sql", + "original_file_path": "macros/adapters/indexes.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/relation.sql", + "path": "macros/adapters/indexes.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__make_backup_relation" + "unique_id": "macro.dbt.default__get_show_indexes_sql" }, - "macro.dbt.default__make_intermediate_relation": { + "macro.dbt.default__get_table_columns_and_constraints": { "arguments": [], - "created_at": 1696458269.812382, + "created_at": 1719485736.497637, "depends_on": { "macros": [ - "macro.dbt.default__make_temp_relation" + "macro.dbt.table_columns_and_constraints" ] }, "description": "", @@ -2970,22 +3540,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", + "macro_sql": "{% macro default__get_table_columns_and_constraints() -%}\n {{ return(table_columns_and_constraints()) }}\n{%- endmacro %}", "meta": {}, - "name": "default__make_intermediate_relation", - "original_file_path": "macros/adapters/relation.sql", + "name": "default__get_table_columns_and_constraints", + "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/relation.sql", + "path": "macros/relations/column/columns_spec_ddl.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__make_intermediate_relation" + "unique_id": "macro.dbt.default__get_table_columns_and_constraints" }, - "macro.dbt.default__make_temp_relation": { + "macro.dbt.default__get_test_sql": { "arguments": [], - "created_at": 1696458269.8132212, + "created_at": 1719485736.372639, "depends_on": { "macros": [] }, @@ -2994,74 +3562,66 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", + "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "meta": {}, - "name": "default__make_temp_relation", - "original_file_path": "macros/adapters/relation.sql", + "name": "default__get_test_sql", + "original_file_path": "macros/materializations/tests/helpers.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/relation.sql", + "path": "macros/materializations/tests/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__make_temp_relation" + "unique_id": "macro.dbt.default__get_test_sql" }, - "macro.dbt.default__persist_docs": { + "macro.dbt.default__get_true_sql": { "arguments": [], - "created_at": 1696458269.834239, + "created_at": 1719485736.3555431, "depends_on": { - "macros": [ - "macro.dbt.run_query", - "macro.dbt.alter_relation_comment", - "macro.dbt.alter_column_comment" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", + "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", "meta": {}, - "name": "default__persist_docs", - "original_file_path": "macros/adapters/persist_docs.sql", + "name": "default__get_true_sql", + "original_file_path": "macros/materializations/snapshots/helpers.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/persist_docs.sql", + "path": "macros/materializations/snapshots/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__persist_docs" + "unique_id": "macro.dbt.default__get_true_sql" }, - "macro.dbt.default__position": { + "macro.dbt.default__get_unit_test_sql": { "arguments": [], - "created_at": 1696458269.785729, + "created_at": 1719485736.3736708, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.string_literal" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", + "macro_sql": "{% macro default__get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n-- Build actual result given inputs\nwith dbt_internal_unit_test_actual as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%},{% endif %}{%- endfor -%}, {{ dbt.string_literal(\"actual\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ main_sql }}\n ) _dbt_internal_unit_test_actual\n),\n-- Build expected result\ndbt_internal_unit_test_expected as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%}, {% endif %}{%- endfor -%}, {{ dbt.string_literal(\"expected\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ expected_fixture_sql }}\n ) _dbt_internal_unit_test_expected\n)\n-- Union actual and expected results\nselect * from dbt_internal_unit_test_actual\nunion all\nselect * from dbt_internal_unit_test_expected\n{%- endmacro %}", "meta": {}, - "name": "default__position", - "original_file_path": "macros/utils/position.sql", + "name": "default__get_unit_test_sql", + "original_file_path": "macros/materializations/tests/helpers.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/position.sql", + "path": "macros/materializations/tests/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__position" + "unique_id": "macro.dbt.default__get_unit_test_sql" }, - "macro.dbt.default__post_snapshot": { + "macro.dbt.default__get_where_subquery": { "arguments": [], - "created_at": 1696458269.6169112, + "created_at": 1719485736.374502, "depends_on": { "macros": [] }, @@ -3070,48 +3630,42 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", + "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "meta": {}, - "name": "default__post_snapshot", - "original_file_path": "macros/materializations/snapshots/helpers.sql", + "name": "default__get_where_subquery", + "original_file_path": "macros/materializations/tests/where_subquery.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/snapshots/helpers.sql", + "path": "macros/materializations/tests/where_subquery.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__post_snapshot" + "unique_id": "macro.dbt.default__get_where_subquery" }, - "macro.dbt.default__rename_relation": { + "macro.dbt.default__handle_existing_table": { "arguments": [], - "created_at": 1696458269.8161411, + "created_at": 1719485736.514176, "depends_on": { - "macros": [ - "macro.dbt.statement" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", + "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "meta": {}, - "name": "default__rename_relation", - "original_file_path": "macros/adapters/relation.sql", + "name": "default__handle_existing_table", + "original_file_path": "macros/relations/view/replace.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/relation.sql", + "path": "macros/relations/view/replace.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__rename_relation" + "unique_id": "macro.dbt.default__handle_existing_table" }, - "macro.dbt.default__replace": { + "macro.dbt.default__hash": { "arguments": [], - "created_at": 1696458269.77388, + "created_at": 1719485736.541434, "depends_on": { "macros": [] }, @@ -3120,48 +3674,42 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", + "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", "meta": {}, - "name": "default__replace", - "original_file_path": "macros/utils/replace.sql", + "name": "default__hash", + "original_file_path": "macros/utils/hash.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/replace.sql", + "path": "macros/utils/hash.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__replace" + "unique_id": "macro.dbt.default__hash" }, - "macro.dbt.default__reset_csv_table": { + "macro.dbt.default__information_schema_name": { "arguments": [], - "created_at": 1696458269.748949, + "created_at": 1719485736.590823, "depends_on": { - "macros": [ - "macro.dbt.create_csv_table" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", + "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "meta": {}, - "name": "default__reset_csv_table", - "original_file_path": "macros/materializations/seeds/helpers.sql", + "name": "default__information_schema_name", + "original_file_path": "macros/adapters/metadata.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/seeds/helpers.sql", + "path": "macros/adapters/metadata.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__reset_csv_table" + "unique_id": "macro.dbt.default__information_schema_name" }, - "macro.dbt.default__right": { + "macro.dbt.default__intersect": { "arguments": [], - "created_at": 1696458269.778592, + "created_at": 1719485736.537451, "depends_on": { "macros": [] }, @@ -3170,72 +3718,66 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", + "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", "meta": {}, - "name": "default__right", - "original_file_path": "macros/utils/right.sql", + "name": "default__intersect", + "original_file_path": "macros/utils/intersect.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/right.sql", + "path": "macros/utils/intersect.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__right" + "unique_id": "macro.dbt.default__intersect" }, - "macro.dbt.default__safe_cast": { + "macro.dbt.default__last_day": { "arguments": [], - "created_at": 1696458269.782259, + "created_at": 1719485736.55349, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.default_last_day" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", + "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", "meta": {}, - "name": "default__safe_cast", - "original_file_path": "macros/utils/safe_cast.sql", + "name": "default__last_day", + "original_file_path": "macros/utils/last_day.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/safe_cast.sql", + "path": "macros/utils/last_day.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__safe_cast" + "unique_id": "macro.dbt.default__last_day" }, - "macro.dbt.default__snapshot_get_time": { + "macro.dbt.default__length": { "arguments": [], - "created_at": 1696458269.8029382, + "created_at": 1719485736.536426, "depends_on": { - "macros": [ - "macro.dbt.current_timestamp" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__snapshot_get_time() %}\n {{ current_timestamp() }}\n{% endmacro %}", + "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", "meta": {}, - "name": "default__snapshot_get_time", - "original_file_path": "macros/adapters/timestamps.sql", + "name": "default__length", + "original_file_path": "macros/utils/length.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/timestamps.sql", + "path": "macros/utils/length.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__snapshot_get_time" + "unique_id": "macro.dbt.default__length" }, - "macro.dbt.default__snapshot_hash_arguments": { + "macro.dbt.default__list_relations_without_caching": { "arguments": [], - "created_at": 1696458269.601996, + "created_at": 1719485736.592287, "depends_on": { "macros": [] }, @@ -3244,96 +3786,93 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", + "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "meta": {}, - "name": "default__snapshot_hash_arguments", - "original_file_path": "macros/materializations/snapshots/strategies.sql", + "name": "default__list_relations_without_caching", + "original_file_path": "macros/adapters/metadata.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/snapshots/strategies.sql", + "path": "macros/adapters/metadata.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__snapshot_hash_arguments" + "unique_id": "macro.dbt.default__list_relations_without_caching" }, - "macro.dbt.default__snapshot_merge_sql": { + "macro.dbt.default__list_schemas": { "arguments": [], - "created_at": 1696458269.595372, + "created_at": 1719485736.591269, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.information_schema_name", + "macro.dbt.run_query" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", + "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "meta": {}, - "name": "default__snapshot_merge_sql", - "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", + "name": "default__list_schemas", + "original_file_path": "macros/adapters/metadata.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/snapshots/snapshot_merge.sql", + "path": "macros/adapters/metadata.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__snapshot_merge_sql" + "unique_id": "macro.dbt.default__list_schemas" }, - "macro.dbt.default__snapshot_staging_table": { + "macro.dbt.default__listagg": { "arguments": [], - "created_at": 1696458269.619116, + "created_at": 1719485736.539655, "depends_on": { - "macros": [ - "macro.dbt.snapshot_get_time" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n\n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", + "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", "meta": {}, - "name": "default__snapshot_staging_table", - "original_file_path": "macros/materializations/snapshots/helpers.sql", + "name": "default__listagg", + "original_file_path": "macros/utils/listagg.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/snapshots/helpers.sql", + "path": "macros/utils/listagg.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__snapshot_staging_table" + "unique_id": "macro.dbt.default__listagg" }, - "macro.dbt.default__snapshot_string_as_time": { + "macro.dbt.default__load_csv_rows": { "arguments": [], - "created_at": 1696458269.60375, + "created_at": 1719485736.465408, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.get_batch_size", + "macro.dbt.get_seed_column_quoted_csv", + "macro.dbt.get_binding_char" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", + "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "meta": {}, - "name": "default__snapshot_string_as_time", - "original_file_path": "macros/materializations/snapshots/strategies.sql", + "name": "default__load_csv_rows", + "original_file_path": "macros/materializations/seeds/helpers.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/snapshots/strategies.sql", + "path": "macros/materializations/seeds/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__snapshot_string_as_time" + "unique_id": "macro.dbt.default__load_csv_rows" }, - "macro.dbt.default__split_part": { + "macro.dbt.default__make_backup_relation": { "arguments": [], - "created_at": 1696458269.796368, + "created_at": 1719485736.5670989, "depends_on": { "macros": [] }, @@ -3342,46 +3881,44 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", + "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", "meta": {}, - "name": "default__split_part", - "original_file_path": "macros/utils/split_part.sql", + "name": "default__make_backup_relation", + "original_file_path": "macros/adapters/relation.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/split_part.sql", + "path": "macros/adapters/relation.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__split_part" + "unique_id": "macro.dbt.default__make_backup_relation" }, - "macro.dbt.default__string_literal": { + "macro.dbt.default__make_intermediate_relation": { "arguments": [], - "created_at": 1696458269.786437, + "created_at": 1719485736.56591, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.default__make_temp_relation" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", + "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", "meta": {}, - "name": "default__string_literal", - "original_file_path": "macros/utils/literal.sql", + "name": "default__make_intermediate_relation", + "original_file_path": "macros/adapters/relation.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/literal.sql", + "path": "macros/adapters/relation.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__string_literal" + "unique_id": "macro.dbt.default__make_intermediate_relation" }, - "macro.dbt.default__support_multiple_grantees_per_dcl_statement": { + "macro.dbt.default__make_temp_relation": { "arguments": [], - "created_at": 1696458269.823607, + "created_at": 1719485736.566462, "depends_on": { "macros": [] }, @@ -3390,72 +3927,68 @@ "node_color": null, "show": true }, - "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", + "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", "meta": {}, - "name": "default__support_multiple_grantees_per_dcl_statement", - "original_file_path": "macros/adapters/apply_grants.sql", + "name": "default__make_temp_relation", + "original_file_path": "macros/adapters/relation.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/apply_grants.sql", + "path": "macros/adapters/relation.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement" + "unique_id": "macro.dbt.default__make_temp_relation" }, - "macro.dbt.default__test_accepted_values": { + "macro.dbt.default__persist_docs": { "arguments": [], - "created_at": 1696458269.761441, + "created_at": 1719485736.586432, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.run_query", + "macro.dbt.alter_relation_comment", + "macro.dbt.alter_column_comment" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", + "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "meta": {}, - "name": "default__test_accepted_values", - "original_file_path": "macros/generic_test_sql/accepted_values.sql", + "name": "default__persist_docs", + "original_file_path": "macros/adapters/persist_docs.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/generic_test_sql/accepted_values.sql", + "path": "macros/adapters/persist_docs.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__test_accepted_values" + "unique_id": "macro.dbt.default__persist_docs" }, - "macro.dbt.default__test_not_null": { + "macro.dbt.default__position": { "arguments": [], - "created_at": 1696458269.7597172, + "created_at": 1719485736.543534, "depends_on": { - "macros": [ - "macro.dbt.should_store_failures" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", + "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", "meta": {}, - "name": "default__test_not_null", - "original_file_path": "macros/generic_test_sql/not_null.sql", + "name": "default__position", + "original_file_path": "macros/utils/position.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/generic_test_sql/not_null.sql", + "path": "macros/utils/position.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__test_not_null" + "unique_id": "macro.dbt.default__position" }, - "macro.dbt.default__test_relationships": { + "macro.dbt.default__post_snapshot": { "arguments": [], - "created_at": 1696458269.759, + "created_at": 1719485736.3552608, "depends_on": { "macros": [] }, @@ -3464,22 +3997,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", + "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "meta": {}, - "name": "default__test_relationships", - "original_file_path": "macros/generic_test_sql/relationships.sql", + "name": "default__post_snapshot", + "original_file_path": "macros/materializations/snapshots/helpers.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/generic_test_sql/relationships.sql", + "path": "macros/materializations/snapshots/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__test_relationships" + "unique_id": "macro.dbt.default__post_snapshot" }, - "macro.dbt.default__test_unique": { + "macro.dbt.default__refresh_materialized_view": { "arguments": [], - "created_at": 1696458269.760334, + "created_at": 1719485736.49013, "depends_on": { "macros": [] }, @@ -3488,22 +4019,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", + "macro_sql": "{% macro default__refresh_materialized_view(relation) %}\n {{ exceptions.raise_compiler_error(\"`refresh_materialized_view` has not been implemented for this adapter.\") }}\n{% endmacro %}", "meta": {}, - "name": "default__test_unique", - "original_file_path": "macros/generic_test_sql/unique.sql", + "name": "default__refresh_materialized_view", + "original_file_path": "macros/relations/materialized_view/refresh.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/generic_test_sql/unique.sql", + "path": "macros/relations/materialized_view/refresh.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__test_unique" + "unique_id": "macro.dbt.default__refresh_materialized_view" }, - "macro.dbt.default__truncate_relation": { + "macro.dbt.default__rename_relation": { "arguments": [], - "created_at": 1696458269.815354, + "created_at": 1719485736.4786189, "depends_on": { "macros": [ "macro.dbt.statement" @@ -3514,22 +4043,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", + "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "meta": {}, - "name": "default__truncate_relation", - "original_file_path": "macros/adapters/relation.sql", + "name": "default__rename_relation", + "original_file_path": "macros/relations/rename.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/relation.sql", + "path": "macros/relations/rename.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__truncate_relation" + "unique_id": "macro.dbt.default__rename_relation" }, - "macro.dbt.default__type_bigint": { + "macro.dbt.default__replace": { "arguments": [], - "created_at": 1696458269.790592, + "created_at": 1719485736.533091, "depends_on": { "macros": [] }, @@ -3538,46 +4065,44 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", + "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", "meta": {}, - "name": "default__type_bigint", - "original_file_path": "macros/utils/data_types.sql", + "name": "default__replace", + "original_file_path": "macros/utils/replace.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/data_types.sql", + "path": "macros/utils/replace.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__type_bigint" + "unique_id": "macro.dbt.default__replace" }, - "macro.dbt.default__type_boolean": { + "macro.dbt.default__reset_csv_table": { "arguments": [], - "created_at": 1696458269.791612, + "created_at": 1719485736.460783, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.create_csv_table" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{%- macro default__type_boolean() -%}\n {{ return(api.Column.translate_type(\"boolean\")) }}\n{%- endmacro -%}\n\n", + "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "meta": {}, - "name": "default__type_boolean", - "original_file_path": "macros/utils/data_types.sql", + "name": "default__reset_csv_table", + "original_file_path": "macros/materializations/seeds/helpers.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/data_types.sql", + "path": "macros/materializations/seeds/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__type_boolean" + "unique_id": "macro.dbt.default__reset_csv_table" }, - "macro.dbt.default__type_float": { + "macro.dbt.default__resolve_model_name": { "arguments": [], - "created_at": 1696458269.789509, + "created_at": 1719485736.615371, "depends_on": { "macros": [] }, @@ -3586,22 +4111,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", + "macro_sql": "\n\n{%- macro default__resolve_model_name(input_model_name) -%}\n {{ input_model_name | string | replace('\"', '\\\"') }}\n{%- endmacro -%}\n\n", "meta": {}, - "name": "default__type_float", - "original_file_path": "macros/utils/data_types.sql", + "name": "default__resolve_model_name", + "original_file_path": "macros/python_model/python.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/data_types.sql", + "path": "macros/python_model/python.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__type_float" + "unique_id": "macro.dbt.default__resolve_model_name" }, - "macro.dbt.default__type_int": { + "macro.dbt.default__right": { "arguments": [], - "created_at": 1696458269.7910988, + "created_at": 1719485736.538501, "depends_on": { "macros": [] }, @@ -3610,22 +4133,20 @@ "node_color": null, "show": true }, - "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", + "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", "meta": {}, - "name": "default__type_int", - "original_file_path": "macros/utils/data_types.sql", + "name": "default__right", + "original_file_path": "macros/utils/right.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/data_types.sql", + "path": "macros/utils/right.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__type_int" + "unique_id": "macro.dbt.default__right" }, - "macro.dbt.default__type_numeric": { + "macro.dbt.default__safe_cast": { "arguments": [], - "created_at": 1696458269.790067, + "created_at": 1719485736.540907, "depends_on": { "macros": [] }, @@ -3634,46 +4155,44 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", + "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", "meta": {}, - "name": "default__type_numeric", - "original_file_path": "macros/utils/data_types.sql", + "name": "default__safe_cast", + "original_file_path": "macros/utils/safe_cast.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/data_types.sql", + "path": "macros/utils/safe_cast.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__type_numeric" + "unique_id": "macro.dbt.default__safe_cast" }, - "macro.dbt.default__type_string": { + "macro.dbt.default__snapshot_get_time": { "arguments": [], - "created_at": 1696458269.788338, + "created_at": 1719485736.559646, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.current_timestamp" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", + "macro_sql": "{% macro default__snapshot_get_time() %}\n {{ current_timestamp() }}\n{% endmacro %}", "meta": {}, - "name": "default__type_string", - "original_file_path": "macros/utils/data_types.sql", + "name": "default__snapshot_get_time", + "original_file_path": "macros/adapters/timestamps.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/data_types.sql", + "path": "macros/adapters/timestamps.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__type_string" + "unique_id": "macro.dbt.default__snapshot_get_time" }, - "macro.dbt.default__type_timestamp": { + "macro.dbt.default__snapshot_hash_arguments": { "arguments": [], - "created_at": 1696458269.788992, + "created_at": 1719485736.345648, "depends_on": { "macros": [] }, @@ -3682,73 +4201,66 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", + "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "meta": {}, - "name": "default__type_timestamp", - "original_file_path": "macros/utils/data_types.sql", + "name": "default__snapshot_hash_arguments", + "original_file_path": "macros/materializations/snapshots/strategies.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/data_types.sql", + "path": "macros/materializations/snapshots/strategies.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default__type_timestamp" + "unique_id": "macro.dbt.default__snapshot_hash_arguments" }, - "macro.dbt.default_last_day": { + "macro.dbt.default__snapshot_merge_sql": { "arguments": [], - "created_at": 1696458269.7946408, + "created_at": 1719485736.340331, "depends_on": { - "macros": [ - "macro.dbt.dateadd", - "macro.dbt.date_trunc" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", + "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "meta": {}, - "name": "default_last_day", - "original_file_path": "macros/utils/last_day.sql", + "name": "default__snapshot_merge_sql", + "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/last_day.sql", + "path": "macros/materializations/snapshots/snapshot_merge.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.default_last_day" + "unique_id": "macro.dbt.default__snapshot_merge_sql" }, - "macro.dbt.diff_column_data_types": { + "macro.dbt.default__snapshot_staging_table": { "arguments": [], - "created_at": 1696458269.6687229, + "created_at": 1719485736.356719, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.snapshot_get_time" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type and not sc.can_expand_to(other_column=tc) %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", + "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n\n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", "meta": {}, - "name": "diff_column_data_types", - "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", + "name": "default__snapshot_staging_table", + "original_file_path": "macros/materializations/snapshots/helpers.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/incremental/column_helpers.sql", + "path": "macros/materializations/snapshots/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.diff_column_data_types" + "unique_id": "macro.dbt.default__snapshot_staging_table" }, - "macro.dbt.diff_columns": { + "macro.dbt.default__snapshot_string_as_time": { "arguments": [], - "created_at": 1696458269.667535, + "created_at": 1719485736.346766, "depends_on": { "macros": [] }, @@ -3757,48 +4269,42 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", + "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "meta": {}, - "name": "diff_columns", - "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", + "name": "default__snapshot_string_as_time", + "original_file_path": "macros/materializations/snapshots/strategies.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/incremental/column_helpers.sql", + "path": "macros/materializations/snapshots/strategies.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.diff_columns" + "unique_id": "macro.dbt.default__snapshot_string_as_time" }, - "macro.dbt.drop_relation": { + "macro.dbt.default__split_part": { "arguments": [], - "created_at": 1696458269.814443, + "created_at": 1719485736.554402, "depends_on": { - "macros": [ - "macro.dbt.default__drop_relation" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", + "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", "meta": {}, - "name": "drop_relation", - "original_file_path": "macros/adapters/relation.sql", + "name": "default__split_part", + "original_file_path": "macros/utils/split_part.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/relation.sql", + "path": "macros/utils/split_part.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.drop_relation" + "unique_id": "macro.dbt.default__split_part" }, - "macro.dbt.drop_relation_if_exists": { + "macro.dbt.default__string_literal": { "arguments": [], - "created_at": 1696458269.8185081, + "created_at": 1719485736.544776, "depends_on": { "macros": [] }, @@ -3807,77 +4313,67 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", + "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", "meta": {}, - "name": "drop_relation_if_exists", - "original_file_path": "macros/adapters/relation.sql", + "name": "default__string_literal", + "original_file_path": "macros/utils/literal.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/relation.sql", + "path": "macros/utils/literal.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.drop_relation_if_exists" + "unique_id": "macro.dbt.default__string_literal" }, - "macro.dbt.drop_schema": { + "macro.dbt.default__support_multiple_grantees_per_dcl_statement": { "arguments": [], - "created_at": 1696458269.8010921, + "created_at": 1719485736.573099, "depends_on": { - "macros": [ - "macro.dbt_postgres.postgres__drop_schema" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", + "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", "meta": {}, - "name": "drop_schema", - "original_file_path": "macros/adapters/schema.sql", + "name": "default__support_multiple_grantees_per_dcl_statement", + "original_file_path": "macros/adapters/apply_grants.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/schema.sql", + "path": "macros/adapters/apply_grants.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.drop_schema" + "unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement" }, - "macro.dbt.escape_single_quotes": { + "macro.dbt.default__test_accepted_values": { "arguments": [], - "created_at": 1696458269.777445, + "created_at": 1719485736.517527, "depends_on": { - "macros": [ - "macro.dbt.default__escape_single_quotes" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", + "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "meta": {}, - "name": "escape_single_quotes", - "original_file_path": "macros/utils/escape_single_quotes.sql", + "name": "default__test_accepted_values", + "original_file_path": "macros/generic_test_sql/accepted_values.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/escape_single_quotes.sql", + "path": "macros/generic_test_sql/accepted_values.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.escape_single_quotes" + "unique_id": "macro.dbt.default__test_accepted_values" }, - "macro.dbt.except": { + "macro.dbt.default__test_not_null": { "arguments": [], - "created_at": 1696458269.7728372, + "created_at": 1719485736.516648, "depends_on": { "macros": [ - "macro.dbt.default__except" + "macro.dbt.should_store_failures" ] }, "description": "", @@ -3885,77 +4381,67 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", + "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "meta": {}, - "name": "except", - "original_file_path": "macros/utils/except.sql", + "name": "default__test_not_null", + "original_file_path": "macros/generic_test_sql/not_null.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/except.sql", + "path": "macros/generic_test_sql/not_null.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.except" + "unique_id": "macro.dbt.default__test_not_null" }, - "macro.dbt.generate_alias_name": { + "macro.dbt.default__test_relationships": { "arguments": [], - "created_at": 1696458269.7544892, + "created_at": 1719485736.516331, "depends_on": { - "macros": [ - "macro.dbt.default__generate_alias_name" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", + "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "meta": {}, - "name": "generate_alias_name", - "original_file_path": "macros/get_custom_name/get_custom_alias.sql", + "name": "default__test_relationships", + "original_file_path": "macros/generic_test_sql/relationships.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/get_custom_name/get_custom_alias.sql", + "path": "macros/generic_test_sql/relationships.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.generate_alias_name" + "unique_id": "macro.dbt.default__test_relationships" }, - "macro.dbt.generate_database_name": { + "macro.dbt.default__test_unique": { "arguments": [], - "created_at": 1696458269.757828, + "created_at": 1719485736.5169122, "depends_on": { - "macros": [ - "macro.dbt.default__generate_database_name" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", + "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "meta": {}, - "name": "generate_database_name", - "original_file_path": "macros/get_custom_name/get_custom_database.sql", + "name": "default__test_unique", + "original_file_path": "macros/generic_test_sql/unique.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/get_custom_name/get_custom_database.sql", + "path": "macros/generic_test_sql/unique.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.generate_database_name" + "unique_id": "macro.dbt.default__test_unique" }, - "macro.dbt.generate_schema_name": { + "macro.dbt.default__truncate_relation": { "arguments": [], - "created_at": 1696458269.756017, + "created_at": 1719485736.5674748, "depends_on": { "macros": [ - "macro.dbt.default__generate_schema_name" + "macro.dbt.statement" ] }, "description": "", @@ -3963,22 +4449,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", + "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", "meta": {}, - "name": "generate_schema_name", - "original_file_path": "macros/get_custom_name/get_custom_schema.sql", + "name": "default__truncate_relation", + "original_file_path": "macros/adapters/relation.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/get_custom_name/get_custom_schema.sql", + "path": "macros/adapters/relation.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.generate_schema_name" + "unique_id": "macro.dbt.default__truncate_relation" }, - "macro.dbt.generate_schema_name_for_env": { + "macro.dbt.default__type_bigint": { "arguments": [], - "created_at": 1696458269.756963, + "created_at": 1719485736.5486028, "depends_on": { "macros": [] }, @@ -3987,181 +4471,155 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", + "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", "meta": {}, - "name": "generate_schema_name_for_env", - "original_file_path": "macros/get_custom_name/get_custom_schema.sql", + "name": "default__type_bigint", + "original_file_path": "macros/utils/data_types.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/get_custom_name/get_custom_schema.sql", + "path": "macros/utils/data_types.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.generate_schema_name_for_env" + "unique_id": "macro.dbt.default__type_bigint" }, - "macro.dbt.get_batch_size": { + "macro.dbt.default__type_boolean": { "arguments": [], - "created_at": 1696458269.750157, + "created_at": 1719485736.550734, "depends_on": { - "macros": [ - "macro.dbt.default__get_batch_size" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", + "macro_sql": "{%- macro default__type_boolean() -%}\n {{ return(api.Column.translate_type(\"boolean\")) }}\n{%- endmacro -%}\n\n", "meta": {}, - "name": "get_batch_size", - "original_file_path": "macros/materializations/seeds/helpers.sql", + "name": "default__type_boolean", + "original_file_path": "macros/utils/data_types.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/seeds/helpers.sql", + "path": "macros/utils/data_types.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_batch_size" + "unique_id": "macro.dbt.default__type_boolean" }, - "macro.dbt.get_binding_char": { + "macro.dbt.default__type_float": { "arguments": [], - "created_at": 1696458269.749717, + "created_at": 1719485736.5472598, "depends_on": { - "macros": [ - "macro.dbt.default__get_binding_char" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", + "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", "meta": {}, - "name": "get_binding_char", - "original_file_path": "macros/materializations/seeds/helpers.sql", + "name": "default__type_float", + "original_file_path": "macros/utils/data_types.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/seeds/helpers.sql", + "path": "macros/utils/data_types.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_binding_char" + "unique_id": "macro.dbt.default__type_float" }, - "macro.dbt.get_catalog": { + "macro.dbt.default__type_int": { "arguments": [], - "created_at": 1696458269.836847, + "created_at": 1719485736.549174, "depends_on": { - "macros": [ - "macro.dbt_postgres.postgres__get_catalog" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", + "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", "meta": {}, - "name": "get_catalog", - "original_file_path": "macros/adapters/metadata.sql", + "name": "default__type_int", + "original_file_path": "macros/utils/data_types.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/metadata.sql", + "path": "macros/utils/data_types.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_catalog" + "unique_id": "macro.dbt.default__type_int" }, - "macro.dbt.get_columns_in_query": { + "macro.dbt.default__type_numeric": { "arguments": [], - "created_at": 1696458269.844657, + "created_at": 1719485736.547792, "depends_on": { - "macros": [ - "macro.dbt.default__get_columns_in_query" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", + "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", "meta": {}, - "name": "get_columns_in_query", - "original_file_path": "macros/adapters/columns.sql", + "name": "default__type_numeric", + "original_file_path": "macros/utils/data_types.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/columns.sql", + "path": "macros/utils/data_types.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_columns_in_query" + "unique_id": "macro.dbt.default__type_numeric" }, - "macro.dbt.get_columns_in_relation": { + "macro.dbt.default__type_string": { "arguments": [], - "created_at": 1696458269.843055, + "created_at": 1719485736.5461628, "depends_on": { - "macros": [ - "macro.dbt_postgres.postgres__get_columns_in_relation" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", + "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", "meta": {}, - "name": "get_columns_in_relation", - "original_file_path": "macros/adapters/columns.sql", + "name": "default__type_string", + "original_file_path": "macros/utils/data_types.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/columns.sql", + "path": "macros/utils/data_types.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_columns_in_relation" + "unique_id": "macro.dbt.default__type_string" }, - "macro.dbt.get_create_index_sql": { + "macro.dbt.default__type_timestamp": { "arguments": [], - "created_at": 1696458269.8050802, + "created_at": 1719485736.5468001, "depends_on": { - "macros": [ - "macro.dbt_postgres.postgres__get_create_index_sql" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", + "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", "meta": {}, - "name": "get_create_index_sql", - "original_file_path": "macros/adapters/indexes.sql", + "name": "default__type_timestamp", + "original_file_path": "macros/utils/data_types.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/indexes.sql", + "path": "macros/utils/data_types.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_create_index_sql" + "unique_id": "macro.dbt.default__type_timestamp" }, - "macro.dbt.get_create_table_as_sql": { + "macro.dbt.default__validate_sql": { "arguments": [], - "created_at": 1696458269.719821, + "created_at": 1719485736.570561, "depends_on": { "macros": [ - "macro.dbt.default__get_create_table_as_sql" + "macro.dbt.statement" ] }, "description": "", @@ -4169,25 +4627,24 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", + "macro_sql": "{% macro default__validate_sql(sql) -%}\n {% call statement('validate_sql') -%}\n explain {{ sql }}\n {% endcall %}\n {{ return(load_result('validate_sql')) }}\n{% endmacro %}", "meta": {}, - "name": "get_create_table_as_sql", - "original_file_path": "macros/materializations/models/table/create_table_as.sql", + "name": "default__validate_sql", + "original_file_path": "macros/adapters/validate_sql.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/table/create_table_as.sql", + "path": "macros/adapters/validate_sql.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_create_table_as_sql" + "unique_id": "macro.dbt.default__validate_sql" }, - "macro.dbt.get_create_view_as_sql": { + "macro.dbt.default_last_day": { "arguments": [], - "created_at": 1696458269.730824, + "created_at": 1719485736.5533261, "depends_on": { "macros": [ - "macro.dbt.default__get_create_view_as_sql" + "macro.dbt.dateadd", + "macro.dbt.date_trunc" ] }, "description": "", @@ -4195,77 +4652,67 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", + "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", "meta": {}, - "name": "get_create_view_as_sql", - "original_file_path": "macros/materializations/models/view/create_view_as.sql", + "name": "default_last_day", + "original_file_path": "macros/utils/last_day.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/view/create_view_as.sql", + "path": "macros/utils/last_day.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_create_view_as_sql" + "unique_id": "macro.dbt.default_last_day" }, - "macro.dbt.get_csv_sql": { + "macro.dbt.diff_column_data_types": { "arguments": [], - "created_at": 1696458269.749263, + "created_at": 1719485736.397441, "depends_on": { - "macros": [ - "macro.dbt.default__get_csv_sql" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", + "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type and not sc.can_expand_to(other_column=tc) %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "meta": {}, - "name": "get_csv_sql", - "original_file_path": "macros/materializations/seeds/helpers.sql", + "name": "diff_column_data_types", + "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/seeds/helpers.sql", + "path": "macros/materializations/models/incremental/column_helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_csv_sql" + "unique_id": "macro.dbt.diff_column_data_types" }, - "macro.dbt.get_dcl_statement_list": { + "macro.dbt.diff_columns": { "arguments": [], - "created_at": 1696458269.826365, + "created_at": 1719485736.396736, "depends_on": { - "macros": [ - "macro.dbt.default__get_dcl_statement_list" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", + "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "meta": {}, - "name": "get_dcl_statement_list", - "original_file_path": "macros/adapters/apply_grants.sql", + "name": "diff_columns", + "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/apply_grants.sql", + "path": "macros/materializations/models/incremental/column_helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_dcl_statement_list" + "unique_id": "macro.dbt.diff_columns" }, - "macro.dbt.get_delete_insert_merge_sql": { + "macro.dbt.drop_materialized_view": { "arguments": [], - "created_at": 1696458269.6812491, + "created_at": 1719485736.486141, "depends_on": { "macros": [ - "macro.dbt.default__get_delete_insert_merge_sql" + "macro.dbt_postgres.postgres__drop_materialized_view" ] }, "description": "", @@ -4273,25 +4720,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns) }}\n{%- endmacro %}", + "macro_sql": "{% macro drop_materialized_view(relation) -%}\n {{- adapter.dispatch('drop_materialized_view', 'dbt')(relation) -}}\n{%- endmacro %}", "meta": {}, - "name": "get_delete_insert_merge_sql", - "original_file_path": "macros/materializations/models/incremental/merge.sql", + "name": "drop_materialized_view", + "original_file_path": "macros/relations/materialized_view/drop.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/incremental/merge.sql", + "path": "macros/relations/materialized_view/drop.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_delete_insert_merge_sql" + "unique_id": "macro.dbt.drop_materialized_view" }, - "macro.dbt.get_grant_sql": { + "macro.dbt.drop_relation": { "arguments": [], - "created_at": 1696458269.8250072, + "created_at": 1719485736.471232, "depends_on": { "macros": [ - "macro.dbt.default__get_grant_sql" + "macro.dbt.default__drop_relation" ] }, "description": "", @@ -4299,51 +4744,45 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", + "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "meta": {}, - "name": "get_grant_sql", - "original_file_path": "macros/adapters/apply_grants.sql", + "name": "drop_relation", + "original_file_path": "macros/relations/drop.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/apply_grants.sql", + "path": "macros/relations/drop.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_grant_sql" + "unique_id": "macro.dbt.drop_relation" }, - "macro.dbt.get_incremental_append_sql": { + "macro.dbt.drop_relation_if_exists": { "arguments": [], - "created_at": 1696458269.686978, + "created_at": 1719485736.471669, "depends_on": { - "macros": [ - "macro.dbt.default__get_incremental_append_sql" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro get_incremental_append_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "meta": {}, - "name": "get_incremental_append_sql", - "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "name": "drop_relation_if_exists", + "original_file_path": "macros/relations/drop.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/incremental/strategies.sql", + "path": "macros/relations/drop.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_incremental_append_sql" + "unique_id": "macro.dbt.drop_relation_if_exists" }, - "macro.dbt.get_incremental_default_sql": { + "macro.dbt.drop_schema": { "arguments": [], - "created_at": 1696458269.690001, + "created_at": 1719485736.558478, "depends_on": { "macros": [ - "macro.dbt_postgres.postgres__get_incremental_default_sql" + "macro.dbt_postgres.postgres__drop_schema" ] }, "description": "", @@ -4351,25 +4790,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_incremental_default_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "meta": {}, - "name": "get_incremental_default_sql", - "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "name": "drop_schema", + "original_file_path": "macros/adapters/schema.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/incremental/strategies.sql", + "path": "macros/adapters/schema.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_incremental_default_sql" + "unique_id": "macro.dbt.drop_schema" }, - "macro.dbt.get_incremental_delete_insert_sql": { + "macro.dbt.drop_schema_named": { "arguments": [], - "created_at": 1696458269.6876879, + "created_at": 1719485736.4761379, "depends_on": { "macros": [ - "macro.dbt.default__get_incremental_delete_insert_sql" + "macro.dbt.default__drop_schema_named" ] }, "description": "", @@ -4377,25 +4814,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_incremental_delete_insert_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "macro_sql": "{% macro drop_schema_named(schema_name) %}\n {{ return(adapter.dispatch('drop_schema_named', 'dbt') (schema_name)) }}\n{% endmacro %}", "meta": {}, - "name": "get_incremental_delete_insert_sql", - "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "name": "drop_schema_named", + "original_file_path": "macros/relations/schema.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/incremental/strategies.sql", + "path": "macros/relations/schema.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_incremental_delete_insert_sql" + "unique_id": "macro.dbt.drop_schema_named" }, - "macro.dbt.get_incremental_insert_overwrite_sql": { + "macro.dbt.drop_table": { "arguments": [], - "created_at": 1696458269.6892319, + "created_at": 1719485736.505447, "depends_on": { "macros": [ - "macro.dbt.default__get_incremental_insert_overwrite_sql" + "macro.dbt_postgres.postgres__drop_table" ] }, "description": "", @@ -4403,25 +4838,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "macro_sql": "{% macro drop_table(relation) -%}\n {{- adapter.dispatch('drop_table', 'dbt')(relation) -}}\n{%- endmacro %}", "meta": {}, - "name": "get_incremental_insert_overwrite_sql", - "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "name": "drop_table", + "original_file_path": "macros/relations/table/drop.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/incremental/strategies.sql", + "path": "macros/relations/table/drop.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_incremental_insert_overwrite_sql" + "unique_id": "macro.dbt.drop_table" }, - "macro.dbt.get_incremental_merge_sql": { + "macro.dbt.drop_view": { "arguments": [], - "created_at": 1696458269.688441, + "created_at": 1719485736.511067, "depends_on": { "macros": [ - "macro.dbt.default__get_incremental_merge_sql" + "macro.dbt_postgres.postgres__drop_view" ] }, "description": "", @@ -4429,25 +4862,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_incremental_merge_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "macro_sql": "{% macro drop_view(relation) -%}\n {{- adapter.dispatch('drop_view', 'dbt')(relation) -}}\n{%- endmacro %}", "meta": {}, - "name": "get_incremental_merge_sql", - "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "name": "drop_view", + "original_file_path": "macros/relations/view/drop.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/incremental/strategies.sql", + "path": "macros/relations/view/drop.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_incremental_merge_sql" + "unique_id": "macro.dbt.drop_view" }, - "macro.dbt.get_insert_into_sql": { + "macro.dbt.escape_single_quotes": { "arguments": [], - "created_at": 1696458269.6907241, + "created_at": 1719485736.537864, "depends_on": { "macros": [ - "macro.dbt.get_quoted_csv" + "macro.dbt.default__escape_single_quotes" ] }, "description": "", @@ -4455,25 +4886,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ temp_relation }}\n )\n\n{% endmacro %}", + "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", "meta": {}, - "name": "get_insert_into_sql", - "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "name": "escape_single_quotes", + "original_file_path": "macros/utils/escape_single_quotes.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/incremental/strategies.sql", + "path": "macros/utils/escape_single_quotes.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_insert_into_sql" + "unique_id": "macro.dbt.escape_single_quotes" }, - "macro.dbt.get_insert_overwrite_merge_sql": { + "macro.dbt.except": { "arguments": [], - "created_at": 1696458269.683062, + "created_at": 1719485736.5260952, "depends_on": { "macros": [ - "macro.dbt.default__get_insert_overwrite_merge_sql" + "macro.dbt.default__except" ] }, "description": "", @@ -4481,25 +4910,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", + "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", "meta": {}, - "name": "get_insert_overwrite_merge_sql", - "original_file_path": "macros/materializations/models/incremental/merge.sql", + "name": "except", + "original_file_path": "macros/utils/except.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/incremental/merge.sql", + "path": "macros/utils/except.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_insert_overwrite_merge_sql" + "unique_id": "macro.dbt.except" }, - "macro.dbt.get_merge_sql": { + "macro.dbt.format_columns": { "arguments": [], - "created_at": 1696458269.67798, + "created_at": 1719485736.504094, "depends_on": { "macros": [ - "macro.dbt.default__get_merge_sql" + "macro.dbt.default__format_column" ] }, "description": "", @@ -4507,25 +4934,25 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, predicates=none) -%}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, predicates) }}\n{%- endmacro %}", + "macro_sql": "{% macro format_columns(columns) %}\n {% set formatted_columns = [] %}\n {% for column in columns %}\n {%- set formatted_column = adapter.dispatch('format_column', 'dbt')(column) -%}\n {%- do formatted_columns.append(formatted_column) -%}\n {% endfor %}\n {{ return(formatted_columns) }}\n{% endmacro %}", "meta": {}, - "name": "get_merge_sql", - "original_file_path": "macros/materializations/models/incremental/merge.sql", + "name": "format_columns", + "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/incremental/merge.sql", + "path": "macros/relations/column/columns_spec_ddl.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_merge_sql" + "unique_id": "macro.dbt.format_columns" }, - "macro.dbt.get_merge_update_columns": { + "macro.dbt.format_row": { "arguments": [], - "created_at": 1696458269.6691232, + "created_at": 1719485736.611443, "depends_on": { "macros": [ - "macro.dbt.default__get_merge_update_columns" + "macro.dbt.string_literal", + "macro.dbt.escape_single_quotes", + "macro.dbt.safe_cast" ] }, "description": "", @@ -4533,25 +4960,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}\n{% endmacro %}", + "macro_sql": "\n\n{%- macro format_row(row, column_name_to_data_types) -%}\n {#-- generate case-insensitive formatted row --#}\n {% set formatted_row = {} %}\n {%- for column_name, column_value in row.items() -%}\n {% set column_name = column_name|lower %}\n\n {%- if column_name not in column_name_to_data_types %}\n {#-- if user-provided row contains column name that relation does not contain, raise an error --#}\n {% set fixture_name = \"expected output\" if model.resource_type == 'unit_test' else (\"'\" ~ model.name ~ \"'\") %}\n {{ exceptions.raise_compiler_error(\n \"Invalid column name: '\" ~ column_name ~ \"' in unit test fixture for \" ~ fixture_name ~ \".\"\n \"\\nAccepted columns for \" ~ fixture_name ~ \" are: \" ~ (column_name_to_data_types.keys()|list)\n ) }}\n {%- endif -%}\n\n {%- set column_type = column_name_to_data_types[column_name] %}\n\n {#-- sanitize column_value: wrap yaml strings in quotes, apply cast --#}\n {%- set column_value_clean = column_value -%}\n {%- if column_value is string -%}\n {%- set column_value_clean = dbt.string_literal(dbt.escape_single_quotes(column_value)) -%}\n {%- elif column_value is none -%}\n {%- set column_value_clean = 'null' -%}\n {%- endif -%}\n\n {%- set row_update = {column_name: safe_cast(column_value_clean, column_type) } -%}\n {%- do formatted_row.update(row_update) -%}\n {%- endfor -%}\n {{ return(formatted_row) }}\n{%- endmacro -%}", "meta": {}, - "name": "get_merge_update_columns", - "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", + "name": "format_row", + "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/incremental/column_helpers.sql", + "path": "macros/unit_test_sql/get_fixture_sql.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_merge_update_columns" + "unique_id": "macro.dbt.format_row" }, - "macro.dbt.get_or_create_relation": { + "macro.dbt.generate_alias_name": { "arguments": [], - "created_at": 1696458269.816565, + "created_at": 1719485736.466082, "depends_on": { "macros": [ - "macro.dbt.default__get_or_create_relation" + "macro.dbt.default__generate_alias_name" ] }, "description": "", @@ -4559,49 +4984,47 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", + "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "meta": {}, - "name": "get_or_create_relation", - "original_file_path": "macros/adapters/relation.sql", + "name": "generate_alias_name", + "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/relation.sql", + "path": "macros/get_custom_name/get_custom_alias.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_or_create_relation" + "unique_id": "macro.dbt.generate_alias_name" }, - "macro.dbt.get_quoted_csv": { + "macro.dbt.generate_database_name": { "arguments": [], - "created_at": 1696458269.666624, + "created_at": 1719485736.468485, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.default__generate_database_name" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", + "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "meta": {}, - "name": "get_quoted_csv", - "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", + "name": "generate_database_name", + "original_file_path": "macros/get_custom_name/get_custom_database.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/incremental/column_helpers.sql", + "path": "macros/get_custom_name/get_custom_database.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_quoted_csv" + "unique_id": "macro.dbt.generate_database_name" }, - "macro.dbt.get_revoke_sql": { + "macro.dbt.generate_schema_name": { "arguments": [], - "created_at": 1696458269.825689, + "created_at": 1719485736.467374, "depends_on": { "macros": [ - "macro.dbt.default__get_revoke_sql" + "macro.dbt.default__generate_schema_name" ] }, "description": "", @@ -4609,22 +5032,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", + "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "meta": {}, - "name": "get_revoke_sql", - "original_file_path": "macros/adapters/apply_grants.sql", + "name": "generate_schema_name", + "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/apply_grants.sql", + "path": "macros/get_custom_name/get_custom_schema.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_revoke_sql" + "unique_id": "macro.dbt.generate_schema_name" }, - "macro.dbt.get_seed_column_quoted_csv": { + "macro.dbt.generate_schema_name_for_env": { "arguments": [], - "created_at": 1696458269.75114, + "created_at": 1719485736.46803, "depends_on": { "macros": [] }, @@ -4633,25 +5054,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", + "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "meta": {}, - "name": "get_seed_column_quoted_csv", - "original_file_path": "macros/materializations/seeds/helpers.sql", + "name": "generate_schema_name_for_env", + "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/seeds/helpers.sql", + "path": "macros/get_custom_name/get_custom_schema.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_seed_column_quoted_csv" + "unique_id": "macro.dbt.generate_schema_name_for_env" }, - "macro.dbt.get_show_grant_sql": { + "macro.dbt.generate_series": { "arguments": [], - "created_at": 1696458269.8244731, + "created_at": 1719485736.5352, "depends_on": { "macros": [ - "macro.dbt_postgres.postgres__get_show_grant_sql" + "macro.dbt.default__generate_series" ] }, "description": "", @@ -4659,25 +5078,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", + "macro_sql": "{% macro generate_series(upper_bound) %}\n {{ return(adapter.dispatch('generate_series', 'dbt')(upper_bound)) }}\n{% endmacro %}", "meta": {}, - "name": "get_show_grant_sql", - "original_file_path": "macros/adapters/apply_grants.sql", + "name": "generate_series", + "original_file_path": "macros/utils/generate_series.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/apply_grants.sql", + "path": "macros/utils/generate_series.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_show_grant_sql" + "unique_id": "macro.dbt.generate_series" }, - "macro.dbt.get_test_sql": { + "macro.dbt.get_alter_materialized_view_as_sql": { "arguments": [], - "created_at": 1696458269.662025, + "created_at": 1719485736.492841, "depends_on": { "macros": [ - "macro.dbt.default__get_test_sql" + "macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql" ] }, "description": "", @@ -4685,25 +5102,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", + "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {{- adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n ) -}}\n{% endmacro %}", "meta": {}, - "name": "get_test_sql", - "original_file_path": "macros/materializations/tests/helpers.sql", + "name": "get_alter_materialized_view_as_sql", + "original_file_path": "macros/relations/materialized_view/alter.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/tests/helpers.sql", + "path": "macros/relations/materialized_view/alter.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_test_sql" + "unique_id": "macro.dbt.get_alter_materialized_view_as_sql" }, - "macro.dbt.get_true_sql": { + "macro.dbt.get_assert_columns_equivalent": { "arguments": [], - "created_at": 1696458269.617147, + "created_at": 1719485736.5009642, "depends_on": { "macros": [ - "macro.dbt.default__get_true_sql" + "macro.dbt.default__get_assert_columns_equivalent" ] }, "description": "", @@ -4711,25 +5126,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", + "macro_sql": "\n\n{%- macro get_assert_columns_equivalent(sql) -%}\n {{ adapter.dispatch('get_assert_columns_equivalent', 'dbt')(sql) }}\n{%- endmacro -%}\n\n", "meta": {}, - "name": "get_true_sql", - "original_file_path": "macros/materializations/snapshots/helpers.sql", + "name": "get_assert_columns_equivalent", + "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/snapshots/helpers.sql", + "path": "macros/relations/column/columns_spec_ddl.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_true_sql" + "unique_id": "macro.dbt.get_assert_columns_equivalent" }, - "macro.dbt.get_where_subquery": { + "macro.dbt.get_batch_size": { "arguments": [], - "created_at": 1696458269.663287, + "created_at": 1719485736.461688, "depends_on": { "macros": [ - "macro.dbt.default__get_where_subquery" + "macro.dbt.default__get_batch_size" ] }, "description": "", @@ -4737,25 +5150,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", + "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "meta": {}, - "name": "get_where_subquery", - "original_file_path": "macros/materializations/tests/where_subquery.sql", + "name": "get_batch_size", + "original_file_path": "macros/materializations/seeds/helpers.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/tests/where_subquery.sql", + "path": "macros/materializations/seeds/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.get_where_subquery" + "unique_id": "macro.dbt.get_batch_size" }, - "macro.dbt.handle_existing_table": { + "macro.dbt.get_binding_char": { "arguments": [], - "created_at": 1696458269.7268028, + "created_at": 1719485736.461306, "depends_on": { "macros": [ - "macro.dbt.default__handle_existing_table" + "macro.dbt.default__get_binding_char" ] }, "description": "", @@ -4763,25 +5174,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", + "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "meta": {}, - "name": "handle_existing_table", - "original_file_path": "macros/materializations/models/view/helpers.sql", + "name": "get_binding_char", + "original_file_path": "macros/materializations/seeds/helpers.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/view/helpers.sql", + "path": "macros/materializations/seeds/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.handle_existing_table" + "unique_id": "macro.dbt.get_binding_char" }, - "macro.dbt.hash": { + "macro.dbt.get_catalog": { "arguments": [], - "created_at": 1696458269.7828329, + "created_at": 1719485736.590185, "depends_on": { "macros": [ - "macro.dbt.default__hash" + "macro.dbt_postgres.postgres__get_catalog" ] }, "description": "", @@ -4789,25 +5198,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", + "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "meta": {}, - "name": "hash", - "original_file_path": "macros/utils/hash.sql", + "name": "get_catalog", + "original_file_path": "macros/adapters/metadata.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/hash.sql", + "path": "macros/adapters/metadata.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.hash" + "unique_id": "macro.dbt.get_catalog" }, - "macro.dbt.in_transaction": { + "macro.dbt.get_catalog_relations": { "arguments": [], - "created_at": 1696458269.591626, + "created_at": 1719485736.5896802, "depends_on": { "macros": [ - "macro.dbt.make_hook_config" + "macro.dbt_postgres.postgres__get_catalog_relations" ] }, "description": "", @@ -4815,49 +5222,47 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", + "macro_sql": "{% macro get_catalog_relations(information_schema, relations) -%}\n {{ return(adapter.dispatch('get_catalog_relations', 'dbt')(information_schema, relations)) }}\n{%- endmacro %}", "meta": {}, - "name": "in_transaction", - "original_file_path": "macros/materializations/hooks.sql", + "name": "get_catalog_relations", + "original_file_path": "macros/adapters/metadata.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/hooks.sql", + "path": "macros/adapters/metadata.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.in_transaction" + "unique_id": "macro.dbt.get_catalog_relations" }, - "macro.dbt.incremental_validate_on_schema_change": { + "macro.dbt.get_column_schema_from_query": { "arguments": [], - "created_at": 1696458269.708918, + "created_at": 1719485736.598866, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.get_empty_subquery_sql" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", + "macro_sql": "{% macro get_column_schema_from_query(select_sql, select_sql_header=none) -%}\n {% set columns = [] %}\n {# -- Using an 'empty subquery' here to get the same schema as the given select_sql statement, without necessitating a data scan.#}\n {% set sql = get_empty_subquery_sql(select_sql, select_sql_header) %}\n {% set column_schema = adapter.get_column_schema_from_query(sql) %}\n {{ return(column_schema) }}\n{% endmacro %}", "meta": {}, - "name": "incremental_validate_on_schema_change", - "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", + "name": "get_column_schema_from_query", + "original_file_path": "macros/adapters/columns.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/incremental/on_schema_change.sql", + "path": "macros/adapters/columns.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.incremental_validate_on_schema_change" + "unique_id": "macro.dbt.get_column_schema_from_query" }, - "macro.dbt.information_schema_name": { + "macro.dbt.get_columns_in_query": { "arguments": [], - "created_at": 1696458269.837573, + "created_at": 1719485736.5990841, "depends_on": { "macros": [ - "macro.dbt_postgres.postgres__information_schema_name" + "macro.dbt.default__get_columns_in_query" ] }, "description": "", @@ -4865,25 +5270,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", + "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "meta": {}, - "name": "information_schema_name", - "original_file_path": "macros/adapters/metadata.sql", + "name": "get_columns_in_query", + "original_file_path": "macros/adapters/columns.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/metadata.sql", + "path": "macros/adapters/columns.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.information_schema_name" + "unique_id": "macro.dbt.get_columns_in_query" }, - "macro.dbt.intersect": { + "macro.dbt.get_columns_in_relation": { "arguments": [], - "created_at": 1696458269.776766, + "created_at": 1719485736.596093, "depends_on": { "macros": [ - "macro.dbt.default__intersect" + "macro.dbt_postgres.postgres__get_columns_in_relation" ] }, "description": "", @@ -4891,25 +5294,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", + "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "meta": {}, - "name": "intersect", - "original_file_path": "macros/utils/intersect.sql", + "name": "get_columns_in_relation", + "original_file_path": "macros/adapters/columns.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/intersect.sql", + "path": "macros/adapters/columns.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.intersect" + "unique_id": "macro.dbt.get_columns_in_relation" }, - "macro.dbt.is_incremental": { + "macro.dbt.get_create_backup_sql": { "arguments": [], - "created_at": 1696458269.6853771, + "created_at": 1719485736.478982, "depends_on": { "macros": [ - "macro.dbt.should_full_refresh" + "macro.dbt.default__get_create_backup_sql" ] }, "description": "", @@ -4917,25 +5318,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", + "macro_sql": "{%- macro get_create_backup_sql(relation) -%}\n {{- log('Applying CREATE BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "meta": {}, - "name": "is_incremental", - "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", + "name": "get_create_backup_sql", + "original_file_path": "macros/relations/create_backup.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/incremental/is_incremental.sql", + "path": "macros/relations/create_backup.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.is_incremental" + "unique_id": "macro.dbt.get_create_backup_sql" }, - "macro.dbt.last_day": { + "macro.dbt.get_create_index_sql": { "arguments": [], - "created_at": 1696458269.794115, + "created_at": 1719485736.561203, "depends_on": { "macros": [ - "macro.dbt_postgres.postgres__last_day" + "macro.dbt_postgres.postgres__get_create_index_sql" ] }, "description": "", @@ -4943,25 +5342,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", + "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "meta": {}, - "name": "last_day", - "original_file_path": "macros/utils/last_day.sql", + "name": "get_create_index_sql", + "original_file_path": "macros/adapters/indexes.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/last_day.sql", + "path": "macros/adapters/indexes.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.last_day" + "unique_id": "macro.dbt.get_create_index_sql" }, - "macro.dbt.length": { + "macro.dbt.get_create_intermediate_sql": { "arguments": [], - "created_at": 1696458269.775149, + "created_at": 1719485736.4755762, "depends_on": { "macros": [ - "macro.dbt.default__length" + "macro.dbt.default__get_create_intermediate_sql" ] }, "description": "", @@ -4969,25 +5366,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", + "macro_sql": "{%- macro get_create_intermediate_sql(relation, sql) -%}\n {{- log('Applying CREATE INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_intermediate_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "meta": {}, - "name": "length", - "original_file_path": "macros/utils/length.sql", + "name": "get_create_intermediate_sql", + "original_file_path": "macros/relations/create_intermediate.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/length.sql", + "path": "macros/relations/create_intermediate.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.length" + "unique_id": "macro.dbt.get_create_intermediate_sql" }, - "macro.dbt.list_relations_without_caching": { + "macro.dbt.get_create_materialized_view_as_sql": { "arguments": [], - "created_at": 1696458269.839705, + "created_at": 1719485736.4946408, "depends_on": { "macros": [ - "macro.dbt_postgres.postgres__list_relations_without_caching" + "macro.dbt_postgres.postgres__get_create_materialized_view_as_sql" ] }, "description": "", @@ -4995,25 +5390,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", + "macro_sql": "{% macro get_create_materialized_view_as_sql(relation, sql) -%}\n {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}}\n{%- endmacro %}", "meta": {}, - "name": "list_relations_without_caching", - "original_file_path": "macros/adapters/metadata.sql", + "name": "get_create_materialized_view_as_sql", + "original_file_path": "macros/relations/materialized_view/create.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/metadata.sql", + "path": "macros/relations/materialized_view/create.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.list_relations_without_caching" + "unique_id": "macro.dbt.get_create_materialized_view_as_sql" }, - "macro.dbt.list_schemas": { + "macro.dbt.get_create_sql": { "arguments": [], - "created_at": 1696458269.838121, + "created_at": 1719485736.481704, "depends_on": { "macros": [ - "macro.dbt_postgres.postgres__list_schemas" + "macro.dbt.default__get_create_sql" ] }, "description": "", @@ -5021,25 +5414,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", + "macro_sql": "{%- macro get_create_sql(relation, sql) -%}\n {{- log('Applying CREATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "meta": {}, - "name": "list_schemas", - "original_file_path": "macros/adapters/metadata.sql", + "name": "get_create_sql", + "original_file_path": "macros/relations/create.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/metadata.sql", + "path": "macros/relations/create.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.list_schemas" + "unique_id": "macro.dbt.get_create_sql" }, - "macro.dbt.listagg": { + "macro.dbt.get_create_table_as_sql": { "arguments": [], - "created_at": 1696458269.779763, + "created_at": 1719485736.50801, "depends_on": { "macros": [ - "macro.dbt_postgres.postgres__listagg" + "macro.dbt.default__get_create_table_as_sql" ] }, "description": "", @@ -5047,49 +5438,47 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", + "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "meta": {}, - "name": "listagg", - "original_file_path": "macros/utils/listagg.sql", + "name": "get_create_table_as_sql", + "original_file_path": "macros/relations/table/create.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/listagg.sql", + "path": "macros/relations/table/create.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.listagg" + "unique_id": "macro.dbt.get_create_table_as_sql" }, - "macro.dbt.load_cached_relation": { + "macro.dbt.get_create_view_as_sql": { "arguments": [], - "created_at": 1696458269.817961, + "created_at": 1719485736.515145, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.default__get_create_view_as_sql" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", + "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "meta": {}, - "name": "load_cached_relation", - "original_file_path": "macros/adapters/relation.sql", + "name": "get_create_view_as_sql", + "original_file_path": "macros/relations/view/create.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/relation.sql", + "path": "macros/relations/view/create.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.load_cached_relation" + "unique_id": "macro.dbt.get_create_view_as_sql" }, - "macro.dbt.load_csv_rows": { + "macro.dbt.get_csv_sql": { "arguments": [], - "created_at": 1696458269.751443, + "created_at": 1719485736.460989, "depends_on": { "macros": [ - "macro.dbt.default__load_csv_rows" + "macro.dbt.default__get_csv_sql" ] }, "description": "", @@ -5097,25 +5486,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", + "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", "meta": {}, - "name": "load_csv_rows", + "name": "get_csv_sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "package_name": "dbt", "patch_path": null, "path": "macros/materializations/seeds/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.load_csv_rows" + "unique_id": "macro.dbt.get_csv_sql" }, - "macro.dbt.load_relation": { + "macro.dbt.get_dcl_statement_list": { "arguments": [], - "created_at": 1696458269.8181918, + "created_at": 1719485736.574908, "depends_on": { "macros": [ - "macro.dbt.load_cached_relation" + "macro.dbt.default__get_dcl_statement_list" ] }, "description": "", @@ -5123,25 +5510,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", + "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", "meta": {}, - "name": "load_relation", - "original_file_path": "macros/adapters/relation.sql", + "name": "get_dcl_statement_list", + "original_file_path": "macros/adapters/apply_grants.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/relation.sql", + "path": "macros/adapters/apply_grants.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.load_relation" + "unique_id": "macro.dbt.get_dcl_statement_list" }, - "macro.dbt.make_backup_relation": { + "macro.dbt.get_delete_insert_merge_sql": { "arguments": [], - "created_at": 1696458269.813613, + "created_at": 1719485736.414207, "depends_on": { "macros": [ - "macro.dbt_postgres.postgres__make_backup_relation" + "macro.dbt.default__get_delete_insert_merge_sql" ] }, "description": "", @@ -5149,49 +5534,47 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", + "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "meta": {}, - "name": "make_backup_relation", - "original_file_path": "macros/adapters/relation.sql", + "name": "get_delete_insert_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/relation.sql", + "path": "macros/materializations/models/incremental/merge.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.make_backup_relation" + "unique_id": "macro.dbt.get_delete_insert_merge_sql" }, - "macro.dbt.make_hook_config": { + "macro.dbt.get_drop_backup_sql": { "arguments": [], - "created_at": 1696458269.591149, + "created_at": 1719485736.476695, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.default__get_drop_backup_sql" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", + "macro_sql": "{%- macro get_drop_backup_sql(relation) -%}\n {{- log('Applying DROP BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "meta": {}, - "name": "make_hook_config", - "original_file_path": "macros/materializations/hooks.sql", + "name": "get_drop_backup_sql", + "original_file_path": "macros/relations/drop_backup.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/hooks.sql", + "path": "macros/relations/drop_backup.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.make_hook_config" + "unique_id": "macro.dbt.get_drop_backup_sql" }, - "macro.dbt.make_intermediate_relation": { + "macro.dbt.get_drop_index_sql": { "arguments": [], - "created_at": 1696458269.8121102, + "created_at": 1719485736.562388, "depends_on": { "macros": [ - "macro.dbt_postgres.postgres__make_intermediate_relation" + "macro.dbt_postgres.postgres__get_drop_index_sql" ] }, "description": "", @@ -5199,25 +5582,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", + "macro_sql": "{% macro get_drop_index_sql(relation, index_name) -%}\n {{ adapter.dispatch('get_drop_index_sql', 'dbt')(relation, index_name) }}\n{%- endmacro %}", "meta": {}, - "name": "make_intermediate_relation", - "original_file_path": "macros/adapters/relation.sql", + "name": "get_drop_index_sql", + "original_file_path": "macros/adapters/indexes.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/relation.sql", + "path": "macros/adapters/indexes.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.make_intermediate_relation" + "unique_id": "macro.dbt.get_drop_index_sql" }, - "macro.dbt.make_temp_relation": { + "macro.dbt.get_drop_sql": { "arguments": [], - "created_at": 1696458269.812738, + "created_at": 1719485736.470603, "depends_on": { "macros": [ - "macro.dbt_postgres.postgres__make_temp_relation" + "macro.dbt.default__get_drop_sql" ] }, "description": "", @@ -5225,40 +5606,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", + "macro_sql": "{%- macro get_drop_sql(relation) -%}\n {{- log('Applying DROP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "meta": {}, - "name": "make_temp_relation", - "original_file_path": "macros/adapters/relation.sql", + "name": "get_drop_sql", + "original_file_path": "macros/relations/drop.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/relation.sql", + "path": "macros/relations/drop.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.make_temp_relation" + "unique_id": "macro.dbt.get_drop_sql" }, - "macro.dbt.materialization_incremental_default": { + "macro.dbt.get_empty_schema_sql": { "arguments": [], - "created_at": 1696458269.699362, + "created_at": 1719485736.597257, "depends_on": { "macros": [ - "macro.dbt.load_cached_relation", - "macro.dbt.make_temp_relation", - "macro.dbt.make_intermediate_relation", - "macro.dbt.make_backup_relation", - "macro.dbt.should_full_refresh", - "macro.dbt.incremental_validate_on_schema_change", - "macro.dbt.drop_relation_if_exists", - "macro.dbt.run_hooks", - "macro.dbt.get_create_table_as_sql", - "macro.dbt.run_query", - "macro.dbt.process_schema_changes", - "macro.dbt.statement", - "macro.dbt.should_revoke", - "macro.dbt.apply_grants", - "macro.dbt.persist_docs", - "macro.dbt.create_indexes" + "macro.dbt.default__get_empty_schema_sql" ] }, "description": "", @@ -5266,37 +5630,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparison later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set incremental_predicates = config.get('incremental_predicates', none) %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", + "macro_sql": "{% macro get_empty_schema_sql(columns) -%}\n {{ return(adapter.dispatch('get_empty_schema_sql', 'dbt')(columns)) }}\n{% endmacro %}", "meta": {}, - "name": "materialization_incremental_default", - "original_file_path": "macros/materializations/models/incremental/incremental.sql", + "name": "get_empty_schema_sql", + "original_file_path": "macros/adapters/columns.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/incremental/incremental.sql", + "path": "macros/adapters/columns.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", - "supported_languages": [ - "sql" - ], - "tags": [], - "unique_id": "macro.dbt.materialization_incremental_default" + "supported_languages": null, + "unique_id": "macro.dbt.get_empty_schema_sql" }, - "macro.dbt.materialization_seed_default": { + "macro.dbt.get_empty_subquery_sql": { "arguments": [], - "created_at": 1696458269.737411, + "created_at": 1719485736.596842, "depends_on": { "macros": [ - "macro.dbt.should_full_refresh", - "macro.dbt.run_hooks", - "macro.dbt.reset_csv_table", - "macro.dbt.create_csv_table", - "macro.dbt.load_csv_rows", - "macro.dbt.noop_statement", - "macro.dbt.get_csv_sql", - "macro.dbt.should_revoke", - "macro.dbt.apply_grants", - "macro.dbt.persist_docs", - "macro.dbt.create_indexes" + "macro.dbt.default__get_empty_subquery_sql" ] }, "description": "", @@ -5304,40 +5654,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparison later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", + "macro_sql": "{% macro get_empty_subquery_sql(select_sql, select_sql_header=none) -%}\n {{ return(adapter.dispatch('get_empty_subquery_sql', 'dbt')(select_sql, select_sql_header)) }}\n{% endmacro %}", "meta": {}, - "name": "materialization_seed_default", - "original_file_path": "macros/materializations/seeds/seed.sql", + "name": "get_empty_subquery_sql", + "original_file_path": "macros/adapters/columns.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/seeds/seed.sql", + "path": "macros/adapters/columns.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", - "supported_languages": [ - "sql" - ], - "tags": [], - "unique_id": "macro.dbt.materialization_seed_default" + "supported_languages": null, + "unique_id": "macro.dbt.get_empty_subquery_sql" }, - "macro.dbt.materialization_snapshot_default": { + "macro.dbt.get_expected_sql": { "arguments": [], - "created_at": 1696458269.65698, + "created_at": 1719485736.608841, "depends_on": { "macros": [ - "macro.dbt.get_or_create_relation", - "macro.dbt.run_hooks", - "macro.dbt.strategy_dispatch", - "macro.dbt.build_snapshot_table", - "macro.dbt.create_table_as", - "macro.dbt.build_snapshot_staging_table", - "macro.dbt.create_columns", - "macro.dbt.snapshot_merge_sql", - "macro.dbt.statement", - "macro.dbt.should_revoke", - "macro.dbt.apply_grants", - "macro.dbt.persist_docs", - "macro.dbt.create_indexes", - "macro.dbt.post_snapshot" + "macro.dbt.format_row" ] }, "description": "", @@ -5345,37 +5678,25 @@ "node_color": null, "show": true }, - "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparison later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", + "macro_sql": "{% macro get_expected_sql(rows, column_name_to_data_types) %}\n\n{%- if (rows | length) == 0 -%}\n select * from dbt_internal_unit_test_actual\n limit 0\n{%- else -%}\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\nselect\n{%- for column_name, column_value in formatted_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n{%- endif -%}\n\n{% endmacro %}", "meta": {}, - "name": "materialization_snapshot_default", - "original_file_path": "macros/materializations/snapshots/snapshot.sql", + "name": "get_expected_sql", + "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/snapshots/snapshot.sql", + "path": "macros/unit_test_sql/get_fixture_sql.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", - "supported_languages": [ - "sql" - ], - "tags": [], - "unique_id": "macro.dbt.materialization_snapshot_default" + "supported_languages": null, + "unique_id": "macro.dbt.get_expected_sql" }, - "macro.dbt.materialization_table_default": { + "macro.dbt.get_fixture_sql": { "arguments": [], - "created_at": 1696458269.718855, + "created_at": 1719485736.607797, "depends_on": { "macros": [ - "macro.dbt.load_cached_relation", - "macro.dbt.make_intermediate_relation", - "macro.dbt.make_backup_relation", - "macro.dbt.drop_relation_if_exists", - "macro.dbt.run_hooks", - "macro.dbt.statement", - "macro.dbt.get_create_table_as_sql", - "macro.dbt.create_indexes", - "macro.dbt.should_revoke", - "macro.dbt.apply_grants", - "macro.dbt.persist_docs" + "macro.dbt.load_relation", + "macro.dbt.safe_cast", + "macro.dbt.format_row" ] }, "description": "", @@ -5383,30 +5704,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparison later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", + "macro_sql": "{% macro get_fixture_sql(rows, column_name_to_data_types) %}\n-- Fixture for {{ model.name }}\n{% set default_row = {} %}\n\n{%- if not column_name_to_data_types -%}\n{#-- Use defer_relation IFF it is available in the manifest and 'this' is missing from the database --#}\n{%- set this_or_defer_relation = defer_relation if (defer_relation and not load_relation(this)) else this -%}\n{%- set columns_in_relation = adapter.get_columns_in_relation(this_or_defer_relation) -%}\n\n{%- set column_name_to_data_types = {} -%}\n{%- for column in columns_in_relation -%}\n{#-- This needs to be a case-insensitive comparison --#}\n{%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n{%- endfor -%}\n{%- endif -%}\n\n{%- if not column_name_to_data_types -%}\n {{ exceptions.raise_compiler_error(\"Not able to get columns for unit test '\" ~ model.name ~ \"' from relation \" ~ this ~ \" because the relation doesn't exist\") }}\n{%- endif -%}\n\n{%- for column_name, column_type in column_name_to_data_types.items() -%}\n {%- do default_row.update({column_name: (safe_cast(\"null\", column_type) | trim )}) -%}\n{%- endfor -%}\n\n\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\n{%- set default_row_copy = default_row.copy() -%}\n{%- do default_row_copy.update(formatted_row) -%}\nselect\n{%- for column_name, column_value in default_row_copy.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n\n{%- if (rows | length) == 0 -%}\n select\n {%- for column_name, column_value in default_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%},{%- endif %}\n {%- endfor %}\n limit 0\n{%- endif -%}\n{% endmacro %}", "meta": {}, - "name": "materialization_table_default", - "original_file_path": "macros/materializations/models/table/table.sql", + "name": "get_fixture_sql", + "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/table/table.sql", + "path": "macros/unit_test_sql/get_fixture_sql.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", - "supported_languages": [ - "sql" - ], - "tags": [], - "unique_id": "macro.dbt.materialization_table_default" + "supported_languages": null, + "unique_id": "macro.dbt.get_fixture_sql" }, - "macro.dbt.materialization_test_default": { + "macro.dbt.get_grant_sql": { "arguments": [], - "created_at": 1696458269.660993, + "created_at": 1719485736.574021, "depends_on": { "macros": [ - "macro.dbt.should_store_failures", - "macro.dbt.statement", - "macro.dbt.create_table_as", - "macro.dbt.get_test_sql" + "macro.dbt.default__get_grant_sql" ] }, "description": "", @@ -5414,36 +5728,23 @@ "node_color": null, "show": true }, - "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type='table') -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ create_table_as(False, target_relation, sql) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql %}\n\n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", + "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "meta": {}, - "name": "materialization_test_default", - "original_file_path": "macros/materializations/tests/test.sql", + "name": "get_grant_sql", + "original_file_path": "macros/adapters/apply_grants.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/tests/test.sql", + "path": "macros/adapters/apply_grants.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", - "supported_languages": [ - "sql" - ], - "tags": [], - "unique_id": "macro.dbt.materialization_test_default" + "supported_languages": null, + "unique_id": "macro.dbt.get_grant_sql" }, - "macro.dbt.materialization_view_default": { + "macro.dbt.get_incremental_append_sql": { "arguments": [], - "created_at": 1696458269.726018, + "created_at": 1719485736.418218, "depends_on": { "macros": [ - "macro.dbt.load_cached_relation", - "macro.dbt.make_intermediate_relation", - "macro.dbt.make_backup_relation", - "macro.dbt.run_hooks", - "macro.dbt.drop_relation_if_exists", - "macro.dbt.statement", - "macro.dbt.get_create_view_as_sql", - "macro.dbt.should_revoke", - "macro.dbt.apply_grants", - "macro.dbt.persist_docs" + "macro.dbt.default__get_incremental_append_sql" ] }, "description": "", @@ -5451,51 +5752,47 @@ "node_color": null, "show": true }, - "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparison later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", + "macro_sql": "{% macro get_incremental_append_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "meta": {}, - "name": "materialization_view_default", - "original_file_path": "macros/materializations/models/view/view.sql", + "name": "get_incremental_append_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/view/view.sql", + "path": "macros/materializations/models/incremental/strategies.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", - "supported_languages": [ - "sql" - ], - "tags": [], - "unique_id": "macro.dbt.materialization_view_default" + "supported_languages": null, + "unique_id": "macro.dbt.get_incremental_append_sql" }, - "macro.dbt.noop_statement": { + "macro.dbt.get_incremental_default_sql": { "arguments": [], - "created_at": 1696458269.7650838, + "created_at": 1719485736.421412, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt_postgres.postgres__get_incremental_default_sql" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", + "macro_sql": "{% macro get_incremental_default_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "meta": {}, - "name": "noop_statement", - "original_file_path": "macros/etc/statement.sql", + "name": "get_incremental_default_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/etc/statement.sql", + "path": "macros/materializations/models/incremental/strategies.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.noop_statement" + "unique_id": "macro.dbt.get_incremental_default_sql" }, - "macro.dbt.partition_range": { + "macro.dbt.get_incremental_delete_insert_sql": { "arguments": [], - "created_at": 1696458269.7719522, + "created_at": 1719485736.4188569, "depends_on": { "macros": [ - "macro.dbt.dates_in_range" + "macro.dbt.default__get_incremental_delete_insert_sql" ] }, "description": "", @@ -5503,25 +5800,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", + "macro_sql": "{% macro get_incremental_delete_insert_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "meta": {}, - "name": "partition_range", - "original_file_path": "macros/etc/datetime.sql", + "name": "get_incremental_delete_insert_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/etc/datetime.sql", + "path": "macros/materializations/models/incremental/strategies.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.partition_range" + "unique_id": "macro.dbt.get_incremental_delete_insert_sql" }, - "macro.dbt.persist_docs": { + "macro.dbt.get_incremental_insert_overwrite_sql": { "arguments": [], - "created_at": 1696458269.833451, + "created_at": 1719485736.420912, "depends_on": { "macros": [ - "macro.dbt.default__persist_docs" + "macro.dbt.default__get_incremental_insert_overwrite_sql" ] }, "description": "", @@ -5529,25 +5824,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", + "macro_sql": "{% macro get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "meta": {}, - "name": "persist_docs", - "original_file_path": "macros/adapters/persist_docs.sql", + "name": "get_incremental_insert_overwrite_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/persist_docs.sql", + "path": "macros/materializations/models/incremental/strategies.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.persist_docs" + "unique_id": "macro.dbt.get_incremental_insert_overwrite_sql" }, - "macro.dbt.position": { + "macro.dbt.get_incremental_merge_sql": { "arguments": [], - "created_at": 1696458269.7855082, + "created_at": 1719485736.419648, "depends_on": { "macros": [ - "macro.dbt.default__position" + "macro.dbt.default__get_incremental_merge_sql" ] }, "description": "", @@ -5555,25 +5848,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", + "macro_sql": "{% macro get_incremental_merge_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "meta": {}, - "name": "position", - "original_file_path": "macros/utils/position.sql", + "name": "get_incremental_merge_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/position.sql", + "path": "macros/materializations/models/incremental/strategies.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.position" + "unique_id": "macro.dbt.get_incremental_merge_sql" }, - "macro.dbt.post_snapshot": { + "macro.dbt.get_insert_into_sql": { "arguments": [], - "created_at": 1696458269.6167622, + "created_at": 1719485736.4218738, "depends_on": { "macros": [ - "macro.dbt.default__post_snapshot" + "macro.dbt.get_quoted_csv" ] }, "description": "", @@ -5581,26 +5872,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", + "macro_sql": "{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ temp_relation }}\n )\n\n{% endmacro %}", "meta": {}, - "name": "post_snapshot", - "original_file_path": "macros/materializations/snapshots/helpers.sql", + "name": "get_insert_into_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/snapshots/helpers.sql", + "path": "macros/materializations/models/incremental/strategies.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.post_snapshot" + "unique_id": "macro.dbt.get_insert_into_sql" }, - "macro.dbt.process_schema_changes": { + "macro.dbt.get_insert_overwrite_merge_sql": { "arguments": [], - "created_at": 1696458269.714473, + "created_at": 1719485736.415607, "depends_on": { "macros": [ - "macro.dbt.check_for_schema_changes", - "macro.dbt.sync_column_schemas" + "macro.dbt.default__get_insert_overwrite_merge_sql" ] }, "description": "", @@ -5608,77 +5896,71 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n\n Additional troubleshooting context:\n Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}\n Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}\n New column types: {{ schema_changes_dict['new_target_types'] }}\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", + "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "meta": {}, - "name": "process_schema_changes", - "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", + "name": "get_insert_overwrite_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/incremental/on_schema_change.sql", + "path": "macros/materializations/models/incremental/merge.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.process_schema_changes" + "unique_id": "macro.dbt.get_insert_overwrite_merge_sql" }, - "macro.dbt.py_current_timestring": { + "macro.dbt.get_intervals_between": { "arguments": [], - "created_at": 1696458269.7723362, + "created_at": 1719485736.527589, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.default__get_intervals_between" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", + "macro_sql": "{% macro get_intervals_between(start_date, end_date, datepart) -%}\n {{ return(adapter.dispatch('get_intervals_between', 'dbt')(start_date, end_date, datepart)) }}\n{%- endmacro %}", "meta": {}, - "name": "py_current_timestring", - "original_file_path": "macros/etc/datetime.sql", + "name": "get_intervals_between", + "original_file_path": "macros/utils/date_spine.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/etc/datetime.sql", + "path": "macros/utils/date_spine.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.py_current_timestring" + "unique_id": "macro.dbt.get_intervals_between" }, - "macro.dbt.py_script_comment": { + "macro.dbt.get_limit_subquery_sql": { "arguments": [], - "created_at": 1696458269.853185, + "created_at": 1719485736.58177, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.default__get_limit_subquery_sql" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{%macro py_script_comment()%}\n{%endmacro%}", + "macro_sql": "{% macro get_limit_subquery_sql(sql, limit) %}\n {{ adapter.dispatch('get_limit_subquery_sql', 'dbt')(sql, limit) }}\n{% endmacro %}", "meta": {}, - "name": "py_script_comment", - "original_file_path": "macros/python_model/python.sql", + "name": "get_limit_subquery_sql", + "original_file_path": "macros/adapters/show.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/python_model/python.sql", + "path": "macros/adapters/show.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.py_script_comment" + "unique_id": "macro.dbt.get_limit_subquery_sql" }, - "macro.dbt.py_script_postfix": { + "macro.dbt.get_materialized_view_configuration_changes": { "arguments": [], - "created_at": 1696458269.85307, + "created_at": 1719485736.494153, "depends_on": { "macros": [ - "macro.dbt.build_ref_function", - "macro.dbt.build_source_function", - "macro.dbt.build_config_dict", - "macro.dbt.is_incremental", - "macro.dbt.py_script_comment" + "macro.dbt_postgres.postgres__get_materialized_view_configuration_changes" ] }, "description": "", @@ -5686,25 +5968,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro py_script_postfix(model) %}\n# This part is user provided model code\n# you will need to copy the next section to run the code\n# COMMAND ----------\n# this part is dbt logic for get ref work, do not modify\n\n{{ build_ref_function(model ) }}\n{{ build_source_function(model ) }}\n{{ build_config_dict(model) }}\n\nclass config:\n def __init__(self, *args, **kwargs):\n pass\n\n @staticmethod\n def get(key, default=None):\n return config_dict.get(key, default)\n\nclass this:\n \"\"\"dbt.this() or dbt.this.identifier\"\"\"\n database = '{{ this.database }}'\n schema = '{{ this.schema }}'\n identifier = '{{ this.identifier }}'\n def __repr__(self):\n return '{{ this }}'\n\n\nclass dbtObj:\n def __init__(self, load_df_function) -> None:\n self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function)\n self.ref = lambda *args: ref(*args, dbt_load_df_function=load_df_function)\n self.config = config\n self.this = this()\n self.is_incremental = {{ is_incremental() }}\n\n# COMMAND ----------\n{{py_script_comment()}}\n{% endmacro %}", + "macro_sql": "{% macro get_materialized_view_configuration_changes(existing_relation, new_config) %}\n /* {#\n It's recommended that configuration changes be formatted as follows:\n {\"\": [{\"action\": \"\", \"context\": ...}]}\n\n For example:\n {\n \"indexes\": [\n {\"action\": \"drop\", \"context\": \"index_abc\"},\n {\"action\": \"create\", \"context\": {\"columns\": [\"column_1\", \"column_2\"], \"type\": \"hash\", \"unique\": True}},\n ],\n }\n\n Either way, `get_materialized_view_configuration_changes` needs to align with `get_alter_materialized_view_as_sql`.\n #} */\n {{- log('Determining configuration changes on: ' ~ existing_relation) -}}\n {%- do return(adapter.dispatch('get_materialized_view_configuration_changes', 'dbt')(existing_relation, new_config)) -%}\n{% endmacro %}", "meta": {}, - "name": "py_script_postfix", - "original_file_path": "macros/python_model/python.sql", + "name": "get_materialized_view_configuration_changes", + "original_file_path": "macros/relations/materialized_view/alter.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/python_model/python.sql", + "path": "macros/relations/materialized_view/alter.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.py_script_postfix" + "unique_id": "macro.dbt.get_materialized_view_configuration_changes" }, - "macro.dbt.rename_relation": { + "macro.dbt.get_merge_sql": { "arguments": [], - "created_at": 1696458269.815678, + "created_at": 1719485736.410901, "depends_on": { "macros": [ - "macro.dbt.default__rename_relation" + "macro.dbt.default__get_merge_sql" ] }, "description": "", @@ -5712,25 +5992,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", + "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n -- back compat for old kwarg name\n {% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "meta": {}, - "name": "rename_relation", - "original_file_path": "macros/adapters/relation.sql", + "name": "get_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/relation.sql", + "path": "macros/materializations/models/incremental/merge.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.rename_relation" + "unique_id": "macro.dbt.get_merge_sql" }, - "macro.dbt.replace": { + "macro.dbt.get_merge_update_columns": { "arguments": [], - "created_at": 1696458269.7736168, + "created_at": 1719485736.397727, "depends_on": { "macros": [ - "macro.dbt.default__replace" + "macro.dbt.default__get_merge_update_columns" ] }, "description": "", @@ -5738,25 +6016,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", + "macro_sql": "{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}\n{% endmacro %}", "meta": {}, - "name": "replace", - "original_file_path": "macros/utils/replace.sql", + "name": "get_merge_update_columns", + "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/replace.sql", + "path": "macros/materializations/models/incremental/column_helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.replace" + "unique_id": "macro.dbt.get_merge_update_columns" }, - "macro.dbt.reset_csv_table": { + "macro.dbt.get_or_create_relation": { "arguments": [], - "created_at": 1696458269.748157, + "created_at": 1719485736.567741, "depends_on": { "macros": [ - "macro.dbt.default__reset_csv_table" + "macro.dbt.default__get_or_create_relation" ] }, "description": "", @@ -5764,25 +6040,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", + "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", "meta": {}, - "name": "reset_csv_table", - "original_file_path": "macros/materializations/seeds/helpers.sql", + "name": "get_or_create_relation", + "original_file_path": "macros/adapters/relation.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/seeds/helpers.sql", + "path": "macros/adapters/relation.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.reset_csv_table" + "unique_id": "macro.dbt.get_or_create_relation" }, - "macro.dbt.right": { + "macro.dbt.get_powers_of_two": { "arguments": [], - "created_at": 1696458269.778283, + "created_at": 1719485736.534465, "depends_on": { "macros": [ - "macro.dbt.default__right" + "macro.dbt.default__get_powers_of_two" ] }, "description": "", @@ -5790,51 +6064,45 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", + "macro_sql": "{% macro get_powers_of_two(upper_bound) %}\n {{ return(adapter.dispatch('get_powers_of_two', 'dbt')(upper_bound)) }}\n{% endmacro %}", "meta": {}, - "name": "right", - "original_file_path": "macros/utils/right.sql", + "name": "get_powers_of_two", + "original_file_path": "macros/utils/generate_series.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/right.sql", + "path": "macros/utils/generate_series.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.right" + "unique_id": "macro.dbt.get_powers_of_two" }, - "macro.dbt.run_hooks": { + "macro.dbt.get_quoted_csv": { "arguments": [], - "created_at": 1696458269.590806, + "created_at": 1719485736.395989, "depends_on": { - "macros": [ - "macro.dbt.statement" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", + "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", "meta": {}, - "name": "run_hooks", - "original_file_path": "macros/materializations/hooks.sql", + "name": "get_quoted_csv", + "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/hooks.sql", + "path": "macros/materializations/models/incremental/column_helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.run_hooks" + "unique_id": "macro.dbt.get_quoted_csv" }, - "macro.dbt.run_query": { + "macro.dbt.get_relation_last_modified": { "arguments": [], - "created_at": 1696458269.765571, + "created_at": 1719485736.593024, "depends_on": { "macros": [ - "macro.dbt.statement" + "macro.dbt.default__get_relation_last_modified" ] }, "description": "", @@ -5842,25 +6110,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", + "macro_sql": "{% macro get_relation_last_modified(information_schema, relations) %}\n {{ return(adapter.dispatch('get_relation_last_modified', 'dbt')(information_schema, relations)) }}\n{% endmacro %}", "meta": {}, - "name": "run_query", - "original_file_path": "macros/etc/statement.sql", + "name": "get_relation_last_modified", + "original_file_path": "macros/adapters/metadata.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/etc/statement.sql", + "path": "macros/adapters/metadata.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.run_query" + "unique_id": "macro.dbt.get_relation_last_modified" }, - "macro.dbt.safe_cast": { + "macro.dbt.get_relations": { "arguments": [], - "created_at": 1696458269.7820318, + "created_at": 1719485736.592459, "depends_on": { "macros": [ - "macro.dbt.default__safe_cast" + "macro.dbt_postgres.postgres__get_relations" ] }, "description": "", @@ -5868,73 +6134,71 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", + "macro_sql": "{% macro get_relations() %}\n {{ return(adapter.dispatch('get_relations', 'dbt')()) }}\n{% endmacro %}", "meta": {}, - "name": "safe_cast", - "original_file_path": "macros/utils/safe_cast.sql", + "name": "get_relations", + "original_file_path": "macros/adapters/metadata.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/safe_cast.sql", + "path": "macros/adapters/metadata.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.safe_cast" + "unique_id": "macro.dbt.get_relations" }, - "macro.dbt.set_sql_header": { + "macro.dbt.get_rename_intermediate_sql": { "arguments": [], - "created_at": 1696458269.592764, + "created_at": 1719485736.484606, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.default__get_rename_intermediate_sql" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", + "macro_sql": "{%- macro get_rename_intermediate_sql(relation) -%}\n {{- log('Applying RENAME INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_intermediate_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "meta": {}, - "name": "set_sql_header", - "original_file_path": "macros/materializations/configs.sql", + "name": "get_rename_intermediate_sql", + "original_file_path": "macros/relations/rename_intermediate.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/configs.sql", + "path": "macros/relations/rename_intermediate.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.set_sql_header" + "unique_id": "macro.dbt.get_rename_intermediate_sql" }, - "macro.dbt.should_full_refresh": { + "macro.dbt.get_rename_materialized_view_sql": { "arguments": [], - "created_at": 1696458269.593334, + "created_at": 1719485736.4909282, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt_postgres.postgres__get_rename_materialized_view_sql" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", + "macro_sql": "{% macro get_rename_materialized_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "meta": {}, - "name": "should_full_refresh", - "original_file_path": "macros/materializations/configs.sql", + "name": "get_rename_materialized_view_sql", + "original_file_path": "macros/relations/materialized_view/rename.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/configs.sql", + "path": "macros/relations/materialized_view/rename.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.should_full_refresh" + "unique_id": "macro.dbt.get_rename_materialized_view_sql" }, - "macro.dbt.should_revoke": { + "macro.dbt.get_rename_sql": { "arguments": [], - "created_at": 1696458269.824169, + "created_at": 1719485736.47766, "depends_on": { "macros": [ - "macro.dbt.copy_grants" + "macro.dbt.default__get_rename_sql" ] }, "description": "", @@ -5942,49 +6206,47 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", + "macro_sql": "{%- macro get_rename_sql(relation, new_name) -%}\n {{- log('Applying RENAME to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_sql', 'dbt')(relation, new_name) -}}\n{%- endmacro -%}\n\n\n", "meta": {}, - "name": "should_revoke", - "original_file_path": "macros/adapters/apply_grants.sql", + "name": "get_rename_sql", + "original_file_path": "macros/relations/rename.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/apply_grants.sql", + "path": "macros/relations/rename.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.should_revoke" + "unique_id": "macro.dbt.get_rename_sql" }, - "macro.dbt.should_store_failures": { + "macro.dbt.get_rename_table_sql": { "arguments": [], - "created_at": 1696458269.593961, + "created_at": 1719485736.5068932, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt_postgres.postgres__get_rename_table_sql" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", + "macro_sql": "{% macro get_rename_table_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_table_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "meta": {}, - "name": "should_store_failures", - "original_file_path": "macros/materializations/configs.sql", + "name": "get_rename_table_sql", + "original_file_path": "macros/relations/table/rename.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/configs.sql", + "path": "macros/relations/table/rename.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.should_store_failures" + "unique_id": "macro.dbt.get_rename_table_sql" }, - "macro.dbt.snapshot_check_all_get_existing_columns": { + "macro.dbt.get_rename_view_sql": { "arguments": [], - "created_at": 1696458269.6062112, + "created_at": 1719485736.5145411, "depends_on": { "macros": [ - "macro.dbt.get_columns_in_query" + "macro.dbt_postgres.postgres__get_rename_view_sql" ] }, "description": "", @@ -5992,28 +6254,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n select {{ check_cols_config | join(', ') }} from ({{ node['compiled_code'] }}) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", + "macro_sql": "{% macro get_rename_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "meta": {}, - "name": "snapshot_check_all_get_existing_columns", - "original_file_path": "macros/materializations/snapshots/strategies.sql", + "name": "get_rename_view_sql", + "original_file_path": "macros/relations/view/rename.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/snapshots/strategies.sql", + "path": "macros/relations/view/rename.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns" + "unique_id": "macro.dbt.get_rename_view_sql" }, - "macro.dbt.snapshot_check_strategy": { + "macro.dbt.get_replace_materialized_view_sql": { "arguments": [], - "created_at": 1696458269.6088378, + "created_at": 1719485736.487545, "depends_on": { "macros": [ - "macro.dbt.snapshot_get_time", - "macro.dbt.snapshot_check_all_get_existing_columns", - "macro.dbt.get_true_sql", - "macro.dbt.snapshot_hash_arguments" + "macro.dbt.default__get_replace_materialized_view_sql" ] }, "description": "", @@ -6021,25 +6278,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n {% set updated_at = config.get('updated_at', snapshot_get_time()) %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", + "macro_sql": "{% macro get_replace_materialized_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_materialized_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "meta": {}, - "name": "snapshot_check_strategy", - "original_file_path": "macros/materializations/snapshots/strategies.sql", + "name": "get_replace_materialized_view_sql", + "original_file_path": "macros/relations/materialized_view/replace.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/snapshots/strategies.sql", + "path": "macros/relations/materialized_view/replace.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.snapshot_check_strategy" + "unique_id": "macro.dbt.get_replace_materialized_view_sql" }, - "macro.dbt.snapshot_get_time": { + "macro.dbt.get_replace_sql": { "arguments": [], - "created_at": 1696458269.80277, + "created_at": 1719485736.4724941, "depends_on": { "macros": [ - "macro.dbt_postgres.postgres__snapshot_get_time" + "macro.dbt.default__get_replace_sql" ] }, "description": "", @@ -6047,25 +6302,23 @@ "node_color": null, "show": true }, - "macro_sql": "\n\n{%- macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro -%}\n\n", + "macro_sql": "{% macro get_replace_sql(existing_relation, target_relation, sql) %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {{- adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql) -}}\n{% endmacro %}", "meta": {}, - "name": "snapshot_get_time", - "original_file_path": "macros/adapters/timestamps.sql", + "name": "get_replace_sql", + "original_file_path": "macros/relations/replace.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/timestamps.sql", + "path": "macros/relations/replace.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.snapshot_get_time" + "unique_id": "macro.dbt.get_replace_sql" }, - "macro.dbt.snapshot_hash_arguments": { + "macro.dbt.get_replace_table_sql": { "arguments": [], - "created_at": 1696458269.6016371, + "created_at": 1719485736.506175, "depends_on": { "macros": [ - "macro.dbt.default__snapshot_hash_arguments" + "macro.dbt_postgres.postgres__get_replace_table_sql" ] }, "description": "", @@ -6073,25 +6326,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", + "macro_sql": "{% macro get_replace_table_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_table_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "meta": {}, - "name": "snapshot_hash_arguments", - "original_file_path": "macros/materializations/snapshots/strategies.sql", + "name": "get_replace_table_sql", + "original_file_path": "macros/relations/table/replace.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/snapshots/strategies.sql", + "path": "macros/relations/table/replace.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.snapshot_hash_arguments" + "unique_id": "macro.dbt.get_replace_table_sql" }, - "macro.dbt.snapshot_merge_sql": { + "macro.dbt.get_replace_view_sql": { "arguments": [], - "created_at": 1696458269.594904, + "created_at": 1719485736.5121448, "depends_on": { "macros": [ - "macro.dbt_postgres.postgres__snapshot_merge_sql" + "macro.dbt_postgres.postgres__get_replace_view_sql" ] }, "description": "", @@ -6099,25 +6350,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", + "macro_sql": "{% macro get_replace_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "meta": {}, - "name": "snapshot_merge_sql", - "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", + "name": "get_replace_view_sql", + "original_file_path": "macros/relations/view/replace.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/snapshots/snapshot_merge.sql", + "path": "macros/relations/view/replace.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.snapshot_merge_sql" + "unique_id": "macro.dbt.get_replace_view_sql" }, - "macro.dbt.snapshot_staging_table": { + "macro.dbt.get_revoke_sql": { "arguments": [], - "created_at": 1696458269.617667, + "created_at": 1719485736.574466, "depends_on": { "macros": [ - "macro.dbt.default__snapshot_staging_table" + "macro.dbt.default__get_revoke_sql" ] }, "description": "", @@ -6125,51 +6374,45 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", + "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "meta": {}, - "name": "snapshot_staging_table", - "original_file_path": "macros/materializations/snapshots/helpers.sql", + "name": "get_revoke_sql", + "original_file_path": "macros/adapters/apply_grants.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/snapshots/helpers.sql", + "path": "macros/adapters/apply_grants.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.snapshot_staging_table" + "unique_id": "macro.dbt.get_revoke_sql" }, - "macro.dbt.snapshot_string_as_time": { + "macro.dbt.get_seed_column_quoted_csv": { "arguments": [], - "created_at": 1696458269.603466, + "created_at": 1719485736.462852, "depends_on": { - "macros": [ - "macro.dbt_postgres.postgres__snapshot_string_as_time" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", + "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "meta": {}, - "name": "snapshot_string_as_time", - "original_file_path": "macros/materializations/snapshots/strategies.sql", + "name": "get_seed_column_quoted_csv", + "original_file_path": "macros/materializations/seeds/helpers.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/snapshots/strategies.sql", + "path": "macros/materializations/seeds/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.snapshot_string_as_time" + "unique_id": "macro.dbt.get_seed_column_quoted_csv" }, - "macro.dbt.snapshot_timestamp_strategy": { + "macro.dbt.get_select_subquery": { "arguments": [], - "created_at": 1696458269.603195, + "created_at": 1719485736.510276, "depends_on": { "macros": [ - "macro.dbt.snapshot_hash_arguments" + "macro.dbt.default__get_select_subquery" ] }, "description": "", @@ -6177,25 +6420,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", + "macro_sql": "{% macro get_select_subquery(sql) %}\n {{ return(adapter.dispatch('get_select_subquery', 'dbt')(sql)) }}\n{% endmacro %}", "meta": {}, - "name": "snapshot_timestamp_strategy", - "original_file_path": "macros/materializations/snapshots/strategies.sql", + "name": "get_select_subquery", + "original_file_path": "macros/relations/table/create.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/snapshots/strategies.sql", + "path": "macros/relations/table/create.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.snapshot_timestamp_strategy" + "unique_id": "macro.dbt.get_select_subquery" }, - "macro.dbt.split_part": { + "macro.dbt.get_show_grant_sql": { "arguments": [], - "created_at": 1696458269.795978, + "created_at": 1719485736.573664, "depends_on": { "macros": [ - "macro.dbt_postgres.postgres__split_part" + "macro.dbt_postgres.postgres__get_show_grant_sql" ] }, "description": "", @@ -6203,97 +6444,95 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", + "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", "meta": {}, - "name": "split_part", - "original_file_path": "macros/utils/split_part.sql", + "name": "get_show_grant_sql", + "original_file_path": "macros/adapters/apply_grants.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/split_part.sql", + "path": "macros/adapters/apply_grants.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.split_part" + "unique_id": "macro.dbt.get_show_grant_sql" }, - "macro.dbt.sql_convert_columns_in_relation": { + "macro.dbt.get_show_indexes_sql": { "arguments": [], - "created_at": 1696458269.844328, + "created_at": 1719485736.562955, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt_postgres.postgres__get_show_indexes_sql" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", + "macro_sql": "{% macro get_show_indexes_sql(relation) -%}\n {{ adapter.dispatch('get_show_indexes_sql', 'dbt')(relation) }}\n{%- endmacro %}", "meta": {}, - "name": "sql_convert_columns_in_relation", - "original_file_path": "macros/adapters/columns.sql", + "name": "get_show_indexes_sql", + "original_file_path": "macros/adapters/indexes.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/columns.sql", + "path": "macros/adapters/indexes.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.sql_convert_columns_in_relation" + "unique_id": "macro.dbt.get_show_indexes_sql" }, - "macro.dbt.statement": { + "macro.dbt.get_show_sql": { "arguments": [], - "created_at": 1696458269.764031, + "created_at": 1719485736.581328, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.get_limit_subquery_sql" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "\n{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%}\n {%- if execute: -%}\n {%- set compiled_code = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime {} for node \"{}\"'.format(language, model['unique_id'])) }}\n {{ write(compiled_code) }}\n {%- endif -%}\n {%- if language == 'sql'-%}\n {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- elif language == 'python' -%}\n {%- set res = submit_python_job(model, compiled_code) -%}\n {#-- TODO: What should table be for python models? --#}\n {%- set table = None -%}\n {%- else -%}\n {% do exceptions.raise_compiler_error(\"statement macro didn't get supported language\") %}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", + "macro_sql": "{% macro get_show_sql(compiled_code, sql_header, limit) -%}\n {%- if sql_header -%}\n {{ sql_header }}\n {%- endif -%}\n {%- if limit is not none -%}\n {{ get_limit_subquery_sql(compiled_code, limit) }}\n {%- else -%}\n {{ compiled_code }}\n {%- endif -%}\n{% endmacro %}", "meta": {}, - "name": "statement", - "original_file_path": "macros/etc/statement.sql", + "name": "get_show_sql", + "original_file_path": "macros/adapters/show.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/etc/statement.sql", + "path": "macros/adapters/show.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.statement" + "unique_id": "macro.dbt.get_show_sql" }, - "macro.dbt.strategy_dispatch": { + "macro.dbt.get_table_columns_and_constraints": { "arguments": [], - "created_at": 1696458269.601364, + "created_at": 1719485736.497471, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.default__get_table_columns_and_constraints" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", + "macro_sql": "{%- macro get_table_columns_and_constraints() -%}\n {{ adapter.dispatch('get_table_columns_and_constraints', 'dbt')() }}\n{%- endmacro -%}\n\n", "meta": {}, - "name": "strategy_dispatch", - "original_file_path": "macros/materializations/snapshots/strategies.sql", + "name": "get_table_columns_and_constraints", + "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/snapshots/strategies.sql", + "path": "macros/relations/column/columns_spec_ddl.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.strategy_dispatch" + "unique_id": "macro.dbt.get_table_columns_and_constraints" }, - "macro.dbt.string_literal": { + "macro.dbt.get_test_sql": { "arguments": [], - "created_at": 1696458269.786274, + "created_at": 1719485736.37231, "depends_on": { "macros": [ - "macro.dbt.default__string_literal" + "macro.dbt.default__get_test_sql" ] }, "description": "", @@ -6301,25 +6540,23 @@ "node_color": null, "show": true }, - "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", + "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "meta": {}, - "name": "string_literal", - "original_file_path": "macros/utils/literal.sql", + "name": "get_test_sql", + "original_file_path": "macros/materializations/tests/helpers.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/literal.sql", + "path": "macros/materializations/tests/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.string_literal" + "unique_id": "macro.dbt.get_test_sql" }, - "macro.dbt.support_multiple_grantees_per_dcl_statement": { + "macro.dbt.get_true_sql": { "arguments": [], - "created_at": 1696458269.823423, + "created_at": 1719485736.355415, "depends_on": { "macros": [ - "macro.dbt.default__support_multiple_grantees_per_dcl_statement" + "macro.dbt.default__get_true_sql" ] }, "description": "", @@ -6327,26 +6564,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", + "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", "meta": {}, - "name": "support_multiple_grantees_per_dcl_statement", - "original_file_path": "macros/adapters/apply_grants.sql", + "name": "get_true_sql", + "original_file_path": "macros/materializations/snapshots/helpers.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/apply_grants.sql", + "path": "macros/materializations/snapshots/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement" + "unique_id": "macro.dbt.get_true_sql" }, - "macro.dbt.sync_column_schemas": { + "macro.dbt.get_unit_test_sql": { "arguments": [], - "created_at": 1696458269.713077, + "created_at": 1719485736.3730109, "depends_on": { "macros": [ - "macro.dbt.alter_relation_add_remove_columns", - "macro.dbt.alter_column_type" + "macro.dbt.default__get_unit_test_sql" ] }, "description": "", @@ -6354,25 +6588,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", + "macro_sql": "{% macro get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n {{ adapter.dispatch('get_unit_test_sql', 'dbt')(main_sql, expected_fixture_sql, expected_column_names) }}\n{%- endmacro %}", "meta": {}, - "name": "sync_column_schemas", - "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", + "name": "get_unit_test_sql", + "original_file_path": "macros/materializations/tests/helpers.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/materializations/models/incremental/on_schema_change.sql", + "path": "macros/materializations/tests/helpers.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.sync_column_schemas" + "unique_id": "macro.dbt.get_unit_test_sql" }, - "macro.dbt.test_accepted_values": { + "macro.dbt.get_where_subquery": { "arguments": [], - "created_at": 1696458269.854999, + "created_at": 1719485736.3740978, "depends_on": { "macros": [ - "macro.dbt.default__test_accepted_values" + "macro.dbt.default__get_where_subquery" ] }, "description": "", @@ -6380,25 +6612,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", + "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "meta": {}, - "name": "test_accepted_values", - "original_file_path": "tests/generic/builtin.sql", + "name": "get_where_subquery", + "original_file_path": "macros/materializations/tests/where_subquery.sql", "package_name": "dbt", "patch_path": null, - "path": "tests/generic/builtin.sql", + "path": "macros/materializations/tests/where_subquery.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.test_accepted_values" + "unique_id": "macro.dbt.get_where_subquery" }, - "macro.dbt.test_not_null": { + "macro.dbt.handle_existing_table": { "arguments": [], - "created_at": 1696458269.854527, + "created_at": 1719485736.513942, "depends_on": { "macros": [ - "macro.dbt.default__test_not_null" + "macro.dbt.default__handle_existing_table" ] }, "description": "", @@ -6406,25 +6636,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", + "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "meta": {}, - "name": "test_not_null", - "original_file_path": "tests/generic/builtin.sql", + "name": "handle_existing_table", + "original_file_path": "macros/relations/view/replace.sql", "package_name": "dbt", "patch_path": null, - "path": "tests/generic/builtin.sql", + "path": "macros/relations/view/replace.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.test_not_null" + "unique_id": "macro.dbt.handle_existing_table" }, - "macro.dbt.test_relationships": { + "macro.dbt.hash": { "arguments": [], - "created_at": 1696458269.855469, + "created_at": 1719485736.5412428, "depends_on": { "macros": [ - "macro.dbt.default__test_relationships" + "macro.dbt.default__hash" ] }, "description": "", @@ -6432,25 +6660,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", + "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", "meta": {}, - "name": "test_relationships", - "original_file_path": "tests/generic/builtin.sql", + "name": "hash", + "original_file_path": "macros/utils/hash.sql", "package_name": "dbt", "patch_path": null, - "path": "tests/generic/builtin.sql", + "path": "macros/utils/hash.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.test_relationships" + "unique_id": "macro.dbt.hash" }, - "macro.dbt.test_unique": { + "macro.dbt.in_transaction": { "arguments": [], - "created_at": 1696458269.854142, + "created_at": 1719485736.336884, "depends_on": { "macros": [ - "macro.dbt.default__test_unique" + "macro.dbt.make_hook_config" ] }, "description": "", @@ -6458,25 +6684,45 @@ "node_color": null, "show": true }, - "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", + "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "meta": {}, - "name": "test_unique", - "original_file_path": "tests/generic/builtin.sql", + "name": "in_transaction", + "original_file_path": "macros/materializations/hooks.sql", "package_name": "dbt", "patch_path": null, - "path": "tests/generic/builtin.sql", + "path": "macros/materializations/hooks.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.test_unique" + "unique_id": "macro.dbt.in_transaction" }, - "macro.dbt.truncate_relation": { + "macro.dbt.incremental_validate_on_schema_change": { + "arguments": [], + "created_at": 1719485736.437581, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", + "meta": {}, + "name": "incremental_validate_on_schema_change", + "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/on_schema_change.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.incremental_validate_on_schema_change" + }, + "macro.dbt.information_schema_name": { "arguments": [], - "created_at": 1696458269.8150861, + "created_at": 1719485736.590652, "depends_on": { "macros": [ - "macro.dbt.default__truncate_relation" + "macro.dbt_postgres.postgres__information_schema_name" ] }, "description": "", @@ -6484,25 +6730,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", + "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "meta": {}, - "name": "truncate_relation", - "original_file_path": "macros/adapters/relation.sql", + "name": "information_schema_name", + "original_file_path": "macros/adapters/metadata.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/adapters/relation.sql", + "path": "macros/adapters/metadata.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.truncate_relation" + "unique_id": "macro.dbt.information_schema_name" }, - "macro.dbt.type_bigint": { + "macro.dbt.intersect": { "arguments": [], - "created_at": 1696458269.7903318, + "created_at": 1719485736.537264, "depends_on": { "macros": [ - "macro.dbt.default__type_bigint" + "macro.dbt.default__intersect" ] }, "description": "", @@ -6510,25 +6754,23 @@ "node_color": null, "show": true }, - "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", "meta": {}, - "name": "type_bigint", - "original_file_path": "macros/utils/data_types.sql", + "name": "intersect", + "original_file_path": "macros/utils/intersect.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/data_types.sql", + "path": "macros/utils/intersect.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.type_bigint" + "unique_id": "macro.dbt.intersect" }, - "macro.dbt.type_boolean": { + "macro.dbt.is_incremental": { "arguments": [], - "created_at": 1696458269.791368, + "created_at": 1719485736.417088, "depends_on": { "macros": [ - "macro.dbt.default__type_boolean" + "macro.dbt.should_full_refresh" ] }, "description": "", @@ -6536,25 +6778,23 @@ "node_color": null, "show": true }, - "macro_sql": "\n\n{%- macro type_boolean() -%}\n {{ return(adapter.dispatch('type_boolean', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "meta": {}, - "name": "type_boolean", - "original_file_path": "macros/utils/data_types.sql", + "name": "is_incremental", + "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/data_types.sql", + "path": "macros/materializations/models/incremental/is_incremental.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.type_boolean" + "unique_id": "macro.dbt.is_incremental" }, - "macro.dbt.type_float": { + "macro.dbt.last_day": { "arguments": [], - "created_at": 1696458269.789262, + "created_at": 1719485736.553028, "depends_on": { "macros": [ - "macro.dbt.default__type_float" + "macro.dbt_postgres.postgres__last_day" ] }, "description": "", @@ -6562,25 +6802,23 @@ "node_color": null, "show": true }, - "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", "meta": {}, - "name": "type_float", - "original_file_path": "macros/utils/data_types.sql", + "name": "last_day", + "original_file_path": "macros/utils/last_day.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/data_types.sql", + "path": "macros/utils/last_day.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.type_float" + "unique_id": "macro.dbt.last_day" }, - "macro.dbt.type_int": { + "macro.dbt.length": { "arguments": [], - "created_at": 1696458269.790855, + "created_at": 1719485736.536302, "depends_on": { "macros": [ - "macro.dbt.default__type_int" + "macro.dbt.default__length" ] }, "description": "", @@ -6588,25 +6826,2248 @@ "node_color": null, "show": true }, - "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", "meta": {}, - "name": "type_int", - "original_file_path": "macros/utils/data_types.sql", + "name": "length", + "original_file_path": "macros/utils/length.sql", "package_name": "dbt", "patch_path": null, - "path": "macros/utils/data_types.sql", + "path": "macros/utils/length.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.type_int" + "unique_id": "macro.dbt.length" + }, + "macro.dbt.list_relations_without_caching": { + "arguments": [], + "created_at": 1719485736.592111, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__list_relations_without_caching" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "list_relations_without_caching", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.list_relations_without_caching" + }, + "macro.dbt.list_schemas": { + "arguments": [], + "created_at": 1719485736.591011, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__list_schemas" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", + "meta": {}, + "name": "list_schemas", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.list_schemas" + }, + "macro.dbt.listagg": { + "arguments": [], + "created_at": 1719485736.5392041, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__listagg" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", + "meta": {}, + "name": "listagg", + "original_file_path": "macros/utils/listagg.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/listagg.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.listagg" + }, + "macro.dbt.load_cached_relation": { + "arguments": [], + "created_at": 1719485736.568574, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", + "meta": {}, + "name": "load_cached_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.load_cached_relation" + }, + "macro.dbt.load_csv_rows": { + "arguments": [], + "created_at": 1719485736.4631271, + "depends_on": { + "macros": [ + "macro.dbt.default__load_csv_rows" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", + "meta": {}, + "name": "load_csv_rows", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.load_csv_rows" + }, + "macro.dbt.load_relation": { + "arguments": [], + "created_at": 1719485736.568728, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "load_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.load_relation" + }, + "macro.dbt.make_backup_relation": { + "arguments": [], + "created_at": 1719485736.566729, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__make_backup_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", + "meta": {}, + "name": "make_backup_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.make_backup_relation" + }, + "macro.dbt.make_hook_config": { + "arguments": [], + "created_at": 1719485736.336577, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", + "meta": {}, + "name": "make_hook_config", + "original_file_path": "macros/materializations/hooks.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/hooks.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.make_hook_config" + }, + "macro.dbt.make_intermediate_relation": { + "arguments": [], + "created_at": 1719485736.565727, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__make_intermediate_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", + "meta": {}, + "name": "make_intermediate_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.make_intermediate_relation" + }, + "macro.dbt.make_temp_relation": { + "arguments": [], + "created_at": 1719485736.5661461, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__make_temp_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", + "meta": {}, + "name": "make_temp_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.make_temp_relation" + }, + "macro.dbt.materialization_clone_default": { + "arguments": [], + "created_at": 1719485736.448061, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation", + "macro.dbt.can_clone_table", + "macro.dbt.drop_relation_if_exists", + "macro.dbt.statement", + "macro.dbt.create_or_replace_clone", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- materialization clone, default -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {% if target_relation and defer_relation and target_relation == defer_relation %}\n {{ log(\"Target relation and defer relation are the same, skipping clone for relation: \" ~ target_relation) }}\n {% else %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endif %}\n\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n\n {%- endif -%}\n\n{%- endmaterialization -%}", + "meta": {}, + "name": "materialization_clone_default", + "original_file_path": "macros/materializations/models/clone/clone.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/clone/clone.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_clone_default" + }, + "macro.dbt.materialization_incremental_default": { + "arguments": [], + "created_at": 1719485736.4306219, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation", + "macro.dbt.make_temp_relation", + "macro.dbt.make_intermediate_relation", + "macro.dbt.make_backup_relation", + "macro.dbt.should_full_refresh", + "macro.dbt.incremental_validate_on_schema_change", + "macro.dbt.drop_relation_if_exists", + "macro.dbt.run_hooks", + "macro.dbt.get_create_table_as_sql", + "macro.dbt.run_query", + "macro.dbt.process_schema_changes", + "macro.dbt.statement", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs", + "macro.dbt.create_indexes" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", + "meta": {}, + "name": "materialization_incremental_default", + "original_file_path": "macros/materializations/models/incremental/incremental.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/incremental.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_incremental_default" + }, + "macro.dbt.materialization_materialized_view_default": { + "arguments": [], + "created_at": 1719485736.383056, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation", + "macro.dbt.make_intermediate_relation", + "macro.dbt.make_backup_relation", + "macro.dbt.materialized_view_setup", + "macro.dbt.materialized_view_get_build_sql", + "macro.dbt.materialized_view_execute_no_op", + "macro.dbt.materialized_view_execute_build_sql", + "macro.dbt.materialized_view_teardown" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization materialized_view, default %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n {% set intermediate_relation = make_intermediate_relation(target_relation) %}\n {% set backup_relation_type = target_relation.MaterializedView if existing_relation is none else existing_relation.type %}\n {% set backup_relation = make_backup_relation(target_relation, backup_relation_type) %}\n\n {{ materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% if build_sql == '' %}\n {{ materialized_view_execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", + "meta": {}, + "name": "materialization_materialized_view_default", + "original_file_path": "macros/materializations/models/materialized_view.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/materialized_view.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_materialized_view_default" + }, + "macro.dbt.materialization_seed_default": { + "arguments": [], + "created_at": 1719485736.4520829, + "depends_on": { + "macros": [ + "macro.dbt.should_full_refresh", + "macro.dbt.run_hooks", + "macro.dbt.reset_csv_table", + "macro.dbt.create_csv_table", + "macro.dbt.load_csv_rows", + "macro.dbt.noop_statement", + "macro.dbt.get_csv_sql", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs", + "macro.dbt.create_indexes" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparison later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", + "meta": {}, + "name": "materialization_seed_default", + "original_file_path": "macros/materializations/seeds/seed.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/seed.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_seed_default" + }, + "macro.dbt.materialization_snapshot_default": { + "arguments": [], + "created_at": 1719485736.3679821, + "depends_on": { + "macros": [ + "macro.dbt.get_or_create_relation", + "macro.dbt.run_hooks", + "macro.dbt.strategy_dispatch", + "macro.dbt.build_snapshot_table", + "macro.dbt.create_table_as", + "macro.dbt.build_snapshot_staging_table", + "macro.dbt.create_columns", + "macro.dbt.snapshot_merge_sql", + "macro.dbt.statement", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs", + "macro.dbt.create_indexes", + "macro.dbt.post_snapshot" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparision later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", + "meta": {}, + "name": "materialization_snapshot_default", + "original_file_path": "macros/materializations/snapshots/snapshot.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/snapshot.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_snapshot_default" + }, + "macro.dbt.materialization_table_default": { + "arguments": [], + "created_at": 1719485736.3933172, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation", + "macro.dbt.make_intermediate_relation", + "macro.dbt.make_backup_relation", + "macro.dbt.drop_relation_if_exists", + "macro.dbt.run_hooks", + "macro.dbt.statement", + "macro.dbt.get_create_table_as_sql", + "macro.dbt.create_indexes", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", + "meta": {}, + "name": "materialization_table_default", + "original_file_path": "macros/materializations/models/table.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_table_default" + }, + "macro.dbt.materialization_test_default": { + "arguments": [], + "created_at": 1719485736.370986, + "depends_on": { + "macros": [ + "macro.dbt.should_store_failures", + "macro.dbt.statement", + "macro.dbt.get_create_sql", + "macro.dbt.get_test_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% set store_failures_as = config.get('store_failures_as') %}\n -- if `--store-failures` is invoked via command line and `store_failures_as` is not set,\n -- config.get('store_failures_as', 'table') returns None, not 'table'\n {% if store_failures_as == none %}{% set store_failures_as = 'table' %}{% endif %}\n {% if store_failures_as not in ['table', 'view'] %}\n {{ exceptions.raise_compiler_error(\n \"'\" ~ store_failures_as ~ \"' is not a valid value for `store_failures_as`. \"\n \"Accepted values are: ['ephemeral', 'table', 'view']\"\n ) }}\n {% endif %}\n\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type=store_failures_as) -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ get_create_sql(target_relation, sql) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql %}\n\n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", + "meta": {}, + "name": "materialization_test_default", + "original_file_path": "macros/materializations/tests/test.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/tests/test.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_test_default" + }, + "macro.dbt.materialization_unit_default": { + "arguments": [], + "created_at": 1719485736.3763871, + "depends_on": { + "macros": [ + "macro.dbt.get_columns_in_query", + "macro.dbt.make_temp_relation", + "macro.dbt.run_query", + "macro.dbt.get_create_table_as_sql", + "macro.dbt.get_empty_subquery_sql", + "macro.dbt.get_expected_sql", + "macro.dbt.get_unit_test_sql", + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- materialization unit, default -%}\n\n {% set relations = [] %}\n\n {% set expected_rows = config.get('expected_rows') %}\n {% set expected_sql = config.get('expected_sql') %}\n {% set tested_expected_column_names = expected_rows[0].keys() if (expected_rows | length ) > 0 else get_columns_in_query(sql) %} %}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {% do run_query(get_create_table_as_sql(True, temp_relation, get_empty_subquery_sql(sql))) %}\n {%- set columns_in_relation = adapter.get_columns_in_relation(temp_relation) -%}\n {%- set column_name_to_data_types = {} -%}\n {%- for column in columns_in_relation -%}\n {%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n {%- endfor -%}\n\n {% if not expected_sql %}\n {% set expected_sql = get_expected_sql(expected_rows, column_name_to_data_types) %}\n {% endif %}\n {% set unit_test_sql = get_unit_test_sql(sql, expected_sql, tested_expected_column_names) %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ unit_test_sql }}\n\n {%- endcall %}\n\n {% do adapter.drop_relation(temp_relation) %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", + "meta": {}, + "name": "materialization_unit_default", + "original_file_path": "macros/materializations/tests/unit.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/tests/unit.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_unit_default" + }, + "macro.dbt.materialization_view_default": { + "arguments": [], + "created_at": 1719485736.390083, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation", + "macro.dbt.make_intermediate_relation", + "macro.dbt.make_backup_relation", + "macro.dbt.run_hooks", + "macro.dbt.drop_relation_if_exists", + "macro.dbt.statement", + "macro.dbt.get_create_view_as_sql", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", + "meta": {}, + "name": "materialization_view_default", + "original_file_path": "macros/materializations/models/view.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/view.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_view_default" + }, + "macro.dbt.materialized_view_execute_build_sql": { + "arguments": [], + "created_at": 1719485736.386875, + "depends_on": { + "macros": [ + "macro.dbt.run_hooks", + "macro.dbt.statement", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n{% endmacro %}", + "meta": {}, + "name": "materialized_view_execute_build_sql", + "original_file_path": "macros/materializations/models/materialized_view.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/materialized_view.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.materialized_view_execute_build_sql" + }, + "macro.dbt.materialized_view_execute_no_op": { + "arguments": [], + "created_at": 1719485736.385866, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro materialized_view_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", + "meta": {}, + "name": "materialized_view_execute_no_op", + "original_file_path": "macros/materializations/models/materialized_view.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/materialized_view.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.materialized_view_execute_no_op" + }, + "macro.dbt.materialized_view_get_build_sql": { + "arguments": [], + "created_at": 1719485736.385609, + "depends_on": { + "macros": [ + "macro.dbt.should_full_refresh", + "macro.dbt.get_create_materialized_view_as_sql", + "macro.dbt.get_replace_sql", + "macro.dbt.get_materialized_view_configuration_changes", + "macro.dbt.refresh_materialized_view", + "macro.dbt.get_alter_materialized_view_as_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = '' %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", + "meta": {}, + "name": "materialized_view_get_build_sql", + "original_file_path": "macros/materializations/models/materialized_view.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/materialized_view.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.materialized_view_get_build_sql" + }, + "macro.dbt.materialized_view_setup": { + "arguments": [], + "created_at": 1719485736.3838332, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation", + "macro.dbt.drop_relation_if_exists", + "macro.dbt.run_hooks" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) %}\n\n -- backup_relation and intermediate_relation should not already exist in the database\n -- it's possible these exist because of a previous run that exited unexpectedly\n {% set preexisting_backup_relation = load_cached_relation(backup_relation) %}\n {% set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% endmacro %}", + "meta": {}, + "name": "materialized_view_setup", + "original_file_path": "macros/materializations/models/materialized_view.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/materialized_view.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.materialized_view_setup" + }, + "macro.dbt.materialized_view_teardown": { + "arguments": [], + "created_at": 1719485736.384107, + "depends_on": { + "macros": [ + "macro.dbt.drop_relation_if_exists", + "macro.dbt.run_hooks" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) %}\n\n -- drop the temp relations if they exist to leave the database clean for the next run\n {{ drop_relation_if_exists(backup_relation) }}\n {{ drop_relation_if_exists(intermediate_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n{% endmacro %}", + "meta": {}, + "name": "materialized_view_teardown", + "original_file_path": "macros/materializations/models/materialized_view.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/materialized_view.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.materialized_view_teardown" + }, + "macro.dbt.noop_statement": { + "arguments": [], + "created_at": 1719485736.5199249, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "noop_statement", + "original_file_path": "macros/etc/statement.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/statement.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.noop_statement" + }, + "macro.dbt.partition_range": { + "arguments": [], + "created_at": 1719485736.5252612, + "depends_on": { + "macros": [ + "macro.dbt.dates_in_range" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", + "meta": {}, + "name": "partition_range", + "original_file_path": "macros/etc/datetime.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/datetime.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.partition_range" + }, + "macro.dbt.persist_docs": { + "arguments": [], + "created_at": 1719485736.585922, + "depends_on": { + "macros": [ + "macro.dbt.default__persist_docs" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", + "meta": {}, + "name": "persist_docs", + "original_file_path": "macros/adapters/persist_docs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/persist_docs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.persist_docs" + }, + "macro.dbt.position": { + "arguments": [], + "created_at": 1719485736.543356, + "depends_on": { + "macros": [ + "macro.dbt.default__position" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", + "meta": {}, + "name": "position", + "original_file_path": "macros/utils/position.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/position.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.position" + }, + "macro.dbt.post_snapshot": { + "arguments": [], + "created_at": 1719485736.355165, + "depends_on": { + "macros": [ + "macro.dbt.default__post_snapshot" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "post_snapshot", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.post_snapshot" + }, + "macro.dbt.process_schema_changes": { + "arguments": [], + "created_at": 1719485736.44261, + "depends_on": { + "macros": [ + "macro.dbt.check_for_schema_changes", + "macro.dbt.sync_column_schemas" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n\n Additional troubleshooting context:\n Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}\n Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}\n New column types: {{ schema_changes_dict['new_target_types'] }}\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", + "meta": {}, + "name": "process_schema_changes", + "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/on_schema_change.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.process_schema_changes" + }, + "macro.dbt.py_current_timestring": { + "arguments": [], + "created_at": 1719485736.5255191, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", + "meta": {}, + "name": "py_current_timestring", + "original_file_path": "macros/etc/datetime.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/datetime.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.py_current_timestring" + }, + "macro.dbt.py_script_comment": { + "arguments": [], + "created_at": 1719485736.6181471, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%macro py_script_comment()%}\n{%endmacro%}", + "meta": {}, + "name": "py_script_comment", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.py_script_comment" + }, + "macro.dbt.py_script_postfix": { + "arguments": [], + "created_at": 1719485736.618067, + "depends_on": { + "macros": [ + "macro.dbt.build_ref_function", + "macro.dbt.build_source_function", + "macro.dbt.build_config_dict", + "macro.dbt.resolve_model_name", + "macro.dbt.is_incremental", + "macro.dbt.py_script_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro py_script_postfix(model) %}\n# This part is user provided model code\n# you will need to copy the next section to run the code\n# COMMAND ----------\n# this part is dbt logic for get ref work, do not modify\n\n{{ build_ref_function(model ) }}\n{{ build_source_function(model ) }}\n{{ build_config_dict(model) }}\n\nclass config:\n def __init__(self, *args, **kwargs):\n pass\n\n @staticmethod\n def get(key, default=None):\n return config_dict.get(key, default)\n\nclass this:\n \"\"\"dbt.this() or dbt.this.identifier\"\"\"\n database = \"{{ this.database }}\"\n schema = \"{{ this.schema }}\"\n identifier = \"{{ this.identifier }}\"\n {% set this_relation_name = resolve_model_name(this) %}\n def __repr__(self):\n return '{{ this_relation_name }}'\n\n\nclass dbtObj:\n def __init__(self, load_df_function) -> None:\n self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function)\n self.ref = lambda *args, **kwargs: ref(*args, **kwargs, dbt_load_df_function=load_df_function)\n self.config = config\n self.this = this()\n self.is_incremental = {{ is_incremental() }}\n\n# COMMAND ----------\n{{py_script_comment()}}\n{% endmacro %}", + "meta": {}, + "name": "py_script_postfix", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.py_script_postfix" + }, + "macro.dbt.refresh_materialized_view": { + "arguments": [], + "created_at": 1719485736.489407, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__refresh_materialized_view" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro refresh_materialized_view(relation) %}\n {{- log('Applying REFRESH to: ' ~ relation) -}}\n {{- adapter.dispatch('refresh_materialized_view', 'dbt')(relation) -}}\n{% endmacro %}", + "meta": {}, + "name": "refresh_materialized_view", + "original_file_path": "macros/relations/materialized_view/refresh.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/materialized_view/refresh.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.refresh_materialized_view" + }, + "macro.dbt.rename_relation": { + "arguments": [], + "created_at": 1719485736.478327, + "depends_on": { + "macros": [ + "macro.dbt.default__rename_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "rename_relation", + "original_file_path": "macros/relations/rename.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/rename.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.rename_relation" + }, + "macro.dbt.replace": { + "arguments": [], + "created_at": 1719485736.532748, + "depends_on": { + "macros": [ + "macro.dbt.default__replace" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", + "meta": {}, + "name": "replace", + "original_file_path": "macros/utils/replace.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.replace" + }, + "macro.dbt.reset_csv_table": { + "arguments": [], + "created_at": 1719485736.460277, + "depends_on": { + "macros": [ + "macro.dbt.default__reset_csv_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", + "meta": {}, + "name": "reset_csv_table", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.reset_csv_table" + }, + "macro.dbt.resolve_model_name": { + "arguments": [], + "created_at": 1719485736.614536, + "depends_on": { + "macros": [ + "macro.dbt.default__resolve_model_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro resolve_model_name(input_model_name) %}\n {{ return(adapter.dispatch('resolve_model_name', 'dbt')(input_model_name)) }}\n{% endmacro %}", + "meta": {}, + "name": "resolve_model_name", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.resolve_model_name" + }, + "macro.dbt.right": { + "arguments": [], + "created_at": 1719485736.538353, + "depends_on": { + "macros": [ + "macro.dbt.default__right" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", + "meta": {}, + "name": "right", + "original_file_path": "macros/utils/right.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/right.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.right" + }, + "macro.dbt.run_hooks": { + "arguments": [], + "created_at": 1719485736.336357, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", + "meta": {}, + "name": "run_hooks", + "original_file_path": "macros/materializations/hooks.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/hooks.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.run_hooks" + }, + "macro.dbt.run_query": { + "arguments": [], + "created_at": 1719485736.5202482, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", + "meta": {}, + "name": "run_query", + "original_file_path": "macros/etc/statement.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/statement.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.run_query" + }, + "macro.dbt.safe_cast": { + "arguments": [], + "created_at": 1719485736.540742, + "depends_on": { + "macros": [ + "macro.dbt.default__safe_cast" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", + "meta": {}, + "name": "safe_cast", + "original_file_path": "macros/utils/safe_cast.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/safe_cast.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.safe_cast" + }, + "macro.dbt.set_sql_header": { + "arguments": [], + "created_at": 1719485736.338798, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", + "meta": {}, + "name": "set_sql_header", + "original_file_path": "macros/materializations/configs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/configs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.set_sql_header" + }, + "macro.dbt.should_full_refresh": { + "arguments": [], + "created_at": 1719485736.339242, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", + "meta": {}, + "name": "should_full_refresh", + "original_file_path": "macros/materializations/configs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/configs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.should_full_refresh" + }, + "macro.dbt.should_revoke": { + "arguments": [], + "created_at": 1719485736.573465, + "depends_on": { + "macros": [ + "macro.dbt.copy_grants" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", + "meta": {}, + "name": "should_revoke", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.should_revoke" + }, + "macro.dbt.should_store_failures": { + "arguments": [], + "created_at": 1719485736.339565, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", + "meta": {}, + "name": "should_store_failures", + "original_file_path": "macros/materializations/configs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/configs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.should_store_failures" + }, + "macro.dbt.snapshot_check_all_get_existing_columns": { + "arguments": [], + "created_at": 1719485736.3482602, + "depends_on": { + "macros": [ + "macro.dbt.get_columns_in_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n {#-- N.B. The whitespace below is necessary to avoid edge case issue with comments --#}\n {#-- See: https://github.com/dbt-labs/dbt-core/issues/6781 --#}\n select {{ check_cols_config | join(', ') }} from (\n {{ node['compiled_code'] }}\n ) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", + "meta": {}, + "name": "snapshot_check_all_get_existing_columns", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns" + }, + "macro.dbt.snapshot_check_strategy": { + "arguments": [], + "created_at": 1719485736.3500328, + "depends_on": { + "macros": [ + "macro.dbt.snapshot_get_time", + "macro.dbt.snapshot_check_all_get_existing_columns", + "macro.dbt.get_true_sql", + "macro.dbt.snapshot_hash_arguments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n {% set updated_at = config.get('updated_at', snapshot_get_time()) %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", + "meta": {}, + "name": "snapshot_check_strategy", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_check_strategy" + }, + "macro.dbt.snapshot_get_time": { + "arguments": [], + "created_at": 1719485736.55953, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__snapshot_get_time" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "snapshot_get_time", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_get_time" + }, + "macro.dbt.snapshot_hash_arguments": { + "arguments": [], + "created_at": 1719485736.3454158, + "depends_on": { + "macros": [ + "macro.dbt.default__snapshot_hash_arguments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", + "meta": {}, + "name": "snapshot_hash_arguments", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_hash_arguments" + }, + "macro.dbt.snapshot_merge_sql": { + "arguments": [], + "created_at": 1719485736.3400362, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__snapshot_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", + "meta": {}, + "name": "snapshot_merge_sql", + "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/snapshot_merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_merge_sql" + }, + "macro.dbt.snapshot_staging_table": { + "arguments": [], + "created_at": 1719485736.355765, + "depends_on": { + "macros": [ + "macro.dbt.default__snapshot_staging_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "snapshot_staging_table", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_staging_table" + }, + "macro.dbt.snapshot_string_as_time": { + "arguments": [], + "created_at": 1719485736.3465831, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__snapshot_string_as_time" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", + "meta": {}, + "name": "snapshot_string_as_time", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_string_as_time" + }, + "macro.dbt.snapshot_timestamp_strategy": { + "arguments": [], + "created_at": 1719485736.346405, + "depends_on": { + "macros": [ + "macro.dbt.snapshot_hash_arguments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", + "meta": {}, + "name": "snapshot_timestamp_strategy", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_timestamp_strategy" + }, + "macro.dbt.split_part": { + "arguments": [], + "created_at": 1719485736.55408, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__split_part" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", + "meta": {}, + "name": "split_part", + "original_file_path": "macros/utils/split_part.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/split_part.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.split_part" + }, + "macro.dbt.sql_convert_columns_in_relation": { + "arguments": [], + "created_at": 1719485736.59661, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", + "meta": {}, + "name": "sql_convert_columns_in_relation", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.sql_convert_columns_in_relation" + }, + "macro.dbt.statement": { + "arguments": [], + "created_at": 1719485736.51912, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%}\n {%- if execute: -%}\n {%- set compiled_code = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime {} for node \"{}\"'.format(language, model['unique_id'])) }}\n {{ write(compiled_code) }}\n {%- endif -%}\n {%- if language == 'sql'-%}\n {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- elif language == 'python' -%}\n {%- set res = submit_python_job(model, compiled_code) -%}\n {#-- TODO: What should table be for python models? --#}\n {%- set table = None -%}\n {%- else -%}\n {% do exceptions.raise_compiler_error(\"statement macro didn't get supported language\") %}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", + "meta": {}, + "name": "statement", + "original_file_path": "macros/etc/statement.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/statement.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.statement" + }, + "macro.dbt.strategy_dispatch": { + "arguments": [], + "created_at": 1719485736.345131, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", + "meta": {}, + "name": "strategy_dispatch", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.strategy_dispatch" + }, + "macro.dbt.string_literal": { + "arguments": [], + "created_at": 1719485736.543961, + "depends_on": { + "macros": [ + "macro.dbt.default__string_literal" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "string_literal", + "original_file_path": "macros/utils/literal.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/literal.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.string_literal" + }, + "macro.dbt.support_multiple_grantees_per_dcl_statement": { + "arguments": [], + "created_at": 1719485736.572978, + "depends_on": { + "macros": [ + "macro.dbt.default__support_multiple_grantees_per_dcl_statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "support_multiple_grantees_per_dcl_statement", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement" + }, + "macro.dbt.sync_column_schemas": { + "arguments": [], + "created_at": 1719485736.4416761, + "depends_on": { + "macros": [ + "macro.dbt.alter_relation_add_remove_columns", + "macro.dbt.alter_column_type" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", + "meta": {}, + "name": "sync_column_schemas", + "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/on_schema_change.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.sync_column_schemas" + }, + "macro.dbt.table_columns_and_constraints": { + "arguments": [], + "created_at": 1719485736.500673, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro table_columns_and_constraints() %}\n {# loop through user_provided_columns to create DDL with data types and constraints #}\n {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%}\n {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%}\n (\n {% for c in raw_column_constraints -%}\n {{ c }}{{ \",\" if not loop.last or raw_model_constraints }}\n {% endfor %}\n {% for c in raw_model_constraints -%}\n {{ c }}{{ \",\" if not loop.last }}\n {% endfor -%}\n )\n{% endmacro %}", + "meta": {}, + "name": "table_columns_and_constraints", + "original_file_path": "macros/relations/column/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/column/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.table_columns_and_constraints" + }, + "macro.dbt.test_accepted_values": { + "arguments": [], + "created_at": 1719485736.6202009, + "depends_on": { + "macros": [ + "macro.dbt.default__test_accepted_values" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", + "meta": {}, + "name": "test_accepted_values", + "original_file_path": "tests/generic/builtin.sql", + "package_name": "dbt", + "patch_path": null, + "path": "tests/generic/builtin.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.test_accepted_values" + }, + "macro.dbt.test_not_null": { + "arguments": [], + "created_at": 1719485736.619859, + "depends_on": { + "macros": [ + "macro.dbt.default__test_not_null" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", + "meta": {}, + "name": "test_not_null", + "original_file_path": "tests/generic/builtin.sql", + "package_name": "dbt", + "patch_path": null, + "path": "tests/generic/builtin.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.test_not_null" + }, + "macro.dbt.test_relationships": { + "arguments": [], + "created_at": 1719485736.620512, + "depends_on": { + "macros": [ + "macro.dbt.default__test_relationships" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", + "meta": {}, + "name": "test_relationships", + "original_file_path": "tests/generic/builtin.sql", + "package_name": "dbt", + "patch_path": null, + "path": "tests/generic/builtin.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.test_relationships" + }, + "macro.dbt.test_unique": { + "arguments": [], + "created_at": 1719485736.619491, + "depends_on": { + "macros": [ + "macro.dbt.default__test_unique" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", + "meta": {}, + "name": "test_unique", + "original_file_path": "tests/generic/builtin.sql", + "package_name": "dbt", + "patch_path": null, + "path": "tests/generic/builtin.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.test_unique" + }, + "macro.dbt.truncate_relation": { + "arguments": [], + "created_at": 1719485736.567298, + "depends_on": { + "macros": [ + "macro.dbt.default__truncate_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "truncate_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.truncate_relation" + }, + "macro.dbt.type_bigint": { + "arguments": [], + "created_at": 1719485736.548424, + "depends_on": { + "macros": [ + "macro.dbt.default__type_bigint" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_bigint", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_bigint" + }, + "macro.dbt.type_boolean": { + "arguments": [], + "created_at": 1719485736.55002, + "depends_on": { + "macros": [ + "macro.dbt.default__type_boolean" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_boolean() -%}\n {{ return(adapter.dispatch('type_boolean', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_boolean", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_boolean" + }, + "macro.dbt.type_float": { + "arguments": [], + "created_at": 1719485736.547037, + "depends_on": { + "macros": [ + "macro.dbt.default__type_float" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_float", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_float" + }, + "macro.dbt.type_int": { + "arguments": [], + "created_at": 1719485736.54878, + "depends_on": { + "macros": [ + "macro.dbt.default__type_int" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_int", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_int" + }, + "macro.dbt.type_numeric": { + "arguments": [], + "created_at": 1719485736.547442, + "depends_on": { + "macros": [ + "macro.dbt.default__type_numeric" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_numeric", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_numeric" + }, + "macro.dbt.type_string": { + "arguments": [], + "created_at": 1719485736.545998, + "depends_on": { + "macros": [ + "macro.dbt.default__type_string" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_string", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_string" + }, + "macro.dbt.type_timestamp": { + "arguments": [], + "created_at": 1719485736.546634, + "depends_on": { + "macros": [ + "macro.dbt.default__type_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_timestamp", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_timestamp" + }, + "macro.dbt.validate_sql": { + "arguments": [], + "created_at": 1719485736.5702772, + "depends_on": { + "macros": [ + "macro.dbt.default__validate_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro validate_sql(sql) -%}\n {{ return(adapter.dispatch('validate_sql', 'dbt')(sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "validate_sql", + "original_file_path": "macros/adapters/validate_sql.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/validate_sql.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.validate_sql" + }, + "macro.dbt_postgres.postgres__alter_column_comment": { + "arguments": [], + "created_at": 1719485736.314291, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres_escape_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% for column_name in column_dict if (column_name in existing_columns) %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", + "meta": {}, + "name": "postgres__alter_column_comment", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__alter_column_comment" + }, + "macro.dbt_postgres.postgres__alter_relation_comment": { + "arguments": [], + "created_at": 1719485736.312385, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres_escape_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", + "meta": {}, + "name": "postgres__alter_relation_comment", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__alter_relation_comment" + }, + "macro.dbt_postgres.postgres__any_value": { + "arguments": [], + "created_at": 1719485736.332639, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__any_value(expression) -%}\n\n min({{ expression }})\n\n{%- endmacro %}", + "meta": {}, + "name": "postgres__any_value", + "original_file_path": "macros/utils/any_value.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/utils/any_value.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__any_value" + }, + "macro.dbt_postgres.postgres__check_schema_exists": { + "arguments": [], + "created_at": 1719485736.309761, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__check_schema_exists(information_schema, schema) -%}\n {% if information_schema.database -%}\n {{ adapter.verify_database(information_schema.database) }}\n {%- endif -%}\n {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}\n select count(*) from pg_namespace where nspname = '{{ schema }}'\n {% endcall %}\n {{ return(load_result('check_schema_exists').table) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__check_schema_exists", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__check_schema_exists" + }, + "macro.dbt_postgres.postgres__copy_grants": { + "arguments": [], + "created_at": 1719485736.3146439, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__copy_grants() %}\n {{ return(False) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__copy_grants", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__copy_grants" + }, + "macro.dbt_postgres.postgres__create_schema": { + "arguments": [], + "created_at": 1719485736.306344, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__create_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier().include(database=False) }}\n {%- endcall -%}\n{% endmacro %}", + "meta": {}, + "name": "postgres__create_schema", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__create_schema" + }, + "macro.dbt_postgres.postgres__create_table_as": { + "arguments": [], + "created_at": 1719485736.3053398, + "depends_on": { + "macros": [ + "macro.dbt.get_assert_columns_equivalent", + "macro.dbt.get_table_columns_and_constraints", + "macro.dbt.default__get_column_names", + "macro.dbt.get_select_subquery" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__create_table_as(temporary, relation, sql) -%}\n {%- set unlogged = config.get('unlogged', default=false) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary -%}\n temporary\n {%- elif unlogged -%}\n unlogged\n {%- endif %} table {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {% endif -%}\n {% if contract_config.enforced and (not temporary) -%}\n {{ get_table_columns_and_constraints() }} ;\n insert into {{ relation }} (\n {{ adapter.dispatch('get_column_names', 'dbt')() }}\n )\n {%- set sql = get_select_subquery(sql) %}\n {% else %}\n as\n {% endif %}\n (\n {{ sql }}\n );\n{%- endmacro %}", + "meta": {}, + "name": "postgres__create_table_as", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__create_table_as" + }, + "macro.dbt_postgres.postgres__current_timestamp": { + "arguments": [], + "created_at": 1719485736.2888992, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__current_timestamp() -%}\n now()\n{%- endmacro %}", + "meta": {}, + "name": "postgres__current_timestamp", + "original_file_path": "macros/timestamps.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__current_timestamp" + }, + "macro.dbt_postgres.postgres__current_timestamp_backcompat": { + "arguments": [], + "created_at": 1719485736.289357, + "depends_on": { + "macros": [ + "macro.dbt.type_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__current_timestamp_backcompat() %}\n current_timestamp::{{ type_timestamp() }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__current_timestamp_backcompat", + "original_file_path": "macros/timestamps.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__current_timestamp_backcompat" + }, + "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat": { + "arguments": [], + "created_at": 1719485736.289477, + "depends_on": { + "macros": [ + "macro.dbt.type_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__current_timestamp_in_utc_backcompat() %}\n (current_timestamp at time zone 'utc')::{{ type_timestamp() }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__current_timestamp_in_utc_backcompat", + "original_file_path": "macros/timestamps.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat" + }, + "macro.dbt_postgres.postgres__dateadd": { + "arguments": [], + "created_at": 1719485736.328303, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }}))\n\n{% endmacro %}", + "meta": {}, + "name": "postgres__dateadd", + "original_file_path": "macros/utils/dateadd.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/utils/dateadd.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__dateadd" + }, + "macro.dbt_postgres.postgres__datediff": { + "arguments": [], + "created_at": 1719485736.332466, + "depends_on": { + "macros": [ + "macro.dbt.datediff" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__datediff(first_date, second_date, datepart) -%}\n\n {% if datepart == 'year' %}\n (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date))\n {% elif datepart == 'quarter' %}\n ({{ datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date))\n {% elif datepart == 'month' %}\n ({{ datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date))\n {% elif datepart == 'day' %}\n (({{second_date}})::date - ({{first_date}})::date)\n {% elif datepart == 'week' %}\n ({{ datediff(first_date, second_date, 'day') }} / 7 + case\n when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then\n case when {{first_date}} <= {{second_date}} then 0 else -1 end\n else\n case when {{first_date}} <= {{second_date}} then 1 else 0 end\n end)\n {% elif datepart == 'hour' %}\n ({{ datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp))\n {% elif datepart == 'minute' %}\n ({{ datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp))\n {% elif datepart == 'second' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp)))\n {% elif datepart == 'millisecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp)))\n {% elif datepart == 'microsecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp)))\n {% else %}\n {{ exceptions.raise_compiler_error(\"Unsupported datepart for macro datediff in postgres: {!r}\".format(datepart)) }}\n {% endif %}\n\n{%- endmacro %}", + "meta": {}, + "name": "postgres__datediff", + "original_file_path": "macros/utils/datediff.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/utils/datediff.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__datediff" + }, + "macro.dbt_postgres.postgres__describe_materialized_view": { + "arguments": [], + "created_at": 1719485736.320213, + "depends_on": { + "macros": [ + "macro.dbt.run_query", + "macro.dbt.get_show_indexes_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__describe_materialized_view(relation) %}\n -- for now just get the indexes, we don't need the name or the query yet\n {% set _indexes = run_query(get_show_indexes_sql(relation)) %}\n {% do return({'indexes': _indexes}) %}\n{% endmacro %}", + "meta": {}, + "name": "postgres__describe_materialized_view", + "original_file_path": "macros/relations/materialized_view/describe.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations/materialized_view/describe.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__describe_materialized_view" + }, + "macro.dbt_postgres.postgres__drop_materialized_view": { + "arguments": [], + "created_at": 1719485736.319324, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation }} cascade\n{%- endmacro %}", + "meta": {}, + "name": "postgres__drop_materialized_view", + "original_file_path": "macros/relations/materialized_view/drop.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations/materialized_view/drop.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__drop_materialized_view" }, - "macro.dbt.type_numeric": { + "macro.dbt_postgres.postgres__drop_schema": { "arguments": [], - "created_at": 1696458269.789772, + "created_at": 1719485736.307206, "depends_on": { "macros": [ - "macro.dbt.default__type_numeric" + "macro.dbt.statement" ] }, "description": "", @@ -6614,77 +9075,68 @@ "node_color": null, "show": true }, - "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "macro_sql": "{% macro postgres__drop_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade\n {%- endcall -%}\n{% endmacro %}", "meta": {}, - "name": "type_numeric", - "original_file_path": "macros/utils/data_types.sql", - "package_name": "dbt", + "name": "postgres__drop_schema", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", "patch_path": null, - "path": "macros/utils/data_types.sql", + "path": "macros/adapters.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.type_numeric" + "unique_id": "macro.dbt_postgres.postgres__drop_schema" }, - "macro.dbt.type_string": { + "macro.dbt_postgres.postgres__drop_table": { "arguments": [], - "created_at": 1696458269.788082, + "created_at": 1719485736.3256729, "depends_on": { - "macros": [ - "macro.dbt.default__type_string" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "macro_sql": "{% macro postgres__drop_table(relation) -%}\n drop table if exists {{ relation }} cascade\n{%- endmacro %}", "meta": {}, - "name": "type_string", - "original_file_path": "macros/utils/data_types.sql", - "package_name": "dbt", + "name": "postgres__drop_table", + "original_file_path": "macros/relations/table/drop.sql", + "package_name": "dbt_postgres", "patch_path": null, - "path": "macros/utils/data_types.sql", + "path": "macros/relations/table/drop.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.type_string" + "unique_id": "macro.dbt_postgres.postgres__drop_table" }, - "macro.dbt.type_timestamp": { + "macro.dbt_postgres.postgres__drop_view": { "arguments": [], - "created_at": 1696458269.788734, + "created_at": 1719485736.327039, "depends_on": { - "macros": [ - "macro.dbt.default__type_timestamp" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "macro_sql": "{% macro postgres__drop_view(relation) -%}\n drop view if exists {{ relation }} cascade\n{%- endmacro %}", "meta": {}, - "name": "type_timestamp", - "original_file_path": "macros/utils/data_types.sql", - "package_name": "dbt", + "name": "postgres__drop_view", + "original_file_path": "macros/relations/view/drop.sql", + "package_name": "dbt_postgres", "patch_path": null, - "path": "macros/utils/data_types.sql", + "path": "macros/relations/view/drop.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/global_project", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt.type_timestamp" + "unique_id": "macro.dbt_postgres.postgres__drop_view" }, - "macro.dbt_postgres.postgres__alter_column_comment": { + "macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql": { "arguments": [], - "created_at": 1696458269.575566, + "created_at": 1719485736.3214989, "depends_on": { "macros": [ - "macro.dbt_postgres.postgres_escape_comment" + "macro.dbt.get_replace_sql", + "macro.dbt_postgres.postgres__update_indexes_on_materialized_view" ] }, "description": "", @@ -6692,25 +9144,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% for column_name in column_dict if (column_name in existing_columns) %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", + "macro_sql": "{% macro postgres__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n\n {{ get_replace_sql(existing_relation, relation, sql) }}\n\n -- otherwise apply individual changes as needed\n {% else %}\n\n {{ postgres__update_indexes_on_materialized_view(relation, configuration_changes.indexes) }}\n\n {%- endif -%}\n\n{% endmacro %}", "meta": {}, - "name": "postgres__alter_column_comment", - "original_file_path": "macros/adapters.sql", + "name": "postgres__get_alter_materialized_view_as_sql", + "original_file_path": "macros/relations/materialized_view/alter.sql", "package_name": "dbt_postgres", "patch_path": null, - "path": "macros/adapters.sql", + "path": "macros/relations/materialized_view/alter.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt_postgres.postgres__alter_column_comment" + "unique_id": "macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql" }, - "macro.dbt_postgres.postgres__alter_relation_comment": { + "macro.dbt_postgres.postgres__get_catalog": { "arguments": [], - "created_at": 1696458269.574534, + "created_at": 1719485736.292309, "depends_on": { "macros": [ - "macro.dbt_postgres.postgres_escape_comment" + "macro.dbt_postgres.postgres__get_catalog_relations" ] }, "description": "", @@ -6718,49 +9168,48 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", + "macro_sql": "{% macro postgres__get_catalog(information_schema, schemas) -%}\n {%- set relations = [] -%}\n {%- for schema in schemas -%}\n {%- set dummy = relations.append({'schema': schema}) -%}\n {%- endfor -%}\n {{ return(postgres__get_catalog_relations(information_schema, relations)) }}\n{%- endmacro %}", "meta": {}, - "name": "postgres__alter_relation_comment", - "original_file_path": "macros/adapters.sql", + "name": "postgres__get_catalog", + "original_file_path": "macros/catalog.sql", "package_name": "dbt_postgres", "patch_path": null, - "path": "macros/adapters.sql", + "path": "macros/catalog.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt_postgres.postgres__alter_relation_comment" + "unique_id": "macro.dbt_postgres.postgres__get_catalog" }, - "macro.dbt_postgres.postgres__any_value": { + "macro.dbt_postgres.postgres__get_catalog_relations": { "arguments": [], - "created_at": 1696458269.586868, + "created_at": 1719485736.2918808, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.statement" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__any_value(expression) -%}\n\n min({{ expression }})\n\n{%- endmacro %}", + "macro_sql": "{% macro postgres__get_catalog_relations(information_schema, relations) -%}\n {%- call statement('catalog', fetch_result=True) -%}\n\n {#\n If the user has multiple databases set and the first one is wrong, this will fail.\n But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.\n #}\n {% set database = information_schema.database %}\n {{ adapter.verify_database(database) }}\n\n select\n '{{ database }}' as table_database,\n sch.nspname as table_schema,\n tbl.relname as table_name,\n case tbl.relkind\n when 'v' then 'VIEW'\n when 'm' then 'MATERIALIZED VIEW'\n else 'BASE TABLE'\n end as table_type,\n tbl_desc.description as table_comment,\n col.attname as column_name,\n col.attnum as column_index,\n pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,\n col_desc.description as column_comment,\n pg_get_userbyid(tbl.relowner) as table_owner\n\n from pg_catalog.pg_namespace sch\n join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid\n join pg_catalog.pg_attribute col on col.attrelid = tbl.oid\n left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)\n left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)\n where (\n {%- for relation in relations -%}\n {%- if relation.identifier -%}\n (upper(sch.nspname) = upper('{{ relation.schema }}') and\n upper(tbl.relname) = upper('{{ relation.identifier }}'))\n {%- else-%}\n upper(sch.nspname) = upper('{{ relation.schema }}')\n {%- endif -%}\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session\n and tbl.relpersistence in ('p', 'u') -- [p]ermanent table or [u]nlogged table. Exclude [t]emporary tables\n and tbl.relkind in ('r', 'v', 'f', 'p', 'm') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table, [m]aterialized view. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table\n and col.attnum > 0 -- negative numbers are used for system columns such as oid\n and not col.attisdropped -- column as not been dropped\n\n order by\n sch.nspname,\n tbl.relname,\n col.attnum\n\n {%- endcall -%}\n\n {{ return(load_result('catalog').table) }}\n{%- endmacro %}", "meta": {}, - "name": "postgres__any_value", - "original_file_path": "macros/utils/any_value.sql", + "name": "postgres__get_catalog_relations", + "original_file_path": "macros/catalog.sql", "package_name": "dbt_postgres", "patch_path": null, - "path": "macros/utils/any_value.sql", + "path": "macros/catalog.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt_postgres.postgres__any_value" + "unique_id": "macro.dbt_postgres.postgres__get_catalog_relations" }, - "macro.dbt_postgres.postgres__check_schema_exists": { + "macro.dbt_postgres.postgres__get_columns_in_relation": { "arguments": [], - "created_at": 1696458269.570358, + "created_at": 1719485736.308002, "depends_on": { "macros": [ - "macro.dbt.statement" + "macro.dbt.statement", + "macro.dbt.sql_convert_columns_in_relation" ] }, "description": "", @@ -6768,22 +9217,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__check_schema_exists(information_schema, schema) -%}\n {% if information_schema.database -%}\n {{ adapter.verify_database(information_schema.database) }}\n {%- endif -%}\n {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}\n select count(*) from pg_namespace where nspname = '{{ schema }}'\n {% endcall %}\n {{ return(load_result('check_schema_exists').table) }}\n{% endmacro %}", + "macro_sql": "{% macro postgres__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n select\n column_name,\n data_type,\n character_maximum_length,\n numeric_precision,\n numeric_scale\n\n from {{ relation.information_schema('columns') }}\n where table_name = '{{ relation.identifier }}'\n {% if relation.schema %}\n and table_schema = '{{ relation.schema }}'\n {% endif %}\n order by ordinal_position\n\n {% endcall %}\n {% set table = load_result('get_columns_in_relation').table %}\n {{ return(sql_convert_columns_in_relation(table)) }}\n{% endmacro %}", "meta": {}, - "name": "postgres__check_schema_exists", + "name": "postgres__get_columns_in_relation", "original_file_path": "macros/adapters.sql", "package_name": "dbt_postgres", "patch_path": null, "path": "macros/adapters.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt_postgres.postgres__check_schema_exists" + "unique_id": "macro.dbt_postgres.postgres__get_columns_in_relation" }, - "macro.dbt_postgres.postgres__copy_grants": { + "macro.dbt_postgres.postgres__get_create_index_sql": { "arguments": [], - "created_at": 1696458269.57614, + "created_at": 1719485736.30595, "depends_on": { "macros": [] }, @@ -6792,25 +9239,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__copy_grants() %}\n {{ return(False) }}\n{% endmacro %}", + "macro_sql": "{% macro postgres__get_create_index_sql(relation, index_dict) -%}\n {%- set index_config = adapter.parse_index(index_dict) -%}\n {%- set comma_separated_columns = \", \".join(index_config.columns) -%}\n {%- set index_name = index_config.render(relation) -%}\n\n create {% if index_config.unique -%}\n unique\n {%- endif %} index if not exists\n \"{{ index_name }}\"\n on {{ relation }} {% if index_config.type -%}\n using {{ index_config.type }}\n {%- endif %}\n ({{ comma_separated_columns }})\n{%- endmacro %}", "meta": {}, - "name": "postgres__copy_grants", + "name": "postgres__get_create_index_sql", "original_file_path": "macros/adapters.sql", "package_name": "dbt_postgres", "patch_path": null, "path": "macros/adapters.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt_postgres.postgres__copy_grants" + "unique_id": "macro.dbt_postgres.postgres__get_create_index_sql" }, - "macro.dbt_postgres.postgres__create_schema": { + "macro.dbt_postgres.postgres__get_create_materialized_view_as_sql": { "arguments": [], - "created_at": 1696458269.566682, + "created_at": 1719485736.3255181, "depends_on": { "macros": [ - "macro.dbt.statement" + "macro.dbt.get_create_index_sql" ] }, "description": "", @@ -6818,22 +9263,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__create_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier().include(database=False) }}\n {%- endcall -%}\n{% endmacro %}", + "macro_sql": "{% macro postgres__get_create_materialized_view_as_sql(relation, sql) %}\n create materialized view if not exists {{ relation }} as {{ sql }};\n\n {% for _index_dict in config.get('indexes', []) -%}\n {{- get_create_index_sql(relation, _index_dict) -}}\n {%- endfor -%}\n\n{% endmacro %}", "meta": {}, - "name": "postgres__create_schema", - "original_file_path": "macros/adapters.sql", + "name": "postgres__get_create_materialized_view_as_sql", + "original_file_path": "macros/relations/materialized_view/create.sql", "package_name": "dbt_postgres", "patch_path": null, - "path": "macros/adapters.sql", + "path": "macros/relations/materialized_view/create.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt_postgres.postgres__create_schema" + "unique_id": "macro.dbt_postgres.postgres__get_create_materialized_view_as_sql" }, - "macro.dbt_postgres.postgres__create_table_as": { + "macro.dbt_postgres.postgres__get_drop_index_sql": { "arguments": [], - "created_at": 1696458269.5653162, + "created_at": 1719485736.3150148, "depends_on": { "macros": [] }, @@ -6842,49 +9285,48 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__create_table_as(temporary, relation, sql) -%}\n {%- set unlogged = config.get('unlogged', default=false) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary -%}\n temporary\n {%- elif unlogged -%}\n unlogged\n {%- endif %} table {{ relation }}\n as (\n {{ sql }}\n );\n{%- endmacro %}", + "macro_sql": "\n\n\n{%- macro postgres__get_drop_index_sql(relation, index_name) -%}\n drop index if exists \"{{ relation.schema }}\".\"{{ index_name }}\"\n{%- endmacro -%}", "meta": {}, - "name": "postgres__create_table_as", + "name": "postgres__get_drop_index_sql", "original_file_path": "macros/adapters.sql", "package_name": "dbt_postgres", "patch_path": null, "path": "macros/adapters.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt_postgres.postgres__create_table_as" + "unique_id": "macro.dbt_postgres.postgres__get_drop_index_sql" }, - "macro.dbt_postgres.postgres__current_timestamp": { + "macro.dbt_postgres.postgres__get_incremental_default_sql": { "arguments": [], - "created_at": 1696458269.5499048, + "created_at": 1719485736.316301, "depends_on": { - "macros": [] + "macros": [ + "macro.dbt.get_incremental_delete_insert_sql", + "macro.dbt.get_incremental_append_sql" + ] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__current_timestamp() -%}\n now()\n{%- endmacro %}", + "macro_sql": "{% macro postgres__get_incremental_default_sql(arg_dict) %}\n\n {% if arg_dict[\"unique_key\"] %}\n {% do return(get_incremental_delete_insert_sql(arg_dict)) %}\n {% else %}\n {% do return(get_incremental_append_sql(arg_dict)) %}\n {% endif %}\n\n{% endmacro %}", "meta": {}, - "name": "postgres__current_timestamp", - "original_file_path": "macros/timestamps.sql", + "name": "postgres__get_incremental_default_sql", + "original_file_path": "macros/materializations/incremental_strategies.sql", "package_name": "dbt_postgres", "patch_path": null, - "path": "macros/timestamps.sql", + "path": "macros/materializations/incremental_strategies.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt_postgres.postgres__current_timestamp" + "unique_id": "macro.dbt_postgres.postgres__get_incremental_default_sql" }, - "macro.dbt_postgres.postgres__current_timestamp_backcompat": { + "macro.dbt_postgres.postgres__get_materialized_view_configuration_changes": { "arguments": [], - "created_at": 1696458269.551879, + "created_at": 1719485736.324945, "depends_on": { "macros": [ - "macro.dbt.type_timestamp" + "macro.dbt_postgres.postgres__describe_materialized_view" ] }, "description": "", @@ -6892,25 +9334,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__current_timestamp_backcompat() %}\n current_timestamp::{{ type_timestamp() }}\n{% endmacro %}", + "macro_sql": "{% macro postgres__get_materialized_view_configuration_changes(existing_relation, new_config) %}\n {% set _existing_materialized_view = postgres__describe_materialized_view(existing_relation) %}\n {% set _configuration_changes = existing_relation.get_materialized_view_config_change_collection(_existing_materialized_view, new_config.model) %}\n {% do return(_configuration_changes) %}\n{% endmacro %}", "meta": {}, - "name": "postgres__current_timestamp_backcompat", - "original_file_path": "macros/timestamps.sql", + "name": "postgres__get_materialized_view_configuration_changes", + "original_file_path": "macros/relations/materialized_view/alter.sql", "package_name": "dbt_postgres", "patch_path": null, - "path": "macros/timestamps.sql", + "path": "macros/relations/materialized_view/alter.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt_postgres.postgres__current_timestamp_backcompat" + "unique_id": "macro.dbt_postgres.postgres__get_materialized_view_configuration_changes" }, - "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat": { + "macro.dbt_postgres.postgres__get_relations": { "arguments": [], - "created_at": 1696458269.5520551, + "created_at": 1719485736.293338, "depends_on": { "macros": [ - "macro.dbt.type_timestamp" + "macro.dbt.statement" ] }, "description": "", @@ -6918,22 +9358,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__current_timestamp_in_utc_backcompat() %}\n (current_timestamp at time zone 'utc')::{{ type_timestamp() }}\n{% endmacro %}", + "macro_sql": "{% macro postgres__get_relations() -%}\n\n {#\n -- in pg_depend, objid is the dependent, refobjid is the referenced object\n -- > a pg_depend entry indicates that the referenced object cannot be\n -- > dropped without also dropping the dependent object.\n #}\n\n {%- call statement('relations', fetch_result=True) -%}\n with relation as (\n select\n pg_rewrite.ev_class as class,\n pg_rewrite.oid as id\n from pg_rewrite\n ),\n class as (\n select\n oid as id,\n relname as name,\n relnamespace as schema,\n relkind as kind\n from pg_class\n ),\n dependency as (\n select distinct\n pg_depend.objid as id,\n pg_depend.refobjid as ref\n from pg_depend\n ),\n schema as (\n select\n pg_namespace.oid as id,\n pg_namespace.nspname as name\n from pg_namespace\n where nspname != 'information_schema' and nspname not like 'pg\\_%'\n ),\n referenced as (\n select\n relation.id AS id,\n referenced_class.name ,\n referenced_class.schema ,\n referenced_class.kind\n from relation\n join class as referenced_class on relation.class=referenced_class.id\n where referenced_class.kind in ('r', 'v', 'm')\n ),\n relationships as (\n select\n referenced.name as referenced_name,\n referenced.schema as referenced_schema_id,\n dependent_class.name as dependent_name,\n dependent_class.schema as dependent_schema_id,\n referenced.kind as kind\n from referenced\n join dependency on referenced.id=dependency.id\n join class as dependent_class on dependency.ref=dependent_class.id\n where\n (referenced.name != dependent_class.name or\n referenced.schema != dependent_class.schema)\n )\n\n select\n referenced_schema.name as referenced_schema,\n relationships.referenced_name as referenced_name,\n dependent_schema.name as dependent_schema,\n relationships.dependent_name as dependent_name\n from relationships\n join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id\n join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id\n group by referenced_schema, referenced_name, dependent_schema, dependent_name\n order by referenced_schema, referenced_name, dependent_schema, dependent_name;\n\n {%- endcall -%}\n\n {{ return(load_result('relations').table) }}\n{% endmacro %}", "meta": {}, - "name": "postgres__current_timestamp_in_utc_backcompat", - "original_file_path": "macros/timestamps.sql", + "name": "postgres__get_relations", + "original_file_path": "macros/relations.sql", "package_name": "dbt_postgres", "patch_path": null, - "path": "macros/timestamps.sql", + "path": "macros/relations.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat" + "unique_id": "macro.dbt_postgres.postgres__get_relations" }, - "macro.dbt_postgres.postgres__dateadd": { + "macro.dbt_postgres.postgres__get_rename_materialized_view_sql": { "arguments": [], - "created_at": 1696458269.579446, + "created_at": 1719485736.3205602, "depends_on": { "macros": [] }, @@ -6942,77 +9380,69 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }}))\n\n{% endmacro %}", + "macro_sql": "{% macro postgres__get_rename_materialized_view_sql(relation, new_name) %}\n alter materialized view {{ relation }} rename to {{ new_name }}\n{% endmacro %}", "meta": {}, - "name": "postgres__dateadd", - "original_file_path": "macros/utils/dateadd.sql", + "name": "postgres__get_rename_materialized_view_sql", + "original_file_path": "macros/relations/materialized_view/rename.sql", "package_name": "dbt_postgres", "patch_path": null, - "path": "macros/utils/dateadd.sql", + "path": "macros/relations/materialized_view/rename.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt_postgres.postgres__dateadd" + "unique_id": "macro.dbt_postgres.postgres__get_rename_materialized_view_sql" }, - "macro.dbt_postgres.postgres__datediff": { + "macro.dbt_postgres.postgres__get_rename_table_sql": { "arguments": [], - "created_at": 1696458269.586445, + "created_at": 1719485736.326884, "depends_on": { - "macros": [ - "macro.dbt.datediff" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__datediff(first_date, second_date, datepart) -%}\n\n {% if datepart == 'year' %}\n (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date))\n {% elif datepart == 'quarter' %}\n ({{ datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date))\n {% elif datepart == 'month' %}\n ({{ datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date))\n {% elif datepart == 'day' %}\n (({{second_date}})::date - ({{first_date}})::date)\n {% elif datepart == 'week' %}\n ({{ datediff(first_date, second_date, 'day') }} / 7 + case\n when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then\n case when {{first_date}} <= {{second_date}} then 0 else -1 end\n else\n case when {{first_date}} <= {{second_date}} then 1 else 0 end\n end)\n {% elif datepart == 'hour' %}\n ({{ datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp))\n {% elif datepart == 'minute' %}\n ({{ datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp))\n {% elif datepart == 'second' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp)))\n {% elif datepart == 'millisecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp)))\n {% elif datepart == 'microsecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp)))\n {% else %}\n {{ exceptions.raise_compiler_error(\"Unsupported datepart for macro datediff in postgres: {!r}\".format(datepart)) }}\n {% endif %}\n\n{%- endmacro %}", + "macro_sql": "{% macro postgres__get_rename_table_sql(relation, new_name) %}\n alter table {{ relation }} rename to {{ new_name }}\n{% endmacro %}", "meta": {}, - "name": "postgres__datediff", - "original_file_path": "macros/utils/datediff.sql", + "name": "postgres__get_rename_table_sql", + "original_file_path": "macros/relations/table/rename.sql", "package_name": "dbt_postgres", "patch_path": null, - "path": "macros/utils/datediff.sql", + "path": "macros/relations/table/rename.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt_postgres.postgres__datediff" + "unique_id": "macro.dbt_postgres.postgres__get_rename_table_sql" }, - "macro.dbt_postgres.postgres__drop_schema": { + "macro.dbt_postgres.postgres__get_rename_view_sql": { "arguments": [], - "created_at": 1696458269.567198, + "created_at": 1719485736.32807, "depends_on": { - "macros": [ - "macro.dbt.statement" - ] + "macros": [] }, "description": "", "docs": { "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__drop_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade\n {%- endcall -%}\n{% endmacro %}", + "macro_sql": "{% macro postgres__get_rename_view_sql(relation, new_name) %}\n alter view {{ relation }} rename to {{ new_name }}\n{% endmacro %}", "meta": {}, - "name": "postgres__drop_schema", - "original_file_path": "macros/adapters.sql", + "name": "postgres__get_rename_view_sql", + "original_file_path": "macros/relations/view/rename.sql", "package_name": "dbt_postgres", "patch_path": null, - "path": "macros/adapters.sql", + "path": "macros/relations/view/rename.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt_postgres.postgres__drop_schema" + "unique_id": "macro.dbt_postgres.postgres__get_rename_view_sql" }, - "macro.dbt_postgres.postgres__get_catalog": { + "macro.dbt_postgres.postgres__get_replace_table_sql": { "arguments": [], - "created_at": 1696458269.5540092, + "created_at": 1719485736.326431, "depends_on": { "macros": [ - "macro.dbt.statement" + "macro.dbt.get_assert_columns_equivalent", + "macro.dbt.get_table_columns_and_constraints", + "macro.dbt.get_select_subquery" ] }, "description": "", @@ -7020,26 +9450,23 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__get_catalog(information_schema, schemas) -%}\n\n {%- call statement('catalog', fetch_result=True) -%}\n {#\n If the user has multiple databases set and the first one is wrong, this will fail.\n But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.\n #}\n {% set database = information_schema.database %}\n {{ adapter.verify_database(database) }}\n\n select\n '{{ database }}' as table_database,\n sch.nspname as table_schema,\n tbl.relname as table_name,\n case tbl.relkind\n when 'v' then 'VIEW'\n else 'BASE TABLE'\n end as table_type,\n tbl_desc.description as table_comment,\n col.attname as column_name,\n col.attnum as column_index,\n pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,\n col_desc.description as column_comment,\n pg_get_userbyid(tbl.relowner) as table_owner\n\n from pg_catalog.pg_namespace sch\n join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid\n join pg_catalog.pg_attribute col on col.attrelid = tbl.oid\n left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)\n left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)\n\n where (\n {%- for schema in schemas -%}\n upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session\n and tbl.relpersistence in ('p', 'u') -- [p]ermanent table or [u]nlogged table. Exclude [t]emporary tables\n and tbl.relkind in ('r', 'v', 'f', 'p') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table, [m]aterialized view\n and col.attnum > 0 -- negative numbers are used for system columns such as oid\n and not col.attisdropped -- column as not been dropped\n\n order by\n sch.nspname,\n tbl.relname,\n col.attnum\n\n {%- endcall -%}\n\n {{ return(load_result('catalog').table) }}\n\n{%- endmacro %}", + "macro_sql": "{% macro postgres__get_replace_table_sql(relation, sql) -%}\n\n {%- set sql_header = config.get('sql_header', none) -%}\n {{ sql_header if sql_header is not none }}\n\n create or replace table {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n\n{%- endmacro %}", "meta": {}, - "name": "postgres__get_catalog", - "original_file_path": "macros/catalog.sql", + "name": "postgres__get_replace_table_sql", + "original_file_path": "macros/relations/table/replace.sql", "package_name": "dbt_postgres", "patch_path": null, - "path": "macros/catalog.sql", + "path": "macros/relations/table/replace.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt_postgres.postgres__get_catalog" + "unique_id": "macro.dbt_postgres.postgres__get_replace_table_sql" }, - "macro.dbt_postgres.postgres__get_columns_in_relation": { + "macro.dbt_postgres.postgres__get_replace_view_sql": { "arguments": [], - "created_at": 1696458269.568068, + "created_at": 1719485736.327884, "depends_on": { "macros": [ - "macro.dbt.statement", - "macro.dbt.sql_convert_columns_in_relation" + "macro.dbt.get_assert_columns_equivalent" ] }, "description": "", @@ -7047,22 +9474,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n select\n column_name,\n data_type,\n character_maximum_length,\n numeric_precision,\n numeric_scale\n\n from {{ relation.information_schema('columns') }}\n where table_name = '{{ relation.identifier }}'\n {% if relation.schema %}\n and table_schema = '{{ relation.schema }}'\n {% endif %}\n order by ordinal_position\n\n {% endcall %}\n {% set table = load_result('get_columns_in_relation').table %}\n {{ return(sql_convert_columns_in_relation(table)) }}\n{% endmacro %}", + "macro_sql": "{% macro postgres__get_replace_view_sql(relation, sql) -%}\n\n {%- set sql_header = config.get('sql_header', none) -%}\n {{ sql_header if sql_header is not none }}\n\n create or replace view {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n\n{%- endmacro %}", "meta": {}, - "name": "postgres__get_columns_in_relation", - "original_file_path": "macros/adapters.sql", + "name": "postgres__get_replace_view_sql", + "original_file_path": "macros/relations/view/replace.sql", "package_name": "dbt_postgres", "patch_path": null, - "path": "macros/adapters.sql", + "path": "macros/relations/view/replace.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt_postgres.postgres__get_columns_in_relation" + "unique_id": "macro.dbt_postgres.postgres__get_replace_view_sql" }, - "macro.dbt_postgres.postgres__get_create_index_sql": { + "macro.dbt_postgres.postgres__get_show_grant_sql": { "arguments": [], - "created_at": 1696458269.566161, + "created_at": 1719485736.314513, "depends_on": { "macros": [] }, @@ -7071,49 +9496,20 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__get_create_index_sql(relation, index_dict) -%}\n {%- set index_config = adapter.parse_index(index_dict) -%}\n {%- set comma_separated_columns = \", \".join(index_config.columns) -%}\n {%- set index_name = index_config.render(relation) -%}\n\n create {% if index_config.unique -%}\n unique\n {%- endif %} index if not exists\n \"{{ index_name }}\"\n on {{ relation }} {% if index_config.type -%}\n using {{ index_config.type }}\n {%- endif %}\n ({{ comma_separated_columns }});\n{%- endmacro %}", + "macro_sql": "\n\n{%- macro postgres__get_show_grant_sql(relation) -%}\n select grantee, privilege_type\n from {{ relation.information_schema('role_table_grants') }}\n where grantor = current_role\n and grantee != current_role\n and table_schema = '{{ relation.schema }}'\n and table_name = '{{ relation.identifier }}'\n{%- endmacro -%}\n\n", "meta": {}, - "name": "postgres__get_create_index_sql", + "name": "postgres__get_show_grant_sql", "original_file_path": "macros/adapters.sql", "package_name": "dbt_postgres", "patch_path": null, "path": "macros/adapters.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", - "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt_postgres.postgres__get_create_index_sql" - }, - "macro.dbt_postgres.postgres__get_incremental_default_sql": { - "arguments": [], - "created_at": 1696458269.577235, - "depends_on": { - "macros": [ - "macro.dbt.get_incremental_delete_insert_sql", - "macro.dbt.get_incremental_append_sql" - ] - }, - "description": "", - "docs": { - "node_color": null, - "show": true - }, - "macro_sql": "{% macro postgres__get_incremental_default_sql(arg_dict) %}\n\n {% if arg_dict[\"unique_key\"] %}\n {% do return(get_incremental_delete_insert_sql(arg_dict)) %}\n {% else %}\n {% do return(get_incremental_append_sql(arg_dict)) %}\n {% endif %}\n\n{% endmacro %}", - "meta": {}, - "name": "postgres__get_incremental_default_sql", - "original_file_path": "macros/materializations/incremental_strategies.sql", - "package_name": "dbt_postgres", - "patch_path": null, - "path": "macros/materializations/incremental_strategies.sql", - "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt_postgres.postgres__get_incremental_default_sql" + "unique_id": "macro.dbt_postgres.postgres__get_show_grant_sql" }, - "macro.dbt_postgres.postgres__get_show_grant_sql": { + "macro.dbt_postgres.postgres__get_show_indexes_sql": { "arguments": [], - "created_at": 1696458269.575934, + "created_at": 1719485736.31486, "depends_on": { "macros": [] }, @@ -7122,22 +9518,20 @@ "node_color": null, "show": true }, - "macro_sql": "\n\n{%- macro postgres__get_show_grant_sql(relation) -%}\n select grantee, privilege_type\n from {{ relation.information_schema('role_table_grants') }}\n where grantor = current_role\n and grantee != current_role\n and table_schema = '{{ relation.schema }}'\n and table_name = '{{ relation.identifier }}'\n{%- endmacro -%}\n\n", + "macro_sql": "{% macro postgres__get_show_indexes_sql(relation) %}\n select\n i.relname as name,\n m.amname as method,\n ix.indisunique as \"unique\",\n array_to_string(array_agg(a.attname), ',') as column_names\n from pg_index ix\n join pg_class i\n on i.oid = ix.indexrelid\n join pg_am m\n on m.oid=i.relam\n join pg_class t\n on t.oid = ix.indrelid\n join pg_namespace n\n on n.oid = t.relnamespace\n join pg_attribute a\n on a.attrelid = t.oid\n and a.attnum = ANY(ix.indkey)\n where t.relname = '{{ relation.identifier }}'\n and n.nspname = '{{ relation.schema }}'\n and t.relkind in ('r', 'm')\n group by 1, 2, 3\n order by 1, 2, 3\n{% endmacro %}", "meta": {}, - "name": "postgres__get_show_grant_sql", + "name": "postgres__get_show_indexes_sql", "original_file_path": "macros/adapters.sql", "package_name": "dbt_postgres", "patch_path": null, "path": "macros/adapters.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], - "unique_id": "macro.dbt_postgres.postgres__get_show_grant_sql" + "unique_id": "macro.dbt_postgres.postgres__get_show_indexes_sql" }, "macro.dbt_postgres.postgres__information_schema_name": { "arguments": [], - "created_at": 1696458269.568956, + "created_at": 1719485736.308933, "depends_on": { "macros": [] }, @@ -7154,14 +9548,12 @@ "patch_path": null, "path": "macros/adapters.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_postgres.postgres__information_schema_name" }, "macro.dbt_postgres.postgres__last_day": { "arguments": [], - "created_at": 1696458269.587851, + "created_at": 1719485736.333154, "depends_on": { "macros": [ "macro.dbt.dateadd", @@ -7182,14 +9574,12 @@ "patch_path": null, "path": "macros/utils/last_day.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_postgres.postgres__last_day" }, "macro.dbt_postgres.postgres__list_relations_without_caching": { "arguments": [], - "created_at": 1696458269.568678, + "created_at": 1719485736.308728, "depends_on": { "macros": [ "macro.dbt.statement" @@ -7200,7 +9590,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n select\n '{{ schema_relation.database }}' as database,\n tablename as name,\n schemaname as schema,\n 'table' as type\n from pg_tables\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n viewname as name,\n schemaname as schema,\n 'view' as type\n from pg_views\n where schemaname ilike '{{ schema_relation.schema }}'\n {% endcall %}\n {{ return(load_result('list_relations_without_caching').table) }}\n{% endmacro %}", + "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n select\n '{{ schema_relation.database }}' as database,\n tablename as name,\n schemaname as schema,\n 'table' as type\n from pg_tables\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n viewname as name,\n schemaname as schema,\n 'view' as type\n from pg_views\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n matviewname as name,\n schemaname as schema,\n 'materialized_view' as type\n from pg_matviews\n where schemaname ilike '{{ schema_relation.schema }}'\n {% endcall %}\n {{ return(load_result('list_relations_without_caching').table) }}\n{% endmacro %}", "meta": {}, "name": "postgres__list_relations_without_caching", "original_file_path": "macros/adapters.sql", @@ -7208,14 +9598,12 @@ "patch_path": null, "path": "macros/adapters.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_postgres.postgres__list_relations_without_caching" }, "macro.dbt_postgres.postgres__list_schemas": { "arguments": [], - "created_at": 1696458269.569512, + "created_at": 1719485736.309331, "depends_on": { "macros": [ "macro.dbt.statement" @@ -7234,14 +9622,12 @@ "patch_path": null, "path": "macros/adapters.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_postgres.postgres__list_schemas" }, "macro.dbt_postgres.postgres__listagg": { "arguments": [], - "created_at": 1696458269.5807512, + "created_at": 1719485736.329083, "depends_on": { "macros": [] }, @@ -7258,14 +9644,12 @@ "patch_path": null, "path": "macros/utils/listagg.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_postgres.postgres__listagg" }, "macro.dbt_postgres.postgres__make_backup_relation": { "arguments": [], - "created_at": 1696458269.5734138, + "created_at": 1719485736.311583, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__make_relation_with_suffix" @@ -7284,14 +9668,12 @@ "patch_path": null, "path": "macros/adapters.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_postgres.postgres__make_backup_relation" }, "macro.dbt_postgres.postgres__make_intermediate_relation": { "arguments": [], - "created_at": 1696458269.5723772, + "created_at": 1719485736.3109372, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__make_relation_with_suffix" @@ -7310,14 +9692,12 @@ "patch_path": null, "path": "macros/adapters.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_postgres.postgres__make_intermediate_relation" }, "macro.dbt_postgres.postgres__make_relation_with_suffix": { "arguments": [], - "created_at": 1696458269.5720162, + "created_at": 1719485736.31072, "depends_on": { "macros": [] }, @@ -7334,14 +9714,12 @@ "patch_path": null, "path": "macros/adapters.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_postgres.postgres__make_relation_with_suffix" }, "macro.dbt_postgres.postgres__make_temp_relation": { "arguments": [], - "created_at": 1696458269.572947, + "created_at": 1719485736.31129, "depends_on": { "macros": [ "macro.dbt_postgres.postgres__make_relation_with_suffix" @@ -7360,14 +9738,34 @@ "patch_path": null, "path": "macros/adapters.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_postgres.postgres__make_temp_relation" }, + "macro.dbt_postgres.postgres__refresh_materialized_view": { + "arguments": [], + "created_at": 1719485736.32038, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__refresh_materialized_view(relation) %}\n refresh materialized view {{ relation }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__refresh_materialized_view", + "original_file_path": "macros/relations/materialized_view/refresh.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations/materialized_view/refresh.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__refresh_materialized_view" + }, "macro.dbt_postgres.postgres__snapshot_get_time": { "arguments": [], - "created_at": 1696458269.5516999, + "created_at": 1719485736.289238, "depends_on": { "macros": [ "macro.dbt.current_timestamp" @@ -7386,14 +9784,12 @@ "patch_path": null, "path": "macros/timestamps.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_postgres.postgres__snapshot_get_time" }, "macro.dbt_postgres.postgres__snapshot_merge_sql": { "arguments": [], - "created_at": 1696458269.5788622, + "created_at": 1719485736.318985, "depends_on": { "macros": [] }, @@ -7410,14 +9806,12 @@ "patch_path": null, "path": "macros/materializations/snapshot_merge.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_postgres.postgres__snapshot_merge_sql" }, "macro.dbt_postgres.postgres__snapshot_string_as_time": { "arguments": [], - "created_at": 1696458269.551509, + "created_at": 1719485736.289119, "depends_on": { "macros": [] }, @@ -7434,14 +9828,12 @@ "patch_path": null, "path": "macros/timestamps.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_postgres.postgres__snapshot_string_as_time" }, "macro.dbt_postgres.postgres__split_part": { "arguments": [], - "created_at": 1696458269.588754, + "created_at": 1719485736.334007, "depends_on": { "macros": [ "macro.dbt.default__split_part", @@ -7461,14 +9853,37 @@ "patch_path": null, "path": "macros/utils/split_part.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_postgres.postgres__split_part" }, + "macro.dbt_postgres.postgres__update_indexes_on_materialized_view": { + "arguments": [], + "created_at": 1719485736.3239639, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_drop_index_sql", + "macro.dbt_postgres.postgres__get_create_index_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n\n{%- macro postgres__update_indexes_on_materialized_view(relation, index_changes) -%}\n {{- log(\"Applying UPDATE INDEXES to: \" ~ relation) -}}\n\n {%- for _index_change in index_changes -%}\n {%- set _index = _index_change.context -%}\n\n {%- if _index_change.action == \"drop\" -%}\n\n {{ postgres__get_drop_index_sql(relation, _index.name) }};\n\n {%- elif _index_change.action == \"create\" -%}\n\n {{ postgres__get_create_index_sql(relation, _index.as_node_config) }}\n\n {%- endif -%}\n\n {%- endfor -%}\n\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "postgres__update_indexes_on_materialized_view", + "original_file_path": "macros/relations/materialized_view/alter.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations/materialized_view/alter.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__update_indexes_on_materialized_view" + }, "macro.dbt_postgres.postgres_escape_comment": { "arguments": [], - "created_at": 1696458269.5741508, + "created_at": 1719485736.3121421, "depends_on": { "macros": [] }, @@ -7485,17 +9900,15 @@ "patch_path": null, "path": "macros/adapters.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_postgres.postgres_escape_comment" }, "macro.dbt_postgres.postgres_get_relations": { "arguments": [], - "created_at": 1696458269.555246, + "created_at": 1719485736.2934961, "depends_on": { "macros": [ - "macro.dbt.statement" + "macro.dbt_postgres.postgres__get_relations" ] }, "description": "", @@ -7503,7 +9916,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro postgres_get_relations () -%}\n\n {#\n -- in pg_depend, objid is the dependent, refobjid is the referenced object\n -- > a pg_depend entry indicates that the referenced object cannot be\n -- > dropped without also dropping the dependent object.\n #}\n\n {%- call statement('relations', fetch_result=True) -%}\n with relation as (\n select\n pg_rewrite.ev_class as class,\n pg_rewrite.oid as id\n from pg_rewrite\n ),\n class as (\n select\n oid as id,\n relname as name,\n relnamespace as schema,\n relkind as kind\n from pg_class\n ),\n dependency as (\n select distinct\n pg_depend.objid as id,\n pg_depend.refobjid as ref\n from pg_depend\n ),\n schema as (\n select\n pg_namespace.oid as id,\n pg_namespace.nspname as name\n from pg_namespace\n where nspname != 'information_schema' and nspname not like 'pg\\_%'\n ),\n referenced as (\n select\n relation.id AS id,\n referenced_class.name ,\n referenced_class.schema ,\n referenced_class.kind\n from relation\n join class as referenced_class on relation.class=referenced_class.id\n where referenced_class.kind in ('r', 'v')\n ),\n relationships as (\n select\n referenced.name as referenced_name,\n referenced.schema as referenced_schema_id,\n dependent_class.name as dependent_name,\n dependent_class.schema as dependent_schema_id,\n referenced.kind as kind\n from referenced\n join dependency on referenced.id=dependency.id\n join class as dependent_class on dependency.ref=dependent_class.id\n where\n (referenced.name != dependent_class.name or\n referenced.schema != dependent_class.schema)\n )\n\n select\n referenced_schema.name as referenced_schema,\n relationships.referenced_name as referenced_name,\n dependent_schema.name as dependent_schema,\n relationships.dependent_name as dependent_name\n from relationships\n join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id\n join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id\n group by referenced_schema, referenced_name, dependent_schema, dependent_name\n order by referenced_schema, referenced_name, dependent_schema, dependent_name;\n\n {%- endcall -%}\n\n {{ return(load_result('relations').table) }}\n{% endmacro %}", + "macro_sql": "{% macro postgres_get_relations() %}\n {{ return(postgres__get_relations()) }}\n{% endmacro %}", "meta": {}, "name": "postgres_get_relations", "original_file_path": "macros/relations.sql", @@ -7511,14 +9924,12 @@ "patch_path": null, "path": "macros/relations.sql", "resource_type": "macro", - "root_path": "/Users/julian/.pyenv/versions/3.10.0/lib/python3.10/site-packages/dbt/include/postgres", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_postgres.postgres_get_relations" }, "macro.dbt_utils._bigquery__get_matching_schemata": { "arguments": [], - "created_at": 1696458269.967883, + "created_at": 1719485736.7140338, "depends_on": { "macros": [ "macro.dbt.run_query" @@ -7537,14 +9948,12 @@ "patch_path": null, "path": "macros/sql/get_tables_by_pattern_sql.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils._bigquery__get_matching_schemata" }, "macro.dbt_utils._is_ephemeral": { "arguments": [], - "created_at": 1696458269.911734, + "created_at": 1719485736.667329, "depends_on": { "macros": [] }, @@ -7561,14 +9970,12 @@ "patch_path": null, "path": "macros/jinja_helpers/_is_ephemeral.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils._is_ephemeral" }, "macro.dbt_utils._is_relation": { "arguments": [], - "created_at": 1696458269.907017, + "created_at": 1719485736.6636631, "depends_on": { "macros": [] }, @@ -7585,14 +9992,12 @@ "patch_path": null, "path": "macros/jinja_helpers/_is_relation.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils._is_relation" }, "macro.dbt_utils.bigquery__deduplicate": { "arguments": [], - "created_at": 1696458269.95776, + "created_at": 1719485736.703784, "depends_on": { "macros": [] }, @@ -7609,14 +10014,12 @@ "patch_path": null, "path": "macros/sql/deduplicate.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.bigquery__deduplicate" }, "macro.dbt_utils.bigquery__get_tables_by_pattern_sql": { "arguments": [], - "created_at": 1696458269.9670799, + "created_at": 1719485736.713356, "depends_on": { "macros": [ "macro.dbt_utils._bigquery__get_matching_schemata", @@ -7636,14 +10039,12 @@ "patch_path": null, "path": "macros/sql/get_tables_by_pattern_sql.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.bigquery__get_tables_by_pattern_sql" }, "macro.dbt_utils.bigquery__haversine_distance": { "arguments": [], - "created_at": 1696458269.9946911, + "created_at": 1719485736.73806, "depends_on": { "macros": [ "macro.dbt_utils.degrees_to_radians" @@ -7662,14 +10063,12 @@ "patch_path": null, "path": "macros/sql/haversine_distance.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.bigquery__haversine_distance" }, "macro.dbt_utils.databricks__get_table_types_sql": { "arguments": [], - "created_at": 1696458269.98752, + "created_at": 1719485736.73151, "depends_on": { "macros": [] }, @@ -7686,14 +10085,12 @@ "patch_path": null, "path": "macros/sql/get_table_types_sql.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.databricks__get_table_types_sql" }, "macro.dbt_utils.date_spine": { "arguments": [], - "created_at": 1696458269.914428, + "created_at": 1719485736.669255, "depends_on": { "macros": [ "macro.dbt_utils.default__date_spine" @@ -7712,14 +10109,12 @@ "patch_path": null, "path": "macros/sql/date_spine.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.date_spine" }, "macro.dbt_utils.deduplicate": { "arguments": [], - "created_at": 1696458269.9561112, + "created_at": 1719485736.7012029, "depends_on": { "macros": [ "macro.dbt_utils.postgres__deduplicate" @@ -7738,14 +10133,12 @@ "patch_path": null, "path": "macros/sql/deduplicate.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.deduplicate" }, "macro.dbt_utils.default__date_spine": { "arguments": [], - "created_at": 1696458269.915016, + "created_at": 1719485736.669651, "depends_on": { "macros": [ "macro.dbt_utils.generate_series", @@ -7766,14 +10159,12 @@ "patch_path": null, "path": "macros/sql/date_spine.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__date_spine" }, "macro.dbt_utils.default__deduplicate": { "arguments": [], - "created_at": 1696458269.956494, + "created_at": 1719485736.701886, "depends_on": { "macros": [] }, @@ -7790,14 +10181,12 @@ "patch_path": null, "path": "macros/sql/deduplicate.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__deduplicate" }, "macro.dbt_utils.default__generate_series": { "arguments": [], - "created_at": 1696458269.925334, + "created_at": 1719485736.678598, "depends_on": { "macros": [ "macro.dbt_utils.get_powers_of_two" @@ -7816,14 +10205,12 @@ "patch_path": null, "path": "macros/sql/generate_series.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__generate_series" }, "macro.dbt_utils.default__generate_surrogate_key": { "arguments": [], - "created_at": 1696458269.985795, + "created_at": 1719485736.730099, "depends_on": { "macros": [ "macro.dbt.type_string", @@ -7844,14 +10231,12 @@ "patch_path": null, "path": "macros/sql/generate_surrogate_key.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__generate_surrogate_key" }, "macro.dbt_utils.default__get_column_values": { "arguments": [], - "created_at": 1696458269.9728918, + "created_at": 1719485736.717495, "depends_on": { "macros": [ "macro.dbt_utils._is_ephemeral", @@ -7872,14 +10257,12 @@ "patch_path": null, "path": "macros/sql/get_column_values.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__get_column_values" }, "macro.dbt_utils.default__get_filtered_columns_in_relation": { "arguments": [], - "created_at": 1696458269.979146, + "created_at": 1719485736.722564, "depends_on": { "macros": [ "macro.dbt_utils._is_relation", @@ -7899,14 +10282,12 @@ "patch_path": null, "path": "macros/sql/get_filtered_columns_in_relation.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__get_filtered_columns_in_relation" }, "macro.dbt_utils.default__get_intervals_between": { "arguments": [], - "created_at": 1696458269.914051, + "created_at": 1719485736.668746, "depends_on": { "macros": [ "macro.dbt.statement", @@ -7926,14 +10307,12 @@ "patch_path": null, "path": "macros/sql/date_spine.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__get_intervals_between" }, "macro.dbt_utils.default__get_powers_of_two": { "arguments": [], - "created_at": 1696458269.923839, + "created_at": 1719485736.677862, "depends_on": { "macros": [] }, @@ -7950,14 +10329,12 @@ "patch_path": null, "path": "macros/sql/generate_series.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__get_powers_of_two" }, "macro.dbt_utils.default__get_query_results_as_dict": { "arguments": [], - "created_at": 1696458269.9838238, + "created_at": 1719485736.727696, "depends_on": { "macros": [ "macro.dbt.statement" @@ -7976,14 +10353,12 @@ "patch_path": null, "path": "macros/sql/get_query_results_as_dict.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__get_query_results_as_dict" }, "macro.dbt_utils.default__get_relations_by_pattern": { "arguments": [], - "created_at": 1696458269.921258, + "created_at": 1719485736.674046, "depends_on": { "macros": [ "macro.dbt.statement", @@ -8003,14 +10378,12 @@ "patch_path": null, "path": "macros/sql/get_relations_by_pattern.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__get_relations_by_pattern" }, "macro.dbt_utils.default__get_relations_by_prefix": { "arguments": [], - "created_at": 1696458269.928546, + "created_at": 1719485736.680293, "depends_on": { "macros": [ "macro.dbt.statement", @@ -8030,14 +10403,12 @@ "patch_path": null, "path": "macros/sql/get_relations_by_prefix.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__get_relations_by_prefix" }, "macro.dbt_utils.default__get_single_value": { "arguments": [], - "created_at": 1696458269.989894, + "created_at": 1719485736.7332969, "depends_on": { "macros": [ "macro.dbt.statement" @@ -8056,14 +10427,12 @@ "patch_path": null, "path": "macros/sql/get_single_value.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__get_single_value" }, "macro.dbt_utils.default__get_table_types_sql": { "arguments": [], - "created_at": 1696458269.987062, + "created_at": 1719485736.73105, "depends_on": { "macros": [] }, @@ -8080,14 +10449,12 @@ "patch_path": null, "path": "macros/sql/get_table_types_sql.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__get_table_types_sql" }, "macro.dbt_utils.default__get_tables_by_pattern_sql": { "arguments": [], - "created_at": 1696458269.965835, + "created_at": 1719485736.7122319, "depends_on": { "macros": [ "macro.dbt_utils.get_table_types_sql" @@ -8106,14 +10473,12 @@ "patch_path": null, "path": "macros/sql/get_tables_by_pattern_sql.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__get_tables_by_pattern_sql" }, "macro.dbt_utils.default__get_tables_by_prefix_sql": { "arguments": [], - "created_at": 1696458269.930016, + "created_at": 1719485736.681414, "depends_on": { "macros": [ "macro.dbt_utils.get_tables_by_pattern_sql" @@ -8132,14 +10497,12 @@ "patch_path": null, "path": "macros/sql/get_tables_by_prefix_sql.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__get_tables_by_prefix_sql" }, "macro.dbt_utils.default__get_url_host": { "arguments": [], - "created_at": 1696458269.857218, + "created_at": 1719485736.621399, "depends_on": { "macros": [ "macro.dbt.split_part", @@ -8161,14 +10524,12 @@ "patch_path": null, "path": "macros/web/get_url_host.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__get_url_host" }, "macro.dbt_utils.default__get_url_parameter": { "arguments": [], - "created_at": 1696458269.860918, + "created_at": 1719485736.624197, "depends_on": { "macros": [ "macro.dbt.split_part" @@ -8187,14 +10548,12 @@ "patch_path": null, "path": "macros/web/get_url_parameter.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__get_url_parameter" }, "macro.dbt_utils.default__get_url_path": { "arguments": [], - "created_at": 1696458269.859402, + "created_at": 1719485736.6228778, "depends_on": { "macros": [ "macro.dbt.replace", @@ -8219,14 +10578,12 @@ "patch_path": null, "path": "macros/web/get_url_path.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__get_url_path" }, "macro.dbt_utils.default__group_by": { "arguments": [], - "created_at": 1696458269.954616, + "created_at": 1719485736.700402, "depends_on": { "macros": [] }, @@ -8243,14 +10600,12 @@ "patch_path": null, "path": "macros/sql/groupby.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__group_by" }, "macro.dbt_utils.default__haversine_distance": { "arguments": [], - "created_at": 1696458269.993296, + "created_at": 1719485736.736367, "depends_on": { "macros": [] }, @@ -8267,14 +10622,12 @@ "patch_path": null, "path": "macros/sql/haversine_distance.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__haversine_distance" }, "macro.dbt_utils.default__log_info": { "arguments": [], - "created_at": 1696458269.908833, + "created_at": 1719485736.665316, "depends_on": { "macros": [ "macro.dbt_utils.pretty_log_format" @@ -8293,14 +10646,12 @@ "patch_path": null, "path": "macros/jinja_helpers/log_info.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__log_info" }, "macro.dbt_utils.default__nullcheck": { "arguments": [], - "created_at": 1696458269.9623308, + "created_at": 1719485736.7089212, "depends_on": { "macros": [] }, @@ -8317,14 +10668,12 @@ "patch_path": null, "path": "macros/sql/nullcheck.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__nullcheck" }, "macro.dbt_utils.default__nullcheck_table": { "arguments": [], - "created_at": 1696458269.9181361, + "created_at": 1719485736.6715178, "depends_on": { "macros": [ "macro.dbt_utils._is_relation", @@ -8345,14 +10694,12 @@ "patch_path": null, "path": "macros/sql/nullcheck_table.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__nullcheck_table" }, "macro.dbt_utils.default__pivot": { "arguments": [], - "created_at": 1696458269.976574, + "created_at": 1719485736.720366, "depends_on": { "macros": [ "macro.dbt.escape_single_quotes", @@ -8372,14 +10719,12 @@ "patch_path": null, "path": "macros/sql/pivot.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__pivot" }, "macro.dbt_utils.default__pretty_log_format": { "arguments": [], - "created_at": 1696458269.906094, + "created_at": 1719485736.66297, "depends_on": { "macros": [ "macro.dbt_utils.pretty_time" @@ -8398,14 +10743,12 @@ "patch_path": null, "path": "macros/jinja_helpers/pretty_log_format.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__pretty_log_format" }, "macro.dbt_utils.default__pretty_time": { "arguments": [], - "created_at": 1696458269.90797, + "created_at": 1719485736.66434, "depends_on": { "macros": [] }, @@ -8422,14 +10765,12 @@ "patch_path": null, "path": "macros/jinja_helpers/pretty_time.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__pretty_time" }, "macro.dbt_utils.default__safe_add": { "arguments": [], - "created_at": 1696458269.961039, + "created_at": 1719485736.707396, "depends_on": { "macros": [] }, @@ -8446,14 +10787,12 @@ "patch_path": null, "path": "macros/sql/safe_add.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__safe_add" }, "macro.dbt_utils.default__safe_divide": { "arguments": [], - "created_at": 1696458269.941708, + "created_at": 1719485736.689852, "depends_on": { "macros": [] }, @@ -8470,14 +10809,12 @@ "patch_path": null, "path": "macros/sql/safe_divide.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__safe_divide" }, "macro.dbt_utils.default__safe_subtract": { "arguments": [], - "created_at": 1696458269.916886, + "created_at": 1719485736.670645, "depends_on": { "macros": [] }, @@ -8494,14 +10831,12 @@ "patch_path": null, "path": "macros/sql/safe_subtract.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__safe_subtract" }, "macro.dbt_utils.default__star": { "arguments": [], - "created_at": 1696458269.935112, + "created_at": 1719485736.685639, "depends_on": { "macros": [ "macro.dbt_utils._is_relation", @@ -8522,14 +10857,12 @@ "patch_path": null, "path": "macros/sql/star.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__star" }, "macro.dbt_utils.default__surrogate_key": { "arguments": [], - "created_at": 1696458269.959094, + "created_at": 1719485736.7051, "depends_on": { "macros": [] }, @@ -8546,14 +10879,12 @@ "patch_path": null, "path": "macros/sql/surrogate_key.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__surrogate_key" }, "macro.dbt_utils.default__test_accepted_range": { "arguments": [], - "created_at": 1696458269.8764522, + "created_at": 1719485736.6358259, "depends_on": { "macros": [] }, @@ -8570,14 +10901,12 @@ "patch_path": null, "path": "macros/generic_tests/accepted_range.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__test_accepted_range" }, "macro.dbt_utils.default__test_at_least_one": { "arguments": [], - "created_at": 1696458269.880932, + "created_at": 1719485736.638953, "depends_on": { "macros": [] }, @@ -8594,14 +10923,12 @@ "patch_path": null, "path": "macros/generic_tests/at_least_one.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__test_at_least_one" }, "macro.dbt_utils.default__test_cardinality_equality": { "arguments": [], - "created_at": 1696458269.884924, + "created_at": 1719485736.646685, "depends_on": { "macros": [ "macro.dbt.except" @@ -8620,14 +10947,12 @@ "patch_path": null, "path": "macros/generic_tests/cardinality_equality.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__test_cardinality_equality" }, "macro.dbt_utils.default__test_equal_rowcount": { "arguments": [], - "created_at": 1696458269.867796, + "created_at": 1719485736.629295, "depends_on": { "macros": [] }, @@ -8644,14 +10969,12 @@ "patch_path": null, "path": "macros/generic_tests/equal_rowcount.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__test_equal_rowcount" }, "macro.dbt_utils.default__test_equality": { "arguments": [], - "created_at": 1696458269.8951738, + "created_at": 1719485736.654316, "depends_on": { "macros": [ "macro.dbt_utils._is_relation", @@ -8672,14 +10995,12 @@ "patch_path": null, "path": "macros/generic_tests/equality.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__test_equality" }, "macro.dbt_utils.default__test_expression_is_true": { "arguments": [], - "created_at": 1696458269.8862998, + "created_at": 1719485736.6476219, "depends_on": { "macros": [ "macro.dbt.should_store_failures" @@ -8698,14 +11019,12 @@ "patch_path": null, "path": "macros/generic_tests/expression_is_true.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__test_expression_is_true" }, "macro.dbt_utils.default__test_fewer_rows_than": { "arguments": [], - "created_at": 1696458269.864535, + "created_at": 1719485736.626648, "depends_on": { "macros": [] }, @@ -8722,14 +11041,12 @@ "patch_path": null, "path": "macros/generic_tests/fewer_rows_than.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__test_fewer_rows_than" }, "macro.dbt_utils.default__test_mutually_exclusive_ranges": { "arguments": [], - "created_at": 1696458269.90514, + "created_at": 1719485736.662267, "depends_on": { "macros": [] }, @@ -8746,14 +11063,12 @@ "patch_path": null, "path": "macros/generic_tests/mutually_exclusive_ranges.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__test_mutually_exclusive_ranges" }, "macro.dbt_utils.default__test_not_accepted_values": { "arguments": [], - "created_at": 1696458269.8782852, + "created_at": 1719485736.636965, "depends_on": { "macros": [] }, @@ -8770,14 +11085,12 @@ "patch_path": null, "path": "macros/generic_tests/not_accepted_values.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__test_not_accepted_values" }, "macro.dbt_utils.default__test_not_constant": { "arguments": [], - "created_at": 1696458269.874454, + "created_at": 1719485736.634388, "depends_on": { "macros": [] }, @@ -8794,14 +11107,12 @@ "patch_path": null, "path": "macros/generic_tests/not_constant.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__test_not_constant" }, "macro.dbt_utils.default__test_not_empty_string": { "arguments": [], - "created_at": 1696458269.8967302, + "created_at": 1719485736.655891, "depends_on": { "macros": [] }, @@ -8818,14 +11129,12 @@ "patch_path": null, "path": "macros/generic_tests/not_empty_string.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__test_not_empty_string" }, "macro.dbt_utils.default__test_not_null_proportion": { "arguments": [], - "created_at": 1696458269.8888562, + "created_at": 1719485736.6493912, "depends_on": { "macros": [] }, @@ -8842,14 +11151,12 @@ "patch_path": null, "path": "macros/generic_tests/not_null_proportion.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__test_not_null_proportion" }, "macro.dbt_utils.default__test_recency": { "arguments": [], - "created_at": 1696458269.8727272, + "created_at": 1719485736.633423, "depends_on": { "macros": [ "macro.dbt.dateadd", @@ -8870,14 +11177,12 @@ "patch_path": null, "path": "macros/generic_tests/recency.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__test_recency" }, "macro.dbt_utils.default__test_relationships_where": { "arguments": [], - "created_at": 1696458269.8695161, + "created_at": 1719485736.630768, "depends_on": { "macros": [] }, @@ -8894,14 +11199,12 @@ "patch_path": null, "path": "macros/generic_tests/relationships_where.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__test_relationships_where" }, "macro.dbt_utils.default__test_sequential_values": { "arguments": [], - "created_at": 1696458269.892125, + "created_at": 1719485736.652041, "depends_on": { "macros": [ "macro.dbt_utils.slugify", @@ -8922,14 +11225,12 @@ "patch_path": null, "path": "macros/generic_tests/sequential_values.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__test_sequential_values" }, "macro.dbt_utils.default__test_unique_combination_of_columns": { "arguments": [], - "created_at": 1696458269.883317, + "created_at": 1719485736.643641, "depends_on": { "macros": [] }, @@ -8946,14 +11247,12 @@ "patch_path": null, "path": "macros/generic_tests/unique_combination_of_columns.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__test_unique_combination_of_columns" }, "macro.dbt_utils.default__union_relations": { "arguments": [], - "created_at": 1696458269.953401, + "created_at": 1719485736.6993861, "depends_on": { "macros": [ "macro.dbt_utils._is_relation", @@ -8975,14 +11274,12 @@ "patch_path": null, "path": "macros/sql/union.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__union_relations" }, "macro.dbt_utils.default__unpivot": { "arguments": [], - "created_at": 1696458269.940675, + "created_at": 1719485736.689354, "depends_on": { "macros": [ "macro.dbt_utils._is_relation", @@ -9004,14 +11301,12 @@ "patch_path": null, "path": "macros/sql/unpivot.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__unpivot" }, "macro.dbt_utils.default__width_bucket": { "arguments": [], - "created_at": 1696458269.981664, + "created_at": 1719485736.7257888, "depends_on": { "macros": [ "macro.dbt.safe_cast", @@ -9031,14 +11326,12 @@ "patch_path": null, "path": "macros/sql/width_bucket.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.default__width_bucket" }, "macro.dbt_utils.degrees_to_radians": { "arguments": [], - "created_at": 1696458269.991828, + "created_at": 1719485736.735244, "depends_on": { "macros": [] }, @@ -9055,14 +11348,12 @@ "patch_path": null, "path": "macros/sql/haversine_distance.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.degrees_to_radians" }, "macro.dbt_utils.generate_series": { "arguments": [], - "created_at": 1696458269.924184, + "created_at": 1719485736.678059, "depends_on": { "macros": [ "macro.dbt_utils.default__generate_series" @@ -9081,14 +11372,12 @@ "patch_path": null, "path": "macros/sql/generate_series.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.generate_series" }, "macro.dbt_utils.generate_surrogate_key": { "arguments": [], - "created_at": 1696458269.984725, + "created_at": 1719485736.7287118, "depends_on": { "macros": [ "macro.dbt_utils.default__generate_surrogate_key" @@ -9107,14 +11396,12 @@ "patch_path": null, "path": "macros/sql/generate_surrogate_key.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.generate_surrogate_key" }, "macro.dbt_utils.get_column_values": { "arguments": [], - "created_at": 1696458269.9700558, + "created_at": 1719485736.7158039, "depends_on": { "macros": [ "macro.dbt_utils.default__get_column_values" @@ -9133,14 +11420,12 @@ "patch_path": null, "path": "macros/sql/get_column_values.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.get_column_values" }, "macro.dbt_utils.get_filtered_columns_in_relation": { "arguments": [], - "created_at": 1696458269.977875, + "created_at": 1719485736.7211049, "depends_on": { "macros": [ "macro.dbt_utils.default__get_filtered_columns_in_relation" @@ -9159,14 +11444,12 @@ "patch_path": null, "path": "macros/sql/get_filtered_columns_in_relation.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.get_filtered_columns_in_relation" }, "macro.dbt_utils.get_intervals_between": { "arguments": [], - "created_at": 1696458269.913064, + "created_at": 1719485736.668108, "depends_on": { "macros": [ "macro.dbt_utils.default__get_intervals_between" @@ -9185,14 +11468,12 @@ "patch_path": null, "path": "macros/sql/date_spine.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.get_intervals_between" }, "macro.dbt_utils.get_powers_of_two": { "arguments": [], - "created_at": 1696458269.923075, + "created_at": 1719485736.677409, "depends_on": { "macros": [ "macro.dbt_utils.default__get_powers_of_two" @@ -9211,14 +11492,12 @@ "patch_path": null, "path": "macros/sql/generate_series.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.get_powers_of_two" }, "macro.dbt_utils.get_query_results_as_dict": { "arguments": [], - "created_at": 1696458269.982841, + "created_at": 1719485736.726866, "depends_on": { "macros": [ "macro.dbt_utils.default__get_query_results_as_dict" @@ -9237,14 +11516,12 @@ "patch_path": null, "path": "macros/sql/get_query_results_as_dict.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.get_query_results_as_dict" }, "macro.dbt_utils.get_relations_by_pattern": { "arguments": [], - "created_at": 1696458269.9195552, + "created_at": 1719485736.6725519, "depends_on": { "macros": [ "macro.dbt_utils.default__get_relations_by_pattern" @@ -9263,14 +11540,12 @@ "patch_path": null, "path": "macros/sql/get_relations_by_pattern.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.get_relations_by_pattern" }, "macro.dbt_utils.get_relations_by_prefix": { "arguments": [], - "created_at": 1696458269.926965, + "created_at": 1719485736.67938, "depends_on": { "macros": [ "macro.dbt_utils.default__get_relations_by_prefix" @@ -9289,14 +11564,12 @@ "patch_path": null, "path": "macros/sql/get_relations_by_prefix.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.get_relations_by_prefix" }, "macro.dbt_utils.get_single_value": { "arguments": [], - "created_at": 1696458269.9885209, + "created_at": 1719485736.732079, "depends_on": { "macros": [ "macro.dbt_utils.default__get_single_value" @@ -9315,14 +11588,12 @@ "patch_path": null, "path": "macros/sql/get_single_value.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.get_single_value" }, "macro.dbt_utils.get_table_types_sql": { "arguments": [], - "created_at": 1696458269.986836, + "created_at": 1719485736.730877, "depends_on": { "macros": [ "macro.dbt_utils.postgres__get_table_types_sql" @@ -9341,14 +11612,12 @@ "patch_path": null, "path": "macros/sql/get_table_types_sql.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.get_table_types_sql" }, "macro.dbt_utils.get_tables_by_pattern_sql": { "arguments": [], - "created_at": 1696458269.9652421, + "created_at": 1719485736.7118368, "depends_on": { "macros": [ "macro.dbt_utils.default__get_tables_by_pattern_sql" @@ -9367,14 +11636,12 @@ "patch_path": null, "path": "macros/sql/get_tables_by_pattern_sql.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.get_tables_by_pattern_sql" }, "macro.dbt_utils.get_tables_by_prefix_sql": { "arguments": [], - "created_at": 1696458269.929534, + "created_at": 1719485736.680733, "depends_on": { "macros": [ "macro.dbt_utils.default__get_tables_by_prefix_sql" @@ -9393,14 +11660,12 @@ "patch_path": null, "path": "macros/sql/get_tables_by_prefix_sql.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.get_tables_by_prefix_sql" }, "macro.dbt_utils.get_url_host": { "arguments": [], - "created_at": 1696458269.856265, + "created_at": 1719485736.6209002, "depends_on": { "macros": [ "macro.dbt_utils.default__get_url_host" @@ -9419,14 +11684,12 @@ "patch_path": null, "path": "macros/web/get_url_host.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.get_url_host" }, "macro.dbt_utils.get_url_parameter": { "arguments": [], - "created_at": 1696458269.860372, + "created_at": 1719485736.6233978, "depends_on": { "macros": [ "macro.dbt_utils.default__get_url_parameter" @@ -9445,14 +11708,12 @@ "patch_path": null, "path": "macros/web/get_url_parameter.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.get_url_parameter" }, "macro.dbt_utils.get_url_path": { "arguments": [], - "created_at": 1696458269.858241, + "created_at": 1719485736.62192, "depends_on": { "macros": [ "macro.dbt_utils.default__get_url_path" @@ -9471,14 +11732,12 @@ "patch_path": null, "path": "macros/web/get_url_path.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.get_url_path" }, "macro.dbt_utils.group_by": { "arguments": [], - "created_at": 1696458269.954188, + "created_at": 1719485736.700075, "depends_on": { "macros": [ "macro.dbt_utils.default__group_by" @@ -9497,14 +11756,12 @@ "patch_path": null, "path": "macros/sql/groupby.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.group_by" }, "macro.dbt_utils.haversine_distance": { "arguments": [], - "created_at": 1696458269.992314, + "created_at": 1719485736.7356942, "depends_on": { "macros": [ "macro.dbt_utils.default__haversine_distance" @@ -9523,14 +11780,12 @@ "patch_path": null, "path": "macros/sql/haversine_distance.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.haversine_distance" }, "macro.dbt_utils.log_info": { "arguments": [], - "created_at": 1696458269.90854, + "created_at": 1719485736.664978, "depends_on": { "macros": [ "macro.dbt_utils.default__log_info" @@ -9549,14 +11804,12 @@ "patch_path": null, "path": "macros/jinja_helpers/log_info.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.log_info" }, "macro.dbt_utils.nullcheck": { "arguments": [], - "created_at": 1696458269.961787, + "created_at": 1719485736.708089, "depends_on": { "macros": [ "macro.dbt_utils.default__nullcheck" @@ -9575,14 +11828,12 @@ "patch_path": null, "path": "macros/sql/nullcheck.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.nullcheck" }, "macro.dbt_utils.nullcheck_table": { "arguments": [], - "created_at": 1696458269.917547, + "created_at": 1719485736.671076, "depends_on": { "macros": [ "macro.dbt_utils.default__nullcheck_table" @@ -9601,14 +11852,12 @@ "patch_path": null, "path": "macros/sql/nullcheck_table.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.nullcheck_table" }, "macro.dbt_utils.pivot": { "arguments": [], - "created_at": 1696458269.975067, + "created_at": 1719485736.719448, "depends_on": { "macros": [ "macro.dbt_utils.default__pivot" @@ -9627,14 +11876,12 @@ "patch_path": null, "path": "macros/sql/pivot.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.pivot" }, "macro.dbt_utils.postgres__deduplicate": { "arguments": [], - "created_at": 1696458269.957185, + "created_at": 1719485736.703091, "depends_on": { "macros": [] }, @@ -9651,14 +11898,12 @@ "patch_path": null, "path": "macros/sql/deduplicate.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.postgres__deduplicate" }, "macro.dbt_utils.postgres__get_table_types_sql": { "arguments": [], - "created_at": 1696458269.9872892, + "created_at": 1719485736.731321, "depends_on": { "macros": [] }, @@ -9675,14 +11920,12 @@ "patch_path": null, "path": "macros/sql/get_table_types_sql.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.postgres__get_table_types_sql" }, "macro.dbt_utils.pretty_log_format": { "arguments": [], - "created_at": 1696458269.9058151, + "created_at": 1719485736.6625931, "depends_on": { "macros": [ "macro.dbt_utils.default__pretty_log_format" @@ -9701,14 +11944,12 @@ "patch_path": null, "path": "macros/jinja_helpers/pretty_log_format.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.pretty_log_format" }, "macro.dbt_utils.pretty_time": { "arguments": [], - "created_at": 1696458269.9076412, + "created_at": 1719485736.664063, "depends_on": { "macros": [ "macro.dbt_utils.default__pretty_time" @@ -9727,14 +11968,12 @@ "patch_path": null, "path": "macros/jinja_helpers/pretty_time.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.pretty_time" }, "macro.dbt_utils.redshift__deduplicate": { "arguments": [], - "created_at": 1696458269.9568481, + "created_at": 1719485736.7025979, "depends_on": { "macros": [ "macro.dbt_utils.default__deduplicate" @@ -9753,14 +11992,12 @@ "patch_path": null, "path": "macros/sql/deduplicate.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.redshift__deduplicate" }, "macro.dbt_utils.safe_add": { "arguments": [], - "created_at": 1696458269.95998, + "created_at": 1719485736.7061272, "depends_on": { "macros": [ "macro.dbt_utils.default__safe_add" @@ -9779,14 +12016,12 @@ "patch_path": null, "path": "macros/sql/safe_add.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.safe_add" }, "macro.dbt_utils.safe_divide": { "arguments": [], - "created_at": 1696458269.9414778, + "created_at": 1719485736.689705, "depends_on": { "macros": [ "macro.dbt_utils.default__safe_divide" @@ -9805,14 +12040,12 @@ "patch_path": null, "path": "macros/sql/safe_divide.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.safe_divide" }, "macro.dbt_utils.safe_subtract": { "arguments": [], - "created_at": 1696458269.915779, + "created_at": 1719485736.670065, "depends_on": { "macros": [ "macro.dbt_utils.default__safe_subtract" @@ -9831,14 +12064,12 @@ "patch_path": null, "path": "macros/sql/safe_subtract.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.safe_subtract" }, "macro.dbt_utils.slugify": { "arguments": [], - "created_at": 1696458269.9100301, + "created_at": 1719485736.6662788, "depends_on": { "macros": [] }, @@ -9855,14 +12086,12 @@ "patch_path": null, "path": "macros/jinja_helpers/slugify.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.slugify" }, "macro.dbt_utils.snowflake__deduplicate": { "arguments": [], - "created_at": 1696458269.95747, + "created_at": 1719485736.703385, "depends_on": { "macros": [] }, @@ -9879,14 +12108,12 @@ "patch_path": null, "path": "macros/sql/deduplicate.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.snowflake__deduplicate" }, "macro.dbt_utils.snowflake__width_bucket": { "arguments": [], - "created_at": 1696458269.9819782, + "created_at": 1719485736.72608, "depends_on": { "macros": [] }, @@ -9903,14 +12130,12 @@ "patch_path": null, "path": "macros/sql/width_bucket.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.snowflake__width_bucket" }, "macro.dbt_utils.star": { "arguments": [], - "created_at": 1696458269.9325912, + "created_at": 1719485736.6838112, "depends_on": { "macros": [ "macro.dbt_utils.default__star" @@ -9929,14 +12154,12 @@ "patch_path": null, "path": "macros/sql/star.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.star" }, "macro.dbt_utils.surrogate_key": { "arguments": [], - "created_at": 1696458269.9586718, + "created_at": 1719485736.704803, "depends_on": { "macros": [ "macro.dbt_utils.default__surrogate_key" @@ -9955,14 +12178,12 @@ "patch_path": null, "path": "macros/sql/surrogate_key.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.surrogate_key" }, "macro.dbt_utils.test_accepted_range": { "arguments": [], - "created_at": 1696458269.875644, + "created_at": 1719485736.6351202, "depends_on": { "macros": [ "macro.dbt_utils.default__test_accepted_range" @@ -9981,14 +12202,12 @@ "patch_path": null, "path": "macros/generic_tests/accepted_range.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.test_accepted_range" }, "macro.dbt_utils.test_at_least_one": { "arguments": [], - "created_at": 1696458269.8795462, + "created_at": 1719485736.637673, "depends_on": { "macros": [ "macro.dbt_utils.default__test_at_least_one" @@ -10007,14 +12226,12 @@ "patch_path": null, "path": "macros/generic_tests/at_least_one.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.test_at_least_one" }, "macro.dbt_utils.test_cardinality_equality": { "arguments": [], - "created_at": 1696458269.884375, + "created_at": 1719485736.64627, "depends_on": { "macros": [ "macro.dbt_utils.default__test_cardinality_equality" @@ -10033,14 +12250,12 @@ "patch_path": null, "path": "macros/generic_tests/cardinality_equality.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.test_cardinality_equality" }, "macro.dbt_utils.test_equal_rowcount": { "arguments": [], - "created_at": 1696458269.866045, + "created_at": 1719485736.6276648, "depends_on": { "macros": [ "macro.dbt_utils.default__test_equal_rowcount" @@ -10059,14 +12274,12 @@ "patch_path": null, "path": "macros/generic_tests/equal_rowcount.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.test_equal_rowcount" }, "macro.dbt_utils.test_equality": { "arguments": [], - "created_at": 1696458269.893547, + "created_at": 1719485736.6529899, "depends_on": { "macros": [ "macro.dbt_utils.default__test_equality" @@ -10085,14 +12298,12 @@ "patch_path": null, "path": "macros/generic_tests/equality.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.test_equality" }, "macro.dbt_utils.test_expression_is_true": { "arguments": [], - "created_at": 1696458269.885707, + "created_at": 1719485736.64721, "depends_on": { "macros": [ "macro.dbt_utils.default__test_expression_is_true" @@ -10111,14 +12322,12 @@ "patch_path": null, "path": "macros/generic_tests/expression_is_true.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.test_expression_is_true" }, "macro.dbt_utils.test_fewer_rows_than": { "arguments": [], - "created_at": 1696458269.862845, + "created_at": 1719485736.625401, "depends_on": { "macros": [ "macro.dbt_utils.default__test_fewer_rows_than" @@ -10137,14 +12346,12 @@ "patch_path": null, "path": "macros/generic_tests/fewer_rows_than.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.test_fewer_rows_than" }, "macro.dbt_utils.test_mutually_exclusive_ranges": { "arguments": [], - "created_at": 1696458269.902503, + "created_at": 1719485736.6603332, "depends_on": { "macros": [ "macro.dbt_utils.default__test_mutually_exclusive_ranges" @@ -10163,14 +12370,12 @@ "patch_path": null, "path": "macros/generic_tests/mutually_exclusive_ranges.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.test_mutually_exclusive_ranges" }, "macro.dbt_utils.test_not_accepted_values": { "arguments": [], - "created_at": 1696458269.877504, + "created_at": 1719485736.6365619, "depends_on": { "macros": [ "macro.dbt_utils.default__test_not_accepted_values" @@ -10189,14 +12394,12 @@ "patch_path": null, "path": "macros/generic_tests/not_accepted_values.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.test_not_accepted_values" }, "macro.dbt_utils.test_not_constant": { "arguments": [], - "created_at": 1696458269.873664, + "created_at": 1719485736.6339169, "depends_on": { "macros": [ "macro.dbt_utils.default__test_not_constant" @@ -10215,14 +12418,12 @@ "patch_path": null, "path": "macros/generic_tests/not_constant.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.test_not_constant" }, "macro.dbt_utils.test_not_empty_string": { "arguments": [], - "created_at": 1696458269.896234, + "created_at": 1719485736.655252, "depends_on": { "macros": [ "macro.dbt_utils.default__test_not_empty_string" @@ -10241,14 +12442,12 @@ "patch_path": null, "path": "macros/generic_tests/not_empty_string.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.test_not_empty_string" }, "macro.dbt_utils.test_not_null_proportion": { "arguments": [], - "created_at": 1696458269.88746, + "created_at": 1719485736.64849, "depends_on": { "macros": [ "macro.dbt_utils.default__test_not_null_proportion" @@ -10267,14 +12466,12 @@ "patch_path": null, "path": "macros/generic_tests/not_null_proportion.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.test_not_null_proportion" }, "macro.dbt_utils.test_recency": { "arguments": [], - "created_at": 1696458269.8711538, + "created_at": 1719485736.632009, "depends_on": { "macros": [ "macro.dbt_utils.default__test_recency" @@ -10293,14 +12490,12 @@ "patch_path": null, "path": "macros/generic_tests/recency.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.test_recency" }, "macro.dbt_utils.test_relationships_where": { "arguments": [], - "created_at": 1696458269.868972, + "created_at": 1719485736.6303911, "depends_on": { "macros": [ "macro.dbt_utils.default__test_relationships_where" @@ -10319,14 +12514,12 @@ "patch_path": null, "path": "macros/generic_tests/relationships_where.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.test_relationships_where" }, "macro.dbt_utils.test_sequential_values": { "arguments": [], - "created_at": 1696458269.890569, + "created_at": 1719485736.650635, "depends_on": { "macros": [ "macro.dbt_utils.default__test_sequential_values" @@ -10345,14 +12538,12 @@ "patch_path": null, "path": "macros/generic_tests/sequential_values.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.test_sequential_values" }, "macro.dbt_utils.test_unique_combination_of_columns": { "arguments": [], - "created_at": 1696458269.882238, + "created_at": 1719485736.642268, "depends_on": { "macros": [ "macro.dbt_utils.default__test_unique_combination_of_columns" @@ -10371,14 +12562,12 @@ "patch_path": null, "path": "macros/generic_tests/unique_combination_of_columns.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.test_unique_combination_of_columns" }, "macro.dbt_utils.union_relations": { "arguments": [], - "created_at": 1696458269.9474702, + "created_at": 1719485736.6939468, "depends_on": { "macros": [ "macro.dbt_utils.default__union_relations" @@ -10397,14 +12586,12 @@ "patch_path": null, "path": "macros/sql/union.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.union_relations" }, "macro.dbt_utils.unpivot": { "arguments": [], - "created_at": 1696458269.937675, + "created_at": 1719485736.68751, "depends_on": { "macros": [ "macro.dbt_utils.default__unpivot" @@ -10423,14 +12610,12 @@ "patch_path": null, "path": "macros/sql/unpivot.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.unpivot" }, "macro.dbt_utils.width_bucket": { "arguments": [], - "created_at": 1696458269.980886, + "created_at": 1719485736.724451, "depends_on": { "macros": [ "macro.dbt_utils.default__width_bucket" @@ -10449,14 +12634,12 @@ "patch_path": null, "path": "macros/sql/width_bucket.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop/dbt_packages/dbt_utils", "supported_languages": null, - "tags": [], "unique_id": "macro.dbt_utils.width_bucket" }, - "macro.jaffle_shop.drop_table": { + "macro.jaffle_shop.drop_table_by_name": { "arguments": [], - "created_at": 1696458269.549238, + "created_at": 1719485736.288547, "depends_on": { "macros": [ "macro.dbt.run_query" @@ -10467,42 +12650,43 @@ "node_color": null, "show": true }, - "macro_sql": "{%- macro drop_table(table_name) -%}\n {%- set drop_query -%}\n DROP TABLE IF EXISTS {{ target.schema }}.{{ table_name }} CASCADE\n {%- endset -%}\n {% do run_query(drop_query) %}\n{%- endmacro -%}", + "macro_sql": "{%- macro drop_table_by_name(table_name) -%}\n {%- set drop_query -%}\n DROP TABLE IF EXISTS {{ target.schema }}.{{ table_name }} CASCADE\n {%- endset -%}\n {% do run_query(drop_query) %}\n{%- endmacro -%}", "meta": {}, - "name": "drop_table", + "name": "drop_table_by_name", "original_file_path": "macros/drop_table.sql", "package_name": "jaffle_shop", "patch_path": null, "path": "macros/drop_table.sql", "resource_type": "macro", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "supported_languages": null, - "tags": [], - "unique_id": "macro.jaffle_shop.drop_table" + "unique_id": "macro.jaffle_shop.drop_table_by_name" } }, "metadata": { "adapter_type": "postgres", - "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v7.json", - "dbt_version": "1.3.1", + "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v12.json", + "dbt_version": "1.8.0", "env": {}, - "generated_at": "2023-10-04T22:24:29.500301Z", - "invocation_id": "fc9520c6-dfc1-4650-9483-1fd3751dfe42", + "generated_at": "2024-06-27T10:55:36.063508Z", + "invocation_id": "31cdaee6-885b-4bdf-b794-4065f9530edb", "project_id": "06e5b98c2db46f8a72cc4f66410e9b3b", + "project_name": "jaffle_shop", "send_anonymous_usage_stats": true, - "user_id": "43f90c72-db98-41b6-8fce-1337e4d59f98" + "user_id": "f5b1bc43-6cc6-4fd4-849c-18b31ffa1e2d" }, "metrics": {}, "nodes": { "model.jaffle_shop.customers": { + "access": "protected", "alias": "customers", "build_path": null, "checksum": { - "checksum": "455b90a31f418ae776213ad9932c7cb72d19a5269a8c722bd9f4e44957313ce8", + "checksum": "60bd72e33da43fff3a7e7609135c17cd4468bd22afec0735dd36018bfb5af30a", "name": "sha256" }, "columns": { "customer_id": { + "constraints": [], "data_type": null, "description": "This is a unique identifier for a customer", "meta": {}, @@ -10511,6 +12695,7 @@ "tags": [] }, "first_name": { + "constraints": [], "data_type": null, "description": "Customer's first name. PII.", "meta": {}, @@ -10519,6 +12704,7 @@ "tags": [] }, "first_order": { + "constraints": [], "data_type": null, "description": "Date (UTC) of a customer's first order", "meta": {}, @@ -10527,6 +12713,7 @@ "tags": [] }, "last_name": { + "constraints": [], "data_type": null, "description": "Customer's last name. PII.", "meta": {}, @@ -10535,6 +12722,7 @@ "tags": [] }, "most_recent_order": { + "constraints": [], "data_type": null, "description": "Date (UTC) of a customer's most recent order", "meta": {}, @@ -10543,6 +12731,7 @@ "tags": [] }, "number_of_orders": { + "constraints": [], "data_type": null, "description": "Count of the number of orders a customer has placed", "meta": {}, @@ -10551,6 +12740,7 @@ "tags": [] }, "total_order_amount": { + "constraints": [], "data_type": null, "description": "Total value (AUD) of a customer's orders", "meta": {}, @@ -10561,8 +12751,13 @@ }, "compiled_path": null, "config": { + "access": "protected", "alias": null, "column_types": {}, + "contract": { + "alias_types": true, + "enforced": false + }, "database": null, "docs": { "node_color": null, @@ -10571,9 +12766,11 @@ "enabled": true, "full_refresh": null, "grants": {}, + "group": null, "incremental_strategy": null, "materialized": "table", "meta": {}, + "on_configuration_change": "apply", "on_schema_change": "ignore", "packages": [], "persist_docs": {}, @@ -10584,9 +12781,14 @@ "tags": [], "unique_key": null }, - "created_at": 1696458270.331094, + "constraints": [], + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.5078778, "database": "postgres", - "deferred": false, "depends_on": { "macros": [], "nodes": [ @@ -10595,6 +12797,7 @@ "model.jaffle_shop.stg_payments" ] }, + "deprecation_date": null, "description": "This table has basic information about a customer, as well as some derived facts based on a customer's orders", "docs": { "node_color": null, @@ -10604,7 +12807,9 @@ "jaffle_shop", "customers" ], + "group": null, "language": "sql", + "latest_version": null, "meta": {}, "metrics": [], "name": "customers", @@ -10614,35 +12819,44 @@ "path": "customers.sql", "raw_code": "with customers as (\n\n select * from {{ ref('stg_customers') }}\n\n),\n\norders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\ncustomer_orders as (\n\n select\n customer_id,\n\n min(order_date) as first_order,\n max(order_date) as most_recent_order,\n count(order_id) as number_of_orders\n from orders\n\n group by customer_id\n\n),\n\ncustomer_payments as (\n\n select\n orders.customer_id,\n sum(amount) as total_amount\n\n from payments\n\n left join orders on\n payments.order_id = orders.order_id\n\n group by orders.customer_id\n\n),\n\nfinal as (\n\n select\n customers.customer_id,\n customers.first_name,\n customers.last_name,\n customer_orders.first_order,\n customer_orders.most_recent_order,\n customer_orders.number_of_orders,\n customer_payments.total_amount as customer_lifetime_value\n\n from customers\n\n left join customer_orders\n on customers.customer_id = customer_orders.customer_id\n\n left join customer_payments\n on customers.customer_id = customer_payments.customer_id\n\n)\n\nselect * from final", "refs": [ - [ - "stg_customers" - ], - [ - "stg_orders" - ], - [ - "stg_payments" - ] + { + "name": "stg_customers", + "package": null, + "version": null + }, + { + "name": "stg_orders", + "package": null, + "version": null + }, + { + "name": "stg_payments", + "package": null, + "version": null + } ], + "relation_name": "\"postgres\".\"public\".\"customers\"", "resource_type": "model", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public", "sources": [], "tags": [], "unique_id": "model.jaffle_shop.customers", "unrendered_config": { "materialized": "table" - } + }, + "version": null }, "model.jaffle_shop.orders": { + "access": "protected", "alias": "orders", "build_path": null, "checksum": { - "checksum": "53950235d8e29690d259e95ee49bda6a5b7911b44c739b738a646dc6014bcfcd", + "checksum": "27f8c79aad1cfd8411ab9c3d2ce8da1d787f7f05c58bbee1d247510dc426be0f", "name": "sha256" }, "columns": { "amount": { + "constraints": [], "data_type": null, "description": "Total amount (AUD) of the order", "meta": {}, @@ -10651,6 +12865,7 @@ "tags": [] }, "bank_transfer_amount": { + "constraints": [], "data_type": null, "description": "Amount of the order (AUD) paid for by bank transfer", "meta": {}, @@ -10659,6 +12874,7 @@ "tags": [] }, "coupon_amount": { + "constraints": [], "data_type": null, "description": "Amount of the order (AUD) paid for by coupon", "meta": {}, @@ -10667,6 +12883,7 @@ "tags": [] }, "credit_card_amount": { + "constraints": [], "data_type": null, "description": "Amount of the order (AUD) paid for by credit card", "meta": {}, @@ -10675,6 +12892,7 @@ "tags": [] }, "customer_id": { + "constraints": [], "data_type": null, "description": "Foreign key to the customers table", "meta": {}, @@ -10683,6 +12901,7 @@ "tags": [] }, "gift_card_amount": { + "constraints": [], "data_type": null, "description": "Amount of the order (AUD) paid for by gift card", "meta": {}, @@ -10691,6 +12910,7 @@ "tags": [] }, "order_date": { + "constraints": [], "data_type": null, "description": "Date (UTC) that the order was placed", "meta": {}, @@ -10699,6 +12919,7 @@ "tags": [] }, "order_id": { + "constraints": [], "data_type": null, "description": "This is a unique identifier for an order", "meta": {}, @@ -10707,6 +12928,7 @@ "tags": [] }, "status": { + "constraints": [], "data_type": null, "description": "Orders can be one of the following statuses:\n\n| status | description |\n|----------------|------------------------------------------------------------------------------------------------------------------------|\n| placed | The order has been placed but has not yet left the warehouse |\n| shipped | The order has ben shipped to the customer and is currently in transit |\n| completed | The order has been received by the customer |\n| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse |\n| returned | The order has been returned by the customer and received at the warehouse |", "meta": {}, @@ -10717,8 +12939,13 @@ }, "compiled_path": null, "config": { + "access": "protected", "alias": null, "column_types": {}, + "contract": { + "alias_types": true, + "enforced": false + }, "database": null, "docs": { "node_color": null, @@ -10727,9 +12954,11 @@ "enabled": true, "full_refresh": null, "grants": {}, + "group": null, "incremental_strategy": null, "materialized": "table", "meta": {}, + "on_configuration_change": "apply", "on_schema_change": "ignore", "packages": [], "persist_docs": {}, @@ -10740,9 +12969,14 @@ "tags": [], "unique_key": null }, - "created_at": 1696458270.3345122, + "constraints": [], + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.511349, "database": "postgres", - "deferred": false, "depends_on": { "macros": [], "nodes": [ @@ -10750,6 +12984,7 @@ "model.jaffle_shop.stg_payments" ] }, + "deprecation_date": null, "description": "This table has basic information about orders, as well as some derived facts based on payments", "docs": { "node_color": null, @@ -10759,7 +12994,9 @@ "jaffle_shop", "orders" ], + "group": null, "language": "sql", + "latest_version": null, "meta": {}, "metrics": [], "name": "orders", @@ -10769,32 +13006,39 @@ "path": "orders.sql", "raw_code": "{% set payment_methods = ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] %}\n\nwith orders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\norder_payments as (\n\n select\n order_id,\n\n {% for payment_method in payment_methods -%}\n sum(case when payment_method = '{{ payment_method }}' then amount else 0 end) as {{ payment_method }}_amount,\n {% endfor -%}\n\n sum(amount) as total_amount\n\n from payments\n\n group by order_id\n\n),\n\nfinal as (\n\n select\n orders.order_id,\n orders.customer_id,\n orders.order_date,\n orders.status,\n\n {% for payment_method in payment_methods -%}\n\n order_payments.{{ payment_method }}_amount,\n\n {% endfor -%}\n\n order_payments.total_amount as amount\n\n from orders\n\n\n left join order_payments\n on orders.order_id = order_payments.order_id\n\n)\n\nselect * from final", "refs": [ - [ - "stg_orders" - ], - [ - "stg_payments" - ] + { + "name": "stg_orders", + "package": null, + "version": null + }, + { + "name": "stg_payments", + "package": null, + "version": null + } ], + "relation_name": "\"postgres\".\"public\".\"orders\"", "resource_type": "model", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public", "sources": [], "tags": [], "unique_id": "model.jaffle_shop.orders", "unrendered_config": { "materialized": "table" - } + }, + "version": null }, "model.jaffle_shop.stg_customers": { + "access": "protected", "alias": "stg_customers", "build_path": null, "checksum": { - "checksum": "6f18a29204dad1de6dbb0c288144c4990742e0a1e065c3b2a67b5f98334c22ba", + "checksum": "80e3223cd54387e11fa16cd0f4cbe15f8ff74dcd9900b93856b9e39416178c9d", "name": "sha256" }, "columns": { "customer_id": { + "constraints": [], "data_type": null, "description": "", "meta": {}, @@ -10805,8 +13049,13 @@ }, "compiled_path": null, "config": { + "access": "protected", "alias": null, "column_types": {}, + "contract": { + "alias_types": true, + "enforced": false + }, "database": null, "docs": { "node_color": null, @@ -10815,9 +13064,11 @@ "enabled": true, "full_refresh": null, "grants": {}, + "group": null, "incremental_strategy": null, "materialized": "view", "meta": {}, + "on_configuration_change": "apply", "on_schema_change": "ignore", "packages": [], "persist_docs": {}, @@ -10828,15 +13079,21 @@ "tags": [], "unique_key": null }, - "created_at": 1696458270.372876, + "constraints": [], + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.629529, "database": "postgres", - "deferred": false, "depends_on": { "macros": [], "nodes": [ "seed.jaffle_shop.raw_customers" ] }, + "deprecation_date": null, "description": "", "docs": { "node_color": null, @@ -10847,7 +13104,9 @@ "staging", "stg_customers" ], + "group": null, "language": "sql", + "latest_version": null, "meta": {}, "metrics": [], "name": "stg_customers", @@ -10857,29 +13116,34 @@ "path": "staging/stg_customers.sql", "raw_code": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_customers') }}\n\n),\n\nrenamed as (\n\n select\n id as customer_id,\n first_name,\n last_name\n\n from source\n\n)\n\nselect * from renamed", "refs": [ - [ - "raw_customers" - ] + { + "name": "raw_customers", + "package": null, + "version": null + } ], + "relation_name": "\"postgres\".\"public\".\"stg_customers\"", "resource_type": "model", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public", "sources": [], "tags": [], "unique_id": "model.jaffle_shop.stg_customers", "unrendered_config": { "materialized": "view" - } + }, + "version": null }, "model.jaffle_shop.stg_orders": { + "access": "protected", "alias": "stg_orders", "build_path": null, "checksum": { - "checksum": "afffa9cbc57e5fd2cf5898ebf571d444a62c9d6d7929d8133d30567fb9a2ce97", + "checksum": "f4f881cb09d2c4162200fc331d7401df6d1abd4fed492554a7db70dede347108", "name": "sha256" }, "columns": { "order_id": { + "constraints": [], "data_type": null, "description": "", "meta": {}, @@ -10888,6 +13152,7 @@ "tags": [] }, "status": { + "constraints": [], "data_type": null, "description": "", "meta": {}, @@ -10898,8 +13163,13 @@ }, "compiled_path": null, "config": { + "access": "protected", "alias": null, "column_types": {}, + "contract": { + "alias_types": true, + "enforced": false + }, "database": null, "docs": { "node_color": null, @@ -10908,9 +13178,11 @@ "enabled": true, "full_refresh": null, "grants": {}, + "group": null, "incremental_strategy": null, "materialized": "view", "meta": {}, + "on_configuration_change": "apply", "on_schema_change": "ignore", "packages": [], "persist_docs": {}, @@ -10921,15 +13193,21 @@ "tags": [], "unique_key": null }, - "created_at": 1696458270.374181, + "constraints": [], + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.630356, "database": "postgres", - "deferred": false, "depends_on": { "macros": [], "nodes": [ "seed.jaffle_shop.raw_orders" ] }, + "deprecation_date": null, "description": "", "docs": { "node_color": null, @@ -10940,7 +13218,9 @@ "staging", "stg_orders" ], + "group": null, "language": "sql", + "latest_version": null, "meta": {}, "metrics": [], "name": "stg_orders", @@ -10950,29 +13230,34 @@ "path": "staging/stg_orders.sql", "raw_code": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_orders') }}\n\n),\n\nrenamed as (\n\n select\n id as order_id,\n user_id as customer_id,\n order_date,\n status\n\n from source\n\n)\n\nselect * from renamed", "refs": [ - [ - "raw_orders" - ] + { + "name": "raw_orders", + "package": null, + "version": null + } ], + "relation_name": "\"postgres\".\"public\".\"stg_orders\"", "resource_type": "model", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public", "sources": [], "tags": [], "unique_id": "model.jaffle_shop.stg_orders", "unrendered_config": { "materialized": "view" - } + }, + "version": null }, "model.jaffle_shop.stg_payments": { + "access": "protected", "alias": "stg_payments", "build_path": null, "checksum": { - "checksum": "a626f1ad5883c5391dc123be01f81491be5f496654d226d876651a0b8c036362", + "checksum": "30f346f66ef7bca4c8865a471086303720c3daab58870c805b6f45e92d19fd65", "name": "sha256" }, "columns": { "payment_id": { + "constraints": [], "data_type": null, "description": "", "meta": {}, @@ -10981,6 +13266,7 @@ "tags": [] }, "payment_method": { + "constraints": [], "data_type": null, "description": "", "meta": {}, @@ -10991,8 +13277,13 @@ }, "compiled_path": null, "config": { + "access": "protected", "alias": null, "column_types": {}, + "contract": { + "alias_types": true, + "enforced": false + }, "database": null, "docs": { "node_color": null, @@ -11001,9 +13292,11 @@ "enabled": true, "full_refresh": null, "grants": {}, + "group": null, "incremental_strategy": null, "materialized": "view", "meta": {}, + "on_configuration_change": "apply", "on_schema_change": "ignore", "packages": [], "persist_docs": {}, @@ -11014,15 +13307,21 @@ "tags": [], "unique_key": null }, - "created_at": 1696458270.375178, + "constraints": [], + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.6316102, "database": "postgres", - "deferred": false, "depends_on": { "macros": [], "nodes": [ "seed.jaffle_shop.raw_payments" ] }, + "deprecation_date": null, "description": "", "docs": { "node_color": null, @@ -11033,7 +13332,9 @@ "staging", "stg_payments" ], + "group": null, "language": "sql", + "latest_version": null, "meta": {}, "metrics": [], "name": "stg_payments", @@ -11043,33 +13344,40 @@ "path": "staging/stg_payments.sql", "raw_code": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_payments') }}\n\n),\n\nrenamed as (\n\n select\n id as payment_id,\n order_id,\n payment_method,\n\n -- `amount` is currently stored in cents, so we convert it to dollars\n amount / 100 as amount\n\n from source\n\n)\n\nselect * from renamed", "refs": [ - [ - "raw_payments" - ] + { + "name": "raw_payments", + "package": null, + "version": null + } ], + "relation_name": "\"postgres\".\"public\".\"stg_payments\"", "resource_type": "model", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public", "sources": [], "tags": [], "unique_id": "model.jaffle_shop.stg_payments", "unrendered_config": { "materialized": "view" - } + }, + "version": null }, "seed.jaffle_shop.raw_customers": { "alias": "raw_customers", "build_path": null, "checksum": { - "checksum": "24579b4b26098d43265376f3c50be8b10faf8e8fd95f5508074f10f76a12671d", + "checksum": "357d173dda65a741ad97d6683502286cc2655bb396ab5f4dfad12b8c39bd2a63", "name": "sha256" }, "columns": {}, - "compiled_path": null, "config": { "alias": null, "column_types": {}, + "contract": { + "alias_types": true, + "enforced": false + }, "database": null, + "delimiter": ",", "docs": { "node_color": null, "show": true @@ -11077,9 +13385,11 @@ "enabled": true, "full_refresh": null, "grants": {}, + "group": null, "incremental_strategy": null, "materialized": "seed", "meta": {}, + "on_configuration_change": "apply", "on_schema_change": "ignore", "packages": [], "persist_docs": {}, @@ -11091,12 +13401,10 @@ "tags": [], "unique_key": null }, - "created_at": 1696458270.302308, + "created_at": 1719485737.409044, "database": "postgres", - "deferred": false, "depends_on": { - "macros": [], - "nodes": [] + "macros": [] }, "description": "", "docs": { @@ -11107,20 +13415,18 @@ "jaffle_shop", "raw_customers" ], - "language": "sql", + "group": null, "meta": {}, - "metrics": [], "name": "raw_customers", "original_file_path": "seeds/raw_customers.csv", "package_name": "jaffle_shop", "patch_path": null, "path": "raw_customers.csv", "raw_code": "", - "refs": [], + "relation_name": "\"postgres\".\"public\".\"raw_customers\"", "resource_type": "seed", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", + "root_path": "/Users/tati/Code/cosmos-clean/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public", - "sources": [], "tags": [], "unique_id": "seed.jaffle_shop.raw_customers", "unrendered_config": {} @@ -11129,15 +13435,19 @@ "alias": "raw_orders", "build_path": null, "checksum": { - "checksum": "c5f309d84ba32f2a39235c59f2d4f6c855aedba7e215847c957f1a5f2fa80d3e", + "checksum": "6228dde8e17b9621f35c13e272ec67d3ff55b55499433f47d303adf2be72c17f", "name": "sha256" }, "columns": {}, - "compiled_path": null, "config": { "alias": null, "column_types": {}, + "contract": { + "alias_types": true, + "enforced": false + }, "database": null, + "delimiter": ",", "docs": { "node_color": null, "show": true @@ -11145,9 +13455,11 @@ "enabled": true, "full_refresh": null, "grants": {}, + "group": null, "incremental_strategy": null, "materialized": "seed", "meta": {}, + "on_configuration_change": "apply", "on_schema_change": "ignore", "packages": [], "persist_docs": {}, @@ -11159,12 +13471,10 @@ "tags": [], "unique_key": null }, - "created_at": 1696458270.3040352, + "created_at": 1719485737.411613, "database": "postgres", - "deferred": false, "depends_on": { - "macros": [], - "nodes": [] + "macros": [] }, "description": "", "docs": { @@ -11175,20 +13485,18 @@ "jaffle_shop", "raw_orders" ], - "language": "sql", + "group": null, "meta": {}, - "metrics": [], "name": "raw_orders", "original_file_path": "seeds/raw_orders.csv", "package_name": "jaffle_shop", "patch_path": null, "path": "raw_orders.csv", "raw_code": "", - "refs": [], + "relation_name": "\"postgres\".\"public\".\"raw_orders\"", "resource_type": "seed", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", + "root_path": "/Users/tati/Code/cosmos-clean/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public", - "sources": [], "tags": [], "unique_id": "seed.jaffle_shop.raw_orders", "unrendered_config": {} @@ -11197,15 +13505,19 @@ "alias": "raw_payments", "build_path": null, "checksum": { - "checksum": "03fd407f3135f84456431a923f22fc185a2154079e210c20b690e3ab11687d11", + "checksum": "6de0626a8db9c1750eefd1b2e17fac4c2a4b9f778eb50532d8b377b90de395e6", "name": "sha256" }, "columns": {}, - "compiled_path": null, "config": { "alias": null, "column_types": {}, + "contract": { + "alias_types": true, + "enforced": false + }, "database": null, + "delimiter": ",", "docs": { "node_color": null, "show": true @@ -11213,9 +13525,11 @@ "enabled": true, "full_refresh": null, "grants": {}, + "group": null, "incremental_strategy": null, "materialized": "seed", "meta": {}, + "on_configuration_change": "apply", "on_schema_change": "ignore", "packages": [], "persist_docs": {}, @@ -11227,12 +13541,10 @@ "tags": [], "unique_key": null }, - "created_at": 1696458270.30559, + "created_at": 1719485737.4141219, "database": "postgres", - "deferred": false, "depends_on": { - "macros": [], - "nodes": [] + "macros": [] }, "description": "", "docs": { @@ -11243,26 +13555,25 @@ "jaffle_shop", "raw_payments" ], - "language": "sql", + "group": null, "meta": {}, - "metrics": [], "name": "raw_payments", "original_file_path": "seeds/raw_payments.csv", "package_name": "jaffle_shop", "patch_path": null, "path": "raw_payments.csv", "raw_code": "", - "refs": [], + "relation_name": "\"postgres\".\"public\".\"raw_payments\"", "resource_type": "seed", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", + "root_path": "/Users/tati/Code/cosmos-clean/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public", - "sources": [], "tags": [], "unique_id": "seed.jaffle_shop.raw_payments", "unrendered_config": {} }, "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3": { "alias": "accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758", + "attached_node": "model.jaffle_shop.orders", "build_path": null, "checksum": { "checksum": "", @@ -11277,19 +13588,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.35778, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.607407, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_accepted_values", @@ -11309,6 +13626,7 @@ "jaffle_shop", "accepted_values_orders_status__placed__shipped__completed__return_pending__returned" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -11319,12 +13637,14 @@ "path": "accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758.sql", "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758\") }}", "refs": [ - [ - "orders" - ] + { + "name": "orders", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -11350,6 +13670,7 @@ }, "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad": { "alias": "accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58", + "attached_node": "model.jaffle_shop.stg_orders", "build_path": null, "checksum": { "checksum": "", @@ -11364,19 +13685,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.381258, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.637758, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_accepted_values", @@ -11397,6 +13724,7 @@ "staging", "accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -11407,12 +13735,14 @@ "path": "accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58.sql", "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58\") }}", "refs": [ - [ - "stg_orders" - ] + { + "name": "stg_orders", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -11438,6 +13768,7 @@ }, "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278": { "alias": "accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef", + "attached_node": "model.jaffle_shop.stg_payments", "build_path": null, "checksum": { "checksum": "", @@ -11452,19 +13783,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.3883939, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.6424909, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_accepted_values", @@ -11485,6 +13822,7 @@ "staging", "accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -11495,12 +13833,14 @@ "path": "accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef.sql", "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef\") }}", "refs": [ - [ - "stg_payments" - ] + { + "name": "stg_payments", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -11525,6 +13865,7 @@ }, "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d": { "alias": "not_null_customers_customer_id", + "attached_node": "model.jaffle_shop.customers", "build_path": null, "checksum": { "checksum": "", @@ -11539,19 +13880,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.342222, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.5844128, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_not_null" @@ -11570,6 +13917,7 @@ "jaffle_shop", "not_null_customers_customer_id" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -11580,12 +13928,14 @@ "path": "not_null_customers_customer_id.sql", "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "refs": [ - [ - "customers" - ] + { + "name": "customers", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -11602,6 +13952,7 @@ }, "test.jaffle_shop.not_null_orders_amount.106140f9fd": { "alias": "not_null_orders_amount", + "attached_node": "model.jaffle_shop.orders", "build_path": null, "checksum": { "checksum": "", @@ -11616,19 +13967,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.365846, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.6199622, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_not_null" @@ -11647,6 +14004,7 @@ "jaffle_shop", "not_null_orders_amount" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -11657,12 +14015,14 @@ "path": "not_null_orders_amount.sql", "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "refs": [ - [ - "orders" - ] + { + "name": "orders", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -11679,6 +14039,7 @@ }, "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49": { "alias": "not_null_orders_bank_transfer_amount", + "attached_node": "model.jaffle_shop.orders", "build_path": null, "checksum": { "checksum": "", @@ -11693,19 +14054,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.369942, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.6258051, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_not_null" @@ -11724,6 +14091,7 @@ "jaffle_shop", "not_null_orders_bank_transfer_amount" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -11734,12 +14102,14 @@ "path": "not_null_orders_bank_transfer_amount.sql", "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "refs": [ - [ - "orders" - ] + { + "name": "orders", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -11756,6 +14126,7 @@ }, "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625": { "alias": "not_null_orders_coupon_amount", + "attached_node": "model.jaffle_shop.orders", "build_path": null, "checksum": { "checksum": "", @@ -11770,19 +14141,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.368623, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.624228, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_not_null" @@ -11801,6 +14178,7 @@ "jaffle_shop", "not_null_orders_coupon_amount" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -11811,12 +14189,14 @@ "path": "not_null_orders_coupon_amount.sql", "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "refs": [ - [ - "orders" - ] + { + "name": "orders", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -11833,6 +14213,7 @@ }, "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59": { "alias": "not_null_orders_credit_card_amount", + "attached_node": "model.jaffle_shop.orders", "build_path": null, "checksum": { "checksum": "", @@ -11847,19 +14228,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.367114, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.622118, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_not_null" @@ -11878,6 +14265,7 @@ "jaffle_shop", "not_null_orders_credit_card_amount" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -11888,12 +14276,14 @@ "path": "not_null_orders_credit_card_amount.sql", "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "refs": [ - [ - "orders" - ] + { + "name": "orders", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -11910,6 +14300,7 @@ }, "test.jaffle_shop.not_null_orders_customer_id.c5f02694af": { "alias": "not_null_orders_customer_id", + "attached_node": "model.jaffle_shop.orders", "build_path": null, "checksum": { "checksum": "", @@ -11924,19 +14315,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.345952, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.587272, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_not_null" @@ -11955,6 +14352,7 @@ "jaffle_shop", "not_null_orders_customer_id" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -11965,12 +14363,14 @@ "path": "not_null_orders_customer_id.sql", "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "refs": [ - [ - "orders" - ] + { + "name": "orders", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -11987,6 +14387,7 @@ }, "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a": { "alias": "not_null_orders_gift_card_amount", + "attached_node": "model.jaffle_shop.orders", "build_path": null, "checksum": { "checksum": "", @@ -12001,19 +14402,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.371219, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.6277661, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_not_null" @@ -12032,6 +14439,7 @@ "jaffle_shop", "not_null_orders_gift_card_amount" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -12042,12 +14450,14 @@ "path": "not_null_orders_gift_card_amount.sql", "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "refs": [ - [ - "orders" - ] + { + "name": "orders", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -12064,6 +14474,7 @@ }, "test.jaffle_shop.not_null_orders_order_id.cf6c17daed": { "alias": "not_null_orders_order_id", + "attached_node": "model.jaffle_shop.orders", "build_path": null, "checksum": { "checksum": "", @@ -12078,19 +14489,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.344596, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.586345, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_not_null" @@ -12109,6 +14526,7 @@ "jaffle_shop", "not_null_orders_order_id" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -12119,12 +14537,14 @@ "path": "not_null_orders_order_id.sql", "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "refs": [ - [ - "orders" - ] + { + "name": "orders", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -12141,6 +14561,7 @@ }, "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa": { "alias": "not_null_stg_customers_customer_id", + "attached_node": "model.jaffle_shop.stg_customers", "build_path": null, "checksum": { "checksum": "", @@ -12155,19 +14576,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.3773491, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.633585, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_not_null" @@ -12187,6 +14614,7 @@ "staging", "not_null_stg_customers_customer_id" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -12197,12 +14625,14 @@ "path": "not_null_stg_customers_customer_id.sql", "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "refs": [ - [ - "stg_customers" - ] + { + "name": "stg_customers", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -12219,6 +14649,7 @@ }, "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64": { "alias": "not_null_stg_orders_order_id", + "attached_node": "model.jaffle_shop.stg_orders", "build_path": null, "checksum": { "checksum": "", @@ -12233,19 +14664,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.380042, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.6366348, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_not_null" @@ -12265,6 +14702,7 @@ "staging", "not_null_stg_orders_order_id" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -12275,12 +14713,14 @@ "path": "not_null_stg_orders_order_id.sql", "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "refs": [ - [ - "stg_orders" - ] + { + "name": "stg_orders", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -12297,6 +14737,7 @@ }, "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075": { "alias": "not_null_stg_payments_payment_id", + "attached_node": "model.jaffle_shop.stg_payments", "build_path": null, "checksum": { "checksum": "", @@ -12311,19 +14752,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.387188, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.641508, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_not_null" @@ -12343,6 +14790,7 @@ "staging", "not_null_stg_payments_payment_id" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -12353,12 +14801,14 @@ "path": "not_null_stg_payments_payment_id.sql", "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "refs": [ - [ - "stg_payments" - ] + { + "name": "stg_payments", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -12375,6 +14825,7 @@ }, "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2": { "alias": "relationships_orders_customer_id__customer_id__ref_customers_", + "attached_node": "model.jaffle_shop.orders", "build_path": null, "checksum": { "checksum": "", @@ -12389,19 +14840,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.347137, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.590184, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_relationships", @@ -12422,6 +14879,7 @@ "jaffle_shop", "relationships_orders_customer_id__customer_id__ref_customers_" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -12432,15 +14890,19 @@ "path": "relationships_orders_customer_id__customer_id__ref_customers_.sql", "raw_code": "{{ test_relationships(**_dbt_generic_test_kwargs) }}", "refs": [ - [ - "customers" - ], - [ - "orders" - ] + { + "name": "customers", + "package": null, + "version": null + }, + { + "name": "orders", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -12459,6 +14921,7 @@ }, "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1": { "alias": "unique_customers_customer_id", + "attached_node": "model.jaffle_shop.customers", "build_path": null, "checksum": { "checksum": "", @@ -12473,19 +14936,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.340836, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.583136, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_unique" @@ -12504,6 +14973,7 @@ "jaffle_shop", "unique_customers_customer_id" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -12514,12 +14984,14 @@ "path": "unique_customers_customer_id.sql", "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "refs": [ - [ - "customers" - ] + { + "name": "customers", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -12536,6 +15008,7 @@ }, "test.jaffle_shop.unique_orders_order_id.fed79b3a6e": { "alias": "unique_orders_order_id", + "attached_node": "model.jaffle_shop.orders", "build_path": null, "checksum": { "checksum": "", @@ -12550,19 +15023,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.343404, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.5853949, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_unique" @@ -12581,6 +15060,7 @@ "jaffle_shop", "unique_orders_order_id" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -12591,12 +15071,14 @@ "path": "unique_orders_order_id.sql", "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "refs": [ - [ - "orders" - ] + { + "name": "orders", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -12613,6 +15095,7 @@ }, "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada": { "alias": "unique_stg_customers_customer_id", + "attached_node": "model.jaffle_shop.stg_customers", "build_path": null, "checksum": { "checksum": "", @@ -12627,19 +15110,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.375885, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.6324039, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_unique" @@ -12659,6 +15148,7 @@ "staging", "unique_stg_customers_customer_id" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -12669,12 +15159,14 @@ "path": "unique_stg_customers_customer_id.sql", "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "refs": [ - [ - "stg_customers" - ] + { + "name": "stg_customers", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -12691,6 +15183,7 @@ }, "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a": { "alias": "unique_stg_orders_order_id", + "attached_node": "model.jaffle_shop.stg_orders", "build_path": null, "checksum": { "checksum": "", @@ -12705,19 +15198,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.3786829, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.635364, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_unique" @@ -12737,6 +15236,7 @@ "staging", "unique_stg_orders_order_id" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -12747,12 +15247,14 @@ "path": "unique_stg_orders_order_id.sql", "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "refs": [ - [ - "stg_orders" - ] + { + "name": "stg_orders", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -12769,6 +15271,7 @@ }, "test.jaffle_shop.unique_stg_payments_payment_id.3744510712": { "alias": "unique_stg_payments_payment_id", + "attached_node": "model.jaffle_shop.stg_payments", "build_path": null, "checksum": { "checksum": "", @@ -12783,19 +15286,25 @@ "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", + "group": null, "limit": null, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": null, + "store_failures_as": null, "tags": [], "warn_if": "!= 0", "where": null }, - "created_at": 1696458270.385969, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1719485737.640539, "database": "postgres", - "deferred": false, "depends_on": { "macros": [ "macro.dbt.test_unique" @@ -12815,6 +15324,7 @@ "staging", "unique_stg_payments_payment_id" ], + "group": null, "language": "sql", "meta": {}, "metrics": [], @@ -12825,12 +15335,14 @@ "path": "unique_stg_payments_payment_id.sql", "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "refs": [ - [ - "stg_payments" - ] + { + "name": "stg_payments", + "package": null, + "version": null + } ], + "relation_name": null, "resource_type": "test", - "root_path": "/Users/julian/Astronomer/astronomer-cosmos/dev/dags/dbt/jaffle_shop", "schema": "public_dbt_test__audit", "sources": [], "tags": [], @@ -12930,6 +15442,9 @@ "model.jaffle_shop.stg_payments" ] }, + "saved_queries": {}, "selectors": {}, - "sources": {} + "semantic_models": {}, + "sources": {}, + "unit_tests": {} } diff --git a/dev/dags/dbt/model_version/models/schema.yml b/dev/dags/dbt/model_version/models/schema.yml index 40a5a4055b..66f1ccedd7 100644 --- a/dev/dags/dbt/model_version/models/schema.yml +++ b/dev/dags/dbt/model_version/models/schema.yml @@ -37,7 +37,11 @@ models: - include: all exclude: - full_name + config: + alias: '{{ "customers_" ~ var("division", "USA") ~ "_v1" }}' - v: 2 + config: + alias: '{{ "customers_" ~ var("division", "USA") ~ "_v2" }}' - name: orders description: This table has basic information about orders, as well as some derived facts based on payments diff --git a/dev/dags/dbt/perf/.gitignore b/dev/dags/dbt/perf/.gitignore new file mode 100644 index 0000000000..49f147cb98 --- /dev/null +++ b/dev/dags/dbt/perf/.gitignore @@ -0,0 +1,4 @@ + +target/ +dbt_packages/ +logs/ diff --git a/dev/dags/dbt/perf/README.md b/dev/dags/dbt/perf/README.md new file mode 100644 index 0000000000..31e67f483b --- /dev/null +++ b/dev/dags/dbt/perf/README.md @@ -0,0 +1,3 @@ +dbt project for running performance tests. + +The `models` directory gets populated by an integration test defined in `tests/perf`. diff --git a/dev/dags/dbt/perf/analyses/.gitkeep b/dev/dags/dbt/perf/analyses/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dev/dags/dbt/perf/dbt_project.yml b/dev/dags/dbt/perf/dbt_project.yml new file mode 100644 index 0000000000..ebb0275743 --- /dev/null +++ b/dev/dags/dbt/perf/dbt_project.yml @@ -0,0 +1,17 @@ +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: "perf" +version: "1.0.0" +config-version: 2 + +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" diff --git a/dev/dags/dbt/perf/macros/.gitkeep b/dev/dags/dbt/perf/macros/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dev/dags/dbt/perf/profiles.yml b/dev/dags/dbt/perf/profiles.yml new file mode 100644 index 0000000000..224f565f4a --- /dev/null +++ b/dev/dags/dbt/perf/profiles.yml @@ -0,0 +1,12 @@ +default: + target: dev + outputs: + dev: + type: postgres + host: "{{ env_var('POSTGRES_HOST') }}" + user: "{{ env_var('POSTGRES_USER') }}" + password: "{{ env_var('POSTGRES_PASSWORD') }}" + port: "{{ env_var('POSTGRES_PORT') | int }}" + dbname: "{{ env_var('POSTGRES_DB') }}" + schema: "{{ env_var('POSTGRES_SCHEMA') }}" + threads: 4 diff --git a/dev/dags/dbt/perf/seeds/.gitkeep b/dev/dags/dbt/perf/seeds/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dev/dags/dbt/perf/snapshots/.gitkeep b/dev/dags/dbt/perf/snapshots/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dev/dags/dbt/perf/tests/.gitkeep b/dev/dags/dbt/perf/tests/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dev/dags/dbt/simple/convert_csv_to_db.py b/dev/dags/dbt/simple/convert_csv_to_db.py index 7424db32de..ef0ca301ec 100644 --- a/dev/dags/dbt/simple/convert_csv_to_db.py +++ b/dev/dags/dbt/simple/convert_csv_to_db.py @@ -1,6 +1,7 @@ -import pandas as pd import sqlite3 +import pandas as pd + df = pd.read_csv("imdb.csv") conn = sqlite3.connect("imdb.db") df.to_sql("movies_ratings", conn, if_exists="replace", index=False) diff --git a/dev/dags/dbt/simple/models/movies_ratings_simplified.sql b/dev/dags/dbt/simple/models/movies_ratings_simplified.sql index 342782e347..671a5764ff 100644 --- a/dev/dags/dbt/simple/models/movies_ratings_simplified.sql +++ b/dev/dags/dbt/simple/models/movies_ratings_simplified.sql @@ -11,4 +11,4 @@ select "Domestic", "Foreign", "Worldwide" -from {{ source('imdb', 'movies_ratings') }} +from {{ source('main', 'movies_ratings') }} diff --git a/dev/dags/dbt/simple/models/source.yml b/dev/dags/dbt/simple/models/source.yml index efb9b15608..b240ba542e 100644 --- a/dev/dags/dbt/simple/models/source.yml +++ b/dev/dags/dbt/simple/models/source.yml @@ -1,7 +1,7 @@ version: 2 sources: - - name: imdb + - name: main description: Example of IMDB SQlite database tables: - name: movies_ratings diff --git a/dev/dags/dbt/simple/models/top_animations.sql b/dev/dags/dbt/simple/models/top_animations.sql index 2b365b09cb..cfae1c5952 100644 --- a/dev/dags/dbt/simple/models/top_animations.sql +++ b/dev/dags/dbt/simple/models/top_animations.sql @@ -1,4 +1,8 @@ -{{ config(materialized='table') }} +{{ config( + materialized='table', + alias=var('animation_alias', 'top_animations') + ) +}} SELECT Title, Rating FROM {{ ref('movies_ratings_simplified') }} diff --git a/dev/dags/dbt/simple/profiles.yml b/dev/dags/dbt/simple/profiles.yml index 5de7e781a6..24841363e6 100644 --- a/dev/dags/dbt/simple/profiles.yml +++ b/dev/dags/dbt/simple/profiles.yml @@ -7,5 +7,5 @@ simple: database: 'database' schema: 'main' schemas_and_paths: - main: "{{ env_var('DBT_SQLITE_PATH', '.') }}/data/imdb.db" - schema_directory: 'data' + main: "{{ env_var('DBT_SQLITE_PATH') }}/imdb.db" + schema_directory: "{{ env_var('DBT_SQLITE_PATH') }}" diff --git a/dev/dags/dbt_docs.py b/dev/dags/dbt_docs.py index edf89bdab6..e26e10d531 100644 --- a/dev/dags/dbt_docs.py +++ b/dev/dags/dbt_docs.py @@ -11,16 +11,16 @@ from pathlib import Path from airflow import DAG -from airflow.hooks.base import BaseHook -from airflow.exceptions import AirflowNotFoundException from airflow.decorators import task +from airflow.exceptions import AirflowNotFoundException +from airflow.hooks.base import BaseHook from pendulum import datetime from cosmos import ProfileConfig from cosmos.operators import ( DbtDocsAzureStorageOperator, - DbtDocsS3Operator, DbtDocsGCSOperator, + DbtDocsS3Operator, ) from cosmos.profiles import PostgresUserPasswordProfileMapping @@ -35,7 +35,7 @@ profile_name="default", target_name="dev", profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="airflow_db", + conn_id="example_conn", profile_args={"schema": "public"}, ), ) @@ -43,7 +43,7 @@ @task.branch(task_id="which_upload") def which_upload(): - "Only run the docs tasks if we have the proper connections set up" + """Only run the docs tasks if we have the proper connections set up""" downstream_tasks_to_run = [] try: diff --git a/dev/dags/example_cosmos_cleanup_dag.py b/dev/dags/example_cosmos_cleanup_dag.py new file mode 100644 index 0000000000..c93bdf0020 --- /dev/null +++ b/dev/dags/example_cosmos_cleanup_dag.py @@ -0,0 +1,34 @@ +""" +Example of cleanup DAG that can be used to clear cache originated from running the dbt ls command while +parsing the DbtDag or DbtTaskGroup since Cosmos 1.5. +""" + +# [START cache_example] +from datetime import datetime, timedelta + +from airflow.decorators import dag, task + +from cosmos.cache import delete_unused_dbt_ls_cache + + +@dag( + schedule_interval="0 0 * * 0", # Runs every Sunday + start_date=datetime(2023, 1, 1), + catchup=False, + tags=["example"], +) +def example_cosmos_cleanup_dag(): + + @task() + def clear_db_ls_cache(session=None): + """ + Delete the dbt ls cache that has not been used for the last five days. + """ + delete_unused_dbt_ls_cache(max_age_last_usage=timedelta(days=5)) + + clear_db_ls_cache() + + +# [END cache_example] + +example_cosmos_cleanup_dag() diff --git a/dev/dags/example_cosmos_python_models.py b/dev/dags/example_cosmos_python_models.py index 7d9a61465e..90c8cc0ca9 100644 --- a/dev/dags/example_cosmos_python_models.py +++ b/dev/dags/example_cosmos_python_models.py @@ -17,7 +17,7 @@ from datetime import datetime from pathlib import Path -from cosmos import DbtDag, ProjectConfig, ProfileConfig +from cosmos import DbtDag, ProfileConfig, ProjectConfig from cosmos.profiles import DatabricksTokenProfileMapping DEFAULT_DBT_ROOT_PATH = Path(__file__).parent / "dbt" @@ -29,7 +29,7 @@ target_name="dev", profile_mapping=DatabricksTokenProfileMapping( conn_id="databricks_default", - profile_args={"schema": SCHEMA}, + profile_args={"schema": SCHEMA, "connect_retries": 3}, ), ) diff --git a/dev/dags/example_cosmos_sources.py b/dev/dags/example_cosmos_sources.py index ddeb9ab15b..346f373704 100644 --- a/dev/dags/example_cosmos_sources.py +++ b/dev/dags/example_cosmos_sources.py @@ -11,22 +11,27 @@ It will dynamically add the new type to the enumeration ``DbtResourceType`` so that Cosmos can parse these dbt nodes and convert them into the Airflow DAG. """ + import os from datetime import datetime from pathlib import Path -from airflow.operators.dummy import DummyOperator from airflow.models.dag import DAG + +try: # available since Airflow 2.4.0 + from airflow.operators.empty import EmptyOperator +except ImportError: + from airflow.operators.dummy import DummyOperator as EmptyOperator from airflow.utils.task_group import TaskGroup -from cosmos import DbtDag, ProjectConfig, ProfileConfig, RenderConfig +from cosmos import DbtDag, ProfileConfig, ProjectConfig, RenderConfig from cosmos.constants import DbtResourceType from cosmos.dbt.graph import DbtNode DEFAULT_DBT_ROOT_PATH = Path(__file__).parent / "dbt" DBT_ROOT_PATH = Path(os.getenv("DBT_ROOT_PATH", DEFAULT_DBT_ROOT_PATH)) -os.environ["DBT_SQLITE_PATH"] = str(DEFAULT_DBT_ROOT_PATH / "simple") +DBT_SQLITE_PATH = str(DEFAULT_DBT_ROOT_PATH / "data") profile_config = ProfileConfig( @@ -37,21 +42,21 @@ # [START custom_dbt_nodes] -# Cosmos will use this function to generate a DummyOperator task when it finds a source node, in the manifest. +# Cosmos will use this function to generate an empty task when it finds a source node, in the manifest. # A more realistic use case could be to use an Airflow sensor to represent a source. def convert_source(dag: DAG, task_group: TaskGroup, node: DbtNode, **kwargs): """ - Return an instance of DummyOperator to represent a dbt "source" node. + Return an instance of a desired operator to represent a dbt "source" node. """ - return DummyOperator(dag=dag, task_group=task_group, task_id=f"{node.name}_source") + return EmptyOperator(dag=dag, task_group=task_group, task_id=f"{node.name}_source") -# Cosmos will use this function to generate a DummyOperator task when it finds a exposure node, in the manifest. +# Cosmos will use this function to generate an empty task when it finds a exposure node, in the manifest. def convert_exposure(dag: DAG, task_group: TaskGroup, node: DbtNode, **kwargs): """ - Return an instance of DummyOperator to represent a dbt "exposure" node. + Return an instance of a desired operator to represent a dbt "exposure" node. """ - return DummyOperator(dag=dag, task_group=task_group, task_id=f"{node.name}_exposure") + return EmptyOperator(dag=dag, task_group=task_group, task_id=f"{node.name}_exposure") # Use `RenderConfig` to tell Cosmos, given a node type, how to convert a dbt node into an Airflow task or task group. @@ -65,15 +70,21 @@ def convert_exposure(dag: DAG, task_group: TaskGroup, node: DbtNode, **kwargs): } ) +# `ProjectConfig` can pass dbt variables and environment variables to dbt commands. Below is an example of +# passing a required env var for the profiles.yml file and a dbt variable that is used for rendering and +# executing dbt models. +project_config = ProjectConfig( + DBT_ROOT_PATH / "simple", + env_vars={"DBT_SQLITE_PATH": DBT_SQLITE_PATH}, + dbt_vars={"animation_alias": "top_5_animated_movies"}, +) + example_cosmos_sources = DbtDag( # dbt/cosmos-specific parameters - project_config=ProjectConfig( - DBT_ROOT_PATH / "simple", - ), + project_config=project_config, profile_config=profile_config, render_config=render_config, - operator_args={"append_env": True}, # normal dag parameters schedule_interval="@daily", start_date=datetime(2023, 1, 1), diff --git a/dev/dags/example_model_version.py b/dev/dags/example_model_version.py index 78f38647dd..df9205f3ef 100644 --- a/dev/dags/example_model_version.py +++ b/dev/dags/example_model_version.py @@ -1,12 +1,12 @@ """ -An example DAG that uses Cosmos to render a dbt project. +An example DAG that uses Cosmos to render a dbt project as an Airflow DAG. """ import os from datetime import datetime from pathlib import Path -from cosmos import DbtDag, ProjectConfig, ProfileConfig +from cosmos import DbtDag, ProfileConfig, ProjectConfig from cosmos.profiles import PostgresUserPasswordProfileMapping DEFAULT_DBT_ROOT_PATH = Path(__file__).parent / "dbt" @@ -16,7 +16,7 @@ profile_name="default", target_name="dev", profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="airflow_db", + conn_id="example_conn", profile_args={"schema": "public"}, ), ) diff --git a/dev/dags/example_virtualenv.py b/dev/dags/example_virtualenv.py index 7b1368f8c4..cd38cba9ec 100644 --- a/dev/dags/example_virtualenv.py +++ b/dev/dags/example_virtualenv.py @@ -1,11 +1,12 @@ """ -An example DAG that uses Cosmos to render a dbt project. +An example DAG that uses Cosmos to render a dbt project as an Airflow DAG. """ + import os from datetime import datetime from pathlib import Path -from cosmos import DbtDag, ExecutionMode, ExecutionConfig, ProjectConfig, ProfileConfig +from cosmos import DbtDag, ExecutionConfig, ExecutionMode, ProfileConfig, ProjectConfig from cosmos.profiles import PostgresUserPasswordProfileMapping DEFAULT_DBT_ROOT_PATH = Path(__file__).parent / "dbt" @@ -16,7 +17,7 @@ profile_name="default", target_name="dev", profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="airflow_db", + conn_id="example_conn", profile_args={"schema": "public"}, ), ) @@ -35,6 +36,7 @@ "py_system_site_packages": False, "py_requirements": ["dbt-postgres==1.6.0b1"], "install_deps": True, + "emit_datasets": False, # Example of how to not set inlets and outlets }, # normal dag parameters schedule_interval="@daily", diff --git a/dev/dags/performance_dag.py b/dev/dags/performance_dag.py new file mode 100644 index 0000000000..3ed23c94c4 --- /dev/null +++ b/dev/dags/performance_dag.py @@ -0,0 +1,38 @@ +""" +An Airflow DAG that uses Cosmos to render a dbt project for performance testing. +""" + +import os +from datetime import datetime +from pathlib import Path + +from cosmos import DbtDag, ProfileConfig, ProjectConfig, RenderConfig +from cosmos.profiles import PostgresUserPasswordProfileMapping + +DEFAULT_DBT_ROOT_PATH = Path(__file__).parent / "dbt" +DBT_ROOT_PATH = Path(os.getenv("DBT_ROOT_PATH", DEFAULT_DBT_ROOT_PATH)) + + +profile_config = ProfileConfig( + profile_name="default", + target_name="dev", + profile_mapping=PostgresUserPasswordProfileMapping( + conn_id="example_conn", + profile_args={"schema": "public"}, + ), +) + +cosmos_perf_dag = DbtDag( + project_config=ProjectConfig( + DBT_ROOT_PATH / "perf", + ), + profile_config=profile_config, + render_config=RenderConfig( + dbt_deps=False, + ), + # normal dag parameters + schedule_interval=None, + start_date=datetime(2024, 1, 1), + catchup=False, + dag_id="performance_dag", +) diff --git a/dev/dags/user_defined_profile.py b/dev/dags/user_defined_profile.py index ab30cdb2fe..a88354b049 100644 --- a/dev/dags/user_defined_profile.py +++ b/dev/dags/user_defined_profile.py @@ -1,6 +1,7 @@ """ A DAG that uses Cosmos with a custom profile. """ + import os from datetime import datetime from pathlib import Path @@ -8,11 +9,12 @@ from airflow.decorators import dag from airflow.operators.empty import EmptyOperator -from cosmos import DbtTaskGroup, ProjectConfig, ProfileConfig +from cosmos import DbtTaskGroup, LoadMode, ProfileConfig, ProjectConfig, RenderConfig DEFAULT_DBT_ROOT_PATH = Path(__file__).parent / "dbt" DBT_ROOT_PATH = Path(os.getenv("DBT_ROOT_PATH", DEFAULT_DBT_ROOT_PATH)) PROFILES_FILE_PATH = Path(DBT_ROOT_PATH, "jaffle_shop", "profiles.yml") +DBT_LS_PATH = Path(DBT_ROOT_PATH, "jaffle_shop", "dbt_ls_models_staging.txt") @dag( @@ -35,6 +37,10 @@ def user_defined_profile() -> None: target_name="dev", profiles_yml_filepath=PROFILES_FILE_PATH, ), + render_config=RenderConfig( + load_method=LoadMode.DBT_LS_FILE, + dbt_ls_path=DBT_LS_PATH, + ), operator_args={"append_env": True, "install_deps": True}, default_args={"retries": 2}, ) diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 23b012d153..5345f4b134 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -10,6 +10,7 @@ x-airflow-common: environment: &airflow-common-env DB_BACKEND: postgres + AIRFLOW__COSMOS__DBT_DOCS_DIR: http://cosmos-docs.s3-website-us-east-1.amazonaws.com/ AIRFLOW__CORE__EXECUTOR: LocalExecutor AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:pg_password@postgres:5432/airflow AIRFLOW__CORE__FERNET_KEY: '' diff --git a/docs/_static/cosmos_aci_schematic.png b/docs/_static/cosmos_aci_schematic.png new file mode 100644 index 0000000000..ece19a4269 Binary files /dev/null and b/docs/_static/cosmos_aci_schematic.png differ diff --git a/docs/_static/jaffle_shop_azure_container_instance.png b/docs/_static/jaffle_shop_azure_container_instance.png new file mode 100644 index 0000000000..48d8e450e8 Binary files /dev/null and b/docs/_static/jaffle_shop_azure_container_instance.png differ diff --git a/docs/_static/jaffle_shop_dag.png b/docs/_static/jaffle_shop_dag.png new file mode 100644 index 0000000000..1a8cdf5cb0 Binary files /dev/null and b/docs/_static/jaffle_shop_dag.png differ diff --git a/docs/_static/location_of_dbt_docs_in_airflow.png b/docs/_static/location_of_dbt_docs_in_airflow.png new file mode 100644 index 0000000000..348a53c8ea Binary files /dev/null and b/docs/_static/location_of_dbt_docs_in_airflow.png differ diff --git a/docs/configuration/caching.rst b/docs/configuration/caching.rst new file mode 100644 index 0000000000..7dba9933f8 --- /dev/null +++ b/docs/configuration/caching.rst @@ -0,0 +1,132 @@ +.. _caching: + +Caching +======= + +This page explains the caching strategies in ``astronomer-cosmos`` Astronomer Cosmos behavior. + +All Cosmos caching mechanisms can be enabled or turned off in the ``airflow.cfg`` file or using environment variables. + +.. note:: + For more information, see `configuring a Cosmos project <./project-config.html>`_. + +Depending on the Cosmos version, it creates a cache for two types of data: + +- The ``dbt ls`` output +- The dbt ``partial_parse.msgpack`` file + +It is possible to turn off any cache in Cosmos by exporting the environment variable ``AIRFLOW__COSMOS__ENABLE_CACHE=0``. +Disabling individual types of cache in Cosmos is also possible, as explained below. + +Caching the dbt ls output +~~~~~~~~~~~~~ + +(Introduced in Cosmos 1.5) + +While parsing a dbt project using `LoadMode.DBT_LS <./parsing-methods.html#dbt-ls>`_, Cosmos uses subprocess to run ``dbt ls``. +This operation can be very costly; it can increase the DAG parsing times and affect not only the scheduler DAG processing but +also the tasks queueing time. + +Cosmos 1.5 introduced a feature to mitigate the performance issue associated with ``LoadMode.DBT_LS`` by caching the output +of this command as an `Airflow Variable `_. +Based on an initial `analysis `_, enabling this setting reduced some DAGs ask queueing from 30s to 0s. Additionally, some users `reported improvements of 84% `_ in the DAG run time. + +This feature is on by default. To turn it off, export the following environment variable: ``AIRFLOW__COSMOS__ENABLE_CACHE_DBT_LS=0``. + +**How the cache is refreshed** + +Users can purge or delete the cache via Airflow UI by identifying and deleting the cache key. + +Cosmos will refresh the cache in a few circumstances: + +* if any files of the dbt project change +* if one of the arguments that affect the dbt ls command execution changes + +To evaluate if the dbt project changed, it calculates the changes using a few of the MD5 of all the files in the directory. + +Additionally, if any of the following DAG configurations are changed, we'll automatically purge the cache of the DAGs that use that specific configuration: + +* ``ProjectConfig.dbt_vars`` +* ``ProjectConfig.env_vars`` +* ``ProjectConfig.partial_parse`` +* ``RenderConfig.env_vars`` +* ``RenderConfig.exclude`` +* ``RenderConfig.select`` +* ``RenderConfig.selector`` + +Finally, if users would like to define specific Airflow variables that, if changed, will cause the recreation of the cache, they can specify those by using: + +* ``RenderConfig.airflow_vars_to_purge_cache`` + +Example: + +.. code-block:: python + + RenderConfig(airflow_vars_to_purge_cache == ["refresh_cache"]) + +**Cleaning up stale cache** + +Not rarely, Cosmos DbtDags and DbtTaskGroups may be renamed or deleted. In those cases, to clean up the Airflow metadata database, it is possible to use the method ``delete_unused_dbt_ls_cache``. + +The method deletes the Cosmos cache stored in Airflow Variables based on the last execution of their associated DAGs. + +As an example, the following clean-up DAG will delete any cache associated with Cosmos that has not been used for the last five days: + +.. literalinclude:: ../../dev/dags/example_cosmos_cleanup_dag.py + :language: python + :start-after: [START cache_example] + :end-before: [END cache_example] + +**Cache key** + +The Airflow variables that represent the dbt ls cache are prefixed by ``cosmos_cache``. +When using ``DbtDag``, the keys use the DAG name. When using ``DbtTaskGroup``, they contain the ``TaskGroup`` and parent task groups and DAG. + +Examples: + +* The ``DbtDag`` "cosmos_dag" will have the cache represented by "cosmos_cache__basic_cosmos_dag". +* The ``DbtTaskGroup`` "customers" declared inside the DAG "basic_cosmos_task_group" will have the cache key "cosmos_cache__basic_cosmos_task_group__customers". + +**Cache value** + +The cache values contain a few properties: + +* ``last_modified`` timestamp, represented using the ISO 8601 format. +* ``version`` is a hash that represents the version of the dbt project and arguments used to run dbt ls by the time Cosmos created the cache +* ``dbt_ls_compressed`` represents the dbt ls output compressed using zlib and encoded to base64 so Cosmos can record the value as a compressed string in the Airflow metadata database. +* ``dag_id`` is the DAG associated to this cache +* ``task_group_id`` is the TaskGroup associated to this cache +* ``cosmos_type`` is either ``DbtDag`` or ``DbtTaskGroup`` + + +Caching the partial parse file +~~~~~~~~~~~~~ + +(Introduced in Cosmos 1.4) + +After parsing the dbt project, dbt stores an internal project manifest in a file called ``partial_parse.msgpack`` (`official docs `_). +This file contributes significantly to the performance of running dbt commands when the dbt project did not change. + +Cosmos 1.4 introduced `support to partial parse files `_ both +provided by the user, and also by storing in the disk temporary folder in the Airflow scheduler and worker node the file +generated after running dbt commands. + +Users can customize where to store the cache using the setting ``AIRFLOW__COSMOS__CACHE_DIR``. + +It is possible to switch off this feature by exporting the environment variable ``AIRFLOW__COSMOS__ENABLE_CACHE_PARTIAL_PARSE=0``. + +For more information, read the `Cosmos partial parsing documentation <./partial-parsing.html>`_ + + +Caching the profiles +~~~~~~~~~~~~~~~~~~~~~~~~ + +(Introduced in Cosmos 1.5) + +Cosmos 1.5 introduced `support to profile caching `_, +enabling caching for the profile mapping in the path specified by env ``AIRFLOW__COSMOS__CACHE_DIR`` and ``AIRFLOW__COSMOS__PROFILE_CACHE_DIR_NAME``. +This feature facilitates the reuse of Airflow connections and ``profiles.yml``. + +Users have the flexibility to customize the cache storage location using the settings ``AIRFLOW__COSMOS__CACHE_DIR`` and ``AIRFLOW__COSMOS__PROFILE_CACHE_DIR_NAME``. + +To disable this feature, users can set the environment variable ``AIRFLOW__COSMOS__ENABLE_CACHE_PROFILE=False`` diff --git a/docs/configuration/cosmos-conf.rst b/docs/configuration/cosmos-conf.rst new file mode 100644 index 0000000000..8dc90a5c18 --- /dev/null +++ b/docs/configuration/cosmos-conf.rst @@ -0,0 +1,112 @@ +Cosmos Config +============= + +This page lists all available Airflow configurations that affect ``astronomer-cosmos`` Astronomer Cosmos behavior. They can be set in the ``airflow.cfg file`` or using environment variables. + +.. note:: + For more information, see `Setting Configuration Options `_. + +**Sections:** + +- [cosmos] +- [openlineage] + +[cosmos] +~~~~~~~~ + +.. _cache_dir: + +`cache_dir`_: + The directory used for caching Cosmos data. + + - Default: ``{TMPDIR}/cosmos_cache`` (where ``{TMPDIR}`` is the system temporary directory) + - Environment Variable: ``AIRFLOW__COSMOS__CACHE_DIR`` + +.. _enable_cache: + +`enable_cache`_: + Enable or disable caching of Cosmos data. + + - Default: ``True`` + - Environment Variable: ``AIRFLOW__COSMOS__ENABLE_CACHE`` + +.. _enable_cache_dbt_ls: + +`enable_cache_dbt_ls`_: + Enable or disable caching of the dbt ls command in case using ``LoadMode.DBT_LS`` in an Airflow Variable. + + - Default: ``True`` + - Environment Variable: ``AIRFLOW__COSMOS__ENABLE_CACHE_DBT_LS`` + +.. _enable_cache_partial_parse: + +`enable_cache_partial_parse`_: + Enable or disable caching of dbt partial parse files in the local disk. + + - Default: ``True`` + - Environment Variable: ``AIRFLOW__COSMOS__ENABLE_CACHE_PARTIAL_PARSE`` + +.. _propagate_logs: + +`propagate_logs`_: + Whether to propagate logs in the Cosmos module. + + - Default: ``True`` + - Environment Variable: ``AIRFLOW__COSMOS__PROPAGATE_LOGS`` + +.. _dbt_docs_dir: + +`dbt_docs_dir`_: + The directory path for dbt documentation. + + - Default: ``None`` + - Environment Variable: ``AIRFLOW__COSMOS__DBT_DOCS_DIR`` + +.. _dbt_docs_conn_id: + +`dbt_docs_conn_id`_: + The connection ID for dbt documentation. + + - Default: ``None`` + - Environment Variable: ``AIRFLOW__COSMOS__DBT_DOCS_CONN_ID`` + +.. _enable_cache_profile: + +`enable_cache_profile`_: + Enable caching for the DBT profile. + + - Default: ``True`` + - Environment Variable: ``AIRFLOW__COSMOS__ENABLE_CACHE_PROFILE`` + +.. _profile_cache_dir_name: + +`profile_cache_dir_name`_: + Folder name to store the DBT cached profiles. This will be a sub-folder of ``cache_dir`` + + - Default: ``profile`` + - Environment Variable: ``AIRFLOW__COSMOS__PROFILE_CACHE_DIR_NAME`` + + +[openlineage] +~~~~~~~~~~~~~ + +.. _namespace: + +`namespace`_: + The OpenLineage namespace for tracking lineage. + + - Default: If not configured in Airflow configuration, it falls back to the environment variable ``OPENLINEAGE_NAMESPACE``, otherwise it uses ``DEFAULT_OPENLINEAGE_NAMESPACE``. + - Environment Variable: ``AIRFLOW__OPENLINEAGE__NAMESPACE`` + +.. note:: + For more information, see `Openlieage Configuration Options `_. + +Environment Variables +~~~~~~~~~~~~~~~~~~~~~ + +.. _LINEAGE_NAMESPACE: + +`LINEAGE_NAMESPACE`_: + The OpenLineage namespace for tracking lineage. + + - Default: If not configured in Airflow configuration, it falls back to the environment variable ``OPENLINEAGE_NAMESPACE``, otherwise it uses ``DEFAULT_OPENLINEAGE_NAMESPACE``. diff --git a/docs/configuration/execution-config.rst b/docs/configuration/execution-config.rst index c118590d85..dd9758d558 100644 --- a/docs/configuration/execution-config.rst +++ b/docs/configuration/execution-config.rst @@ -7,6 +7,7 @@ It does this by exposing a ``cosmos.config.ExecutionConfig`` class that you can The ``ExecutionConfig`` class takes the following arguments: - ``execution_mode``: The way dbt is run when executing within airflow. For more information, see the `execution modes <../getting_started/execution-modes.html>`_ page. +- ``invocation_mode`` (new in v1.4): The way dbt is invoked within the execution mode. This is only configurable for ``ExecutionMode.LOCAL``. For more information, see `invocation modes <../getting_started/execution-modes.html#invocation-modes>`_. - ``test_indirect_selection``: The mode to configure the test behavior when performing indirect selection. - ``dbt_executable_path``: The path to the dbt executable for dag generation. Defaults to dbt if available on the path. -- ``dbt_project_path``: Configures the DBT project location accessible on their airflow controller for DAG rendering - Required when using ``load_method=LoadMode.DBT_LS`` or ``load_method=LoadMode.CUSTOM`` +- ``dbt_project_path``: Configures the dbt project location accessible at runtime for dag execution. This is the project path in a docker container for ``ExecutionMode.DOCKER`` or ``ExecutionMode.KUBERNETES``. Mutually exclusive with ``ProjectConfig.dbt_project_path``. diff --git a/docs/configuration/generating-docs.rst b/docs/configuration/generating-docs.rst index 88459fd14e..208280cbb9 100644 --- a/docs/configuration/generating-docs.rst +++ b/docs/configuration/generating-docs.rst @@ -5,7 +5,9 @@ Generating Docs dbt allows you to generate static documentation on your models, tables, and more. You can read more about it in the `official dbt documentation `_. For an example of what the docs look like with the ``jaffle_shop`` project, check out `this site `_. -Many users choose to generate and serve these docs on a static website. This is a great way to share your data models with your team and other stakeholders. +After generating the dbt docs, you can host them natively within Airflow via the Cosmos Airflow plugin; see `Hosting Docs `__ for more information. + +Alternatively, many users choose to serve these docs on a separate static website. This is a great way to share your data models with a broad array of stakeholders. Cosmos offers two pre-built ways of generating and uploading dbt docs and a fallback option to run custom code after the docs are generated: @@ -67,7 +69,40 @@ Upload to GCS GCS supports serving static files directly from a bucket. To learn more (and to set it up), check out the `official GCS documentation `_. -You can use the :class:`~cosmos.operators.DbtDocsGCSOperator` to generate and upload docs to a S3 bucket. The following code snippet shows how to do this with the default jaffle_shop project: +You can use the :class:`~cosmos.operators.DbtDocsGCSOperator` to generate and upload docs to a GCS bucket. The following code snippet shows how to do this with the default jaffle_shop project: + +.. code-block:: python + + from cosmos.operators import DbtDocsGCSOperator + + # then, in your DAG code: + generate_dbt_docs_aws = DbtDocsGCSOperator( + task_id="generate_dbt_docs_gcs", + project_dir="path/to/jaffle_shop", + profile_config=profile_config, + # docs-specific arguments + connection_id="test_gcs", + bucket_name="test_bucket", + ) + +Choosing a folder +~~~~~~~~~~~~~~~~~~~~~~~ + +All the DbtDocsOperators support specification of a custom folder (prefix) to place documentation in on the target cloud storage. This can be done by +adding a ``folder_dir`` parameter to the operator definition. + +Static Flag +~~~~~~~~~~~~~~~~~~~~~~~ + +All of the DbtDocsOperator accept the ``--static`` flag. To learn more about the static flag, check out the `original PR on dbt-core `_. +The static flag is used to generate a single doc file that can be hosted directly from cloud storage. +By having a single documentation file, you can make use of Access control can be configured through Identity-Aware Proxy (IAP), and making it easy to host. + +.. note:: + The static flag is only available from dbt-core >=1.7 + +The following code snippet shows how to provide this flag with the default jaffle_shop project: + .. code-block:: python @@ -81,6 +116,7 @@ You can use the :class:`~cosmos.operators.DbtDocsGCSOperator` to generate and up # docs-specific arguments connection_id="test_gcs", bucket_name="test_bucket", + dbt_cmd_flags=["--static"], ) Custom Callback diff --git a/docs/configuration/hosting-docs.rst b/docs/configuration/hosting-docs.rst new file mode 100644 index 0000000000..755bfe815d --- /dev/null +++ b/docs/configuration/hosting-docs.rst @@ -0,0 +1,153 @@ +.. hosting-docs: + +Hosting Docs +============ + +dbt docs can be served directly from the Apache Airflow webserver with the Cosmos Airflow plugin, without requiring the user to set up anything outside of Airflow. This page describes how to host docs in the Airflow webserver directly, although some users may opt to host docs externally. + +Overview +~~~~~~~~ + +The dbt docs are available in the Airflow menu under ``Browse > dbt docs``: + +.. image:: /_static/location_of_dbt_docs_in_airflow.png + :alt: Airflow UI - Location of dbt docs in menu + :align: center + +In order to access the dbt docs, you must specify the following config variables: + +- ``cosmos.dbt_docs_dir``: A path to where the docs are being hosted. +- (Optional) ``cosmos.dbt_docs_conn_id``: A conn ID to use for a cloud storage deployment. If not specified _and_ the URI points to a cloud storage platform, then the default conn ID for the AWS/Azure/GCP hook will be used. + +.. code-block:: cfg + + [cosmos] + dbt_docs_dir = path/to/docs/here + dbt_docs_conn_id = my_conn_id + +or as an environment variable: + +.. code-block:: shell + + AIRFLOW__COSMOS__DBT_DOCS_DIR="path/to/docs/here" + AIRFLOW__COSMOS__DBT_DOCS_CONN_ID="my_conn_id" + +The path can be either a folder in the local file system the webserver is running on, or a URI to a cloud storage platform (S3, GCS, Azure). + +If your docs were generated using the ``--static`` flag, you can set the index filename using ``dbt_docs_index_file_name``: + +.. code-block:: cfg + + [cosmos] + dbt_docs_index_file_name = static_index.html + + +Host from Cloud Storage +~~~~~~~~~~~~~~~~~~~~~~~ + +For typical users, the recommended setup for hosting dbt docs would look like this: + +1. Generate the docs via one of Cosmos' pre-built operators for generating dbt docs (see `Generating Docs `__ for more information) +2. Wherever you dumped the docs, set your ``cosmos.dbt_docs_dir`` to that location. +3. If you want to use a conn ID other than the default connection, set your ``cosmos.dbt_docs_conn_id``. Otherwise, leave this blank. + +AWS S3 Example +^^^^^^^^^^^^^^ + +.. code-block:: cfg + + [cosmos] + dbt_docs_dir = s3://my-bucket/path/to/docs + dbt_docs_conn_id = aws_default + +.. code-block:: shell + + AIRFLOW__COSMOS__DBT_DOCS_DIR="s3://my-bucket/path/to/docs" + AIRFLOW__COSMOS__DBT_DOCS_CONN_ID="aws_default" + +Google Cloud Storage Example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: cfg + + [cosmos] + dbt_docs_dir = gs://my-bucket/path/to/docs + dbt_docs_conn_id = google_cloud_default + +.. code-block:: shell + + AIRFLOW__COSMOS__DBT_DOCS_DIR="gs://my-bucket/path/to/docs" + AIRFLOW__COSMOS__DBT_DOCS_CONN_ID="google_cloud_default" + +Azure Blob Storage Example +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: cfg + + [cosmos] + dbt_docs_dir = wasb://my-container/path/to/docs + dbt_docs_conn_id = wasb_default + +.. code-block:: shell + + AIRFLOW__COSMOS__DBT_DOCS_DIR="wasb://my-container/path/to/docs" + AIRFLOW__COSMOS__DBT_DOCS_CONN_ID="wasb_default" + +Host from Local Storage +~~~~~~~~~~~~~~~~~~~~~~~ + +By default, Cosmos will not generate docs on the fly. Local storage only works if you are pre-compiling your dbt project before deployment. + +If your Airflow deployment process involves running ``dbt compile``, you will also want to add ``dbt docs generate`` to your deployment process as well to generate all the artifacts necessary to run the dbt docs from local storage. + +By default, dbt docs are generated in the ``target`` folder; so that will also be your docs folder by default. + +For example, if your dbt project directory is ``/usr/local/airflow/dags/my_dbt_project``, then by default your dbt docs directory will be ``/usr/local/airflow/dags/my_dbt_project/target``: + +.. code-block:: cfg + + [cosmos] + dbt_docs_dir = /usr/local/airflow/dags/my_dbt_project/target + +.. code-block:: shell + + AIRFLOW__COSMOS__DBT_DOCS_DIR="/usr/local/airflow/dags/my_dbt_project/target" + +Using docs out of local storage has a couple downsides. First, some values in the dbt docs can become stale, unless the docs are periodically refreshed and redeployed: + +- Counts of the numbers of rows. +- The compiled SQL for incremental models before and after the first run. + +Second, deployment from local storage may only be partially compatible with some managed Airflow systems. +Compatibility will depend on the managed Airflow system, as each one works differently. + +For example, Astronomer does not update the resources available to the webserver instance when ``--dags`` is specified during deployment, meaning that the dbt dcs will not be updated when this flag is used. + +.. note:: + Managed Airflow on Astronomer Cloud does not provide the webserver access to the DAGs folder. + If you want to host your docs in local storage with Astro, you should host them in a directory other than ``dags/``. + For example, you can set your ``AIRFLOW__COSMOS__DBT_DOCS_DIR`` to ``/usr/local/airflow/dbt_docs_dir`` with the following pre-deployment script: + + .. code-block:: bash + + dbt docs generate + mkdir dbt_docs_dir + cp dags/dbt/target/manifest.json dbt_docs_dir/manifest.json + cp dags/dbt/target/catalog.json dbt_docs_dir/catalog.json + cp dags/dbt/target/index.html dbt_docs_dir/index.html + +Host from HTTP/HTTPS +~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: cfg + + [cosmos] + dbt_docs_dir = https://my-site.com/path/to/docs + +.. code-block:: shell + + AIRFLOW__COSMOS__DBT_DOCS_DIR="https://my-site.com/path/to/docs" + + +You do not need to set a ``dbt_docs_conn_id`` when using HTTP/HTTPS. +If you do set the ``dbt_docs_conn_id``, then the ``HttpHook`` will be used. diff --git a/docs/configuration/index.rst b/docs/configuration/index.rst index 2c027e32d0..90f1959385 100644 --- a/docs/configuration/index.rst +++ b/docs/configuration/index.rst @@ -14,10 +14,15 @@ Cosmos offers a number of configuration options to customize its behavior. For m Render Config Parsing Methods + Configuring in Airflow Configuring Lineage Generating Docs + Hosting Docs Scheduling Testing Behavior Selecting & Excluding + Partial Parsing Operator Args Compiled SQL + Logging + Caching diff --git a/docs/configuration/lineage.rst b/docs/configuration/lineage.rst index bf099f344e..8dfcae51b9 100644 --- a/docs/configuration/lineage.rst +++ b/docs/configuration/lineage.rst @@ -9,7 +9,7 @@ and virtualenv execution methods (read `execution modes <../getting_started/exec To emit lineage events, Cosmos can use one of the following: -1. Airflow 2.7 `built-in support to OpenLineage `_, or +1. Airflow `official OpenLineage provider `_, or 2. `Additional libraries `_. No change to the user DAG files is required to use OpenLineage. @@ -18,7 +18,7 @@ No change to the user DAG files is required to use OpenLineage. Installation ------------ -If using Airflow 2.7, no other dependency is required. +If using Airflow 2.7 or higher, install ``apache-airflow-providers-openlineage``. Otherwise, install Cosmos using ``astronomer-cosmos[openlineage]``. diff --git a/docs/configuration/logging.rst b/docs/configuration/logging.rst new file mode 100644 index 0000000000..9c27a950c2 --- /dev/null +++ b/docs/configuration/logging.rst @@ -0,0 +1,19 @@ +.. _logging: + +Logging +==================== + +Cosmos uses a custom logger implementation so that all log messages are clearly tagged with ``(astronomer-cosmos)``. By default this logger has propagation enabled. + +In some environments (for example when running Celery workers) this can cause duplicated log messages to appear in the logs. In this case log propagation can be disabled via airflow configuration using the boolean option ``propagate_logs`` under a ``cosmos`` section. + +.. code-block:: cfg + + [cosmos] + propagate_logs = False + +or + +.. code-block:: python + + AIRFLOW__COSMOS__PROPAGATE_LOGS = "False" diff --git a/docs/configuration/operator-args.rst b/docs/configuration/operator-args.rst index 9d533bf137..d78b6f1b79 100644 --- a/docs/configuration/operator-args.rst +++ b/docs/configuration/operator-args.rst @@ -43,17 +43,18 @@ Summary of Cosmos-specific arguments dbt-related ........... -- ``append_env``: Expose the operating system environment variables to the ``dbt`` command. The default is ``False``. +- ``append_env``: Expose the operating system environment variables to the ``dbt`` command. For most execution modes, the default is ``False``, however, for execution modes ExecuteMode.LOCAL and ExecuteMode.VIRTUALENV, the default is True. This behavior is consistent with the LoadMode.DBT_LS command in forwarding the environment variables to the subprocess by default. - ``dbt_cmd_flags``: List of command flags to pass to ``dbt`` command, added after dbt subcommand - ``dbt_cmd_global_flags``: List of ``dbt`` `global flags `_ to be passed to the ``dbt`` command, before the subcommand - ``dbt_executable_path``: Path to dbt executable. -- ``env``: Declare, using a Python dictionary, values to be set as environment variables when running ``dbt`` commands. +- ``env``: (Deprecated since Cosmos 1.3 use ``ProjectConfig.env_vars`` instead) Declare, using a Python dictionary, values to be set as environment variables when running ``dbt`` commands. - ``fail_fast``: ``dbt`` exits immediately if ``dbt`` fails to process a resource. - ``models``: Specifies which nodes to include. - ``no_version_check``: If set, skip ensuring ``dbt``'s version matches the one specified in the ``dbt_project.yml``. - ``quiet``: run ``dbt`` in silent mode, only displaying its error logs. -- ``vars``: Supply variables to the project. This argument overrides variables defined in the ``dbt_project.yml``. +- ``vars``: (Deprecated since Cosmos 1.3 use ``ProjectConfig.dbt_vars`` instead) Supply variables to the project. This argument overrides variables defined in the ``dbt_project.yml``. - ``warn_error``: convert ``dbt`` warnings into errors. +- ``full_refresh``: If True, then full refresh the node. This only applies to model and seed nodes. Airflow-related ............... @@ -88,3 +89,33 @@ Sample usage "skip_exit_code": 1, } ) + + +Template fields +--------------- + +Some of the operator args are `template fields `_ for your convenience. + +These template fields can be useful for hooking into Airflow `Params `_, or for more advanced customization with `XComs `_. + +The following operator args support templating, and are accessible both through the ``DbtDag`` and ``DbtTaskGroup`` constructors in addition to being accessible standalone: + +- ``env`` +- ``vars`` +- ``full_refresh`` (for the ``build``, ``seed``, and ``run`` operators since Cosmos 1.4.) + +.. note:: + Using Jinja templating for ``env`` and ``vars`` may cause problems when using ``LoadMode.DBT_LS`` to render your DAG. + +The following template fields are only selectable when using the operators in a standalone context (starting in Cosmos 1.4): + +- ``select`` +- ``exclude`` +- ``selector`` +- ``models`` + +Since Airflow resolves template fields during Airflow DAG execution and not DAG parsing, the args above cannot be templated via ``DbtDag`` and ``DbtTaskGroup`` because both need to select dbt nodes during DAG parsing. + +Additionally, the SQL for compiled dbt models is stored in the template fields, which is viewable in the Airflow UI for each task run. +This is provided for telemetry on task execution, and is not an operator arg. +For more information about this, see the `Compiled SQL `_ docs. diff --git a/docs/configuration/parsing-methods.rst b/docs/configuration/parsing-methods.rst index fbf6e43bfd..ebd6030e6b 100644 --- a/docs/configuration/parsing-methods.rst +++ b/docs/configuration/parsing-methods.rst @@ -8,13 +8,15 @@ Cosmos offers several options to parse your dbt project: - ``automatic``. Tries to find a user-supplied ``manifest.json`` file. If it can't find one, it will run ``dbt ls`` to generate one. If that fails, it will use Cosmos' dbt parser. - ``dbt_manifest``. Parses a user-supplied ``manifest.json`` file. This can be generated manually with dbt commands or via a CI/CD process. - ``dbt_ls``. Parses a dbt project directory using the ``dbt ls`` command. +- ``dbt_ls_file``. Parses a dbt project directory using the output of ``dbt ls`` command from a file. - ``custom``. Uses Cosmos' custom dbt parser, which extracts dependencies from your dbt's model code. There are benefits and drawbacks to each method: - ``dbt_manifest``: You have to generate the manifest file on your own. When using the manifest, Cosmos gets a complete set of metadata about your models. However, Cosmos uses its own selecting & excluding logic to determine which models to run, which may not be as robust as dbt's. - ``dbt_ls``: Cosmos will generate the manifest file for you. This method uses dbt's metadata AND dbt's selecting/excluding logic. This is the most robust method. However, this requires the dbt executable to be installed on your machine (either on the host directly or in a virtual environment). -- ``custom``: Cosmos will parse your project and model files for you. This means that Cosmos will not have access to dbt's metadata. However, this method does not require the dbt executable to be installed on your machine. +- ``dbt_ls_file`` (new in 1.3): Path to a file containing the ``dbt ls`` output. To use this method, run ``dbt ls`` using ``--output json`` and store the output in a file. ``RenderConfig.select`` and ``RenderConfig.exclude`` will not work using this method. +- ``custom``: Cosmos will parse your project and model files. This means that Cosmos will not have access to dbt's metadata. However, this method does not require the dbt executable to be installed on your machine, and does not require the user to provide any dbt artifacts. If you're using the ``local`` mode, you should use the ``dbt_ls`` method. @@ -49,7 +51,7 @@ To use this: ), render_config=RenderConfig( load_method=LoadMode.DBT_MANIFEST, - ) + ), # ..., ) @@ -58,12 +60,15 @@ To use this: .. note:: - This only works for the ``local`` execution mode. + This only works if a dbt command / executable is available to the scheduler. If you don't have a ``manifest.json`` file, Cosmos will attempt to generate one from your dbt project. It does this by running ``dbt ls`` and parsing the output. When Cosmos runs ``dbt ls``, it also passes your ``select`` and ``exclude`` arguments to the command. This means that Cosmos will only generate a manifest for the models you want to run. +Starting in Cosmos 1.5, Cosmos will cache the output of the ``dbt ls`` command, to improve the performance of this +parsing method. Learn more `here <./caching.html>`_. + To use this: .. code-block:: python @@ -75,6 +80,26 @@ To use this: # ..., ) +``dbt_ls_file`` +---------------- + +.. note:: + New in Cosmos 1.3. + +If you provide the output of ``dbt ls --output json`` as a file, you can use this to parse similar to ``dbt_ls``. +You can supply a ``dbt_ls_path`` parameter on the DbtDag / DbtTaskGroup with a path to a ``dbt_ls_output.txt`` file. +Check `this Dag `_ for an example. + +To use this: + +.. code-block:: python + + DbtDag( + render_config=RenderConfig( + load_method=LoadMode.DBT_LS_FILE, dbt_ls_path="/path/to/dbt_ls_file.txt" + ) + # ..., + ) ``custom`` ---------- diff --git a/docs/configuration/partial-parsing.rst b/docs/configuration/partial-parsing.rst new file mode 100644 index 0000000000..911e828b3f --- /dev/null +++ b/docs/configuration/partial-parsing.rst @@ -0,0 +1,68 @@ +.. _partial-parsing: + +Partial parsing +=============== + +Starting in the 1.4 version, Cosmos tries to leverage dbt's partial parsing (``partial_parse.msgpack``) to speed up both the task execution and the DAG parsing (if using ``LoadMode.DBT_LS``). + +This feature is bound to `dbt partial parsing limitations `_. +As an example, ``dbt`` requires the same ``--vars``, ``--target``, ``--profile``, and ``profile.yml`` environment variables (as called by the ``env_var()`` macro) while running dbt commands, otherwise it will reparse the project from scratch. + +Profile configuration +--------------------- + +To respect the dbt requirement of having the same profile to benefit from partial parsing, Cosmos users should either: +* If using Cosmos profile mapping (``ProfileConfig(profile_mapping=...``), disable using mocked profile mappings by setting ``render_config=RenderConfig(enable_mock_profile=False)`` +* Declare their own ``profiles.yml`` file, via ``ProfileConfig(profiles_yml_filepath=...)`` + +If users don't follow these guidelines, Cosmos will use different profiles to parse the dbt project and to run tasks, and the user won't leverage dbt partial parsing. +Their logs will contain multiple ``INFO`` messages similar to the following, meaning that Cosmos is not using partial parsing: + +.. code-block:: + + 13:33:16 Unable to do partial parsing because profile has changed + 13:33:16 Unable to do partial parsing because env vars used in profiles.yml have changed + +dbt vars +-------- + +If the Airflow scheduler and worker processes run in the same node, users must ensure the dbt ``--vars`` flag is the same in the ``RenderConfig`` and ``ExecutionConfig``. + +Otherwise, users may see messages similar to the following in their logs: + +.. code-block:: + + [2024-03-14, 17:04:57 GMT] {{subprocess.py:94}} INFO - Unable to do partial parsing because config vars, config profile, or config target have changed + + +Caching +------- + +If the dbt project ``target`` directory has a ``partial_parse.msgpack``, Cosmos will attempt to use it. + +There is a chance, however, that the file is stale or was generated in a way that is different to how Cosmos runs the dbt commands. + +Therefore, Cosmos also caches the most up-to-date ``partial_parse.msgpack`` file after running a dbt command in the `system temporary directory `_. +With this, unless there are code changes, each Airflow node should only run the dbt command with a full dbt project parse once, and benefit from partial parsing from then onwards. + + +Caching is enabled by default. +It is possible to disable caching or override the directory that Cosmos uses caching with the Airflow configuration: + +.. code-block:: cfg + + [cosmos] + cache_dir = path/to/docs/here # to override default caching directory (by default, uses the system temporary directory) + enable_cache = False # to disable caching (enabled by default) + +Or environment variable: + +.. code-block:: cfg + + AIRFLOW__COSMOS__CACHE_DIR="path/to/docs/here" # to override default caching directory (by default, uses the system temporary directory) + AIRFLOW__COSMOS__ENABLE_CACHE="False" # to disable caching (enabled by default) + +Disabling +--------- + +To switch off partial parsing in Cosmos, use the argument ``partial_parse=False`` in the ``ProjectConfig``. diff --git a/docs/configuration/project-config.rst b/docs/configuration/project-config.rst index c1d952f6e1..2882ee9cc3 100644 --- a/docs/configuration/project-config.rst +++ b/docs/configuration/project-config.rst @@ -1,8 +1,8 @@ Project Config ================ -The ``cosmos.config.ProjectConfig`` allows you to specify information about where your dbt project is located. It -takes the following arguments: +The ``cosmos.config.ProjectConfig`` allows you to specify information about where your dbt project is located and project +variables that should be used for rendering and execution. It takes the following arguments: - ``dbt_project_path``: The full path to your dbt project. This directory should have a ``dbt_project.yml`` file - ``models_relative_path``: The path to your models directory, relative to the ``dbt_project_path``. This defaults to @@ -16,7 +16,16 @@ takes the following arguments: - ``project_name`` : The name of the project. If ``dbt_project_path`` is provided, the ``project_name`` defaults to the folder name containing ``dbt_project.yml``. If ``dbt_project_path`` is not provided, and ``manifest_path`` is provided, ``project_name`` is required as the name can not be inferred from ``dbt_project_path`` - +- ``dbt_vars``: (new in v1.3) A dictionary of dbt variables for the project rendering and execution. This argument overrides variables + defined in the dbt_project.yml file. The dictionary of variables is dumped to a yaml string and passed to dbt commands + as the --vars argument. Variables are only supported for rendering when using ``RenderConfig.LoadMode.DBT_LS`` and + ``RenderConfig.LoadMode.CUSTOM`` load mode. Variables using `Airflow templating `_ + will only be rendered at execution time, not at render time. +- ``env_vars``: (new in v1.3) A dictionary of environment variables used for rendering and execution. Rendering with + env vars is only supported when using ``RenderConfig.LoadMode.DBT_LS`` load mode. +- ``partial_parse``: (new in v1.4) If True, then attempt to use the ``partial_parse.msgpack`` if it exists. This is only used + for the ``LoadMode.DBT_LS`` load mode, and for the ``ExecutionMode.LOCAL`` and ``ExecutionMode.VIRTUALENV`` + execution modes. Due to the way that dbt `partial parsing works `_, it does not work with Cosmos profile mapping classes. To benefit from this feature, users have to set the ``profiles_yml_filepath`` argument in ``ProfileConfig``. Project Config Example ---------------------- @@ -31,4 +40,10 @@ Project Config Example seeds_relative_path="data", snapshots_relative_path="snapshots", manifest_path="/path/to/manifests", + env_vars={"MY_ENV_VAR": "my_env_value"}, + dbt_vars={ + "my_dbt_var": "my_value", + "start_time": "{{ data_interval_start.strftime('%Y%m%d%H%M%S') }}", + "end_time": "{{ data_interval_end.strftime('%Y%m%d%H%M%S') }}", + }, ) diff --git a/docs/configuration/render-config.rst b/docs/configuration/render-config.rst index de0a08cdbe..4b2535e076 100644 --- a/docs/configuration/render-config.rst +++ b/docs/configuration/render-config.rst @@ -7,14 +7,17 @@ It does this by exposing a ``cosmos.config.RenderConfig`` class that you can use The ``RenderConfig`` class takes the following arguments: -- ``emit_datasets``: whether or not to emit Airflow datasets to be used for data-aware scheduling. Defaults to True +- ``emit_datasets``: whether or not to emit Airflow datasets to be used for data-aware scheduling. Defaults to True. Depends on `additional dependencies `_. - ``test_behavior``: how to run tests. Defaults to running a model's tests immediately after the model is run. For more information, see the `Testing Behavior `_ section. - ``load_method``: how to load your dbt project. See `Parsing Methods `_ for more information. - ``select`` and ``exclude``: which models to include or exclude from your DAGs. See `Selecting & Excluding `_ for more information. +- ``selector``: (new in v1.3) name of a dbt YAML selector to use for DAG parsing. Only supported when using ``load_method=LoadMode.DBT_LS``. See `Selecting & Excluding `_ for more information. - ``dbt_deps``: A Boolean to run dbt deps when using dbt ls for dag parsing. Default True - ``node_converters``: a dictionary mapping a ``DbtResourceType`` into a callable. Users can control how to render dbt nodes in Airflow. Only supported when using ``load_method=LoadMode.DBT_MANIFEST`` or ``LoadMode.DBT_LS``. Find more information below. - ``dbt_executable_path``: The path to the dbt executable for dag generation. Defaults to dbt if available on the path. +- ``env_vars``: (available in v1.2.5, use``ProjectConfig.env_vars`` for v1.3.0 onwards) A dictionary of environment variables for rendering. Only supported when using ``load_method=LoadMode.DBT_LS``. - ``dbt_project_path``: Configures the DBT project location accessible on their airflow controller for DAG rendering - Required when using ``load_method=LoadMode.DBT_LS`` or ``load_method=LoadMode.CUSTOM`` +- ``airflow_vars_to_purge_cache``: (new in v1.5) Specify Airflow variables that will affect the ``LoadMode.DBT_LS`` cache. See `Caching <./caching.html>`_ for more information. Customizing how nodes are rendered (experimental) ------------------------------------------------- diff --git a/docs/configuration/scheduling.rst b/docs/configuration/scheduling.rst index de21f84950..d96930395e 100644 --- a/docs/configuration/scheduling.rst +++ b/docs/configuration/scheduling.rst @@ -17,18 +17,24 @@ To schedule a dbt project on a time-based schedule, you can use Airflow's schedu jaffle_shop = DbtDag( # ... start_date=datetime(2023, 1, 1), - schedule_interval="@daily", + schedule="@daily", ) Data-Aware Scheduling --------------------- -By default, Cosmos emits `Airflow Datasets `_ when running dbt projects. This allows you to use Airflow's data-aware scheduling capabilities to schedule your dbt projects. Cosmos emits datasets in the following format: +Apache Airflow 2.4 introduced the concept of `scheduling based on Datasets `_. + +By default, if Airflow 2.4 or higher is used, Cosmos emits `Airflow Datasets `_ when running dbt projects. This allows you to use Airflow's data-aware scheduling capabilities to schedule your dbt projects. Cosmos emits datasets using the OpenLineage URI format, as detailed in the `OpenLineage Naming Convention `_. + +Cosmos calculates these URIs during the task execution, by using the library `OpenLineage Integration Common `_. + +This block illustrates a Cosmos-generated dataset for Postgres: .. code-block:: python - Dataset("DBT://{connection_id}/{project_name}/{model_name}") + Dataset("postgres://host:5432/database.schema.table") For example, let's say you have: @@ -36,21 +42,22 @@ For example, let's say you have: - A dbt project (``project_one``) with a model called ``my_model`` that runs daily - A second dbt project (``project_two``) with a model called ``my_other_model`` that you want to run immediately after ``my_model`` +We are assuming that the Database used is Postgres, the host is ``host``, the database is ``database`` and the schema is ``schema``. + Then, you can use Airflow's data-aware scheduling capabilities to schedule ``my_other_model`` to run after ``my_model``. For example, you can use the following DAGs: .. code-block:: python - from cosmos import DbtDag, get_dbt_dataset + from cosmos import DbtDag project_one = DbtDag( # ... start_date=datetime(2023, 1, 1), - schedule_interval="@daily", + schedule="@daily", ) project_two = DbtDag( - # ... - schedule_interval=[get_dbt_dataset("my_conn", "project_one", "my_model")], + schedule=[Dataset("postgres://host:5432/database.schema.my_model")], dbt_project_name="project_two", ) diff --git a/docs/configuration/selecting-excluding.rst b/docs/configuration/selecting-excluding.rst index fadea1485f..01ee536b0a 100644 --- a/docs/configuration/selecting-excluding.rst +++ b/docs/configuration/selecting-excluding.rst @@ -3,14 +3,22 @@ Selecting & Excluding ======================= -Cosmos allows you to filter to a subset of your dbt project in each ``DbtDag`` / ``DbtTaskGroup`` using the ``select`` and ``exclude`` parameters in the ``RenderConfig`` class. +Cosmos allows you to filter to a subset of your dbt project in each ``DbtDag`` / ``DbtTaskGroup`` using the ``select `` and ``exclude`` parameters in the ``RenderConfig`` class. + + Since Cosmos 1.3, the ``selector`` parameter is also available in ``RenderConfig`` when using the ``LoadMode.DBT_LS`` to parse the dbt project into Airflow. + + +Using ``select`` and ``exclude`` +-------------------------------- The ``select`` and ``exclude`` parameters are lists, with values like the following: - ``tag:my_tag``: include/exclude models with the tag ``my_tag`` - ``config.materialized:table``: include/exclude models with the config ``materialized: table`` - ``path:analytics/tables``: include/exclude models in the ``analytics/tables`` directory - +- ``+node_name+1`` (graph operators): include/exclude the node with name ``node_name``, all its parents, and its first generation of children (`dbt graph selector docs `_) +- ``tag:my_tag,+node_name`` (intersection): include/exclude ``node_name`` and its parents if they have the tag ``my_tag`` (`dbt set operator docs `_) +- ``['tag:first_tag', 'tag:second_tag']`` (union): include/exclude nodes that have either ``tag:first_tag`` or ``tag:second_tag`` .. note:: @@ -51,3 +59,55 @@ Examples: select=["path:analytics/tables"], ) ) + + +.. code-block:: python + + from cosmos import DbtDag, RenderConfig + + jaffle_shop = DbtDag( + render_config=RenderConfig( + select=["tag:include_tag1", "tag:include_tag2"], # union + ) + ) + +.. code-block:: python + + from cosmos import DbtDag, RenderConfig + + jaffle_shop = DbtDag( + render_config=RenderConfig( + select=["tag:include_tag1,tag:include_tag2"], # intersection + ) + ) + +.. code-block:: python + + from cosmos import DbtDag, RenderConfig + + jaffle_shop = DbtDag( + render_config=RenderConfig( + exclude=["node_name+"], # node_name and its children + ) + ) + +Using ``selector`` +-------------------------------- +.. note:: + Only currently supported using the ``dbt_ls`` parsing method since Cosmos 1.3 where the selector is passed directly to the dbt CLI command. \ + If ``select`` and/or ``exclude`` are used with ``selector``, dbt will ignore the ``select`` and ``exclude`` parameters. + +The ``selector`` parameter is a string that references a `dbt YAML selector `_ already defined in a dbt project. + +Examples: + +.. code-block:: python + + from cosmos import DbtDag, RenderConfig, LoadMode + + jaffle_shop = DbtDag( + render_config=RenderConfig( + selector="my_selector", # this selector must be defined in your dbt project + load_method=LoadMode.DBT_LS, + ) + ) diff --git a/docs/configuration/testing-behavior.rst b/docs/configuration/testing-behavior.rst index 951c90ca57..7af9ceedd9 100644 --- a/docs/configuration/testing-behavior.rst +++ b/docs/configuration/testing-behavior.rst @@ -37,7 +37,7 @@ Warning Behavior .. note:: - As of now, this feature is only available for the default execution mode ``local`` + As of now, this feature is only available for the default execution mode ``local`` and for ``virtualenv`` Cosmos enables you to receive warning notifications from tests and process them using a callback function. The ``on_warning_callback`` parameter adds two extra context variables to the callback function: ``test_names`` and ``test_results``. diff --git a/docs/contributing.rst b/docs/contributing.rst index f875538839..56826cfc8d 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -1,3 +1,5 @@ +.. _contributing: + Cosmos Contributing Guide ========================= @@ -6,6 +8,8 @@ All contributions, bug reports, bug fixes, documentation improvements, enhanceme As contributors and maintainers to this project, you are expected to abide by the `Contributor Code of Conduct `_. +Learn more about the contributors roles in :ref:`contributors-roles`. + Overview ________ @@ -31,8 +35,8 @@ Then install ``airflow`` and ``astronomer-cosmos`` using python-venv: .. code-block:: bash python3 -m venv env && source env/bin/activate - pip3 install apache-airflow[cncf.kubernetes,openlineage] - pip3 install -e .[dbt-postgres,dbt-databricks] + pip3 install "apache-airflow[cncf.kubernetes,openlineage]" + pip3 install -e ".[dbt-postgres,dbt-databricks]" Set airflow home to the ``dev/`` directory and disabled loading example DAGs: diff --git a/docs/contributors-roles.rst b/docs/contributors-roles.rst new file mode 100644 index 0000000000..7cddfb95fd --- /dev/null +++ b/docs/contributors-roles.rst @@ -0,0 +1,56 @@ +.. _contributors-roles: + +Contributor roles +================== + +Contributors are welcome and are greatly appreciated! Every little bit helps, and we give credit to them. + +This document aims to explain the current roles in the Astronomer Cosmos project. +For more information, check :ref:`contributing` and :ref:`contributors`. + + +Contributors +------------ + +A contributor is anyone who wants to contribute code, documentation, tests, ideas, or anything to the Astronomer Cosmos project. + +Cosmos contributors can be found in the Astronomer Cosmos Github `insights page `_ and in the `#airflow-dbt `_ Slack channel. + +Contributors are responsible for: + +* Fixing bugs +* Refactoring code +* Improving processes and tooling +* Adding features +* Improving the documentation +* Making/answering questions in the #airflow-dbt Slack channel + + +Committers +---------------------- + +Committers are community members with write access to the `Astronomer Cosmos Github repository `_. +They can modify the code and the documentation and accept others' contributions to the repo. + +Check :ref:`contributors` for the official list of Astronomer Cosmos committers. + +Committers have the same responsibilities as standard contributors and also perform the following actions: + +* Reviewing & merging pull-requests +* Scanning and responding to GitHub issues, helping triaging them + +If you know you are not going to be able to contribute for a long time (for instance, due to a change of job or circumstances), you should inform other maintainers, and we will mark you as "emeritus". +Emeritus committers will no longer have write access to the repo. +As merit earned never expires, once an emeritus committer becomes active again, they can simply email another maintainer from Astronomer and ask to be reinstated. + +Pre-requisites to becoming a committer +....................................... + +General prerequisites that we look for in all candidates: + +1. Consistent contribution over last few months +2. Visibility on discussions on the Slack channel or GitHub issues/discussions +3. Contributions to community health and project's sustainability for the long-term +4. Understands the project's contributors guidelines :ref:`contributing`. +Astronomer is responsible and accountable for releasing new versions of Cosmos in PyPI , following the milestones . +Astronomer has the right to grant and revoke write access permissions to the project's official repository for any reason it sees fit. diff --git a/docs/contributors.rst b/docs/contributors.rst new file mode 100644 index 0000000000..865d648748 --- /dev/null +++ b/docs/contributors.rst @@ -0,0 +1,27 @@ +.. _contributors: +Contributors +============ + +There are different ways people can contribute to Astronomer Cosmos. +Learn more about the project contributors roles in :ref:`contributors-roles`. + +Committers +---------------------- + +* Daniel Reeves (`@dwreeves `_) +* Julian LaNeve (`@jlaneve `_) +* Justin Bandoro (`@jbandoro `_) +* Tatiana Al-Chueyr (`@tatiana `_) + + +Emeritus Committers +------------------------------- + +* Chris Hronek (`@chrishronek `_) + + +Contributors +------------ + +Many people are improving Astronomer Cosmos each day. +Find more contributors `in our Github page `_ and in the `#airflow-dbt `_ Slack channel. diff --git a/docs/generate_mappings.py b/docs/generate_mappings.py index f7f0696cf7..049eb8b4e1 100644 --- a/docs/generate_mappings.py +++ b/docs/generate_mappings.py @@ -1,6 +1,7 @@ """ Script to generate a dedicated docs page per profile mapping. """ + from __future__ import annotations import os @@ -8,12 +9,14 @@ from typing import Type from jinja2 import Environment, FileSystemLoader -from cosmos.profiles import profile_mappings, BaseProfileMapping + +from cosmos.profiles import BaseProfileMapping, profile_mappings @dataclass class Field: - "Represents a field in a profile mapping." + """Represents a field in a profile mapping.""" + dbt_name: str required: bool = False airflow_name: str | list[str] | None = None diff --git a/docs/getting_started/astro.rst b/docs/getting_started/astro.rst index c0bedc7e64..f7e9942bc5 100644 --- a/docs/getting_started/astro.rst +++ b/docs/getting_started/astro.rst @@ -5,6 +5,10 @@ Getting Started on Astro While it is possible to use Cosmos on Astro with all :ref:`Execution Modes `, we recommend using the ``local`` execution mode. It's the simplest to set up and use. +If you'd like to see a fully functional project to run in Astro (CLI or Cloud), check out `cosmos-demo `_. + +Below you can find a step-by-step guide to run your own dbt project within Astro. + Pre-requisites ~~~~~~~~~~~~~~ @@ -12,6 +16,7 @@ To get started, you should have: - The Astro CLI installed. You can find installation instructions `here `_. - An Astro CLI project. You can initialize a new project with ``astro dev init``. +- A dbt project. The `jaffle shop example `_ is a good example. Create a virtual environment ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -20,12 +25,13 @@ Create a virtual environment in your ``Dockerfile`` using the sample below. Be s .. code-block:: docker - FROM quay.io/astronomer/astro-runtime:8.8.0 + FROM quay.io/astronomer/astro-runtime:11.3.0 # install dbt into a virtual environment RUN python -m venv dbt_venv && source dbt_venv/bin/activate && \ pip install --no-cache-dir && deactivate +An example of dbt adapter is ``dbt-postgres``. Install Cosmos ~~~~~~~~~~~~~~ diff --git a/docs/getting_started/azure-container-instance.rst b/docs/getting_started/azure-container-instance.rst new file mode 100644 index 0000000000..8f979f514c --- /dev/null +++ b/docs/getting_started/azure-container-instance.rst @@ -0,0 +1,135 @@ +.. _azure-container-instance + +Azure Container Instance Execution Mode +======================================= +.. versionadded:: 1.4 +This tutorial will guide you through the steps required to use Azure Container Instance as the Execution Mode for your dbt code with Astronomer Cosmos. Schematically, the guide will walk you through the steps required to build the following architecture: + +.. figure:: https://github.com/astronomer/astronomer-cosmos/raw/main/docs/_static/cosmos_aci_schematic.png + :width: 800 + +Prerequisites ++++++++++++++ +1. Docker with docker daemon (Docker Desktop on MacOS). Follow the `Docker installation guide `_. +2. Airflow +3. Azure CLI (install guide here: `Azure CLI `_) +4. Astronomer-cosmos package containing the dbt Azure Container Instance operators +5. Azure account with: + 1. A resource group + 2. A service principal with `Contributor` permissions on the resource group + 3. A Container Registry + 4. A Postgres instance accessible from Azure. (we use an Azure Postgres instance in the example) +6. Docker image built with required dbt project and dbt DAG +7. dbt DAG with dbt Azure Container Instance operators in the Airflow DAGs directory to run in Airflow + +More information on how to achieve 2-6 is detailed below. + +Note that the steps below will walk you through an example, for which the code can be found HERE + +Step-by-step guide +++++++++++++++++++ + +**Install Airflow and Cosmos** + +Create a python virtualenv, activate it, upgrade pip to the latest version and install apache airflow & astronomer-postgres + +.. code-block:: bash + + python -m venv venv + source venv/bin/activate + pip install --upgrade pip + pip install apache-airflow + pip install "astronomer-cosmos[dbt-postgres,azure-container-instance]" + +**Setup Postgres database** + +You will need a postgres database running to be used as the database for the dbt project. In order to have it accessible from Azure Container Instance, the easiest way is to create an Azure Postgres instance. For this, run the following (assuming you are logged into your Azure account) + +.. code-block:: bash + + az postgres server create -l westeurope -g <<>> -n <<>> -u dbadmin -p <<>> --sku-name B_Gen5_1 --ssl-enforcement Enabled + + +**Setup Azure Container Registry** +In order to run a container in Azure Container Instance, it needs access to the container image. In our setup, we will use Azure Container Registry for this. To set an Azure Container Registry up, you can use the following bash command: + +.. code-block:: bash + az acr create --name <<>> --resource-group <<>> --sku Basic --admin-enabled + +**Build the dbt Docker image** + +For the Docker operators to work, you need to create a docker image that will be supplied as image parameter to the dbt docker operators used in the DAG. + +Clone the `cosmos-example `_ repo + +.. code-block:: bash + + git clone https://github.com/astronomer/cosmos-example.git + cd cosmos-example + +Create a docker image containing the dbt project files and dbt profile by using the `Dockerfile `_, which will be supplied to the Docker operators. + +.. code-block:: bash + + docker build -t <<>:1.0.0 -f Dockerfile.azure_container_instance . + +After this, the image needs to be pushed to the registry of your choice. Note that your image name should contain the name of your registry: +.. code-block:: bash + + docker push <<>>:1.0.0 + +.. note:: + + You may need to ensure docker knows to use the right credentials. If using Azure Container Registry, this can be done by running the following command: + .. code-block:: bash + az acr login + +.. note:: + + If running on M1, you may need to set the following envvars for running the docker build command in case it fails + + .. code-block:: bash + + export DOCKER_BUILDKIT=0 + export COMPOSE_DOCKER_CLI_BUILD=0 + export DOCKER_DEFAULT_PLATFORM=linux/amd64 + +Take a read of the Dockerfile to understand what it does so that you could use it as a reference in your project. + + - The `dbt profile `_ file is added to the image + - The dags directory containing the `dbt project jaffle_shop `_ is added to the image + - The dbt_project.yml is replaced with `postgres_profile_dbt_project.yml `_ which contains the profile key pointing to postgres_profile as profile creation is not handled at the moment for K8s operators like in local mode. + +**Setup Airflow Connections** +Now you have the required Azure infrastructure, you still need to add configuration to Airflow to ensure the infrastructure can be used. You'll need 3 connections: + +1. ``aci_db``: a Postgres connection to your Azure Postgres instance. +2. ``aci``: an Azure Container Instance connection configured with a Service Principal with sufficient permissions (i.e. ``Contributor`` on the resource group in which you will use Azure Container Instances). +3. ``acr``: an Azure Container Registry connection configured for your Azure Container Registry. + +Check out the ``airflow-settings.yml`` file `here `_ for an example. If you are using Astro CLI, filling in the right values here will be enough for this to work. + +**Setup and Trigger the DAG with Airflow** + +Copy the dags directory from cosmos-example repo to your Airflow home + +.. code-block:: bash + + cp -r dags $AIRFLOW_HOME/ + +Run Airflow + +.. code-block:: bash + + airflow standalone + +.. note:: + + You might need to run airflow standalone with ``sudo`` if your Airflow user is not able to access the docker socket URL or pull the images in the Kind cluster. + +Log in to Airflow through a web browser ``http://localhost:8080/``, using the user ``airflow`` and the password described in the ``standalone_admin_password.txt`` file. + +Enable and trigger a run of the `jaffle_shop_azure_container_instance `_ DAG. You will be able to see the following successful DAG run. + +.. figure:: https://github.com/astronomer/astronomer-cosmos/raw/main/docs/_static/jaffle_shop_azure_container_instance.png + :width: 800 diff --git a/docs/getting_started/execution-modes-local-conflicts.rst b/docs/getting_started/execution-modes-local-conflicts.rst index 0f23ef8245..3f537baba9 100644 --- a/docs/getting_started/execution-modes-local-conflicts.rst +++ b/docs/getting_started/execution-modes-local-conflicts.rst @@ -1,32 +1,51 @@ .. _execution-modes-local-conflicts: -Airflow and DBT dependencies conflicts +Airflow and dbt dependencies conflicts ====================================== When using the `Local Execution Mode `__, users may face dependency conflicts between -Apache Airflow and DBT. The amount of conflicts may increase depending on the Airflow providers and DBT plugins being used. +Apache Airflow and dbt. The conflicts may increase depending on the Airflow providers and dbt adapters being used. If you find errors, we recommend users look into using `alternative execution modes `__. In the following table, ``x`` represents combinations that lead to conflicts (vanilla ``apache-airflow`` and ``dbt-core`` packages): -+---------------+-----+-----+-----+-----+-----+-----+---------+ -| Airflow \ DBT | 1.0 | 1.1 | 1.2 | 1.3 | 1.4 | 1.5 | 1.6.0b6 | -+===============+=====+=====+=====+=====+=====+=====+=========+ -| 2.2 | | | | x | x | x | x | -+---------------+-----+-----+-----+-----+-----+-----+---------+ -| 2.3 | x | x | | x | x | x | x | -+---------------+-----+-----+-----+-----+-----+-----+---------+ -| 2.4 | x | x | x | | | | | -+---------------+-----+-----+-----+-----+-----+-----+---------+ -| 2.5 | x | x | x | | | | | -+---------------+-----+-----+-----+-----+-----+-----+---------+ -| 2.6 | x | x | x | x | x | | x | -+---------------+-----+-----+-----+-----+-----+-----+---------+ ++---------------+-----+-----+-----+-----+-----+-----+-----+-----+-----+ +| Airflow / DBT | 1.0 | 1.1 | 1.2 | 1.3 | 1.4 | 1.5 | 1.6 | 1.7 | 1.8 | ++===============+=====+=====+=====+=====+=====+=====+=====+=====+=====+ +| 2.2 | | | | x | x | x | x | x | x | ++---------------+-----+-----+-----+-----+-----+-----+-----+-----+-----+ +| 2.3 | x | x | | x | x | x | x | x | X | ++---------------+-----+-----+-----+-----+-----+-----+-----+-----+-----+ +| 2.4 | x | x | x | | | | | | | ++---------------+-----+-----+-----+-----+-----+-----+-----+-----+-----+ +| 2.5 | x | x | x | | | | | | | ++---------------+-----+-----+-----+-----+-----+-----+-----+-----+-----+ +| 2.6 | x | x | x | x | x | | | | | ++---------------+-----+-----+-----+-----+-----+-----+-----+-----+-----+ +| 2.7 | x | x | x | x | x | | | | | ++---------------+-----+-----+-----+-----+-----+-----+-----+-----+-----+ +| 2.8 | x | x | x | x | x | | x | | | ++---------------+-----+-----+-----+-----+-----+-----+-----+-----+-----+ +| 2.9 | x | x | x | x | x | | | | | ++---------------+-----+-----+-----+-----+-----+-----+-----+-----+-----+ Examples of errors ----------------------------------- +.. code-block:: bash + + The conflict is caused by: + apache-airflow 2.8.0 depends on pydantic>=2.3.0 + dbt-semantic-interfaces 0.4.2 depends on pydantic~=1.10 + apache-airflow 2.8.0 depends on pydantic>=2.3.0 + dbt-semantic-interfaces 0.4.2.dev0 depends on pydantic~=1.10 + apache-airflow 2.8.0 depends on pydantic>=2.3.0 + dbt-semantic-interfaces 0.4.1 depends on pydantic~=1.10 + apache-airflow 2.8.0 depends on pydantic>=2.3.0 + dbt-semantic-interfaces 0.4.0 depends on pydantic~=1.10 + + .. code-block:: bash ERROR: Cannot install apache-airflow==2.2.4 and dbt-core==1.5.0 because these package versions have conflicting dependencies. @@ -41,6 +60,22 @@ Examples of errors apache-airflow 2.6.0 depends on importlib-metadata<5.0.0 and >=1.7; python_version < "3.9" dbt-semantic-interfaces 0.1.0.dev7 depends on importlib-metadata==6.6.0 +.. code-block:: bash + + ERROR: Cannot install apache-airflow, apache-airflow==2.7.0 and dbt-core==1.4.0 because these package versions have conflicting dependencies. + + The conflict is caused by: + dbt-core 1.4.0 depends on pyyaml>=6.0 + connexion 2.12.0 depends on PyYAML<6 and >=5.1 + dbt-core 1.4.0 depends on pyyaml>=6.0 + connexion 2.11.2 depends on PyYAML<6 and >=5.1 + dbt-core 1.4.0 depends on pyyaml>=6.0 + connexion 2.11.1 depends on PyYAML<6 and >=5.1 + dbt-core 1.4.0 depends on pyyaml>=6.0 + connexion 2.11.0 depends on PyYAML<6 and >=5.1 + apache-airflow 2.7.0 depends on jsonschema>=4.18.0 + flask-appbuilder 4.3.3 depends on jsonschema<5 and >=3 + connexion 2.10.0 depends on jsonschema<4 and >=2.5.1 How to reproduce ---------------- @@ -57,8 +92,12 @@ The table was created by running `nox `__ wi @nox.session(python=["3.10"]) - @nox.parametrize("dbt_version", ["1.0", "1.1", "1.2", "1.3", "1.4", "1.5", "1.6.0b6"]) - @nox.parametrize("airflow_version", ["2.2.4", "2.3", "2.4", "2.5", "2.6"]) + @nox.parametrize( + "dbt_version", ["1.0", "1.1", "1.2", "1.3", "1.4", "1.5", "1.6", "1.7", "1.8"] + ) + @nox.parametrize( + "airflow_version", ["2.2.4", "2.3", "2.4", "2.5", "2.6", "2.7", "2.8", "2.9"] + ) def compatibility(session: nox.Session, airflow_version, dbt_version) -> None: """Run both unit and integration tests.""" session.run( diff --git a/docs/getting_started/execution-modes.rst b/docs/getting_started/execution-modes.rst index 5925853fe1..266b40f323 100644 --- a/docs/getting_started/execution-modes.rst +++ b/docs/getting_started/execution-modes.rst @@ -3,18 +3,20 @@ Execution Modes =============== -Cosmos can run ``dbt`` commands using four different approaches, called ``execution modes``: +Cosmos can run ``dbt`` commands using five different approaches, called ``execution modes``: 1. **local**: Run ``dbt`` commands using a local ``dbt`` installation (default) 2. **virtualenv**: Run ``dbt`` commands from Python virtual environments managed by Cosmos 3. **docker**: Run ``dbt`` commands from Docker containers managed by Cosmos (requires a pre-existing Docker image) 4. **kubernetes**: Run ``dbt`` commands from Kubernetes Pods managed by Cosmos (requires a pre-existing Docker image) +5. **aws_eks**: Run ``dbt`` commands from AWS EKS Pods managed by Cosmos (requires a pre-existing Docker image) +6. **azure_container_instance**: Run ``dbt`` commands from Azure Container Instances managed by Cosmos (requires a pre-existing Docker image) The choice of the ``execution mode`` can vary based on each user's needs and concerns. For more details, check each execution mode described below. .. list-table:: Execution Modes Comparison - :widths: 25 25 25 25 + :widths: 20 20 20 20 20 :header-rows: 1 * - Execution Mode @@ -37,6 +39,14 @@ The choice of the ``execution mode`` can vary based on each user's needs and con - Slow - High - No + * - AWS_EKS + - Slow + - High + - No + * - Azure Container Instance + - Slow + - High + - No Local ----- @@ -51,6 +61,11 @@ The ``local`` execution mode assumes a ``dbt`` binary is reachable within the Ai If ``dbt`` was not installed as part of the Cosmos packages, users can define a custom path to ``dbt`` by declaring the argument ``dbt_executable_path``. +.. note:: + Starting in the 1.4 version, Cosmos tries to leverage the dbt partial parsing (``partial_parse.msgpack``) to speed up task execution. + This feature is bound to `dbt partial parsing limitations `_. + Learn more: :ref:`partial-parsing`. + When using the ``local`` execution mode, Cosmos converts Airflow Connections into a native ``dbt`` profiles file (``profiles.yml``). Example of how to use, for instance, when ``dbt`` was installed together with Cosmos: @@ -71,8 +86,13 @@ The ``virtualenv`` mode isolates the Airflow worker dependencies from ``dbt`` by In this case, users are responsible for declaring which version of ``dbt`` they want to use by giving the argument ``py_requirements``. This argument can be set directly in operator instances or when instantiating ``DbtDag`` and ``DbtTaskGroup`` as part of ``operator_args``. Similar to the ``local`` execution mode, Cosmos converts Airflow Connections into a way ``dbt`` understands them by creating a ``dbt`` profile file (``profiles.yml``). +Also similar to the ``local`` execution mode, Cosmos will by default attempt to use a ``partial_parse.msgpack`` if one exists to speed up parsing. + +Some drawbacks of this approach: -A drawback with this approach is that it is slower than ``local`` because it creates a new Python virtual environment for each Cosmos dbt task run. +- It is slower than ``local`` because it creates a new Python virtual environment for each Cosmos dbt task run. +- If dbt is unavailable in the Airflow scheduler, the default ``LoadMode.DBT_LS`` will not work. In this scenario, users must use a `parsing method `_ that does not rely on dbt, such as ``LoadMode.MANIFEST``. +- Only ``InvocationMode.SUBPROCESS`` is supported currently, attempt to use ``InvocationMode.DBT_RUNNER`` will raise error. Example of how to use: @@ -91,6 +111,7 @@ The user has better environment isolation than when using ``local`` or ``virtual The other challenge with the ``docker`` approach is if the Airflow worker is already running in Docker, which sometimes can lead to challenges running `Docker in Docker `__. This approach can be significantly slower than ``virtualenv`` since it may have to build the ``Docker`` container, which is slower than creating a Virtualenv with ``dbt-core``. +If dbt is unavailable in the Airflow scheduler, the default ``LoadMode.DBT_LS`` will not work. In this scenario, users must use a `parsing method `_ that does not rely on dbt, such as ``LoadMode.MANIFEST``. Check the step-by-step guide on using the ``docker`` execution mode at :ref:`docker`. @@ -113,7 +134,7 @@ Example DAG: Kubernetes ---------- -Lastly, the ``kubernetes`` approach is the most isolated way of running ``dbt`` since the ``dbt`` run commands from within a Kubernetes Pod, usually in a separate host. +The ``kubernetes`` approach is a very isolated way of running ``dbt`` since the ``dbt`` run commands from within a Kubernetes Pod, usually in a separate host. It assumes the user has a Kubernetes cluster. It also expects the user to ensure the Docker container has up-to-date ``dbt`` pipelines and profiles, potentially leading the user to declare secrets in two places (Airflow and Docker container). @@ -144,3 +165,92 @@ Example DAG: "secrets": [postgres_password_secret], }, ) +AWS_EKS +---------- + +The ``aws_eks`` approach is very similar to the ``kubernetes`` approach, but it is specifically designed to run on AWS EKS clusters. +It uses the `EKSPodOperator `_ +to run the dbt commands. You need to provide the ``cluster_name`` in your operator_args to connect to the AWS EKS cluster. + + +Example DAG: + +.. code-block:: python + + postgres_password_secret = Secret( + deploy_type="env", + deploy_target="POSTGRES_PASSWORD", + secret="postgres-secrets", + key="password", + ) + + docker_cosmos_dag = DbtDag( + # ... + execution_config=ExecutionConfig( + execution_mode=ExecutionMode.AWS_EKS, + ), + operator_args={ + "image": "dbt-jaffle-shop:1.0.0", + "cluster_name": CLUSTER_NAME, + "get_logs": True, + "is_delete_operator_pod": False, + "secrets": [postgres_password_secret], + }, + ) + +Azure Container Instance +------------------------ +.. versionadded:: 1.4 +Similar to the ``kubernetes`` approach, using ``Azure Container Instances`` as the execution mode gives a very isolated way of running ``dbt``, since the ``dbt`` run itself is run within a container running in an Azure Container Instance. + +This execution mode requires the user has an Azure environment that can be used to run Azure Container Groups in (see :ref:`azure-container-instance` for more details on the exact requirements). Similarly to the ``Docker`` and ``Kubernetes`` execution modes, a Docker container should be available, containing the up-to-date ``dbt`` pipelines and profiles. + +Each task will create a new container on Azure, giving full isolation. This, however, comes at the cost of speed, as this separation of tasks introduces some overhead. Please checkout the step-by-step guide for using Azure Container Instance as the execution mode + + +.. code-block:: python + + docker_cosmos_dag = DbtDag( + # ... + execution_config=ExecutionConfig( + execution_mode=ExecutionMode.AZURE_CONTAINER_INSTANCE + ), + operator_args={ + "ci_conn_id": "aci", + "registry_conn_id": "acr", + "resource_group": "my-rg", + "name": "my-aci-{{ ti.task_id.replace('.','-').replace('_','-') }}", + "region": "West Europe", + "image": "dbt-jaffle-shop:1.0.0", + }, + ) + + +.. _invocation_modes: +Invocation Modes +================ +.. versionadded:: 1.4 + +For ``ExecutionMode.LOCAL`` execution mode, Cosmos supports two invocation modes for running dbt: + +1. ``InvocationMode.SUBPROCESS``: In this mode, Cosmos runs dbt cli commands using the Python ``subprocess`` module and parses the output to capture logs and to raise exceptions. + +2. ``InvocationMode.DBT_RUNNER``: In this mode, Cosmos uses the ``dbtRunner`` available for `dbt programmatic invocations `__ to run dbt commands. \ + In order to use this mode, dbt must be installed in the same local environment. This mode does not have the overhead of spawning new subprocesses or parsing the output of dbt commands and is faster than ``InvocationMode.SUBPROCESS``. \ + This mode requires dbt version 1.5.0 or higher. It is up to the user to resolve :ref:`execution-modes-local-conflicts` when using this mode. + +The invocation mode can be set in the ``ExecutionConfig`` as shown below: + +.. code-block:: python + + from cosmos.constants import InvocationMode + + dag = DbtDag( + # ... + execution_config=ExecutionConfig( + execution_mode=ExecutionMode.LOCAL, + invocation_mode=InvocationMode.DBT_RUNNER, + ), + ) + +If the invocation mode is not set, Cosmos will attempt to use ``InvocationMode.DBT_RUNNER`` if dbt is installed in the same environment as the worker, otherwise it will fall back to ``InvocationMode.SUBPROCESS``. diff --git a/docs/getting_started/gcc.rst b/docs/getting_started/gcc.rst index 1ec056e842..ed4b931ce1 100644 --- a/docs/getting_started/gcc.rst +++ b/docs/getting_started/gcc.rst @@ -22,6 +22,8 @@ Make a new folder, ``dbt``, inside your local ``dags`` folder. Then, copy/paste Note: your dbt projects can go anywhere that Airflow can read. By default, Cosmos looks in the ``/usr/local/airflow/dags/dbt`` directory, but you can change this by setting the ``dbt_project_dir`` argument when you create your DAG instance. +For more accurate parsing of your dbt project, you should pre-compile your dbt project's ``manifest.json`` (include ``dbt deps && dbt compile`` as part of your deployment process). + For example, if you wanted to put your dbt project in the ``/usr/local/airflow/dags/my_dbt_project`` directory, you would do: .. code-block:: python @@ -31,11 +33,15 @@ For example, if you wanted to put your dbt project in the ``/usr/local/airflow/d my_cosmos_dag = DbtDag( project_config=ProjectConfig( dbt_project_path="/usr/local/airflow/dags/my_dbt_project", + manifest_path="/usr/local/airflow/dags/my_dbt_project/target/manifest.json", ), # ..., ) +.. note:: + You can also exclude the ``manifest_path=...`` from the ``ProjectConfig``. Excluding a ``manifest_path`` file will by default use Cosmos's ``custom`` parsing method, which may be less accurate at parsing a dbt project compared to providing a ``manifest.json``. + Create your DAG --------------- @@ -43,6 +49,8 @@ In your ``my_cosmos_dag.py`` file, import the ``DbtDag`` class from Cosmos and c Make sure to rename the ```` value below to your adapter's Python package (i.e. ``dbt-snowflake`` or ``dbt-bigquery``) +If you need to modify the pip install options, you can do so by adding ``pip_install_options`` to the ``operator_args`` dictionary. For example, if you wanted to install packages from local wheels you could set it too: ``["--no-index", "--find-links=/path/to/wheels"]``. All options can be found here: + .. code-block:: python from cosmos import DbtDag, ProjectConfig, ProfileConfig, ExecutionConfig diff --git a/docs/getting_started/mwaa.rst b/docs/getting_started/mwaa.rst index f7a5693021..0def60f496 100644 --- a/docs/getting_started/mwaa.rst +++ b/docs/getting_started/mwaa.rst @@ -87,8 +87,11 @@ In your ``my_cosmos_dag.py`` file, import the ``DbtDag`` class from Cosmos and c .. code-block:: python - from cosmos import DbtDag, ProjectConfig, ProfileConfig + import os + from datetime import datetime + from cosmos import DbtDag, ProjectConfig, ProfileConfig, ExecutionConfig from cosmos.profiles import PostgresUserPasswordProfileMapping + from cosmos.constants import ExecutionMode profile_config = ProfileConfig( profile_name="default", @@ -99,11 +102,16 @@ In your ``my_cosmos_dag.py`` file, import the ``DbtDag`` class from Cosmos and c ), ) + execution_config = ExecutionConfig( + dbt_executable_path=f"{os.environ['AIRFLOW_HOME']}/dbt_venv/bin/dbt", + ) + my_cosmos_dag = DbtDag( project_config=ProjectConfig( "", ), profile_config=profile_config, + execution_config=execution_config, # normal dag parameters schedule_interval="@daily", start_date=datetime(2023, 1, 1), diff --git a/docs/index.rst b/docs/index.rst index 3c61b645dc..c22de1a7a2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -40,53 +40,25 @@ Run your dbt Core projects as `Apache Airflow `_ DA Example Usage ___________________ -You can render an Airflow Task Group using the ``DbtTaskGroup`` class. Here's an example with the jaffle_shop project: +You can render a Cosmos Airflow DAG using the ``DbtDag`` class. Here's an example with the `jaffle_shop project `_: -.. code-block:: python +.. + The following renders in Sphinx but not Github: - from pendulum import datetime +.. literalinclude:: ./dev/dags/basic_cosmos_dag.py + :language: python + :start-after: [START local_example] + :end-before: [END local_example] - from airflow import DAG - from airflow.operators.empty import EmptyOperator - from cosmos import DbtTaskGroup +This will generate an Airflow DAG that looks like this: - profile_config = ProfileConfig( - profile_name="default", - target_name="dev", - profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="airflow_db", - profile_args={"schema": "public"}, - ), - ) - - with DAG( - dag_id="extract_dag", - start_date=datetime(2022, 11, 27), - schedule_interval="@daily", - ): - e1 = EmptyOperator(task_id="pre_dbt") - - dbt_tg = DbtTaskGroup( - project_config=ProjectConfig("jaffle_shop"), - profile_config=profile_config, - default_args={"retries": 2}, - ) - - e2 = EmptyOperator(task_id="post_dbt") - - e1 >> dbt_tg >> e2 - - -This will generate an Airflow Task Group that looks like this: - -.. image:: https://raw.githubusercontent.com/astronomer/astronomer-cosmos/main/docs/_static/jaffle_shop_task_group.png - +.. image:: https://raw.githubusercontent.com/astronomer/astronomer-cosmos/main/docs/_static/jaffle_shop_dag.png Getting Started _______________ -To get started now, check out the `Getting Started Guide `_. +Check out the Quickstart guide on our `docs `_. See more examples at `/dev/dags `_ and at the `cosmos-demo repo `_. Changelog @@ -101,7 +73,8 @@ __________________ All contributions, bug reports, bug fixes, documentation improvements, enhancements are welcome. -A detailed overview an how to contribute can be found in the `Contributing Guide `_. +A detailed overview on how to contribute can be found in the `Contributing Guide `_. +Find out more about `our contributors `_. As contributors and maintainers to this project, you are expected to abide by the `Contributor Code of Conduct `_. diff --git a/docs/requirements.txt b/docs/requirements.txt index 6ead434854..81a7084e4a 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,11 @@ aenum -sphinx -pydata-sphinx-theme -sphinx-autobuild -sphinx-autoapi apache-airflow +apache-airflow-providers-cncf-kubernetes>=5.1.1 +google-re2==1.1 +msgpack openlineage-airflow +pydantic +pydata-sphinx-theme +sphinx +sphinx-autoapi +sphinx-autobuild diff --git a/docs/templates/index.rst.jinja2 b/docs/templates/index.rst.jinja2 index 5c0152363b..802b075ed9 100644 --- a/docs/templates/index.rst.jinja2 +++ b/docs/templates/index.rst.jinja2 @@ -83,6 +83,77 @@ but override the ``database`` and ``schema`` values: Note that when using a profile mapping, the profiles.yml file gets generated with the profile name and target name you specify in ``ProfileConfig``. +Disabling dbt event tracking +-------------------------------- + +.. note: + Deprecated in v.1.4 and will be removed in v2.0.0. Use dbt_config_vars=DbtProfileConfigVars(send_anonymous_usage_stats=False) instead. +.. versionadded:: 1.3 + +By default `dbt will track events `_ by sending anonymous usage data +when dbt commands are invoked. Users have an option to opt out of event tracking by updating their ``profiles.yml`` file. + +If you'd like to disable this behavior in the Cosmos generated profile, you can pass ``disable_event_tracking=True`` to the profile mapping like in +the example below: + +.. code-block:: python + + from cosmos.profiles import SnowflakeUserPasswordProfileMapping + + profile_config = ProfileConfig( + profile_name="my_profile_name", + target_name="my_target_name", + profile_mapping=SnowflakeUserPasswordProfileMapping( + conn_id="my_snowflake_conn_id", + profile_args={ + "database": "my_snowflake_database", + "schema": "my_snowflake_schema", + }, + disable_event_tracking=True, + ), + ) + + dag = DbtDag(profile_config=profile_config, ...) + +Dbt profile config variables +-------------------------------- +.. versionadded:: 1.4.0 + +The parts of ``profiles.yml``, which aren't specific to a particular data platform `dbt docs `_ + +.. code-block:: python + + from cosmos.profiles import SnowflakeUserPasswordProfileMapping, DbtProfileConfigVars + + profile_config = ProfileConfig( + profile_name="my_profile_name", + target_name="my_target_name", + profile_mapping=SnowflakeUserPasswordProfileMapping( + conn_id="my_snowflake_conn_id", + profile_args={ + "database": "my_snowflake_database", + "schema": "my_snowflake_schema", + }, + dbt_config_vars=DbtProfileConfigVars( + send_anonymous_usage_stats=False, + partial_parse=True, + use_experimental_parse=True, + static_parser=True, + printer_width=120, + write_json=True, + warn_error=True, + warn_error_options={"include": "all"}, + log_format='text', + debug=True, + version_check=True, + ), + ), + ) + + dag = DbtDag(profile_config=profile_config, ...) + + + Using your own profiles.yml file ++++++++++++++++++++++++++++++++++++ diff --git a/pyproject.toml b/pyproject.toml index cd90c8d969..5cbc93a988 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,20 +5,12 @@ build-backend = "hatchling.build" [project] name = "astronomer-cosmos" dynamic = ["version"] -description = "Render 3rd party workflows in Airflow" +description = "Orchestrate your dbt projects in Airflow" readme = "README.rst" license = "Apache-2.0" requires-python = ">=3.8" -authors = [ - { name = "Astronomer", email = "humans@astronomer.io" }, -] -keywords = [ - "airflow", - "apache-airflow", - "astronomer", - "dags", - "dbt", -] +authors = [{ name = "Astronomer", email = "humans@astronomer.io" }] +keywords = ["airflow", "apache-airflow", "astronomer", "dags", "dbt"] classifiers = [ "Development Status :: 3 - Alpha", "Environment :: Web Environment", @@ -32,15 +24,18 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] dependencies = [ - # Airflow & Pydantic issue: https://github.com/apache/airflow/issues/32311 "aenum", "attrs", - "pydantic>=1.10.0,<2.0.0", - "apache-airflow>=2.3.0", + "apache-airflow>=2.4.0", "importlib-metadata; python_version < '3.8'", "Jinja2>=3.0.0", + "msgpack", + "packaging", + "pydantic>=1.10.0", "typing-extensions; python_version < '3.8'", "virtualenv", ] @@ -49,54 +44,35 @@ dependencies = [ dbt-all = [ "dbt-athena", "dbt-bigquery", + "dbt-clickhouse", "dbt-databricks", "dbt-exasol", "dbt-postgres", "dbt-redshift", "dbt-snowflake", "dbt-spark", + "dbt-teradata", "dbt-vertica", ] -dbt-athena = [ - "dbt-athena-community", -] -dbt-bigquery = [ - "dbt-bigquery", -] -dbt-databricks = [ - "dbt-databricks", -] -dbt-exasol = [ - "dbt-exasol", -] -dbt-postgres = [ - "dbt-postgres", -] -dbt-redshift = [ - "dbt-redshift", -] -dbt-snowflake = [ - "dbt-snowflake", -] -dbt-spark = [ - "dbt-spark", -] -dbt-vertica = [ - "dbt-vertica<=1.5.4", -] -openlineage = [ - "openlineage-integration-common", - "openlineage-airflow", -] -all = [ - "astronomer-cosmos[dbt-all]", - "astronomer-cosmos[openlineage]" -] -docs =[ +dbt-athena = ["dbt-athena-community", "apache-airflow-providers-amazon>=8.0.0"] +dbt-bigquery = ["dbt-bigquery"] +dbt-clickhouse = ["dbt-clickhouse"] +dbt-databricks = ["dbt-databricks"] +dbt-exasol = ["dbt-exasol"] +dbt-postgres = ["dbt-postgres"] +dbt-redshift = ["dbt-redshift"] +dbt-snowflake = ["dbt-snowflake"] +dbt-spark = ["dbt-spark"] +dbt-teradata = ["dbt-teradata"] +dbt-vertica = ["dbt-vertica<=1.5.4"] +openlineage = ["openlineage-integration-common!=1.15.0", "openlineage-airflow"] +all = ["astronomer-cosmos[dbt-all]", "astronomer-cosmos[openlineage]"] +docs = [ "sphinx", "pydata-sphinx-theme", "sphinx-autobuild", - "sphinx-autoapi" + "sphinx-autoapi", + "apache-airflow-providers-cncf-kubernetes>=5.1.1", ] tests = [ "packaging", @@ -107,18 +83,29 @@ tests = [ "pytest-cov", "pytest-describe", "sqlalchemy-stubs", # Change when sqlalchemy is upgraded https://docs.sqlalchemy.org/en/14/orm/extensions/mypy.html + "types-pytz", "types-requests", - "mypy", "sqlalchemy-stubs", # Change when sqlalchemy is upgraded https://docs.sqlalchemy.org/en/14/orm/extensions/mypy.html + "pre-commit", ] - docker = [ "apache-airflow-providers-docker>=3.5.0", ] kubernetes = [ "apache-airflow-providers-cncf-kubernetes>=5.1.1", ] +aws_eks = [ + "apache-airflow-providers-amazon>=8.0.0,<8.20.0", # https://github.com/apache/airflow/issues/39103 +] +azure-container-instance = [ + "apache-airflow-providers-microsoft-azure>=8.4.0", +] + +[project.entry-points.cosmos] +provider_info = "cosmos:get_provider_info" +[project.entry-points."airflow.plugins"] +cosmos = "cosmos.plugin:CosmosPlugin" [project.urls] Homepage = "https://github.com/astronomer/astronomer-cosmos" @@ -129,9 +116,10 @@ Documentation = "https://astronomer.github.io/astronomer-cosmos" path = "cosmos/__init__.py" [tool.hatch.build.targets.sdist] -include = [ - "/cosmos", -] +include = ["/cosmos"] + +[tool.hatch.build.targets.wheel] +packages = ["/cosmos"] ###################################### # TESTING @@ -140,77 +128,48 @@ include = [ [tool.hatch.envs.tests] dependencies = [ "astronomer-cosmos[tests]", - "apache-airflow-providers-docker>=3.5.0", "apache-airflow-providers-cncf-kubernetes>=5.1.1", + "apache-airflow-providers-amazon>=3.0.0,<8.20.0", # https://github.com/apache/airflow/issues/39103 + "apache-airflow-providers-docker>=3.5.0", + "apache-airflow-providers-google", + "apache-airflow-providers-microsoft-azure", + "apache-airflow-providers-postgres", "types-PyYAML", "types-attrs", "types-requests", "types-python-dateutil", - "apache-airflow", "Werkzeug<3.0.0", + "apache-airflow~={matrix:airflow}.0,!=2.9.0,!=2.9.1", # https://github.com/apache/airflow/pull/39670 ] +pre-install-commands = ["sh scripts/test/pre-install-airflow.sh {matrix:airflow} {matrix:python}"] [[tool.hatch.envs.tests.matrix]] -python = ["3.8", "3.9", "3.10"] -airflow = ["2.3", "2.4", "2.5", "2.6", "2.7"] +python = ["3.8", "3.9", "3.10", "3.11", "3.12"] +airflow = ["2.4", "2.5", "2.6", "2.7", "2.8", "2.9"] [tool.hatch.envs.tests.overrides] matrix.airflow.dependencies = [ - { value = "apache-airflow==2.3", if = ["2.3"] }, - { value = "apache-airflow==2.4", if = ["2.4"] }, - { value = "apache-airflow==2.5", if = ["2.5"] }, - { value = "apache-airflow==2.6", if = ["2.6"] }, - { value = "apache-airflow==2.7", if = ["2.7"] }, + { value = "typing_extensions<4.6", if = ["2.6"] } ] [tool.hatch.envs.tests.scripts] freeze = "pip freeze" -type-check = "mypy cosmos" -test = 'pytest -vv --durations=0 . -m "not integration" --ignore=tests/test_example_dags.py --ignore=tests/test_example_dags_no_connections.py' -test-cov = """pytest -vv --cov=cosmos --cov-report=term-missing --cov-report=xml --durations=0 -m "not integration" --ignore=tests/test_example_dags.py --ignore=tests/test_example_dags_no_connections.py""" -# we install using the following workaround to overcome installation conflicts, such as: -# apache-airflow 2.3.0 and dbt-core [0.13.0 - 1.5.2] and jinja2>=3.0.0 because these package versions have conflicting dependencies -test-integration-setup = """pip uninstall dbt-postgres dbt-databricks dbt-vertica; \ -rm -rf airflow.*; \ -airflow db init; \ -pip install 'dbt-core' 'dbt-databricks' 'dbt-postgres' 'dbt-vertica' 'openlineage-airflow'""" -test-integration = """rm -rf dbt/jaffle_shop/dbt_packages; -pytest -vv \ ---cov=cosmos \ ---cov-report=term-missing \ ---cov-report=xml \ ---durations=0 \ --m integration \ --k 'not (sqlite or example_cosmos_sources or example_cosmos_python_models or example_virtualenv)'""" -test-integration-expensive = """pytest -vv \ ---cov=cosmos \ ---cov-report=term-missing \ ---cov-report=xml \ ---durations=0 \ --m integration \ --k 'example_cosmos_python_models or example_virtualenv'""" -test-integration-sqlite-setup = """pip uninstall -y dbt-core dbt-sqlite openlineage-airflow openlineage-integration-common; \ -rm -rf airflow.*; \ -airflow db init; \ -pip install 'dbt-core==1.4' 'dbt-sqlite<=1.4' 'dbt-databricks<=1.4' 'dbt-postgres<=1.4' """ -test-integration-sqlite = """ -pytest -vv \ ---cov=cosmos \ ---cov-report=term-missing \ ---cov-report=xml \ ---durations=0 \ --m integration \ --k 'example_cosmos_sources or sqlite'""" +test = 'sh scripts/test/unit.sh' +test-cov = 'sh scripts/test/unit-cov.sh' +test-integration = 'sh scripts/test/integration.sh' +test-integration-dbt-1-5-4 = 'sh scripts/test/integration-dbt-1-5-4.sh' +test-integration-expensive = 'sh scripts/test/integration-expensive.sh' +test-integration-setup = 'sh scripts/test/integration-setup.sh' +test-integration-sqlite = 'sh scripts/test/integration-sqlite.sh' +test-integration-sqlite-setup = 'sh scripts/test/integration-sqlite-setup.sh' +test-performance = 'sh scripts/test/performance.sh' +test-performance-setup = 'sh scripts/test/performance-setup.sh' +type-check = " pre-commit run mypy --files cosmos/**/*" [tool.pytest.ini_options] -filterwarnings = [ - "ignore::DeprecationWarning", -] +filterwarnings = ["ignore::DeprecationWarning"] minversion = "6.0" -markers = [ - "integration", - "sqlite" -] +markers = ["integration", "sqlite", "perf"] ###################################### # DOCS @@ -219,11 +178,14 @@ markers = [ [tool.hatch.envs.docs] dependencies = [ "aenum", - "sphinx", + "apache-airflow-providers-cncf-kubernetes>=5.1.1", + "msgpack", + "openlineage-airflow", + "pydantic>=1.10.0", "pydata-sphinx-theme", - "sphinx-autobuild", + "sphinx", "sphinx-autoapi", - "openlineage-airflow", + "sphinx-autobuild", ] [tool.hatch.envs.docs.scripts] @@ -248,10 +210,11 @@ no_warn_unused_ignores = true [tool.ruff] line-length = 120 +[tool.ruff.lint] +select = ["C901", "D300", "I", "F"] +ignore = ["F541"] +[tool.ruff.lint.mccabe] +max-complexity = 10 [tool.distutils.bdist_wheel] universal = true - -[tool.flake8] -max-complexity = 10 -select = "C" diff --git a/scripts/test/integration-dbt-1-5-4.sh b/scripts/test/integration-dbt-1-5-4.sh new file mode 100644 index 0000000000..0875330820 --- /dev/null +++ b/scripts/test/integration-dbt-1-5-4.sh @@ -0,0 +1,12 @@ +pip uninstall dbt-adapters dbt-common dbt-core dbt-extractor dbt-postgres dbt-semantic-interfaces -y +pip install dbt-postgres==1.5.4 dbt-databricks==1.5.4 +rm -rf airflow.*; \ +airflow db init; \ +pytest -vv \ + --cov=cosmos \ + --cov-report=term-missing \ + --cov-report=xml \ + --durations=0 \ + -m integration \ + --ignore=tests/perf \ + -k 'basic_cosmos_task_group' diff --git a/scripts/test/integration-expensive.sh b/scripts/test/integration-expensive.sh new file mode 100644 index 0000000000..24bace86d4 --- /dev/null +++ b/scripts/test/integration-expensive.sh @@ -0,0 +1,8 @@ +pytest -vv \ + --cov=cosmos \ + --cov-report=term-missing \ + --cov-report=xml \ + --durations=0 \ + -m integration \ + --ignore=tests/perf \ + -k 'example_cosmos_python_models or example_virtualenv' diff --git a/scripts/test/integration-setup.sh b/scripts/test/integration-setup.sh new file mode 100644 index 0000000000..c6e106fd56 --- /dev/null +++ b/scripts/test/integration-setup.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -v +set -x +set -e + +# we install using the following workaround to overcome installation conflicts, such as: +# apache-airflow 2.3.0 and dbt-core [0.13.0 - 1.5.2] and jinja2>=3.0.0 because these package versions have conflicting dependencies +pip uninstall -y dbt-postgres dbt-databricks dbt-vertica +rm -rf airflow.* +pip freeze | grep airflow +airflow db reset -y +airflow db init +pip install 'dbt-core' 'dbt-databricks' 'dbt-postgres' 'dbt-vertica' 'openlineage-airflow' diff --git a/scripts/test/integration-sqlite-setup.sh b/scripts/test/integration-sqlite-setup.sh new file mode 100644 index 0000000000..b8bce035c0 --- /dev/null +++ b/scripts/test/integration-sqlite-setup.sh @@ -0,0 +1,4 @@ +pip uninstall -y dbt-core dbt-sqlite openlineage-airflow openlineage-integration-common; \ +rm -rf airflow.*; \ +airflow db init; \ +pip install 'dbt-core==1.4' 'dbt-sqlite<=1.4' 'dbt-databricks<=1.4' 'dbt-postgres<=1.4' diff --git a/scripts/test/integration-sqlite.sh b/scripts/test/integration-sqlite.sh new file mode 100644 index 0000000000..dc32324d47 --- /dev/null +++ b/scripts/test/integration-sqlite.sh @@ -0,0 +1,8 @@ +pytest -vv \ + --cov=cosmos \ + --cov-report=term-missing \ + --cov-report=xml \ + --durations=0 \ + -m integration \ + --ignore=tests/perf \ + -k 'example_cosmos_sources or sqlite' diff --git a/scripts/test/integration.sh b/scripts/test/integration.sh new file mode 100644 index 0000000000..1d8264768a --- /dev/null +++ b/scripts/test/integration.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -x +set -e + +pip freeze | grep airflow +echo $AIRFLOW_HOME +ls $AIRFLOW_HOME + +airflow db check + + +rm -rf dbt/jaffle_shop/dbt_packages; +pytest -vv \ + --cov=cosmos \ + --cov-report=term-missing \ + --cov-report=xml \ + --durations=0 \ + -m integration \ + --ignore=tests/perf \ + -k 'not (sqlite or example_cosmos_sources or example_cosmos_python_models or example_virtualenv)' diff --git a/scripts/test/performance-setup.sh b/scripts/test/performance-setup.sh new file mode 100644 index 0000000000..7efb917c1e --- /dev/null +++ b/scripts/test/performance-setup.sh @@ -0,0 +1,4 @@ +pip uninstall -y dbt-core dbt-sqlite dbt-postgres openlineage-airflow openlineage-integration-common; \ +rm -rf airflow.*; \ +airflow db init; \ +pip install 'dbt-postgres' diff --git a/scripts/test/performance.sh b/scripts/test/performance.sh new file mode 100644 index 0000000000..ea58c19600 --- /dev/null +++ b/scripts/test/performance.sh @@ -0,0 +1,5 @@ +pytest -vv \ + -s \ + -m 'perf' \ + --ignore=tests/test_example_dags.py \ + --ignore=tests/test_example_dags_no_connections.py diff --git a/scripts/test/pre-install-airflow.sh b/scripts/test/pre-install-airflow.sh new file mode 100755 index 0000000000..08dcf042d6 --- /dev/null +++ b/scripts/test/pre-install-airflow.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +AIRFLOW_VERSION="$1" +PYTHON_VERSION="$2" + +# Use this to set the appropriate Python environment in Github Actions, +# while also not assuming --system when running locally. +if [ "$GITHUB_ACTIONS" = "true" ] && [ -z "${VIRTUAL_ENV}" ]; then + py_path=$(which python) + virtual_env_dir=$(dirname "$(dirname "$py_path")") + export VIRTUAL_ENV="$virtual_env_dir" +fi + +echo "${VIRTUAL_ENV}" + +CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-$AIRFLOW_VERSION.0/constraints-$PYTHON_VERSION.txt" +curl -sSL $CONSTRAINT_URL -o /tmp/constraint.txt +# Workaround to remove PyYAML constraint that will work on both Linux and MacOS +sed '/PyYAML==/d' /tmp/constraint.txt > /tmp/constraint.txt.tmp +mv /tmp/constraint.txt.tmp /tmp/constraint.txt +# Install Airflow with constraints +pip install uv +uv pip install "apache-airflow==$AIRFLOW_VERSION" --constraint /tmp/constraint.txt +uv pip install pydantic --constraint /tmp/constraint.txt +rm /tmp/constraint.txt diff --git a/scripts/test/unit-cov.sh b/scripts/test/unit-cov.sh new file mode 100644 index 0000000000..89a6244ba5 --- /dev/null +++ b/scripts/test/unit-cov.sh @@ -0,0 +1,10 @@ +pytest \ + -vv \ + --cov=cosmos \ + --cov-report=term-missing \ + --cov-report=xml \ + --durations=0 \ + -m "not (integration or perf)" \ + --ignore=tests/perf \ + --ignore=tests/test_example_dags.py \ + --ignore=tests/test_example_dags_no_connections.py diff --git a/scripts/test/unit.sh b/scripts/test/unit.sh new file mode 100644 index 0000000000..ecc1a049a2 --- /dev/null +++ b/scripts/test/unit.sh @@ -0,0 +1,7 @@ +pytest \ + -vv \ + --durations=0 \ + -m "not (integration or perf)" \ + --ignore=tests/perf \ + --ignore=tests/test_example_dags.py \ + --ignore=tests/test_example_dags_no_connections.py diff --git a/tests/airflow/test_graph.py b/tests/airflow/test_graph.py index 6bc244b6b6..a238475c2c 100644 --- a/tests/airflow/test_graph.py +++ b/tests/airflow/test_graph.py @@ -9,6 +9,7 @@ from packaging import version from cosmos.airflow.graph import ( + _snake_case_to_camelcase, build_airflow_graph, calculate_leaves, calculate_operator_class, @@ -16,44 +17,59 @@ create_test_task_metadata, generate_task_or_group, ) -from cosmos.config import ProfileConfig -from cosmos.constants import DbtResourceType, ExecutionMode, TestBehavior, TestIndirectSelection +from cosmos.config import ProfileConfig, RenderConfig +from cosmos.constants import ( + DbtResourceType, + ExecutionMode, + TestBehavior, + TestIndirectSelection, +) +from cosmos.converter import airflow_kwargs from cosmos.dbt.graph import DbtNode from cosmos.profiles import PostgresUserPasswordProfileMapping SAMPLE_PROJ_PATH = Path("/home/user/path/dbt-proj/") parent_seed = DbtNode( - name="seed_parent", - unique_id="seed_parent", + unique_id=f"{DbtResourceType.SEED.value}.{SAMPLE_PROJ_PATH.stem}.seed_parent", resource_type=DbtResourceType.SEED, depends_on=[], file_path="", ) parent_node = DbtNode( - name="parent", - unique_id="parent", + unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.parent", resource_type=DbtResourceType.MODEL, - depends_on=["seed_parent"], + depends_on=[parent_seed.unique_id], file_path=SAMPLE_PROJ_PATH / "gen2/models/parent.sql", tags=["has_child"], config={"materialized": "view"}, has_test=True, ) test_parent_node = DbtNode( - name="test_parent", unique_id="test_parent", resource_type=DbtResourceType.TEST, depends_on=["parent"], file_path="" + unique_id=f"{DbtResourceType.TEST.value}.{SAMPLE_PROJ_PATH.stem}.test_parent", + resource_type=DbtResourceType.TEST, + depends_on=[parent_node.unique_id], + file_path="", ) child_node = DbtNode( - name="child", - unique_id="child", + unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.child", resource_type=DbtResourceType.MODEL, - depends_on=["parent"], + depends_on=[parent_node.unique_id], file_path=SAMPLE_PROJ_PATH / "gen3/models/child.sql", tags=["nightly"], config={"materialized": "table"}, ) -sample_nodes_list = [parent_seed, parent_node, test_parent_node, child_node] +child2_node = DbtNode( + unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.child2.v2", + resource_type=DbtResourceType.MODEL, + depends_on=[parent_node.unique_id], + file_path=SAMPLE_PROJ_PATH / "gen3/models/child2_v2.sql", + tags=["nightly"], + config={"materialized": "table"}, +) + +sample_nodes_list = [parent_seed, parent_node, test_parent_node, child_node, child2_node] sample_nodes = {node.unique_id: node for node in sample_nodes_list} @@ -82,7 +98,9 @@ def test_build_airflow_graph_with_after_each(): execution_mode=ExecutionMode.LOCAL, test_indirect_selection=TestIndirectSelection.EAGER, task_args=task_args, - test_behavior=TestBehavior.AFTER_EACH, + render_config=RenderConfig( + test_behavior=TestBehavior.AFTER_EACH, + ), dbt_project_name="astro_shop", ) topological_sort = [task.task_id for task in dag.topological_sort()] @@ -91,6 +109,7 @@ def test_build_airflow_graph_with_after_each(): "parent.run", "parent.test", "child_run", + "child2_v2_run", ] assert topological_sort == expected_sort @@ -100,15 +119,16 @@ def test_build_airflow_graph_with_after_each(): assert task_groups["parent"].upstream_task_ids == {"seed_parent_seed"} assert list(task_groups["parent"].children.keys()) == ["parent.run", "parent.test"] - assert len(dag.leaves) == 1 + assert len(dag.leaves) == 2 assert dag.leaves[0].task_id == "child_run" + assert dag.leaves[1].task_id == "child2_v2_run" @pytest.mark.parametrize( "node_type,task_suffix", [(DbtResourceType.MODEL, "run"), (DbtResourceType.SEED, "seed"), (DbtResourceType.SNAPSHOT, "snapshot")], ) -def test_create_task_group_for_after_each_supported_nodes(node_type, task_suffix): +def test_create_task_group_for_after_each_supported_nodes(node_type: DbtResourceType, task_suffix): """ dbt test runs tests defined on models, sources, snapshots, and seeds. It expects that you have already created those resources through the appropriate commands. @@ -116,8 +136,7 @@ def test_create_task_group_for_after_each_supported_nodes(node_type, task_suffix """ with DAG("test-task-group-after-each", start_date=datetime(2022, 1, 1)) as dag: node = DbtNode( - name="dbt_node", - unique_id="dbt_node", + unique_id=f"{node_type.value}.{SAMPLE_PROJ_PATH.stem}.dbt_node", resource_type=node_type, file_path=SAMPLE_PROJ_PATH / "gen2/models/parent.sql", tags=["has_child"], @@ -168,17 +187,21 @@ def test_build_airflow_graph_with_after_all(): ), ), } + render_config = RenderConfig( + select=["tag:some"], + test_behavior=TestBehavior.AFTER_ALL, + ) build_airflow_graph( nodes=sample_nodes, dag=dag, execution_mode=ExecutionMode.LOCAL, test_indirect_selection=TestIndirectSelection.EAGER, task_args=task_args, - test_behavior=TestBehavior.AFTER_ALL, dbt_project_name="astro_shop", + render_config=render_config, ) topological_sort = [task.task_id for task in dag.topological_sort()] - expected_sort = ["seed_parent_seed", "parent_run", "child_run", "astro_shop_test"] + expected_sort = ["seed_parent_seed", "parent_run", "child_run", "child2_v2_run", "astro_shop_test"] assert topological_sort == expected_sort task_groups = dag.task_group_dict @@ -186,6 +209,7 @@ def test_build_airflow_graph_with_after_all(): assert len(dag.leaves) == 1 assert dag.leaves[0].task_id == "astro_shop_test" + assert dag.leaves[0].select == ["tag:some"] def test_calculate_operator_class(): @@ -195,8 +219,7 @@ def test_calculate_operator_class(): def test_calculate_leaves(): grandparent_node = DbtNode( - name="grandparent", - unique_id="grandparent", + unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.grandparent", resource_type=DbtResourceType.MODEL, depends_on=[], file_path="", @@ -204,28 +227,25 @@ def test_calculate_leaves(): config={}, ) parent1_node = DbtNode( - name="parent1", - unique_id="parent1", + unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.parent1", resource_type=DbtResourceType.MODEL, - depends_on=["grandparent"], + depends_on=[grandparent_node.unique_id], file_path="", tags=[], config={}, ) parent2_node = DbtNode( - name="parent2", - unique_id="parent2", + unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.parent2", resource_type=DbtResourceType.MODEL, - depends_on=["grandparent"], + depends_on=[parent1_node.unique_id], file_path="", tags=[], config={}, ) child_node = DbtNode( - name="child", - unique_id="child", + unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.child", resource_type=DbtResourceType.MODEL, - depends_on=["parent1", "parent2"], + depends_on=[parent1_node.unique_id, parent2_node.unique_id], file_path="", tags=[], config={}, @@ -235,14 +255,13 @@ def test_calculate_leaves(): nodes = {node.unique_id: node for node in nodes_list} leaves = calculate_leaves(nodes.keys(), nodes) - assert leaves == ["child"] + assert leaves == [f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.child"] @patch("cosmos.airflow.graph.logger.propagate", True) def test_create_task_metadata_unsupported(caplog): child_node = DbtNode( - name="unsupported", - unique_id="unsupported", + unique_id=f"unsupported.{SAMPLE_PROJ_PATH.stem}.unsupported", resource_type="unsupported", depends_on=[], file_path="", @@ -252,16 +271,106 @@ def test_create_task_metadata_unsupported(caplog): response = create_task_metadata(child_node, execution_mode="", args={}) assert response is None expected_msg = ( - "Unavailable conversion function for (node ). " + "Unavailable conversion function for (node ). " "Define a converter function using render_config.node_converters." ) assert caplog.messages[0] == expected_msg -def test_create_task_metadata_model(caplog): +@pytest.mark.parametrize( + "unique_id, resource_type, expected_id, expected_operator_class, expected_arguments, expected_extra_context", + [ + ( + f"{DbtResourceType.MODEL.value}.my_folder.my_model", + DbtResourceType.MODEL, + "my_model_run", + "cosmos.operators.local.DbtRunLocalOperator", + {"models": "my_model"}, + { + "dbt_node_config": { + "unique_id": "model.my_folder.my_model", + "resource_type": "model", + "depends_on": [], + "file_path": ".", + "tags": [], + "config": {}, + "has_test": False, + "resource_name": "my_model", + "name": "my_model", + } + }, + ), + ( + f"{DbtResourceType.SOURCE.value}.my_folder.my_source", + DbtResourceType.SOURCE, + "my_source_run", + "cosmos.operators.local.DbtRunLocalOperator", + {"models": "my_source"}, + { + "dbt_node_config": { + "unique_id": "model.my_folder.my_source", + "resource_type": "source", + "depends_on": [], + "file_path": ".", + "tags": [], + "config": {}, + "has_test": False, + "resource_name": "my_source", + "name": "my_source", + } + }, + ), + ( + f"{DbtResourceType.SNAPSHOT.value}.my_folder.my_snapshot", + DbtResourceType.SNAPSHOT, + "my_snapshot_snapshot", + "cosmos.operators.local.DbtSnapshotLocalOperator", + {"models": "my_snapshot"}, + { + "dbt_node_config": { + "unique_id": "snapshot.my_folder.my_snapshot", + "resource_type": "snapshot", + "depends_on": [], + "file_path": ".", + "tags": [], + "config": {}, + "has_test": False, + "resource_name": "my_snapshot", + "name": "my_snapshot", + }, + }, + ), + ], +) +def test_create_task_metadata_model( + unique_id, + resource_type, + expected_id, + expected_operator_class, + expected_arguments, + expected_extra_context, + caplog, +): + child_node = DbtNode( + unique_id=unique_id, + resource_type=resource_type, + depends_on=[], + file_path=Path(""), + tags=[], + config={}, + ) + + metadata = create_task_metadata(child_node, execution_mode=ExecutionMode.LOCAL, args={}) + if metadata: + assert metadata.id == expected_id + assert metadata.operator_class == expected_operator_class + assert metadata.arguments == expected_arguments + assert metadata.extra_context == expected_extra_context + + +def test_create_task_metadata_model_with_versions(caplog): child_node = DbtNode( - name="my_model", - unique_id="my_folder.my_model", + unique_id=f"{DbtResourceType.MODEL.value}.my_folder.my_model.v1", resource_type=DbtResourceType.MODEL, depends_on=[], file_path="", @@ -269,15 +378,14 @@ def test_create_task_metadata_model(caplog): config={}, ) metadata = create_task_metadata(child_node, execution_mode=ExecutionMode.LOCAL, args={}) - assert metadata.id == "my_model_run" + assert metadata.id == "my_model_v1_run" assert metadata.operator_class == "cosmos.operators.local.DbtRunLocalOperator" - assert metadata.arguments == {"models": "my_model"} + assert metadata.arguments == {"models": "my_model.v1"} def test_create_task_metadata_model_use_task_group(caplog): child_node = DbtNode( - name="my_model", - unique_id="my_folder.my_model", + unique_id=f"{DbtResourceType.MODEL.value}.my_folder.my_model", resource_type=DbtResourceType.MODEL, depends_on=[], file_path=Path(""), @@ -291,8 +399,7 @@ def test_create_task_metadata_model_use_task_group(caplog): @pytest.mark.parametrize("use_task_group", (None, True, False)) def test_create_task_metadata_seed(caplog, use_task_group): sample_node = DbtNode( - name="my_seed", - unique_id="my_folder.my_seed", + unique_id=f"{DbtResourceType.SEED.value}.my_folder.my_seed", resource_type=DbtResourceType.SEED, depends_on=[], file_path="", @@ -320,8 +427,7 @@ def test_create_task_metadata_seed(caplog, use_task_group): def test_create_task_metadata_snapshot(caplog): sample_node = DbtNode( - name="my_snapshot", - unique_id="my_folder.my_snapshot", + unique_id=f"{DbtResourceType.SNAPSHOT.value}.my_folder.my_snapshot", resource_type=DbtResourceType.SNAPSHOT, depends_on=[], file_path="", @@ -337,22 +443,33 @@ def test_create_task_metadata_snapshot(caplog): @pytest.mark.parametrize( "node_type,node_unique_id,test_indirect_selection,additional_arguments", [ - (DbtResourceType.MODEL, "node_name", TestIndirectSelection.EAGER, {"models": "node_name"}), + ( + DbtResourceType.MODEL, + f"{DbtResourceType.MODEL.value}.my_folder.node_name", + TestIndirectSelection.EAGER, + {"models": "node_name"}, + ), + ( + DbtResourceType.MODEL, + f"{DbtResourceType.MODEL.value}.my_folder.node_name.v1", + TestIndirectSelection.EAGER, + {"models": "node_name.v1"}, + ), ( DbtResourceType.SEED, - "node_name", + f"{DbtResourceType.SEED.value}.my_folder.node_name", TestIndirectSelection.CAUTIOUS, {"select": "node_name", "indirect_selection": "cautious"}, ), ( DbtResourceType.SOURCE, - "source.node_name", + f"{DbtResourceType.SOURCE.value}.my_folder.node_name", TestIndirectSelection.BUILDABLE, {"select": "source:node_name", "indirect_selection": "buildable"}, ), ( DbtResourceType.SNAPSHOT, - "node_name", + f"{DbtResourceType.SNAPSHOT.value}.my_folder.node_name", TestIndirectSelection.EMPTY, {"select": "node_name", "indirect_selection": "empty"}, ), @@ -360,7 +477,6 @@ def test_create_task_metadata_snapshot(caplog): ) def test_create_test_task_metadata(node_type, node_unique_id, test_indirect_selection, additional_arguments): sample_node = DbtNode( - name="node_name", unique_id=node_unique_id, resource_type=node_type, depends_on=[], @@ -385,3 +501,35 @@ def test_create_test_task_metadata(node_type, node_unique_id, test_indirect_sele }, **additional_arguments, } + + +@pytest.mark.parametrize( + "input,expected", [("snake_case", "SnakeCase"), ("snake_case_with_underscores", "SnakeCaseWithUnderscores")] +) +def test_snake_case_to_camelcase(input, expected): + assert _snake_case_to_camelcase(input) == expected + + +def test_airflow_kwargs_generation(): + """ + airflow_kwargs_generation should always contain dag. + """ + task_args = { + "group_id": "fake_group_id", + "project_dir": SAMPLE_PROJ_PATH, + "conn_id": "fake_conn", + "render_config": RenderConfig(select=["fake-render"]), + "default_args": {"retries": 2}, + "profile_config": ProfileConfig( + profile_name="default", + target_name="default", + profile_mapping=PostgresUserPasswordProfileMapping( + conn_id="fake_conn", + profile_args={"schema": "public"}, + ), + ), + "dag": DAG(dag_id="fake_dag_name"), + } + result = airflow_kwargs(**task_args) + + assert "dag" in result diff --git a/tests/dbt/parser/test_output.py b/tests/dbt/parser/test_output.py index 0f4ba56cde..4fe11669f5 100644 --- a/tests/dbt/parser/test_output.py +++ b/tests/dbt/parser/test_output.py @@ -1,18 +1,53 @@ +import logging +from unittest.mock import MagicMock + +import pytest from airflow.hooks.subprocess import SubprocessResult from cosmos.dbt.parser.output import ( + extract_dbt_runner_issues, extract_log_issues, - parse_output, + parse_number_of_warnings_dbt_runner, + parse_number_of_warnings_subprocess, ) -def test_parse_output() -> None: - for warnings in range(0, 3): - output_str = f"Done. PASS=15 WARN={warnings} ERROR=0 SKIP=0 TOTAL=16" - keyword = "WARN" +@pytest.mark.parametrize( + "output_str, expected_warnings", + [ + ("Done. PASS=15 WARN=1 ERROR=0 SKIP=0 TOTAL=16", 1), + ("Done. PASS=15 WARN=0 ERROR=0 SKIP=0 TOTAL=16", 0), + ("Done. PASS=15 WARN=2 ERROR=0 SKIP=0 TOTAL=16", 2), + ("Nothing to do. Exiting without running tests.", 0), + ], +) +def test_parse_number_of_warnings_subprocess(output_str: str, expected_warnings): + result = SubprocessResult(exit_code=0, output=output_str) + num_warns = parse_number_of_warnings_subprocess(result) + assert num_warns == expected_warnings + + +def test_parse_number_of_warnings_subprocess_error_logged(caplog): + output_str = "WARN= should log an error." + with caplog.at_level(logging.ERROR): result = SubprocessResult(exit_code=0, output=output_str) - num_warns = parse_output(result, keyword) - assert num_warns == warnings + parse_number_of_warnings_subprocess(result) + expected_error_log = ( + "Could not parse number of WARNs. Check your dbt/airflow version or if --quiet is not being used" + ) + assert expected_error_log in caplog.text + + +def test_parse_number_of_warnings_dbt_runner_with_warnings(): + runner_result = MagicMock() + runner_result.result.results = [ + MagicMock(status="pass"), + MagicMock(status="warn"), + MagicMock(status="pass"), + MagicMock(status="warn"), + ] + num_warns = parse_number_of_warnings_dbt_runner(runner_result) + assert num_warns == 2 def test_extract_log_issues() -> None: @@ -37,3 +72,43 @@ def test_extract_log_issues() -> None: test_names_no_warns, test_results_no_warns = extract_log_issues(log_list_no_warning) assert test_names_no_warns == [] assert test_results_no_warns == [] + + +def test_extract_dbt_runner_issues(): + """Tests that the function extracts the correct node names and messages from a dbt runner result + for warnings by default. + """ + runner_result = MagicMock() + runner_result.result.results = [ + MagicMock(status="pass"), + MagicMock(status="warn", message="A warning message", node=MagicMock()), + MagicMock(status="pass"), + MagicMock(status="warn", message="A different warning message", node=MagicMock()), + ] + runner_result.result.results[1].node.name = "a_test" + runner_result.result.results[3].node.name = "another_test" + + node_names, node_results = extract_dbt_runner_issues(runner_result) + + assert node_names == ["a_test", "another_test"] + assert node_results == ["A warning message", "A different warning message"] + + +def test_extract_dbt_runner_issues_with_status_levels(): + """Tests that the function extracts the correct test names and results from a dbt runner result + for status levels. + """ + runner_result = MagicMock() + runner_result.result.results = [ + MagicMock(status="pass"), + MagicMock(status="error", message="An error message", node=MagicMock()), + MagicMock(status="warn"), + MagicMock(status="fail", message="A failure message", node=MagicMock()), + ] + runner_result.result.results[1].node.name = "node1" + runner_result.result.results[3].node.name = "node2" + + node_names, node_results = extract_dbt_runner_issues(runner_result, status_levels=["error", "fail"]) + + assert node_names == ["node1", "node2"] + assert node_results == ["An error message", "A failure message"] diff --git a/tests/dbt/parser/test_project.py b/tests/dbt/parser/test_project.py index 4f13a3eb36..31fe7e18d0 100644 --- a/tests/dbt/parser/test_project.py +++ b/tests/dbt/parser/test_project.py @@ -219,6 +219,6 @@ def test_dbtmodelconfig_with_vars(tmp_path): name="some_name", type=DbtModelType.DBT_MODEL, path=path_with_sources, - operator_args={"vars": {"country_code": "us"}}, + dbt_vars={"country_code": "us"}, ) assert "stg_customers_us" in dbt_model.config.upstream_models diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index b108878fc9..05aad822ad 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -1,20 +1,27 @@ +import importlib +import logging +import os import shutil +import sys import tempfile +from datetime import datetime from pathlib import Path -from unittest.mock import patch +from subprocess import PIPE, Popen +from unittest.mock import MagicMock, patch import pytest +from airflow.models import Variable -from cosmos.config import ExecutionConfig, ProfileConfig, ProjectConfig, RenderConfig -from cosmos.constants import DbtResourceType, ExecutionMode +from cosmos import settings +from cosmos.config import CosmosConfigException, ExecutionConfig, ProfileConfig, ProjectConfig, RenderConfig +from cosmos.constants import DBT_TARGET_DIR_NAME, DbtResourceType, ExecutionMode from cosmos.dbt.graph import ( CosmosLoadDbtException, DbtGraph, DbtNode, LoadMode, - create_symlinks, - run_command, parse_dbt_ls_output, + run_command, ) from cosmos.profiles import PostgresUserPasswordProfileMapping @@ -24,6 +31,7 @@ SAMPLE_MANIFEST_PY = Path(__file__).parent.parent / "sample/manifest_python.json" SAMPLE_MANIFEST_MODEL_VERSION = Path(__file__).parent.parent / "sample/manifest_model_version.json" SAMPLE_MANIFEST_SOURCE = Path(__file__).parent.parent / "sample/manifest_source.json" +SAMPLE_DBT_LS_OUTPUT = Path(__file__).parent.parent / "sample/sample_dbt_ls.txt" @pytest.fixture @@ -43,6 +51,73 @@ def tmp_dbt_project_dir(): shutil.rmtree(tmp_dir, ignore_errors=True) # delete directory +@pytest.fixture +def postgres_profile_config() -> ProfileConfig: + return ProfileConfig( + profile_name="default", + target_name="default", + profile_mapping=PostgresUserPasswordProfileMapping( + conn_id="example_conn", + profile_args={"schema": "public"}, + ), + ) + + +@pytest.mark.parametrize( + "unique_id,expected_name, expected_select", + [ + ("model.my_project.customers", "customers", "customers"), + ("model.my_project.customers.v1", "customers_v1", "customers.v1"), + ("model.my_project.orders.v2", "orders_v2", "orders.v2"), + ], +) +def test_dbt_node_name_and_select(unique_id, expected_name, expected_select): + node = DbtNode(unique_id=unique_id, resource_type=DbtResourceType.MODEL, depends_on=[], file_path="") + assert node.name == expected_name + assert node.resource_name == expected_select + + +@pytest.mark.parametrize( + "unique_id,expected_dict", + [ + ( + "model.my_project.customers", + { + "unique_id": "model.my_project.customers", + "resource_type": "model", + "depends_on": [], + "file_path": "", + "tags": [], + "config": {}, + "has_test": False, + "resource_name": "customers", + "name": "customers", + }, + ), + ( + "model.my_project.customers.v1", + { + "unique_id": "model.my_project.customers.v1", + "resource_type": "model", + "depends_on": [], + "file_path": "", + "tags": [], + "config": {}, + "has_test": False, + "resource_name": "customers.v1", + "name": "customers_v1", + }, + ), + ], +) +def test_dbt_node_context_dict( + unique_id, + expected_dict, +): + node = DbtNode(unique_id=unique_id, resource_type=DbtResourceType.MODEL, depends_on=[], file_path="") + assert node.context_dict == expected_dict + + @pytest.mark.parametrize( "project_name,manifest_filepath,model_filepath", [(DBT_PROJECT_NAME, SAMPLE_MANIFEST, "customers.sql"), ("jaffle_shop_python", SAMPLE_MANIFEST_PY, "customers.py")], @@ -82,6 +157,77 @@ def test_load_via_manifest_with_exclude(project_name, manifest_filepath, model_f assert sample_node.file_path == DBT_PROJECTS_ROOT_DIR / f"{project_name}/models/{model_filepath}" +@pytest.mark.parametrize( + "project_name,manifest_filepath,model_filepath", + [(DBT_PROJECT_NAME, SAMPLE_MANIFEST, "customers.sql"), ("jaffle_shop_python", SAMPLE_MANIFEST_PY, "customers.py")], +) +def test_load_via_manifest_with_select(project_name, manifest_filepath, model_filepath): + project_config = ProjectConfig( + dbt_project_path=DBT_PROJECTS_ROOT_DIR / project_name, manifest_path=manifest_filepath + ) + profile_config = ProfileConfig( + profile_name="test", + target_name="test", + profiles_yml_filepath=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME / "profiles.yml", + ) + render_config = RenderConfig(select=["+customers"]) + execution_config = ExecutionConfig(dbt_project_path=project_config.dbt_project_path) + dbt_graph = DbtGraph( + project=project_config, + execution_config=execution_config, + profile_config=profile_config, + render_config=render_config, + ) + dbt_graph.load_from_dbt_manifest() + + expected_keys = [ + "model.jaffle_shop.customers", + "model.jaffle_shop.orders", + "model.jaffle_shop.stg_customers", + "model.jaffle_shop.stg_orders", + "model.jaffle_shop.stg_payments", + "seed.jaffle_shop.raw_customers", + "seed.jaffle_shop.raw_orders", + "seed.jaffle_shop.raw_payments", + "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3", + "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad", + "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278", + "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d", + "test.jaffle_shop.not_null_orders_amount.106140f9fd", + "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49", + "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625", + "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59", + "test.jaffle_shop.not_null_orders_customer_id.c5f02694af", + "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a", + "test.jaffle_shop.not_null_orders_order_id.cf6c17daed", + "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa", + "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64", + "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075", + "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2", + "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1", + "test.jaffle_shop.unique_orders_order_id.fed79b3a6e", + "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada", + "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a", + "test.jaffle_shop.unique_stg_payments_payment_id.3744510712", + ] + assert sorted(dbt_graph.nodes.keys()) == expected_keys + + assert len(dbt_graph.nodes) == 28 + assert len(dbt_graph.filtered_nodes) == 7 + assert "model.jaffle_shop.orders" not in dbt_graph.filtered_nodes + + sample_node = dbt_graph.nodes["model.jaffle_shop.customers"] + assert sample_node.name == "customers" + assert sample_node.unique_id == "model.jaffle_shop.customers" + assert sample_node.resource_type == DbtResourceType.MODEL + assert sample_node.depends_on == [ + "model.jaffle_shop.stg_customers", + "model.jaffle_shop.stg_orders", + "model.jaffle_shop.stg_payments", + ] + assert sample_node.file_path == DBT_PROJECTS_ROOT_DIR / f"{project_name}/models/{model_filepath}" + + @patch("cosmos.dbt.graph.DbtGraph.load_from_dbt_manifest", return_value=None) def test_load_automatic_manifest_is_available(mock_load_from_dbt_manifest): project_config = ProjectConfig( @@ -97,6 +243,52 @@ def test_load_automatic_manifest_is_available(mock_load_from_dbt_manifest): assert mock_load_from_dbt_manifest.called +@patch("cosmos.dbt.graph.DbtGraph.load_via_dbt_ls_file", return_value=None) +def test_load_automatic_dbt_ls_file_is_available(mock_load_via_dbt_ls_file): + project_config = ProjectConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) + profile_config = ProfileConfig( + profile_name="test", + target_name="test", + profiles_yml_filepath=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME / "profiles.yml", + ) + render_config = RenderConfig(dbt_ls_path=SAMPLE_DBT_LS_OUTPUT) + dbt_graph = DbtGraph(project=project_config, profile_config=profile_config, render_config=render_config) + dbt_graph.load(method=LoadMode.DBT_LS_FILE, execution_mode=ExecutionMode.LOCAL) + assert mock_load_via_dbt_ls_file.called + + +def test_load_dbt_ls_file_without_file(): + project_config = ProjectConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) + profile_config = ProfileConfig( + profile_name="test", + target_name="test", + profiles_yml_filepath=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME / "profiles.yml", + ) + render_config = RenderConfig(dbt_ls_path=None) + dbt_graph = DbtGraph(project=project_config, profile_config=profile_config, render_config=render_config) + with pytest.raises(CosmosLoadDbtException) as err_info: + dbt_graph.load(execution_mode=ExecutionMode.LOCAL, method=LoadMode.DBT_LS_FILE) + assert err_info.value.args[0] == "Unable to load dbt ls file using None" + + +def test_load_dbt_ls_file_without_project_path(): + project_config = ProjectConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) + profile_config = ProfileConfig( + profile_name="test", + target_name="test", + profiles_yml_filepath=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME / "profiles.yml", + ) + render_config = RenderConfig(dbt_ls_path=SAMPLE_DBT_LS_OUTPUT, dbt_project_path=None) + dbt_graph = DbtGraph( + project=project_config, + profile_config=profile_config, + render_config=render_config, + ) + with pytest.raises(CosmosLoadDbtException) as err_info: + dbt_graph.load(execution_mode=ExecutionMode.LOCAL, method=LoadMode.DBT_LS_FILE) + assert err_info.value.args[0] == "Unable to load dbt ls file without RenderConfig.project_path" + + @patch("cosmos.dbt.graph.DbtGraph.load_via_custom_parser", side_effect=None) @patch("cosmos.dbt.graph.DbtGraph.load_via_dbt_ls", return_value=None) def test_load_automatic_without_manifest_with_profile_yml(mock_load_via_dbt_ls, mock_load_via_custom_parser): @@ -120,7 +312,7 @@ def test_load_automatic_without_manifest_with_profile_mapping(mock_load_via_dbt_ profile_name="test", target_name="test", profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="airflow_db", + conn_id="example_conn", profile_args={"schema": "public"}, ), ) @@ -184,11 +376,20 @@ def test_load_manifest_with_manifest(mock_load_from_dbt_manifest): (ExecutionMode.LOCAL, LoadMode.CUSTOM, "mock_load_via_custom_parser"), ], ) +@patch("cosmos.dbt.graph.DbtGraph.update_node_dependency") @patch("cosmos.dbt.graph.DbtGraph.load_via_custom_parser", return_value=None) @patch("cosmos.dbt.graph.DbtGraph.load_via_dbt_ls", return_value=None) @patch("cosmos.dbt.graph.DbtGraph.load_from_dbt_manifest", return_value=None) +@patch("cosmos.dbt.graph.DbtGraph.load_via_dbt_ls_file", return_value=None) def test_load( - mock_load_from_dbt_manifest, mock_load_via_dbt_ls, mock_load_via_custom_parser, exec_mode, method, expected_function + mock_load_from_dbt_manifest, + mock_load_via_dbt_ls_file, + mock_load_via_dbt_ls, + mock_load_via_custom_parser, + mock_update_node_dependency, + exec_mode, + method, + expected_function, ): project_config = ProjectConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) profile_config = ProfileConfig( @@ -201,11 +402,17 @@ def test_load( dbt_graph.load(method=method, execution_mode=exec_mode) load_function = locals()[expected_function] assert load_function.called + assert mock_update_node_dependency.called @pytest.mark.integration +@pytest.mark.parametrize("enable_cache_profile", [True, False]) +@patch("cosmos.config.is_profile_cache_enabled") @patch("cosmos.dbt.graph.Popen") -def test_load_via_dbt_ls_does_not_create_target_logs_in_original_folder(mock_popen, tmp_dbt_project_dir): +def test_load_via_dbt_ls_does_not_create_target_logs_in_original_folder( + mock_popen, is_profile_cache_enabled, enable_cache_profile, tmp_dbt_project_dir, postgres_profile_config +): + is_profile_cache_enabled.return_value = enable_cache_profile mock_popen().communicate.return_value = ("", "") mock_popen().returncode = 0 assert not (tmp_dbt_project_dir / "target").exists() @@ -218,26 +425,19 @@ def test_load_via_dbt_ls_does_not_create_target_logs_in_original_folder(mock_pop project=project_config, render_config=render_config, execution_config=execution_config, - profile_config=ProfileConfig( - profile_name="default", - target_name="default", - profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="airflow_db", - profile_args={"schema": "public"}, - ), - ), + profile_config=postgres_profile_config, ) dbt_graph.load_via_dbt_ls() assert not (tmp_dbt_project_dir / "target").exists() assert not (tmp_dbt_project_dir / "logs").exists() - used_cwd = Path(mock_popen.call_args[0][0][-5]) + used_cwd = Path(mock_popen.call_args[0][0][5]) assert used_cwd != project_config.dbt_project_path assert not used_cwd.exists() @pytest.mark.integration -def test_load_via_dbt_ls_with_exclude(): +def test_load_via_dbt_ls_with_exclude(postgres_profile_config): project_config = ProjectConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) render_config = RenderConfig( dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME, select=["*customers*"], exclude=["*orders*"] @@ -247,14 +447,7 @@ def test_load_via_dbt_ls_with_exclude(): project=project_config, render_config=render_config, execution_config=execution_config, - profile_config=ProfileConfig( - profile_name="default", - target_name="default", - profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="airflow_db", - profile_args={"schema": "public"}, - ), - ), + profile_config=postgres_profile_config, ) dbt_graph.load_via_dbt_ls() @@ -286,7 +479,7 @@ def test_load_via_dbt_ls_with_exclude(): @pytest.mark.integration @pytest.mark.parametrize("project_name", ("jaffle_shop", "jaffle_shop_python")) -def test_load_via_dbt_ls_without_exclude(project_name): +def test_load_via_dbt_ls_without_exclude(project_name, postgres_profile_config): project_config = ProjectConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / project_name) render_config = RenderConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) execution_config = ExecutionConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) @@ -294,14 +487,7 @@ def test_load_via_dbt_ls_without_exclude(project_name): project=project_config, render_config=render_config, execution_config=execution_config, - profile_config=ProfileConfig( - profile_name="default", - target_name="default", - profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="airflow_db", - profile_args={"schema": "public"}, - ), - ), + profile_config=postgres_profile_config, ) dbt_graph.load_via_dbt_ls() @@ -312,9 +498,8 @@ def test_load_via_dbt_ls_without_exclude(project_name): def test_load_via_custom_without_project_path(): project_config = ProjectConfig(manifest_path=SAMPLE_MANIFEST, project_name="test") execution_config = ExecutionConfig() - render_config = RenderConfig() + render_config = RenderConfig(dbt_executable_path="/inexistent/dbt") dbt_graph = DbtGraph( - dbt_cmd="/inexistent/dbt", project=project_config, execution_config=execution_config, render_config=render_config, @@ -326,12 +511,14 @@ def test_load_via_custom_without_project_path(): assert err_info.value.args[0] == expected -def test_load_via_dbt_ls_without_profile(): +@patch("cosmos.config.RenderConfig.validate_dbt_command", return_value=None) +def test_load_via_dbt_ls_without_profile(mock_validate_dbt_command): project_config = ProjectConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) execution_config = ExecutionConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) - render_config = RenderConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) + render_config = RenderConfig( + dbt_executable_path="existing-dbt-cmd", dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME + ) dbt_graph = DbtGraph( - dbt_cmd="/inexistent/dbt", project=project_config, execution_config=execution_config, render_config=render_config, @@ -343,13 +530,15 @@ def test_load_via_dbt_ls_without_profile(): assert err_info.value.args[0] == expected -def test_load_via_dbt_ls_with_invalid_dbt_path(): +@patch("cosmos.dbt.executable.shutil.which", return_value=None) +def test_load_via_dbt_ls_with_invalid_dbt_path(mock_which): project_config = ProjectConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) execution_config = ExecutionConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) - render_config = RenderConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) + render_config = RenderConfig( + dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME, dbt_executable_path="/inexistent/dbt" + ) with patch("pathlib.Path.exists", return_value=True): dbt_graph = DbtGraph( - dbt_cmd="/inexistent/dbt", project=project_config, execution_config=execution_config, render_config=render_config, @@ -359,10 +548,10 @@ def test_load_via_dbt_ls_with_invalid_dbt_path(): profiles_yml_filepath=Path(__file__).parent.parent / "sample/profiles.yml", ), ) - with pytest.raises(CosmosLoadDbtException) as err_info: + with pytest.raises(CosmosConfigException) as err_info: dbt_graph.load_via_dbt_ls() - expected = "Unable to find the dbt executable: /inexistent/dbt" + expected = "Unable to find the dbt executable, attempted: and ." assert err_info.value.args[0] == expected @@ -376,7 +565,11 @@ def test_load_via_dbt_ls_with_sources(load_method): dbt_project_path=DBT_PROJECTS_ROOT_DIR / project_name, manifest_path=SAMPLE_MANIFEST_SOURCE if load_method == "load_from_dbt_manifest" else None, ), - render_config=RenderConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / project_name, dbt_deps=False), + render_config=RenderConfig( + dbt_project_path=DBT_PROJECTS_ROOT_DIR / project_name, + dbt_deps=False, + env_vars={"DBT_SQLITE_PATH": str(DBT_PROJECTS_ROOT_DIR / "data")}, + ), execution_config=ExecutionConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / project_name), profile_config=ProfileConfig( profile_name="simple", @@ -385,13 +578,13 @@ def test_load_via_dbt_ls_with_sources(load_method): ), ) getattr(dbt_graph, load_method)() - assert len(dbt_graph.nodes) == 4 - assert "source.simple.imdb.movies_ratings" in dbt_graph.nodes + assert len(dbt_graph.nodes) >= 4 + assert "source.simple.main.movies_ratings" in dbt_graph.nodes assert "exposure.simple.weekly_metrics" in dbt_graph.nodes @pytest.mark.integration -def test_load_via_dbt_ls_without_dbt_deps(): +def test_load_via_dbt_ls_without_dbt_deps(postgres_profile_config): project_config = ProjectConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) render_config = RenderConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME, dbt_deps=False) execution_config = ExecutionConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) @@ -399,26 +592,148 @@ def test_load_via_dbt_ls_without_dbt_deps(): project=project_config, render_config=render_config, execution_config=execution_config, - profile_config=ProfileConfig( - profile_name="default", - target_name="default", - profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="airflow_db", - profile_args={"schema": "public"}, - ), - ), + profile_config=postgres_profile_config, ) with pytest.raises(CosmosLoadDbtException) as err_info: - dbt_graph.load_via_dbt_ls() + dbt_graph.load_via_dbt_ls_without_cache() expected = "Unable to run dbt ls command due to missing dbt_packages. Set RenderConfig.dbt_deps=True." assert err_info.value.args[0] == expected +@pytest.mark.integration +def test_load_via_dbt_ls_without_dbt_deps_and_preinstalled_dbt_packages( + tmp_dbt_project_dir, postgres_profile_config, caplog +): + local_flags = [ + "--project-dir", + tmp_dbt_project_dir / DBT_PROJECT_NAME, + "--profiles-dir", + tmp_dbt_project_dir / DBT_PROJECT_NAME, + "--profile", + "default", + "--target", + "dev", + ] + + deps_command = ["dbt", "deps"] + deps_command.extend(local_flags) + process = Popen( + deps_command, + stdout=PIPE, + stderr=PIPE, + cwd=tmp_dbt_project_dir / DBT_PROJECT_NAME, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + + project_config = ProjectConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) + render_config = RenderConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME, dbt_deps=False) + execution_config = ExecutionConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) + dbt_graph = DbtGraph( + project=project_config, + render_config=render_config, + execution_config=execution_config, + profile_config=postgres_profile_config, + ) + + assert dbt_graph.load_via_dbt_ls() is None # Doesn't raise any exceptions + + +@pytest.mark.integration +@pytest.mark.parametrize("enable_cache_profile", [True, False]) +@patch("cosmos.config.is_profile_cache_enabled") +def test_load_via_dbt_ls_caching_partial_parsing( + is_profile_cache_enabled, enable_cache_profile, tmp_dbt_project_dir, postgres_profile_config, caplog, tmp_path +): + """ + When using RenderConfig.enable_mock_profile=False and defining DbtGraph.cache_dir, + Cosmos should leverage dbt partial parsing. + """ + caplog.set_level(logging.DEBUG) + + is_profile_cache_enabled.return_value = enable_cache_profile + + project_config = ProjectConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) + render_config = RenderConfig( + dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME, dbt_deps=True, enable_mock_profile=False + ) + execution_config = ExecutionConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) + dbt_graph = DbtGraph( + project=project_config, + render_config=render_config, + execution_config=execution_config, + profile_config=postgres_profile_config, + cache_dir=tmp_path, + ) + + (tmp_path / DBT_TARGET_DIR_NAME).mkdir(parents=True, exist_ok=True) + + # First time dbt ls is run, partial parsing was not cached, so we don't benefit from this + dbt_graph.load_via_dbt_ls_without_cache() + assert "Unable to do partial parsing" in caplog.text + + # From the second time we run dbt ls onwards, we benefit from partial parsing + caplog.clear() + dbt_graph.load_via_dbt_ls_without_cache() # should not not raise exception + assert not "Unable to do partial parsing" in caplog.text + + +@pytest.mark.integration +def test_load_via_dbt_ls_uses_partial_parse_when_cache_is_disabled( + tmp_dbt_project_dir, postgres_profile_config, caplog, tmp_path +): + """ + When using RenderConfig.enable_mock_profile=False and defining DbtGraph.cache_dir, + Cosmos should leverage dbt partial parsing. + """ + target_dir = tmp_dbt_project_dir / DBT_PROJECT_NAME / DBT_TARGET_DIR_NAME + target_dir.mkdir(parents=True, exist_ok=True) + + partial_parse_cache_dir = Path("/tmp/cosmos") + partial_parse_cache_dir.mkdir(parents=True, exist_ok=True) + + caplog.set_level(logging.DEBUG) + project_config = ProjectConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) + render_config = RenderConfig( + dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME, dbt_deps=True, enable_mock_profile=False + ) + execution_config = ExecutionConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) + dbt_graph = DbtGraph( + project=project_config, + render_config=render_config, + execution_config=execution_config, + profile_config=postgres_profile_config, + cache_dir=partial_parse_cache_dir, + ) + # Creates partial parse file with the expected version + dbt_graph.load_via_dbt_ls_without_cache() # should not not raise exception + + caplog.clear() + + # We copy a valid partial parse to the project directory + shutil.copy(partial_parse_cache_dir / "target/partial_parse.msgpack", target_dir / "partial_parse.msgpack") + + # Run dbt ls without cache_dir, which disables cache: + dbt_graph = DbtGraph( + project=project_config, + render_config=render_config, + execution_config=execution_config, + profile_config=postgres_profile_config, + cache_dir="", # Cache is disabled + ) + # Should use the partial parse available in the original project folder + dbt_graph.load_via_dbt_ls_without_cache() # should not not raise exception + + assert not "Unable to do partial parsing" in caplog.text + + @pytest.mark.integration @patch("cosmos.dbt.graph.Popen") -def test_load_via_dbt_ls_with_zero_returncode_and_non_empty_stderr(mock_popen, tmp_dbt_project_dir): +def test_load_via_dbt_ls_with_zero_returncode_and_non_empty_stderr( + mock_popen, tmp_dbt_project_dir, postgres_profile_config +): mock_popen().communicate.return_value = ("", "Some stderr warnings") mock_popen().returncode = 0 @@ -429,14 +744,7 @@ def test_load_via_dbt_ls_with_zero_returncode_and_non_empty_stderr(mock_popen, t project=project_config, render_config=render_config, execution_config=execution_config, - profile_config=ProfileConfig( - profile_name="default", - target_name="default", - profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="airflow_db", - profile_args={"schema": "public"}, - ), - ), + profile_config=postgres_profile_config, ) dbt_graph.load_via_dbt_ls() # does not raise exception @@ -444,7 +752,7 @@ def test_load_via_dbt_ls_with_zero_returncode_and_non_empty_stderr(mock_popen, t @pytest.mark.integration @patch("cosmos.dbt.graph.Popen") -def test_load_via_dbt_ls_with_non_zero_returncode(mock_popen): +def test_load_via_dbt_ls_with_non_zero_returncode(mock_popen, postgres_profile_config): mock_popen().communicate.return_value = ("", "Some stderr message") mock_popen().returncode = 1 @@ -455,14 +763,7 @@ def test_load_via_dbt_ls_with_non_zero_returncode(mock_popen): project=project_config, render_config=render_config, execution_config=execution_config, - profile_config=ProfileConfig( - profile_name="default", - target_name="default", - profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="airflow_db", - profile_args={"schema": "public"}, - ), - ), + profile_config=postgres_profile_config, ) expected = r"Unable to run \['.+dbt', 'deps', .*\] due to the error:\nSome stderr message" with pytest.raises(CosmosLoadDbtException, match=expected): @@ -471,7 +772,7 @@ def test_load_via_dbt_ls_with_non_zero_returncode(mock_popen): @pytest.mark.integration @patch("cosmos.dbt.graph.Popen.communicate", return_value=("Some Runtime Error", "")) -def test_load_via_dbt_ls_with_runtime_error_in_stdout(mock_popen_communicate): +def test_load_via_dbt_ls_with_runtime_error_in_stdout(mock_popen_communicate, postgres_profile_config): # It may seem strange, but at least until dbt 1.6.0, there are circumstances when it outputs errors to stdout project_config = ProjectConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) render_config = RenderConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) @@ -480,14 +781,7 @@ def test_load_via_dbt_ls_with_runtime_error_in_stdout(mock_popen_communicate): project=project_config, render_config=render_config, execution_config=execution_config, - profile_config=ProfileConfig( - profile_name="default", - target_name="default", - profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="airflow_db", - profile_args={"schema": "public"}, - ), - ), + profile_config=postgres_profile_config, ) expected = r"Unable to run \['.+dbt', 'deps', .*\] due to the error:\nSome Runtime Error" with pytest.raises(CosmosLoadDbtException, match=expected): @@ -595,8 +889,7 @@ def test_tag_selected_node_test_exist(): profile_config=profile_config, render_config=render_config, ) - dbt_graph.load_from_dbt_manifest() - + dbt_graph.load() assert len(dbt_graph.filtered_nodes) > 0 for _, node in dbt_graph.filtered_nodes.items(): @@ -607,7 +900,7 @@ def test_tag_selected_node_test_exist(): @pytest.mark.integration @pytest.mark.parametrize("load_method", ["load_via_dbt_ls", "load_from_dbt_manifest"]) -def test_load_dbt_ls_and_manifest_with_model_version(load_method): +def test_load_dbt_ls_and_manifest_with_model_version(load_method, postgres_profile_config): dbt_graph = DbtGraph( project=ProjectConfig( dbt_project_path=DBT_PROJECTS_ROOT_DIR / "model_version", @@ -615,14 +908,7 @@ def test_load_dbt_ls_and_manifest_with_model_version(load_method): ), render_config=RenderConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / "model_version"), execution_config=ExecutionConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / "model_version"), - profile_config=ProfileConfig( - profile_name="default", - target_name="default", - profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="airflow_db", - profile_args={"schema": "public"}, - ), - ), + profile_config=postgres_profile_config, ) getattr(dbt_graph, load_method)() expected_dbt_nodes = { @@ -659,21 +945,43 @@ def test_load_dbt_ls_and_manifest_with_model_version(load_method): } == set(dbt_graph.nodes["model.jaffle_shop.orders"].depends_on) -def test_create_symlinks(tmp_path): - """Tests that symlinks are created for expected files in the dbt project directory.""" - tmp_dir = tmp_path / "dbt-project" - tmp_dir.mkdir() +@pytest.mark.integration +def test_load_via_dbt_ls_file(): + project_config = ProjectConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) + profile_config = ProfileConfig( + profile_name="test", + target_name="test", + profiles_yml_filepath=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME / "profiles.yml", + ) + render_config = RenderConfig( + dbt_ls_path=SAMPLE_DBT_LS_OUTPUT, dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME + ) + dbt_graph = DbtGraph( + project=project_config, + profile_config=profile_config, + render_config=render_config, + ) + dbt_graph.load(method=LoadMode.DBT_LS_FILE, execution_mode=ExecutionMode.LOCAL) - create_symlinks(DBT_PROJECTS_ROOT_DIR / "jaffle_shop", tmp_dir) - for child in tmp_dir.iterdir(): - assert child.is_symlink() - assert child.name not in ("logs", "target", "profiles.yml", "dbt_packages") + expected_dbt_nodes = { + "model.jaffle_shop.stg_customers": "stg_customers", + "model.jaffle_shop.stg_orders": "stg_orders", + "model.jaffle_shop.stg_payments": "stg_payments", + } + for unique_id, name in expected_dbt_nodes.items(): + assert unique_id in dbt_graph.nodes + assert name == dbt_graph.nodes[unique_id].name + # Test dependencies + assert {"seed.jaffle_shop.raw_customers"} == set(dbt_graph.nodes["model.jaffle_shop.stg_customers"].depends_on) + assert {"seed.jaffle_shop.raw_orders"} == set(dbt_graph.nodes["model.jaffle_shop.stg_orders"].depends_on) + assert {"seed.jaffle_shop.raw_payments"} == set(dbt_graph.nodes["model.jaffle_shop.stg_payments"].depends_on) @pytest.mark.parametrize( "stdout,returncode", [ ("all good", None), + ("WarnErrorOptions", None), pytest.param("fail", 599, marks=pytest.mark.xfail(raises=CosmosLoadDbtException)), pytest.param("Error", None, marks=pytest.mark.xfail(raises=CosmosLoadDbtException)), ], @@ -701,7 +1009,6 @@ def test_parse_dbt_ls_output(): expected_nodes = { "fake-unique-id": DbtNode( - name="fake-name", unique_id="fake-unique-id", resource_type=DbtResourceType.MODEL, file_path=Path("fake-project/fake-file-path.sql"), @@ -713,3 +1020,480 @@ def test_parse_dbt_ls_output(): nodes = parse_dbt_ls_output(Path("fake-project"), fake_ls_stdout) assert expected_nodes == nodes + + +def test_parse_dbt_ls_output_with_json_without_tags_or_config(): + some_ls_stdout = '{"resource_type": "model", "name": "some-name", "original_file_path": "some-file-path.sql", "unique_id": "some-unique-id", "config": {}}' + + expected_nodes = { + "some-unique-id": DbtNode( + unique_id="some-unique-id", + resource_type=DbtResourceType.MODEL, + file_path=Path("some-project/some-file-path.sql"), + tags=[], + config={}, + depends_on=[], + ), + } + nodes = parse_dbt_ls_output(Path("some-project"), some_ls_stdout) + + assert expected_nodes == nodes + + +@patch("cosmos.dbt.graph.DbtGraph.should_use_dbt_ls_cache", return_value=False) +@patch("cosmos.dbt.graph.Popen") +@patch("cosmos.dbt.graph.DbtGraph.update_node_dependency") +@patch("cosmos.config.RenderConfig.validate_dbt_command") +def test_load_via_dbt_ls_project_config_env_vars( + mock_validate, mock_update_nodes, mock_popen, mock_enable_cache, tmp_dbt_project_dir +): + """Tests that the dbt ls command in the subprocess has the project config env vars set.""" + mock_popen().communicate.return_value = ("", "") + mock_popen().returncode = 0 + env_vars = {"MY_ENV_VAR": "my_value"} + project_config = ProjectConfig(env_vars=env_vars) + render_config = RenderConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) + profile_config = ProfileConfig( + profile_name="test", + target_name="test", + profiles_yml_filepath=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME / "profiles.yml", + ) + execution_config = ExecutionConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) + dbt_graph = DbtGraph( + project=project_config, + render_config=render_config, + execution_config=execution_config, + profile_config=profile_config, + ) + dbt_graph.load_via_dbt_ls() + + assert "MY_ENV_VAR" in mock_popen.call_args.kwargs["env"] + assert mock_popen.call_args.kwargs["env"]["MY_ENV_VAR"] == "my_value" + + +@patch("cosmos.dbt.graph.DbtGraph.should_use_dbt_ls_cache", return_value=False) +@patch("cosmos.config.is_profile_cache_enabled", return_value=False) +@patch("cosmos.dbt.graph.Popen") +@patch("cosmos.dbt.graph.DbtGraph.update_node_dependency") +@patch("cosmos.config.RenderConfig.validate_dbt_command") +def test_profile_created_correctly_with_profile_mapping( + mock_validate, + mock_update_nodes, + mock_popen, + mock_enable_profile_cache, + mock_enable_cache, + tmp_dbt_project_dir, + postgres_profile_config, +): + """Tests that the temporary profile is created without errors.""" + mock_popen().communicate.return_value = ("", "") + mock_popen().returncode = 0 + project_config = ProjectConfig(env_vars={}) + render_config = RenderConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) + profile_config = postgres_profile_config + execution_config = ExecutionConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) + dbt_graph = DbtGraph( + project=project_config, + render_config=render_config, + execution_config=execution_config, + profile_config=profile_config, + ) + + assert dbt_graph.load_via_dbt_ls() == None + + +@patch("cosmos.dbt.graph.DbtGraph.should_use_dbt_ls_cache", return_value=False) +@patch("cosmos.dbt.graph.Popen") +@patch("cosmos.dbt.graph.DbtGraph.update_node_dependency") +@patch("cosmos.config.RenderConfig.validate_dbt_command") +def test_load_via_dbt_ls_project_config_dbt_vars( + mock_validate, mock_update_nodes, mock_popen, mock_use_case, tmp_dbt_project_dir +): + """Tests that the dbt ls command in the subprocess has "--vars" with the project config dbt_vars.""" + mock_popen().communicate.return_value = ("", "") + mock_popen().returncode = 0 + dbt_vars = {"my_var1": "my_value1", "my_var2": "my_value2"} + project_config = ProjectConfig(dbt_vars=dbt_vars) + render_config = RenderConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) + profile_config = ProfileConfig( + profile_name="test", + target_name="test", + profiles_yml_filepath=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME / "profiles.yml", + ) + execution_config = ExecutionConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) + dbt_graph = DbtGraph( + project=project_config, + render_config=render_config, + execution_config=execution_config, + profile_config=profile_config, + ) + dbt_graph.load_via_dbt_ls() + ls_command = mock_popen.call_args.args[0] + assert "--vars" in ls_command + assert ls_command[ls_command.index("--vars") + 1] == '{"my_var1": "my_value1", "my_var2": "my_value2"}' + + +@patch("cosmos.dbt.graph.DbtGraph.should_use_dbt_ls_cache", return_value=False) +@patch("cosmos.dbt.graph.Popen") +@patch("cosmos.dbt.graph.DbtGraph.update_node_dependency") +@patch("cosmos.config.RenderConfig.validate_dbt_command") +def test_load_via_dbt_ls_render_config_selector_arg_is_used( + mock_validate, mock_update_nodes, mock_popen, mock_enable_cache, tmp_dbt_project_dir +): + """Tests that the dbt ls command in the subprocess has "--selector" with the RenderConfig.selector.""" + mock_popen().communicate.return_value = ("", "") + mock_popen().returncode = 0 + selector = "my_selector" + project_config = ProjectConfig() + render_config = RenderConfig( + dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME, + load_method=LoadMode.DBT_LS, + selector=selector, + ) + profile_config = ProfileConfig( + profile_name="test", + target_name="test", + profiles_yml_filepath=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME / "profiles.yml", + ) + execution_config = MagicMock() + dbt_graph = DbtGraph( + project=project_config, + render_config=render_config, + execution_config=execution_config, + profile_config=profile_config, + ) + dbt_graph.load_via_dbt_ls() + ls_command = mock_popen.call_args.args[0] + assert "--selector" in ls_command + assert ls_command[ls_command.index("--selector") + 1] == selector + + +@patch("cosmos.dbt.graph.DbtGraph.should_use_dbt_ls_cache", return_value=False) +@patch("cosmos.dbt.graph.Popen") +@patch("cosmos.dbt.graph.DbtGraph.update_node_dependency") +@patch("cosmos.config.RenderConfig.validate_dbt_command") +def test_load_via_dbt_ls_render_config_no_partial_parse( + mock_validate, mock_update_nodes, mock_popen, mock_enable_cache, tmp_dbt_project_dir +): + """Tests that --no-partial-parse appears when partial_parse=False.""" + mock_popen().communicate.return_value = ("", "") + mock_popen().returncode = 0 + project_config = ProjectConfig(partial_parse=False) + render_config = RenderConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME, load_method=LoadMode.DBT_LS) + profile_config = ProfileConfig( + profile_name="test", + target_name="test", + profiles_yml_filepath=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME / "profiles.yml", + ) + execution_config = MagicMock() + dbt_graph = DbtGraph( + project=project_config, + render_config=render_config, + execution_config=execution_config, + profile_config=profile_config, + ) + dbt_graph.load_via_dbt_ls() + ls_command = mock_popen.call_args.args[0] + assert "--no-partial-parse" in ls_command + + +@pytest.mark.parametrize("load_method", [LoadMode.DBT_MANIFEST, LoadMode.CUSTOM]) +def test_load_method_with_unsupported_render_config_selector_arg(load_method): + """Tests that error is raised when RenderConfig.selector is used with LoadMode.DBT_MANIFEST or LoadMode.CUSTOM.""" + + expected_error_msg = ( + f"RenderConfig.selector is not yet supported when loading dbt projects using the {load_method} parser." + ) + dbt_graph = DbtGraph( + render_config=RenderConfig(load_method=load_method, selector="my_selector"), + project=MagicMock(), + ) + with pytest.raises(CosmosLoadDbtException, match=expected_error_msg): + dbt_graph.load(method=load_method) + + +@pytest.mark.sqlite +@pytest.mark.integration +def test_load_via_dbt_ls_with_project_config_vars(): + """ + Integration that tests that the dbt ls command is successful and that the node affected by the dbt_vars is + rendered correctly. + """ + project_name = "simple" + dbt_graph = DbtGraph( + project=ProjectConfig( + dbt_project_path=DBT_PROJECTS_ROOT_DIR / project_name, + env_vars={"DBT_SQLITE_PATH": str(DBT_PROJECTS_ROOT_DIR / "data")}, + dbt_vars={"animation_alias": "top_5_animated_movies"}, + ), + render_config=RenderConfig( + dbt_project_path=DBT_PROJECTS_ROOT_DIR / project_name, + dbt_deps=False, + ), + execution_config=ExecutionConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / project_name), + profile_config=ProfileConfig( + profile_name="simple", + target_name="dev", + profiles_yml_filepath=(DBT_PROJECTS_ROOT_DIR / project_name / "profiles.yml"), + ), + ) + dbt_graph.load_via_dbt_ls() + assert dbt_graph.nodes["model.simple.top_animations"].config["alias"] == "top_5_animated_movies" + + +@pytest.mark.integration +def test_load_via_dbt_ls_with_selector_arg(tmp_dbt_project_dir, postgres_profile_config): + """ + Tests that the dbt ls load method is successful if a selector arg is used with RenderConfig + and that the filtered nodes are expected. + """ + # Add a selectors yaml file to the project that will select the stg_customers model and all + # parents (raw_customers) + selectors_yaml = """ + selectors: + - name: stage_customers + definition: + method: fqn + value: stg_customers + parents: true + """ + with open(tmp_dbt_project_dir / DBT_PROJECT_NAME / "selectors.yml", "w") as f: + f.write(selectors_yaml) + + project_config = ProjectConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) + execution_config = ExecutionConfig(dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME) + render_config = RenderConfig( + dbt_project_path=tmp_dbt_project_dir / DBT_PROJECT_NAME, + selector="stage_customers", + ) + + dbt_graph = DbtGraph( + project=project_config, + render_config=render_config, + execution_config=execution_config, + profile_config=postgres_profile_config, + ) + dbt_graph.load_via_dbt_ls() + + filtered_nodes = dbt_graph.filtered_nodes.keys() + assert len(filtered_nodes) == 4 + assert "model.jaffle_shop.stg_customers" in filtered_nodes + assert "seed.jaffle_shop.raw_customers" in filtered_nodes + # Two tests should be filtered + assert sum(node.startswith("test.jaffle_shop") for node in filtered_nodes) == 2 + + +@pytest.mark.parametrize( + "render_config,project_config,expected_envvars", + [ + (RenderConfig(), ProjectConfig(), {}), + (RenderConfig(env_vars={"a": 1}), ProjectConfig(), {"a": 1}), + (RenderConfig(), ProjectConfig(env_vars={"b": 2}), {"b": 2}), + (RenderConfig(env_vars={"a": 1}), ProjectConfig(env_vars={"b": 2}), {"a": 1}), + ], +) +def test_env_vars(render_config, project_config, expected_envvars): + graph = DbtGraph( + project=project_config, + render_config=render_config, + ) + assert graph.env_vars == expected_envvars + + +def test_project_path_fails(): + graph = DbtGraph(project=ProjectConfig()) + with pytest.raises(CosmosLoadDbtException) as e: + graph.project_path + + expected = "Unable to load project via dbt ls without RenderConfig.dbt_project_path, ProjectConfig.dbt_project_path or ExecutionConfig.dbt_project_path" + assert e.value.args[0] == expected + + +@pytest.mark.parametrize( + "render_config,project_config,expected_dbt_ls_args", + [ + (RenderConfig(), ProjectConfig(), []), + (RenderConfig(exclude=["package:snowplow"]), ProjectConfig(), ["--exclude", "package:snowplow"]), + ( + RenderConfig(select=["tag:prod", "config.materialized:incremental"]), + ProjectConfig(), + ["--select", "tag:prod", "config.materialized:incremental"], + ), + (RenderConfig(selector="nightly"), ProjectConfig(), ["--selector", "nightly"]), + (RenderConfig(), ProjectConfig(dbt_vars={"a": 1}), ["--vars", '{"a": 1}']), + (RenderConfig(), ProjectConfig(partial_parse=False), ["--no-partial-parse"]), + ( + RenderConfig(exclude=["1", "2"], select=["a", "b"], selector="nightly"), + ProjectConfig(dbt_vars={"a": 1}, partial_parse=False), + [ + "--exclude", + "1", + "2", + "--select", + "a", + "b", + "--vars", + '{"a": 1}', + "--selector", + "nightly", + "--no-partial-parse", + ], + ), + ], +) +def test_dbt_ls_args(render_config, project_config, expected_dbt_ls_args): + graph = DbtGraph( + project=project_config, + render_config=render_config, + ) + assert graph.dbt_ls_args == expected_dbt_ls_args + + +def test_dbt_ls_cache_key_args_sorts_envvars(): + project_config = ProjectConfig(env_vars={11: "November", 12: "December", 5: "May"}) + graph = DbtGraph(project=project_config) + assert graph.dbt_ls_cache_key_args == ['{"5": "May", "11": "November", "12": "December"}'] + + +@pytest.fixture() +def airflow_variable(): + key = "cosmos_cache__undefined" + value = "some_value" + Variable.set(key, value) + + yield key, value + + Variable.delete(key) + + +@pytest.mark.integration +def test_dbt_ls_cache_key_args_uses_airflow_vars_to_purge_dbt_ls_cache(airflow_variable): + key, value = airflow_variable + graph = DbtGraph(project=ProjectConfig(), render_config=RenderConfig(airflow_vars_to_purge_dbt_ls_cache=[key])) + assert graph.dbt_ls_cache_key_args == [key, value] + + +@patch("cosmos.dbt.graph.datetime") +@patch("cosmos.dbt.graph.Variable.set") +def test_save_dbt_ls_cache(mock_variable_set, mock_datetime, tmp_dbt_project_dir): + mock_datetime.datetime.now.return_value = datetime(2022, 1, 1, 12, 0, 0) + graph = DbtGraph(cache_identifier="something", project=ProjectConfig(dbt_project_path=tmp_dbt_project_dir)) + dbt_ls_output = "some output" + graph.save_dbt_ls_cache(dbt_ls_output) + assert mock_variable_set.call_args[0][0] == "cosmos_cache__something" + assert mock_variable_set.call_args[0][1]["dbt_ls_compressed"] == "eJwrzs9NVcgvLSkoLQEAGpAEhg==" + assert mock_variable_set.call_args[0][1]["last_modified"] == "2022-01-01T12:00:00" + version = mock_variable_set.call_args[0][1].get("version") + hash_dir, hash_args = version.split(",") + assert hash_args == "d41d8cd98f00b204e9800998ecf8427e" + if sys.platform == "darwin": + assert hash_dir == "cdc6f0bec00f4edc616f3aa755a34330" + else: + assert hash_dir == "77d08d6da374330ac1b49438ff2873f7" + + +@pytest.mark.integration +def test_get_dbt_ls_cache_returns_empty_if_non_json_var(airflow_variable): + graph = DbtGraph(project=ProjectConfig()) + assert graph.get_dbt_ls_cache() == {} + + +@patch("cosmos.dbt.graph.Variable.get", return_value={"dbt_ls_compressed": "eJwrzs9NVcgvLSkoLQEAGpAEhg=="}) +def test_get_dbt_ls_cache_returns_decoded_and_decompressed_value(mock_variable_get): + graph = DbtGraph(project=ProjectConfig()) + assert graph.get_dbt_ls_cache() == {"dbt_ls": "some output"} + + +@patch("cosmos.dbt.graph.Variable.get", return_value={}) +def test_get_dbt_ls_cache_returns_empty_dict_if_empty_dict_var(mock_variable_get): + graph = DbtGraph(project=ProjectConfig()) + assert graph.get_dbt_ls_cache() == {} + + +@patch("cosmos.dbt.graph.DbtGraph.load_via_dbt_ls_without_cache") +@patch("cosmos.dbt.graph.DbtGraph.load_via_dbt_ls_cache", return_value=True) +def test_load_via_dbt_ls_does_not_call_without_cache(mock_cache, mock_without_cache): + graph = DbtGraph(project=ProjectConfig()) + graph.load_via_dbt_ls() + assert mock_cache.called + assert not mock_without_cache.called + + +@patch("cosmos.dbt.graph.DbtGraph.load_via_dbt_ls_without_cache") +@patch("cosmos.dbt.graph.DbtGraph.load_via_dbt_ls_cache", return_value=False) +def test_load_via_dbt_ls_calls_without_cache(mock_cache, mock_without_cache): + graph = DbtGraph(project=ProjectConfig()) + graph.load_via_dbt_ls() + assert mock_cache.called + assert mock_without_cache.called + + +@patch("cosmos.dbt.graph.DbtGraph.should_use_dbt_ls_cache", return_value=False) +def test_load_via_dbt_ls_cache_is_false_if_disabled(mock_should_use_dbt_ls_cache): + graph = DbtGraph(project=ProjectConfig()) + assert not graph.load_via_dbt_ls_cache() + assert mock_should_use_dbt_ls_cache.called + + +@patch("cosmos.dbt.graph.DbtGraph.get_dbt_ls_cache", return_value={}) +@patch("cosmos.dbt.graph.DbtGraph.should_use_dbt_ls_cache", return_value=True) +def test_load_via_dbt_ls_cache_is_false_if_no_cache(mock_should_use_dbt_ls_cache, mock_get_dbt_ls_cache): + graph = DbtGraph(project=ProjectConfig(dbt_project_path="/tmp")) + assert not graph.load_via_dbt_ls_cache() + assert mock_should_use_dbt_ls_cache.called + assert mock_get_dbt_ls_cache.called + + +@patch("cosmos.dbt.graph.cache._calculate_dbt_ls_cache_current_version", return_value=1) +@patch("cosmos.dbt.graph.DbtGraph.get_dbt_ls_cache", return_value={"version": 2, "dbt_ls": "output"}) +@patch("cosmos.dbt.graph.DbtGraph.should_use_dbt_ls_cache", return_value=True) +def test_load_via_dbt_ls_cache_is_false_if_cache_is_outdated( + mock_should_use_dbt_ls_cache, mock_get_dbt_ls_cache, mock_calculate_current_version +): + graph = DbtGraph(project=ProjectConfig(dbt_project_path="/tmp")) + assert not graph.load_via_dbt_ls_cache() + assert mock_should_use_dbt_ls_cache.called + assert mock_get_dbt_ls_cache.called + assert mock_calculate_current_version.called + + +@patch("cosmos.dbt.graph.parse_dbt_ls_output", return_value={"some-node": {}}) +@patch("cosmos.dbt.graph.cache._calculate_dbt_ls_cache_current_version", return_value=1) +@patch("cosmos.dbt.graph.DbtGraph.get_dbt_ls_cache", return_value={"version": 1, "dbt_ls": "output"}) +@patch("cosmos.dbt.graph.DbtGraph.should_use_dbt_ls_cache", return_value=True) +def test_load_via_dbt_ls_cache_is_true( + mock_should_use_dbt_ls_cache, mock_get_dbt_ls_cache, mock_calculate_current_version, mock_parse_dbt_ls_output +): + graph = DbtGraph(project=ProjectConfig(dbt_project_path="/tmp")) + assert graph.load_via_dbt_ls_cache() + assert graph.load_method == LoadMode.DBT_LS_CACHE + assert graph.nodes == {"some-node": {}} + assert graph.filtered_nodes == {"some-node": {}} + assert mock_should_use_dbt_ls_cache.called + assert mock_get_dbt_ls_cache.called + assert mock_calculate_current_version.called + assert mock_parse_dbt_ls_output.called + + +@pytest.mark.parametrize( + "enable_cache,enable_cache_dbt_ls,cache_id,should_use", + [ + (False, True, "id", False), + (True, False, "id", False), + (False, False, "id", False), + (True, True, "", False), + (True, True, "id", True), + ], +) +def test_should_use_dbt_ls_cache(enable_cache, enable_cache_dbt_ls, cache_id, should_use): + with patch.dict( + os.environ, + { + "AIRFLOW__COSMOS__ENABLE_CACHE": str(enable_cache), + "AIRFLOW__COSMOS__ENABLE_CACHE_DBT_LS": str(enable_cache_dbt_ls), + }, + ): + importlib.reload(settings) + graph = DbtGraph(cache_identifier=cache_id, project=ProjectConfig(dbt_project_path="/tmp")) + graph.should_use_dbt_ls_cache.cache_clear() + assert graph.should_use_dbt_ls_cache() == should_use diff --git a/tests/dbt/test_project.py b/tests/dbt/test_project.py new file mode 100644 index 0000000000..df625182fc --- /dev/null +++ b/tests/dbt/test_project.py @@ -0,0 +1,71 @@ +import os +from pathlib import Path +from unittest.mock import patch + +import pytest + +from cosmos.dbt.project import change_working_directory, create_symlinks, environ, has_non_empty_dependencies_file + +DBT_PROJECTS_ROOT_DIR = Path(__file__).parent.parent.parent / "dev/dags/dbt" + + +def test_create_symlinks(tmp_path): + """Tests that symlinks are created for expected files in the dbt project directory.""" + tmp_dir = tmp_path / "dbt-project" + tmp_dir.mkdir() + + create_symlinks(DBT_PROJECTS_ROOT_DIR / "jaffle_shop", tmp_dir, False) + for child in tmp_dir.iterdir(): + assert child.is_symlink() + assert child.name not in ("logs", "target", "profiles.yml", "dbt_packages") + + +@patch.dict(os.environ, {"VAR1": "value1", "VAR2": "value2"}) +def test_environ_context_manager(): + # Define the expected environment variables + expected_env_vars = {"VAR2": "new_value2", "VAR3": "value3"} + # Use the environ context manager + with environ(expected_env_vars): + # Check if the environment variables are set correctly + for key, value in expected_env_vars.items(): + assert value == os.environ.get(key) + # Check if the original non-overlapping environment variable is still set + assert "value1" == os.environ.get("VAR1") + # Check if the environment variables are unset after exiting the context manager + assert os.environ.get("VAR3") is None + # Check if the original environment variables are still set + assert "value1" == os.environ.get("VAR1") + assert "value2" == os.environ.get("VAR2") + + +@patch("os.chdir") +def test_change_working_directory(mock_chdir): + """Tests that the working directory is changed and then restored correctly.""" + # Define the path to change the working directory to + path = "/path/to/directory" + + # Use the change_working_directory context manager + with change_working_directory(path): + # Check if os.chdir is called with the correct path + mock_chdir.assert_called_once_with(path) + + # Check if os.chdir is called with the previous working directory + mock_chdir.assert_called_with(os.getcwd()) + + +@pytest.mark.parametrize("filename", ["packages.yml", "dependencies.yml"]) +def test_has_non_empty_dependencies_file_is_true(tmpdir, filename): + filepath = Path(tmpdir) / filename + filepath.write_text("content") + assert has_non_empty_dependencies_file(tmpdir) + + +@pytest.mark.parametrize("filename", ["packages.yml", "dependencies.yml"]) +def test_has_non_empty_dependencies_file_is_false(tmpdir, filename): + filepath = Path(tmpdir) / filename + filepath.touch() + assert not has_non_empty_dependencies_file(tmpdir) + + +def test_has_non_empty_dependencies_file_is_false_in_empty_dir(tmpdir): + assert not has_non_empty_dependencies_file(tmpdir) diff --git a/tests/dbt/test_selector.py b/tests/dbt/test_selector.py index 9f6071a20b..ece32ac951 100644 --- a/tests/dbt/test_selector.py +++ b/tests/dbt/test_selector.py @@ -2,10 +2,9 @@ import pytest -from cosmos.dbt.selector import SelectorConfig from cosmos.constants import DbtResourceType from cosmos.dbt.graph import DbtNode -from cosmos.dbt.selector import select_nodes +from cosmos.dbt.selector import NodeSelector, SelectorConfig, select_nodes from cosmos.exceptions import CosmosValueError SAMPLE_PROJ_PATH = Path("/home/user/path/dbt-proj/") @@ -39,59 +38,76 @@ def test_is_empty_config(selector_config, paths, tags, config, other, expected): grandparent_node = DbtNode( - name="grandparent", - unique_id="grandparent", + unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.grandparent", resource_type=DbtResourceType.MODEL, depends_on=[], file_path=SAMPLE_PROJ_PATH / "gen1/models/grandparent.sql", tags=["has_child"], config={"materialized": "view", "tags": ["has_child"]}, ) + +another_grandparent_node = DbtNode( + unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.another_grandparent_node", + resource_type=DbtResourceType.MODEL, + depends_on=[], + file_path=SAMPLE_PROJ_PATH / "gen1/models/another_grandparent_node.sql", + tags=[], + config={}, +) + parent_node = DbtNode( - name="parent", - unique_id="parent", + unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.parent", resource_type=DbtResourceType.MODEL, - depends_on=["grandparent"], + depends_on=[grandparent_node.unique_id, another_grandparent_node.unique_id], file_path=SAMPLE_PROJ_PATH / "gen2/models/parent.sql", tags=["has_child", "is_child"], config={"materialized": "view", "tags": ["has_child", "is_child"]}, ) + child_node = DbtNode( - name="child", - unique_id="child", + unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.child", resource_type=DbtResourceType.MODEL, - depends_on=["parent"], + depends_on=[parent_node.unique_id], file_path=SAMPLE_PROJ_PATH / "gen3/models/child.sql", tags=["nightly", "is_child"], config={"materialized": "table", "tags": ["nightly", "is_child"]}, ) -grandchild_1_test_node = DbtNode( - name="grandchild_1", - unique_id="grandchild_1", +sibling1_node = DbtNode( + unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.sibling1", resource_type=DbtResourceType.MODEL, - depends_on=["parent"], - file_path=SAMPLE_PROJ_PATH / "gen3/models/grandchild_1.sql", + depends_on=[parent_node.unique_id], + file_path=SAMPLE_PROJ_PATH / "gen3/models/sibling1.sql", tags=["nightly", "deprecated", "test"], config={"materialized": "table", "tags": ["nightly", "deprecated", "test"]}, ) -grandchild_2_test_node = DbtNode( - name="grandchild_2", - unique_id="grandchild_2", +sibling2_node = DbtNode( + unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.sibling2", resource_type=DbtResourceType.MODEL, - depends_on=["parent"], - file_path=SAMPLE_PROJ_PATH / "gen3/models/grandchild_2.sql", + depends_on=[parent_node.unique_id], + file_path=SAMPLE_PROJ_PATH / "gen3/models/sibling2.sql", tags=["nightly", "deprecated", "test2"], config={"materialized": "table", "tags": ["nightly", "deprecated", "test2"]}, ) +orphaned_node = DbtNode( + unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.orphaned", + resource_type=DbtResourceType.MODEL, + depends_on=[], + file_path=SAMPLE_PROJ_PATH / "gen3/models/orphaned.sql", + tags=[], + config={}, +) + sample_nodes = { grandparent_node.unique_id: grandparent_node, + another_grandparent_node.unique_id: another_grandparent_node, parent_node.unique_id: parent_node, child_node.unique_id: child_node, - grandchild_1_test_node.unique_id: grandchild_1_test_node, - grandchild_2_test_node.unique_id: grandchild_2_test_node, + sibling1_node.unique_id: sibling1_node, + sibling2_node.unique_id: sibling2_node, + orphaned_node.unique_id: orphaned_node, } @@ -105,8 +121,8 @@ def test_select_nodes_by_select_config(): selected = select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, select=["config.materialized:table"]) expected = { child_node.unique_id: child_node, - grandchild_1_test_node.unique_id: grandchild_1_test_node, - grandchild_2_test_node.unique_id: grandchild_2_test_node, + sibling1_node.unique_id: sibling1_node, + sibling2_node.unique_id: sibling2_node, } assert selected == expected @@ -141,8 +157,8 @@ def test_select_nodes_by_select_union_config_test_tags(): expected = { grandparent_node.unique_id: grandparent_node, parent_node.unique_id: parent_node, - grandchild_1_test_node.unique_id: grandchild_1_test_node, - grandchild_2_test_node.unique_id: grandchild_2_test_node, + sibling1_node.unique_id: sibling1_node, + sibling2_node.unique_id: sibling2_node, } assert selected == expected @@ -181,8 +197,8 @@ def test_select_nodes_by_select_union(): grandparent_node.unique_id: grandparent_node, parent_node.unique_id: parent_node, child_node.unique_id: child_node, - grandchild_1_test_node.unique_id: grandchild_1_test_node, - grandchild_2_test_node.unique_id: grandchild_2_test_node, + sibling1_node.unique_id: sibling1_node, + sibling2_node.unique_id: sibling2_node, } assert selected == expected @@ -196,8 +212,10 @@ def test_select_nodes_by_exclude_tag(): selected = select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, exclude=["tag:has_child"]) expected = { child_node.unique_id: child_node, - grandchild_1_test_node.unique_id: grandchild_1_test_node, - grandchild_2_test_node.unique_id: grandchild_2_test_node, + sibling1_node.unique_id: sibling1_node, + sibling2_node.unique_id: sibling2_node, + another_grandparent_node.unique_id: another_grandparent_node, + orphaned_node.unique_id: orphaned_node, } assert selected == expected @@ -222,8 +240,10 @@ def test_select_nodes_by_exclude_union_config_test_tags(): ) expected = { grandparent_node.unique_id: grandparent_node, + another_grandparent_node.unique_id: another_grandparent_node, parent_node.unique_id: parent_node, child_node.unique_id: child_node, + orphaned_node.unique_id: orphaned_node, } assert selected == expected @@ -232,15 +252,183 @@ def test_select_nodes_by_path_dir(): selected = select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, select=["path:gen3/models"]) expected = { child_node.unique_id: child_node, - grandchild_1_test_node.unique_id: grandchild_1_test_node, - grandchild_2_test_node.unique_id: grandchild_2_test_node, + sibling1_node.unique_id: sibling1_node, + sibling2_node.unique_id: sibling2_node, + orphaned_node.unique_id: orphaned_node, } assert selected == expected def test_select_nodes_by_path_file(): selected = select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, select=["path:gen2/models/parent.sql"]) - expected = { - parent_node.unique_id: parent_node, - } - assert selected == expected + expected = [parent_node.unique_id] + assert list(selected.keys()) == expected + + +def test_select_nodes_by_child_and_precursors(): + selected = select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, select=["+child"]) + expected = [ + another_grandparent_node.unique_id, + child_node.unique_id, + grandparent_node.unique_id, + parent_node.unique_id, + ] + assert sorted(selected.keys()) == expected + + +def test_select_nodes_by_child_and_precursors_exclude_tags(): + selected = select_nodes( + project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, select=["+child"], exclude=["tag:has_child"] + ) + expected = [another_grandparent_node.unique_id, child_node.unique_id] + assert sorted(selected.keys()) == expected + + +def test_select_node_by_child_and_precursors_partial_tree(): + selected = select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, select=["+parent"]) + expected = [another_grandparent_node.unique_id, grandparent_node.unique_id, parent_node.unique_id] + assert sorted(selected.keys()) == expected + + +def test_select_node_by_precursors_with_orphaned_node(): + selected = select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, select=["+orphaned"]) + expected = [orphaned_node.unique_id] + assert list(selected.keys()) == expected + + +def test_select_nodes_by_child_and_first_degree_precursors(): + selected = select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, select=["1+child"]) + expected = [ + child_node.unique_id, + parent_node.unique_id, + ] + assert sorted(selected.keys()) == expected + + +def test_select_nodes_by_child_and_second_degree_precursors(): + selected = select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, select=["2+child"]) + expected = [ + another_grandparent_node.unique_id, + child_node.unique_id, + grandparent_node.unique_id, + parent_node.unique_id, + ] + assert sorted(selected.keys()) == expected + + +def test_select_node_by_exact_node_name(): + selected = select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, select=["child"]) + expected = [child_node.unique_id] + assert list(selected.keys()) == expected + + +def test_select_node_by_child_and_precursors_no_node(): + selected = select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, select=["+modelDoesntExist"]) + expected = [] + assert list(selected.keys()) == expected + + +def test_select_node_by_descendants(): + selected = select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, select=["grandparent+"]) + expected = [ + "model.dbt-proj.child", + "model.dbt-proj.grandparent", + "model.dbt-proj.parent", + "model.dbt-proj.sibling1", + "model.dbt-proj.sibling2", + ] + assert sorted(selected.keys()) == expected + + +def test_select_node_by_descendants_depth_first_degree(): + selected = select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, select=["grandparent+1"]) + expected = [ + "model.dbt-proj.grandparent", + "model.dbt-proj.parent", + ] + assert sorted(selected.keys()) == expected + + +def test_select_node_by_descendants_union(): + selected = select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, select=["grandparent+1", "parent+1"]) + expected = [ + "model.dbt-proj.child", + "model.dbt-proj.grandparent", + "model.dbt-proj.parent", + "model.dbt-proj.sibling1", + "model.dbt-proj.sibling2", + ] + assert sorted(selected.keys()) == expected + + +def test_select_node_by_descendants_intersection(): + selected = select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, select=["grandparent+1,parent+1"]) + expected = [ + "model.dbt-proj.parent", + ] + assert sorted(selected.keys()) == expected + + +def test_select_node_by_descendants_intersection_with_tag(): + selected = select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, select=["parent+1,tag:has_child"]) + expected = [ + "model.dbt-proj.parent", + ] + assert sorted(selected.keys()) == expected + + +def test_select_node_by_descendants_and_tag_union(): + selected = select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, select=["child", "tag:has_child"]) + expected = [ + "model.dbt-proj.child", + "model.dbt-proj.grandparent", + "model.dbt-proj.parent", + ] + assert sorted(selected.keys()) == expected + + +def test_exclude_by_graph_selector(): + selected = select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, exclude=["+parent"]) + expected = [ + "model.dbt-proj.child", + "model.dbt-proj.orphaned", + "model.dbt-proj.sibling1", + "model.dbt-proj.sibling2", + ] + assert sorted(selected.keys()) == expected + + +def test_exclude_by_union_graph_selector_and_tag(): + selected = select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=sample_nodes, exclude=["+parent", "tag:deprecated"]) + expected = [ + "model.dbt-proj.child", + "model.dbt-proj.orphaned", + ] + assert sorted(selected.keys()) == expected + + +def test_node_without_depends_on_with_tag_selector_should_not_raise_exception(): + standalone_test_node = DbtNode( + unique_id=f"{DbtResourceType.TEST.value}.{SAMPLE_PROJ_PATH.stem}.standalone", + resource_type=DbtResourceType.TEST, + depends_on=[], + tags=[], + config={}, + file_path=SAMPLE_PROJ_PATH / "tests/generic/builtin.sql", + ) + nodes = {standalone_test_node.unique_id: standalone_test_node} + assert not select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=nodes, select=["tag:some-tag"]) + + +def test_should_include_node_without_depends_on(selector_config): + node = DbtNode( + unique_id=f"{DbtResourceType.TEST.value}.{SAMPLE_PROJ_PATH.stem}.standalone", + resource_type=DbtResourceType.TEST, + depends_on=None, + tags=[], + config={}, + file_path=SAMPLE_PROJ_PATH / "tests/generic/builtin.sql", + ) + selector = NodeSelector({}, selector_config) + selector.visited_nodes = set() + selector._should_include_node(node.unique_id, node) diff --git a/tests/hooks/__init__.py b/tests/hooks/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/hooks/test_subprocess.py b/tests/hooks/test_subprocess.py new file mode 100644 index 0000000000..e8b16d387b --- /dev/null +++ b/tests/hooks/test_subprocess.py @@ -0,0 +1,70 @@ +import signal +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import MagicMock, patch + +import pytest + +from cosmos.hooks.subprocess import FullOutputSubprocessHook + +OS_ENV_KEY = "SUBPROCESS_ENV_TEST" +OS_ENV_VAL = "this-is-from-os-environ" + + +@pytest.mark.parametrize( + "env,expected", + [ + ({"ABC": "123", "AAA": "456"}, {"ABC": "123", "AAA": "456", OS_ENV_KEY: ""}), + ({}, {OS_ENV_KEY: ""}), + (None, {OS_ENV_KEY: OS_ENV_VAL}), + ], + ids=["with env", "empty env", "no env"], +) +def test_env(env, expected): + """ + Test that env variables are exported correctly to the command environment. + When ``env`` is ``None``, ``os.environ`` should be passed to ``Popen``. + Otherwise, the variables in ``env`` should be available, and ``os.environ`` should not. + """ + hook = FullOutputSubprocessHook() + + def build_cmd(keys, filename): + """ + Produce bash command to echo env vars into filename. + Will always echo the special test var named ``OS_ENV_KEY`` into the file to test whether + ``os.environ`` is passed or not. + """ + return "\n".join(f"echo {k}=${k}>> {filename}" for k in [*keys, OS_ENV_KEY]) + + with TemporaryDirectory() as tmp_dir, patch.dict("os.environ", {OS_ENV_KEY: OS_ENV_VAL}): + tmp_file = Path(tmp_dir, "test.txt") + command = build_cmd(env and env.keys() or [], tmp_file.as_posix()) + hook.run_command(command=["bash", "-c", command], env=env) + actual = dict([x.split("=") for x in tmp_file.read_text().splitlines()]) + assert actual == expected + + +def test_subprocess_hook(): + hook = FullOutputSubprocessHook() + result = hook.run_command(command=["bash", "-c", f'echo "foo"']) + assert result.exit_code == 0 + assert result.output == "foo" + assert result.full_output == ["foo"] + + +@patch("os.getpgid", return_value=123) +@patch("os.killpg") +def test_send_sigint(mock_killpg, mock_getpgid): + hook = FullOutputSubprocessHook() + hook.sub_process = MagicMock() + hook.send_sigint() + mock_killpg.assert_called_with(123, signal.SIGINT) + + +@patch("os.getpgid", return_value=123) +@patch("os.killpg") +def test_send_sigterm(mock_killpg, mock_getpgid): + hook = FullOutputSubprocessHook() + hook.sub_process = MagicMock() + hook.send_sigterm() + mock_killpg.assert_called_with(123, signal.SIGTERM) diff --git a/tests/operators/test_aws_eks.py b/tests/operators/test_aws_eks.py new file mode 100644 index 0000000000..35717a0617 --- /dev/null +++ b/tests/operators/test_aws_eks.py @@ -0,0 +1,97 @@ +from unittest.mock import MagicMock, patch + +import pytest +from airflow.exceptions import AirflowException + +from cosmos.operators.aws_eks import ( + DbtBuildAwsEksOperator, + DbtLSAwsEksOperator, + DbtRunAwsEksOperator, + DbtSeedAwsEksOperator, + DbtTestAwsEksOperator, +) + + +@pytest.fixture() +def mock_kubernetes_execute(): + with patch("cosmos.operators.kubernetes.KubernetesPodOperator.execute") as mock_execute: + yield mock_execute + + +base_kwargs = { + "conn_id": "my_airflow_connection", + "cluster_name": "my-cluster", + "task_id": "my-task", + "image": "my_image", + "project_dir": "my/dir", + "vars": { + "start_time": "{{ data_interval_start.strftime('%Y%m%d%H%M%S') }}", + "end_time": "{{ data_interval_end.strftime('%Y%m%d%H%M%S') }}", + }, + "no_version_check": True, +} + + +def test_dbt_kubernetes_build_command(): + """ + Since we know that the KubernetesOperator is tested, we can just test that the + command is built correctly and added to the "arguments" parameter. + """ + + result_map = { + "ls": DbtLSAwsEksOperator(**base_kwargs), + "run": DbtRunAwsEksOperator(**base_kwargs), + "test": DbtTestAwsEksOperator(**base_kwargs), + "build": DbtBuildAwsEksOperator(**base_kwargs), + "seed": DbtSeedAwsEksOperator(**base_kwargs), + } + + for command_name, command_operator in result_map.items(): + command_operator.build_kube_args(context=MagicMock(), cmd_flags=MagicMock()) + assert command_operator.arguments == [ + "dbt", + command_name, + "--vars", + "end_time: '{{ data_interval_end.strftime(''%Y%m%d%H%M%S'') }}'\n" + "start_time: '{{ data_interval_start.strftime(''%Y%m%d%H%M%S'') }}'\n", + "--no-version-check", + "--project-dir", + "my/dir", + ] + + +@patch("cosmos.operators.kubernetes.DbtKubernetesBaseOperator.build_kube_args") +@patch("cosmos.operators.aws_eks.EksHook.generate_config_file") +def test_dbt_kubernetes_operator_execute(mock_generate_config_file, mock_build_kube_args, mock_kubernetes_execute): + """Tests that the execute method call results in both the build_kube_args method and the kubernetes execute method being called.""" + operator = DbtLSAwsEksOperator( + conn_id="my_airflow_connection", + cluster_name="my-cluster", + task_id="my-task", + image="my_image", + project_dir="my/dir", + ) + operator.execute(context={}) + # Assert that the build_kube_args method was called in the execution + mock_build_kube_args.assert_called_once() + + # Assert that the generate_config_file method was called in the execution to create the kubeconfig for eks + mock_generate_config_file.assert_called_once_with(eks_cluster_name="my-cluster", pod_namespace="default") + + # Assert that the kubernetes execute method was called in the execution + mock_kubernetes_execute.assert_called_once() + assert mock_kubernetes_execute.call_args.args[-1] == {} + + +def test_provided_config_file_fails(): + """Tests that the constructor fails if it is called with a config_file.""" + with pytest.raises(AirflowException) as err_context: + DbtLSAwsEksOperator( + conn_id="my_airflow_connection", + cluster_name="my-cluster", + task_id="my-task", + image="my_image", + project_dir="my/dir", + config_file="my/config", + ) + assert "The config_file is not an allowed parameter for the EksPodOperator." in str(err_context.value) diff --git a/tests/operators/test_azure_container_instance.py b/tests/operators/test_azure_container_instance.py new file mode 100644 index 0000000000..836ca45961 --- /dev/null +++ b/tests/operators/test_azure_container_instance.py @@ -0,0 +1,167 @@ +from pathlib import Path +from unittest.mock import MagicMock, patch + +from airflow.utils.context import Context +from pendulum import datetime + +from cosmos.operators.azure_container_instance import ( + DbtAzureContainerInstanceBaseOperator, + DbtLSAzureContainerInstanceOperator, + DbtRunAzureContainerInstanceOperator, + DbtSeedAzureContainerInstanceOperator, + DbtTestAzureContainerInstanceOperator, +) + + +class ConcreteDbtAzureContainerInstanceOperator(DbtAzureContainerInstanceBaseOperator): + base_cmd = ["cmd"] + + +def test_dbt_azure_container_instance_operator_add_global_flags() -> None: + dbt_base_operator = ConcreteDbtAzureContainerInstanceOperator( + ci_conn_id="my_airflow_connection", + task_id="my-task", + image="my_image", + region="Mordor", + name="my-aci", + resource_group="my-rg", + project_dir="my/dir", + vars={ + "start_time": "{{ data_interval_start.strftime('%Y%m%d%H%M%S') }}", + "end_time": "{{ data_interval_end.strftime('%Y%m%d%H%M%S') }}", + }, + no_version_check=True, + ) + assert dbt_base_operator.add_global_flags() == [ + "--vars", + "end_time: '{{ data_interval_end.strftime(''%Y%m%d%H%M%S'') }}'\n" + "start_time: '{{ data_interval_start.strftime(''%Y%m%d%H%M%S'') }}'\n", + "--no-version-check", + ] + + +@patch("cosmos.operators.base.context_to_airflow_vars") +def test_dbt_azure_container_instance_operator_get_env(p_context_to_airflow_vars: MagicMock) -> None: + """ + If an end user passes in a variable via the context that is also a global flag, validate that the both are kept + """ + dbt_base_operator = ConcreteDbtAzureContainerInstanceOperator( + ci_conn_id="my_airflow_connection", + task_id="my-task", + image="my_image", + region="Mordor", + name="my-aci", + resource_group="my-rg", + project_dir="my/dir", + ) + dbt_base_operator.env = { + "start_date": "20220101", + "end_date": "20220102", + "some_path": Path(__file__), + "retries": 3, + ("tuple", "key"): "some_value", + } + p_context_to_airflow_vars.return_value = {"START_DATE": "2023-02-15 12:30:00"} + env = dbt_base_operator.get_env( + Context(execution_date=datetime(2023, 2, 15, 12, 30)), + ) + expected_env = { + "start_date": "20220101", + "end_date": "20220102", + "some_path": Path(__file__), + "START_DATE": "2023-02-15 12:30:00", + } + assert env == expected_env + + +@patch("cosmos.operators.base.context_to_airflow_vars") +def test_dbt_azure_container_instance_operator_check_environment_variables( + p_context_to_airflow_vars: MagicMock, +) -> None: + """ + If an end user passes in a variable via the context that is also a global flag, validate that the both are kept + """ + dbt_base_operator = ConcreteDbtAzureContainerInstanceOperator( + ci_conn_id="my_airflow_connection", + task_id="my-task", + image="my_image", + region="Mordor", + name="my-aci", + resource_group="my-rg", + project_dir="my/dir", + environment_variables={"FOO": "BAR"}, + ) + dbt_base_operator.env = { + "start_date": "20220101", + "end_date": "20220102", + "some_path": Path(__file__), + "retries": 3, + "FOO": "foo", + ("tuple", "key"): "some_value", + } + expected_env = {"start_date": "20220101", "end_date": "20220102", "some_path": Path(__file__), "FOO": "BAR"} + dbt_base_operator.build_command(context=MagicMock()) + + assert dbt_base_operator.environment_variables == expected_env + + +base_kwargs = { + "ci_conn_id": "my_airflow_connection", + "name": "my-aci", + "region": "Mordor", + "resource_group": "my-rg", + "task_id": "my-task", + "image": "my_image", + "project_dir": "my/dir", + "vars": { + "start_time": "{{ data_interval_start.strftime('%Y%m%d%H%M%S') }}", + "end_time": "{{ data_interval_end.strftime('%Y%m%d%H%M%S') }}", + }, + "no_version_check": True, +} + +result_map = { + "ls": DbtLSAzureContainerInstanceOperator(**base_kwargs), + "run": DbtRunAzureContainerInstanceOperator(**base_kwargs), + "test": DbtTestAzureContainerInstanceOperator(**base_kwargs), + "seed": DbtSeedAzureContainerInstanceOperator(**base_kwargs), +} + + +def test_dbt_azure_container_instance_build_command(): + """ + Since we know that the AzureContainerInstanceOperator is tested, we can just test that the + command is built correctly. + """ + for command_name, command_operator in result_map.items(): + command_operator.build_command(context=MagicMock(), cmd_flags=MagicMock()) + assert command_operator.command == [ + "dbt", + command_name, + "--vars", + "end_time: '{{ data_interval_end.strftime(''%Y%m%d%H%M%S'') }}'\n" + "start_time: '{{ data_interval_start.strftime(''%Y%m%d%H%M%S'') }}'\n", + "--no-version-check", + ] + + +@patch("cosmos.operators.azure_container_instance.AzureContainerInstancesOperator.execute") +def test_dbt_azure_container_instance_build_and_run_cmd(mock_execute): + dbt_base_operator = ConcreteDbtAzureContainerInstanceOperator( + ci_conn_id="my_airflow_connection", + task_id="my-task", + image="my_image", + region="Mordor", + name="my-aci", + resource_group="my-rg", + project_dir="my/dir", + environment_variables={"FOO": "BAR"}, + ) + mock_build_command = MagicMock() + dbt_base_operator.build_command = mock_build_command + + mock_context = MagicMock() + dbt_base_operator.build_and_run_cmd(context=mock_context) + + mock_build_command.assert_called_with(mock_context, None) + mock_execute.assert_called_once_with(dbt_base_operator, mock_context) diff --git a/tests/operators/test_base.py b/tests/operators/test_base.py new file mode 100644 index 0000000000..6f44252820 --- /dev/null +++ b/tests/operators/test_base.py @@ -0,0 +1,175 @@ +import sys +from datetime import datetime +from unittest.mock import patch + +import pytest +from airflow.utils.context import Context + +from cosmos.operators.base import ( + AbstractDbtBaseOperator, + DbtBuildMixin, + DbtLSMixin, + DbtRunMixin, + DbtRunOperationMixin, + DbtSeedMixin, + DbtSnapshotMixin, + DbtTestMixin, +) + + +@pytest.mark.skipif( + (sys.version_info.major, sys.version_info.minor) == (3, 12), + reason="The error message for the abstract class instantiation seems to have changed between Python 3.11 and 3.12", +) +def test_dbt_base_operator_is_abstract(): + """Tests that the abstract base operator cannot be instantiated since the base_cmd is not defined.""" + expected_error = ( + "Can't instantiate abstract class AbstractDbtBaseOperator with abstract methods base_cmd, build_and_run_cmd" + ) + with pytest.raises(TypeError, match=expected_error): + AbstractDbtBaseOperator() + + +@pytest.mark.skipif( + (sys.version_info.major, sys.version_info.minor) != (3, 12), + reason="The error message for the abstract class instantiation seems to have changed between Python 3.11 and 3.12", +) +def test_dbt_base_operator_is_abstract_py12(): + """Tests that the abstract base operator cannot be instantiated since the base_cmd is not defined.""" + expected_error = ( + "Can't instantiate abstract class AbstractDbtBaseOperator without an implementation for abstract methods " + "'base_cmd', 'build_and_run_cmd'" + ) + with pytest.raises(TypeError, match=expected_error): + AbstractDbtBaseOperator() + + +@pytest.mark.parametrize("cmd_flags", [["--some-flag"], []]) +@patch("cosmos.operators.base.AbstractDbtBaseOperator.build_and_run_cmd") +def test_dbt_base_operator_execute(mock_build_and_run_cmd, cmd_flags, monkeypatch): + """Tests that the base operator execute method calls the build_and_run_cmd method with the expected arguments.""" + monkeypatch.setattr(AbstractDbtBaseOperator, "add_cmd_flags", lambda _: cmd_flags) + AbstractDbtBaseOperator.__abstractmethods__ = set() + + base_operator = AbstractDbtBaseOperator(task_id="fake_task", project_dir="fake_dir") + + base_operator.execute(context={}) + mock_build_and_run_cmd.assert_called_once_with(context={}, cmd_flags=cmd_flags) + + +@patch("cosmos.operators.base.context_merge") +def test_dbt_base_operator_context_merge_called(mock_context_merge): + """Tests that the base operator execute method calls the context_merge method with the expected arguments.""" + base_operator = AbstractDbtBaseOperator( + task_id="fake_task", + project_dir="fake_dir", + extra_context={"extra": "extra"}, + ) + + base_operator.execute(context={}) + mock_context_merge.assert_called_once_with({}, {"extra": "extra"}) + + +@pytest.mark.parametrize( + "context, extra_context, expected_context", + [ + ( + Context( + start_date=datetime(2021, 1, 1), + ), + { + "extra": "extra", + }, + Context( + start_date=datetime(2021, 1, 1), + extra="extra", + ), + ), + ( + Context( + start_date=datetime(2021, 1, 1), + end_date=datetime(2023, 1, 1), + ), + { + "extra": "extra", + "extra_2": "extra_2", + }, + Context( + start_date=datetime(2021, 1, 1), + end_date=datetime(2023, 1, 1), + extra="extra", + extra_2="extra_2", + ), + ), + ( + Context( + overwrite="to_overwrite", + start_date=datetime(2021, 1, 1), + end_date=datetime(2023, 1, 1), + ), + { + "overwrite": "overwritten", + }, + Context( + start_date=datetime(2021, 1, 1), + end_date=datetime(2023, 1, 1), + overwrite="overwritten", + ), + ), + ], +) +def test_dbt_base_operator_context_merge( + context, + extra_context, + expected_context, +): + """Tests that the base operator execute method calls and update context""" + base_operator = AbstractDbtBaseOperator( + task_id="fake_task", + project_dir="fake_dir", + extra_context=extra_context, + ) + + base_operator.execute(context=context) + assert context == expected_context + + +@pytest.mark.parametrize( + "dbt_command, dbt_operator_class", + [ + ("test", DbtTestMixin), + ("snapshot", DbtSnapshotMixin), + ("ls", DbtLSMixin), + ("seed", DbtSeedMixin), + ("run", DbtRunMixin), + ("build", DbtBuildMixin), + ], +) +def test_dbt_mixin_base_cmd(dbt_command, dbt_operator_class): + assert [dbt_command] == dbt_operator_class.base_cmd + + +@pytest.mark.parametrize("dbt_operator_class", [DbtSeedMixin, DbtRunMixin, DbtBuildMixin]) +@pytest.mark.parametrize( + "full_refresh, expected_flags", [("True", ["--full-refresh"]), (True, ["--full-refresh"]), (False, [])] +) +def test_dbt_mixin_add_cmd_flags_full_refresh(full_refresh, expected_flags, dbt_operator_class): + dbt_mixin = dbt_operator_class(full_refresh=full_refresh) + flags = dbt_mixin.add_cmd_flags() + assert flags == expected_flags + + +@pytest.mark.parametrize("args, expected_flags", [(None, []), ({"arg1": "val1"}, ["--args", "arg1: val1\n"])]) +def test_dbt_mixin_add_cmd_flags_run_operator(args, expected_flags): + macro_name = "some_macro" + run_operation = DbtRunOperationMixin(macro_name=macro_name, args=args) + assert run_operation.base_cmd == ["run-operation", "some_macro"] + + flags = run_operation.add_cmd_flags() + assert flags == expected_flags + + +def test_abstract_dbt_base_operator_append_env_is_false_by_default(): + """Tests that the append_env attribute is set to False by default.""" + base_operator = AbstractDbtBaseOperator(task_id="fake_task", project_dir="fake_dir") + assert base_operator.append_env is False diff --git a/tests/operators/test_docker.py b/tests/operators/test_docker.py index 234878e07a..2cfb6b835e 100644 --- a/tests/operators/test_docker.py +++ b/tests/operators/test_docker.py @@ -1,11 +1,12 @@ from pathlib import Path from unittest.mock import MagicMock, patch +import pytest from airflow.utils.context import Context from pendulum import datetime from cosmos.operators.docker import ( - DbtDockerBaseOperator, + DbtBuildDockerOperator, DbtLSDockerOperator, DbtRunDockerOperator, DbtSeedDockerOperator, @@ -13,8 +14,24 @@ ) -def test_dbt_docker_operator_add_global_flags() -> None: - dbt_base_operator = DbtDockerBaseOperator( +@pytest.fixture() +def mock_docker_execute(): + with patch("cosmos.operators.docker.DockerOperator.execute") as mock_execute: + yield mock_execute + + +@pytest.fixture() +def base_operator(mock_docker_execute): + from cosmos.operators.docker import DbtDockerBaseOperator + + class ConcreteDbtDockerBaseOperator(DbtDockerBaseOperator): + base_cmd = ["cmd"] + + return ConcreteDbtDockerBaseOperator + + +def test_dbt_docker_operator_add_global_flags(base_operator) -> None: + dbt_base_operator = base_operator( conn_id="my_airflow_connection", task_id="my-task", image="my_image", @@ -33,17 +50,31 @@ def test_dbt_docker_operator_add_global_flags() -> None: ] -@patch("cosmos.operators.base.context_to_airflow_vars") -def test_dbt_docker_operator_get_env(p_context_to_airflow_vars: MagicMock) -> None: - """ - If an end user passes in a - """ - dbt_base_operator = DbtDockerBaseOperator( +@patch("cosmos.operators.docker.DbtDockerBaseOperator.build_command") +def test_dbt_docker_operator_execute(mock_build_command, base_operator, mock_docker_execute): + """Tests that the execute method call results in both the build_command method and the docker execute method being called.""" + operator = base_operator( conn_id="my_airflow_connection", task_id="my-task", image="my_image", project_dir="my/dir", ) + operator.execute(context={}) + # Assert that the build_command method was called in the execution + mock_build_command.assert_called_once() + # Assert that the docker execute method was called in the execution + mock_docker_execute.assert_called_once() + assert mock_docker_execute.call_args.args[-1] == {} + + +@patch("cosmos.operators.base.context_to_airflow_vars") +def test_dbt_docker_operator_get_env(p_context_to_airflow_vars: MagicMock, base_operator) -> None: + """ + If an end user passes in a + """ + dbt_base_operator = base_operator( + conn_id="my_airflow_connection", task_id="my-task", image="my_image", project_dir="my/dir", append_env=False + ) dbt_base_operator.env = { "start_date": "20220101", "end_date": "20220102", @@ -80,6 +111,7 @@ def test_dbt_docker_operator_get_env(p_context_to_airflow_vars: MagicMock) -> No "ls": DbtLSDockerOperator(**base_kwargs), "run": DbtRunDockerOperator(**base_kwargs), "test": DbtTestDockerOperator(**base_kwargs), + "build": DbtBuildDockerOperator(**base_kwargs), "seed": DbtSeedDockerOperator(**base_kwargs), } diff --git a/tests/operators/test_kubernetes.py b/tests/operators/test_kubernetes.py index 7ef606cfe4..d0be2acad9 100644 --- a/tests/operators/test_kubernetes.py +++ b/tests/operators/test_kubernetes.py @@ -1,20 +1,45 @@ from pathlib import Path from unittest.mock import MagicMock, patch -from airflow.utils.context import Context +import pytest +from airflow.models import TaskInstance +from airflow.utils.context import Context, context_merge from pendulum import datetime from cosmos.operators.kubernetes import ( - DbtKubernetesBaseOperator, + DbtBuildKubernetesOperator, DbtLSKubernetesOperator, DbtRunKubernetesOperator, DbtSeedKubernetesOperator, DbtTestKubernetesOperator, ) +try: + from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction -def test_dbt_kubernetes_operator_add_global_flags() -> None: - dbt_kube_operator = DbtKubernetesBaseOperator( + module_available = True +except ImportError: + module_available = False + + +@pytest.fixture() +def mock_kubernetes_execute(): + with patch("cosmos.operators.kubernetes.KubernetesPodOperator.execute") as mock_execute: + yield mock_execute + + +@pytest.fixture() +def base_operator(mock_kubernetes_execute): + from cosmos.operators.kubernetes import DbtKubernetesBaseOperator + + class ConcreteDbtKubernetesBaseOperator(DbtKubernetesBaseOperator): + base_cmd = ["cmd"] + + return ConcreteDbtKubernetesBaseOperator + + +def test_dbt_kubernetes_operator_add_global_flags(base_operator) -> None: + dbt_kube_operator = base_operator( conn_id="my_airflow_connection", task_id="my-task", image="my_image", @@ -33,17 +58,31 @@ def test_dbt_kubernetes_operator_add_global_flags() -> None: ] -@patch("cosmos.operators.base.context_to_airflow_vars") -def test_dbt_kubernetes_operator_get_env(p_context_to_airflow_vars: MagicMock) -> None: - """ - If an end user passes in a - """ - dbt_kube_operator = DbtKubernetesBaseOperator( +@patch("cosmos.operators.kubernetes.DbtKubernetesBaseOperator.build_kube_args") +def test_dbt_kubernetes_operator_execute(mock_build_kube_args, base_operator, mock_kubernetes_execute): + """Tests that the execute method call results in both the build_kube_args method and the kubernetes execute method being called.""" + operator = base_operator( conn_id="my_airflow_connection", task_id="my-task", image="my_image", project_dir="my/dir", ) + operator.execute(context={}) + # Assert that the build_kube_args method was called in the execution + mock_build_kube_args.assert_called_once() + # Assert that the kubernetes execute method was called in the execution + mock_kubernetes_execute.assert_called_once() + assert mock_kubernetes_execute.call_args.args[-1] == {} + + +@patch("cosmos.operators.base.context_to_airflow_vars") +def test_dbt_kubernetes_operator_get_env(p_context_to_airflow_vars: MagicMock, base_operator) -> None: + """ + If an end user passes in a + """ + dbt_kube_operator = base_operator( + conn_id="my_airflow_connection", task_id="my-task", image="my_image", project_dir="my/dir", append_env=False + ) dbt_kube_operator.env = { "start_date": "20220101", "end_date": "20220102", @@ -80,6 +119,7 @@ def test_dbt_kubernetes_operator_get_env(p_context_to_airflow_vars: MagicMock) - "ls": DbtLSKubernetesOperator(**base_kwargs), "run": DbtRunKubernetesOperator(**base_kwargs), "test": DbtTestKubernetesOperator(**base_kwargs), + "build": DbtBuildKubernetesOperator(**base_kwargs), "seed": DbtSeedKubernetesOperator(**base_kwargs), } @@ -103,118 +143,145 @@ def test_dbt_kubernetes_build_command(): ] -@patch("airflow.providers.cncf.kubernetes.operators.pod.KubernetesPodOperator.hook") -def test_created_pod(test_hook): - test_hook.is_in_cluster = False - test_hook._get_namespace.return_value.to_dict.return_value = "foo" - ls_kwargs = {"env_vars": {"FOO": "BAR"}} +@pytest.mark.parametrize( + "additional_kwargs,expected_results", + [ + ({"on_success_callback": None, "is_delete_operator_pod": True}, (1, 1, True, "delete_pod")), + ( + {"on_success_callback": (lambda **kwargs: None), "is_delete_operator_pod": False}, + (2, 1, False, "keep_pod"), + ), + ( + {"on_success_callback": [(lambda **kwargs: None), (lambda **kwargs: None)], "is_delete_operator_pod": None}, + (3, 1, True, "delete_pod"), + ), + ( + {"on_failure_callback": None, "is_delete_operator_pod": True, "on_finish_action": "keep_pod"}, + (1, 1, True, "delete_pod"), + ), + ( + { + "on_failure_callback": (lambda **kwargs: None), + "is_delete_operator_pod": None, + "on_finish_action": "delete_pod", + }, + (1, 2, True, "delete_pod"), + ), + ( + { + "on_failure_callback": [(lambda **kwargs: None), (lambda **kwargs: None)], + "is_delete_operator_pod": None, + "on_finish_action": "delete_succeeded_pod", + }, + (1, 3, False, "delete_succeeded_pod"), + ), + ({"is_delete_operator_pod": None, "on_finish_action": "keep_pod"}, (1, 1, False, "keep_pod")), + ({}, (1, 1, True, "delete_pod")), + ], +) +@pytest.mark.skipif( + not module_available, reason="Kubernetes module `airflow.providers.cncf.kubernetes.utils.pod_manager` not available" +) +def test_dbt_test_kubernetes_operator_constructor(additional_kwargs, expected_results): + test_operator = DbtTestKubernetesOperator( + on_warning_callback=(lambda **kwargs: None), **additional_kwargs, **base_kwargs + ) + + print(additional_kwargs, test_operator.__dict__) + + assert isinstance(test_operator.on_success_callback, list) + assert isinstance(test_operator.on_failure_callback, list) + assert test_operator._handle_warnings in test_operator.on_success_callback + assert test_operator._cleanup_pod in test_operator.on_failure_callback + assert len(test_operator.on_success_callback) == expected_results[0] + assert len(test_operator.on_failure_callback) == expected_results[1] + assert test_operator.is_delete_operator_pod_original == expected_results[2] + assert test_operator.on_finish_action_original == OnFinishAction(expected_results[3]) + + +class FakePodManager: + def read_pod_logs(self, pod, container): + assert pod == "pod" + assert container == "base" + log_string = """ +19:48:25 Concurrency: 4 threads (target='target') +19:48:25 +19:48:25 1 of 2 START test dbt_utils_accepted_range_table_col__12__0 ................... [RUN] +19:48:25 2 of 2 START test unique_table__uuid .......................................... [RUN] +19:48:27 1 of 2 WARN 252 dbt_utils_accepted_range_table_col__12__0 ..................... [WARN 117 in 1.83s] +19:48:27 2 of 2 PASS unique_table__uuid ................................................ [PASS in 1.85s] +19:48:27 +19:48:27 Finished running 2 tests, 1 hook in 0 hours 0 minutes and 12.86 seconds (12.86s). +19:48:27 +19:48:27 Completed with 1 warning: +19:48:27 +19:48:27 Warning in test dbt_utils_accepted_range_table_col__12__0 (models/ads/ads.yaml) +19:48:27 Got 252 results, configured to warn if >0 +19:48:27 +19:48:27 compiled Code at target/compiled/model/models/table/table.yaml/dbt_utils_accepted_range_table_col__12__0.sql +19:48:27 +19:48:27 Done. PASS=1 WARN=1 ERROR=0 SKIP=0 TOTAL=2 +""" + return (log.encode("utf-8") for log in log_string.split("\n")) + + +@pytest.mark.skipif( + not module_available, reason="Kubernetes module `airflow.providers.cncf.kubernetes.utils.pod_manager` not available" +) +def test_dbt_test_kubernetes_operator_handle_warnings_and_cleanup_pod(): + def on_warning_callback(context: Context): + assert context["test_names"] == ["dbt_utils_accepted_range_table_col__12__0"] + assert context["test_results"] == ["Got 252 results, configured to warn if >0"] + + def cleanup(pod: str, remote_pod: str): + assert pod == remote_pod + + test_operator = DbtTestKubernetesOperator( + is_delete_operator_pod=True, on_warning_callback=on_warning_callback, **base_kwargs + ) + task_instance = TaskInstance(test_operator) + task_instance.task.pod_manager = FakePodManager() + task_instance.task.pod = task_instance.task.remote_pod = "pod" + task_instance.task.cleanup = cleanup + + context = Context() + context_merge(context, task_instance=task_instance) + + test_operator._handle_warnings(context) + + +def test_created_pod(): + ls_kwargs = {"env_vars": {"FOO": "BAR"}, "namespace": "foo", "append_env": False} ls_kwargs.update(base_kwargs) ls_operator = DbtLSKubernetesOperator(**ls_kwargs) + ls_operator.hook = MagicMock() + ls_operator.hook.is_in_cluster = False ls_operator.build_kube_args(context={}, cmd_flags=MagicMock()) pod_obj = ls_operator.build_pod_request_obj() - expected_result = { - "api_version": "v1", - "kind": "Pod", - "metadata": { - "annotations": {}, - "cluster_name": None, - "creation_timestamp": None, - "deletion_grace_period_seconds": None, - "deletion_timestamp": None, - "finalizers": None, - "generate_name": None, - "generation": None, - "labels": { - "airflow_kpo_in_cluster": "False", - "airflow_version": pod_obj.metadata.labels["airflow_version"], - }, - "managed_fields": None, - "name": pod_obj.metadata.name, - "owner_references": None, - "resource_version": None, - "self_link": None, - "uid": None, - }, - "spec": { - "active_deadline_seconds": None, - "affinity": {}, - "automount_service_account_token": None, - "containers": [ - { - "args": [ - "dbt", - "ls", - "--vars", - "end_time: '{{ " - "data_interval_end.strftime(''%Y%m%d%H%M%S'') " - "}}'\n" - "start_time: '{{ " - "data_interval_start.strftime(''%Y%m%d%H%M%S'') " - "}}'\n", - "--no-version-check", - "--project-dir", - "my/dir", - ], - "command": [], - "env": [{"name": "FOO", "value": "BAR", "value_from": None}], - "env_from": [], - "image": "my_image", - "image_pull_policy": None, - "lifecycle": None, - "liveness_probe": None, - "name": "base", - "ports": [], - "readiness_probe": None, - "resources": None, - "security_context": None, - "startup_probe": None, - "stdin": None, - "stdin_once": None, - "termination_message_path": None, - # "termination_message_policy": None, - "tty": None, - "volume_devices": None, - "volume_mounts": [], - "working_dir": None, - } - ], - "dns_config": None, - "dns_policy": None, - "enable_service_links": None, - "ephemeral_containers": None, - "host_aliases": None, - "host_ipc": None, - "host_network": False, - "host_pid": None, - "hostname": None, - "image_pull_secrets": [], - "init_containers": [], - "node_name": None, - "node_selector": {}, - "os": None, - "overhead": None, - "preemption_policy": None, - "priority": None, - "priority_class_name": None, - "readiness_gates": None, - "restart_policy": "Never", - "runtime_class_name": None, - "scheduler_name": None, - "security_context": {}, - "service_account": None, - "service_account_name": None, - "set_hostname_as_fqdn": None, - "share_process_namespace": None, - "subdomain": None, - "termination_grace_period_seconds": None, - "tolerations": [], - "topology_spread_constraints": None, - "volumes": [], - }, - "status": None, - } - computed_result = pod_obj.to_dict() - computed_result["spec"]["containers"][0].pop("termination_message_policy") - computed_result["metadata"].pop("namespace") - assert computed_result == expected_result + metadata = pod_obj.metadata + assert metadata.labels["airflow_kpo_in_cluster"] == "False" + assert metadata.namespace == "foo" + + container = pod_obj.spec.containers[0] + assert container.env[0].name == "FOO" + assert container.env[0].value == "BAR" + assert container.env[0].value_from is None + assert container.image == "my_image" + + expected_container_args = [ + "dbt", + "ls", + "--vars", + "end_time: '{{ " + "data_interval_end.strftime(''%Y%m%d%H%M%S'') " + "}}'\n" + "start_time: '{{ " + "data_interval_start.strftime(''%Y%m%d%H%M%S'') " + "}}'\n", + "--no-version-check", + "--project-dir", + "my/dir", + ] + assert container.args == expected_container_args + assert container.command == [] diff --git a/tests/operators/test_local.py b/tests/operators/test_local.py index 580d49e6ce..ed46caf887 100644 --- a/tests/operators/test_local.py +++ b/tests/operators/test_local.py @@ -1,10 +1,10 @@ import logging import os -import sys import shutil +import sys import tempfile from pathlib import Path -from unittest.mock import MagicMock, patch, call +from unittest.mock import MagicMock, call, patch import pytest from airflow import DAG @@ -15,24 +15,32 @@ from packaging import version from pendulum import datetime +from cosmos import cache from cosmos.config import ProfileConfig +from cosmos.constants import PARTIALLY_SUPPORTED_AIRFLOW_VERSIONS, InvocationMode +from cosmos.dbt.parser.output import ( + extract_dbt_runner_issues, + parse_number_of_warnings_dbt_runner, + parse_number_of_warnings_subprocess, +) +from cosmos.hooks.subprocess import FullOutputSubprocessResult from cosmos.operators.local import ( + DbtBuildLocalOperator, + DbtDocsAzureStorageLocalOperator, + DbtDocsGCSLocalOperator, + DbtDocsLocalOperator, + DbtDocsS3LocalOperator, DbtLocalBaseOperator, DbtLSLocalOperator, - DbtSnapshotLocalOperator, DbtRunLocalOperator, - DbtTestLocalOperator, - DbtDocsLocalOperator, - DbtDocsS3LocalOperator, - DbtDocsAzureStorageLocalOperator, - DbtDocsGCSLocalOperator, - DbtSeedLocalOperator, DbtRunOperationLocalOperator, + DbtSeedLocalOperator, + DbtSnapshotLocalOperator, + DbtTestLocalOperator, ) from cosmos.profiles import PostgresUserPasswordProfileMapping from tests.utils import test_dag as run_test_dag - DBT_PROJ_DIR = Path(__file__).parent.parent.parent / "dev/dags/dbt/jaffle_shop" MINI_DBT_PROJ_DIR = Path(__file__).parent.parent / "sample/mini" MINI_DBT_PROJ_DIR_FAILING_SCHEMA = MINI_DBT_PROJ_DIR / "schema_failing_test.yml" @@ -48,7 +56,7 @@ profile_name="default", target_name="dev", profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="airflow_db", + conn_id="example_conn", profile_args={"schema": "public"}, ), ) @@ -68,8 +76,19 @@ def failing_test_dbt_project(tmp_path): tmp_dir.cleanup() +class ConcreteDbtLocalBaseOperator(DbtLocalBaseOperator): + base_cmd = ["cmd"] + + +def test_install_deps_in_empty_dir_becomes_false(tmpdir): + dbt_base_operator = ConcreteDbtLocalBaseOperator( + profile_config=profile_config, task_id="my-task", project_dir=tmpdir, install_deps=True + ) + assert not dbt_base_operator.install_deps + + def test_dbt_base_operator_add_global_flags() -> None: - dbt_base_operator = DbtLocalBaseOperator( + dbt_base_operator = ConcreteDbtLocalBaseOperator( profile_config=profile_config, task_id="my-task", project_dir="my/dir", @@ -78,8 +97,11 @@ def test_dbt_base_operator_add_global_flags() -> None: "end_time": "{{ data_interval_end.strftime('%Y%m%d%H%M%S') }}", }, no_version_check=True, + select=["my_first_model", "my_second_model"], ) assert dbt_base_operator.add_global_flags() == [ + "--select", + "my_first_model my_second_model", "--vars", "end_time: '{{ data_interval_end.strftime(''%Y%m%d%H%M%S'') }}'\n" "start_time: '{{ data_interval_start.strftime(''%Y%m%d%H%M%S'') }}'\n", @@ -87,28 +109,42 @@ def test_dbt_base_operator_add_global_flags() -> None: ] +def test_dbt_local_operator_append_env_is_true_by_default() -> None: + dbt_local_operator = ConcreteDbtLocalBaseOperator( + profile_config=profile_config, + task_id="my-task", + project_dir="my/dir", + vars={ + "start_time": "{{ data_interval_start.strftime('%Y%m%d%H%M%S') }}", + "end_time": "{{ data_interval_end.strftime('%Y%m%d%H%M%S') }}", + }, + no_version_check=True, + select=["my_first_model", "my_second_model"], + ) + + assert dbt_local_operator.append_env == True + + def test_dbt_base_operator_add_user_supplied_flags() -> None: - dbt_base_operator = DbtLocalBaseOperator( + dbt_base_operator = ConcreteDbtLocalBaseOperator( profile_config=profile_config, task_id="my-task", project_dir="my/dir", - base_cmd=["run"], dbt_cmd_flags=["--full-refresh"], ) cmd, _ = dbt_base_operator.build_cmd( Context(execution_date=datetime(2023, 2, 15, 12, 30)), ) - assert cmd[-2] == "run" + assert cmd[-2] == "cmd" assert cmd[-1] == "--full-refresh" def test_dbt_base_operator_add_user_supplied_global_flags() -> None: - dbt_base_operator = DbtLocalBaseOperator( + dbt_base_operator = ConcreteDbtLocalBaseOperator( profile_config=profile_config, task_id="my-task", project_dir="my/dir", - base_cmd=["run"], dbt_cmd_global_flags=["--cache-selected-only"], ) @@ -116,7 +152,52 @@ def test_dbt_base_operator_add_user_supplied_global_flags() -> None: Context(execution_date=datetime(2023, 2, 15, 12, 30)), ) assert cmd[-2] == "--cache-selected-only" - assert cmd[-1] == "run" + assert cmd[-1] == "cmd" + + +@pytest.mark.parametrize( + "invocation_mode, invoke_dbt_method, handle_exception_method", + [ + (InvocationMode.SUBPROCESS, "run_subprocess", "handle_exception_subprocess"), + (InvocationMode.DBT_RUNNER, "run_dbt_runner", "handle_exception_dbt_runner"), + ], +) +def test_dbt_base_operator_set_invocation_methods(invocation_mode, invoke_dbt_method, handle_exception_method): + """Tests that the right methods are mapped to DbtLocalBaseOperator.invoke_dbt and + DbtLocalBaseOperator.handle_exception when a known invocation mode passed. + """ + dbt_base_operator = ConcreteDbtLocalBaseOperator( + profile_config=profile_config, task_id="my-task", project_dir="my/dir", invocation_mode=invocation_mode + ) + dbt_base_operator._set_invocation_methods() + assert dbt_base_operator.invoke_dbt.__name__ == invoke_dbt_method + assert dbt_base_operator.handle_exception.__name__ == handle_exception_method + + +@pytest.mark.parametrize( + "can_import_dbt, invoke_dbt_method, handle_exception_method", + [ + (False, "run_subprocess", "handle_exception_subprocess"), + (True, "run_dbt_runner", "handle_exception_dbt_runner"), + ], +) +def test_dbt_base_operator_discover_invocation_mode(can_import_dbt, invoke_dbt_method, handle_exception_method): + """Tests that the right methods are mapped to DbtLocalBaseOperator.invoke_dbt and + DbtLocalBaseOperator.handle_exception if dbt can be imported or not. + """ + dbt_base_operator = ConcreteDbtLocalBaseOperator( + profile_config=profile_config, task_id="my-task", project_dir="my/dir" + ) + with patch.dict(sys.modules, {"dbt.cli.main": MagicMock()} if can_import_dbt else {"dbt.cli.main": None}): + dbt_base_operator = ConcreteDbtLocalBaseOperator( + profile_config=profile_config, task_id="my-task", project_dir="my/dir" + ) + dbt_base_operator._discover_invocation_mode() + assert dbt_base_operator.invocation_mode == ( + InvocationMode.DBT_RUNNER if can_import_dbt else InvocationMode.SUBPROCESS + ) + assert dbt_base_operator.invoke_dbt.__name__ == invoke_dbt_method + assert dbt_base_operator.handle_exception.__name__ == handle_exception_method @pytest.mark.parametrize( @@ -124,11 +205,10 @@ def test_dbt_base_operator_add_user_supplied_global_flags() -> None: [None, "cautious", "buildable", "empty"], ) def test_dbt_base_operator_use_indirect_selection(indirect_selection_type) -> None: - dbt_base_operator = DbtLocalBaseOperator( + dbt_base_operator = ConcreteDbtLocalBaseOperator( profile_config=profile_config, task_id="my-task", project_dir="my/dir", - base_cmd=["run"], indirect_selection=indirect_selection_type, ) @@ -140,7 +220,70 @@ def test_dbt_base_operator_use_indirect_selection(indirect_selection_type) -> No assert cmd[-1] == indirect_selection_type else: assert cmd[0].endswith("dbt") - assert cmd[1] == "run" + assert cmd[1] == "cmd" + + +def test_dbt_base_operator_run_dbt_runner_cannot_import(): + """Tests that the right error message is raised if dbtRunner cannot be imported.""" + dbt_base_operator = ConcreteDbtLocalBaseOperator( + profile_config=profile_config, + task_id="my-task", + project_dir="my/dir", + invocation_mode=InvocationMode.DBT_RUNNER, + ) + expected_error_message = "Could not import dbt core. Ensure that dbt-core >= v1.5 is installed and available in the environment where the operator is running." + with patch.dict(sys.modules, {"dbt.cli.main": None}): + with pytest.raises(ImportError, match=expected_error_message): + dbt_base_operator.run_dbt_runner(command=["cmd"], env={}, cwd="some-project") + + +@patch("cosmos.dbt.project.os.environ") +@patch("cosmos.dbt.project.os.chdir") +def test_dbt_base_operator_run_dbt_runner(mock_chdir, mock_environ): + """Tests that dbtRunner.invoke() is called with the expected cli args, that the + cwd is changed to the expected directory, and env variables are set.""" + dbt_base_operator = ConcreteDbtLocalBaseOperator( + profile_config=profile_config, + task_id="my-task", + project_dir="my/dir", + invocation_mode=InvocationMode.DBT_RUNNER, + ) + full_dbt_cmd = ["dbt", "run", "some_model"] + env_vars = {"VAR1": "value1", "VAR2": "value2"} + + mock_dbt = MagicMock() + with patch.dict(sys.modules, {"dbt.cli.main": mock_dbt}): + dbt_base_operator.run_dbt_runner(command=full_dbt_cmd, env=env_vars, cwd="some-dir") + + mock_dbt_runner = mock_dbt.dbtRunner.return_value + expected_cli_args = ["run", "some_model"] + # Assert dbtRunner.invoke was called with the expected cli args + assert mock_dbt_runner.invoke.call_count == 1 + assert mock_dbt_runner.invoke.call_args[0][0] == expected_cli_args + # Assert cwd was changed to the expected directory + assert mock_chdir.call_count == 2 + assert mock_chdir.call_args_list[0][0][0] == "some-dir" + # Assert env variables were updated + assert mock_environ.update.call_count == 1 + assert mock_environ.update.call_args[0][0] == env_vars + + +@patch("cosmos.dbt.project.os.chdir") +def test_dbt_base_operator_run_dbt_runner_is_cached(mock_chdir): + """Tests that if run_dbt_runner is called multiple times a cached runner is used.""" + dbt_base_operator = ConcreteDbtLocalBaseOperator( + profile_config=profile_config, + task_id="my-task", + project_dir="my/dir", + invocation_mode=InvocationMode.DBT_RUNNER, + ) + mock_dbt = MagicMock() + with patch.dict(sys.modules, {"dbt.cli.main": mock_dbt}): + for _ in range(3): + dbt_base_operator.run_dbt_runner(command=["cmd"], env={}, cwd="some-project") + mock_dbt_runner = mock_dbt.dbtRunner + assert mock_dbt_runner.call_count == 1 + assert dbt_base_operator._dbt_runner is not None @pytest.mark.parametrize( @@ -156,17 +299,56 @@ def test_dbt_base_operator_use_indirect_selection(indirect_selection_type) -> No "No exception raised", ], ) -def test_dbt_base_operator_exception_handling(skip_exception, exception_code_returned, expected_exception) -> None: - dbt_base_operator = DbtLocalBaseOperator( +def test_dbt_base_operator_exception_handling_subprocess( + skip_exception, exception_code_returned, expected_exception +) -> None: + dbt_base_operator = ConcreteDbtLocalBaseOperator( profile_config=profile_config, task_id="my-task", project_dir="my/dir", + invocation_mode=InvocationMode.SUBPROCESS, ) if expected_exception: with pytest.raises(expected_exception): - dbt_base_operator.exception_handling(SubprocessResult(exception_code_returned, None)) + dbt_base_operator.handle_exception(SubprocessResult(exception_code_returned, None)) else: - dbt_base_operator.exception_handling(SubprocessResult(exception_code_returned, None)) + dbt_base_operator.handle_exception(SubprocessResult(exception_code_returned, None)) + + +def test_dbt_base_operator_handle_exception_dbt_runner_unhandled_error(): + """Tests that an AirflowException is raised if the dbtRunner result is not successful with an unhandled error.""" + operator = ConcreteDbtLocalBaseOperator( + profile_config=MagicMock(), + task_id="my-task", + project_dir="my/dir", + ) + result = MagicMock() + result.success = False + result.exception = "some exception" + expected_error_message = "dbt invocation did not complete with unhandled error: some exception" + + with pytest.raises(AirflowException, match=expected_error_message): + operator.handle_exception_dbt_runner(result) + + +@patch("cosmos.operators.local.extract_dbt_runner_issues", return_value=(["node1", "node2"], ["error1", "error2"])) +def test_dbt_base_operator_handle_exception_dbt_runner_handled_error(mock_extract_dbt_runner_issues): + """Tests that an AirflowException is raised if the dbtRunner result is not successful and with handled errors.""" + operator = ConcreteDbtLocalBaseOperator( + profile_config=MagicMock(), + task_id="my-task", + project_dir="my/dir", + ) + result = MagicMock() + result.success = False + result.exception = None + + expected_error_message = "dbt invocation completed with errors: node1: error1\nnode2: error2" + + with pytest.raises(AirflowException, match=expected_error_message): + operator.handle_exception_dbt_runner(result) + + mock_extract_dbt_runner_issues.assert_called_once() @patch("cosmos.operators.base.context_to_airflow_vars") @@ -174,10 +356,8 @@ def test_dbt_base_operator_get_env(p_context_to_airflow_vars: MagicMock) -> None """ If an end user passes in a """ - dbt_base_operator = DbtLocalBaseOperator( - profile_config=profile_config, - task_id="my-task", - project_dir="my/dir", + dbt_base_operator = ConcreteDbtLocalBaseOperator( + profile_config=profile_config, task_id="my-task", project_dir="my/dir", append_env=False ) dbt_base_operator.env = { "start_date": "20220101", @@ -199,15 +379,51 @@ def test_dbt_base_operator_get_env(p_context_to_airflow_vars: MagicMock) -> None assert env == expected_env +@patch("cosmos.operators.local.extract_log_issues") +def test_dbt_test_local_operator_invocation_mode_methods(mock_extract_log_issues): + # test subprocess invocation mode + operator = DbtTestLocalOperator( + profile_config=profile_config, + invocation_mode=InvocationMode.SUBPROCESS, + task_id="my-task", + project_dir="my/dir", + ) + operator._set_test_result_parsing_methods() + assert operator.parse_number_of_warnings == parse_number_of_warnings_subprocess + result = MagicMock(full_output="some output") + operator.extract_issues(result) + mock_extract_log_issues.assert_called_once_with("some output") + + # test dbt runner invocation mode + operator = DbtTestLocalOperator( + profile_config=profile_config, + invocation_mode=InvocationMode.DBT_RUNNER, + task_id="my-task", + project_dir="my/dir", + ) + operator._set_test_result_parsing_methods() + assert operator.extract_issues == extract_dbt_runner_issues + assert operator.parse_number_of_warnings == parse_number_of_warnings_dbt_runner + + @pytest.mark.skipif( - version.parse(airflow_version) < version.parse("2.4"), - reason="Airflow DAG did not have datasets until the 2.4 release", + version.parse(airflow_version) < version.parse("2.4") + or version.parse(airflow_version) in PARTIALLY_SUPPORTED_AIRFLOW_VERSIONS, + reason="Airflow DAG did not have datasets until the 2.4 release, inlets and outlets do not work by default in Airflow 2.9.0 and 2.9.1", ) @pytest.mark.integration -def test_run_operator_dataset_inlets_and_outlets(): +def test_run_operator_dataset_inlets_and_outlets(caplog): from airflow.datasets import Dataset with DAG("test-id-1", start_date=datetime(2022, 1, 1)) as dag: + seed_operator = DbtSeedLocalOperator( + profile_config=real_profile_config, + project_dir=DBT_PROJ_DIR, + task_id="seed", + dbt_cmd_flags=["--select", "raw_customers"], + install_deps=True, + append_env=True, + ) run_operator = DbtRunLocalOperator( profile_config=real_profile_config, project_dir=DBT_PROJ_DIR, @@ -224,16 +440,98 @@ def test_run_operator_dataset_inlets_and_outlets(): install_deps=True, append_env=True, ) - run_operator + seed_operator >> run_operator >> test_operator + run_test_dag(dag) + assert run_operator.inlets == [] assert run_operator.outlets == [Dataset(uri="postgres://0.0.0.0:5432/postgres.public.stg_customers", extra=None)] assert test_operator.inlets == [Dataset(uri="postgres://0.0.0.0:5432/postgres.public.stg_customers", extra=None)] assert test_operator.outlets == [] +@pytest.mark.skipif( + version.parse(airflow_version) not in PARTIALLY_SUPPORTED_AIRFLOW_VERSIONS, + reason="Airflow 2.9.0 and 2.9.1 have a breaking change in Dataset URIs", + # https://github.com/apache/airflow/issues/39486 +) +@pytest.mark.integration +def test_run_operator_dataset_emission_is_skipped(caplog): + + with DAG("test-id-1", start_date=datetime(2022, 1, 1)) as dag: + seed_operator = DbtSeedLocalOperator( + profile_config=real_profile_config, + project_dir=DBT_PROJ_DIR, + task_id="seed", + dbt_cmd_flags=["--select", "raw_customers"], + install_deps=True, + append_env=True, + emit_datasets=False, + ) + run_operator = DbtRunLocalOperator( + profile_config=real_profile_config, + project_dir=DBT_PROJ_DIR, + task_id="run", + dbt_cmd_flags=["--models", "stg_customers"], + install_deps=True, + append_env=True, + emit_datasets=False, + ) + + seed_operator >> run_operator + + run_test_dag(dag) + + assert run_operator.inlets == [] + assert run_operator.outlets == [] + + +@pytest.mark.integration +def test_run_operator_caches_partial_parsing(caplog, tmp_path): + caplog.set_level(logging.DEBUG) + with DAG("test-partial-parsing", start_date=datetime(2022, 1, 1)) as dag: + seed_operator = DbtSeedLocalOperator( + profile_config=real_profile_config, + project_dir=DBT_PROJ_DIR, + task_id="seed", + dbt_cmd_flags=["--select", "raw_customers"], + install_deps=True, + append_env=True, + cache_dir=cache._obtain_cache_dir_path("test-partial-parsing", tmp_path), + invocation_mode=InvocationMode.SUBPROCESS, + ) + seed_operator + + run_test_dag(dag) + + # Unable to do partial parsing because saved manifest not found. Starting full parse. + assert "Unable to do partial parsing" in caplog.text + + caplog.clear() + run_test_dag(dag) + + assert not "Unable to do partial parsing" in caplog.text + + +def test_dbt_base_operator_no_partial_parse() -> None: + + dbt_base_operator = ConcreteDbtLocalBaseOperator( + profile_config=profile_config, + task_id="my-task", + project_dir="my/dir", + partial_parse=False, + ) + + cmd, _ = dbt_base_operator.build_cmd( + Context(execution_date=datetime(2023, 2, 15, 12, 30)), + ) + + assert "--no-partial-parse" in cmd + + @pytest.mark.integration -def test_run_test_operator_with_callback(failing_test_dbt_project): +@pytest.mark.parametrize("invocation_mode", [InvocationMode.SUBPROCESS, InvocationMode.DBT_RUNNER]) +def test_run_test_operator_with_callback(invocation_mode, failing_test_dbt_project): on_warning_callback = MagicMock() with DAG("test-id-2", start_date=datetime(2022, 1, 1)) as dag: @@ -249,6 +547,7 @@ def test_run_test_operator_with_callback(failing_test_dbt_project): task_id="test", append_env=True, on_warning_callback=on_warning_callback, + invocation_mode=invocation_mode, ) run_operator >> test_operator run_test_dag(dag) @@ -256,7 +555,8 @@ def test_run_test_operator_with_callback(failing_test_dbt_project): @pytest.mark.integration -def test_run_test_operator_without_callback(): +@pytest.mark.parametrize("invocation_mode", [InvocationMode.SUBPROCESS, InvocationMode.DBT_RUNNER]) +def test_run_test_operator_without_callback(invocation_mode): on_warning_callback = MagicMock() with DAG("test-id-3", start_date=datetime(2022, 1, 1)) as dag: @@ -265,6 +565,7 @@ def test_run_test_operator_without_callback(): project_dir=MINI_DBT_PROJ_DIR, task_id="run", append_env=True, + invocation_mode=invocation_mode, ) test_operator = DbtTestLocalOperator( profile_config=mini_profile_config, @@ -272,6 +573,7 @@ def test_run_test_operator_without_callback(): task_id="test", append_env=True, on_warning_callback=on_warning_callback, + invocation_mode=invocation_mode, ) run_operator >> test_operator run_test_dag(dag) @@ -292,7 +594,7 @@ class MockEvent: run = MockRun() job = MockJob() - dbt_base_operator = DbtLocalBaseOperator( + dbt_base_operator = ConcreteDbtLocalBaseOperator( profile_config=profile_config, task_id="my-task", project_dir="my/dir", @@ -307,7 +609,7 @@ class MockEvent: def test_run_operator_emits_events_without_openlineage_events_completes(caplog): - dbt_base_operator = DbtLocalBaseOperator( + dbt_base_operator = ConcreteDbtLocalBaseOperator( profile_config=profile_config, task_id="my-task", project_dir="my/dir", @@ -324,7 +626,7 @@ def test_run_operator_emits_events_without_openlineage_events_completes(caplog): def test_store_compiled_sql() -> None: - dbt_base_operator = DbtLocalBaseOperator( + dbt_base_operator = ConcreteDbtLocalBaseOperator( profile_config=profile_config, task_id="my-task", project_dir="my/dir", @@ -337,7 +639,7 @@ def test_store_compiled_sql() -> None: context=Context(execution_date=datetime(2023, 2, 15, 12, 30)), ) - dbt_base_operator = DbtLocalBaseOperator( + dbt_base_operator = ConcreteDbtLocalBaseOperator( profile_config=profile_config, task_id="my-task", project_dir="my/dir", @@ -356,21 +658,62 @@ def test_store_compiled_sql() -> None: @pytest.mark.parametrize( "operator_class,kwargs,expected_call_kwargs", [ - (DbtSeedLocalOperator, {"full_refresh": True}, {"context": {}, "cmd_flags": ["--full-refresh"]}), - (DbtRunLocalOperator, {"full_refresh": True}, {"context": {}, "cmd_flags": ["--full-refresh"]}), - (DbtTestLocalOperator, {"full_refresh": True}, {"context": {}}), + ( + DbtSeedLocalOperator, + {"full_refresh": True}, + {"context": {}, "env": {}, "cmd_flags": ["seed", "--full-refresh"]}, + ), + ( + DbtBuildLocalOperator, + {"full_refresh": True}, + {"context": {}, "env": {}, "cmd_flags": ["build", "--full-refresh"]}, + ), + ( + DbtRunLocalOperator, + {"full_refresh": True}, + {"context": {}, "env": {}, "cmd_flags": ["run", "--full-refresh"]}, + ), + ( + DbtTestLocalOperator, + {}, + {"context": {}, "env": {}, "cmd_flags": ["test"]}, + ), + ( + DbtTestLocalOperator, + {"select": []}, + {"context": {}, "env": {}, "cmd_flags": ["test"]}, + ), + ( + DbtTestLocalOperator, + {"full_refresh": True, "select": ["tag:daily"], "exclude": ["tag:disabled"]}, + {"context": {}, "env": {}, "cmd_flags": ["test", "--select", "tag:daily", "--exclude", "tag:disabled"]}, + ), + ( + DbtTestLocalOperator, + {"full_refresh": True, "selector": "nightly_snowplow"}, + {"context": {}, "env": {}, "cmd_flags": ["test", "--selector", "nightly_snowplow"]}, + ), ( DbtRunOperationLocalOperator, {"args": {"days": 7, "dry_run": True}, "macro_name": "bla"}, - {"context": {}, "cmd_flags": ["--args", "days: 7\ndry_run: true\n"]}, + {"context": {}, "env": {}, "cmd_flags": ["run-operation", "bla", "--args", "days: 7\ndry_run: true\n"]}, ), ], ) -@patch("cosmos.operators.local.DbtLocalBaseOperator.build_and_run_cmd") -def test_operator_execute_with_flags(mock_build_and_run_cmd, operator_class, kwargs, expected_call_kwargs): - task = operator_class(profile_config=profile_config, task_id="my-task", project_dir="my/dir", **kwargs) +@patch("cosmos.operators.local.DbtLocalBaseOperator.run_command") +def test_operator_execute_with_flags(mock_run_cmd, operator_class, kwargs, expected_call_kwargs): + task = operator_class( + profile_config=profile_config, + task_id="my-task", + project_dir="my/dir", + invocation_mode=InvocationMode.DBT_RUNNER, + **kwargs, + ) + task.get_env = MagicMock(return_value={}) task.execute(context={}) - mock_build_and_run_cmd.assert_called_once_with(**expected_call_kwargs) + mock_run_cmd.assert_called_once_with( + cmd=[task.dbt_executable_path, *expected_call_kwargs.pop("cmd_flags")], **expected_call_kwargs + ) @pytest.mark.parametrize( @@ -379,6 +722,7 @@ def test_operator_execute_with_flags(mock_build_and_run_cmd, operator_class, kwa DbtLSLocalOperator, DbtSnapshotLocalOperator, DbtTestLocalOperator, + DbtBuildLocalOperator, DbtDocsLocalOperator, DbtDocsS3LocalOperator, DbtDocsAzureStorageLocalOperator, @@ -396,10 +740,11 @@ def test_operator_execute_without_flags(mock_build_and_run_cmd, operator_class): profile_config=profile_config, task_id="my-task", project_dir="my/dir", + invocation_mode=InvocationMode.DBT_RUNNER, **operator_class_kwargs.get(operator_class, {}), ) task.execute(context={}) - mock_build_and_run_cmd.assert_called_once_with(context={}) + mock_build_and_run_cmd.assert_called_once_with(context={}, cmd_flags=[]) @patch("cosmos.operators.local.DbtLocalArtifactProcessor") @@ -407,7 +752,7 @@ def test_calculate_openlineage_events_completes_openlineage_errors(mock_processo instance = mock_processor.return_value instance.parse = MagicMock(side_effect=KeyError) caplog.set_level(logging.DEBUG) - dbt_base_operator = DbtLocalBaseOperator( + dbt_base_operator = ConcreteDbtLocalBaseOperator( profile_config=profile_config, task_id="my-task", project_dir=DBT_PROJ_DIR, @@ -420,6 +765,29 @@ def test_calculate_openlineage_events_completes_openlineage_errors(mock_processo assert err_msg in caplog.text +@pytest.mark.parametrize( + "operator_class,expected_template", + [ + ( + DbtSeedLocalOperator, + ("env", "select", "exclude", "selector", "vars", "models", "compiled_sql", "full_refresh"), + ), + ( + DbtRunLocalOperator, + ("env", "select", "exclude", "selector", "vars", "models", "compiled_sql", "full_refresh"), + ), + ( + DbtBuildLocalOperator, + ("env", "select", "exclude", "selector", "vars", "models", "compiled_sql", "full_refresh"), + ), + ], +) +def test_dbt_base_operator_template_fields(operator_class, expected_template): + # Check if value of template fields is what we expect for the operators we're validating + dbt_base_operator = operator_class(profile_config=profile_config, task_id="my-task", project_dir="my/dir") + assert dbt_base_operator.template_fields == expected_template + + @patch.object(DbtDocsGCSLocalOperator, "required_files", ["file1", "file2"]) def test_dbt_docs_gcs_local_operator(): mock_gcs = MagicMock() @@ -445,3 +813,131 @@ def test_dbt_docs_gcs_local_operator(): call(filename="fake-dir/target/file2", bucket_name="fake-bucket", object_name="fake-folder/file2"), ] mock_hook.upload.assert_has_calls(expected_upload_calls) + + +@patch("cosmos.operators.local.DbtLocalBaseOperator.store_compiled_sql") +@patch("cosmos.operators.local.DbtLocalBaseOperator.handle_exception_subprocess") +@patch("cosmos.config.ProfileConfig.ensure_profile") +@patch("cosmos.operators.local.DbtLocalBaseOperator.run_subprocess") +@patch("cosmos.operators.local.DbtLocalBaseOperator.run_dbt_runner") +@patch("cosmos.operators.local.tempfile.TemporaryDirectory") +@pytest.mark.parametrize("invocation_mode", [InvocationMode.SUBPROCESS, InvocationMode.DBT_RUNNER]) +def test_operator_execute_deps_parameters( + mock_temporary_directory, + mock_dbt_runner, + mock_subprocess, + mock_ensure_profile, + mock_exception_handling, + mock_store_compiled_sql, + invocation_mode, + tmp_path, +): + project_dir = tmp_path / "mock_project_tmp_dir" + project_dir.mkdir() + + expected_call_kwargs = [ + "/usr/local/bin/dbt", + "deps", + "--project-dir", + project_dir.as_posix(), + "--profiles-dir", + "/path/to", + "--profile", + "default", + "--target", + "dev", + ] + task = DbtRunLocalOperator( + dag=DAG("sample_dag", start_date=datetime(2024, 4, 16)), + profile_config=real_profile_config, + task_id="my-task", + project_dir=DBT_PROJ_DIR, + install_deps=True, + emit_datasets=False, + dbt_executable_path="/usr/local/bin/dbt", + invocation_mode=invocation_mode, + ) + mock_ensure_profile.return_value.__enter__.return_value = (Path("/path/to/profile"), {"ENV_VAR": "value"}) + mock_temporary_directory.return_value.__enter__.return_value = project_dir.as_posix() + task.execute(context={"task_instance": MagicMock()}) + if invocation_mode == InvocationMode.SUBPROCESS: + assert mock_subprocess.call_args_list[0].kwargs["command"] == expected_call_kwargs + elif invocation_mode == InvocationMode.DBT_RUNNER: + mock_dbt_runner.all_args_list[0].kwargs["command"] == expected_call_kwargs + + +def test_dbt_docs_local_operator_with_static_flag(): + # Check when static flag is passed, the required files are correctly adjusted to a single file + operator = DbtDocsLocalOperator( + task_id="fake-task", + project_dir="fake-dir", + profile_config=profile_config, + dbt_cmd_flags=["--static"], + ) + assert operator.required_files == ["static_index.html"] + + +def test_dbt_docs_local_operator_ignores_graph_gpickle(): + # Check when --no-write-json is passed, graph.gpickle is removed. + # This is only currently relevant for subclasses, but will become more generally relevant in the future. + class CustomDbtDocsLocalOperator(DbtDocsLocalOperator): + required_files = ["index.html", "manifest.json", "graph.gpickle", "catalog.json"] + + operator = CustomDbtDocsLocalOperator( + task_id="fake-task", + project_dir="fake-dir", + profile_config=profile_config, + dbt_cmd_global_flags=["--no-write-json"], + ) + assert operator.required_files == ["index.html", "manifest.json", "catalog.json"] + + +@patch("cosmos.hooks.subprocess.FullOutputSubprocessHook.send_sigint") +def test_dbt_local_operator_on_kill_sigint(mock_send_sigint) -> None: + + dbt_base_operator = ConcreteDbtLocalBaseOperator( + profile_config=profile_config, + task_id="my-task", + project_dir="my/dir", + cancel_query_on_kill=True, + invocation_mode=InvocationMode.SUBPROCESS, + ) + + dbt_base_operator.on_kill() + + mock_send_sigint.assert_called_once() + + +@patch("cosmos.hooks.subprocess.FullOutputSubprocessHook.send_sigterm") +def test_dbt_local_operator_on_kill_sigterm(mock_send_sigterm) -> None: + + dbt_base_operator = ConcreteDbtLocalBaseOperator( + profile_config=profile_config, + task_id="my-task", + project_dir="my/dir", + cancel_query_on_kill=False, + invocation_mode=InvocationMode.SUBPROCESS, + ) + + dbt_base_operator.on_kill() + + mock_send_sigterm.assert_called_once() + + +def test_handle_exception_subprocess(caplog): + """ + Test the handle_exception_subprocess method of the DbtLocalBaseOperator class for non-zero dbt exit code. + """ + operator = ConcreteDbtLocalBaseOperator( + profile_config=None, + task_id="my-task", + project_dir="my/dir", + ) + result = FullOutputSubprocessResult(exit_code=1, output="test", full_output=["n" * n for n in range(1, 1000)]) + + caplog.set_level(logging.ERROR) + # Test when exit_code is non-zero + with pytest.raises(AirflowException) as err_context: + operator.handle_exception_subprocess(result) + assert len(str(err_context.value)) < 100 # Ensure the error message is not too long + assert len(caplog.text) > 1000 # Ensure the log message is not truncated diff --git a/tests/operators/test_virtualenv.py b/tests/operators/test_virtualenv.py index 142a251a7c..deb7151e5d 100644 --- a/tests/operators/test_virtualenv.py +++ b/tests/operators/test_virtualenv.py @@ -1,11 +1,12 @@ -from unittest.mock import patch, MagicMock - -from cosmos.operators.virtualenv import DbtVirtualenvBaseOperator +from datetime import datetime +from unittest.mock import MagicMock, patch +from airflow.models import DAG from airflow.models.connection import Connection from cosmos.config import ProfileConfig - +from cosmos.constants import InvocationMode +from cosmos.operators.virtualenv import DbtVirtualenvBaseOperator from cosmos.profiles import PostgresUserPasswordProfileMapping profile_config = ProfileConfig( @@ -18,10 +19,14 @@ ) +class ConcreteDbtVirtualenvBaseOperator(DbtVirtualenvBaseOperator): + base_cmd = ["cmd"] + + @patch("airflow.utils.python_virtualenv.execute_in_subprocess") @patch("cosmos.operators.virtualenv.DbtLocalBaseOperator.calculate_openlineage_events_completes") @patch("cosmos.operators.virtualenv.DbtLocalBaseOperator.store_compiled_sql") -@patch("cosmos.operators.virtualenv.DbtLocalBaseOperator.exception_handling") +@patch("cosmos.operators.virtualenv.DbtLocalBaseOperator.handle_exception_subprocess") @patch("cosmos.operators.virtualenv.DbtLocalBaseOperator.subprocess_hook") @patch("airflow.hooks.base.BaseHook.get_connection") def test_run_command( @@ -41,14 +46,17 @@ def test_run_command( password="fake_password", schema="fake_schema", ) - venv_operator = DbtVirtualenvBaseOperator( + venv_operator = ConcreteDbtVirtualenvBaseOperator( + dag=DAG("sample_dag", start_date=datetime(2024, 4, 16)), profile_config=profile_config, task_id="fake_task", install_deps=True, project_dir="./dev/dags/dbt/jaffle_shop", py_system_site_packages=False, + pip_install_options=["--test-flag"], py_requirements=["dbt-postgres==1.6.0b1"], emit_datasets=False, + invocation_mode=InvocationMode.SUBPROCESS, ) assert venv_operator._venv_tmp_dir is None # Otherwise we are creating empty directories during DAG parsing time # and not deleting them @@ -56,12 +64,29 @@ def test_run_command( run_command_args = mock_subprocess_hook.run_command.call_args_list assert len(run_command_args) == 3 python_cmd = run_command_args[0] - dbt_deps = run_command_args[1] - dbt_cmd = run_command_args[2] + dbt_deps = run_command_args[1].kwargs + dbt_cmd = run_command_args[2].kwargs assert python_cmd[0][0][0].endswith("/bin/python") assert python_cmd[0][-1][-1] == "from importlib.metadata import version; print(version('dbt-core'))" - assert dbt_deps[0][0][-1] == "deps" - assert dbt_deps[0][0][0].endswith("/bin/dbt") - assert dbt_deps[0][0][0] == dbt_cmd[0][0][0] - assert dbt_cmd[0][0][1] == "do-something" + assert dbt_deps["command"][1] == "deps" + assert dbt_deps["command"][0].endswith("/bin/dbt") + assert dbt_deps["command"][0] == dbt_cmd["command"][0] + assert dbt_cmd["command"][1] == "do-something" assert mock_execute.call_count == 2 + + +def test_virtualenv_operator_append_env_is_true_by_default(): + venv_operator = ConcreteDbtVirtualenvBaseOperator( + dag=DAG("sample_dag", start_date=datetime(2024, 4, 16)), + profile_config=profile_config, + task_id="fake_task", + install_deps=True, + project_dir="./dev/dags/dbt/jaffle_shop", + py_system_site_packages=False, + pip_install_options=["--test-flag"], + py_requirements=["dbt-postgres==1.6.0b1"], + emit_datasets=False, + invocation_mode=InvocationMode.SUBPROCESS, + ) + + assert venv_operator.append_env is True diff --git a/tests/perf/test_performance.py b/tests/perf/test_performance.py new file mode 100644 index 0000000000..995c33a740 --- /dev/null +++ b/tests/perf/test_performance.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import os +import time +from contextlib import contextmanager +from pathlib import Path +from typing import Generator + +try: + from functools import cache +except ImportError: + from functools import lru_cache as cache + +import pytest +from airflow.models.dagbag import DagBag +from dbt.version import get_installed_version as get_dbt_version +from packaging.version import Version + +EXAMPLE_DAGS_DIR = Path(__file__).parent.parent.parent / "dev/dags" +AIRFLOW_IGNORE_FILE = EXAMPLE_DAGS_DIR / ".airflowignore" +DBT_VERSION = Version(get_dbt_version().to_version_string()[1:]) + + +@cache +def get_dag_bag() -> DagBag: + """Create a DagBag by adding the files that are not supported to .airflowignore""" + # add everything to airflow ignore that isn't performance_dag.py + with open(AIRFLOW_IGNORE_FILE, "w+") as f: + for file in EXAMPLE_DAGS_DIR.iterdir(): + if file.is_file() and file.suffix == ".py": + if file.name != "performance_dag.py": + print(f"Adding {file.name} to .airflowignore") + f.write(f"{file.name}\n") + + print(AIRFLOW_IGNORE_FILE.read_text()) + + db = DagBag(EXAMPLE_DAGS_DIR, include_examples=False) + + assert db.dags + assert not db.import_errors + + return db + + +def generate_model_code(model_number: int) -> str: + """ + Generates code for a dbt model with a dependency on the previous model. Runs + a simple select statement on the previous model. + """ + if model_number == 0: + return f""" + {{{{ config(materialized='table') }}}} + + select 1 as id + """ + + return f""" + {{{{ config(materialized='table') }}}} + + select * from {{{{ ref('model_{model_number - 1}') }}}} + """ + + +@contextmanager +def generate_project( + project_path: Path, + num_models: int, +) -> Generator[Path, None, None]: + """ + Generate dbt models in the project directory. + """ + models_dir = project_path / "models" + + try: + # create the models directory + models_dir.mkdir(exist_ok=True) + + # create the models + for i in range(num_models): + model = models_dir / f"model_{i}.sql" + model.write_text(generate_model_code(i)) + + yield project_path + finally: + # clean up the models in the project_path / models directory + for model in models_dir.iterdir(): + model.unlink() + + +@pytest.mark.perf +def test_perf_dag(): + num_models = os.environ.get("MODEL_COUNT", 10) + + if type(num_models) is str: + num_models = int(num_models) + + print(f"Generating dbt project with {num_models} models") + + dbt_project_dir = EXAMPLE_DAGS_DIR / "dbt" / "perf" + + with generate_project(dbt_project_dir, num_models): + dag_bag = get_dag_bag() + + dag = dag_bag.get_dag("performance_dag") + + # verify the integrity of the dag + assert dag.task_count == num_models + + # measure the time before and after the dag is run + + start = time.time() + dag_run = dag.test() + end = time.time() + + # assert the dag run was successful before writing the results + if dag_run.state == "success": + print(f"Ran {num_models} models in {end - start} seconds") + print(f"NUM_MODELS={num_models}\nTIME={end - start}") + + # write the results to a file + with open("/tmp/performance_results.txt", "w") as f: + f.write( + f"NUM_MODELS={num_models}\nTIME={end - start}\nMODELS_PER_SECOND={num_models / (end - start)}\nDBT_VERSION={DBT_VERSION}" + ) + else: + raise Exception("Performance DAG run failed.") diff --git a/tests/plugin/__init__.py b/tests/plugin/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/plugin/test_plugin.py b/tests/plugin/test_plugin.py new file mode 100644 index 0000000000..04dc39cd8b --- /dev/null +++ b/tests/plugin/test_plugin.py @@ -0,0 +1,305 @@ +# dbt-core relies on Jinja2>3, whereas Flask<2 relies on an incompatible version of Jinja2. +# +# This discrepancy causes the automated integration tests to fail, as dbt-core is installed in the same +# environment as apache-airflow. +# +# We can get around this by patching the jinja2 namespace to include the deprecated objects: +try: + import flask # noqa: F401 +except ImportError: + import jinja2 + import markupsafe + + jinja2.Markup = markupsafe.Markup + jinja2.escape = markupsafe.escape + +import sys +from importlib.util import find_spec +from unittest.mock import MagicMock, PropertyMock, mock_open, patch + +import pytest +from airflow.utils.db import initdb, resetdb +from airflow.www.app import cached_app +from airflow.www.extensions.init_appbuilder import AirflowAppBuilder +from flask.testing import FlaskClient + +import cosmos.plugin +from cosmos.plugin import ( + dbt_docs_view, + iframe_script, + open_azure_file, + open_file, + open_gcs_file, + open_http_file, + open_s3_file, +) + + +def _get_text_from_response(response) -> str: + # Airflow < 2.4 uses an old version of Werkzeug that does not have Response.text. + if not hasattr(response, "text"): + return response.get_data(as_text=True) + else: + return response.text + + +@pytest.fixture(scope="module") +def app() -> FlaskClient: + initdb() + + app = cached_app(testing=True) + appbuilder: AirflowAppBuilder = app.extensions["appbuilder"] + + appbuilder.sm.check_authorization = lambda *args, **kwargs: True + + if dbt_docs_view not in appbuilder.baseviews: + appbuilder._check_and_init(dbt_docs_view) + appbuilder.register_blueprint(dbt_docs_view) + + yield app.test_client() + + resetdb() + + +@pytest.mark.integration +def test_dbt_docs(monkeypatch, app): + monkeypatch.setattr("cosmos.plugin.dbt_docs_dir", "path/to/docs/dir") + + response = app.get("/cosmos/dbt_docs") + + assert response.status_code == 200 + assert " None: + pass + + def get_credentials(self) -> Credentials: + return mock_assumed_credentials + + mock_aws_hook.AwsGenericHook = MockAwsGenericHook + + with patch.dict(sys.modules, {"airflow.providers.amazon.aws.hooks.base_aws": mock_aws_hook}): + yield mock_aws_hook + + +def mock_conn_value(token: str | None = None) -> Connection: conn = Connection( conn_id="my_athena_connection", conn_type="aws", @@ -22,14 +50,33 @@ def mock_athena_conn(): # type: ignore password="my_aws_secret_key", extra=json.dumps( { + "aws_session_token": token, "database": "my_database", - "region_name": "my_region", + "region_name": "us-east-1", "s3_staging_dir": "s3://my_bucket/dbt/", "schema": "my_schema", } ), ) + return conn + + +@pytest.fixture() +def mock_athena_conn(): # type: ignore + """ + Sets the connection as an environment variable. + """ + conn = mock_conn_value(token="token123") + with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): + yield conn + +@pytest.fixture() +def mock_athena_conn_without_token(): # type: ignore + """ + Sets the connection as an environment variable. + """ + conn = mock_conn_value(token=None) with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): yield conn @@ -47,6 +94,7 @@ def test_athena_connection_claiming() -> None: # - region_name # - s3_staging_dir # - schema + potential_values = { "conn_type": "aws", "login": "my_aws_access_key_id", @@ -54,7 +102,7 @@ def test_athena_connection_claiming() -> None: "extra": json.dumps( { "database": "my_database", - "region_name": "my_region", + "region_name": "us-east-1", "s3_staging_dir": "s3://my_bucket/dbt/", "schema": "my_schema", } @@ -67,12 +115,14 @@ def test_athena_connection_claiming() -> None: del values[key] conn = Connection(**values) # type: ignore - print("testing with", values) - - with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): - # should raise an InvalidMappingException - profile_mapping = AthenaAccessKeyProfileMapping(conn, {}) - assert not profile_mapping.can_claim_connection() + with patch( + "cosmos.profiles.athena.access_key.AthenaAccessKeyProfileMapping._get_temporary_credentials", + return_value=mock_missing_credentials, + ): + with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): + # should raise an InvalidMappingException + profile_mapping = AthenaAccessKeyProfileMapping(conn, {}) + assert not profile_mapping.can_claim_connection() # if we have them all, it should claim conn = Connection(**potential_values) # type: ignore @@ -87,6 +137,7 @@ def test_athena_profile_mapping_selected( """ Tests that the correct profile mapping is selected for Athena. """ + profile_mapping = get_automatic_profile_mapping( mock_athena_conn.conn_id, ) @@ -99,14 +150,16 @@ def test_athena_profile_args( """ Tests that the profile values get set correctly for Athena. """ + profile_mapping = get_automatic_profile_mapping( mock_athena_conn.conn_id, ) assert profile_mapping.profile == { "type": "athena", - "aws_access_key_id": mock_athena_conn.login, + "aws_access_key_id": mock_assumed_credentials.access_key, "aws_secret_access_key": "{{ env_var('COSMOS_CONN_AWS_AWS_SECRET_ACCESS_KEY') }}", + "aws_session_token": "{{ env_var('COSMOS_CONN_AWS_AWS_SESSION_TOKEN') }}", "database": mock_athena_conn.extra_dejson.get("database"), "region_name": mock_athena_conn.extra_dejson.get("region_name"), "s3_staging_dir": mock_athena_conn.extra_dejson.get("s3_staging_dir"), @@ -114,25 +167,55 @@ def test_athena_profile_args( } +@mock.patch("cosmos.profiles.athena.access_key.AthenaAccessKeyProfileMapping._get_temporary_credentials") +def test_athena_profile_args_without_token(mock_temp_cred, mock_athena_conn_without_token: Connection) -> None: + """ + Tests that the profile values get set correctly for Athena. + """ + ReadOnlyCredentials = namedtuple("ReadOnlyCredentials", ["access_key", "secret_key", "token"]) + credentials = ReadOnlyCredentials(access_key="my_aws_access_key", secret_key="my_aws_secret_key", token=None) + mock_temp_cred.return_value = credentials + + profile_mapping = get_automatic_profile_mapping(mock_athena_conn_without_token.conn_id) + + assert profile_mapping.profile == { + "type": "athena", + "aws_access_key_id": "my_aws_access_key", + "aws_secret_access_key": "{{ env_var('COSMOS_CONN_AWS_AWS_SECRET_ACCESS_KEY') }}", + "database": mock_athena_conn_without_token.extra_dejson.get("database"), + "region_name": mock_athena_conn_without_token.extra_dejson.get("region_name"), + "s3_staging_dir": mock_athena_conn_without_token.extra_dejson.get("s3_staging_dir"), + "schema": mock_athena_conn_without_token.extra_dejson.get("schema"), + } + + def test_athena_profile_args_overrides( mock_athena_conn: Connection, ) -> None: """ Tests that you can override the profile values for Athena. """ + profile_mapping = get_automatic_profile_mapping( mock_athena_conn.conn_id, - profile_args={"schema": "my_custom_schema", "database": "my_custom_db"}, + profile_args={ + "schema": "my_custom_schema", + "database": "my_custom_db", + "aws_session_token": "override_token", + }, ) + assert profile_mapping.profile_args == { "schema": "my_custom_schema", "database": "my_custom_db", + "aws_session_token": "override_token", } assert profile_mapping.profile == { "type": "athena", - "aws_access_key_id": mock_athena_conn.login, + "aws_access_key_id": mock_assumed_credentials.access_key, "aws_secret_access_key": "{{ env_var('COSMOS_CONN_AWS_AWS_SECRET_ACCESS_KEY') }}", + "aws_session_token": "{{ env_var('COSMOS_CONN_AWS_AWS_SESSION_TOKEN') }}", "database": "my_custom_db", "region_name": mock_athena_conn.extra_dejson.get("region_name"), "s3_staging_dir": mock_athena_conn.extra_dejson.get("s3_staging_dir"), @@ -146,9 +229,12 @@ def test_athena_profile_env_vars( """ Tests that the environment variables get set correctly for Athena. """ + profile_mapping = get_automatic_profile_mapping( mock_athena_conn.conn_id, ) + assert profile_mapping.env_vars == { - "COSMOS_CONN_AWS_AWS_SECRET_ACCESS_KEY": mock_athena_conn.password, + "COSMOS_CONN_AWS_AWS_SECRET_ACCESS_KEY": mock_assumed_credentials.secret_key, + "COSMOS_CONN_AWS_AWS_SESSION_TOKEN": mock_assumed_credentials.token, } diff --git a/tests/profiles/bigquery/test_bq_oauth.py b/tests/profiles/bigquery/test_bq_oauth.py index f225f585f7..d48cb8cc4b 100644 --- a/tests/profiles/bigquery/test_bq_oauth.py +++ b/tests/profiles/bigquery/test_bq_oauth.py @@ -1,4 +1,4 @@ -"Tests for the BigQuery profile." +"""Tests for the BigQuery profile.""" import json from unittest.mock import patch diff --git a/tests/profiles/bigquery/test_bq_service_account_file.py b/tests/profiles/bigquery/test_bq_service_account_file.py index 7b4cbd6efc..7c685b50b1 100644 --- a/tests/profiles/bigquery/test_bq_service_account_file.py +++ b/tests/profiles/bigquery/test_bq_service_account_file.py @@ -1,4 +1,4 @@ -"Tests for the BigQuery profile." +"""Tests for the BigQuery profile.""" import json from unittest.mock import patch diff --git a/tests/profiles/bigquery/test_bq_service_account_keyfile_dict.py b/tests/profiles/bigquery/test_bq_service_account_keyfile_dict.py old mode 100644 new mode 100755 index 0647e68000..d30c900216 --- a/tests/profiles/bigquery/test_bq_service_account_keyfile_dict.py +++ b/tests/profiles/bigquery/test_bq_service_account_keyfile_dict.py @@ -1,13 +1,15 @@ import json +from collections import namedtuple from unittest.mock import patch import pytest from airflow.models.connection import Connection -from cosmos.exceptions import CosmosValueError +from cosmos.exceptions import CosmosValueError from cosmos.profiles import get_automatic_profile_mapping from cosmos.profiles.bigquery.service_account_keyfile_dict import GoogleCloudServiceAccountDictProfileMapping +ConnExtraParams = namedtuple("ConnExtraParams", ["keyfile_dict", "keyfile_json_extra_key"]) sample_keyfile_dict = { "type": "service_account", "private_key_id": "my_private_key_id", @@ -15,7 +17,21 @@ } -@pytest.fixture(params=[sample_keyfile_dict, json.dumps(sample_keyfile_dict)]) +def get_fixture_params(): + """ + Make a matrix of params for the fixture that mock connection, as there are multiple fields in + the airflow param mapping for the "keyfile_json" in GoogleCloudServiceAccountDictProfileMapping + """ + fixture_params = [] + for d in (sample_keyfile_dict, json.dumps(sample_keyfile_dict)): + for key in GoogleCloudServiceAccountDictProfileMapping.airflow_param_mapping.get("keyfile_json"): + if key.startswith("extra."): + key = key.replace("extra.", "") + fixture_params.append(ConnExtraParams(keyfile_dict=d, keyfile_json_extra_key=key)) + return fixture_params + + +@pytest.fixture(params=get_fixture_params()) def mock_bigquery_conn_with_dict(request): # type: ignore """ Mocks and returns an Airflow BigQuery connection. @@ -23,14 +39,13 @@ def mock_bigquery_conn_with_dict(request): # type: ignore extra = { "project": "my_project", "dataset": "my_dataset", - "keyfile_dict": request.param, + request.param.keyfile_json_extra_key: request.param.keyfile_dict, } conn = Connection( conn_id="my_bigquery_connection", conn_type="google_cloud_platform", extra=json.dumps(extra), ) - with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): yield conn @@ -49,8 +64,8 @@ def test_connection_claiming_succeeds(mock_bigquery_conn_with_dict: Connection): def test_connection_claiming_fails(mock_bigquery_conn_with_dict: Connection): - # Remove the `dataset` key, which is mandatory - mock_bigquery_conn_with_dict.extra = json.dumps({"project": "my_project", "keyfile_dict": sample_keyfile_dict}) + # Remove the `project` key, which is mandatory + mock_bigquery_conn_with_dict.extra = json.dumps({"dataset": "my_dataset", "keyfile_dict": sample_keyfile_dict}) profile_mapping = GoogleCloudServiceAccountDictProfileMapping(mock_bigquery_conn_with_dict, {}) assert not profile_mapping.can_claim_connection() diff --git a/tests/profiles/clickhouse/__init__.py b/tests/profiles/clickhouse/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/profiles/clickhouse/test_clickhouse_userpass.py b/tests/profiles/clickhouse/test_clickhouse_userpass.py new file mode 100644 index 0000000000..1f623c8030 --- /dev/null +++ b/tests/profiles/clickhouse/test_clickhouse_userpass.py @@ -0,0 +1,117 @@ +"""Tests for the clickhouse profile.""" + +from unittest.mock import patch + +import pytest +from airflow.models.connection import Connection + +from cosmos.profiles import get_automatic_profile_mapping +from cosmos.profiles.clickhouse.user_pass import ( + ClickhouseUserPasswordProfileMapping, +) + + +@pytest.fixture() +def mock_clickhouse_conn(): # type: ignore + """Sets the connection as an environment variable.""" + conn = Connection( + conn_id="clickhouse_connection", + conn_type="generic", + host="my_host", + login="my_user", + password="my_password", + schema="my_database", + extra='{"clickhouse": "True"}', + ) + + with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): + yield conn + + +def test_connection_claiming1() -> None: + """ + Tests that the clickhouse profile mapping claims the correct connection type. + + should only claim when: + - conn_type == generic + And the following exist: + - host + - login + - password + - schema + - extra.clickhouse + """ + required_values = { + "conn_type": "generic", + "host": "my_host", + "login": "my_user", + "schema": "my_database", + "extra": '{"clickhouse": "True"}', + } + + def can_claim_with_missing_key(missing_key: str) -> bool: + values = required_values.copy() + del values[missing_key] + conn = Connection(**values) # type: ignore + with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): + profile_mapping = ClickhouseUserPasswordProfileMapping(conn, {}) + return profile_mapping.can_claim_connection() + + # if we're missing any of the required values, it shouldn't claim + for key in required_values: + assert not can_claim_with_missing_key(key), f"Failed when missing {key}" + + # if we have all the required values, it should claim + conn = Connection(**required_values) # type: ignore + with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): + profile_mapping = ClickhouseUserPasswordProfileMapping(conn, {}) + assert profile_mapping.can_claim_connection() + + +def test_profile_mapping_selected( + mock_clickhouse_conn: Connection, +) -> None: + """Tests that the correct profile mapping is selected.""" + profile_mapping = get_automatic_profile_mapping(mock_clickhouse_conn.conn_id, {}) + assert isinstance(profile_mapping, ClickhouseUserPasswordProfileMapping) + + +def test_profile_args(mock_clickhouse_conn: Connection) -> None: + """Tests that the profile values get set correctly.""" + profile_mapping = get_automatic_profile_mapping(mock_clickhouse_conn.conn_id, profile_args={}) + + assert profile_mapping.profile == { + "type": "clickhouse", + "schema": mock_clickhouse_conn.schema, + "login": mock_clickhouse_conn.login, + "password": "{{ env_var('COSMOS_CONN_GENERIC_PASSWORD') }}", + "driver": "native", + "port": 9000, + "host": mock_clickhouse_conn.host, + "secure": False, + "clickhouse": "True", + } + + +def test_mock_profile() -> None: + """Tests that the mock_profile values get set correctly.""" + profile_mapping = ClickhouseUserPasswordProfileMapping( + "conn_id" + ) # get_automatic_profile_mapping("mock_clickhouse_conn.conn_id", profile_args={}) + + assert profile_mapping.mock_profile == { + "type": "clickhouse", + "schema": "mock_value", + "login": "mock_value", + "driver": "native", + "port": 9000, + "host": "mock_value", + "secure": False, + "clickhouse": "mock_value", + } + + +def test_profile_env_vars(mock_clickhouse_conn: Connection) -> None: + """Tests that the environment variables get set correctly.""" + profile_mapping = get_automatic_profile_mapping(mock_clickhouse_conn.conn_id, profile_args={}) + assert profile_mapping.env_vars == {"COSMOS_CONN_GENERIC_PASSWORD": mock_clickhouse_conn.password} diff --git a/tests/profiles/databricks/test_dbr_token.py b/tests/profiles/databricks/test_dbr_token.py index 3fd36a784c..ada72f0e7b 100644 --- a/tests/profiles/databricks/test_dbr_token.py +++ b/tests/profiles/databricks/test_dbr_token.py @@ -1,4 +1,4 @@ -"Tests for the postgres profile." +"""Tests for the postgres profile.""" from unittest.mock import patch @@ -95,11 +95,15 @@ def test_profile_args( profile_args={ "schema": "my_schema", "catalog": "my_catalog", + "session_properties": {"legacy_time_parser_policy": "corrected"}, + "threads": 4, }, ) assert profile_mapping.profile_args == { "schema": "my_schema", "catalog": "my_catalog", + "session_properties": {"legacy_time_parser_policy": "corrected"}, + "threads": 4, } assert profile_mapping.profile == { @@ -109,7 +113,23 @@ def test_profile_args( "http_path": mock_databricks_conn.extra_dejson.get("http_path"), "schema": "my_schema", "catalog": "my_catalog", + "threads": 4, + "session_properties": {"legacy_time_parser_policy": "corrected"}, } + expected_profile_yml = """example: + outputs: + cosmos_target: + catalog: my_catalog + host: my_host + http_path: my_http_path + schema: my_schema + session_properties: + legacy_time_parser_policy: corrected + threads: 4 + token: '{{ env_var(''COSMOS_CONN_DATABRICKS_TOKEN'') }}' + type: databricks + target: cosmos_target\n""" + assert profile_mapping.get_profile_file_contents("example") == expected_profile_yml def test_profile_args_overrides( diff --git a/tests/profiles/exasol/test_exasol_user_pass.py b/tests/profiles/exasol/test_exasol_user_pass.py index b2880c222d..b4f4d14b49 100644 --- a/tests/profiles/exasol/test_exasol_user_pass.py +++ b/tests/profiles/exasol/test_exasol_user_pass.py @@ -1,4 +1,4 @@ -"Tests for the Exasol profile." +"""Tests for the Exasol profile.""" from unittest.mock import patch diff --git a/tests/profiles/postgres/test_pg_user_pass.py b/tests/profiles/postgres/test_pg_user_pass.py index 3492450b5e..c23e6add68 100644 --- a/tests/profiles/postgres/test_pg_user_pass.py +++ b/tests/profiles/postgres/test_pg_user_pass.py @@ -1,4 +1,4 @@ -"Tests for the postgres profile." +"""Tests for the postgres profile.""" from unittest.mock import patch @@ -83,9 +83,9 @@ def test_connection_claiming() -> None: assert not profile_mapping.can_claim_connection() # also test when there's no schema - conn = Connection(**potential_values) # type: ignore + conn = Connection(**{k: v for k, v in potential_values.items() if k != "schema"}) with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): - profile_mapping = PostgresUserPasswordProfileMapping(conn, {}) + profile_mapping = PostgresUserPasswordProfileMapping(conn, {"schema": None}) assert not profile_mapping.can_claim_connection() # if we have them all, it should claim diff --git a/tests/profiles/redshift/test_redshift_user_pass.py b/tests/profiles/redshift/test_redshift_user_pass.py index 073ce2deec..f1e87b3cdc 100644 --- a/tests/profiles/redshift/test_redshift_user_pass.py +++ b/tests/profiles/redshift/test_redshift_user_pass.py @@ -1,4 +1,4 @@ -"Tests for the Redshift profile." +"""Tests for the Redshift profile.""" from unittest.mock import patch diff --git a/tests/profiles/snowflake/test_snowflake_user_encrypted_privatekey.py b/tests/profiles/snowflake/test_snowflake_user_encrypted_privatekey_env_variable.py similarity index 92% rename from tests/profiles/snowflake/test_snowflake_user_encrypted_privatekey.py rename to tests/profiles/snowflake/test_snowflake_user_encrypted_privatekey_env_variable.py index b61b85094b..d6c1e30aee 100644 --- a/tests/profiles/snowflake/test_snowflake_user_encrypted_privatekey.py +++ b/tests/profiles/snowflake/test_snowflake_user_encrypted_privatekey_env_variable.py @@ -1,4 +1,4 @@ -"Tests for the Snowflake user/private key profile." +"""Tests for the Snowflake user/private key environmentvariable profile.""" import json from unittest.mock import patch @@ -29,7 +29,7 @@ def mock_snowflake_conn(): # type: ignore "region": "my_region", "database": "my_database", "warehouse": "my_warehouse", - "private_key_file": "path/to/private_key.p8", + "private_key_content": "my_private_key", } ), ) @@ -52,7 +52,7 @@ def test_connection_claiming() -> None: "account": "my_account", "database": "my_database", "warehouse": "my_warehouse", - "private_key_file": "path/to/private_key.p8", + "private_key_content": "my_private_key", } ), } @@ -130,8 +130,8 @@ def test_profile_args( assert profile_mapping.profile == { "type": mock_snowflake_conn.conn_type, "user": mock_snowflake_conn.login, + "private_key": "{{ env_var('COSMOS_CONN_SNOWFLAKE_PRIVATE_KEY') }}", "private_key_passphrase": "{{ env_var('COSMOS_CONN_SNOWFLAKE_PRIVATE_KEY_PASSPHRASE') }}", - "private_key_path": mock_snowflake_conn.extra_dejson.get("private_key_file"), "schema": mock_snowflake_conn.schema, "account": f"{mock_account}.{mock_region}", "database": mock_snowflake_conn.extra_dejson.get("database"), @@ -160,7 +160,7 @@ def test_profile_args_overrides( "type": mock_snowflake_conn.conn_type, "user": mock_snowflake_conn.login, "private_key_passphrase": "{{ env_var('COSMOS_CONN_SNOWFLAKE_PRIVATE_KEY_PASSPHRASE') }}", - "private_key_path": mock_snowflake_conn.extra_dejson.get("private_key_file"), + "private_key": "{{ env_var('COSMOS_CONN_SNOWFLAKE_PRIVATE_KEY') }}", "schema": mock_snowflake_conn.schema, "account": f"{mock_account}.{mock_region}", "database": "my_db_override", @@ -178,6 +178,7 @@ def test_profile_env_vars( mock_snowflake_conn.conn_id, ) assert profile_mapping.env_vars == { + "COSMOS_CONN_SNOWFLAKE_PRIVATE_KEY": mock_snowflake_conn.extra_dejson.get("private_key_content"), "COSMOS_CONN_SNOWFLAKE_PRIVATE_KEY_PASSPHRASE": mock_snowflake_conn.password, } @@ -197,7 +198,7 @@ def test_old_snowflake_format() -> None: "extra__snowflake__account": "my_account", "extra__snowflake__database": "my_database", "extra__snowflake__warehouse": "my_warehouse", - "extra__snowflake__private_key_file": "path/to/private_key.p8", + "extra__snowflake__private_key_content": "my_private_key", } ), ) @@ -207,10 +208,10 @@ def test_old_snowflake_format() -> None: assert profile_mapping.profile == { "type": conn.conn_type, "user": conn.login, + "private_key": "{{ env_var('COSMOS_CONN_SNOWFLAKE_PRIVATE_KEY') }}", "private_key_passphrase": "{{ env_var('COSMOS_CONN_SNOWFLAKE_PRIVATE_KEY_PASSPHRASE') }}", - "private_key_path": conn.extra_dejson.get("private_key_file"), "schema": conn.schema, "account": conn.extra_dejson.get("account"), "database": conn.extra_dejson.get("database"), "warehouse": conn.extra_dejson.get("warehouse"), - } + } \ No newline at end of file diff --git a/tests/profiles/snowflake/test_snowflake_user_encrypted_privatekey_file.py b/tests/profiles/snowflake/test_snowflake_user_encrypted_privatekey_file.py new file mode 100644 index 0000000000..762895e7e0 --- /dev/null +++ b/tests/profiles/snowflake/test_snowflake_user_encrypted_privatekey_file.py @@ -0,0 +1,216 @@ +"""Tests for the Snowflake user/private key file profile.""" + +import json +from unittest.mock import patch + +import pytest +from airflow.models.connection import Connection + +from cosmos.profiles import get_automatic_profile_mapping +from cosmos.profiles.snowflake import ( + SnowflakeEncryptedPrivateKeyFilePemProfileMapping, +) + + +@pytest.fixture() +def mock_snowflake_conn(): # type: ignore + """ + Sets the connection as an environment variable. + """ + conn = Connection( + conn_id="my_snowflake_pk_connection", + conn_type="snowflake", + login="my_user", + schema="my_schema", + password="secret", + extra=json.dumps( + { + "account": "my_account", + "region": "my_region", + "database": "my_database", + "warehouse": "my_warehouse", + "private_key_file": "path/to/private_key.p8", + } + ), + ) + + with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): + yield conn + + +def test_connection_claiming() -> None: + """ + Tests that the Snowflake profile mapping claims the correct connection type. + """ + potential_values = { + "conn_type": "snowflake", + "login": "my_user", + "schema": "my_database", + "password": "secret", + "extra": json.dumps( + { + "account": "my_account", + "database": "my_database", + "warehouse": "my_warehouse", + "private_key_file": "path/to/private_key.p8", + } + ), + } + + # if we're missing any of the values, it shouldn't claim + for key in potential_values: + values = potential_values.copy() + del values[key] + conn = Connection(**values) # type: ignore + + print("testing with", values) + + with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): + profile_mapping = SnowflakeEncryptedPrivateKeyFilePemProfileMapping( + conn, + ) + assert not profile_mapping.can_claim_connection() + + # test when we're missing the account + conn = Connection(**potential_values) # type: ignore + conn.extra = '{"database": "my_database", "warehouse": "my_warehouse", "private_key_content": "my_private_key"}' + print("testing with", conn.extra) + with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): + profile_mapping = SnowflakeEncryptedPrivateKeyFilePemProfileMapping(conn) + assert not profile_mapping.can_claim_connection() + + # test when we're missing the database + conn = Connection(**potential_values) # type: ignore + conn.extra = '{"account": "my_account", "warehouse": "my_warehouse", "private_key_content": "my_private_key"}' + print("testing with", conn.extra) + with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): + profile_mapping = SnowflakeEncryptedPrivateKeyFilePemProfileMapping(conn) + assert not profile_mapping.can_claim_connection() + + # test when we're missing the warehouse + conn = Connection(**potential_values) # type: ignore + conn.extra = '{"account": "my_account", "database": "my_database", "private_key_content": "my_private_key"}' + print("testing with", conn.extra) + with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): + profile_mapping = SnowflakeEncryptedPrivateKeyFilePemProfileMapping(conn) + assert not profile_mapping.can_claim_connection() + + # if we have them all, it should claim + conn = Connection(**potential_values) # type: ignore + with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): + profile_mapping = SnowflakeEncryptedPrivateKeyFilePemProfileMapping(conn) + assert profile_mapping.can_claim_connection() + + +def test_profile_mapping_selected( + mock_snowflake_conn: Connection, +) -> None: + """ + Tests that the correct profile mapping is selected. + """ + profile_mapping = get_automatic_profile_mapping( + mock_snowflake_conn.conn_id, + ) + assert isinstance(profile_mapping, SnowflakeEncryptedPrivateKeyFilePemProfileMapping) + + +def test_profile_args( + mock_snowflake_conn: Connection, +) -> None: + """ + Tests that the profile values get set correctly. + """ + profile_mapping = get_automatic_profile_mapping( + mock_snowflake_conn.conn_id, + ) + + mock_account = mock_snowflake_conn.extra_dejson.get("account") + mock_region = mock_snowflake_conn.extra_dejson.get("region") + + assert profile_mapping.profile == { + "type": mock_snowflake_conn.conn_type, + "user": mock_snowflake_conn.login, + "private_key_passphrase": "{{ env_var('COSMOS_CONN_SNOWFLAKE_PRIVATE_KEY_PASSPHRASE') }}", + "private_key_path": mock_snowflake_conn.extra_dejson.get("private_key_file"), + "schema": mock_snowflake_conn.schema, + "account": f"{mock_account}.{mock_region}", + "database": mock_snowflake_conn.extra_dejson.get("database"), + "warehouse": mock_snowflake_conn.extra_dejson.get("warehouse"), + } + + +def test_profile_args_overrides( + mock_snowflake_conn: Connection, +) -> None: + """ + Tests that you can override the profile values. + """ + profile_mapping = get_automatic_profile_mapping( + mock_snowflake_conn.conn_id, + profile_args={"database": "my_db_override"}, + ) + assert profile_mapping.profile_args == { + "database": "my_db_override", + } + + mock_account = mock_snowflake_conn.extra_dejson.get("account") + mock_region = mock_snowflake_conn.extra_dejson.get("region") + + assert profile_mapping.profile == { + "type": mock_snowflake_conn.conn_type, + "user": mock_snowflake_conn.login, + "private_key_passphrase": "{{ env_var('COSMOS_CONN_SNOWFLAKE_PRIVATE_KEY_PASSPHRASE') }}", + "private_key_path": mock_snowflake_conn.extra_dejson.get("private_key_file"), + "schema": mock_snowflake_conn.schema, + "account": f"{mock_account}.{mock_region}", + "database": "my_db_override", + "warehouse": mock_snowflake_conn.extra_dejson.get("warehouse"), + } + + +def test_profile_env_vars( + mock_snowflake_conn: Connection, +) -> None: + """ + Tests that the environment variables get set correctly. + """ + profile_mapping = get_automatic_profile_mapping( + mock_snowflake_conn.conn_id, + ) + assert profile_mapping.env_vars == { + "COSMOS_CONN_SNOWFLAKE_PRIVATE_KEY_PASSPHRASE": mock_snowflake_conn.password, + } + + +def test_old_snowflake_format() -> None: + """ + Tests that the old format still works. + """ + conn = Connection( + conn_id="my_snowflake_connection", + conn_type="snowflake", + login="my_user", + schema="my_schema", + password="secret", + extra=json.dumps( + { + "extra__snowflake__account": "my_account", + "extra__snowflake__database": "my_database", + "extra__snowflake__warehouse": "my_warehouse", + "extra__snowflake__private_key_file": "path/to/private_key.p8", + } + ), + ) + + with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): + profile_mapping = SnowflakeEncryptedPrivateKeyFilePemProfileMapping(conn) + assert profile_mapping.profile == { + "type": conn.conn_type, + "user": conn.login, + "private_key_passphrase": "{{ env_var('COSMOS_CONN_SNOWFLAKE_PRIVATE_KEY_PASSPHRASE') }}", + "private_key_path": conn.extra_dejson.get("private_key_file"), + "schema": conn.schema, + "account": conn.extra_dejson.get("account"), + "database": conn.extra_dejson.get("database"), + "warehouse": conn.extra_dejson.get("warehouse"), + } \ No newline at end of file diff --git a/tests/profiles/snowflake/test_snowflake_user_pass.py b/tests/profiles/snowflake/test_snowflake_user_pass.py index 6276a20f1b..6514bdf8db 100644 --- a/tests/profiles/snowflake/test_snowflake_user_pass.py +++ b/tests/profiles/snowflake/test_snowflake_user_pass.py @@ -1,4 +1,4 @@ -"Tests for the Snowflake user/password profile." +"""Tests for the Snowflake user/password profile.""" import json from unittest.mock import patch @@ -231,3 +231,27 @@ def test_appends_region() -> None: "database": conn.extra_dejson.get("database"), "warehouse": conn.extra_dejson.get("warehouse"), } + + +def test_appends_host_and_port() -> None: + """ + Tests that host/port extras are appended to the connection settings. + """ + conn = Connection( + conn_id="my_snowflake_connection", + conn_type="snowflake", + login="my_user", + password="my_password", + schema="my_schema", + extra=json.dumps( + { + "host": "snowflake.localhost.localstack.cloud", + "port": 4566, + } + ), + ) + + with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): + profile_mapping = SnowflakeUserPasswordProfileMapping(conn) + assert profile_mapping.profile["host"] == "snowflake.localhost.localstack.cloud" + assert profile_mapping.profile["port"] == 4566 diff --git a/tests/profiles/snowflake/test_snowflake_user_privatekey.py b/tests/profiles/snowflake/test_snowflake_user_privatekey.py index 2692792cfa..edbf2b464f 100644 --- a/tests/profiles/snowflake/test_snowflake_user_privatekey.py +++ b/tests/profiles/snowflake/test_snowflake_user_privatekey.py @@ -1,4 +1,4 @@ -"Tests for the Snowflake user/private key profile." +"""Tests for the Snowflake user/private key profile.""" import json from unittest.mock import patch diff --git a/tests/profiles/spark/test_spark_thrift.py b/tests/profiles/spark/test_spark_thrift.py index da733e32fc..2ac8303ca9 100644 --- a/tests/profiles/spark/test_spark_thrift.py +++ b/tests/profiles/spark/test_spark_thrift.py @@ -1,4 +1,4 @@ -"Tests for the Spark profile." +"""Tests for the Spark profile.""" from unittest.mock import patch diff --git a/tests/profiles/teradata/__init__.py b/tests/profiles/teradata/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/profiles/teradata/test_teradata_user_pass.py b/tests/profiles/teradata/test_teradata_user_pass.py new file mode 100644 index 0000000000..795e461a48 --- /dev/null +++ b/tests/profiles/teradata/test_teradata_user_pass.py @@ -0,0 +1,185 @@ +"""Tests for the postgres profile.""" + +from unittest.mock import patch + +import pytest +from airflow.models.connection import Connection + +from cosmos.profiles import get_automatic_profile_mapping +from cosmos.profiles.teradata.user_pass import TeradataUserPasswordProfileMapping + + +@pytest.fixture() +def mock_teradata_conn(): # type: ignore + """ + Sets the connection as an environment variable. + """ + conn = Connection( + conn_id="my_teradata_connection", + conn_type="teradata", + host="my_host", + login="my_user", + password="my_password", + ) + + with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): + yield conn + + +@pytest.fixture() +def mock_teradata_conn_custom_tmode(): # type: ignore + """ + Sets the connection as an environment variable. + """ + conn = Connection( + conn_id="my_teradata_connection", + conn_type="teradata", + host="my_host", + login="my_user", + password="my_password", + schema="my_database", + extra='{"tmode": "TERA"}', + ) + + with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): + yield conn + + +def test_connection_claiming() -> None: + """ + Tests that the teradata profile mapping claims the correct connection type. + """ + # should only claim when: + # - conn_type == teradata + # and the following exist: + # - host + # - user + # - password + potential_values: dict[str, str] = { + "conn_type": "teradata", + "host": "my_host", + "login": "my_user", + "password": "my_password", + } + + # if we're missing any of the values, it shouldn't claim + for key in potential_values: + values = potential_values.copy() + del values[key] + conn = Connection(**values) # type: ignore + + with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): + profile_mapping = TeradataUserPasswordProfileMapping(conn) + assert not profile_mapping.can_claim_connection() + + # Even there is no schema, making user as schema as user itself schema in teradata + conn = Connection(**{k: v for k, v in potential_values.items() if k != "schema"}) + with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): + profile_mapping = TeradataUserPasswordProfileMapping(conn, {"schema": None}) + assert profile_mapping.can_claim_connection() + # if we have them all, it should claim + conn = Connection(**potential_values) # type: ignore + with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): + profile_mapping = TeradataUserPasswordProfileMapping(conn, {"schema": "my_schema"}) + assert profile_mapping.can_claim_connection() + + +def test_profile_mapping_selected( + mock_teradata_conn: Connection, +) -> None: + """ + Tests that the correct profile mapping is selected. + """ + profile_mapping = get_automatic_profile_mapping( + mock_teradata_conn.conn_id, + ) + assert isinstance(profile_mapping, TeradataUserPasswordProfileMapping) + + +def test_profile_mapping_schema_validation(mock_teradata_conn: Connection) -> None: + # port is not handled in airflow connection so adding it as profile_args + profile = TeradataUserPasswordProfileMapping(mock_teradata_conn.conn_id) + assert profile.profile["schema"] == "my_user" + + +def test_profile_mapping_keeps_port(mock_teradata_conn: Connection) -> None: + # port is not handled in airflow connection so adding it as profile_args + profile = TeradataUserPasswordProfileMapping(mock_teradata_conn.conn_id, profile_args={"port": 1025}) + assert profile.profile["port"] == 1025 + + +def test_profile_mapping_keeps_custom_tmode(mock_teradata_conn_custom_tmode: Connection) -> None: + profile = TeradataUserPasswordProfileMapping(mock_teradata_conn_custom_tmode.conn_id) + assert profile.profile["tmode"] == "TERA" + + +def test_profile_args( + mock_teradata_conn: Connection, +) -> None: + """ + Tests that the profile values get set correctly. + """ + profile_mapping = get_automatic_profile_mapping( + mock_teradata_conn.conn_id, + profile_args={"schema": "my_database"}, + ) + assert profile_mapping.profile_args == { + "schema": "my_database", + } + + assert profile_mapping.profile == { + "type": mock_teradata_conn.conn_type, + "host": mock_teradata_conn.host, + "user": mock_teradata_conn.login, + "password": "{{ env_var('COSMOS_CONN_TERADATA_PASSWORD') }}", + "schema": "my_database", + } + + +def test_profile_args_overrides( + mock_teradata_conn: Connection, +) -> None: + """ + Tests that you can override the profile values. + """ + profile_mapping = get_automatic_profile_mapping( + mock_teradata_conn.conn_id, + profile_args={"schema": "my_schema"}, + ) + assert profile_mapping.profile_args == { + "schema": "my_schema", + } + + assert profile_mapping.profile == { + "type": mock_teradata_conn.conn_type, + "host": mock_teradata_conn.host, + "user": mock_teradata_conn.login, + "password": "{{ env_var('COSMOS_CONN_TERADATA_PASSWORD') }}", + "schema": "my_schema", + } + + +def test_profile_env_vars( + mock_teradata_conn: Connection, +) -> None: + """ + Tests that the environment variables get set correctly. + """ + profile_mapping = get_automatic_profile_mapping( + mock_teradata_conn.conn_id, + profile_args={"schema": "my_schema"}, + ) + assert profile_mapping.env_vars == { + "COSMOS_CONN_TERADATA_PASSWORD": mock_teradata_conn.password, + } + + +def test_mock_profile() -> None: + """ + Tests that the mock profile port value get set correctly. + """ + profile = TeradataUserPasswordProfileMapping("mock_conn_id") + assert profile.mock_profile.get("host") == "mock_value" + assert profile.mock_profile.get("user") == "mock_value" + assert profile.mock_profile.get("password") == "mock_value" + assert profile.mock_profile.get("schema") == "mock_value" diff --git a/tests/profiles/test_base_profile.py b/tests/profiles/test_base_profile.py new file mode 100644 index 0000000000..8eeb83537d --- /dev/null +++ b/tests/profiles/test_base_profile.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +from typing import Any + +import pytest +import yaml +from pydantic.error_wrappers import ValidationError + +from cosmos.exceptions import CosmosValueError +from cosmos.profiles.base import BaseProfileMapping, DbtProfileConfigVars + + +class TestProfileMapping(BaseProfileMapping): + dbt_profile_method: str = "fake-method" + dbt_profile_type: str = "fake-type" + + @property + def profile(self) -> dict[str, str]: + return {"some-profile-key": "some-profile-value"} + + +@pytest.mark.parametrize("profile_arg", ["type", "method"]) +def test_validate_profile_args(profile_arg: str): + """ + An error should be raised if the profile_args contains a key that should not be overridden from the class variables. + """ + profile_args = {profile_arg: "fake-value"} + dbt_profile_value = getattr(TestProfileMapping, f"dbt_profile_{profile_arg}") + + expected_cosmos_error = ( + f"`profile_args` for TestProfileMapping has {profile_arg}='fake-value' that will override the dbt profile required value of " + f"'{dbt_profile_value}'. To fix this, remove {profile_arg} from `profile_args`." + ) + + with pytest.raises(CosmosValueError, match=expected_cosmos_error): + TestProfileMapping( + conn_id="fake_conn_id", + profile_args=profile_args, + ) + + +@pytest.mark.parametrize("disable_event_tracking", [True, False]) +def test_disable_event_tracking(disable_event_tracking: bool): + """ + Tests the config block in the profile is set correctly if disable_event_tracking is set. + """ + test_profile = TestProfileMapping( + conn_id="fake_conn_id", + disable_event_tracking=disable_event_tracking, + ) + profile_contents = yaml.safe_load(test_profile.get_profile_file_contents(profile_name="fake-profile-name")) + + assert ("config" in profile_contents) == disable_event_tracking + if disable_event_tracking: + assert profile_contents["config"]["send_anonymous_usage_stats"] is False + + +def test_disable_event_tracking_and_send_anonymous_usage_stats(): + expected_cosmos_error = ( + "Cannot set both disable_event_tracking and " + "dbt_config_vars=DbtProfileConfigVars(send_anonymous_usage_stats ..." + ) + + with pytest.raises(CosmosValueError) as err_info: + TestProfileMapping( + conn_id="fake_conn_id", + dbt_config_vars=DbtProfileConfigVars(send_anonymous_usage_stats=False), + disable_event_tracking=True, + ) + assert err_info.value.args[0] == expected_cosmos_error + + +def test_dbt_profile_config_vars_none(): + """ + Tests the DbtProfileConfigVars return None. + """ + dbt_config_vars = DbtProfileConfigVars( + send_anonymous_usage_stats=None, + partial_parse=None, + use_experimental_parser=None, + static_parser=None, + printer_width=None, + write_json=None, + warn_error=None, + warn_error_options=None, + log_format=None, + debug=None, + version_check=None, + ) + assert dbt_config_vars.as_dict() is None + + +@pytest.mark.parametrize("config", [True, False]) +def test_dbt_config_vars_config(config: bool): + """ + Tests the config block in the profile is set correctly. + """ + + dbt_config_vars = None + if config: + dbt_config_vars = DbtProfileConfigVars(debug=False) + + test_profile = TestProfileMapping( + conn_id="fake_conn_id", + dbt_config_vars=dbt_config_vars, + ) + profile_contents = yaml.safe_load(test_profile.get_profile_file_contents(profile_name="fake-profile-name")) + + assert ("config" in profile_contents) == config + + +@pytest.mark.parametrize("dbt_config_var,dbt_config_value", [("debug", True), ("debug", False)]) +def test_validate_dbt_config_vars(dbt_config_var: str, dbt_config_value: Any): + """ + Tests the config block in the profile is set correctly. + """ + dbt_config_vars = {dbt_config_var: dbt_config_value} + test_profile = TestProfileMapping( + conn_id="fake_conn_id", + dbt_config_vars=DbtProfileConfigVars(**dbt_config_vars), + ) + profile_contents = yaml.safe_load(test_profile.get_profile_file_contents(profile_name="fake-profile-name")) + + assert "config" in profile_contents + assert profile_contents["config"][dbt_config_var] == dbt_config_value + + +@pytest.mark.parametrize( + "dbt_config_var,dbt_config_value", + [("send_anonymous_usage_stats", 2), ("send_anonymous_usage_stats", "aaa")], +) +def test_profile_config_validate_dbt_config_vars_check_unexpected_types(dbt_config_var: str, dbt_config_value: Any): + dbt_config_vars = {dbt_config_var: dbt_config_value} + + with pytest.raises(ValidationError): + TestProfileMapping( + conn_id="fake_conn_id", + dbt_config_vars=DbtProfileConfigVars(**dbt_config_vars), + ) + + +@pytest.mark.parametrize("dbt_config_var,dbt_config_value", [("send_anonymous_usage_stats", True)]) +def test_profile_config_validate_dbt_config_vars_check_expected_types(dbt_config_var: str, dbt_config_value: Any): + dbt_config_vars = {dbt_config_var: dbt_config_value} + + profile_config = TestProfileMapping( + conn_id="fake_conn_id", + dbt_config_vars=DbtProfileConfigVars(**dbt_config_vars), + ) + assert profile_config.dbt_config_vars.as_dict() == dbt_config_vars + + +@pytest.mark.parametrize( + "dbt_config_var,dbt_config_value", + [("log_format", "text2")], +) +def test_profile_config_validate_dbt_config_vars_check_values(dbt_config_var: str, dbt_config_value: Any): + dbt_config_vars = {dbt_config_var: dbt_config_value} + + with pytest.raises(ValidationError): + TestProfileMapping( + conn_id="fake_conn_id", + dbt_config_vars=DbtProfileConfigVars(**dbt_config_vars), + ) + + +def test_profile_version_sha_consistency(): + profile_mapping = TestProfileMapping(conn_id="fake_conn_id") + version = profile_mapping.version(profile_name="dev", target_name="dev") + assert version == "ea3bf1f70b033405ba9ff9cafe65af873fd7a868cac840cdbfd5e8e9a1da9650" diff --git a/tests/profiles/trino/test_trino_base.py b/tests/profiles/trino/test_trino_base.py index 29e4026914..ee03821e2f 100644 --- a/tests/profiles/trino/test_trino_base.py +++ b/tests/profiles/trino/test_trino_base.py @@ -1,7 +1,7 @@ -"Tests for the Trino profile." +"""Tests for the Trino profile.""" -from unittest.mock import patch import json +from unittest.mock import patch from airflow.models.connection import Connection @@ -40,6 +40,7 @@ def test_profile_args() -> None: "schema": "my_schema", "host": "my_host", "port": 8080, + "user": "my_login", "session_properties": {"my_property": "my_value"}, } @@ -80,5 +81,6 @@ def test_profile_args_overrides() -> None: "schema": "my_schema", "host": "my_host_override", "port": 8080, + "user": "my_login", "session_properties": {"my_property": "my_value_override"}, } diff --git a/tests/profiles/trino/test_trino_certificate.py b/tests/profiles/trino/test_trino_certificate.py index 4ab3589e81..81728c32d7 100644 --- a/tests/profiles/trino/test_trino_certificate.py +++ b/tests/profiles/trino/test_trino_certificate.py @@ -1,4 +1,4 @@ -"Tests for the Trino profile." +"""Tests for the Trino profile.""" import json from unittest.mock import patch diff --git a/tests/profiles/trino/test_trino_jwt.py b/tests/profiles/trino/test_trino_jwt.py index d62fefc471..b886120f23 100644 --- a/tests/profiles/trino/test_trino_jwt.py +++ b/tests/profiles/trino/test_trino_jwt.py @@ -1,4 +1,4 @@ -"Tests for the Trino profile." +"""Tests for the Trino profile.""" import json from unittest.mock import patch @@ -6,8 +6,7 @@ import pytest from airflow.models.connection import Connection -from cosmos.profiles import get_automatic_profile_mapping -from cosmos.profiles import TrinoJWTProfileMapping +from cosmos.profiles import TrinoJWTProfileMapping, get_automatic_profile_mapping @pytest.fixture() diff --git a/tests/profiles/trino/test_trino_ldap.py b/tests/profiles/trino/test_trino_ldap.py index a959e9e68a..98bb2a642c 100644 --- a/tests/profiles/trino/test_trino_ldap.py +++ b/tests/profiles/trino/test_trino_ldap.py @@ -1,12 +1,11 @@ -"Tests for the Trino profile." +"""Tests for the Trino profile.""" from unittest.mock import patch import pytest from airflow.models.connection import Connection -from cosmos.profiles import get_automatic_profile_mapping -from cosmos.profiles import TrinoLDAPProfileMapping +from cosmos.profiles import TrinoLDAPProfileMapping, get_automatic_profile_mapping @pytest.fixture() diff --git a/tests/profiles/vertica/test_vertica_user_pass.py b/tests/profiles/vertica/test_vertica_user_pass.py index 953a3c553f..cae259dffe 100644 --- a/tests/profiles/vertica/test_vertica_user_pass.py +++ b/tests/profiles/vertica/test_vertica_user_pass.py @@ -1,4 +1,4 @@ -"Tests for the vertica profile." +"""Tests for the vertica profile.""" from unittest.mock import patch @@ -23,8 +23,7 @@ def mock_vertica_conn(): # type: ignore login="my_user", password="my_password", port=5433, - schema="my_schema", - extra='{"database": "my_database"}', + schema="my_database", ) with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): @@ -43,8 +42,7 @@ def mock_vertica_conn_custom_port(): # type: ignore login="my_user", password="my_password", port=7472, - schema="my_schema", - extra='{"database": "my_database"}', + schema="my_database", ) with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): @@ -59,7 +57,7 @@ def test_connection_claiming() -> None: # - conn_type == vertica # and the following exist: # - host - # - user + # - username # - password # - port # - database or database @@ -69,8 +67,7 @@ def test_connection_claiming() -> None: "host": "my_host", "login": "my_user", "password": "my_password", - "schema": "my_schema", - "extra": '{"database": "my_database"}', + "schema": "my_database", } # if we're missing any of the values, it shouldn't claim @@ -82,20 +79,20 @@ def test_connection_claiming() -> None: print("testing with", values) with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): - profile_mapping = VerticaUserPasswordProfileMapping(conn) + profile_mapping = VerticaUserPasswordProfileMapping(conn, {"schema": "my_schema"}) assert not profile_mapping.can_claim_connection() - # also test when there's no database + # also test when there's no schema conn = Connection(**potential_values) # type: ignore conn.extra = "" with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): - profile_mapping = VerticaUserPasswordProfileMapping(conn) + profile_mapping = VerticaUserPasswordProfileMapping(conn, {}) assert not profile_mapping.can_claim_connection() # if we have them all, it should claim conn = Connection(**potential_values) # type: ignore with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): - profile_mapping = VerticaUserPasswordProfileMapping(conn) + profile_mapping = VerticaUserPasswordProfileMapping(conn, {"schema": "my_schema"}) assert profile_mapping.can_claim_connection() @@ -107,7 +104,7 @@ def test_profile_mapping_selected( """ profile_mapping = get_automatic_profile_mapping( mock_vertica_conn.conn_id, - {"schema": "my_schema"}, + {"schema": "my_database"}, ) assert isinstance(profile_mapping, VerticaUserPasswordProfileMapping) @@ -142,11 +139,11 @@ def test_profile_args( assert profile_mapping.profile == { "type": mock_vertica_conn.conn_type, "host": mock_vertica_conn.host, - "user": mock_vertica_conn.login, + "username": mock_vertica_conn.login, "password": "{{ env_var('COSMOS_CONN_VERTICA_PASSWORD') }}", "port": mock_vertica_conn.port, + "database": mock_vertica_conn.schema, "schema": "my_schema", - "database": mock_vertica_conn.extra_dejson.get("database"), } @@ -168,7 +165,7 @@ def test_profile_args_overrides( assert profile_mapping.profile == { "type": mock_vertica_conn.conn_type, "host": mock_vertica_conn.host, - "user": mock_vertica_conn.login, + "username": mock_vertica_conn.login, "password": "{{ env_var('COSMOS_CONN_VERTICA_PASSWORD') }}", "port": mock_vertica_conn.port, "database": "my_db_override", diff --git a/tests/sample/manifest_source.json b/tests/sample/manifest_source.json index 5326e2ab28..67f57035ef 100644 --- a/tests/sample/manifest_source.json +++ b/tests/sample/manifest_source.json @@ -7,9 +7,11 @@ "model.simple.top_animations": [ "exposure.simple.weekly_metrics" ], - "source.simple.imdb.movies_ratings": [ - "model.simple.movies_ratings_simplified" - ] + "source.simple.main.movies_ratings": [ + "model.simple.movies_ratings_simplified", + "test.simple.source_not_null_imdb_movies_ratings_X.e684bf90f4" + ], + "test.simple.source_not_null_imdb_movies_ratings_X.e684bf90f4": [] }, "disabled": {}, "docs": { @@ -28,7 +30,7 @@ "config": { "enabled": true }, - "created_at": 1696859933.549042, + "created_at": 1697205180.995924, "depends_on": { "macros": [], "nodes": [ @@ -1016,7 +1018,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", + "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partiton start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "meta": {}, "name": "dates_in_range", "original_file_path": "macros/etc/datetime.sql", @@ -6945,8 +6947,8 @@ "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v8.json", "dbt_version": "1.4.0", "env": {}, - "generated_at": "2023-10-09T13:58:53.320926Z", - "invocation_id": "1790dc18-1177-4ca0-b993-a8eeb59f0c4c", + "generated_at": "2023-10-13T13:58:46.591195Z", + "invocation_id": "4f4b6d15-b4ab-4683-bbe5-efd94824b1d9", "project_id": "8dbdda48fb8748d6746f1965824e966a", "send_anonymous_usage_stats": true, "user_id": "4bdc3972-5c9f-4f16-90bd-3769a225fbe6" @@ -6986,13 +6988,13 @@ "tags": [], "unique_key": null }, - "created_at": 1696859933.5317938, + "created_at": 1697205180.9909241, "database": "database", "deferred": false, "depends_on": { "macros": [], "nodes": [ - "source.simple.imdb.movies_ratings" + "source.simple.main.movies_ratings" ] }, "description": "", @@ -7062,7 +7064,7 @@ "tags": [], "unique_key": null }, - "created_at": 1696859933.537527, + "created_at": 1697205180.984051, "database": "database", "deferred": false, "depends_on": { @@ -7103,6 +7105,84 @@ "unrendered_config": { "materialized": "table" } + }, + "test.simple.source_not_null_imdb_movies_ratings_X.e684bf90f4": { + "alias": "source_not_null_imdb_movies_ratings_X", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "X", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "created_at": 1697205181.009168, + "database": "database", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "source.simple.main.movies_ratings" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "sources.imdb", + "fqn": [ + "simple", + "source_not_null_imdb_movies_ratings_X" + ], + "language": "sql", + "meta": {}, + "metrics": [], + "name": "source_not_null_imdb_movies_ratings_X", + "original_file_path": "models/source.yml", + "package_name": "simple", + "patch_path": null, + "path": "source_not_null_imdb_movies_ratings_X.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [], + "relation_name": null, + "resource_type": "test", + "schema": "main_dbt_test__audit", + "sources": [ + [ + "imdb", + "movies_ratings" + ] + ], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "X", + "model": "{{ get_where_subquery(source('imdb', 'movies_ratings')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.simple.source_not_null_imdb_movies_ratings_X.e684bf90f4", + "unrendered_config": {} } }, "parent_map": { @@ -7110,23 +7190,35 @@ "model.simple.top_animations" ], "model.simple.movies_ratings_simplified": [ - "source.simple.imdb.movies_ratings" + "source.simple.main.movies_ratings" ], "model.simple.top_animations": [ "model.simple.movies_ratings_simplified" ], - "source.simple.imdb.movies_ratings": [] + "source.simple.main.movies_ratings": [], + "test.simple.source_not_null_imdb_movies_ratings_X.e684bf90f4": [ + "source.simple.main.movies_ratings" + ] }, "selectors": {}, "sources": { - "source.simple.imdb.movies_ratings": { - "columns": {}, + "source.simple.main.movies_ratings": { + "columns": { + "X": { + "data_type": null, + "description": "", + "meta": {}, + "name": "X", + "quote": null, + "tags": [] + } + }, "config": { "enabled": true }, - "created_at": 1696859933.549542, + "created_at": 1697205181.0098429, "database": "database", - "description": "Ratings by movie", + "description": "Ratings by movie\n", "external": null, "fqn": [ "simple", @@ -7166,7 +7258,7 @@ "source_meta": {}, "source_name": "imdb", "tags": [], - "unique_id": "source.simple.imdb.movies_ratings", + "unique_id": "source.simple.main.movies_ratings", "unrendered_config": {} } } diff --git a/tests/sample/partial_parse.msgpack b/tests/sample/partial_parse.msgpack new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/sample/sample_dbt_ls.txt b/tests/sample/sample_dbt_ls.txt new file mode 100644 index 0000000000..b356a5208c --- /dev/null +++ b/tests/sample/sample_dbt_ls.txt @@ -0,0 +1,6 @@ +14:26:04 Running with dbt=1.6.9 +14:26:04 Registered adapter: exasol=1.6.2 +14:26:04 Found 5 models, 3 seeds, 20 tests, 0 sources, 0 exposures, 0 metrics, 366 macros, 0 groups, 0 semantic models +{"name": "stg_customers", "resource_type": "model", "package_name": "jaffle_shop", "original_file_path": "models/staging/stg_customers.sql", "unique_id": "model.jaffle_shop.stg_customers", "alias": "stg_customers", "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "post-hook": [], "pre-hook": []}, "tags": [], "depends_on": {"macros": [], "nodes": ["seed.jaffle_shop.raw_customers"]}} +{"name": "stg_orders", "resource_type": "model", "package_name": "jaffle_shop", "original_file_path": "models/staging/stg_orders.sql", "unique_id": "model.jaffle_shop.stg_orders", "alias": "stg_orders", "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "post-hook": [], "pre-hook": []}, "tags": [], "depends_on": {"macros": [], "nodes": ["seed.jaffle_shop.raw_orders"]}} +{"name": "stg_payments", "resource_type": "model", "package_name": "jaffle_shop", "original_file_path": "models/staging/stg_payments.sql", "unique_id": "model.jaffle_shop.stg_payments", "alias": "stg_payments", "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "post-hook": [], "pre-hook": []}, "tags": [], "depends_on": {"macros": [], "nodes": ["seed.jaffle_shop.raw_payments"]}} diff --git a/tests/test_cache.py b/tests/test_cache.py new file mode 100644 index 0000000000..738890bcb0 --- /dev/null +++ b/tests/test_cache.py @@ -0,0 +1,286 @@ +import logging +import shutil +import tempfile +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from unittest.mock import call, patch + +import pytest +from airflow import DAG +from airflow.models import DagRun, Variable +from airflow.utils.db import create_session +from airflow.utils.task_group import TaskGroup + +from cosmos.cache import ( + _copy_partial_parse_to_project, + _create_cache_identifier, + _get_latest_partial_parse, + _get_or_create_profile_cache_dir, + _update_partial_parse_cache, + create_cache_profile, + delete_unused_dbt_ls_cache, + get_cached_profile, + is_profile_cache_enabled, +) +from cosmos.constants import DBT_PARTIAL_PARSE_FILE_NAME, DBT_TARGET_DIR_NAME, DEFAULT_PROFILES_FILE_NAME +from cosmos.settings import dbt_profile_cache_dir_name + +START_DATE = datetime(2024, 4, 16) +example_dag = DAG("dag", start_date=START_DATE) +SAMPLE_PARTIAL_PARSE_FILEPATH = Path(__file__).parent / "sample/partial_parse.msgpack" + + +@pytest.mark.parametrize( + "dag, task_group, result_identifier", + [ + (example_dag, None, "dag"), + (None, TaskGroup(dag=example_dag, group_id="inner_tg"), "dag__inner_tg"), + ( + None, + TaskGroup( + dag=example_dag, group_id="child_tg", parent_group=TaskGroup(dag=example_dag, group_id="parent_tg") + ), + "dag__parent_tg__child_tg", + ), + ( + None, + TaskGroup( + dag=example_dag, + group_id="child_tg", + parent_group=TaskGroup( + dag=example_dag, group_id="mum_tg", parent_group=TaskGroup(dag=example_dag, group_id="nana_tg") + ), + ), + "dag__nana_tg__mum_tg__child_tg", + ), + ], +) +def test_create_cache_identifier(dag, task_group, result_identifier): + assert _create_cache_identifier(dag, task_group) == result_identifier + + +def test_get_latest_partial_parse(tmp_path): + old_tmp_dir = tmp_path / "old" + old_tmp_target_dir = old_tmp_dir / DBT_TARGET_DIR_NAME + old_tmp_target_dir.mkdir(parents=True, exist_ok=True) + old_partial_parse_filepath = old_tmp_target_dir / DBT_PARTIAL_PARSE_FILE_NAME + old_partial_parse_filepath.touch() + + # This is necessary in the CI, but not on local MacOS dev env, since the files + # were being created too quickly and sometimes had the same st_mtime + time.sleep(1) + + new_tmp_dir = tmp_path / "new" + new_tmp_target_dir = new_tmp_dir / DBT_TARGET_DIR_NAME + new_tmp_target_dir.mkdir(parents=True, exist_ok=True) + new_partial_parse_filepath = new_tmp_target_dir / DBT_PARTIAL_PARSE_FILE_NAME + new_partial_parse_filepath.touch() + + assert _get_latest_partial_parse(old_tmp_dir, new_tmp_dir) == new_partial_parse_filepath + assert _get_latest_partial_parse(new_tmp_dir, old_tmp_dir) == new_partial_parse_filepath + assert _get_latest_partial_parse(old_tmp_dir, old_tmp_dir) == old_partial_parse_filepath + assert _get_latest_partial_parse(old_tmp_dir, tmp_path) == old_partial_parse_filepath + assert _get_latest_partial_parse(tmp_path, old_tmp_dir) == old_partial_parse_filepath + assert _get_latest_partial_parse(tmp_path, tmp_path) is None + + +@patch("cosmos.cache.msgpack.unpack", side_effect=ValueError) +def test__copy_partial_parse_to_project_msg_fails_msgpack(mock_unpack, tmp_path, caplog): + caplog.set_level(logging.INFO) + source_dir = tmp_path / DBT_TARGET_DIR_NAME + source_dir.mkdir() + partial_parse_filepath = source_dir / DBT_PARTIAL_PARSE_FILE_NAME + shutil.copy(str(SAMPLE_PARTIAL_PARSE_FILEPATH), str(partial_parse_filepath)) + + # actual test + with tempfile.TemporaryDirectory() as tmp_dir: + _copy_partial_parse_to_project(partial_parse_filepath, Path(tmp_dir)) + + assert "Unable to patch the partial_parse.msgpack file due to ValueError()" in caplog.text + + +@patch("cosmos.cache.shutil.copyfile") +@patch("cosmos.cache.get_partial_parse_path") +def test_update_partial_parse_cache(mock_get_partial_parse_path, mock_copyfile): + mock_get_partial_parse_path.side_effect = lambda cache_dir: cache_dir / "partial_parse.yml" + + latest_partial_parse_filepath = Path("/path/to/latest_partial_parse.yml") + cache_dir = Path("/tmp/path/to/cache_directory") + + # Expected paths + cache_path = cache_dir / "partial_parse.yml" + manifest_path = cache_dir / "manifest.json" + + _update_partial_parse_cache(latest_partial_parse_filepath, cache_dir) + + # Assert shutil.copyfile was called twice with the correct arguments + calls = [ + call(str(latest_partial_parse_filepath), str(cache_path)), + call(str(latest_partial_parse_filepath.parent / "manifest.json"), str(manifest_path)), + ] + mock_copyfile.assert_has_calls(calls) + + +@pytest.fixture +def vars_session(): + with create_session() as session: + var1 = Variable(key="cosmos_cache__dag_a", val='{"dag_id": "dag_a"}') + var2 = Variable(key="cosmos_cache__dag_b", val='{"dag_id": "dag_b"}') + var3 = Variable(key="cosmos_cache__dag_c__task_group_1", val='{"dag_id": "dag_c"}') + + dag_run_a = DagRun( + dag_id="dag_a", + run_id="dag_a_run_a_week_ago", + execution_date=datetime.now(timezone.utc) - timedelta(days=7), + state="success", + run_type="manual", + ) + dag_run_b = DagRun( + dag_id="dag_b", + run_id="dag_b_run_yesterday", + execution_date=datetime.now(timezone.utc) - timedelta(days=1), + state="failed", + run_type="manual", + ) + dag_run_c = DagRun( + dag_id="dag_c", + run_id="dag_c_run_on_hour_ago", + execution_date=datetime.now(timezone.utc) - timedelta(hours=1), + state="running", + run_type="manual", + ) + + session.add(var1) + session.add(var2) + session.add(var3) + session.add(dag_run_a) + session.add(dag_run_b) + session.add(dag_run_c) + session.commit() + + yield session + + session.query(Variable).filter_by(key="cosmos_cache__dag_a").delete() + session.query(Variable).filter_by(key="cosmos_cache__dag_b").delete() + session.query(Variable).filter_by(key="cosmos_cache__dag_c__task_group_1").delete() + + session.query(DagRun).filter_by(dag_id="dag_a", run_id="dag_a_run_a_week_ago").delete() + session.query(DagRun).filter_by(dag_id="dag_b", run_id="dag_b_run_yesterday").delete() + session.query(DagRun).filter_by(dag_id="dag_c", run_id="dag_c_run_on_hour_ago").delete() + session.commit() + + +@pytest.mark.integration +def test_delete_unused_dbt_ls_cache_deletes_a_week_ago_cache(vars_session): + assert vars_session.query(Variable).filter_by(key="cosmos_cache__dag_a").first() + assert delete_unused_dbt_ls_cache(max_age_last_usage=timedelta(days=5), session=vars_session) == 1 + assert not vars_session.query(Variable).filter_by(key="cosmos_cache__dag_a").first() + + +@pytest.mark.integration +def test_delete_unused_dbt_ls_cache_deletes_all_cache_five_minutes_ago(vars_session): + assert vars_session.query(Variable).filter_by(key="cosmos_cache__dag_a").first() + assert vars_session.query(Variable).filter_by(key="cosmos_cache__dag_b").first() + assert vars_session.query(Variable).filter_by(key="cosmos_cache__dag_c__task_group_1").first() + assert delete_unused_dbt_ls_cache(max_age_last_usage=timedelta(minutes=5), session=vars_session) == 3 + assert not vars_session.query(Variable).filter_by(key="cosmos_cache__dag_a").first() + assert not vars_session.query(Variable).filter_by(key="cosmos_cache__dag_b").first() + assert not vars_session.query(Variable).filter_by(key="cosmos_cache__dag_c__task_group_1").first() + + +@pytest.mark.parametrize( + "enable_cache, enable_cache_profile, expected_result", + [(True, True, True), (True, False, False), (False, True, False), (False, False, False)], +) +def test_is_profile_cache_enabled(enable_cache, enable_cache_profile, expected_result): + with patch("cosmos.cache.enable_cache", enable_cache), patch( + "cosmos.cache.enable_cache_profile", enable_cache_profile + ): + assert is_profile_cache_enabled() == expected_result + + +def test_get_or_create_profile_cache_dir(): + # Create a temporary directory for cache_dir + with tempfile.TemporaryDirectory() as temp_dir: + temp_dir_path = Path(temp_dir) + + # Test case 1: Directory does not exist, should create it + with patch("cosmos.cache.cache_dir", temp_dir_path): + profile_cache_dir = _get_or_create_profile_cache_dir() + expected_dir = temp_dir_path / dbt_profile_cache_dir_name + assert profile_cache_dir == expected_dir + assert expected_dir.exists() + + # Test case 2: Directory already exists, should return existing path + with patch("cosmos.cache.cache_dir", temp_dir_path): + profile_cache_dir_again = _get_or_create_profile_cache_dir() + expected_dir = temp_dir_path / dbt_profile_cache_dir_name + assert profile_cache_dir_again == expected_dir + assert expected_dir.exists() + + +def test_get_cached_profile_not_exists(): + with tempfile.TemporaryDirectory() as temp_dir: + temp_dir = Path(temp_dir) + # Mock cache_dir to use the temporary directory + with patch("cosmos.cache.cache_dir", temp_dir): + # Create a dummy profile YAML file for version 'v1' + version = "592906f650558ce1dadb75fcce84a2ec09e444441e6af6069f19204d59fe428b" + result = get_cached_profile(version) + assert result is None + + +def test_get_cached_profile(): + profile_content = """ + default: + target: dev + outputs: + dev: + type: postgres + host: localhost + user: myuser + pass: mypassword + dbname: mydatabase + """ + with tempfile.TemporaryDirectory() as temp_dir: + temp_dir = Path(temp_dir) + with patch("cosmos.cache.cache_dir", temp_dir): + # Setup DBT profile + version = "592906f650558ce1dadb75fcce84a2ec09e444441e6af6069f19204d59fe428b" + create_cache_profile(version, profile_content) + + expected_yml_path = temp_dir / dbt_profile_cache_dir_name / version / DEFAULT_PROFILES_FILE_NAME + result = get_cached_profile(version) + assert result == expected_yml_path + + +def test_create_cache_profile(): + version = "592906f650558ce1dadb75fcce84a2ec09e444441e6af6069f19204d59fe428b" + profile_content = """ + default: + target: dev + outputs: + dev: + type: postgres + host: localhost + user: myuser + pass: mypassword + dbname: mydatabase + """ + + with tempfile.TemporaryDirectory() as temp_dir: + temp_dir = Path(temp_dir) + with patch("cosmos.cache.cache_dir", temp_dir): + profile_yml_path = create_cache_profile(version, profile_content) + + expected_dir = temp_dir / dbt_profile_cache_dir_name / version + expected_path = expected_dir / DEFAULT_PROFILES_FILE_NAME + + # Check if the directory and file were created + assert expected_dir.exists() + assert expected_path.exists() + + # Check content of the created file + assert expected_path.read_text() == profile_content + assert profile_yml_path == expected_path diff --git a/tests/test_config.py b/tests/test_config.py index 9eec48055d..5bf0f69b56 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,12 +1,17 @@ +from contextlib import nullcontext as does_not_raise from pathlib import Path +from unittest.mock import Mock, PropertyMock, call, patch import pytest -from cosmos.config import ProfileConfig, ProjectConfig +from cosmos.config import CosmosConfigException, ExecutionConfig, ProfileConfig, ProjectConfig, RenderConfig +from cosmos.constants import ExecutionMode, InvocationMode from cosmos.exceptions import CosmosValueError - +from cosmos.profiles.athena.access_key import AthenaAccessKeyProfileMapping +from cosmos.profiles.postgres.user_pass import PostgresUserPasswordProfileMapping DBT_PROJECTS_ROOT_DIR = Path(__file__).parent / "sample/" +SAMPLE_PROFILE_YML = Path(__file__).parent / "sample/profiles.yml" PIPELINE_FOLDER = "jaffle_shop" @@ -110,14 +115,180 @@ def test_project_name(): assert dbt_project.project_name == "sample" -def test_profile_config_post_init(): +def test_profile_config_validate_none(): with pytest.raises(CosmosValueError) as err_info: - ProfileConfig(profiles_yml_filepath="/tmp/some-profile", profile_name="test", target_name="test") - assert err_info.value.args[0] == "The file /tmp/some-profile does not exist." + ProfileConfig(profile_name="test", target_name="test") + assert err_info.value.args[0] == "Either profiles_yml_filepath or profile_mapping must be set to render a profile" -def test_profile_config_validate(): +def test_profile_config_validate_both(): with pytest.raises(CosmosValueError) as err_info: - profile_config = ProfileConfig(profile_name="test", target_name="test") - assert profile_config.validate_profile() is None - assert err_info.value.args[0] == "Either profiles_yml_filepath or profile_mapping must be set to render a profile" + ProfileConfig( + profile_name="test", + target_name="test", + profiles_yml_filepath=SAMPLE_PROFILE_YML, + profile_mapping=PostgresUserPasswordProfileMapping(conn_id="test", profile_args={}), + ) + assert ( + err_info.value.args[0] + == "Both profiles_yml_filepath and profile_mapping are defined and are mutually exclusive. Ensure only one of these is defined." + ) + + +def test_profile_config_validate_profiles_yml(): + profile_config = ProfileConfig(profile_name="test", target_name="test", profiles_yml_filepath="/tmp/no-exists") + with pytest.raises(CosmosValueError) as err_info: + profile_config.validate_profiles_yml() + + assert err_info.value.args[0] == "The file /tmp/no-exists does not exist." + + +@patch("cosmos.config.is_profile_cache_enabled", return_value=False) +@patch("cosmos.profiles.athena.access_key.AthenaAccessKeyProfileMapping.env_vars", new_callable=PropertyMock) +@patch("cosmos.profiles.athena.access_key.AthenaAccessKeyProfileMapping.get_profile_file_contents") +@patch("cosmos.config.Path") +def test_profile_config_ensure_profile_without_caching_calls_get_profile_file_content_before_env_vars( + mock_path, mock_get_profile_file_contents, mock_env_vars, mock_cache +): + """ + The `env_vars` should not be called if profile file is not populated. + """ + profile_mapping = AthenaAccessKeyProfileMapping(conn_id="test", profile_args={}) + profile_config = ProfileConfig(profile_name="test", target_name="test", profile_mapping=profile_mapping) + mock_manager = Mock() + mock_manager.attach_mock(mock_get_profile_file_contents, "get_profile_file_contents") + mock_manager.attach_mock(mock_env_vars, "env_vars") + + with profile_config.ensure_profile(desired_profile_path=mock_path): + mock_get_profile_file_contents.assert_called_once() + mock_env_vars.assert_called_once() + expected_calls = [ + call.get_profile_file_contents(profile_name="test", target_name="test", use_mock_values=False), + call.env_vars, + ] + mock_manager.assert_has_calls(expected_calls, any_order=False) + + +@patch("cosmos.config.create_cache_profile") +@patch("cosmos.profiles.athena.access_key.AthenaAccessKeyProfileMapping.version") +@patch("cosmos.config.get_cached_profile", return_value=None) +@patch("cosmos.config.is_profile_cache_enabled", return_value=True) +@patch("cosmos.profiles.athena.access_key.AthenaAccessKeyProfileMapping.env_vars", new_callable=PropertyMock) +@patch("cosmos.profiles.athena.access_key.AthenaAccessKeyProfileMapping.get_profile_file_contents") +@patch("cosmos.config.Path") +def test_profile_config_ensure_profile_with_caching_calls_get_profile_file_content_before_env_vars( + mock_path, + mock_get_profile_file_contents, + mock_env_vars, + mock_cache, + mock_get_cached_profile, + mock_version, + mock_create_cache_profile, +): + """ + The `env_vars` should not be called if profile file is not populated. + """ + profile_mapping = AthenaAccessKeyProfileMapping(conn_id="test", profile_args={}) + profile_config = ProfileConfig(profile_name="test", target_name="test", profile_mapping=profile_mapping) + mock_manager = Mock() + mock_manager.attach_mock(mock_get_profile_file_contents, "get_profile_file_contents") + mock_manager.attach_mock(mock_env_vars, "env_vars") + + with profile_config.ensure_profile(desired_profile_path=mock_path): + mock_get_profile_file_contents.assert_called_once() + mock_env_vars.assert_called_once() + expected_calls = [ + call.get_profile_file_contents(profile_name="test", target_name="test", use_mock_values=False), + call.env_vars, + ] + mock_manager.assert_has_calls(expected_calls, any_order=False) + + +@patch("cosmos.config.shutil.which", return_value=None) +def test_render_config_without_dbt_cmd(mock_which): + render_config = RenderConfig() + with pytest.raises(CosmosConfigException) as err_info: + render_config.validate_dbt_command("inexistent-dbt") + + error_msg = err_info.value.args[0] + assert error_msg.startswith("Unable to find the dbt executable, attempted: <") + assert error_msg.endswith("dbt> and .") + + +@patch("cosmos.config.shutil.which", return_value=None) +def test_render_config_with_invalid_dbt_commands(mock_which): + render_config = RenderConfig(dbt_executable_path="invalid-dbt") + with pytest.raises(CosmosConfigException) as err_info: + render_config.validate_dbt_command() + + error_msg = err_info.value.args[0] + assert error_msg == "Unable to find the dbt executable, attempted: ." + + +@patch("cosmos.config.shutil.which", side_effect=(None, "fallback-dbt-path")) +def test_render_config_uses_fallback_if_default_not_found(mock_which): + render_config = RenderConfig() + render_config.validate_dbt_command(Path("/tmp/fallback-dbt-path")) + assert render_config.dbt_executable_path == "/tmp/fallback-dbt-path" + + +@patch("cosmos.config.shutil.which", side_effect=("user-dbt", "fallback-dbt-path")) +def test_render_config_uses_default_if_exists(mock_which): + render_config = RenderConfig(dbt_executable_path="user-dbt") + render_config.validate_dbt_command("fallback-dbt-path") + assert render_config.dbt_executable_path == "user-dbt" + + +def test_is_dbt_ls_file_available_is_true(): + render_config = RenderConfig(dbt_ls_path=DBT_PROJECTS_ROOT_DIR / "sample_dbt_ls.txt") + assert render_config.is_dbt_ls_file_available() + + +def test_is_dbt_ls_file_available_is_true_for_str_path(): + render_config = RenderConfig(dbt_ls_path=str(DBT_PROJECTS_ROOT_DIR / "sample_dbt_ls.txt")) + assert render_config.is_dbt_ls_file_available() + + +def test_is_dbt_ls_file_available_is_false(): + render_config = RenderConfig(dbt_ls_path=None) + assert not render_config.is_dbt_ls_file_available() + + +def test_render_config_env_vars_deprecated(): + """RenderConfig.env_vars is deprecated since Cosmos 1.3, should warn user.""" + with pytest.deprecated_call(): + RenderConfig(env_vars={"VAR": "value"}) + + +@pytest.mark.parametrize( + "execution_mode, invocation_mode, expectation", + [ + (ExecutionMode.LOCAL, InvocationMode.DBT_RUNNER, does_not_raise()), + (ExecutionMode.LOCAL, InvocationMode.SUBPROCESS, does_not_raise()), + (ExecutionMode.LOCAL, None, does_not_raise()), + (ExecutionMode.VIRTUALENV, InvocationMode.DBT_RUNNER, pytest.raises(CosmosValueError)), + (ExecutionMode.VIRTUALENV, InvocationMode.SUBPROCESS, does_not_raise()), + (ExecutionMode.VIRTUALENV, None, does_not_raise()), + (ExecutionMode.KUBERNETES, InvocationMode.DBT_RUNNER, pytest.raises(CosmosValueError)), + (ExecutionMode.DOCKER, InvocationMode.DBT_RUNNER, pytest.raises(CosmosValueError)), + (ExecutionMode.AZURE_CONTAINER_INSTANCE, InvocationMode.DBT_RUNNER, pytest.raises(CosmosValueError)), + ], +) +def test_execution_config_with_invocation_option(execution_mode, invocation_mode, expectation): + with expectation: + ExecutionConfig(execution_mode=execution_mode, invocation_mode=invocation_mode) + + +@pytest.mark.parametrize( + "execution_mode, expected_invocation_mode", + [ + (ExecutionMode.LOCAL, None), + (ExecutionMode.VIRTUALENV, InvocationMode.SUBPROCESS), + (ExecutionMode.KUBERNETES, None), + (ExecutionMode.DOCKER, None), + (ExecutionMode.AZURE_CONTAINER_INSTANCE, None), + ], +) +def test_execution_config_default_config(execution_mode, expected_invocation_mode): + execution_config = ExecutionConfig(execution_mode=execution_mode) + assert execution_config.invocation_mode == expected_invocation_mode diff --git a/tests/test_converter.py b/tests/test_converter.py index 5d89513b31..bef2dc06d4 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -1,14 +1,17 @@ +import tempfile +from datetime import datetime from pathlib import Path +from unittest.mock import MagicMock, patch -from unittest.mock import patch import pytest +from airflow.models import DAG -from cosmos.converter import DbtToAirflowConverter, validate_arguments -from cosmos.constants import DbtResourceType, ExecutionMode -from cosmos.config import ProjectConfig, ProfileConfig, ExecutionConfig, RenderConfig -from cosmos.dbt.graph import DbtNode +from cosmos.config import CosmosConfigException, ExecutionConfig, ProfileConfig, ProjectConfig, RenderConfig +from cosmos.constants import DbtResourceType, ExecutionMode, InvocationMode, LoadMode +from cosmos.converter import DbtToAirflowConverter, validate_arguments, validate_initial_user_config +from cosmos.dbt.graph import DbtGraph, DbtNode from cosmos.exceptions import CosmosValueError - +from cosmos.profiles.postgres import PostgresUserPasswordProfileMapping SAMPLE_PROFILE_YML = Path(__file__).parent / "sample/profiles.yml" SAMPLE_DBT_PROJECT = Path(__file__).parent / "sample/" @@ -20,17 +23,106 @@ def test_validate_arguments_tags(argument_key): selector_name = argument_key[:-1] select = [f"{selector_name}:a,{selector_name}:b"] exclude = [f"{selector_name}:b,{selector_name}:c"] - profile_args = {} + profile_config = ProfileConfig( + profile_name="test", + target_name="test", + profile_mapping=PostgresUserPasswordProfileMapping(conn_id="test", profile_args={}), + ) task_args = {} with pytest.raises(CosmosValueError) as err: - validate_arguments(select, exclude, profile_args, task_args) + validate_arguments(select, exclude, profile_config, task_args, execution_mode=ExecutionMode.LOCAL) expected = f"Can't specify the same {selector_name} in `select` and `exclude`: {{'b'}}" assert err.value.args[0] == expected +@pytest.mark.parametrize( + "execution_mode", + (ExecutionMode.LOCAL, ExecutionMode.VIRTUALENV), +) +def test_validate_initial_user_config_no_profile(execution_mode): + execution_config = ExecutionConfig(execution_mode=execution_mode) + profile_config = None + project_config = ProjectConfig() + with pytest.raises(CosmosValueError) as err_info: + validate_initial_user_config(execution_config, profile_config, project_config, None, {}) + err_msg = f"The profile_config is mandatory when using {execution_mode}" + assert err_info.value.args[0] == err_msg + + +@pytest.mark.parametrize( + "execution_mode", + (ExecutionMode.DOCKER, ExecutionMode.KUBERNETES), +) +def test_validate_initial_user_config_expects_profile(execution_mode): + execution_config = ExecutionConfig(execution_mode=execution_mode) + profile_config = None + project_config = ProjectConfig() + assert validate_initial_user_config(execution_config, profile_config, project_config, None, {}) is None + + +@pytest.mark.parametrize("operator_args", [{"env": {"key": "value"}}, {"vars": {"key": "value"}}]) +def test_validate_user_config_operator_args_deprecated(operator_args): + """Deprecating warnings should be raised when using operator_args with "vars" or "env".""" + project_config = ProjectConfig() + execution_config = ExecutionConfig() + render_config = RenderConfig() + profile_config = MagicMock() + + with pytest.deprecated_call(): + validate_initial_user_config(execution_config, profile_config, project_config, render_config, operator_args) + + +@pytest.mark.parametrize("project_config_arg, operator_arg", [("dbt_vars", "vars"), ("env_vars", "env")]) +def test_validate_user_config_fails_project_config_and_operator_args_overlap(project_config_arg, operator_arg): + """ + The validation should fail if a user specifies both a ProjectConfig and operator_args with dbt_vars/vars or env_vars/env + that overlap. + """ + project_config = ProjectConfig( + project_name="fake-project", + dbt_project_path="/some/project/path", + **{project_config_arg: {"key": "value"}}, # type: ignore + ) + execution_config = ExecutionConfig() + render_config = RenderConfig() + profile_config = MagicMock() + operator_args = {operator_arg: {"key": "value"}} + + expected_error_msg = f"ProjectConfig.{project_config_arg} and operator_args with '{operator_arg}' are mutually exclusive and only one can be used." + with pytest.raises(CosmosValueError, match=expected_error_msg): + validate_initial_user_config(execution_config, profile_config, project_config, render_config, operator_args) + + +def test_validate_user_config_fails_project_config_render_config_env_vars(): + """ + The validation should fail if a user specifies both ProjectConfig.env_vars and RenderConfig.env_vars. + """ + project_config = ProjectConfig(env_vars={"key": "value"}) + execution_config = ExecutionConfig() + render_config = RenderConfig(env_vars={"key": "value"}) + profile_config = MagicMock() + operator_args = {} + + expected_error_match = "Both ProjectConfig.env_vars and RenderConfig.env_vars were provided.*" + with pytest.raises(CosmosValueError, match=expected_error_match): + validate_initial_user_config(execution_config, profile_config, project_config, render_config, operator_args) + + +def test_validate_arguments_schema_in_task_args(): + profile_config = ProfileConfig( + profile_name="test", + target_name="test", + profile_mapping=PostgresUserPasswordProfileMapping(conn_id="test", profile_args={}), + ) + task_args = {"schema": "abcd"} + validate_arguments( + select=[], exclude=[], profile_config=profile_config, task_args=task_args, execution_mode=ExecutionMode.LOCAL + ) + assert profile_config.profile_mapping.profile_args["schema"] == "abcd" + + parent_seed = DbtNode( - name="seed_parent", - unique_id="seed_parent", + unique_id=f"{DbtResourceType.SEED}.{SAMPLE_DBT_PROJECT.stem}.seed_parent", resource_type=DbtResourceType.SEED, depends_on=[], file_path="", @@ -61,6 +153,7 @@ def test_converter_creates_dag_with_seed(mock_load_dbt_graph, execution_mode, op profiles_yml_filepath=SAMPLE_PROFILE_YML, ) converter = DbtToAirflowConverter( + dag=DAG("sample_dag", start_date=datetime(2024, 4, 16)), nodes=nodes, project_config=project_config, profile_config=profile_config, @@ -94,6 +187,7 @@ def test_converter_creates_dag_with_project_path_str(mock_load_dbt_graph, execut profiles_yml_filepath=SAMPLE_PROFILE_YML, ) converter = DbtToAirflowConverter( + dag=DAG("sample_dag", start_date=datetime(2024, 4, 16)), nodes=nodes, project_config=project_config, profile_config=profile_config, @@ -141,6 +235,74 @@ def test_converter_fails_execution_config_no_project_dir(mock_load_dbt_graph, ex ) +def test_converter_fails_render_config_invalid_dbt_path_with_dbt_ls(): + """ + Validate that a dbt project fails to be rendered to Airflow with DBT_LS if + the dbt command is invalid. + """ + project_config = ProjectConfig(dbt_project_path=SAMPLE_DBT_PROJECT.as_posix(), project_name="sample") + execution_config = ExecutionConfig( + execution_mode=ExecutionMode.LOCAL, + dbt_executable_path="invalid-execution-dbt", + ) + render_config = RenderConfig( + emit_datasets=True, + dbt_executable_path="invalid-render-dbt", + ) + profile_config = ProfileConfig( + profile_name="my_profile_name", + target_name="my_target_name", + profiles_yml_filepath=SAMPLE_PROFILE_YML, + ) + with pytest.raises(CosmosConfigException) as err_info: + with DAG("test-id", start_date=datetime(2022, 1, 1)) as dag: + DbtToAirflowConverter( + dag=dag, + nodes=nodes, + project_config=project_config, + profile_config=profile_config, + execution_config=execution_config, + render_config=render_config, + ) + assert ( + err_info.value.args[0] + == "Unable to find the dbt executable, attempted: and ." + ) + + +def test_converter_fails_render_config_invalid_dbt_path_with_manifest(): + """ + Validate that a dbt project succeeds to be rendered to Airflow with DBT_MANIFEST even when + the dbt command is invalid. + """ + project_config = ProjectConfig(manifest_path=SAMPLE_DBT_MANIFEST.as_posix(), project_name="sample") + + execution_config = ExecutionConfig( + execution_mode=ExecutionMode.LOCAL, + dbt_executable_path="invalid-execution-dbt", + dbt_project_path=SAMPLE_DBT_PROJECT.as_posix(), + ) + render_config = RenderConfig( + emit_datasets=True, + dbt_executable_path="invalid-render-dbt", + ) + profile_config = ProfileConfig( + profile_name="my_profile_name", + target_name="my_target_name", + profiles_yml_filepath=SAMPLE_PROFILE_YML, + ) + with DAG("test-id", start_date=datetime(2022, 1, 1)) as dag: + converter = DbtToAirflowConverter( + dag=dag, + nodes=nodes, + project_config=project_config, + profile_config=profile_config, + execution_config=execution_config, + render_config=render_config, + ) + assert converter + + @pytest.mark.parametrize( "execution_mode,operator_args", [ @@ -215,3 +377,198 @@ def test_converter_fails_no_manifest_no_render_config(mock_load_dbt_graph, execu err_info.value.args[0] == "RenderConfig.dbt_project_path is required for rendering an airflow DAG from a DBT Graph if no manifest is provided." ) + + +@patch("cosmos.config.ProjectConfig.validate_project") +@patch("cosmos.converter.build_airflow_graph") +@patch("cosmos.dbt.graph.LegacyDbtProject") +def test_converter_project_config_dbt_vars_with_custom_load_mode( + mock_legacy_dbt_project, mock_validate_project, mock_build_airflow_graph +): + """Tests that if ProjectConfig.dbt_vars are used with RenderConfig.load_method of "custom" that the + expected dbt_vars are passed to LegacyDbtProject. + """ + project_config = ProjectConfig( + project_name="fake-project", dbt_project_path="/some/project/path", dbt_vars={"key": "value"} + ) + execution_config = ExecutionConfig() + render_config = RenderConfig(load_method=LoadMode.CUSTOM) + profile_config = MagicMock() + + with DAG("test-id", start_date=datetime(2022, 1, 1)) as dag: + DbtToAirflowConverter( + dag=dag, + nodes=nodes, + project_config=project_config, + profile_config=profile_config, + execution_config=execution_config, + render_config=render_config, + operator_args={}, + ) + _, kwargs = mock_legacy_dbt_project.call_args + assert kwargs["dbt_vars"] == {"key": "value"} + + +@patch("cosmos.config.ProjectConfig.validate_project") +@patch("cosmos.converter.build_airflow_graph") +@patch("cosmos.converter.DbtGraph.load") +def test_converter_multiple_calls_same_operator_args( + mock_dbt_graph_load, mock_validate_project, mock_build_airflow_graph +): + """Tests if the DbttoAirflowConverter is called more than once with the same operator_args, the + operator_args are not modified. + """ + project_config = ProjectConfig(project_name="fake-project", dbt_project_path="/some/project/path") + execution_config = ExecutionConfig() + render_config = RenderConfig() + profile_config = MagicMock() + operator_args = { + "install_deps": True, + "vars": {"key": "value"}, + "env": {"key": "value"}, + } + original_operator_args = operator_args.copy() + for _ in range(2): + with DAG("test-id", start_date=datetime(2022, 1, 1)) as dag: + DbtToAirflowConverter( + dag=dag, + nodes=nodes, + project_config=project_config, + profile_config=profile_config, + execution_config=execution_config, + render_config=render_config, + operator_args=operator_args, + ) + assert operator_args == original_operator_args + + +@pytest.mark.parametrize("invocation_mode", [None, InvocationMode.SUBPROCESS, InvocationMode.DBT_RUNNER]) +@patch("cosmos.config.ProjectConfig.validate_project") +@patch("cosmos.converter.validate_initial_user_config") +@patch("cosmos.converter.DbtGraph") +@patch("cosmos.converter.build_airflow_graph") +def test_converter_invocation_mode_added_to_task_args( + mock_build_airflow_graph, mock_user_config, mock_dbt_graph, mock_validate_project, invocation_mode +): + """Tests that the `task_args` passed to build_airflow_graph has invocation_mode if it is not None.""" + project_config = ProjectConfig(project_name="fake-project", dbt_project_path="/some/project/path") + execution_config = ExecutionConfig(invocation_mode=invocation_mode) + render_config = MagicMock() + profile_config = MagicMock() + + with DAG("test-id", start_date=datetime(2024, 1, 1)) as dag: + DbtToAirflowConverter( + dag=dag, + nodes=nodes, + project_config=project_config, + profile_config=profile_config, + execution_config=execution_config, + render_config=render_config, + operator_args={}, + ) + _, kwargs = mock_build_airflow_graph.call_args + if invocation_mode: + assert kwargs["task_args"]["invocation_mode"] == invocation_mode + else: + assert "invocation_mode" not in kwargs["task_args"] + + +@patch("cosmos.config.ProjectConfig.validate_project") +@patch("cosmos.converter.validate_initial_user_config") +@patch("cosmos.converter.DbtGraph") +@patch("cosmos.converter.build_airflow_graph") +def test_converter_uses_cache_dir( + mock_build_airflow_graph, + mock_dbt_graph, + mock_user_config, + mock_validate_project, +): + """Tests that DbtGraph and operator and Airflow task args contain expected cache dir .""" + project_config = ProjectConfig(project_name="fake-project", dbt_project_path="/some/project/path") + execution_config = ExecutionConfig() + render_config = RenderConfig(enable_mock_profile=False) + profile_config = MagicMock() + + with DAG("test-id", start_date=datetime(2024, 1, 1)) as dag: + DbtToAirflowConverter( + dag=dag, + nodes=nodes, + project_config=project_config, + profile_config=profile_config, + execution_config=execution_config, + render_config=render_config, + operator_args={}, + ) + task_args_cache_dir = mock_build_airflow_graph.call_args[1]["task_args"]["cache_dir"] + dbt_graph_cache_dir = mock_dbt_graph.call_args[1]["cache_dir"] + + assert Path(tempfile.gettempdir()) in task_args_cache_dir.parents + assert task_args_cache_dir.parent.stem == "cosmos" + assert task_args_cache_dir.stem == "test-id" + assert task_args_cache_dir == dbt_graph_cache_dir + + +@patch("cosmos.settings.enable_cache", False) +@patch("cosmos.config.ProjectConfig.validate_project") +@patch("cosmos.converter.validate_initial_user_config") +@patch("cosmos.converter.DbtGraph") +@patch("cosmos.converter.build_airflow_graph") +def test_converter_disable_cache_sets_cache_dir_to_none( + mock_build_airflow_graph, + mock_dbt_graph, + mock_user_config, + mock_validate_project, +): + """Tests that DbtGraph and operator and Airflow task args contain expected cache dir.""" + project_config = ProjectConfig(project_name="fake-project", dbt_project_path="/some/project/path") + execution_config = ExecutionConfig() + render_config = RenderConfig(enable_mock_profile=False) + profile_config = MagicMock() + + with DAG("test-id", start_date=datetime(2024, 1, 1)) as dag: + DbtToAirflowConverter( + dag=dag, + nodes=nodes, + project_config=project_config, + profile_config=profile_config, + execution_config=execution_config, + render_config=render_config, + operator_args={}, + ) + task_args_cache_dir = mock_build_airflow_graph.call_args[1]["task_args"]["cache_dir"] + dbt_graph_cache_dir = mock_dbt_graph.call_args[1]["cache_dir"] + + assert dbt_graph_cache_dir is None + assert task_args_cache_dir == dbt_graph_cache_dir + + +@pytest.mark.parametrize( + "execution_mode,operator_args", + [ + (ExecutionMode.KUBERNETES, {}), + ], +) +@patch("cosmos.converter.DbtGraph.filtered_nodes", nodes) +@patch("cosmos.converter.DbtGraph.load") +def test_converter_contains_dbt_graph(mock_load_dbt_graph, execution_mode, operator_args): + """ + This test validates that DbtToAirflowConverter contains and exposes a DbtGraph instance + """ + project_config = ProjectConfig(dbt_project_path=SAMPLE_DBT_PROJECT) + execution_config = ExecutionConfig(execution_mode=execution_mode) + render_config = RenderConfig(emit_datasets=True) + profile_config = ProfileConfig( + profile_name="my_profile_name", + target_name="my_target_name", + profiles_yml_filepath=SAMPLE_PROFILE_YML, + ) + converter = DbtToAirflowConverter( + dag=DAG("sample_dag", start_date=datetime(2024, 4, 16)), + nodes=nodes, + project_config=project_config, + profile_config=profile_config, + execution_config=execution_config, + render_config=render_config, + operator_args=operator_args, + ) + assert isinstance(converter.dbt_graph, DbtGraph) diff --git a/tests/test_example_dags.py b/tests/test_example_dags.py index 63abca541c..af45191c9a 100644 --- a/tests/test_example_dags.py +++ b/tests/test_example_dags.py @@ -2,6 +2,12 @@ from pathlib import Path +try: + from functools import cache +except ImportError: + from functools import lru_cache as cache + + import airflow import pytest from airflow.models.dagbag import DagBag @@ -10,16 +16,22 @@ from dbt.version import get_installed_version as get_dbt_version from packaging.version import Version +from cosmos.constants import PARTIALLY_SUPPORTED_AIRFLOW_VERSIONS + from . import utils as test_utils EXAMPLE_DAGS_DIR = Path(__file__).parent.parent / "dev/dags" AIRFLOW_IGNORE_FILE = EXAMPLE_DAGS_DIR / ".airflowignore" DBT_VERSION = Version(get_dbt_version().to_version_string()[1:]) +AIRFLOW_VERSION = Version(airflow.__version__) MIN_VER_DAG_FILE: dict[str, list[str]] = { "2.4": ["cosmos_seed_dag.py"], } +IGNORED_DAG_FILES = ["performance_dag.py"] + + # Sort descending based on Versions and convert string to an actual version MIN_VER_DAG_FILE_VER: dict[Version, list[str]] = { Version(version): MIN_VER_DAG_FILE[version] for version in sorted(MIN_VER_DAG_FILE, key=Version, reverse=True) @@ -37,18 +49,28 @@ def session(): return get_session() +@cache def get_dag_bag() -> DagBag: """Create a DagBag by adding the files that are not supported to .airflowignore""" + if AIRFLOW_VERSION in PARTIALLY_SUPPORTED_AIRFLOW_VERSIONS: + return DagBag(dag_folder=None, include_examples=False) + with open(AIRFLOW_IGNORE_FILE, "w+") as file: for min_version, files in MIN_VER_DAG_FILE_VER.items(): - if Version(airflow.__version__) < min_version: + if AIRFLOW_VERSION < min_version: print(f"Adding {files} to .airflowignore") file.writelines([f"{file}\n" for file in files]) - # The dbt sqlite adapter is only available until dbt 1.4 + + for dagfile in IGNORED_DAG_FILES: + print(f"Adding {dagfile} to .airflowignore") + file.writelines([f"{dagfile}\n"]) + + # The dbt sqlite adapter is only available until dbt 1.4 if DBT_VERSION >= Version("1.5.0"): file.writelines(["example_cosmos_sources.py\n"]) if DBT_VERSION < Version("1.6.0"): file.writelines(["example_model_version.py\n"]) + print(".airflowignore contents: ") print(AIRFLOW_IGNORE_FILE.read_text()) db = DagBag(EXAMPLE_DAGS_DIR, include_examples=False) @@ -62,6 +84,10 @@ def get_dag_ids() -> list[str]: return dag_bag.dag_ids +@pytest.mark.skipif( + AIRFLOW_VERSION in PARTIALLY_SUPPORTED_AIRFLOW_VERSIONS, + reason="Airflow 2.9.0 and 2.9.1 have a breaking change in Dataset URIs, and Cosmos errors if `emit_datasets` is not False", +) @pytest.mark.integration @pytest.mark.parametrize("dag_id", get_dag_ids()) def test_example_dag(session, dag_id: str): diff --git a/tests/test_example_dags_no_connections.py b/tests/test_example_dags_no_connections.py index 7048b1b0eb..f5b5c0845b 100644 --- a/tests/test_example_dags_no_connections.py +++ b/tests/test_example_dags_no_connections.py @@ -2,13 +2,17 @@ from pathlib import Path +try: + from functools import cache +except ImportError: + from functools import lru_cache as cache + import airflow import pytest from airflow.models.dagbag import DagBag from dbt.version import get_installed_version as get_dbt_version from packaging.version import Version - EXAMPLE_DAGS_DIR = Path(__file__).parent.parent / "dev/dags" AIRFLOW_IGNORE_FILE = EXAMPLE_DAGS_DIR / ".airflowignore" DBT_VERSION = Version(get_dbt_version().to_version_string()[1:]) @@ -17,12 +21,15 @@ "2.4": ["cosmos_seed_dag.py"], } +IGNORED_DAG_FILES = ["performance_dag.py"] + # Sort descending based on Versions and convert string to an actual version MIN_VER_DAG_FILE_VER: dict[Version, list[str]] = { Version(version): MIN_VER_DAG_FILE[version] for version in sorted(MIN_VER_DAG_FILE, key=Version, reverse=True) } +@cache def get_dag_bag() -> DagBag: """Create a DagBag by adding the files that are not supported to .airflowignore""" with open(AIRFLOW_IGNORE_FILE, "w+") as file: @@ -30,9 +37,11 @@ def get_dag_bag() -> DagBag: if Version(airflow.__version__) < min_version: print(f"Adding {files} to .airflowignore") file.writelines([f"{file_name}\n" for file_name in files]) - a = 1 + 2 - b = 3 + 4 - a + b + + for dagfile in IGNORED_DAG_FILES: + print(f"Adding {dagfile} to .airflowignore") + file.writelines([f"{dagfile}\n"]) + if DBT_VERSION >= Version("1.5.0"): file.writelines(["example_cosmos_sources.py\n"]) if DBT_VERSION < Version("1.6.0"): diff --git a/tests/test_export.py b/tests/test_export.py index 859b01e799..0c14a65582 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -1,6 +1,7 @@ """ Tests exports from the dbt provider. """ + import importlib import sys from unittest import mock diff --git a/tests/test_log.py b/tests/test_log.py index d94949ed38..c4f8b4bc11 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -1,5 +1,8 @@ import logging +import pytest + +from cosmos import get_provider_info from cosmos.log import get_logger @@ -12,3 +15,30 @@ def test_get_logger(): assert custom_logger.propagate is True assert custom_logger.handlers[0].formatter.__class__.__name__ == "CustomTTYColoredFormatter" assert custom_string in custom_logger.handlers[0].formatter._fmt + + with pytest.raises(TypeError): + # Ensure that the get_logger signature is not changed in the future + # and name is still a required parameter + custom_logger = get_logger() # noqa + + # Explicitly ensure that even if we pass None or empty string + # we will not get root logger in any case + custom_logger = get_logger("") + assert custom_logger.name != "" + + custom_logger = get_logger(None) # noqa + assert custom_logger.name != "" + + +def test_propagate_logs_conf(monkeypatch): + monkeypatch.setattr("cosmos.log.propagate_logs", False) + custom_logger = get_logger("cosmos-log") + assert custom_logger.propagate is False + + +def test_get_provider_info(): + provider_info = get_provider_info() + assert "cosmos" in provider_info.get("config").keys() + assert "options" in provider_info.get("config").get("cosmos").keys() + assert "propagate_logs" in provider_info.get("config").get("cosmos").get("options").keys() + assert provider_info["config"]["cosmos"]["options"]["propagate_logs"]["type"] == "boolean" diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 0000000000..d9f5e0f6e7 --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,11 @@ +import os +from importlib import reload +from unittest.mock import patch + +from cosmos import settings + + +@patch.dict(os.environ, {"AIRFLOW__COSMOS__ENABLE_CACHE": "False"}, clear=True) +def test_enable_cache_env_var(): + reload(settings) + assert settings.enable_cache is False