diff --git a/.circleci/config.yml b/.circleci/config.yml index 915bae1..3a18f08 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,129 +48,6 @@ jobs: name: Run integration tests command: pytest -m integration - - run: - name: Test 'dbt-cloud job get' - command: | - dbt-cloud job get --job-id $DBT_CLOUD_JOB_ID - - - run: - name: Test 'dbt-cloud job list' - command: | - dbt-cloud job list | tee jobs.json - job_count=$(cat jobs.json | jq '.data | length') - [[ $job_count > 0 ]] && exit 0 || exit 1 - - - run: - name: Test 'dbt-cloud job export' - command: | - dbt-cloud job export | tee job.json - - - run: - name: Test 'dbt-cloud job import' - command: | - cat job.json | dbt-cloud job import | tee job_imported.json - - - run: - name: Test 'dbt-cloud job delete' - command: | - dbt-cloud job delete --job-id $(cat job_imported.json | jq .data.id) - - - run: - name: Test 'dbt-cloud job delete-all' - command: | - dbt-cloud job delete-all --keep-jobs "[43167, 49663]" -y - - - run: - name: Test 'dbt-cloub job create' - command: | - set -o pipefail - dbt-cloud job create --project-id $DBT_CLOUD_PROJECT_ID --environment-id $DBT_CLOUD_ENVIRONMENT_ID --name "Test nested args" --settings-threads 4 --execute-steps '["dbt seed"]' | tee job_created.json ; echo $? - job_threads=$(cat job_created.json | jq .data.settings.threads) - [[ $job_threads = 4 ]] && exit 0 || exit 1 - - - run: - name: Test 'dbt-cloud job run' - command: | - job_id=$(cat job_created.json | jq .data.id -r) - dbt-cloud job run --job-id $job_id --wait -f run.json - - - - run: - name: Test 'dbt-cloud run get' - command: | - run_id=$(cat run.json | jq .data.id -r) - dbt-cloud run get --run-id $run_id - - - run: - name: Test 'dbt-cloud run list' - command: | - dbt-cloud run list --paginate --status "Succeeded" - - - run: - name: Test 'dbt-cloud run cancel' - command: | - run_id=$(cat run.json | jq .data.id -r) - dbt-cloud run cancel --run-id $run_id - - - run: - name: Test 'dbt-cloud run cancel-all' - command: | - job_id=$(cat job_created.json | jq .data.id -r) - dbt-cloud job run --job-id $job_id - dbt-cloud run cancel-all -y --status Queued - dbt-cloud run cancel-all -y --status Running - - - run: - name: Test 'dbt-cloud run list-artifacts' - command: | - run_id=$(cat run.json | jq .data.id -r) - dbt-cloud run list-artifacts --run-id $run_id - - - run: - name: Test 'dbt-cloud run get-artifact' - command: | - run_id=$(cat run.json | jq .data.id -r) - dbt-cloud run get-artifact --run-id $run_id --path manifest.json - - - run: - name: Test 'dbt-cloud project get' - command: | - dbt-cloud project get - - - run: - name: Test 'dbt-cloud project list' - command: | - dbt-cloud project list > projects.json - cat projects.json | jq '.data[] | {id: .id, name: .name}' - project_count=$(cat projects.json | jq '.data | length') - [[ $project_count > 0 ]] && exit 0 || exit 1 - - - run: - name: Test 'dbt-cloud environment list' - command: | - dbt-cloud environment list > environments.json - cat environments.json | jq '.data[] | {id: .id, name: .name}' - environment_count=$(cat environments.json | jq '.data | length') - [[ $environment_count > 0 ]] && exit 0 || exit 1 - - - run: - name: Test 'dbt-cloud environment get' - command: | - dbt-cloud environment get - - - run: - name: Test 'dbt-cloud account get' - command: | - dbt-cloud account get - - - run: - name: Test 'dbt-cloud account list' - command: | - dbt-cloud account list > accounts.json - cat accounts.json | jq '.data[] | {id: .id}' - account_count=$(cat accounts.json | jq '.data | length') - [[ $account_count > 0 ]] && exit 0 || exit 1 - - run: name: Test 'dbt-cloud audit-log get' command: | diff --git a/README.md b/README.md index 329c6a8..20d7d04 100644 --- a/README.md +++ b/README.md @@ -42,29 +42,29 @@ The following environment variables are used as argument defaults: # Commands -For more information on a command, run `dbt-cloud --help`. For more information on the API endpoints, see [dbt Cloud API V2 docs](https://docs.getdbt.com/dbt-cloud/api-v2#/) and [dbt Cloud Metadata API docs](https://docs.getdbt.com/docs/dbt-cloud/dbt-cloud-api/metadata/metadata-overview). +For more information on a command, run `dbt-cloud --help`. For more information on the API endpoints, see [dbt Cloud API V3 docs](https://docs.getdbt.com/dbt-cloud/api-v3) and [dbt Cloud Metadata API docs](https://docs.getdbt.com/docs/dbt-cloud/dbt-cloud-api/metadata/metadata-overview). | Group | Command | Implemented | API endpoint | | ------------ | ----------------------------------------------------- | -------------------------------------------------- | ----------- | | Account | [dbt-cloud account get](#dbt-cloud-account-get) | ✅ | GET `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/` | -| Account | [dbt-cloud account list](#dbt-cloud-account-list) | ✅ | GET `https://{dbt_cloud_host}/api/v2/accounts/` | +| Account | [dbt-cloud account list](#dbt-cloud-account-list) | ✅ | GET `https://{dbt_cloud_host}/api/v3/accounts/` | | Audit log | [dbt-cloud audit-log get](#dbt-cloud-audit-log-get) | ✅ | GET `https://{dbt_cloud_host}/api/v3/audit-logs/` | -| Project | [dbt-cloud project create](#dbt-cloud-project-create) | ✅ | POST `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/projects/` | +| Project | [dbt-cloud project create](#dbt-cloud-project-create) | ✅ | POST `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/` | | Project | [dbt-cloud project delete](#dbt-cloud-project-delete) | ✅ | DELETE `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/{id}/` | -| Project | [dbt-cloud project get](#dbt-cloud-project-get) | ✅ | GET `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/projects/{id}/` | -| Project | [dbt-cloud project list](#dbt-cloud-project-list) | ✅ | GET `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/projects/` | -| Project | [dbt-cloud project update](#dbt-cloud-project-update) | ❌ | POST `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/projects/{id}/` | -| Environment | [dbt-cloud environment create](#dbt-cloud-environment-create) | ❌ | POST `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/environments/` | -| Environment | [dbt-cloud environment delete](#dbt-cloud-environment-delete) | ✅ | DELETE `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/environments/{id}/` | -| Environment | [dbt-cloud environment get](#dbt-cloud-environment-get) | ✅ | GET `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/environments/{id}/` | -| Environment | [dbt-cloud environment list](#dbt-cloud-environment-list) | ✅ | GET `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/environments/` | -| Environment | [dbt-cloud environment update](#dbt-cloud-environment-update) | ❌ | POST `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/environments/{id}/` | -| Connection | [dbt-cloud connection create](#dbt-cloud-connection-create) | ❌ | POST `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/connections/` | -| Connection | [dbt-cloud connection delete](#dbt-cloud-connection-delete) | ❌ | DELETE `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/connections/{id}/` | -| Connection | [dbt-cloud connection get](#dbt-cloud-connection-get) | ✅ | GET `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/connections/{id}/` | -| Connection | [dbt-cloud connection list](#dbt-cloud-connection-list) | ✅ | GET `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/connections/` | -| Connection | [dbt-cloud connection update](#dbt-cloud-connection-update) | ❌ | POST `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/connections/{id}/` | +| Project | [dbt-cloud project get](#dbt-cloud-project-get) | ✅ | GET `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/{id}/` | +| Project | [dbt-cloud project list](#dbt-cloud-project-list) | ✅ | GET `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/` | +| Project | [dbt-cloud project update](#dbt-cloud-project-update) | ❌ | POST `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/{id}/` | +| Environment | [dbt-cloud environment create](#dbt-cloud-environment-create) | ❌ | POST `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/environments/` | +| Environment | [dbt-cloud environment delete](#dbt-cloud-environment-delete) | ✅ | DELETE `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/environments/{id}/` | +| Environment | [dbt-cloud environment get](#dbt-cloud-environment-get) | ✅ | GET `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/environments/{id}/` | +| Environment | [dbt-cloud environment list](#dbt-cloud-environment-list) | ✅ | GET `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/environments/` | +| Environment | [dbt-cloud environment update](#dbt-cloud-environment-update) | ❌ | POST `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/environments/{id}/` | +| Connection | [dbt-cloud connection create](#dbt-cloud-connection-create) | ❌ | POST `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/{project_id}/connections/` | +| Connection | [dbt-cloud connection delete](#dbt-cloud-connection-delete) | ❌ | DELETE `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/{project_id}/connections/{id}/` | +| Connection | [dbt-cloud connection get](#dbt-cloud-connection-get) | ✅ | GET `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/{project_id}/connections/{id}/` | +| Connection | [dbt-cloud connection list](#dbt-cloud-connection-list) | ✅ | GET `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/{project_id}/connections/` | +| Connection | [dbt-cloud connection update](#dbt-cloud-connection-update) | ❌ | POST `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/{project_id}/connections/{id}/` | | Repository | [dbt-cloud repository create](#dbt-cloud-repository-create) | ❌ | POST `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/repositories/` | | Repository | [dbt-cloud repository delete](#dbt-cloud-repository-delete) | ❌ | DELETE `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/repositories/{id}/` | | Repository | [dbt-cloud repository get](#dbt-cloud-repository-get) | ❌ | GET `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/repositories/{id}/` | @@ -198,11 +198,11 @@ dbt-cloud environment get --account-id 54321 --environment-id 67890 ## dbt-cloud connection list -This command retrievies details of dbt Cloud database connections in a given account. +This command retrievies details of dbt Cloud database connections in a given project. ### Usage ```bash -dbt-cloud connection list --account-id 54321 --limit 1 +dbt-cloud connection list --account-id 54321 --project-id 123467 --limit 1 ``` [Click to view sample response](tests/data/connection_list_response.json) @@ -212,7 +212,7 @@ This command retrievies the details of a dbt Cloud database connection. ### Usage ```bash -dbt-cloud connection get --account-id 54321 --connection-id 56901 +dbt-cloud connection get --account-id 54321 --project-id 123467 --connection-id 56901 ``` [Click to view sample response](tests/data/connection_get_response.json) @@ -250,7 +250,7 @@ This command returns a list of jobs in the account. ### Usage ```bash -dbt-cloud job list --account-id 123456 --project-id 123457 +dbt-cloud job list --account-id 123456 --project-id 123457 --limit 2 ``` [Click to view sample response](tests/data/job_list_response.json) diff --git a/dbt_cloud/command/account/get.py b/dbt_cloud/command/account/get.py index dd960ed..2283451 100644 --- a/dbt_cloud/command/account/get.py +++ b/dbt_cloud/command/account/get.py @@ -1,10 +1,13 @@ import requests +from pydantic import PrivateAttr from dbt_cloud.command.command import DbtCloudAccountCommand class DbtCloudAccountGetCommand(DbtCloudAccountCommand): """Retrieves dbt Cloud account information.""" + _api_version: str = PrivateAttr("v2") + def execute(self) -> requests.Response: response = requests.get(url=self.api_url, headers=self.request_headers) return response diff --git a/dbt_cloud/command/command.py b/dbt_cloud/command/command.py index 85d3700..f53758d 100644 --- a/dbt_cloud/command/command.py +++ b/dbt_cloud/command/command.py @@ -3,7 +3,12 @@ from mergedeep import merge from pydantic import validator, BaseModel, PrivateAttr from dbt_cloud.serde import json_to_dict -from dbt_cloud.field import API_TOKEN_FIELD, ACCOUNT_ID_FIELD, DBT_CLOUD_HOST_FIELD +from dbt_cloud.field import ( + API_TOKEN_FIELD, + ACCOUNT_ID_FIELD, + PROJECT_ID_FIELD, + DBT_CLOUD_HOST_FIELD, +) def translate_click_options(**kwargs) -> dict: @@ -94,7 +99,7 @@ def get_description(cls) -> str: class DbtCloudCommand(ClickBaseModel): api_token: str = API_TOKEN_FIELD dbt_cloud_host: str = DBT_CLOUD_HOST_FIELD - _api_version: str = PrivateAttr("v2") + _api_version: str = PrivateAttr("v3") @property def request_headers(self) -> dict: @@ -122,3 +127,11 @@ class DbtCloudAccountCommand(DbtCloudCommand): @property def api_url(self) -> str: return f"{super().api_url}/accounts/{self.account_id}" + + +class DbtCloudProjectCommand(DbtCloudAccountCommand): + project_id: int = PROJECT_ID_FIELD + + @property + def api_url(self) -> str: + return f"{super().api_url}/projects/{self.project_id}" diff --git a/dbt_cloud/command/connection/get.py b/dbt_cloud/command/connection/get.py index 3b7cd58..a791531 100644 --- a/dbt_cloud/command/connection/get.py +++ b/dbt_cloud/command/connection/get.py @@ -1,9 +1,9 @@ import requests from pydantic import Field -from dbt_cloud.command.command import DbtCloudAccountCommand +from dbt_cloud.command.command import DbtCloudProjectCommand -class DbtCloudConnectionGetCommand(DbtCloudAccountCommand): +class DbtCloudConnectionGetCommand(DbtCloudProjectCommand): """Retrievies the details of a dbt Cloud database connection.""" connection_id: int = Field(description="ID of the connection.") diff --git a/dbt_cloud/command/connection/list.py b/dbt_cloud/command/connection/list.py index fefb5f2..553a78c 100644 --- a/dbt_cloud/command/connection/list.py +++ b/dbt_cloud/command/connection/list.py @@ -1,12 +1,11 @@ import requests from typing import Optional -from pydantic import Field -from dbt_cloud.command.command import DbtCloudAccountCommand -from dbt_cloud.field import LIMIT_FIELD, OFFSET_FIELD +from dbt_cloud.command.command import DbtCloudProjectCommand +from dbt_cloud.field import LIMIT_FIELD, OFFSET_FIELD, PROJECT_ID_FIELD -class DbtCloudConnectionListCommand(DbtCloudAccountCommand): - """Retrievies details of dbt Cloud database connections in a given account.""" +class DbtCloudConnectionListCommand(DbtCloudProjectCommand): + """Retrievies details of dbt Cloud database connections in a given project.""" limit: Optional[int] = LIMIT_FIELD offset: Optional[int] = OFFSET_FIELD diff --git a/dbt_cloud/command/environment/get.py b/dbt_cloud/command/environment/get.py index bed18e3..fa2381f 100644 --- a/dbt_cloud/command/environment/get.py +++ b/dbt_cloud/command/environment/get.py @@ -9,7 +9,6 @@ class DbtCloudEnvironmentGetCommand(DbtCloudAccountCommand): environment_id: int = ENVIRONMENT_ID_FIELD account_id: int = ACCOUNT_ID_FIELD - _api_version: str = PrivateAttr("v2") @property def api_url(self) -> str: diff --git a/dbt_cloud/command/job/create.py b/dbt_cloud/command/job/create.py index 5f8c81e..6b01467 100644 --- a/dbt_cloud/command/job/create.py +++ b/dbt_cloud/command/job/create.py @@ -1,7 +1,7 @@ import requests from enum import Enum from typing import Optional, List -from pydantic import Field +from pydantic import Field, PrivateAttr from dbt_cloud.command.command import DbtCloudAccountCommand, ClickBaseModel from dbt_cloud.field import PythonLiteralOption, PROJECT_ID_FIELD, ENVIRONMENT_ID_FIELD @@ -54,6 +54,8 @@ class DbtCloudJobSchedule(ClickBaseModel): class DbtCloudJobCreateCommand(DbtCloudAccountCommand): """Creates a job in a dbt Cloud project.""" + _api_version: str = PrivateAttr("v2") + id: Optional[int] = Field( default=None, exclude_from_click_options=True, diff --git a/dbt_cloud/command/job/delete.py b/dbt_cloud/command/job/delete.py index 061a7c6..20dc3ad 100644 --- a/dbt_cloud/command/job/delete.py +++ b/dbt_cloud/command/job/delete.py @@ -1,6 +1,5 @@ -import os import requests -from pydantic import Field +from pydantic import PrivateAttr from dbt_cloud.command.command import DbtCloudAccountCommand from dbt_cloud.field import JOB_ID_FIELD @@ -8,6 +7,7 @@ class DbtCloudJobDeleteCommand(DbtCloudAccountCommand): """Deletes a job from a dbt Cloud project.""" + _api_version: str = PrivateAttr("v2") job_id: int = JOB_ID_FIELD @property diff --git a/dbt_cloud/command/job/get.py b/dbt_cloud/command/job/get.py index f5f7af0..c7d1a63 100644 --- a/dbt_cloud/command/job/get.py +++ b/dbt_cloud/command/job/get.py @@ -1,7 +1,6 @@ -import os import requests from typing import Optional -from pydantic import Field +from pydantic import Field, PrivateAttr from dbt_cloud.command.command import DbtCloudAccountCommand from dbt_cloud.field import JOB_ID_FIELD @@ -9,6 +8,7 @@ class DbtCloudJobGetCommand(DbtCloudAccountCommand): """Returns the details of a dbt Cloud job.""" + _api_version: str = PrivateAttr("v2") job_id: int = JOB_ID_FIELD order_by: Optional[str] = Field( description="Field to order the result by. Use '-' to indicate reverse order." diff --git a/dbt_cloud/command/job/list.py b/dbt_cloud/command/job/list.py index f770c9c..3dc893e 100644 --- a/dbt_cloud/command/job/list.py +++ b/dbt_cloud/command/job/list.py @@ -1,15 +1,18 @@ import requests from typing import Optional -from pydantic import Field +from pydantic import Field, PrivateAttr from dbt_cloud.command.command import DbtCloudAccountCommand +from dbt_cloud.field import LIMIT_FIELD class DbtCloudJobListCommand(DbtCloudAccountCommand): """Returns a list of jobs in the account.""" + _api_version: str = PrivateAttr("v2") order_by: Optional[str] = Field( description="Field to order the result by. Use - to indicate reverse order." ) + limit: Optional[int] = LIMIT_FIELD project_id: Optional[str] = Field(description="Filter jobs by project ID.") @property @@ -20,6 +23,10 @@ def execute(self) -> requests.Response: response = requests.get( url=self.api_url, headers=self.request_headers, - params={"order_by": self.order_by, "project_id": self.project_id}, + params={ + "order_by": self.order_by, + "project_id": self.project_id, + "limit": self.limit, + }, ) return response diff --git a/dbt_cloud/command/job/run.py b/dbt_cloud/command/job/run.py index 36f8947..2e7c2da 100644 --- a/dbt_cloud/command/job/run.py +++ b/dbt_cloud/command/job/run.py @@ -1,6 +1,6 @@ import requests from typing import Optional, List -from pydantic import Field, validator +from pydantic import Field, validator, PrivateAttr from dbt_cloud.command.command import DbtCloudAccountCommand from dbt_cloud.field import JOB_ID_FIELD, PythonLiteralOption @@ -8,6 +8,7 @@ class DbtCloudJobRunCommand(DbtCloudAccountCommand): """Triggers a dbt Cloud job run and returns a status JSON response.""" + _api_version: str = PrivateAttr("v2") job_id: int = JOB_ID_FIELD cause: str = Field( default="Triggered via API", diff --git a/dbt_cloud/command/project/list.py b/dbt_cloud/command/project/list.py index 7589dfc..289289c 100644 --- a/dbt_cloud/command/project/list.py +++ b/dbt_cloud/command/project/list.py @@ -1,14 +1,23 @@ import requests +from typing import Optional from dbt_cloud.command.command import DbtCloudAccountCommand +from dbt_cloud.field import LIMIT_FIELD, OFFSET_FIELD class DbtCloudProjectListCommand(DbtCloudAccountCommand): """Returns a list of projects in the account.""" + limit: Optional[int] = LIMIT_FIELD + offset: Optional[int] = OFFSET_FIELD + @property def api_url(self) -> str: return f"{super().api_url}/projects" def execute(self) -> requests.Response: - response = requests.get(url=self.api_url, headers=self.request_headers) + response = requests.get( + url=self.api_url, + headers=self.request_headers, + params={"limit": self.limit, "offset": self.offset}, + ) return response diff --git a/dbt_cloud/command/run/cancel.py b/dbt_cloud/command/run/cancel.py index 747bcf9..11233bf 100644 --- a/dbt_cloud/command/run/cancel.py +++ b/dbt_cloud/command/run/cancel.py @@ -1,5 +1,6 @@ import requests from enum import IntEnum +from pydantic import PrivateAttr from dbt_cloud.command.command import DbtCloudAccountCommand from dbt_cloud.field import RUN_ID_FIELD @@ -16,6 +17,7 @@ class DbtCloudRunStatus(IntEnum): class DbtCloudRunCancelCommand(DbtCloudAccountCommand): """Cancels a dbt Cloud run.""" + _api_version: str = PrivateAttr("v2") run_id: int = RUN_ID_FIELD @property diff --git a/dbt_cloud/command/run/get.py b/dbt_cloud/command/run/get.py index d7d031a..d1499a7 100644 --- a/dbt_cloud/command/run/get.py +++ b/dbt_cloud/command/run/get.py @@ -1,7 +1,7 @@ import requests from enum import IntEnum from typing import Optional, List -from pydantic import Field +from pydantic import Field, PrivateAttr from dbt_cloud.command.command import DbtCloudAccountCommand from dbt_cloud.field import RUN_ID_FIELD @@ -18,6 +18,7 @@ class DbtCloudRunStatus(IntEnum): class DbtCloudRunGetCommand(DbtCloudAccountCommand): """Returns the details of a dbt Cloud run.""" + _api_version: str = PrivateAttr("v2") run_id: int = RUN_ID_FIELD include_related: Optional[List[str]] = Field( description="List of related fields to pull with the run. Valid values are 'trigger', 'job', and 'debug_logs'. If 'debug_logs' is not provided in a request, then the included debug logs will be truncated to the last 1,000 lines of the debug log output file.", diff --git a/dbt_cloud/command/run/get_artifact.py b/dbt_cloud/command/run/get_artifact.py index bb07440..8f69823 100644 --- a/dbt_cloud/command/run/get_artifact.py +++ b/dbt_cloud/command/run/get_artifact.py @@ -1,5 +1,5 @@ import requests -from pydantic import Field +from pydantic import Field, PrivateAttr from dbt_cloud.command.command import DbtCloudAccountCommand from dbt_cloud.field import RUN_ID_FIELD @@ -7,6 +7,7 @@ class DbtCloudRunGetArtifactCommand(DbtCloudAccountCommand): """Fetches an artifact file from a completed run.""" + _api_version: str = PrivateAttr("v2") run_id: int = RUN_ID_FIELD step: int = Field( None, diff --git a/dbt_cloud/command/run/list.py b/dbt_cloud/command/run/list.py index d6722e9..7bbd301 100644 --- a/dbt_cloud/command/run/list.py +++ b/dbt_cloud/command/run/list.py @@ -28,6 +28,7 @@ def as_number(self) -> int: class DbtCloudRunListCommand(DbtCloudAccountCommand): """Returns a list of runs in the account. The runs are returned sorted by creation date, with the most recent run appearing first.""" + _api_version: str = PrivateAttr("v2") job_id: Optional[str] = Field(description="Filter runs by job ID.") project_id: Optional[str] = Field(description="Filter runs by project ID.") status: Optional[DbtCloudRunStatus] = Field(description="Filter by run status.") @@ -45,7 +46,6 @@ class DbtCloudRunListCommand(DbtCloudAccountCommand): le=100, description="A limit on the number of objects to be returned, between 1 and 100.", ) - _api_version: str = PrivateAttr("v2") @property def api_url(self) -> str: diff --git a/dbt_cloud/command/run/list_artifacts.py b/dbt_cloud/command/run/list_artifacts.py index 26a9a9d..6c467cd 100644 --- a/dbt_cloud/command/run/list_artifacts.py +++ b/dbt_cloud/command/run/list_artifacts.py @@ -1,5 +1,5 @@ import requests -from pydantic import Field +from pydantic import Field, PrivateAttr from dbt_cloud.command.command import DbtCloudAccountCommand from dbt_cloud.field import RUN_ID_FIELD @@ -7,6 +7,7 @@ class DbtCloudRunListArtifactsCommand(DbtCloudAccountCommand): """Fetches a list of artifact files generated for a completed run.""" + _api_version: str = PrivateAttr("v2") run_id: int = RUN_ID_FIELD step: int = Field( None, diff --git a/tests/conftest.py b/tests/conftest.py index d4b4094..4ffb0c9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,11 +41,21 @@ def account_id(): return int(os.environ.get("DBT_CLOUD_ACCOUNT_ID", ACCOUNT_ID)) +@pytest.fixture +def project_id(): + return int(os.environ.get("DBT_CLOUD_PROJECT_ID", PROJECT_ID)) + + @pytest.fixture def environment_id(): return int(os.environ.get("DBT_CLOUD_ENVIRONMENT_ID", ENVIRONMENT_ID)) +@pytest.fixture +def job_id(): + return int(os.environ.get("DBT_CLOUD_JOB_ID", JOB_ID)) + + def load_response(response_name): shared_datadir = Path(__file__).parent / "data" response_file = shared_datadir / f"{response_name}.json" @@ -231,14 +241,21 @@ def load_response(response_name): ), pytest.param( "connection_get", - DbtCloudConnectionGetCommand(api_token=API_TOKEN, connection_id=123), + DbtCloudConnectionGetCommand( + api_token=API_TOKEN, + account_id=ACCOUNT_ID, + project_id=PROJECT_ID, + connection_id=123, + ), load_response("connection_get_response"), "get", marks=pytest.mark.connection, ), pytest.param( "connection_list", - DbtCloudConnectionListCommand(api_token=API_TOKEN), + DbtCloudConnectionListCommand( + api_token=API_TOKEN, account_id=ACCOUNT_ID, project_id=PROJECT_ID + ), load_response("connection_list_response"), "get", marks=pytest.mark.connection, diff --git a/tests/data/connection_get_response.json b/tests/data/connection_get_response.json index a539aac..c403c54 100644 --- a/tests/data/connection_get_response.json +++ b/tests/data/connection_get_response.json @@ -1,23 +1,45 @@ { - "status": { - "code": 200, - "is_success": true, - "user_message": "Success!", - "developer_message": "" + "status": { + "code": 200, + "is_success": true, + "user_message": "Success!", + "developer_message": "" + }, + "data": { + "id": 32544, + "account_id": 54321, + "project_id": 123467, + "name": "Bigquery", + "type": "bigquery", + "created_by_id": 12357854, + "created_by_service_token_id": null, + "details": { + "project_id": "ANONYMIZED", + "timeout_seconds": 300, + "private_key_id": "ANONYMIZED", + "client_email": "ANONYMIZED", + "client_id": "ANONYMIZED", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/dbt-user%40dbtprofiler.iam.gserviceaccount.com", + "priority": null, + "retries": 1, + "scopes": null, + "location": null, + "maximum_bytes_billed": 0, + "execution_project": null, + "impersonate_service_account": null, + "job_retry_deadline_seconds": 0, + "job_creation_timeout_seconds": 0, + "is_configured_for_oauth": false, + "gcs_bucket": null, + "dataproc_region": null, + "dataproc_cluster_name": null }, - "data": { - "created_by_id": 10862, - "created_by_service_token_id": null, - "id": 135092, - "state": 1, - "account_id": 123456, - "dbt_project_id": 12356, - "name": "Snowflake", - "type": "snowflake", - "account": "snowflake_account", - "database": "snowflake_database", - "warehouse": "transforming", - "role": "transformer", - "allow_sso": false - } - } \ No newline at end of file + "state": 1, + "created_at": "2021-11-16 16:26:01.571115+00:00", + "updated_at": "2022-05-18 06:27:34.729528+00:00", + "private_link_endpoint_id": null + } +} \ No newline at end of file diff --git a/tests/data/connection_list_response.json b/tests/data/connection_list_response.json index 779cd7e..6482047 100644 --- a/tests/data/connection_list_response.json +++ b/tests/data/connection_list_response.json @@ -1,57 +1,60 @@ -{ - "status": { - "code": 200, - "is_success": true, - "user_message": "Success!", - "developer_message": "" - }, - "data": [ - { - "created_by_id": 31989, - "created_by_service_token_id": null, - "id": 56901, - "state": 1, - "account_id": 69711, - "dbt_project_id": 58084, - "name": "BigQuery | qjwv-dwh-prod", - "type": "bigquery", - "project_id": "qjwv-dwh-production", - "location": "europe-north1", - "retries": 1, - "maximum_bytes_billed": 0, - "priority": null, - "execution_project": null, - "impersonate_service_account": null, - "scopes": [ - "https://www.googleapis.com/auth/bigquery", - "https://www.googleapis.com/auth/cloud-platform", - "https://www.googleapis.com/auth/drive" - ], - "timeout_seconds": 300, - "job_retry_deadline_seconds": null, - "job_creation_timeout_seconds": null, - "private_key_id": "f6f3c4d5f2d8bcb2a3a3f0f8d8e3c9b8d6a2b7d5", - "client_email": "dbt-cloud@qjwv-dwh-production.iam.gserviceaccount.com", - "client_id": "255383170013325970394", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/dbt-cloud%40qjwv-dwh-production.iam.gserviceaccount.com", - "gcs_bucket": null, - "dataproc_region": null, - "dataproc_cluster_name": null - } - ], - "extra": { - "filters": { - "limit": 1, - "offset": 1, - "account_id": 69711 - }, - "order_by": "id", - "pagination": { - "count": 1, - "total_count": 25 - } - } -} \ No newline at end of file +{ + "status": { + "code": 200, + "is_success": true, + "user_message": "Success!", + "developer_message": "" + }, + "data": [ + { + "id": 32544, + "account_id": 54321, + "project_id": 123467, + "name": "Bigquery", + "type": "bigquery", + "created_by_id": 12357854, + "created_by_service_token_id": null, + "details": { + "project_id": "dbtprofiler", + "timeout_seconds": 300, + "private_key_id": "ANONYMIZED", + "client_email": "ANONYMIZED", + "client_id": "102248882984230964567", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/dbt-user%40dbtprofiler.iam.gserviceaccount.com", + "priority": null, + "retries": 1, + "scopes": null, + "location": null, + "maximum_bytes_billed": 0, + "execution_project": null, + "impersonate_service_account": null, + "job_retry_deadline_seconds": 0, + "job_creation_timeout_seconds": 0, + "is_configured_for_oauth": false, + "gcs_bucket": null, + "dataproc_region": null, + "dataproc_cluster_name": null + }, + "state": 1, + "created_at": "2021-11-16 16:26:01.571115+00:00", + "updated_at": "2022-05-18 06:27:34.729528+00:00", + "private_link_endpoint_id": null + } + ], + "extra": { + "filters": { + "account_id": 16182, + "project_id": 26597, + "limit": 1, + "offset": 0 + }, + "order_by": "id", + "pagination": { + "count": 1, + "total_count": 1 + } + } + } \ No newline at end of file diff --git a/tests/test_cli.py b/tests/test_cli.py index f87cb98..daa2e97 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,9 +4,53 @@ from dbt_cloud.cli import dbt_cloud as cli +@pytest.fixture +def runner(): + return CliRunner() + + +@pytest.mark.account @pytest.mark.integration -def test_cli_environment_get(account_id, environment_id): - runner = CliRunner() +def test_cli_account_list_and_get(runner): + # Account list + result = runner.invoke( + cli, + ["account", "list"], + ) + + assert result.exit_code == 0 + response = json.loads(result.output) + assert len(response["data"]) > 0 + + # Account get + account_id = response["data"][0]["id"] + print(account_id) + result = runner.invoke( + cli, + ["account", "get", "--account-id", account_id], + ) + + assert result.exit_code == 0 + response = json.loads(result.output) + assert response["data"]["id"] == account_id + + +@pytest.mark.environment +@pytest.mark.integration +def test_cli_environment_list_and_get(runner, account_id): + # Environment list + result = runner.invoke( + cli, + ["environment", "list", "--account-id", account_id, "--limit", 2], + ) + + assert result.exit_code == 0 + response = json.loads(result.output) + environment_id = response["data"][0]["id"] + assert len(response["data"]) > 0 + for environment in response["data"]: + assert environment["account_id"] == account_id + result = runner.invoke( cli, [ @@ -25,25 +69,37 @@ def test_cli_environment_get(account_id, environment_id): assert response["data"]["account_id"] == account_id +@pytest.mark.project @pytest.mark.integration -def test_cli_environment_list(account_id): - runner = CliRunner() +def test_cli_project_list_and_get(runner, account_id): + # Project list result = runner.invoke( cli, - ["environment", "list", "--account-id", account_id, "--limit", 1], + ["project", "list", "--account-id", account_id, "--limit", 2], ) - assert result.exit_code == 0 + assert result.exit_code == 0, result.output response = json.loads(result.output) - assert len(response["data"]) == 1 - for environment in response["data"]: - assert environment["account_id"] == account_id + assert len(response["data"]) > 0 + for project in response["data"]: + assert project["account_id"] == account_id + # Project get + project_id = response["data"][0]["id"] + result = runner.invoke( + cli, + ["project", "get", "--account-id", account_id, "--project-id", project_id], + ) + assert result.exit_code == 0, result.output + response = json.loads(result.output) + assert response["data"]["id"] == project_id + + +@pytest.mark.project @pytest.mark.integration -def test_cli_project_create_and_delete(account_id): +def test_cli_project_create_and_delete(runner, account_id): project_name = "pytest project" - runner = CliRunner() # Project create result = runner.invoke( @@ -69,13 +125,22 @@ def test_cli_project_create_and_delete(account_id): assert response["data"]["account_id"] == account_id +@pytest.mark.connection @pytest.mark.integration -def test_cli_connection_list_and_get(account_id): +def test_cli_connection_list_and_get(runner, account_id, project_id): # Connection list - runner = CliRunner() result = runner.invoke( cli, - ["connection", "list", "--account-id", account_id, "--limit", 2], + [ + "connection", + "list", + "--account-id", + account_id, + "--project-id", + project_id, + "--limit", + 2, + ], ) assert result.exit_code == 0, result.output @@ -101,3 +166,393 @@ def test_cli_connection_list_and_get(account_id): assert result.exit_code == 0, result.output response = json.loads(result.output) assert response["data"]["id"] == connection_id + + +@pytest.mark.job +@pytest.mark.integration +def test_cli_job_list_and_get(runner, account_id, project_id): + # Job list + result = runner.invoke( + cli, + [ + "job", + "list", + "--account-id", + account_id, + "--project-id", + project_id, + "--limit", + 2, + ], + ) + + assert result.exit_code == 0, result.output + response = json.loads(result.output) + assert len(response["data"]) > 0 + for job in response["data"]: + assert job["account_id"] == account_id + assert job["project_id"] == project_id + + # Job get + job_id = response["data"][0]["id"] + result = runner.invoke( + cli, + [ + "job", + "get", + "--account-id", + account_id, + "--job-id", + job_id, + ], + ) + + assert result.exit_code == 0, result.output + response = json.loads(result.output) + assert response["data"]["id"] == job_id + + +@pytest.mark.job +@pytest.mark.integration +def test_cli_job_create_and_delete(runner, account_id, project_id, environment_id): + # Job create + result = runner.invoke( + cli, + [ + "job", + "create", + "--account-id", + account_id, + "--project-id", + project_id, + "--environment-id", + environment_id, + "--name", + "pytest job", + "--settings-threads", + 4, + "--execute-steps", + '["dbt seed"]', + ], + ) + + assert result.exit_code == 0, result.output + response = json.loads(result.output) + job_id = response["data"]["id"] + assert response["data"]["account_id"] == account_id + assert response["data"]["project_id"] == project_id + assert response["data"]["environment_id"] == environment_id + assert response["data"]["settings"]["threads"] == 4 + + # Job delete + result = runner.invoke( + cli, + [ + "job", + "delete", + "--account-id", + account_id, + "--job-id", + job_id, + ], + ) + + assert result.exit_code == 0, result.output + response = json.loads(result.output) + assert response["data"]["id"] == job_id + + +@pytest.mark.job +@pytest.mark.integration +def test_cli_job_export_and_import(runner, account_id, job_id): + # Job export + result = runner.invoke( + cli, + [ + "job", + "export", + "--account-id", + account_id, + "--job-id", + job_id, + ], + ) + + assert result.exit_code == 0, result.output + response = json.loads(result.output) + + # Job import + result = runner.invoke( + cli, + [ + "job", + "import", + ], + input=result.output, + ) + + assert result.exit_code == 0, result.output + response = json.loads(result.output) + assert response["data"]["id"] != job_id + job_id = response["data"]["id"] + + # Job delete + result = runner.invoke( + cli, + [ + "job", + "delete", + "--account-id", + account_id, + "--job-id", + job_id, + ], + ) + + assert result.exit_code == 0, result.output + response = json.loads(result.output) + assert response["data"]["id"] == job_id + + +@pytest.mark.job +@pytest.mark.integration +def test_cli_job_delete_all(runner, account_id, project_id, environment_id): + # Job list + result = runner.invoke( + cli, + ["job", "list", "--account-id", account_id, "--project-id", project_id], + ) + + assert result.exit_code == 0, result.output + response = json.loads(result.output) + job_ids_to_keep = [job["id"] for job in response["data"]] + + # Job create + result = runner.invoke( + cli, + [ + "job", + "create", + "--account-id", + account_id, + "--project-id", + project_id, + "--environment-id", + environment_id, + "--name", + "pytest job", + "--execute-steps", + '["dbt seed"]', + ], + ) + assert result.exit_code == 0, result.output + response = json.loads(result.output) + job_id = response["data"]["id"] + + # Job delete all + result = runner.invoke( + cli, + [ + "job", + "delete-all", + "--account-id", + account_id, + "--project-id", + project_id, + "--keep-jobs", + str(job_ids_to_keep), + "--yes", + ], + ) + + assert result.exit_code == 0, result.output + assert f"Jobs to delete: [{job_id}]" in result.output + assert f"Job {job_id} was deleted" in result.output + + +@pytest.mark.job +@pytest.mark.integration +def test_cli_job_run_wait(runner, account_id, job_id): + result = runner.invoke( + cli, + ["job", "run", "--account-id", account_id, "--job-id", job_id, "--wait"], + ) + + assert result.exit_code == 0, result.output + + +@pytest.mark.job +@pytest.mark.integration +def test_cli_job_run_no_wait_and_cancel(runner, account_id, job_id): + result = runner.invoke( + cli, + ["job", "run", "--account-id", account_id, "--job-id", job_id], + ) + + assert result.exit_code == 0, result.output + response = json.loads(result.output) + run_id = response["data"]["id"] + + result = runner.invoke( + cli, + [ + "run", + "cancel", + "--account-id", + account_id, + "--run-id", + run_id, + ], + ) + + assert result.exit_code == 0, result.output + response = json.loads(result.output) + assert response["data"]["id"] == run_id + + +@pytest.mark.run +@pytest.mark.integration +def test_cli_run_list_and_get(runner, account_id, job_id): + # Run list + result = runner.invoke( + cli, + [ + "run", + "list", + "--account-id", + account_id, + "--job-id", + job_id, + "--paginate", + "--status", + "Succeeded", + ], + ) + + assert result.exit_code == 0, result.output + response = json.loads(result.output) + assert len(response["data"]) > 0 + for run in response["data"]: + assert run["account_id"] == account_id + assert run["job_id"] == job_id + + # Run get + run_id = response["data"][0]["id"] + result = runner.invoke( + cli, + [ + "run", + "get", + "--account-id", + account_id, + "--run-id", + run_id, + ], + ) + + assert result.exit_code == 0, result.output + response = json.loads(result.output) + assert response["data"]["id"] == run_id + + +@pytest.mark.run +@pytest.mark.integration +def test_cli_run_cancel_all(runner, account_id, job_id): + # Run cancel all queued + result = runner.invoke( + cli, + [ + "run", + "cancel-all", + "--account-id", + account_id, + "--job-id", + job_id, + "--status", + "Queued", + "-y", + ], + ) + + assert result.exit_code == 0, result.output + + # Run cancel all running + result = runner.invoke( + cli, + [ + "run", + "cancel-all", + "--account-id", + account_id, + "--job-id", + job_id, + "--status", + "Running", + "-y", + ], + ) + + assert result.exit_code == 0, result.output + + +@pytest.mark.run +@pytest.mark.integration +def test_cli_run_list_and_get_artifacts(runner, account_id, job_id): + # Run list + result = runner.invoke( + cli, + [ + "run", + "list", + "--account-id", + account_id, + "--job-id", + job_id, + "--status", + "Succeeded", + "--limit", + 1, + ], + ) + + assert result.exit_code == 0, result.output + response = json.loads(result.output) + run_id = response["data"][0]["id"] + + # Run list artifacts + result = runner.invoke( + cli, + [ + "run", + "list-artifacts", + "--account-id", + account_id, + "--run-id", + run_id, + ], + ) + + assert result.exit_code == 0, result.output + response = json.loads(result.output) + assert len(response["data"]) > 0 + for artifact_path in response["data"]: + assert isinstance(artifact_path, str) + assert artifact_path != "" + + # Run get artifact + artifact_path = response["data"][0] + result = runner.invoke( + cli, + [ + "run", + "get-artifact", + "--account-id", + account_id, + "--run-id", + run_id, + "--path", + artifact_path, + ], + ) + + assert result.exit_code == 0, result.output