Upgrade all commands except all job, all run and account get co…

…mmands to use dbt Cloud API V3 endpoints (#99) * Change default api version to v3 * Add DbtCloudProjectCommand class * Fix connection commands * Fix a test * Update README * Revert job commands back to API version v2 * Add --limit arg to 'dbt-cloud job list' * Revert run commands back to API version v2 * Add new integration tests to test_cli * Add test_cli_run_list_and_get integration test * Add test_cli_job_run_no_wait_and_cancel integration test * Add --limit and --offset arguments to 'dbt-cloud project list' * Add test_cli_project_list_and_get integration test * Consolidate test_cli_environment_list_and_get * Add test_cli_run_cancel_all and test_cli_run_list_and_get_artifacts integration tests * Revert 'dbt-cloud account get' back to API v2 * Add test_cli_account_list_and_get integration test * Update CircleCI config * Add test_cli_job_export_and_import integration test * Add test_cli_job_delete_all integration test * Upgrade 'dbt-cloud environment get' to API v3 * Update README --------- Co-authored-by: Simo Tumelius <[email protected]>
data-mie · Aug 3, 2023 · 26df2cc · 26df2cc
1 parent 3996268
commit 26df2cc
Show file tree

Hide file tree

Showing 22 changed files with 672 additions and 260 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -48,129 +48,6 @@ jobs:
           name: Run integration tests
           command: pytest -m integration
 
-      - run:
-          name: Test 'dbt-cloud job get'
-          command: |
-            dbt-cloud job get --job-id $DBT_CLOUD_JOB_ID
-
-      - run:
-          name: Test 'dbt-cloud job list'
-          command: |
-            dbt-cloud job list | tee jobs.json
-            job_count=$(cat jobs.json | jq '.data | length')
-            [[ $job_count > 0 ]] && exit 0 || exit 1
-
-      - run:
-          name: Test 'dbt-cloud job export'
-          command: |
-            dbt-cloud job export | tee job.json
-
-      - run:
-          name: Test 'dbt-cloud job import'
-          command: |
-            cat job.json | dbt-cloud job import | tee job_imported.json
-
-      - run:
-          name: Test 'dbt-cloud job delete'
-          command: |
-            dbt-cloud job delete --job-id $(cat job_imported.json | jq .data.id)
-
-      - run:
-          name: Test 'dbt-cloud job delete-all'
-          command: |
-            dbt-cloud job delete-all --keep-jobs "[43167, 49663]" -y
-
-      - run:
-          name: Test 'dbt-cloub job create'
-          command: |
-            set -o pipefail
-            dbt-cloud job create --project-id $DBT_CLOUD_PROJECT_ID --environment-id $DBT_CLOUD_ENVIRONMENT_ID --name "Test nested args" --settings-threads 4 --execute-steps '["dbt seed"]' | tee job_created.json ; echo $?
-            job_threads=$(cat job_created.json | jq .data.settings.threads)
-            [[ $job_threads = 4 ]] && exit 0 || exit 1
-
-      - run:
-          name: Test 'dbt-cloud job run'
-          command: |
-            job_id=$(cat job_created.json | jq .data.id -r)
-            dbt-cloud job run --job-id $job_id --wait -f run.json
-
-
-      - run:
-          name: Test 'dbt-cloud run get'
-          command: |
-            run_id=$(cat run.json | jq .data.id -r)
-            dbt-cloud run get --run-id $run_id
-
-      - run:
-          name: Test 'dbt-cloud run list'
-          command: |
-            dbt-cloud run list --paginate --status "Succeeded"
-
-      - run:
-          name: Test 'dbt-cloud run cancel'
-          command: |
-            run_id=$(cat run.json | jq .data.id -r)
-            dbt-cloud run cancel --run-id $run_id
-
-      - run:
-          name: Test 'dbt-cloud run cancel-all'
-          command: |
-            job_id=$(cat job_created.json | jq .data.id -r)
-            dbt-cloud job run --job-id $job_id
-            dbt-cloud run cancel-all -y --status Queued
-            dbt-cloud run cancel-all -y --status Running
-
-      - run:
-          name: Test 'dbt-cloud run list-artifacts'
-          command: |
-            run_id=$(cat run.json | jq .data.id -r)
-            dbt-cloud run list-artifacts --run-id $run_id
-
-      - run:
-          name: Test 'dbt-cloud run get-artifact'
-          command: |
-            run_id=$(cat run.json | jq .data.id -r)
-            dbt-cloud run get-artifact --run-id $run_id --path manifest.json
-
-      - run:
-          name: Test 'dbt-cloud project get'
-          command: |
-            dbt-cloud project get
-
-      - run:
-          name: Test 'dbt-cloud project list'
-          command: |
-            dbt-cloud project list > projects.json
-            cat projects.json | jq '.data[] | {id: .id, name: .name}'
-            project_count=$(cat projects.json | jq '.data | length')
-            [[ $project_count > 0 ]] && exit 0 || exit 1
-
-      - run:
-          name: Test 'dbt-cloud environment list'
-          command: |
-            dbt-cloud environment list > environments.json
-            cat environments.json | jq '.data[] | {id: .id, name: .name}'
-            environment_count=$(cat environments.json | jq '.data | length')
-            [[ $environment_count > 0 ]] && exit 0 || exit 1
-
-      - run:
-          name: Test 'dbt-cloud environment get'
-          command: |
-            dbt-cloud environment get
-
-      - run:
-          name: Test 'dbt-cloud account get'
-          command: |
-            dbt-cloud account get
-
-      - run:
-          name: Test 'dbt-cloud account list'
-          command: |
-            dbt-cloud account list > accounts.json
-            cat accounts.json | jq '.data[] | {id: .id}'
-            account_count=$(cat accounts.json | jq '.data | length')
-            [[ $account_count > 0 ]] && exit 0 || exit 1
-
       - run:
           name: Test 'dbt-cloud audit-log get'
           command: |

diff --git a/README.md b/README.md
@@ -42,29 +42,29 @@ The following environment variables are used as argument defaults:
 
 # Commands
 
-For more information on a command, run `dbt-cloud <command> --help`. For more information on the API endpoints, see [dbt Cloud API V2 docs](https://docs.getdbt.com/dbt-cloud/api-v2#/) and [dbt Cloud Metadata API docs](https://docs.getdbt.com/docs/dbt-cloud/dbt-cloud-api/metadata/metadata-overview).
+For more information on a command, run `dbt-cloud <command> --help`. For more information on the API endpoints, see [dbt Cloud API V3 docs](https://docs.getdbt.com/dbt-cloud/api-v3) and [dbt Cloud Metadata API docs](https://docs.getdbt.com/docs/dbt-cloud/dbt-cloud-api/metadata/metadata-overview).
 
 
 | Group        | Command                                               | Implemented | API endpoint                                        |
 | ------------ | ----------------------------------------------------- | -------------------------------------------------- | ----------- | 
 | Account      | [dbt-cloud account get](#dbt-cloud-account-get)       | ✅           | GET `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/`                   | 
-| Account      | [dbt-cloud account list](#dbt-cloud-account-list)     | ✅           | GET `https://{dbt_cloud_host}/api/v2/accounts/`                                | 
+| Account      | [dbt-cloud account list](#dbt-cloud-account-list)     | ✅           | GET `https://{dbt_cloud_host}/api/v3/accounts/`                                | 
 | Audit log    | [dbt-cloud audit-log get](#dbt-cloud-audit-log-get)   | ✅           | GET `https://{dbt_cloud_host}/api/v3/audit-logs/`                              | 
-| Project      | [dbt-cloud project create](#dbt-cloud-project-create) | ✅           | POST `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/projects/`         | 
+| Project      | [dbt-cloud project create](#dbt-cloud-project-create) | ✅           | POST `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/`         | 
 | Project      | [dbt-cloud project delete](#dbt-cloud-project-delete) | ✅           | DELETE `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/{id}/`  |
-| Project      | [dbt-cloud project get](#dbt-cloud-project-get)       | ✅           | GET `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/projects/{id}/`     | 
-| Project      | [dbt-cloud project list](#dbt-cloud-project-list)     | ✅           | GET `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/projects/`          |  
-| Project      | [dbt-cloud project update](#dbt-cloud-project-update) | ❌           | POST `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/projects/{id}/`     | 
-| Environment  | [dbt-cloud environment create](#dbt-cloud-environment-create) | ❌          | POST `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/environments/` | 
-| Environment  | [dbt-cloud environment delete](#dbt-cloud-environment-delete) | ✅ | DELETE `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/environments/{id}/` |  
-| Environment  | [dbt-cloud environment get](#dbt-cloud-environment-get) | ✅ | GET `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/environments/{id}/` |  
-| Environment  | [dbt-cloud environment list](#dbt-cloud-environment-list) | ✅ | GET `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/environments/` |  
-| Environment  | [dbt-cloud environment update](#dbt-cloud-environment-update) | ❌ | POST `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/environments/{id}/` |  
-| Connection  | [dbt-cloud connection create](#dbt-cloud-connection-create) | ❌ | POST `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/connections/` | 
-| Connection  | [dbt-cloud connection delete](#dbt-cloud-connection-delete) | ❌ | DELETE `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/connections/{id}/` | 
-| Connection  | [dbt-cloud connection get](#dbt-cloud-connection-get) | ✅ | GET `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/connections/{id}/` | 
-| Connection  | [dbt-cloud connection list](#dbt-cloud-connection-list) | ✅ | GET `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/connections/` | 
-| Connection  | [dbt-cloud connection update](#dbt-cloud-connection-update) | ❌ | POST `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/connections/{id}/` | 
+| Project      | [dbt-cloud project get](#dbt-cloud-project-get)       | ✅           | GET `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/{id}/`     | 
+| Project      | [dbt-cloud project list](#dbt-cloud-project-list)     | ✅           | GET `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/`          |  
+| Project      | [dbt-cloud project update](#dbt-cloud-project-update) | ❌           | POST `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/{id}/`     | 
+| Environment  | [dbt-cloud environment create](#dbt-cloud-environment-create) | ❌          | POST `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/environments/` | 
+| Environment  | [dbt-cloud environment delete](#dbt-cloud-environment-delete) | ✅ | DELETE `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/environments/{id}/` |  
+| Environment  | [dbt-cloud environment get](#dbt-cloud-environment-get) | ✅ | GET `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/environments/{id}/` |  
+| Environment  | [dbt-cloud environment list](#dbt-cloud-environment-list) | ✅ | GET `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/environments/` |  
+| Environment  | [dbt-cloud environment update](#dbt-cloud-environment-update) | ❌ | POST `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/environments/{id}/` |  
+| Connection  | [dbt-cloud connection create](#dbt-cloud-connection-create) | ❌ | POST `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/{project_id}/connections/` | 
+| Connection  | [dbt-cloud connection delete](#dbt-cloud-connection-delete) | ❌ | DELETE `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/{project_id}/connections/{id}/` | 
+| Connection  | [dbt-cloud connection get](#dbt-cloud-connection-get) | ✅ | GET `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/{project_id}/connections/{id}/` | 
+| Connection  | [dbt-cloud connection list](#dbt-cloud-connection-list) | ✅ | GET `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/{project_id}/connections/` | 
+| Connection  | [dbt-cloud connection update](#dbt-cloud-connection-update) | ❌ | POST `https://{dbt_cloud_host}/api/v3/accounts/{account_id}/projects/{project_id}/connections/{id}/` | 
 | Repository  | [dbt-cloud repository create](#dbt-cloud-repository-create) | ❌ | POST `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/repositories/` | 
 | Repository  | [dbt-cloud repository delete](#dbt-cloud-repository-delete) | ❌ | DELETE `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/repositories/{id}/` | 
 | Repository  | [dbt-cloud repository get](#dbt-cloud-repository-get) | ❌ | GET `https://{dbt_cloud_host}/api/v2/accounts/{account_id}/repositories/{id}/` | 
@@ -198,11 +198,11 @@ dbt-cloud environment get --account-id 54321 --environment-id 67890
 
 
 ## dbt-cloud connection list
-This command retrievies details of dbt Cloud database connections in a given account.
+This command retrievies details of dbt Cloud database connections in a given project.
 
 ### Usage
 ```bash
-dbt-cloud connection list --account-id 54321 --limit 1
+dbt-cloud connection list --account-id 54321 --project-id 123467 --limit 1
 ```
 
 [Click to view sample response](tests/data/connection_list_response.json)
@@ -212,7 +212,7 @@ This command retrievies the details of a dbt Cloud database connection.
 
 ### Usage
 ```bash
-dbt-cloud connection get --account-id 54321 --connection-id 56901
+dbt-cloud connection get --account-id 54321 --project-id 123467 --connection-id 56901
 ```
 
 [Click to view sample response](tests/data/connection_get_response.json)
@@ -250,7 +250,7 @@ This command returns a list of jobs in the account.
 
 ### Usage
 ```bash
-dbt-cloud job list --account-id 123456 --project-id 123457
+dbt-cloud job list --account-id 123456 --project-id 123457 --limit 2
 ```
 
 [Click to view sample response](tests/data/job_list_response.json)

diff --git a/dbt_cloud/command/account/get.py b/dbt_cloud/command/account/get.py
@@ -1,10 +1,13 @@
 import requests
+from pydantic import PrivateAttr
 from dbt_cloud.command.command import DbtCloudAccountCommand
 
 
 class DbtCloudAccountGetCommand(DbtCloudAccountCommand):
     """Retrieves dbt Cloud account information."""
 
+    _api_version: str = PrivateAttr("v2")
+
     def execute(self) -> requests.Response:
         response = requests.get(url=self.api_url, headers=self.request_headers)
         return response
diff --git a/dbt_cloud/command/command.py b/dbt_cloud/command/command.py
@@ -3,7 +3,12 @@
 from mergedeep import merge
 from pydantic import validator, BaseModel, PrivateAttr
 from dbt_cloud.serde import json_to_dict
-from dbt_cloud.field import API_TOKEN_FIELD, ACCOUNT_ID_FIELD, DBT_CLOUD_HOST_FIELD
+from dbt_cloud.field import (
+    API_TOKEN_FIELD,
+    ACCOUNT_ID_FIELD,
+    PROJECT_ID_FIELD,
+    DBT_CLOUD_HOST_FIELD,
+)
 
 
 def translate_click_options(**kwargs) -> dict:
@@ -94,7 +99,7 @@ def get_description(cls) -> str:
 class DbtCloudCommand(ClickBaseModel):
     api_token: str = API_TOKEN_FIELD
     dbt_cloud_host: str = DBT_CLOUD_HOST_FIELD
-    _api_version: str = PrivateAttr("v2")
+    _api_version: str = PrivateAttr("v3")
 
     @property
     def request_headers(self) -> dict:
@@ -122,3 +127,11 @@ class DbtCloudAccountCommand(DbtCloudCommand):
     @property
     def api_url(self) -> str:
         return f"{super().api_url}/accounts/{self.account_id}"
+
+
+class DbtCloudProjectCommand(DbtCloudAccountCommand):
+    project_id: int = PROJECT_ID_FIELD
+
+    @property
+    def api_url(self) -> str:
+        return f"{super().api_url}/projects/{self.project_id}"
diff --git a/dbt_cloud/command/connection/get.py b/dbt_cloud/command/connection/get.py
@@ -1,9 +1,9 @@
 import requests
 from pydantic import Field
-from dbt_cloud.command.command import DbtCloudAccountCommand
+from dbt_cloud.command.command import DbtCloudProjectCommand
 
 
-class DbtCloudConnectionGetCommand(DbtCloudAccountCommand):
+class DbtCloudConnectionGetCommand(DbtCloudProjectCommand):
     """Retrievies the details of a dbt Cloud database connection."""
 
     connection_id: int = Field(description="ID of the connection.")

diff --git a/dbt_cloud/command/connection/list.py b/dbt_cloud/command/connection/list.py
@@ -1,12 +1,11 @@
 import requests
 from typing import Optional
-from pydantic import Field
-from dbt_cloud.command.command import DbtCloudAccountCommand
-from dbt_cloud.field import LIMIT_FIELD, OFFSET_FIELD
+from dbt_cloud.command.command import DbtCloudProjectCommand
+from dbt_cloud.field import LIMIT_FIELD, OFFSET_FIELD, PROJECT_ID_FIELD
 
 
-class DbtCloudConnectionListCommand(DbtCloudAccountCommand):
-    """Retrievies details of dbt Cloud database connections in a given account."""
+class DbtCloudConnectionListCommand(DbtCloudProjectCommand):
+    """Retrievies details of dbt Cloud database connections in a given project."""
 
     limit: Optional[int] = LIMIT_FIELD
     offset: Optional[int] = OFFSET_FIELD

diff --git a/dbt_cloud/command/environment/get.py b/dbt_cloud/command/environment/get.py
@@ -9,7 +9,6 @@ class DbtCloudEnvironmentGetCommand(DbtCloudAccountCommand):
 
     environment_id: int = ENVIRONMENT_ID_FIELD
     account_id: int = ACCOUNT_ID_FIELD
-    _api_version: str = PrivateAttr("v2")
 
     @property
     def api_url(self) -> str:

diff --git a/dbt_cloud/command/job/create.py b/dbt_cloud/command/job/create.py
@@ -1,7 +1,7 @@
 import requests
 from enum import Enum
 from typing import Optional, List
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 from dbt_cloud.command.command import DbtCloudAccountCommand, ClickBaseModel
 from dbt_cloud.field import PythonLiteralOption, PROJECT_ID_FIELD, ENVIRONMENT_ID_FIELD
 
@@ -54,6 +54,8 @@ class DbtCloudJobSchedule(ClickBaseModel):
 class DbtCloudJobCreateCommand(DbtCloudAccountCommand):
     """Creates a job in a dbt Cloud project."""
 
+    _api_version: str = PrivateAttr("v2")
+
     id: Optional[int] = Field(
         default=None,
         exclude_from_click_options=True,

diff --git a/dbt_cloud/command/job/delete.py b/dbt_cloud/command/job/delete.py
@@ -1,13 +1,13 @@
-import os
 import requests
-from pydantic import Field
+from pydantic import PrivateAttr
 from dbt_cloud.command.command import DbtCloudAccountCommand
 from dbt_cloud.field import JOB_ID_FIELD
 
 
 class DbtCloudJobDeleteCommand(DbtCloudAccountCommand):
     """Deletes a job from a dbt Cloud project."""
 
+    _api_version: str = PrivateAttr("v2")
     job_id: int = JOB_ID_FIELD
 
     @property

diff --git a/dbt_cloud/command/job/get.py b/dbt_cloud/command/job/get.py
@@ -1,14 +1,14 @@
-import os
 import requests
 from typing import Optional
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 from dbt_cloud.command.command import DbtCloudAccountCommand
 from dbt_cloud.field import JOB_ID_FIELD
 
 
 class DbtCloudJobGetCommand(DbtCloudAccountCommand):
     """Returns the details of a dbt Cloud job."""
 
+    _api_version: str = PrivateAttr("v2")
     job_id: int = JOB_ID_FIELD
     order_by: Optional[str] = Field(
         description="Field to order the result by. Use '-' to indicate reverse order."

diff --git a/dbt_cloud/command/job/list.py b/dbt_cloud/command/job/list.py
@@ -1,15 +1,18 @@
 import requests
 from typing import Optional
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 from dbt_cloud.command.command import DbtCloudAccountCommand
+from dbt_cloud.field import LIMIT_FIELD
 
 
 class DbtCloudJobListCommand(DbtCloudAccountCommand):
     """Returns a list of jobs in the account."""
 
+    _api_version: str = PrivateAttr("v2")
     order_by: Optional[str] = Field(
         description="Field to order the result by. Use - to indicate reverse order."
     )
+    limit: Optional[int] = LIMIT_FIELD
     project_id: Optional[str] = Field(description="Filter jobs by project ID.")
 
     @property
@@ -20,6 +23,10 @@ def execute(self) -> requests.Response:
         response = requests.get(
             url=self.api_url,
             headers=self.request_headers,
-            params={"order_by": self.order_by, "project_id": self.project_id},
+            params={
+                "order_by": self.order_by,
+                "project_id": self.project_id,
+                "limit": self.limit,
+            },
         )
         return response
diff --git a/dbt_cloud/command/job/run.py b/dbt_cloud/command/job/run.py
@@ -1,13 +1,14 @@
 import requests
 from typing import Optional, List
-from pydantic import Field, validator
+from pydantic import Field, validator, PrivateAttr
 from dbt_cloud.command.command import DbtCloudAccountCommand
 from dbt_cloud.field import JOB_ID_FIELD, PythonLiteralOption
 
 
 class DbtCloudJobRunCommand(DbtCloudAccountCommand):
     """Triggers a dbt Cloud job run and returns a status JSON response."""
 
+    _api_version: str = PrivateAttr("v2")
     job_id: int = JOB_ID_FIELD
     cause: str = Field(
         default="Triggered via API",