diff --git a/.github/actions/bootstrap/action.yml b/.github/actions/bootstrap/action.yml index 45f608240e..24e69ca30a 100644 --- a/.github/actions/bootstrap/action.yml +++ b/.github/actions/bootstrap/action.yml @@ -4,9 +4,9 @@ runs: using: composite steps: - name: Setup node - uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 + uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 with: - node-version: 16 + node-version: 20 - name: Get yarn cache directory path shell: bash @@ -14,7 +14,7 @@ runs: working-directory: utils run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT - - uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c + - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 id: yarn-cache with: path: ${{ steps.yarn-cache-dir-path.outputs.dir }} diff --git a/.github/workflows/pr-merged.yml b/.github/workflows/pr-merged.yml index 3000b9c819..ff37f490d1 100644 --- a/.github/workflows/pr-merged.yml +++ b/.github/workflows/pr-merged.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 with: token: ${{ secrets.OPENSOURCE_BOT_TOKEN }} ref: "release" diff --git a/.github/workflows/pr-project-board.yml b/.github/workflows/pr-project-board.yml index 6e6fc001e3..6d6c04a1aa 100644 --- a/.github/workflows/pr-project-board.yml +++ b/.github/workflows/pr-project-board.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - name: Move PR to Project Board for triage run: | diff --git a/.github/workflows/preview-links.yml b/.github/workflows/preview-links.yml index 1e5a9407fc..665d3eff74 100644 --- a/.github/workflows/preview-links.yml +++ b/.github/workflows/preview-links.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - name: Setup workspace uses: "./.github/actions/bootstrap" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c720598e54..1773cc5925 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,7 +16,7 @@ jobs: # Checkout fetch-depth: 2 because there's a check to see if package.json # was updated, and need at least 2 commits for the check to function properly - name: Checkout repo - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 with: fetch-depth: 2 diff --git a/.github/workflows/repolinter.yml b/.github/workflows/repolinter.yml index fe5b5e0039..4623f777cd 100644 --- a/.github/workflows/repolinter.yml +++ b/.github/workflows/repolinter.yml @@ -15,14 +15,14 @@ jobs: steps: - name: Test Default Branch id: default-branch - uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea with: script: | const data = await github.rest.repos.get(context.repo) return data.data && data.data.default_branch === context.ref.split('/').slice(-1)[0] - name: Checkout Self if: ${{ steps.default-branch.outputs.result == 'true' }} - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - name: Run Repolinter if: ${{ steps.default-branch.outputs.result == 'true' }} uses: newrelic/repolinter-action@3f4448f855c351e9695b24524a4111c7847b84cb diff --git a/.github/workflows/reusable.quickstart_submission.yml b/.github/workflows/reusable.quickstart_submission.yml index 6aaaff6cf0..bb76fdbe72 100644 --- a/.github/workflows/reusable.quickstart_submission.yml +++ b/.github/workflows/reusable.quickstart_submission.yml @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # Ensure we have the most recent commit to `main` with: ref: "main" @@ -55,7 +55,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # Ensure we have the most recent commit to `main` with: ref: "main" @@ -80,7 +80,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # Ensure we have the most recent commit to `main` with: ref: "main" @@ -105,7 +105,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # Ensure we have the most recent commit to `main` with: ref: "main" @@ -130,7 +130,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # Ensure we have the most recent commit to `main` with: ref: "main" diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index b0c4a83e66..ea0bfdabce 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - name: Setup workspace uses: "./.github/actions/bootstrap" diff --git a/.github/workflows/submit-gate.yml b/.github/workflows/submit-gate.yml index f5d7757511..64ec1e8d71 100644 --- a/.github/workflows/submit-gate.yml +++ b/.github/workflows/submit-gate.yml @@ -21,6 +21,6 @@ jobs: run: echo "${{ github.event.pull_request.number }}" > pr_number_submit.txt - name: upload artifact - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 with: path: pr_number_submit.txt diff --git a/.github/workflows/validate_data_sources.yml b/.github/workflows/validate_data_sources.yml index 9f39d62947..4d0e542d2c 100644 --- a/.github/workflows/validate_data_sources.yml +++ b/.github/workflows/validate_data_sources.yml @@ -24,7 +24,7 @@ jobs: echo "pr-number=$PR_NUMBER" >> $GITHUB_ENV - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 with: ref: "refs/pull/${{ env.pr-number }}/merge" diff --git a/.github/workflows/validate_install_plans.yml b/.github/workflows/validate_install_plans.yml index 3c8dc5e091..92d65891a7 100644 --- a/.github/workflows/validate_install_plans.yml +++ b/.github/workflows/validate_install_plans.yml @@ -24,7 +24,7 @@ jobs: echo "pr-number=$PR_NUMBER" >> $GITHUB_ENV - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 with: ref: "refs/pull/${{ env.pr-number }}/merge" diff --git a/.github/workflows/validate_packs.yml b/.github/workflows/validate_packs.yml index b80f9f71ef..2df675b546 100644 --- a/.github/workflows/validate_packs.yml +++ b/.github/workflows/validate_packs.yml @@ -27,7 +27,7 @@ jobs: echo "pr-number=$PR_NUMBER" >> $GITHUB_ENV - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 with: ref: "refs/pull/${{ env.pr-number }}/merge" @@ -62,7 +62,7 @@ jobs: echo "pr-number=$PR_NUMBER" >> $GITHUB_ENV - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 with: ref: "refs/pull/${{ env.pr-number }}/merge" @@ -97,7 +97,7 @@ jobs: echo "pr-number=$PR_NUMBER" >> $GITHUB_ENV - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 with: ref: "refs/pull/${{ env.pr-number }}/merge" @@ -132,7 +132,7 @@ jobs: echo "pr-number=$PR_NUMBER" >> $GITHUB_ENV - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 with: ref: "refs/pull/${{ env.pr-number }}/merge" @@ -180,7 +180,7 @@ jobs: echo "pr-number=$PR_NUMBER" >> $GITHUB_ENV - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 with: ref: "refs/pull/${{ env.pr-number }}/merge" @@ -220,7 +220,7 @@ jobs: echo "pr-number=$PR_NUMBER" >> $GITHUB_ENV - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 with: ref: "refs/pull/${{ env.pr-number }}/merge" @@ -262,7 +262,7 @@ jobs: echo "pr-number=$PR_NUMBER" >> $GITHUB_ENV - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 with: ref: "refs/pull/${{ env.pr-number }}/merge" @@ -296,7 +296,7 @@ jobs: echo "pr-number=$PR_NUMBER" >> $GITHUB_ENV - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 with: ref: "refs/pull/${{ env.pr-number }}/merge" diff --git a/.github/workflows/validation_gate.yml b/.github/workflows/validation_gate.yml index a06b325d9a..2285f324bb 100644 --- a/.github/workflows/validation_gate.yml +++ b/.github/workflows/validation_gate.yml @@ -18,6 +18,6 @@ jobs: run: echo "${{ github.event.pull_request.number }}" > pr_number.txt - name: upload artifact - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 with: path: pr_number.txt diff --git a/.github/workflows/yaml-lint.yml b/.github/workflows/yaml-lint.yml index 9f49d59877..d190d08e17 100644 --- a/.github/workflows/yaml-lint.yml +++ b/.github/workflows/yaml-lint.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - name: Run yamllint run: | diff --git a/alert-policies/dbt-cloud/dbt-failed-test-row.yml b/alert-policies/dbt-cloud/dbt-failed-test-row.yml new file mode 100644 index 0000000000..162d6f94c6 --- /dev/null +++ b/alert-policies/dbt-cloud/dbt-failed-test-row.yml @@ -0,0 +1,27 @@ +name: dbt Failed Test Row + +description: |+ + Generates alerts for rows in your database that fail dbt tests. This alert is useful for sending notifications with additional context to business users. + +type: STATIC +nrql: + query: "select count(*) from dbt_failed_test_row facet team, name, description, field_1, field_2, field_3, field_4, field_5, field_6, field_7, field_8, field_9, field_10, slack_mentions" + +# Function used to aggregate the NRQL query value(s) for comparison to the terms.threshold (Default: SINGLE_VALUE) +valueFunction: SINGLE_VALUE + +# List of Critical and Warning thresholds for the condition +terms: + - priority: CRITICAL + # Operator used to compare against the threshold. + operator: ABOVE + # Value that triggers a violation + threshold: 5 + # Time in seconds; 120 - 3600 + thresholdDuration: 300 + # How many data points must be in violation for the duration + thresholdOccurrences: ALL + +# Duration after which a violation automatically closes +# Time in seconds; 300 - 2592000 (Default: 86400 [1 day]) +violationTimeLimitSeconds: 86400 \ No newline at end of file diff --git a/alert-policies/dbt-cloud/dbt-job-failed.yml b/alert-policies/dbt-cloud/dbt-job-failed.yml new file mode 100644 index 0000000000..4407781958 --- /dev/null +++ b/alert-policies/dbt-cloud/dbt-job-failed.yml @@ -0,0 +1,27 @@ +name: dbt Job Failed + +description: |+ + Generates critical alerts for dbt jobs that fail. A failed job is any job that does not return status = 10 from dbt Cloud. + +type: STATIC +nrql: + query: "select count(*) from dbt_job_run where run_status != 10 facet run_team, project_name, environment_name, job_name, run_started_at, run_status_humanized, run_status_message, run_href" + +# Function used to aggregate the NRQL query value(s) for comparison to the terms.threshold (Default: SINGLE_VALUE) +valueFunction: SINGLE_VALUE + +# List of Critical and Warning thresholds for the condition +terms: + - priority: CRITICAL + # Operator used to compare against the threshold. + operator: ABOVE + # Value that triggers a violation + threshold: 3 + # Time in seconds; 120 - 3600 + thresholdDuration: 300 + # How many data points must be in violation for the duration + thresholdOccurrences: ALL + +# Duration after which a violation automatically closes +# Time in seconds; 300 - 2592000 (Default: 86400 [1 day]) +violationTimeLimitSeconds: 86400 \ No newline at end of file diff --git a/alert-policies/dbt-cloud/dbt-resource-failed.yml b/alert-policies/dbt-cloud/dbt-resource-failed.yml new file mode 100644 index 0000000000..0dc7587fb3 --- /dev/null +++ b/alert-policies/dbt-cloud/dbt-resource-failed.yml @@ -0,0 +1,27 @@ +name: dbt Resource Failed + +description: |+ + Generates critical alerts for dbt resources fail with an error. + +type: STATIC +nrql: + query: "select count(*) from dbt_resource_run where status in ('fail', 'error') facet team, run_team, project_name, environment_name, job_name, name, resource_type, status, error, slack_mentions, run_href" + +# Function used to aggregate the NRQL query value(s) for comparison to the terms.threshold (Default: SINGLE_VALUE) +valueFunction: SINGLE_VALUE + +# List of Critical and Warning thresholds for the condition +terms: + - priority: CRITICAL + # Operator used to compare against the threshold. + operator: ABOVE + # Value that triggers a violation + threshold: 1 + # Time in seconds; 120 - 3600 + thresholdDuration: 300 + # How many data points must be in violation for the duration + thresholdOccurrences: ALL + +# Duration after which a violation automatically closes +# Time in seconds; 300 - 2592000 (Default: 86400 [1 day]) +violationTimeLimitSeconds: 86400 \ No newline at end of file diff --git a/alert-policies/dbt-cloud/dbt-test-warning.yml b/alert-policies/dbt-cloud/dbt-test-warning.yml new file mode 100644 index 0000000000..bdda234951 --- /dev/null +++ b/alert-policies/dbt-cloud/dbt-test-warning.yml @@ -0,0 +1,27 @@ +name: dbt Test Warning + +description: |+ + Generates alerts for dbt test warnings. A status of "warn" will trigger this alert. + +type: STATIC +nrql: + query: "select count(*) from dbt_resource_run where status = 'warn' facet team, run_team, project_name, environment_name, job_name, name, resource_type, status, error, slack_mentions, run_href" + +# Function used to aggregate the NRQL query value(s) for comparison to the terms.threshold (Default: SINGLE_VALUE) +valueFunction: SINGLE_VALUE + +# List of Critical and Warning thresholds for the condition +terms: + - priority: CRITICAL + # Operator used to compare against the threshold. + operator: ABOVE + # Value that triggers a violation + threshold: 5 + # Time in seconds; 120 - 3600 + thresholdDuration: 300 + # How many data points must be in violation for the duration + thresholdOccurrences: ALL + +# Duration after which a violation automatically closes +# Time in seconds; 300 - 2592000 (Default: 86400 [1 day]) +violationTimeLimitSeconds: 86400 \ No newline at end of file diff --git a/dashboards/dbt-cloud/dbt-cloud.json b/dashboards/dbt-cloud/dbt-cloud.json new file mode 100644 index 0000000000..4cecad79bf --- /dev/null +++ b/dashboards/dbt-cloud/dbt-cloud.json @@ -0,0 +1,790 @@ +{ + "name": "dbt Observability", + "description": "dbt Observability Dashboard with metrics, history and status by run, resource, test and failed test rows", + "pages": [ + { + "name": "dbt Observability", + "description": null, + "widgets": [ + { + "title": "", + "layout": { + "column": 1, + "row": 1, + "width": 2, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select \n percentage(count(*), where run_status = 10) as `Job Pass Rate`\nfrom dbt_job_run \nwhere run_team in ({{select_run_team}})\nand project_name in ({{select_project}})\nand environment_name in ({{select_environment}})\nand job_name in ({{select_job}})\nsince 7 days ago\n\n\n" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "", + "layout": { + "column": 3, + "row": 1, + "width": 2, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select percentage(count(*), where status = 'success') as `Model Pass Rate`\nfrom dbt_resource_run \nwhere resource_type != 'test'\nand status not in ('None', 'skipped')\nand run_team in ({{select_run_team}})\nand project_name in ({{select_project}})\nand environment_name in ({{select_environment}})\nand job_name in ({{select_job}})\nand team in ({{select_team}})\nsince 7 day ago\n\n\n" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "", + "layout": { + "column": 5, + "row": 1, + "width": 2, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select percentage(count(*), where status = 'pass') as `Test Pass Rate`\nfrom dbt_resource_run \nwhere resource_type = 'test'\nand status not in ('None', 'skipped')\nand run_team in ({{select_run_team}})\nand project_name in ({{select_project}})\nand environment_name in ({{select_environment}})\nand job_name in ({{select_job}})\nand team in ({{select_team}})\nsince 7 days ago\n\n\n" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "", + "layout": { + "column": 7, + "row": 1, + "width": 2, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select uniqueCount(run_id) as `Total Runs`\nfrom dbt_job_run\nwhere run_team in ({{select_run_team}})\nand project_name in ({{select_project}})\nand environment_name in ({{select_environment}})\nand job_name in ({{select_job}})\nsince 7 days ago " + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "", + "layout": { + "column": 9, + "row": 1, + "width": 2, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select uniqueCount(unique_id) as `Total Model Runs`\nfrom dbt_resource_run\nwhere run_team in ({{select_run_team}})\nand project_name in ({{select_project}})\nand environment_name in ({{select_environment}})\nand job_name in ({{select_job}})\nand team in ({{select_team}})\nand resource_type != 'test'\nand status not in ('None', 'skipped')\nsince 7 days ago\n\n" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "", + "layout": { + "column": 11, + "row": 1, + "width": 2, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select count(*) as `Total Test Runs`\nfrom dbt_resource_run\nwhere run_team in ({{select_run_team}})\nand project_name in ({{select_project}})\nand environment_name in ({{select_environment}})\nand job_name in ({{select_job}})\nand team in ({{select_team}})\nand resource_type = 'test'\nand status not in ('None', 'skipped')\nsince 7 days ago\n" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "", + "layout": { + "column": 1, + "row": 2, + "width": 2, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select uniqueCount(job_id) as `Total Jobs`\nfrom dbt_job_run\nwhere run_team in ({{select_run_team}})\nand project_name in ({{select_project}})\nand environment_name in ({{select_environment}})\nand job_name in ({{select_job}})\nsince 7 days ago\n" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "", + "layout": { + "column": 3, + "row": 2, + "width": 2, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select \n uniqueCount(unique_id) as `Total Models`\nfrom dbt_resource_run\nwhere resource_type != 'test'\nand run_team in ({{select_run_team}})\nand project_name in ({{select_project}})\nand environment_name in ({{select_environment}})\nand job_name in ({{select_job}})\nand team in ({{select_team}})\nsince 7 days ago" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "", + "layout": { + "column": 5, + "row": 2, + "width": 2, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select \n uniqueCount(unique_id) as `Total Tests`\nfrom dbt_resource_run\nwhere resource_type = 'test'\nand run_team in ({{select_run_team}})\nand project_name in ({{select_project}})\nand environment_name in ({{select_environment}})\nand job_name in ({{select_job}})\nand team in ({{select_team}})\nsince 7 days ago" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "", + "layout": { + "column": 7, + "row": 2, + "width": 6, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": " " + } + }, + { + "title": "Jobs by Status Message", + "layout": { + "column": 1, + "row": 3, + "width": 4, + "height": 2 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.area" + }, + "rawConfiguration": { + "colors": { + "seriesOverrides": [ + { + "color": "#35b171", + "seriesName": "Success" + }, + { + "color": "#d91717", + "seriesName": "Error" + }, + { + "color": "#d6d1d1", + "seriesName": "Cancelled" + } + ] + }, + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select \n count(*)\nfrom dbt_job_run\nwhere run_team in ({{select_run_team}})\nand project_name in ({{select_project}})\nand environment_name in ({{select_environment}})\nand job_name in ({{select_job}})\nfacet run_status_humanized TIMESERIES \nsince 7 days ago " + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Models by Status", + "layout": { + "column": 5, + "row": 3, + "width": 4, + "height": 2 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.area" + }, + "rawConfiguration": { + "colors": { + "seriesOverrides": [ + { + "color": "#35b171", + "seriesName": "success" + }, + { + "color": "#d91717", + "seriesName": "error" + }, + { + "color": "#d6d1d1", + "seriesName": "skipped" + } + ] + }, + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select \n count(*)\nfrom dbt_resource_run\nwhere resource_type != 'test'\nand status != 'None'\nand run_team in ({{select_run_team}})\nand project_name in ({{select_project}})\nand environment_name in ({{select_environment}})\nand job_name in ({{select_job}})\nand team in ({{select_team}})\nfacet status\nTIMESERIES \nsince 7 days ago" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Tests by Status", + "layout": { + "column": 9, + "row": 3, + "width": 4, + "height": 2 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.area" + }, + "rawConfiguration": { + "colors": { + "seriesOverrides": [ + { + "color": "#35b171", + "seriesName": "pass" + }, + { + "color": "#d91717", + "seriesName": "fail" + }, + { + "color": "#d7c614", + "seriesName": "warn" + }, + { + "color": "#d6d1d1", + "seriesName": "skipped" + }, + { + "color": "#d91717", + "seriesName": "error" + } + ] + }, + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select \n count(*)\nfrom dbt_resource_run\nwhere resource_type = 'test'\nand status != 'None'\nand run_team in ({{select_run_team}})\nand project_name in ({{select_project}})\nand environment_name in ({{select_environment}})\nand job_name in ({{select_job}})\nand team in ({{select_team}})\nfacet status\nTIMESERIES \nsince 7 days ago" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Failed Runs", + "layout": { + "column": 1, + "row": 5, + "width": 12, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.table" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select \n run_team,\n project_name,\n environment_name,\n job_name,\n run_started_at,\n run_status,\n run_status_humanized,\n run_status_message,\n run_href\nfrom dbt_job_run\nwhere run_status != 10\nand run_team in ({{select_run_team}})\nand project_name in ({{select_project}})\nand environment_name in ({{select_environment}})\nand job_name in ({{select_job}})\nsince 7 days ago\nlimit 100" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Failed Models", + "layout": { + "column": 1, + "row": 8, + "width": 12, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.table" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select \n name,\n run_team,\n project_name,\n environment_name,\n job_name,\n team,\n run_started_at,\n database_name,\n schema_name,\n alias,\n error,\n run_href\nfrom dbt_resource_run\nwhere resource_type != 'test'\nand status not in ('None', 'skipped', 'success')\nand run_team in ({{select_run_team}})\nand project_name in ({{select_project}})\nand environment_name in ({{select_environment}})\nand job_name in ({{select_job}})\nand team in ({{select_team}})\n\nsince 7 days ago\nlimit 100" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Failed Tests", + "layout": { + "column": 1, + "row": 11, + "width": 12, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.table" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select \n team,\n job_id, \n database_name, \n schema_name, \n test_model_name,\n name, \n status, \n error,\n test_parameters,\n run_started_at,\n run_href \nfrom dbt_resource_run \nwhere resource_type = 'test'\nand status not in ('pass', 'None', 'skipped')\nand run_team in ({{select_run_team}})\nand project_name in ({{select_project}})\nand environment_name in ({{select_environment}})\nand job_name in ({{select_job}})\nand team in ({{select_team}})\n\nlimit 100\nsince 1 day ago" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + } + ] + }, + { + "name": "Tests", + "description": null, + "widgets": [ + { + "title": "", + "layout": { + "column": 1, + "row": 1, + "width": 12, + "height": 1 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "Click on the three visuals below to drill down to tests for individual models, tests, and statuses. " + } + }, + { + "title": "Models", + "layout": { + "column": 1, + "row": 2, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.table" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": true + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select \n latest(status) as status\nfrom dbt_resource_run \nwhere resource_type = 'test' \nand status != 'None'\nand run_team in ({{select_run_team}})\nand project_name in ({{select_project}})\nand environment_name in ({{select_environment}})\nand job_name in ({{select_job}})\nand team in ({{select_team}})\nfacet test_model_name \nsince 7 days ago limit 1000" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Tests", + "layout": { + "column": 5, + "row": 2, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.table" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select \n latest(status) as status\nfrom dbt_resource_run \nwhere resource_type = 'test' \nand status != 'None'\nand run_team in ({{select_run_team}})\nand project_name in ({{select_project}})\nand environment_name in ({{select_environment}})\nand job_name in ({{select_job}})\nand team in ({{select_team}})\nfacet name \nsince 1 day ago limit 1000" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Status", + "layout": { + "column": 9, + "row": 2, + "width": 4, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.bar" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select\n count(*) as count\nfrom dbt_resource_run\nwhere resource_type = 'test'\nand run_team in ({{select_run_team}})\nand project_name in ({{select_project}})\nand environment_name in ({{select_environment}})\nand job_name in ({{select_job}})\nand team in ({{select_team}})\nfacet status\nsince 7 days ago" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Tests by Status", + "layout": { + "column": 1, + "row": 5, + "width": 12, + "height": 3 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.area" + }, + "rawConfiguration": { + "colors": { + "seriesOverrides": [ + { + "color": "#35b171", + "seriesName": "pass" + }, + { + "color": "#d91717", + "seriesName": "fail" + }, + { + "color": "#d7c614", + "seriesName": "warn" + }, + { + "color": "#d6d1d1", + "seriesName": "skipped" + } + ] + }, + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select \n count(*)\nfrom dbt_resource_run\nwhere resource_type = 'test'\nand status not in ('None')\nand run_team in ({{select_run_team}})\nand project_name in ({{select_project}})\nand environment_name in ({{select_environment}})\nand job_name in ({{select_job}})\nand team in ({{select_team}})\nfacet status\nTIMESERIES \nsince 7 days ago" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Failed test rows", + "layout": { + "column": 1, + "row": 8, + "width": 12, + "height": 4 + }, + "linkedEntityGuids": null, + "visualization": { + "id": "viz.table" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountIds": [], + "query": "select \n alias,\n test_model_name,\n field_1,\n field_2,\n field_3,\n field_4,\n field_5\nfrom dbt_failed_test_row\nsince 1 day ago limit 1000" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + } + ] + } + ], + "variables": [ + { + "name": "select_run_team", + "items": null, + "defaultValues": [ + { + "value": { + "string": "*" + } + } + ], + "nrqlQuery": { + "accountIds": [], + "query": "select uniques(run_team) from dbt_job_run since 7 days ago" + }, + "options": { + "ignoreTimeRange": false + }, + "title": "Run Team", + "type": "NRQL", + "isMultiSelection": true, + "replacementStrategy": "STRING" + }, + { + "name": "select_project", + "items": null, + "defaultValues": [ + { + "value": { + "string": "*" + } + } + ], + "nrqlQuery": { + "accountIds": [], + "query": "select uniques(project_name) from dbt_job_run since 7 days ago" + }, + "options": { + "ignoreTimeRange": false + }, + "title": "Project", + "type": "NRQL", + "isMultiSelection": true, + "replacementStrategy": "STRING" + }, + { + "name": "select_environment", + "items": null, + "defaultValues": [ + { + "value": { + "string": "*" + } + } + ], + "nrqlQuery": { + "accountIds": [], + "query": "select uniques(environment_name) from dbt_job_run since 7 days ago" + }, + "options": { + "ignoreTimeRange": false + }, + "title": "Environment", + "type": "NRQL", + "isMultiSelection": true, + "replacementStrategy": "STRING" + }, + { + "name": "select_job", + "items": null, + "defaultValues": [ + { + "value": { + "string": "*" + } + } + ], + "nrqlQuery": { + "accountIds": [], + "query": "select uniques(job_name) from dbt_job_run since 7 days ago" + }, + "options": { + "ignoreTimeRange": false + }, + "title": "Job", + "type": "NRQL", + "isMultiSelection": true, + "replacementStrategy": "STRING" + }, + { + "name": "select_team", + "items": null, + "defaultValues": [ + { + "value": { + "string": "*" + } + } + ], + "nrqlQuery": { + "accountIds": [], + "query": "select uniques(team) from dbt_resource_run since 7 days ago" + }, + "options": { + "ignoreTimeRange": false + }, + "title": "Team", + "type": "NRQL", + "isMultiSelection": true, + "replacementStrategy": "STRING" + } + ] + } \ No newline at end of file diff --git a/dashboards/dbt-cloud/dbt-cloud01.png b/dashboards/dbt-cloud/dbt-cloud01.png new file mode 100644 index 0000000000..11643afe2c Binary files /dev/null and b/dashboards/dbt-cloud/dbt-cloud01.png differ diff --git a/data-sources/dbt-cloud/config.yml b/data-sources/dbt-cloud/config.yml new file mode 100644 index 0000000000..4b46f85432 --- /dev/null +++ b/data-sources/dbt-cloud/config.yml @@ -0,0 +1,20 @@ +id: dbt-cloud +displayName: dbt Cloud +description: | + Integrates dbt Cloud with New Relic using an Airflow DAG. Our dashboard and alerts help you find issues with dbt Cloud runs and resources. +icon: logo.png +install: + primary: + link: + url: https://docs.newrelic.com/docs/more-integrations/dbt-cloud/dbt-cloud-integration/ +keywords: + - dbt cloud + - dbt + +documentation: + - name: Github Repository + description: | + Github repository with the Airflow DAG and related documentation. + url: >- + https://github.com/newrelic-experimental/newrelic-dbt-cloud-integration/ + diff --git a/data-sources/dbt-cloud/logo.png b/data-sources/dbt-cloud/logo.png new file mode 100644 index 0000000000..b30bff37aa Binary files /dev/null and b/data-sources/dbt-cloud/logo.png differ diff --git a/data-sources/nvidia-jetson/config.yml b/data-sources/nvidia-jetson/config.yml index 417c2ef16c..8e34cf838d 100644 --- a/data-sources/nvidia-jetson/config.yml +++ b/data-sources/nvidia-jetson/config.yml @@ -5,11 +5,14 @@ description: | icon: logo.png install: primary: - link: - url: https://docs.newrelic.com/docs/infrastructure/host-integrations/host-integrations-list/nvidia-jetson-integration + nerdlet: + nerdletId: marketplace.install-data-source + nerdletState: + dataSourceId: nvidia-jetson + frameworkConfigId: nvidia-jetson + requiresAccount: false keywords: - jetson - nvidia jetson - tegrastats - - nvidia - \ No newline at end of file + - nvidia \ No newline at end of file diff --git a/data-sources/nvidia-triton/config.yml b/data-sources/nvidia-triton/config.yml index c7d49ab653..26cb280049 100644 --- a/data-sources/nvidia-triton/config.yml +++ b/data-sources/nvidia-triton/config.yml @@ -4,7 +4,10 @@ description: | Monitor and analyze your NVIDIA Triton inference server with New Relic. install: primary: - link: - url: https://docs.newrelic.com/docs/infrastructure/host-integrations/host-integrations-list/nvidia-triton-integration/ -icon: logo.png - + nerdlet: + nerdletId: marketplace.install-data-source + nerdletState: + dataSourceId: nvidia-triton + frameworkConfigId: nvidia-triton + requiresAccount: false +icon: logo.png \ No newline at end of file diff --git a/data-sources/nvml/config.yml b/data-sources/nvml/config.yml index 9ac570d919..aa8acf0675 100644 --- a/data-sources/nvml/config.yml +++ b/data-sources/nvml/config.yml @@ -4,8 +4,12 @@ description: | Monitor and analyze your NVML infrastructure with New Relic. install: primary: - link: - url: https://docs.newrelic.com/docs/infrastructure/host-integrations/host-integrations-list/nvml-integration/ + nerdlet: + nerdletId: marketplace.install-data-source + nerdletState: + dataSourceId: nvml + frameworkConfigId: nvml + requiresAccount: false icon: logo.png keywords: - NVIDIA diff --git a/quickstarts/dbt-cloud/config.yml b/quickstarts/dbt-cloud/config.yml new file mode 100644 index 0000000000..3a5623602a --- /dev/null +++ b/quickstarts/dbt-cloud/config.yml @@ -0,0 +1,32 @@ +slug: dbt-cloud +description: | + Integrates dbt Cloud with New Relic using an Airflow DAG. Our dashboard and alerts help you find issues with dbt Cloud runs and resources (models, seeds, snapshots, tests). Includes the ability to collect failed test rows from dbt tests in Snowflake. + +summary: | + dbt Cloud integration using an Airflow DAG. +icon: logo.png +level: Community +authors: + - New Relic + - Leo Godin +title: dbt Cloud +documentation: + - name: dbt cloud integration documentation + description: | + Installation documentation for dbt Cloud observability using an Airflow DAG + url: "https://docs.newrelic.com/docs/more-integrations/dbt-cloud/dbt-cloud-integration/" + - name: Git repository + description: | + Git repository for the Airflow DAG and additional documentation + url: "https://github.com/newrelic-experimental/newrelic-dbt-cloud-integration" +keywords: + - dbt cloud + - dbt + - NR1_addData + - NR1_sys +dashboards: + - dbt-cloud +alertPolicies: + - dbt-cloud +dataSourceIds: + - dbt-cloud diff --git a/quickstarts/dbt-cloud/logo.png b/quickstarts/dbt-cloud/logo.png new file mode 100644 index 0000000000..b30bff37aa Binary files /dev/null and b/quickstarts/dbt-cloud/logo.png differ