Merge branch 'main' into main

.github/workflows/pr-merged.yml

@@ -26,7 +26,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           token: ${{ secrets.OPENSOURCE_BOT_TOKEN }}
-          ref: "main"
+          ref: "release"
           fetch-depth: 0
 
       - name: Setup workspace
@@ -46,7 +46,7 @@ jobs:
             const result = await github.rest.repos.updateBranchProtection({
               owner: context.repo.owner,
               repo: context.repo.repo,
-              branch: 'main',
+              branch: 'release',
               required_status_checks: null,
               restrictions: null,
               enforce_admins: null,
@@ -60,8 +60,8 @@ jobs:
           git config --local user.email "${{ env.THIRD_PARTY_GIT_AUTHOR_EMAIL }}"
           git config --local user.name "${{ env.THIRD_PARTY_GIT_AUTHOR_NAME }}"
 
-          # main could have been modified since we checked out, so pull before committing
-          git pull --ff-only origin main
+          # release could have been modified since we checked out, so pull before committing
+          git pull --ff-only origin release
 
           git add ./quickstarts/*
           git diff-index --quiet HEAD ./quickstarts/* || { git commit -m 'chore: generate UUID(s) [skip ci]' && echo "commit=true" >> $GITHUB_ENV; }
@@ -80,17 +80,19 @@ jobs:
             const result = await github.rest.repos.updateBranchProtection({
               owner: context.repo.owner,
               repo: context.repo.repo,
-              branch: 'main',
+              branch: 'release',
               required_status_checks: {
                 strict: true,
                 contexts: [
                     'Validation / Image count and extension compliance',
                     'Validation / Ensure icons exist',
                     'Validation / Install plan ids exist',
+                    'Validation / Data source ids exist',
                     'Validation / Install plan schema compliance',
                     'Validation / Data source schema compliance',
                     'Validation / Quickstart id are unique',
-                    'Validation / Validate Quickstart Schema'
+                    'Validation / Validate Quickstart Schema',
+                    'Validation / Quickstart dashboard name is unique'
                 ]
               },
               restrictions: {
@@ -110,56 +112,56 @@ jobs:
             })
             console.log("Result:", result)
 
-  get-pr-number:
-    name: Get PR number
-    runs-on: ubuntu-latest
-    outputs:
-      pr-number: ${{ steps.output_pr_number.outputs.pr-number }}
-    steps:
-      - name: Download artifact
-        uses: dawidd6/action-download-artifact@v2
-        with:
-          workflow: submit_gate.yml
-          run_id: ${{ github.event.workflow_run.id }}
-
-      - name: Get PR number
-        id: output_pr_number
-        run: |
-          export PR_NUMBER=$(cat artifact/pr_number_submit.txt)
-          echo "pr-number=$PR_NUMBER" >> $GITHUB_OUTPUT
-
-  staging:
-    needs: [generate-uuid, get-pr-number]
-    uses: ./.github/workflows/reusable.quickstart_submission.yml
-    with:
-      pr-number: ${{ needs.get-pr-number.outputs.pr-number }}
-      dry-run: false
-    secrets:
-      nr-api-url: ${{ secrets.NR_API_URL_STAGING }}
-      nr-api-token: ${{ secrets.NR_API_TOKEN_STAGING }}
-      github-token: ${{ secrets.GITHUB_TOKEN }}
-      nr-license-key: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
-
-  production:
-    needs: [staging, get-pr-number]
-    uses: ./.github/workflows/reusable.quickstart_submission.yml
-    with:
-      pr-number: ${{ needs.get-pr-number.outputs.pr-number }}
-      dry-run: false
-    secrets:
-      nr-api-url: ${{ secrets.NR_API_URL }}
-      nr-api-token: ${{ secrets.NR_API_TOKEN }}
-      github-token: ${{ secrets.GITHUB_TOKEN }}
-      nr-license-key: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
-
-  eu-production:
-    needs: [staging, get-pr-number]
-    uses: ./.github/workflows/reusable.quickstart_submission.yml
-    with:
-      pr-number: ${{ needs.get-pr-number.outputs.pr-number }}
-      dry-run: false
-    secrets:
-      nr-api-url: ${{ secrets.NR_API_URL_EU }}
-      nr-api-token: ${{ secrets.NR_API_TOKEN_EU }}
-      github-token: ${{ secrets.GITHUB_TOKEN }}
-      nr-license-key: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
+  # get-pr-number:
+  #   name: Get PR number
+  #   runs-on: ubuntu-latest
+  #   outputs:
+  #     pr-number: ${{ steps.output_pr_number.outputs.pr-number }}
+  #   steps:
+  #     - name: Download artifact
+  #       uses: dawidd6/action-download-artifact@v2
+  #       with:
+  #         workflow: submit_gate.yml
+  #         run_id: ${{ github.event.workflow_run.id }}
+
+  #     - name: Get PR number
+  #       id: output_pr_number
+  #       run: |
+  #         export PR_NUMBER=$(cat artifact/pr_number_submit.txt)
+  #         echo "pr-number=$PR_NUMBER" >> $GITHUB_OUTPUT
+
+  # staging:
+  #   needs: [generate-uuid, get-pr-number]
+  #   uses: ./.github/workflows/reusable.quickstart_submission.yml
+  #   with:
+  #     pr-number: ${{ needs.get-pr-number.outputs.pr-number }}
+  #     dry-run: false
+  #   secrets:
+  #     nr-api-url: ${{ secrets.NR_API_URL_STAGING }}
+  #     nr-api-token: ${{ secrets.NR_API_TOKEN_STAGING }}
+  #     github-token: ${{ secrets.GITHUB_TOKEN }}
+  #     nr-license-key: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
+
+  # production:
+  #   needs: [staging, get-pr-number]
+  #   uses: ./.github/workflows/reusable.quickstart_submission.yml
+  #   with:
+  #     pr-number: ${{ needs.get-pr-number.outputs.pr-number }}
+  #     dry-run: false
+  #   secrets:
+  #     nr-api-url: ${{ secrets.NR_API_URL }}
+  #     nr-api-token: ${{ secrets.NR_API_TOKEN }}
+  #     github-token: ${{ secrets.GITHUB_TOKEN }}
+  #     nr-license-key: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
+
+  # eu-production:
+  #   needs: [staging, get-pr-number]
+  #   uses: ./.github/workflows/reusable.quickstart_submission.yml
+  #   with:
+  #     pr-number: ${{ needs.get-pr-number.outputs.pr-number }}
+  #     dry-run: false
+  #   secrets:
+  #     nr-api-url: ${{ secrets.NR_API_URL_EU }}
+  #     nr-api-token: ${{ secrets.NR_API_TOKEN_EU }}
+  #     github-token: ${{ secrets.GITHUB_TOKEN }}
+  #     nr-license-key: ${{ secrets.NEW_RELIC_LICENSE_KEY }}

.github/workflows/release.yml

@@ -79,10 +79,12 @@ jobs:
                     'Validation / Image count and extension compliance',
                     'Validation / Ensure icons exist',
                     'Validation / Install plan ids exist',
+                    'Validation / Data source ids exist',
                     'Validation / Install plan schema compliance',
                     'Validation / Data source schema compliance',
                     'Validation / Quickstart id are unique',
-                    'Validation / Validate Quickstart Schema'
+                    'Validation / Validate Quickstart Schema',
+                    'Validation / Quickstart dashboard name is unique'
                 ]
               },
               restrictions: {

.github/workflows/submit-gate.yml

@@ -7,7 +7,7 @@ name: Submit Gate
 on: 
   pull_request:
     branches:
-      - main
+      - release
     types: [closed]
 
 jobs:
@@ -23,4 +23,4 @@ jobs:
       - name: upload artifact
         uses: actions/upload-artifact@v3
         with:
-          path: pr_number_submit.txt
+          path: pr_number_submit.txt

.github/workflows/validate_packs.yml

@@ -204,6 +204,48 @@ jobs:
           statusContext: "Validation / Install plan ids exist"
           state: ${{ job.status }}
 
+  validate-data-source-ids:
+    name: Validate data source ids
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download artifact
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: validation_gate.yml
+          run_id: ${{ github.event.workflow_run.id }}
+
+      - name: Get PR number
+        id: get_pr_number
+        run: |
+          export PR_NUMBER=$(cat artifact/pr_number.txt)
+          echo "pr-number=$PR_NUMBER" >> $GITHUB_ENV
+
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          ref: "refs/pull/${{ env.pr-number }}/merge"
+
+      - name: Setup workspace
+        uses: './.github/actions/bootstrap'
+
+      - name: Validate new files
+        id: validation
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUMBER: ${{ env.pr-number }}
+          NR_API_URL: ${{ secrets.NR_API_URL }}
+          NR_API_TOKEN: ${{ secrets.NR_API_TOKEN }}
+        run: |
+          URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/$PR_NUMBER/files"
+          cd utils && yarn validate-quickstart-data-sources "$URL"
+
+      - name: Add commit status
+        if: always()
+        uses: './.github/actions/add-commit-status'
+        with:
+          statusContext: "Validation / Data source ids exist"
+          state: ${{ job.status }}
+
   ensure-quickstart-dashboard-names-are-unique:
     name: Ensure quickstart dashboard names are unique
     runs-on: ubuntu-latest

utils/__tests__/validate_install_plans.test.js

@@ -2,7 +2,8 @@
 
 import { getInstallPlanId } from '../create-validate-install-plans';
 
-import * as fs from 'fs';
+import {readFileSync} from 'fs';
+const fs = {readFileSync};
 
 jest.mock('@actions/core');
 jest.spyOn(global.console, 'error').mockImplementation(() => {});

utils/create-validate-data-sources.ts

@@ -33,21 +33,17 @@ export const getDataSourceId = (filename: string) => {
   return config.id;
 };
 
-/**
- * Entrypoint.
- */
-const main = async () => {
-  const [GITHUB_API_URL, dryRun] = passedProcessArguments();
-  const githubToken = process.env.GITHUB_TOKEN;
-  const isDryRun = dryRun === 'true';
-
-  if (!githubToken) {
-    console.error('GITHUB_TOKEN is not defined.');
-    process.exit(1);
+export const createValidateDataSources = async (
+  ghUrl: string,
+  ghToken?: string,
+  isDryRun = false
+) => {
+  if (!ghToken) {
+    console.warn('GITHUB_TOKEN is not defined.');
   }
 
-  logger.info(`Fetching files for pull request ${GITHUB_API_URL}`);
-  const files = await fetchPaginatedGHResults(GITHUB_API_URL, githubToken);
+  logger.info(`Fetching files for pull request ${ghUrl}`);
+  const files = await fetchPaginatedGHResults(ghUrl, ghToken);
   logger.info(`Found ${files.length} files`);
 
   const dataSources = filterOutTestFiles(files)
@@ -79,6 +75,17 @@ const main = async () => {
   );
 
   const hasFailed = failures.length > 0;
+  return hasFailed;
+};
+
+/**
+ * Entrypoint.
+ */
+const main = async () => {
+  const [ghUrl, isDryRun] = passedProcessArguments();
+  const ghToken = process.env.GITHUB_TOKEN;
+  const dryRun = isDryRun === 'true';
+  const hasFailed = await createValidateDataSources(ghUrl, ghToken, dryRun);
 
   const event = isDryRun
     ? CUSTOM_EVENT.VALIDATE_DATA_SOURCES

utils/create_validate_pr_quickstarts.ts

@@ -34,7 +34,7 @@ const installPlanErrorExists = (error: Error | NerdGraphError): boolean =>
 
 const dataSourceErrorExists = (error: Error | NerdGraphError): boolean =>
   'extensions' in error &&
-  error?.extensions?.argumentPath.includes('dataSourceIds') &&
+  error?.extensions?.argumentPath?.includes('dataSourceIds') &&
   error?.message?.includes('contains a data source that does not exist');
 
 export const countAndOutputErrors = (
@@ -64,8 +64,7 @@ export const createValidateQuickstarts = async (
   isDryRun = false
 ): Promise<boolean> => {
   if (!ghToken) {
-    console.error('GITHUB_TOKEN is not defined.');
-    return false;
+    console.warn('GITHUB_TOKEN is not defined.');
   }
 
   if (!ghUrl) {

utils/lib/github-api-helpers.ts

@@ -1,5 +1,5 @@
 import fetch from 'node-fetch';
-import * as parseLinkHeader from 'parse-link-header';
+import parseLinkHeader from 'parse-link-header';
 import {
   QUICKSTART_CONFIG_REGEXP,
   DATA_SOURCE_CONFIG_REGEXP,
@@ -9,6 +9,9 @@ const INSTALL_CONFIG_REGEXP = new RegExp('install/.+/install.+(yml|yaml)');
 const MOCK_FILES_REGEXP = new RegExp('mock_files/.+');
 const TEMPLATE_REGEXP = new RegExp('_template/.+');
 
+export const generatePrUrl = (prNumber: string | number) =>
+  `https://api.github.com/repos/newrelic/newrelic-quickstarts/pulls/${prNumber}/files`;
+
 /**
  * Pulls the next page off of a `Link` header
  * @param {String|Null} linkHeader the `Link` header value
@@ -50,16 +53,18 @@ export interface GithubAPIPullRequestFile {
  */
 export const fetchPaginatedGHResults = async (
   url: string,
-  token: string
+  token?: string
 ): Promise<GithubAPIPullRequestFile[]> => {
   logger.debug(`Running fetch against ${url}`, { url });
   let files: GithubAPIPullRequestFile[] = [];
   let nextPageLink: string | null = url;
   try {
     while (nextPageLink) {
       logger.debug(`Fetching ${nextPageLink}`, { url: nextPageLink });
+
+      const headers = token ? { authorization: `token ${token}` } : undefined;
       const resp = await fetch(nextPageLink, {
-        headers: { authorization: `token ${token}` },
+        headers,
       });
       // TODO: this should happen after the resp.ok check
       const responseJson = await resp.json();

utils/lib/nr-graphql-helpers.ts

@@ -18,9 +18,6 @@ import { Policy } from 'cockatiel';
 import fetch, { Response } from 'node-fetch';
 import logger from '../logger';
 
-const NR_API_URL = process.env.NR_API_URL || '';
-const NR_API_TOKEN = process.env.NR_API_TOKEN || '';
-
 /**
  * Build body param for NR GraphQL request
  * @param {{queryString, variables}} queryBody - query string and corresponding variables for request
@@ -44,8 +41,11 @@ export type ErrorOrNerdGraphError = Error | NerdGraphError;
  * @returns {Promise<Object>} An object with the results or errors of a GraphQL request
  */
 export const fetchNRGraphqlResults = async <Variables, ResponseData>(
-  queryBody: NerdGraphRequest<Variables>
+  queryBody: NerdGraphRequest<Variables>,
 ): Promise<NerdGraphResponseWithLocalErrors<ResponseData>> => {
+  const NR_API_URL = process.env.NR_API_URL || '';
+  const NR_API_TOKEN = process.env.NR_API_TOKEN || '';
+
   let results;
   let graphqlErrors: ErrorOrNerdGraphError[] = [];