Merge branch 'GoogleCloudPlatform:main' into main

hao-nan-li · Jan 9, 2024 · 5a34e2f · 5a34e2f
2 parents 8ea8110 + 603220f
commit 5a34e2f
Show file tree

Hide file tree

Showing 1,039 changed files with 38,109 additions and 32,720 deletions.
diff --git a/.ci/containers/build-environment/Dockerfile b/.ci/containers/build-environment/Dockerfile
@@ -1,5 +1,5 @@
 # Stage 1: Building Go dependencies
-FROM golang:1.19-bullseye AS builder
+FROM golang:1.20-bullseye AS builder
 
 # Set working directory
 WORKDIR /app
@@ -15,7 +15,7 @@ RUN go mod download
 FROM ruby:3.1-bullseye
 
 # golang
-COPY --from=golang:1.19-bullseye /usr/local/go /usr/local/go
+COPY --from=golang:1.20-bullseye /usr/local/go /usr/local/go
 ENV GOPATH /go
 ENV PATH /usr/local/go/bin:$PATH
 ENV PATH $GOPATH/bin:$PATH

diff --git a/.ci/containers/go-plus/Dockerfile b/.ci/containers/go-plus/Dockerfile
@@ -1,5 +1,25 @@
-from golang:1.19-bullseye as resource
+# Stage 1: Download go module cache for builds
+FROM golang:1.20-bullseye AS builder
+ENV GOCACHE=/go/cache
+
+RUN apt-get update && apt-get install -y unzip
+WORKDIR /app1
+# Add the source code and build
+ADD "https://github.com/GoogleCloudPlatform/magic-modules/archive/refs/heads/main.zip" source.zip
+RUN unzip source.zip && rm source.zip
+WORKDIR /app1/magic-modules-main/.ci/magician
+# Build the binary (we won't use it in the final image, but it's cached)
+RUN go build -o /dev/null .
+
+# Stage 2: Creating the final image
+FROM golang:1.20-bullseye
 SHELL ["/bin/bash", "-c"]
+ENV GOCACHE=/go/cache
+
+# Copy Go dependencies and Go build cache
+COPY --from=builder /go/pkg/mod /go/pkg/mod
+COPY --from=builder /go/cache /go/cache
+
 # Set up Github SSH cloning.
 RUN ssh-keyscan github.com >> /known_hosts
 RUN echo "UserKnownHostsFile /known_hosts" >> /etc/ssh/ssh_config
@@ -17,16 +37,3 @@ RUN wget https://releases.hashicorp.com/terraform/1.2.5/terraform_1.2.5_linux_am
     && unzip terraform_1.2.5_linux_amd64.zip \
     && rm terraform_1.2.5_linux_amd64.zip \
     && mv ./terraform /bin/terraform
-
-### the following version of terraform are required for
-### terraform-validator-tester-integration
-RUN wget https://releases.hashicorp.com/terraform/0.13.7/terraform_0.13.7_linux_amd64.zip \
-    && unzip terraform_0.13.7_linux_amd64.zip \
-    && rm terraform_0.13.7_linux_amd64.zip \
-    && mkdir /terraform \
-    && mv terraform /terraform/0.13.7
-
-RUN wget https://releases.hashicorp.com/terraform/0.12.31/terraform_0.12.31_linux_amd64.zip \
-    && unzip terraform_0.12.31_linux_amd64.zip \
-    && rm terraform_0.12.31_linux_amd64.zip \
-    && mv terraform /terraform/0.12.31
diff --git a/.ci/gcb-build-container.yml b/.ci/gcb-build-container.yml
@@ -0,0 +1,15 @@
+---
+# substitutions:
+#   _IMAGE_NAME: 'gcr.io/graphite-docker-images/bash-plus:latest'
+#   _DIRECTORY: '.ci/containers/bash-plus'
+
+steps:
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['build', '-t', '$_IMAGE_NAME', '.']
+    dir: '$_DIRECTORY'
+
+images:
+  - '$_IMAGE_NAME'
+
+logsBucket: 'gs://cloudbuild-container-builder-logs'
+
diff --git a/.ci/gcb-community-checker.yml b/.ci/gcb-community-checker.yml
@@ -47,6 +47,16 @@ steps:
           git diff $base_commit origin/main -- .ci/ > /workspace/ci.diff
           git apply ./ci.diff --allow-empty
         fi
+        if [ "$_BASE_BRANCH" != "main" ]; then
+          echo "Checking out tools/ folder from main"
+          git checkout origin/main -- tools/
+        else
+          base_commit=$(git merge-base origin/main HEAD)
+          echo "Common ancestor commit: $base_commit"
+          git diff $base_commit origin/main -- tools/
+          git diff $base_commit origin/main -- tools/ > /workspace/tools.diff
+          git apply ./tools.diff --allow-empty
+        fi
 
   - name: 'gcr.io/graphite-docker-images/go-plus'
     entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'

diff --git a/.ci/gcb-contributor-membership-checker.yml b/.ci/gcb-contributor-membership-checker.yml
@@ -47,6 +47,16 @@ steps:
           git diff $base_commit origin/main -- .ci/ > /workspace/ci.diff
           git apply ./ci.diff --allow-empty
         fi
+        if [ "$_BASE_BRANCH" != "main" ]; then
+          echo "Checking out tools/ folder from main"
+          git checkout origin/main -- tools/
+        else
+          base_commit=$(git merge-base origin/main HEAD)
+          echo "Common ancestor commit: $base_commit"
+          git diff $base_commit origin/main -- tools/
+          git diff $base_commit origin/main -- tools/ > /workspace/tools.diff
+          git apply ./tools.diff --allow-empty
+        fi
 
   - name: "gcr.io/graphite-docker-images/go-plus"
     entrypoint: "/workspace/.ci/scripts/go-plus/magician/exec.sh"

diff --git a/.ci/gcb-generate-diffs-new.yml b/.ci/gcb-generate-diffs-new.yml
@@ -169,17 +169,17 @@ steps:
           - $_PR_NUMBER
 
     - name: 'gcr.io/graphite-docker-images/go-plus'
-      entrypoint: '/workspace/.ci/scripts/go-plus/github-differ/generate_comment.sh'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       id: diff
       secretEnv: ["GITHUB_TOKEN"]
+      args:
+          - 'generate-comment'
       env:
         - BUILD_ID=$BUILD_ID
         - PROJECT_ID=$PROJECT_ID
         - BUILD_STEP=17
         - COMMIT_SHA=$COMMIT_SHA
         - PR_NUMBER=$_PR_NUMBER
-      args:
-          - $_PR_NUMBER
 
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: tgc-test
@@ -194,13 +194,12 @@ steps:
         - PR_NUMBER=$_PR_NUMBER
 
     - name: 'gcr.io/graphite-docker-images/go-plus'
-      id: tgc-test-integration-0.12.31
+      id: tgc-test-integration
       entrypoint: '/workspace/.ci/scripts/go-plus/tgc-tester-integration/test_tgc_integration.sh'
       allowFailure: true
       secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["tpgb-head", "tpgb-base", "tgc-head", "tgc-base"]
       env:
-        - TERRAFORM_VERSION=0.12.31
         - TEST_PROJECT=$_VALIDATOR_TEST_PROJECT
         - TEST_FOLDER_ID=$_VALIDATOR_TEST_FOLDER
         - TEST_ANCESTRY=$_VALIDATOR_TEST_ANCESTRY
@@ -213,26 +212,6 @@ steps:
           - "18"  # Build step
           - terraform-google-conversion
 
-    - name: 'gcr.io/graphite-docker-images/go-plus'
-      id: tgc-test-integration-0.13.7
-      entrypoint: '/workspace/.ci/scripts/go-plus/tgc-tester-integration/test_tgc_integration.sh'
-      allowFailure: true
-      secretEnv: ["GITHUB_TOKEN"]
-      waitFor: ["tpgb-head", "tpgb-base", "tgc-head", "tgc-base"]
-      env:
-        - TERRAFORM_VERSION=0.13.7
-        - TEST_PROJECT=$_VALIDATOR_TEST_PROJECT
-        - TEST_FOLDER_ID=$_VALIDATOR_TEST_FOLDER
-        - TEST_ANCESTRY=$_VALIDATOR_TEST_ANCESTRY
-        - TEST_ORG_ID=$_VALIDATOR_TEST_ORG
-      args:
-          - $_PR_NUMBER
-          - $COMMIT_SHA
-          - $BUILD_ID
-          - $PROJECT_ID
-          - "19"  # Build step
-          - terraform-google-conversion
-
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: tpgb-test
       allowFailure: true
@@ -273,6 +252,14 @@ steps:
           - $PROJECT_ID
           - "22"  # Build step
 
+    - name: 'gcr.io/graphite-docker-images/go-plus'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
+      secretEnv: ["GITHUB_TOKEN"]
+      waitFor: ["diff"]
+      args:
+          - 'request-service-reviewers'
+          - $_PR_NUMBER
+
 # Long timeout to enable waiting on VCR test
 timeout: 20000s
 options:

diff --git a/.ci/gcb-push-downstream.yml b/.ci/gcb-push-downstream.yml
@@ -175,21 +175,73 @@ steps:
     - name: 'gcr.io/graphite-docker-images/go-plus'
       entrypoint: '/workspace/.ci/scripts/go-plus/vcr-cassette-merger/vcr_merge.sh'
       secretEnv: ["GITHUB_TOKEN", "GOOGLE_PROJECT"]
+      id: vcr-merge
       waitFor: ["tpg-push"]
       env:
         - BASE_BRANCH=$BRANCH_NAME
       args:
           - $COMMIT_SHA
 
+    - name: 'gcr.io/graphite-docker-images/go-plus'
+      id: magician-check-vcr-cassettes
+      waitFor: ["vcr-merge"]
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
+      secretEnv:
+        - "GITHUB_TOKEN"
+        - "GOOGLE_BILLING_ACCOUNT"
+        - "GOOGLE_CUST_ID"
+        - "GOOGLE_FIRESTORE_PROJECT"
+        - "GOOGLE_IDENTITY_USER"
+        - "GOOGLE_MASTER_BILLING_ACCOUNT"
+        - "GOOGLE_ORG"
+        - "GOOGLE_ORG_2"
+        - "GOOGLE_ORG_DOMAIN"
+        - "GOOGLE_PROJECT"
+        - "GOOGLE_PROJECT_NUMBER"
+        - "GOOGLE_SERVICE_ACCOUNT"
+        - "SA_KEY"
+        - "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"
+      env:
+        - "COMMIT_SHA=$COMMIT_SHA"
+        - "GOOGLE_REGION=us-central1"
+        - "GOOGLE_ZONE=us-central1-a"
+      args:
+        - "check-cassettes"
+
 # set extremely long 1 day timeout, in order to ensure that any jams / backlogs can be cleared.
 timeout: 86400s
 options:
     machineType: 'N1_HIGHCPU_32'
 
 
+logsBucket: 'gs://cloudbuild-downstream-builder-logs'
 availableSecrets:
   secretManager:
-    - versionName: projects/673497134629/secrets/github-magician-token/versions/latest
+    - versionName: projects/673497134629/secrets/github-classic--repo-workflow/versions/latest
       env: GITHUB_TOKEN
+    - versionName: projects/673497134629/secrets/ci-test-billing-account/versions/latest
+      env: GOOGLE_BILLING_ACCOUNT
+    - versionName: projects/673497134629/secrets/ci-test-cust-id/versions/latest
+      env: GOOGLE_CUST_ID
+    - versionName: projects/673497134629/secrets/ci-test-firestore-project/versions/latest
+      env: GOOGLE_FIRESTORE_PROJECT
+    - versionName: projects/673497134629/secrets/ci-test-identity-user/versions/latest
+      env: GOOGLE_IDENTITY_USER
+    - versionName: projects/673497134629/secrets/ci-test-master-billing-account/versions/latest
+      env: GOOGLE_MASTER_BILLING_ACCOUNT
+    - versionName: projects/673497134629/secrets/ci-test-org/versions/latest
+      env: GOOGLE_ORG
+    - versionName: projects/673497134629/secrets/ci-test-org-2/versions/latest
+      env: GOOGLE_ORG_2
+    - versionName: projects/673497134629/secrets/ci-test-org-domain/versions/latest
+      env: GOOGLE_ORG_DOMAIN
     - versionName: projects/673497134629/secrets/ci-test-project/versions/latest
       env: GOOGLE_PROJECT
+    - versionName: projects/673497134629/secrets/ci-test-project-number/versions/latest
+      env: GOOGLE_PROJECT_NUMBER
+    - versionName: projects/673497134629/secrets/ci-test-service-account/versions/latest
+      env: GOOGLE_SERVICE_ACCOUNT
+    - versionName: projects/673497134629/secrets/ci-test-service-account-key/versions/latest
+      env: SA_KEY
+    - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest
+      env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION
diff --git a/.ci/gcb-vcr-nightly.yml b/.ci/gcb-vcr-nightly.yml
@@ -0,0 +1,45 @@
+---
+steps:
+    - name: 'gcr.io/graphite-docker-images/go-plus'
+      id: gcb-vcr-nightly
+      entrypoint: '/workspace/.ci/scripts/go-plus/vcr-cassette-update/vcr_cassette_update.sh'
+      secretEnv: ["GITHUB_TOKEN", "GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"]
+      args:
+          - $BUILD_ID
+
+# Long timeout to enable waiting on VCR test
+timeout: 20000s
+options:
+    machineType: 'N1_HIGHCPU_32'
+
+logsBucket: 'gs://cloudbuild-vcr-nightly-logs'
+availableSecrets:
+  secretManager:
+    - versionName: projects/673497134629/secrets/github-magician-token/versions/latest
+      env: GITHUB_TOKEN
+    - versionName: projects/673497134629/secrets/ci-test-billing-account/versions/latest
+      env: GOOGLE_BILLING_ACCOUNT
+    - versionName: projects/673497134629/secrets/ci-test-cust-id/versions/latest
+      env: GOOGLE_CUST_ID
+    - versionName: projects/673497134629/secrets/ci-test-firestore-project/versions/latest
+      env: GOOGLE_FIRESTORE_PROJECT
+    - versionName: projects/673497134629/secrets/ci-test-identity-user/versions/latest
+      env: GOOGLE_IDENTITY_USER
+    - versionName: projects/673497134629/secrets/ci-test-master-billing-account/versions/latest
+      env: GOOGLE_MASTER_BILLING_ACCOUNT
+    - versionName: projects/673497134629/secrets/ci-test-org/versions/latest
+      env: GOOGLE_ORG
+    - versionName: projects/673497134629/secrets/ci-test-org-2/versions/latest
+      env: GOOGLE_ORG_2
+    - versionName: projects/673497134629/secrets/ci-test-org-domain/versions/latest
+      env: GOOGLE_ORG_DOMAIN
+    - versionName: projects/673497134629/secrets/ci-test-project/versions/latest
+      env: GOOGLE_PROJECT
+    - versionName: projects/673497134629/secrets/ci-test-project-number/versions/latest
+      env: GOOGLE_PROJECT_NUMBER
+    - versionName: projects/673497134629/secrets/ci-test-service-account/versions/latest
+      env: GOOGLE_SERVICE_ACCOUNT
+    - versionName: projects/673497134629/secrets/ci-test-service-account-key/versions/latest
+      env: SA_KEY
+    - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest
+      env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION
diff --git a/.ci/infra/terraform/README.md b/.ci/infra/terraform/README.md
@@ -47,6 +47,7 @@ After applying this configuration:
 - Add a new test user in the Google Workspace Admin Console: https://admin.google.com/ac/users
 - Create a `support@` group in the Google Workspace Admin Console, add new service account as a member, and make it an owner
 - Enroll in Cloud Armor Managed Protection Plus tier
+- Add Cloud Identity Premium Plan to the Google Workspace domain
 
 Quotas that will need to be adjusted to support all tests:
 - Project quota for the new service account
@@ -67,3 +68,5 @@ Quotas that will need to be adjusted to support all tests:
 - compute.googleapis.com/regional_static_addresses (us-central1)
 - compute.googleapis.com/routers
 - compute.googleapis.com/c2_cpus (us-central1)
+- compute.googleapis.com/n2_cpus (us-central1) to 36+
+- VMware Engine standard 72 vCPUs nodes per region - southamerica-east1 to 21
diff --git a/.ci/infra/terraform/main.tf b/.ci/infra/terraform/main.tf
@@ -184,6 +184,7 @@ module "project-services" {
     "bigtableadmin.googleapis.com",
     "billingbudgets.googleapis.com",
     "binaryauthorization.googleapis.com",
+    "blockchainnodeengine.googleapis.com",
     "certificatemanager.googleapis.com",
     "cloudapis.googleapis.com",
     "cloudasset.googleapis.com",
@@ -201,6 +202,7 @@ module "project-services" {
     "cloudtrace.googleapis.com",
     "composer.googleapis.com",
     "compute.googleapis.com",
+    "connectors.googleapis.com",
     "container.googleapis.com",
     "containeranalysis.googleapis.com",
     "containerfilesystem.googleapis.com",
@@ -259,10 +261,12 @@ module "project-services" {
     "managedidentities.googleapis.com",
     "memcache.googleapis.com",
     "metastore.googleapis.com",
+    "migrationcenter.googleapis.com",
     "ml.googleapis.com",
     "mobilecrashreporting.googleapis.com",
     "monitoring.googleapis.com",
     "multiclustermetering.googleapis.com",
+    "netapp.googleapis.com",
     "networkconnectivity.googleapis.com",
     "networkmanagement.googleapis.com",
     "networksecurity.googleapis.com",
@@ -284,6 +288,7 @@ module "project-services" {
     "run.googleapis.com",
     "runtimeconfig.googleapis.com",
     "secretmanager.googleapis.com",
+    "securesourcemanager.googleapis.com",
     "securetoken.googleapis.com",
     "securitycenter.googleapis.com",
     "serviceconsumermanagement.googleapis.com",