Refresh infra scripts to address audit followups, reduce output noise

infra/gcp/ensure-conformance-storage.sh

@@ -61,12 +61,10 @@ color 6 "Ensuring project exists: ${PROJECT}"
 ensure_project "${PROJECT}"
 
 # Enable GCS APIs
-color 6 "Enabling the GCS API"
-enable_api "${PROJECT}" storage-component.googleapis.com
-
-# Enable Secret Manager API
-color 6 "Enabling the Secret Manager API"
-enable_api "${PROJECT}" secretmanager.googleapis.com
+color 6 "Ensuring only necessary services are enabled for conformance project: ${PROJECT}"
+ensure_only_services "${PROJECT}" \
+    storage-component.googleapis.com \
+    secretmanager.googleapis.com \
 
 color 6 "Ensuring all conformance buckets"
 for REPO; do
@@ -102,11 +100,7 @@ for REPO; do
         readonly SERVICE_ACCOUNT_EMAIL="$(svc_acct_email "${PROJECT}" \
             "${SERVICE_ACCOUNT_NAME}")"
         readonly SECRET_ID="${SERVICE_ACCOUNT_NAME}-key"
-        readonly TMP_DIR=$(mktemp -d "/tmp/${SERVICE_ACCOUNT_NAME}.XXXXXX")
-        readonly KEY_FILE="${TMP_DIR}/key.json"
-
-        # Clean tmp dir when exit
-        trap 'rm -rf "${TMP_DIR}"' EXIT
+        readonly KEY_FILE="${TMPDIR}/key.json"
 
         if ! gcloud iam service-accounts describe "${SERVICE_ACCOUNT_EMAIL}" \
             --project "${PROJECT}" >/dev/null 2>&1
@@ -126,7 +120,7 @@ for REPO; do
             gcloud iam service-accounts keys create "${KEY_FILE}" \
                 --project "${PROJECT}" \
                 --iam-account "${SERVICE_ACCOUNT_EMAIL}"
-            
+
             color 6 "Creating secret to store private key"
             gcloud secrets create "${SECRET_ID}" \
                 --project "${PROJECT}" \
@@ -138,10 +132,10 @@ for REPO; do
                 --data-file "${KEY_FILE}"
 
             color 6 "Empowering ${BUCKET_WRITERS} for read secret"
-            gcloud secrets add-iam-policy-binding "${SECRET_ID}" \
-                --project "${PROJECT}" \
-                --member "group:${BUCKET_WRITERS}" \
-                --role "roles/secretmanager.secretAccessor"
+            ensure_secret_role_binding \
+                "projects/${PROJECT}/secrets/${SECRET_ID}" \
+                "group:${BUCKET_WRITERS}" \
+                "roles/secretmanager.secretAccessor"
         fi
     )
 done 2>&1 | indent

infra/gcp/ensure-gsuite.sh

@@ -52,15 +52,16 @@ GSUITE_GROUP_ADMINS="k8s-infra-group-admins@kubernetes.io"
 color 6 "Ensuring project exists: ${PROJECT}"
 ensure_project "${PROJECT}"
 
-# Enable GSuite APIs
-color 6 "Enabling the GSuite admin API"
-enable_api "${PROJECT}" admin.googleapis.com
+GSUITE_PROJECT_SERVICES=(
+    admin.googleapis.com
+    groupssettings.googleapis.com
+    secretmanager.googleapis.com
+)
 
-color 6 "Enabling the GSuite groups API"
-enable_api "${PROJECT}" groupssettings.googleapis.com
+# Enable GSuite APIs
+color 6 "Ensure services necessary for GSuite administration are enabled for: ${PROJECT}"
 
-color 6 "Enabling the Secret Manager API"
-enable_api "${PROJECT}" secretmanager.googleapis.com
+ensure_only_services "${PROJECT}" "${GSUITE_PROJECT_SERVICES[@]}"
 
 # Create a service account for gsuite to grant access to.
 color 6 "Creating service account for ${GSUITE_SVCACCT}"
@@ -87,18 +88,16 @@ if ! gcloud --project="${PROJECT}" \
     color 4 "  rm tmp.json"
 else
     color 6 "Empowering ${GSUITE_GROUP_ADMINS} to access the ${GSUITE_SVCACCT}_key secret"
-    gcloud --project="${PROJECT}" \
-        secrets add-iam-policy-binding "${GSUITE_SVCACCT}_key" \
-        --member="group:${GSUITE_GROUP_ADMINS}" \
-        --role="roles/secretmanager.secretAccessor"
+    ensure_secret_role_binding \
+        "projects/${PROJECT}/secrets/${GSUITE_SVCACCT}_key" \
+        "group:${GSUITE_GROUP_ADMINS}" \
+        "roles/secretmanager.secretAccessor"
 fi
 
 # Grant project owner for now because I have no idea exactly which specific
 # permissions are needed, and the UI is really not helping.
 color 6 "Empowering ${GSUITE_USER}"
-gcloud projects add-iam-policy-binding "${PROJECT}" \
-    --member "user:${GSUITE_USER}" \
-    --role roles/owner
+ensure_project_role_binding "${PROJECT}" "user:${GSUITE_USER}" "roles/owner"
 
 color 4 -n "The service account "
 color 6 -n "${GSUITE_SVCACCT}"

infra/gcp/ensure-main-project.sh

@@ -61,33 +61,29 @@ color 6 "Ensuring project exists: ${PROJECT}"
 ensure_project "${PROJECT}"
 
 # Enable APIs we know we need
-color 6 "Enabling the GCE API"
-enable_api "${PROJECT}" compute.googleapis.com
-color 6 "Enabling the StackDriver logging API"
-enable_api "${PROJECT}" logging.googleapis.com
-color 6 "Enabling the StackDriver monitoring API"
-enable_api "${PROJECT}" monitoring.googleapis.com
-color 6 "Enabling the BigQuery API"
-enable_api "${PROJECT}" bigquery-json.googleapis.com
-color 6 "Enabling the GKE API"
-enable_api "${PROJECT}" container.googleapis.com
-color 6 "Enabling the GCS API"
-enable_api "${PROJECT}" storage-component.googleapis.com
-color 6 "Enabling the OSLogin API"
-enable_api "${PROJECT}" oslogin.googleapis.com
-color 6 "Enabling the DNS API"
-enable_api "${PROJECT}" dns.googleapis.com
-color 6 "Enabling the Secret Manager API"
-enable_api "${PROJECT}" secretmanager.googleapis.com
-
+apis=(
+    bigquery-json.googleapis.com
+    compute.googleapis.com
+    container.googleapis.com
+    dns.googleapis.com
+    logging.googleapis.com
+    monitoring.googleapis.com
+    oslogin.googleapis.com
+    secretmanager.googleapis.com
+    storage-component.googleapis.com
+)
+ensure_only_services "${PROJECT}" "${apis[@]}"
 
 color 6 "Ensuring the cluster terraform-state bucket exists"
-ensure_private_gcs_bucket "${PROJECT}" "gs://${CLUSTER_TERRAFORM_BUCKET}"
+ensure_private_gcs_bucket \
+    "${PROJECT}" \
+    "gs://${CLUSTER_TERRAFORM_BUCKET}"
 
 color 6 "Empowering BigQuery admins"
-gcloud projects add-iam-policy-binding "${PROJECT}" \
-    --member "group:${BQ_ADMINS_GROUP}" \
-    --role roles/bigquery.admin
+ensure_project_role_binding \
+    "${PROJECT}" \
+    "group:${BQ_ADMINS_GROUP}" \
+    "roles/bigquery.admin"
 
 color 6 "Empowering cluster admins"
 # TODO: this can also be a custom role
@@ -102,24 +98,29 @@ for role in "${cluster_admin_roles[@]}"; do
 done
 # TODO(spiffxp): remove when bindings for custom project role are gone
 ensure_removed_project_role_binding "${PROJECT}" "group:${CLUSTER_ADMINS_GROUP}" "$(custom_project_role_name "${PROJECT}" ServiceAccountLister)"
-ensure_removed_project_role "${PROJECT}" "ServiceAccountLister"
-
-gsutil iam ch \
-    "group:${CLUSTER_ADMINS_GROUP}:objectAdmin" \
-    "gs://${CLUSTER_TERRAFORM_BUCKET}"
-gsutil iam ch \
-    "group:${CLUSTER_ADMINS_GROUP}:legacyBucketOwner" \
-    "gs://${CLUSTER_TERRAFORM_BUCKET}"
+ensure_removed_custom_project_iam_role "${PROJECT}" "ServiceAccountLister"
+
+color 6 "Empowering cluster admins to own gs://${CLUSTER_TERRAFORM_BUCKET}"
+ensure_gcs_role_binding \
+    "gs://${CLUSTER_TERRAFORM_BUCKET}" \
+    "group:${CLUSTER_ADMINS_GROUP}" \
+    "objectAdmin"
+ensure_gcs_role_binding \
+    "gs://${CLUSTER_TERRAFORM_BUCKET}" \
+    "group:${CLUSTER_ADMINS_GROUP}" \
+    "legacyBucketOwner"
 
 color 6 "Empowering cluster users"
-gcloud projects add-iam-policy-binding "${PROJECT}" \
-    --member "group:${CLUSTER_USERS_GROUP}" \
-    --role "roles/container.clusterViewer"
+ensure_project_role_binding \
+    "${PROJECT}" \
+    "group:${CLUSTER_USERS_GROUP}" \
+    "roles/container.clusterViewer"
 
 color 6 "Empowering GCP accounting"
-gcloud projects add-iam-policy-binding "${PROJECT}" \
-    --member "group:${ACCOUNTING_GROUP}" \
-    --role roles/bigquery.jobUser
+ensure_project_role_binding \
+  "${PROJECT}" \
+  "group:${ACCOUNTING_GROUP}" \
+  "roles/bigquery.jobUser"
 
 color 6 "Ensuring the k8s-infra-gcp-auditor serviceaccount exists"
 ensure_service_account \
@@ -144,17 +145,18 @@ ensure_service_account \
     "k8s-infra-dns-updater" \
     "k8s-infra dns updater"
 
-color 6 -n "Empowering k8s-infra-dns-updater serviceaccount to be used on"
-color 6 " build cluster"
+color 6 "Empowering k8s-infra-dns-updater serviceaccount to be used on build cluster"
 empower_ksa_to_svcacct \
     "k8s-infra-prow-build-trusted.svc.id.goog[test-pods/k8s-infra-dns-updater]" \
     "${PROJECT}" \
     "$(svc_acct_email "${PROJECT}" "k8s-infra-dns-updater")"
 
 color 6 "Empowering ${DNS_GROUP}"
-gcloud projects add-iam-policy-binding "${PROJECT}" \
-    --member "group:${DNS_GROUP}" \
-    --role roles/dns.admin
+color 6 "Empowering BigQuery admins"
+ensure_project_role_binding \
+    "${PROJECT}" \
+    "group:${DNS_GROUP}" \
+    "roles/dns.admin"
 
 # Monitoring
 MONITORING_SVCACCT_NAME="$(svc_acct_email "${PROJECT}" \
@@ -166,17 +168,17 @@ ensure_service_account \
     "k8s-infra-monitoring-viewer" \
     "k8s-infra monitoring viewer"
 
-color 6 -n "Empowering k8s-infra-monitoring-viewer serviceaccount to be used on"
-color 6 " the 'aaa' cluster inside the 'monitoring' namespace"
+color 6 "Empowering k8s-infra-monitoring-viewer serviceaccount to be used on the 'aaa' cluster inside the 'monitoring' namespace"
 empower_ksa_to_svcacct \
     "kubernetes-public.svc.id.goog[monitoring/k8s-infra-monitoring-viewer]" \
     "${PROJECT}" \
     "${MONITORING_SVCACCT_NAME}"
 
 color 6 "Empowering service account ${MONITORING_SVCACCT_NAME}"
-gcloud projects add-iam-policy-binding "${PROJECT}" \
-    --member "serviceAccount:${MONITORING_SVCACCT_NAME}" \
-    --role roles/monitoring.viewer
+ensure_project_role_binding \
+    "${PROJECT}" \
+    "serviceAccount:${MONITORING_SVCACCT_NAME}" \
+    "roles/monitoring.viewer"
 
 # Bootstrap DNS zones
 ensure_dns_zone "${PROJECT}" "k8s-io" "k8s.io"
@@ -200,10 +202,10 @@ color 6 "Setting BigQuery permissions"
 #   * The full list is large and has stuff that is inherited listed in it
 #   * All of our other IAM binding logic calls are additive
 
-CUR=$(mktemp -p /tmp k8s-infra-bq-access-cur-XXXXXX)
+CUR=${TMPDIR}/k8s-infra-bq-access.before.json
 bq show --format=prettyjson "${PROJECT}":"${BQ_BILLING_DATASET}"  > "${CUR}"
 
-ENSURE=$(mktemp -p /tmp k8s-infra-bq-access-new-XXXXXX)
+ENSURE=${TMPDIR}/k8s-infra-bq-access.ensure.json
 cat > "${ENSURE}" << __EOF__
 {
   "access": [
@@ -223,7 +225,7 @@ cat > "${ENSURE}" << __EOF__
 }
 __EOF__
 
-FINAL=$(mktemp -p /tmp k8s-infra-bq-access-new-XXXXXX)
+FINAL=${TMPDIR}/k8s-infra-bq-access.final.json
 jq -s '.[0].access + .[1].access | { access: . }' "${CUR}" "${ENSURE}" > "${FINAL}"
 
 bq update --source "${FINAL}" "${PROJECT}":"${BQ_BILLING_DATASET}"

infra/gcp/ensure-release-projects.sh

@@ -47,6 +47,14 @@ ADMINS="k8s-infra-release-admins@kubernetes.io"
 WRITERS="k8s-infra-release-editors@kubernetes.io"
 VIEWERS="k8s-infra-release-viewers@kubernetes.io"
 
+readonly RELEASE_PROJECT_SERVICES=(
+    cloudbuild.googleapis.com
+    cloudkms.googleapis.com
+    containerregistry.googleapis.com
+    secretmanager.googleapis.com
+    storage-component.googleapis.com
+)
+
 for PROJECT; do
     color 3 "Configuring: ${PROJECT}"
 
@@ -65,11 +73,11 @@ for PROJECT; do
         empower_group_as_viewer "${PROJECT}" "${group}"
     done
 
-    # Every project gets a GCR repo
+    # Enable services for release projects and their direct dependencies; prune anything else
+    color 6 "Ensuring only necessary services are enabled for release project: ${PROJECT}"
+    ensure_only_services "${PROJECT}" "${RELEASE_PROJECT_SERVICES[@]}"
 
-    # Enable container registry APIs
-    color 6 "Enabling the container registry API"
-    enable_api "${PROJECT}" containerregistry.googleapis.com
+    # Every project gets a GCR repo
 
     # Push an image to trigger the bucket to be created
     color 6 "Ensuring the registry exists and is readable"
@@ -87,10 +95,6 @@ for PROJECT; do
 
     # Every project gets some GCS buckets
 
-    # Enable GCS APIs
-    color 6 "Enabling the GCS API"
-    enable_api "${PROJECT}" storage-component.googleapis.com
-
     for BUCKET in "${ALL_BUCKETS[@]}"; do
         color 3 "Configuring bucket: ${BUCKET}"
 
@@ -111,10 +115,6 @@ for PROJECT; do
 
     # Enable GCB and Prow to build and push images.
 
-    # Enable GCB APIs
-    color 6 "Enabling the GCB API"
-    enable_api "${PROJECT}" cloudbuild.googleapis.com
-
     # Let project writers use GCB.
     for group in ${ADMINS} ${WRITERS}; do
         color 6 "Empowering ${group} as GCB editors"
@@ -125,10 +125,6 @@ for PROJECT; do
     color 6 "Empowering Prow"
     empower_prow "${PROJECT}" "${GCB_BUCKET}"
 
-    # Enable KMS APIs
-    color 6 "Enabling the KMS API"
-    enable_api "${PROJECT}" cloudkms.googleapis.com
-
     # Let project admins use KMS.
     color 6 "Empowering ${ADMINS} as KMS admins"
     empower_group_for_kms "${PROJECT}" "${ADMINS}"
@@ -159,8 +155,6 @@ for BUCKET in "${RELEASE_BUCKETS[@]}"; do
     empower_gcs_admins "k8s-release" "${BUCKET}"
 
     # Enable prow to write to the bucket
-    # TODO(spiffxp): I almost guarantee prow will need admin privileges but
-    #                let's start restricted and find out
     empower_svcacct_to_write_gcs_bucket "${PROW_BUILD_SVCACCT}" "${BUCKET}"
 
     # Enable writers on the bucket

infra/gcp/ensure-releng.sh

@@ -59,7 +59,7 @@ for PROJECT; do
 
     # Enable KMS APIs
     color 6 "Enabling the KMS API"
-    enable_api "${PROJECT}" cloudkms.googleapis.com
+    ensure_only_services "${PROJECT}" cloudkms.googleapis.com
 
     # Let project admins use KMS.
     color 6 "Empowering ${RELEASE_ADMINS} as KMS admins"