From dd90c4cc1f5ecff8f66b6e33690964294998a12a Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Thu, 5 Sep 2019 16:37:35 -0700 Subject: [PATCH 1/2] Don't create a gke-nodes svcacct in bash The terraform code will handle this. --- infra/gcp/ensure-main-project.sh | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/infra/gcp/ensure-main-project.sh b/infra/gcp/ensure-main-project.sh index 54e0ce09fb1..3210a6a0f21 100755 --- a/infra/gcp/ensure-main-project.sh +++ b/infra/gcp/ensure-main-project.sh @@ -42,9 +42,6 @@ BQ_BILLING_DATASET="kubernetes_public_billing" # The BigQuery admins group. BQ_ADMINS_GROUP="k8s-infra-bigquery-admins@kubernetes.io" -# The service account for GKE nodes. -NODES_SVCACCT="k8s-nodes" - # The cluster admins group. CLUSTER_ADMINS_GROUP="k8s-infra-cluster-admins@kubernetes.io" @@ -70,22 +67,6 @@ enable_api "${PROJECT}" storage-component.googleapis.com color 6 "Enabling the OSLogin API" enable_api "${PROJECT}" oslogin.googleapis.com -# Make an account for GKE nodes to run as -color 6 "Creating service account for ${NODES_SVCACCT}" -ensure_service_account "${PROJECT}" "${NODES_SVCACCT}" "Least-privilege SA for k8s nodes" - -color 6 "Empowering ${NODES_SVCACCT} with min permissions" -acct=$(svc_acct_email "${PROJECT}" "${NODES_SVCACCT}") -gcloud projects add-iam-policy-binding "${PROJECT}" \ - --member "serviceAccount:${acct}" \ - --role roles/logging.logWriter -gcloud projects add-iam-policy-binding "${PROJECT}" \ - --member "serviceAccount:${acct}" \ - --role roles/monitoring.viewer -gcloud projects add-iam-policy-binding "${PROJECT}" \ - --member "serviceAccount:${acct}" \ - --role roles/monitoring.metricWriter - color 6 "Empowering BigQuery admins" gcloud projects add-iam-policy-binding "${PROJECT}" \ --member "group:${BQ_ADMINS_GROUP}" \ From b2aa0753894722defad3ac5d8a11906ddafcbb5b Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Thu, 5 Sep 2019 16:38:30 -0700 Subject: [PATCH 2/2] Enable GCB builds for staging repos Enable GCB API. Create a scratch bucket for logs and stuff. Allow the prow svcacct to trigger builds and log. Remove retention policy for staging buckets (not needed for most and disallowed for GCB scratch) After this change, a small prow PR and a YAML file in your repo and you can pos-submit build and push to staging GCR without human hands touching it. --- infra/gcp/ensure-staging-storage.sh | 60 ++++++++++++++++------------- infra/gcp/lib.sh | 33 +++++++++++++++- 2 files changed, 65 insertions(+), 28 deletions(-) diff --git a/infra/gcp/ensure-staging-storage.sh b/infra/gcp/ensure-staging-storage.sh index e39ab45ec9b..0b93f583358 100755 --- a/infra/gcp/ensure-staging-storage.sh +++ b/infra/gcp/ensure-staging-storage.sh @@ -62,7 +62,7 @@ if [ $# = 0 ]; then fi for REPO; do - color 3 "${REPO}" + color 3 "Configuring staging: ${REPO}" # The GCP project name. PROJECT="k8s-staging-${REPO}" @@ -70,14 +70,15 @@ for REPO; do # The group that can write to this staging repo. WRITERS="k8s-infra-staging-${REPO}@kubernetes.io" - # The name of the bucket - BUCKET="gs://${PROJECT}" + # The names of the buckets + STAGING_BUCKET="gs://${PROJECT}" # used by humans + GCB_BUCKET="gs://${PROJECT}-gcb" # used by GCB + ALL_BUCKETS=("${STAGING_BUCKET}" "${GCB_BUCKET}") - # A short retention - it can always be raised, but it is hard to lower - # We expect promotion within 30d, or for testing to "move on" - # 30d is also short enough that people should notice occasionally, + # A short expiration - it can always be raised, but it is hard to lower + # We expect promotion within 60d, or for testing to "move on", but + # it is also short enough that people should notice occasionally, # and not accidentally think of the staging buckets as permanent. - RETENTION=30d AUTO_DELETION_DAYS=60 # Make the project, if needed @@ -106,34 +107,41 @@ for REPO; do color 6 "Empowering ${WRITERS} to GCR" empower_group_to_gcr "${PROJECT}" "${WRITERS}" - # Every project gets a GCS bucket + # Every project gets some GCS buckets # Enable GCS APIs color 6 "Enabling the GCS API" enable_api "${PROJECT}" storage-component.googleapis.com - # Create the bucket - color 6 "Ensuring the bucket exists and is world readable" - ensure_gcs_bucket "${PROJECT}" "${BUCKET}" + for BUCKET in "${ALL_BUCKETS[@]}"; do + color 3 "Configuring bucket: ${BUCKET}" - # Set bucket retention - color 6 "Ensuring the bucket has retention of ${RETENTION}" - ensure_gcs_bucket_retention "${BUCKET}" "${RETENTION}" + # Create the bucket + color 6 "Ensuring the bucket exists and is world readable" + ensure_gcs_bucket "${PROJECT}" "${BUCKET}" - # Set bucket auto-deletion - color 6 "Ensuring the bucket has auto-deletion of ${AUTO_DELETION_DAYS} days" - ensure_gcs_bucket_auto_deletion "${BUCKET}" "${AUTO_DELETION_DAYS}" + # Set bucket auto-deletion + color 6 "Ensuring the bucket has auto-deletion of ${AUTO_DELETION_DAYS} days" + ensure_gcs_bucket_auto_deletion "${BUCKET}" "${AUTO_DELETION_DAYS}" - # Enable admins on the bucket - color 6 "Empowering GCS admins" - empower_gcs_admins "${PROJECT}" "${BUCKET}" + # Enable admins on the bucket + color 6 "Empowering GCS admins" + empower_gcs_admins "${PROJECT}" "${BUCKET}" - # Enable writers on the bucket - color 6 "Empowering ${WRITERS} to GCS" - empower_group_to_gcs_bucket "${WRITERS}" "${BUCKET}" + # Enable writers on the bucket + color 6 "Empowering ${WRITERS} to GCS" + empower_group_to_gcs_bucket "${WRITERS}" "${BUCKET}" + done + + # Enable GCB and Prow to build and push images. + + # Enable GCB APIs + color 6 "Enabling the GCB API" + enable_api "${PROJECT}" cloudbuild.googleapis.com + + # Let prow trigger builds and access the scratch bucket + color 6 "Empowering Prow" + empower_prow "${PROJECT}" "${GCB_BUCKET}" color 6 "Done" done - -# Special case: don't use retention on cip-test buckets -gsutil retention clear gs://k8s-staging-cip-test diff --git a/infra/gcp/lib.sh b/infra/gcp/lib.sh index 85bb6f61758..57b94a0917f 100755 --- a/infra/gcp/lib.sh +++ b/infra/gcp/lib.sh @@ -41,6 +41,9 @@ GCS_ADMINS=$GCR_ADMINS # The service account name for the image promoter. PROMOTER_SVCACCT="k8s-infra-gcr-promoter" +# The service account email for Prow (not in this org for now). +PROW_SVCACCT="deployer@k8s-prow.iam.gserviceaccount.com" + # The GCP org stuff needed to turn it all on. GCP_ORG="758905017065" # kubernetes.io GCP_BILLING="018801-93540E-22A20E" @@ -211,12 +214,12 @@ function upload_gcs_static_content() { gsutil rsync -c "${srcdir}" "${bucket}" } -# Grant project viewew privileges to a principal +# Grant project viewer privileges to a principal # $1: The GCP project # $2: The group email function empower_group_as_viewer() { if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then - echo "empower_empower_group_as_viewer(project, group) requires 2 arguments" >&2 + echo "empower_group_as_viewer(project, group) requires 2 arguments" >&2 return 1 fi project="$1" @@ -228,6 +231,32 @@ function empower_group_as_viewer() { --role roles/viewer } +# Grant privileges to prow in a staging project +# $1: The GCP project +# $2: The GCS scratch bucket +function empower_prow() { + if [ $# -lt 2 -o -z "$1" -o -z "$2" ]; then + echo "empower_prow(project, bucket) requires 2 arguments" >&2 + return 1 + fi + project="$1" + bucket="$2" + + # Allow prow to trigger builds. + gcloud \ + projects add-iam-policy-binding "${project}" \ + --member "serviceAccount:${PROW_SVCACCT}" \ + --role roles/cloudbuild.builds.builder + + # Allow prow to push source and access build logs. + gsutil iam ch \ + "serviceAccount:${PROW_SVCACCT}:objectCreator" \ + "${bucket}" + gsutil iam ch \ + "serviceAccount:${PROW_SVCACCT}:objectViewer" \ + "${bucket}" +} + # Grant full privileges to GCR admins # $1: The GCP project # $2: The GCR region (optional)