From 6cee07585911839c7bfa9d3b52ca279a1d6b5c7d Mon Sep 17 00:00:00 2001 From: Aaron Crickenberger Date: Fri, 21 May 2021 11:23:35 -0400 Subject: [PATCH 1/7] infra/gcp/lib_gsm: fix ensure_serviceaccount_key_secret --- infra/gcp/lib_gsm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/gcp/lib_gsm.sh b/infra/gcp/lib_gsm.sh index 7247f5d2c34..04eeb6d25c8 100644 --- a/infra/gcp/lib_gsm.sh +++ b/infra/gcp/lib_gsm.sh @@ -78,7 +78,7 @@ function ensure_serviceaccount_key_secret() { gcloud iam service-accounts keys create "${private_key_file}" \ --project "${project}" \ - --iam-account "${email}" + --iam-account "${serviceaccount}" gcloud secrets versions add "${secret}" \ --project "${project}" \ From 0588d004b2f7d7d6d06eeb34cdac1b49fd02ba20 Mon Sep 17 00:00:00 2001 From: Aaron Crickenberger Date: Fri, 21 May 2021 11:23:51 -0400 Subject: [PATCH 2/7] infra/gcp/lib_gsm: fix shellcheck nits --- infra/gcp/lib_gsm.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/infra/gcp/lib_gsm.sh b/infra/gcp/lib_gsm.sh index 04eeb6d25c8..10945b146e4 100644 --- a/infra/gcp/lib_gsm.sh +++ b/infra/gcp/lib_gsm.sh @@ -23,8 +23,8 @@ # $1: The project id hosting the secret (e.g. "k8s-infra-foo") # $2: The secret name (e.g. "my-secret") function secret_full_name() { - if [ ! $# -eq 2 -o -z "$1" -o -z "$2" ]; then - echo "secret_full_name(project, secret) requires 2 arguments" >&2 + if [ ! $# -eq 2 ] || [ -z "$1" ] || [ -z "$2" ]; then + echo "${FUNCNAME[0]}(project, secret) requires 2 arguments" >&2 return 1 fi @@ -41,15 +41,15 @@ function secret_full_name() { # $1: The project id hosting the secret (e.g. "k8s-infra-foo") # $2: The secret name (e.g. "my-secret") function ensure_secret() { - if [ ! $# -eq 2 -o -z "$1" -o -z "$2" ]; then - echo "ensure_secret(project, secret) requires 2 arguments" >&2 + if [ ! $# -eq 2 ] || [ -z "$1" ] || [ -z "$2" ]; then + echo "${FUNCNAME[0]}(project, secret) requires 2 arguments" >&2 return 1 fi local project="${1}" local secret="${2}" - if ! gcloud secrets describe --project "${project}" "${secret}" > /dev/null; then + if ! gcloud secrets describe --project "${project}" "${secret}" >/dev/null 2>&1; then gcloud secrets create --project "${project}" "${secret}" fi } @@ -62,8 +62,8 @@ function ensure_secret() { # $2: The secret name (e.g. "my-secret") # $3: The service-account (e.g. "foo@k8s-infra.iam.gserviceaccount.com") function ensure_serviceaccount_key_secret() { - if [ ! $# -eq 3 -o -z "$1" -o -z "$2" -o -z "$3" ]; then - echo "ensure_serviceaccount_key_secret(project, secret, serviceaccountt) requires 3 arguments" >&2 + if [ ! $# -eq 3 ] || [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then + echo "${FUNCNAME[0]}(project, secret, serviceaccountt) requires 3 arguments" >&2 return 1 fi @@ -73,7 +73,7 @@ function ensure_serviceaccount_key_secret() { local private_key_file="${TMPDIR}/key.json" - if ! gcloud secrets describe --project "${project}" "${secret}" > /dev/null; then + if ! gcloud secrets describe --project "${project}" "${secret}" >/dev/null 2>&1; then ensure_secret "${project}" "${secret}" gcloud iam service-accounts keys create "${private_key_file}" \ From 393c027d35f2050d73b4ba0b7aa0cff5bcc32bfe Mon Sep 17 00:00:00 2001 From: Aaron Crickenberger Date: Fri, 21 May 2021 12:22:53 -0400 Subject: [PATCH 3/7] infra/gcp/lib_gsm: add convenience funcs Specfically ensure_secret_with_admins, ensure_secret_labels --- infra/gcp/lib_gsm.sh | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/infra/gcp/lib_gsm.sh b/infra/gcp/lib_gsm.sh index 10945b146e4..9a1df0519f8 100644 --- a/infra/gcp/lib_gsm.sh +++ b/infra/gcp/lib_gsm.sh @@ -54,6 +54,46 @@ function ensure_secret() { fi } +# Ensures the give labels exist on the given secret in the given project +# Arguments: +# $1: The project id hosting the secret (e.g. "k8s-infra-foo") +# $2: The secret name (e.g. "my-secret") +# $3+ Labels in the form of key=value (e.g. "app=foo" "sig=awesome") +function ensure_secret_labels() { + if [ $# -lt 3 ] || [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then + echo "${FUNCNAME[0]}(project, secret, labels) requires at least 3 arguments" >&2 + return 1 + fi + + local project="${1}"; shift + local secret="${1}"; shift + + gcloud secrets update --project "${project}" "${secret}" "${@/#/"--update-labels="}" +} + +# Ensures a secret exists in the given project with the given name and that +# its admins are the given group +# Arguments: +# $1: The project id hosting the secret (e.g. "k8s-infra-foo") +# $2: The secret name (e.g. "my-secret") +# $3: The admin group (e.g. "k8s-infra-foo-admins@kubernetes.io") +function ensure_secret_with_admins() { + if [ ! $# -eq 3 ] || [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then + echo "${FUNCNAME[0]}(project, secret, admins) requires 3 arguments" >&2 + return 1 + fi + local project="${1}" + local secret="${2}" + local admins="${3}" + + ensure_secret "${project}" "${secret}" + + ensure_secret_role_binding \ + "$(secret_full_name "${project}" "${secret}")" \ + "group:${admins}" \ + "roles/secretmanager.admin" +} + # Ensures a secret exists in the given project with the given name. If the # secret does not exist, it is pre-populated with a newly created private key # for the given service-account From a8fd682ff56130147a34ab2ab4f01c93caef8a61 Mon Sep 17 00:00:00 2001 From: Aaron Crickenberger Date: Fri, 21 May 2021 12:23:55 -0400 Subject: [PATCH 4/7] infra/gcp/main: manage kubernetes-public secrets --- infra/gcp/ensure-main-project.sh | 48 ++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/infra/gcp/ensure-main-project.sh b/infra/gcp/ensure-main-project.sh index 0fa72eb9c19..a90285c73c6 100755 --- a/infra/gcp/ensure-main-project.sh +++ b/infra/gcp/ensure-main-project.sh @@ -313,6 +313,51 @@ EOF read -rs } +# Eventually we would like to use kubernetes-external-secrets to manage +# all secrets in aaa; not sure how far we are on that. So for now, at least +# ensure that the existing kubernetes-public secrets created for humans +# to manually sync into the aaa cluster are managed by this script. +function ensure_aaa_external_secrets() { + if [ $# -ne 1 ] || [ -z "$1" ]; then + echo "${FUNCNAME[0]}(project) requires 1 argument" >&2 + return 1 + fi + local project="${1}" + local secret_specs=() + + # another sign that we should move to using YAML as source of intent; + # bash and indirect array access don't play nice, so we get this... + local slack_infra_secrets=( + recaptcha + slack-event-log-config + slack-moderator-config + slack-moderator-words-config + slack-welcomer-config + slackin-token + ) + local triageparty_release_secrets=( + triage-party-github-token + ) + mapfile -t secret_specs < <( + printf "%s/slack-infra/sig-contributor-experience\n" "${slack_infra_secrets[@]}" + printf "%s/triageparty-release/sig-release\n" "${triageparty_release_secrets[@]}" + ) + + for spec in "${secret_specs[@]}"; do + local secret app k8s_group + secret="$(echo "${spec}" | cut -d/ -f1)" + app="$(echo "${spec}" | cut -d/ -f2)" + k8s_group="$(echo "${spec}" | cut -d/ -f3)" + + local admins="k8s-infra-rbac-${app}@kubernetes.io" + local labels=("app=${app}" "group=${k8s_group}") + + color 6 "Ensuring '${app}' secret '${secret}' exists in '${project}' and is owned by '${admins}'" + ensure_secret_with_admins "${project}" "${secret}" "${admins}" + ensure_secret_labels "${project}" "${secret}" "${labels[@]}" + done +} + function ensure_main_project() { if [ $# -ne 1 ] || [ -z "$1" ]; then echo "${FUNCNAME[0]}(gcp_project) requires 1 argument" >&2 @@ -365,6 +410,9 @@ function ensure_main_project() { color 6 "Ensuring DNS is configured in: ${project}" ensure_dns "${project}" 2>&1 | indent + color 6 "Ensuring secrets destined for apps in 'aaa' exist in: ${project}" + ensure_aaa_external_secrets "${project}" 2>&1 | indent + color 6 "Ensuring biquery configured for billing and access by appropriate groups in: ${project}" ensure_billing_bigquery "${project}" 2>&1 | indent From 05687ec4eda05c3f598b4efe16eb5213dd24ee70 Mon Sep 17 00:00:00 2001 From: Aaron Crickenberger Date: Fri, 21 May 2021 14:13:40 -0400 Subject: [PATCH 5/7] infra/gcp/prow: refactor into functions ensure_e2e_project could use some work but this is enough to break up the structure and address shellcheck nits --- infra/gcp/prow/ensure-e2e-projects.sh | 130 ++++++++++++++++---------- 1 file changed, 79 insertions(+), 51 deletions(-) diff --git a/infra/gcp/prow/ensure-e2e-projects.sh b/infra/gcp/prow/ensure-e2e-projects.sh index 23750ceb829..2337fd5adc0 100755 --- a/infra/gcp/prow/ensure-e2e-projects.sh +++ b/infra/gcp/prow/ensure-e2e-projects.sh @@ -37,84 +37,66 @@ function usage() { PROW_BUILD_SVCACCT=$(svc_acct_email "k8s-infra-prow-build" "prow-build") BOSKOS_JANITOR_SVCACCT=$(svc_acct_email "k8s-infra-prow-build" "boskos-janitor") -color 6 "Ensuring boskos-janitor is empowered" -( -color 6 "Ensuring external ip address exists for boskos-metrics service in prow build cluster" -# this is so monitoring.prow.k8s.io is able to scrape metrics from boskos -ensure_regional_address \ - "k8s-infra-prow-build" \ - "us-central1" \ - "boskos-metrics" \ - "to allow monitoring.k8s.prow.io to scrape boskos metrics" -) 2>&1 | indent - -color 6 "Ensuring greenhouse is empowered" -( -ensure_regional_address \ - "k8s-infra-prow-build" \ - "us-central1" \ - "greenhouse-metrics" \ - "to allow monitoring.k8s.prow.io to scrape greenhouse metrics" -) 2>&1 | indent - ## setup projects to be used by e2e tests for standing up clusters -E2E_MANUAL_PROJECTS=( - # for manual use during node-e2e job migration, eg: --gcp-project=gce-project - k8s-infra-e2e-gce-project - # for manual use during job migration, eg: --gcp-project=node-e2e-project - k8s-infra-e2e-node-e2e-project - # for manual use during job migration, eg: --gcp-project=scale-project - k8s-infra-e2e-scale-project - # for manual use during job migration, eg: --gcp-project=gpu-project - k8s-infra-e2e-gpu-project - # for manual use during job migration, eg: --gcp-project=ingress-project - k8s-infra-e2e-ingress-project +readonly E2E_MANUAL_PROJECTS=( + # for manual use during node-e2e job migration, eg: --gcp-project=gce-project + k8s-infra-e2e-gce-project + # for manual use during job migration, eg: --gcp-project=node-e2e-project + k8s-infra-e2e-node-e2e-project + # for manual use during job migration, eg: --gcp-project=scale-project + k8s-infra-e2e-scale-project + # for manual use during job migration, eg: --gcp-project=gpu-project + k8s-infra-e2e-gpu-project + # for manual use during job migration, eg: --gcp-project=ingress-project + k8s-infra-e2e-ingress-project ) # general purpose e2e projects, no quota changes E2E_BOSKOS_PROJECTS=() for i in $(seq 1 120); do - E2E_BOSKOS_PROJECTS+=("$(printf "k8s-infra-e2e-boskos-%03i" $i)") + E2E_BOSKOS_PROJECTS+=("$(printf "k8s-infra-e2e-boskos-%03i" "$i")") done +readonly E2E_BOSKOS_PROJECTS # e2e projects for scalability jobs # - us-east1 cpu quota raised to 125 # - us-east1 in-use addresses quota raised to 125 E2E_SCALE_PROJECTS=() for i in $(seq 1 30); do - E2E_SCALE_PROJECTS+=("$(printf "k8s-infra-e2e-boskos-scale-%02i" $i)") + E2E_SCALE_PROJECTS+=("$(printf "k8s-infra-e2e-boskos-scale-%02i" "$i")") done +readonly E2E_SCALE_PROJECTS # e2e projects for gpu jobs # - us-west1 Committed NVIDIA K80 GPUs raised to 2 E2E_GPU_PROJECTS=() for i in $(seq 1 10); do - E2E_GPU_PROJECTS+=("$(printf "k8s-infra-e2e-boskos-gpu-%02i" $i)") + E2E_GPU_PROJECTS+=("$(printf "k8s-infra-e2e-boskos-gpu-%02i" "$i")") done +readonly E2E_GPU_PROJECTS -E2E_PROJECTS=( +readonly E2E_PROJECTS=( "${E2E_MANUAL_PROJECTS[@]}" "${E2E_BOSKOS_PROJECTS[@]}" "${E2E_SCALE_PROJECTS[@]}" "${E2E_GPU_PROJECTS[@]}" ) -if [ $# = 0 ]; then - # default to all e2e projects - set -- "${E2E_PROJECTS[@]}" -fi - -color 6 "Ensuring e2e projects exist and are appropriately configured" -for prj; do +# prow build cluster services that expose metrics endpoints to be scraped +# by monitoring.prow.k8s.io; they each get a regional address +readonly PROW_BUILD_CLUSTER_METRICS_SERVICES=( + "boskos-metrics" + "greenhouse-metrics" +) - if ! (printf '%s\n' "${E2E_PROJECTS[@]}" | grep -q "^${prj}$"); then - color 2 "Skipping unrecognized e2e project name: ${prj}" - continue - fi +function ensure_e2e_project() { + if [ $# != 1 ] || [ -z "$1" ]; then + echo "${FUNCNAME[0]}(project) requires 1 argument" >&2 + return 1 + fi + local prj="${1}" - color 6 "Ensuring e2e project exists and is appropriately configured: ${prj}" - ( ensure_project "${prj}" color 6 "Ensure stale role bindings have been removed from e2e project: ${prj}" @@ -191,11 +173,57 @@ for prj; do done fi - if ! diff ${ssh_keys_before} ${ssh_keys_after} >/dev/null; then + if ! diff "${ssh_keys_before}" "${ssh_keys_after}" >/dev/null; then gcloud compute project-info add-metadata --project="${prj}" \ --metadata-from-file ssh-keys="${ssh_keys_after}" diff_colorized "${ssh_keys_before}" "${ssh_keys_after}" fi +} + +# TODO: this should be moved to the terraform responsible for k8s-infra-prow-build +function ensure_prow_build_cluster_metrics_endpoints() { + local project="k8s-infra-prow-build" + local region="us-central1" + for service in "${PROW_BUILD_CLUSTER_METRICS_SERVICES[@]}"; do + color 6 "Ensuring monitoring.prow.k8s.io can scrape ${service} for: ${project}" + ensure_regional_address \ + "${project}" \ + "${region}" \ + "${service}" \ + "to allow monitoring.k8s.prow.io to scrape ${service}" \ + 2>&1 | indent + done +} + +function ensure_e2e_projects() { + # default to all staging projects + if [ $# = 0 ]; then + set -- "${E2E_PROJECTS[@]}" + fi + + for project in "${@}"; do + if ! (printf '%s\n' "${E2E_PROJECTS[@]}" | grep -q "^${project}$"); then + color 2 "Skipping unrecognized e2e project name: ${project}" + continue + fi + + color 3 "Configuring e2e project: ${project}" + ensure_e2e_project "${project}" 2>&1 | indent + done +} + +# +# main +# + +function main() { + color 6 "Ensuring monitoring.prow.k8s.io can scrape k8s-infra-prow-build metrics endpoints" + ensure_prow_build_cluster_metrics_endpoints 2>&1 | indent + + color 6 "Ensuring e2e projects used by prow..." + ensure_e2e_projects "${@}" 2>&1 | indent + + color 6 "Done" +} - ) 2>&1 | indent -done 2>&1 | indent +main "${@}" From 36c13d159af28204a8c8f54e63cea5dea158f046 Mon Sep 17 00:00:00 2001 From: Aaron Crickenberger Date: Mon, 24 May 2021 13:51:40 -0400 Subject: [PATCH 6/7] infra/gcp/prow: manage prow-build-trusted secrets --- infra/gcp/prow/ensure-e2e-projects.sh | 31 +++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/infra/gcp/prow/ensure-e2e-projects.sh b/infra/gcp/prow/ensure-e2e-projects.sh index 2337fd5adc0..40dac0e1e05 100755 --- a/infra/gcp/prow/ensure-e2e-projects.sh +++ b/infra/gcp/prow/ensure-e2e-projects.sh @@ -195,6 +195,34 @@ function ensure_prow_build_cluster_metrics_endpoints() { done } +# TODO: this should be moved to the terraform responsible for k8s-infra-prow-build-trusted +function ensure_trusted_prow_build_cluster_secrets() { + local project="k8s-infra-prow-build-trusted" + local secret_specs=( + cncf-ci-github-token/sig-testing/k8s-infra-ii-coop@kubernetes.io + ) + + for spec in "${secret_specs[@]}"; do + local secret k8s_group admin_group + secret="$(echo "${spec}" | cut -d/ -f1)" + k8s_group="$(echo "${spec}" | cut -d/ -f2)" + admin_group="$(echo "${spec}" | cut -d/ -f3)" + + local admins=("k8s-infra-prow-oncall@kubernetes.io" "${admin_group}") + local labels=("group=${k8s_group}") + + color 6 "Ensuring secret '${secret}' exists in '${project}' and is owned by '${admin_group}'" + ensure_secret "${project}" "${secret}" + ensure_secret_labels "${project}" "${secret}" "${labels[@]}" + for group in "${admins[@]}"; do + ensure_secret_role_binding \ + "$(secret_full_name "${project}" "${secret}")" \ + "group:${group}" \ + "roles/secretmanager.admin" + done + done +} + function ensure_e2e_projects() { # default to all staging projects if [ $# = 0 ]; then @@ -220,6 +248,9 @@ function main() { color 6 "Ensuring monitoring.prow.k8s.io can scrape k8s-infra-prow-build metrics endpoints" ensure_prow_build_cluster_metrics_endpoints 2>&1 | indent + color 6 "Ensuring external secrets exist for use by k8s-infra-prow-build-trusted" + ensure_trusted_prow_build_cluster_secrets 2>&1 | indent + color 6 "Ensuring e2e projects used by prow..." ensure_e2e_projects "${@}" 2>&1 | indent From e8fea9fca870284cc0e58cee75b1308e2044b83e Mon Sep 17 00:00:00 2001 From: Aaron Crickenberger Date: Mon, 24 May 2021 13:53:31 -0400 Subject: [PATCH 7/7] infra/gcp/prow: add snyk-token secret --- infra/gcp/prow/ensure-e2e-projects.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/infra/gcp/prow/ensure-e2e-projects.sh b/infra/gcp/prow/ensure-e2e-projects.sh index 40dac0e1e05..be448d34f1c 100755 --- a/infra/gcp/prow/ensure-e2e-projects.sh +++ b/infra/gcp/prow/ensure-e2e-projects.sh @@ -200,6 +200,7 @@ function ensure_trusted_prow_build_cluster_secrets() { local project="k8s-infra-prow-build-trusted" local secret_specs=( cncf-ci-github-token/sig-testing/k8s-infra-ii-coop@kubernetes.io + snyk-token/sig-architecture/k8s-infra-code-organization@kubernetes.io ) for spec in "${secret_specs[@]}"; do