Skip to content

Commit

Permalink
Merge pull request #2442 from spiffxp/setup-k8s-metrics-canary
Browse files Browse the repository at this point in the history 
infra/gcp/main: add k8s-project-metrics special-case
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Aug 2, 2021
2 parents e201101 + adbd4e2 commit a075ae2
Show file tree
Hide file tree
Showing 3 changed files with 82 additions and 21 deletions.
36 changes: 29 additions & 7 deletions infra/gcp/clusters/projects/k8s-infra-prow-build-trusted/prow-build-trusted/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,16 @@ This file defines:

locals {
project_id = "k8s-infra-prow-build-trusted"
cluster_name = "prow-build-trusted" // The name of the cluster defined in this file
cluster_location = "us-central1" // The GCP location (region or zone) where the cluster should be created
bigquery_location = "US" // The bigquery specific location where the dataset should be created
pod_namespace = "test-pods" // MUST match whatever prow is configured to use when it schedules to this cluster
cluster_name = "prow-build-trusted" // The name of the cluster defined in this file
cluster_location = "us-central1" // The GCP location (region or zone) where the cluster should be created
bigquery_location = "US" // The bigquery specific location where the dataset should be created
pod_namespace = "test-pods" // MUST match whatever prow is configured to use when it schedules to this cluster

// Service Accounts in ${pod_namespace} (usable via Workload Identity)
cluster_sa_name = "prow-build-trusted" // Pods use this by default
gcb_builder_sa_name = "gcb-builder" // Allowed to run GCB builds and push to GCS buckets
prow_deployer_sa_name = "prow-deployer" // Allowed to deploy to prow build clusters
cluster_sa_name = "prow-build-trusted" // Pods use this by default
gcb_builder_sa_name = "gcb-builder" // Allowed to run GCB builds and push to GCS buckets
prow_deployer_sa_name = "prow-deployer" // Allowed to deploy to prow build clusters
k8s_metrics_sa_name = "k8s-metrics" // Allowed to write to gs://k8s-metrics
}

data "google_organization" "org" {
Expand Down Expand Up @@ -117,6 +118,27 @@ resource "google_service_account_iam_policy" "prow_deployer_sa_iam" {
service_account_id = google_service_account.prow_deployer_sa.name
policy_data = data.google_iam_policy.prow_deployer_sa_workload_identity.policy_data
}
// Create GCP SA for jobs that write to gs://k8s-metrics and gs://k8s-project-metrics
resource "google_service_account" "k8s_metrics_sa" {
project = local.project_id
account_id = local.k8s_metrics_sa_name
display_name = local.k8s_metrics_sa_name
}
// Allow pods using the build cluster KSA to use the GCP SA via workload identity
data "google_iam_policy" "k8s_metrics_sa_workload_identity" {
binding {
role = "roles/iam.workloadIdentityUser"

members = [
"serviceAccount:${local.project_id}.svc.id.goog[${local.pod_namespace}/${local.k8s_metrics_sa_name}]",
]
}
}
// Authoritative iam-policy: replaces any existing policy attached to this service_account
resource "google_service_account_iam_policy" "k8s_metrics_sa_iam" {
service_account_id = google_service_account.k8s_metrics_sa.name
policy_data = data.google_iam_policy.k8s_metrics_sa_workload_identity.policy_data
}

resource "google_project_iam_member" "prow_deployer_for_prow_build_trusted" {
project = local.project_id
Expand Down
38 changes: 26 additions & 12 deletions ...-prow-build-trusted/prow-build-trusted/resources/test-pods/test-pods-serviceaccounts.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
# Prow service accounts
---
apiVersion: v1
kind: ServiceAccount
Expand All @@ -7,20 +8,30 @@ metadata:
name: prow-build-trusted
namespace: test-pods
---
kind: ServiceAccount
apiVersion: v1
metadata:
annotations:
iam.gke.io/gcp-service-account: prow-deployer@k8s-infra-prow-build-trusted.iam.gserviceaccount.com
name: prow-deployer
namespace: test-pods
---
kind: ServiceAccount
apiVersion: v1
metadata:
annotations:
iam.gke.io/gcp-service-account: k8s-infra-gcp-auditor@kubernetes-public.iam.gserviceaccount.com
name: k8s-infra-gcp-auditor
iam.gke.io/gcp-service-account: k8s-metrics@k8s-infra-prow-build-trusted.iam.gserviceaccount.com
name: k8s-metrics
namespace: test-pods

# Infrastructure management service accounts
---
apiVersion: v1
kind: ServiceAccount
metadata:
annotations:
iam.gke.io/gcp-service-account: gcb-builder@k8s-infra-prow-build-trusted.iam.gserviceaccount.com
name: gcb-builder
iam.gke.io/gcp-service-account: k8s-infra-gcp-auditor@kubernetes-public.iam.gserviceaccount.com
name: k8s-infra-gcp-auditor
namespace: test-pods
---
kind: ServiceAccount
Expand All @@ -38,30 +49,33 @@ metadata:
iam.gke.io/gcp-service-account: [email protected]
name: gsuite-groups-manager
namespace: test-pods

# Image promotion service accounts
---
kind: ServiceAccount
apiVersion: v1
metadata:
annotations:
iam.gke.io/gcp-service-account: prow-deployer@k8s-infra-prow-build-trusted.iam.gserviceaccount.com
name: prow-deployer
iam.gke.io/gcp-service-account: k8s-infra-gcr-promoter@k8s-artifacts-prod.iam.gserviceaccount.com
name: k8s-infra-gcr-promoter
namespace: test-pods
---
# Image promotion service accounts
kind: ServiceAccount
apiVersion: v1
metadata:
annotations:
iam.gke.io/gcp-service-account: k8s-infra-gcr-promoter@k8s-artifacts-prod.iam.gserviceaccount.com
name: k8s-infra-gcr-promoter
iam.gke.io/gcp-service-account: k8s-infra-gcr-promoter@k8s-artifacts-prod-bak.iam.gserviceaccount.com
name: k8s-infra-gcr-promoter-bak
namespace: test-pods

# Staging service accounts
---
kind: ServiceAccount
apiVersion: v1
kind: ServiceAccount
metadata:
annotations:
iam.gke.io/gcp-service-account: k8s-infra-gcr-promoter@k8s-artifacts-prod-bak.iam.gserviceaccount.com
name: k8s-infra-gcr-promoter-bak
iam.gke.io/gcp-service-account: gcb-builder@k8s-infra-prow-build-trusted.iam.gserviceaccount.com
name: gcb-builder
namespace: test-pods
---
kind: ServiceAccount
Expand Down
29 changes: 27 additions & 2 deletions infra/gcp/ensure-main-project.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -396,11 +396,36 @@ function ensure_prow_special_cases {
return 1
fi
local project="${1}"

local bucket principal secret

color 6 "Special case: ensuring k8s-infra-ci-robot-github-token accessible by k8s-infra-prow-build-trusted"
local principal secret
principal="serviceAccount:$(svc_acct_email "k8s-infra-prow-build-trusted" "kubernetes-external-secrets")"
secret=$(secret_full_name "${project}" "k8s-infra-ci-robot-github-token")
ensure_secret_role_binding "${secret}" "${principal}" "roles/secretmanager.secretAccessor"
ensure_secret_role_binding "${secret}" "${principal}" "roles/secretmanager.secretAccessor" 2>&1 | indent

color 6 "Special case: ensuring gs://k8s-metrics-canary exists for gs://k8s-metrics migration"
(
bucket="gs://k8s-project-metrics"
owners="[email protected]"
local old_service_account="[email protected]"

ensure_public_gcs_bucket "${project}" "${bucket}"
ensure_gcs_bucket_auto_deletion "${bucket}" "365" # match gs://k8s-metrics
# GCS admins can admin all GCS buckets
empower_gcs_admins "${project}" "${bucket}"
# bucket owners can admin this bucket
empower_group_to_admin_gcs_bucket "${owners}" "${bucket}"
# TODO(spiffxp): copy pasted to flip to ensure_removed when migrated
# k8s-prow-builds can write to this bucket
principal="serviceAccount:${old_service_account}"
ensure_gcs_role_binding "${bucket}" "${principal}" "objectAdmin"
ensure_gcs_role_binding "${bucket}" "${principal}" "legacyBucketWriter"
# k8s-infra-prow-build-trusted can write to this bucket
principal="serviceAccount:$(svc_acc_email "k8s-infra-prow-build-trusted" "k8s-metrics")"
ensure_gcs_role_binding "${bucket}" "${principal}" "objectAdmin"
ensure_gcs_role_binding "${bucket}" "${principal}" "legacyBucketWriter"
) 2>&1 | indent
}

function ensure_main_project() {
Expand Down

0 comments on commit a075ae2

Please sign in to comment.