ensure-organization.sh followup

infra/gcp/clusters/modules/gke-project/README.md

@@ -6,8 +6,9 @@ that is intended to host a GKE cluster created by the [`gke-cluster`] module:
 - Project is linked to CNCF billing account
 - Services necessary to support [`gke-cluster`] are enabled
 - Some default IAM bindings are added:
-  - k8s-infra-cluster-admins@ gets `roles/compute.viewer`, `roles/container.admin`, `roles/ServiceAccountLister`
+  - k8s-infra-cluster-admins@ gets `roles/compute.viewer`, `roles/container.admin`, org role [`ServiceAccountLister`]
   - gke-security-groups@ gets `roles/container.clusterViewer`
 
 [`gke-cluster`]: /infra/gcp/clusters/modules/gke-cluster
 [`gke-nodepool`]: /infra/gcp/clusters/modules/gke-nodepool
+[`ServiceAccountLister`]: /infra/gcp/roles/iam.serviceAccountLister.yaml

infra/gcp/clusters/modules/gke-project/main.tf

@@ -14,12 +14,31 @@
  * limitations under the License.
  */
 
-/// Create the project in which we're creating the cluster
+// hardcoded org and billing account, not sure we want this
+// reusable outside of k8s-infra
+locals {
+  org_domain      = "kubernetes.io"
+  billing_account = "018801-93540E-22A20E"
+}
+
+data "google_organization" "org" {
+  domain = local.org_domain
+}
+
+// TODO(spiffxp): explicitly not using a data source for this until
+// I have a better sense of whether this requires more permissions
+// than (are / should be) available for k8s-infra-prow-oncall and
+// k8s-infra-cluster-admins
+// data google_billing_account {
+// billing_account = locals.billing_account
+// }
+
+// Create the project in which we're creating the cluster
 resource "google_project" "project" {
   name            = var.project_name
   project_id      = var.project_name
-  org_id          = "758905017065"          // kubernetes.io
-  billing_account = "018801-93540E-22A20E"
+  org_id          = data.google_organization.org.org_id
+  billing_account = local.billing_account
 }
 
 // Services we need
@@ -91,16 +110,15 @@ resource "google_project_iam_member" "cluster_admins_as_container_admin" {
   role    = "roles/container.admin"
   member  = "group:${var.cluster_admins_group}"
 }
-resource "google_project_iam_custom_role" "service_account_lister" {
-  project     = google_project.project.project_id
-  role_id     = "ServiceAccountLister"
-  title       = "Service Account Lister"
-  description = "Can list ServiceAccounts."
-  permissions = ["iam.serviceAccounts.list"]
+
+// Role created by infra/gcp/ensure-organization.sh, use a data source to ensure it exists
+data "google_iam_role" "service_account_lister" {
+  name = "${data.google_organization.org.name}/roles/iam.serviceAccountLister"
 }
+
 resource "google_project_iam_member" "cluster_admins_as_service_account_lister" {
   project = google_project.project.project_id
-  role    = "projects/${google_project.project.project_id}/roles/${google_project_iam_custom_role.service_account_lister.role_id}"
+  role    = data.google_iam_role.service_account_lister.name
   member  = "group:${var.cluster_admins_group}"
 }
 

infra/gcp/clusters/projects/k8s-infra-prow-build/prow-build/main.tf

@@ -33,6 +33,10 @@ locals {
   boskos_janitor_sa_name       = "boskos-janitor"       // Name of the GSA and KSA used by boskos-janitor
 }
 
+data "google_organization" "org" {
+  domain = "kubernetes.io"
+}
+
 module "project" {
   source = "../../../modules/gke-project"
   project_id            = local.project_id
@@ -46,11 +50,15 @@ resource "google_project_iam_member" "k8s_infra_prow_oncall" {
   member  = "group:k8s-infra-prow-oncall@kubernetes.io"
 }
 
+// Role created by infra/gcp/ensure-organization.sh, use a data source to ensure it exists
+data "google_iam_role" "prow_viewer" {
+  name = "${data.google_organization.org.name}/roles/prow.viewer"
+}
+
 // Ensure k8s-infra-prow-viewers@kuberentes.io has prow.viewer access to this project
 resource "google_project_iam_member" "k8s_infra_prow_viewers" {
   project = local.project_id
-  # TODO: use data resource to get org role name instead of hardcode
-  role    = "organizations/758905017065/roles/prow.viewer"
+  role    = data.google_iam_role.prow_viewer.name
   member  = "group:k8s-infra-prow-viewers@kubernetes.io"
 }
 

infra/gcp/ensure-main-project.sh

@@ -90,23 +90,20 @@ gcloud projects add-iam-policy-binding "${PROJECT}" \
     --role roles/bigquery.admin
 
 color 6 "Empowering cluster admins"
-gcloud projects add-iam-policy-binding "${PROJECT}" \
-    --member "group:${CLUSTER_ADMINS_GROUP}" \
-    --role roles/compute.viewer
-gcloud projects add-iam-policy-binding "${PROJECT}" \
-    --member "group:${CLUSTER_ADMINS_GROUP}" \
-    --role roles/container.admin
-gcloud projects add-iam-policy-binding "${PROJECT}" \
-    --member "group:${CLUSTER_ADMINS_GROUP}" \
-    --role roles/compute.loadBalancerAdmin
-ensure_custom_iam_role "${PROJECT}" \
-    ServiceAccountLister \
-    "Service Account Lister" \
-    "Can list ServiceAccounts." \
-    iam.serviceAccounts.list
-gcloud projects add-iam-policy-binding "${PROJECT}" \
-    --member "group:${CLUSTER_ADMINS_GROUP}" \
-    --role "projects/${PROJECT}/roles/ServiceAccountLister"
+# TODO: this can also be a custom role
+cluster_admin_roles=(
+    roles/compute.viewer
+    roles/container.admin
+    roles/compute.loadBalancerAdmin
+    $(custom_org_role_name iam.serviceAccountLister)
+)
+for role in "${cluster_admin_roles[@]}"; do
+    ensure_project_role_binding "${PROJECT}" "group:${CLUSTER_ADMINS_GROUP}" "${role}"
+done
+# TODO(spiffxp): remove when bindings for custom project role are gone
+ensure_removed_project_role_binding "${PROJECT}" "group:${CLUSTER_ADMINS_GROUP}" "$(custom_project_role_name "${PROJECT}" ServiceAccountLister)"
+ensure_removed_project_role "${PROJECT}" "ServiceAccountLister"
+
 gsutil iam ch \
     "group:${CLUSTER_ADMINS_GROUP}:objectAdmin" \
     "gs://${CLUSTER_TERRAFORM_BUCKET}"

infra/gcp/ensure-organization.sh

@@ -40,18 +40,32 @@ org_roles=(
     secretmanager.secretLister
     organization.admin
     CustomRole
+    iam.serviceAccountLister
+)
+
+old_org_roles=(
     StorageBucketLister
 )
 
+# TODO(https://github.com/kubernetes/k8s.io/issues/1659): obviated by organization.admin, remove when bindings gone
+old_org_admin_roles=(
+    roles/billing.user
+    roles/iam.organizationRoleAdmin
+    roles/resourcemanager.organizationAdmin
+    roles/resourcemanager.projectCreator
+    roles/resourcemanager.projectDeleter
+    roles/servicemanagement.quotaAdmin
+)
+
 color 6 "Ensuring organization custom roles exist"
 (
     for role in "${org_roles[@]}"; do
       color 6 "Ensuring organization custom role ${role}"
-      ensure_custom_iam_role_from_file "org" "${role}" "${SCRIPT_DIR}/roles/${role}.yaml"
+      ensure_custom_org_iam_role_from_file "${role}" "${SCRIPT_DIR}/roles/${role}.yaml"
     done
 ) 2>&1 | indent
 
-color 6 "Ensuring org-level IAM bindings exist"
+color 6 "Ensuring organization IAM bindings exist"
 (
     # k8s-infra-prow-oncall@kubernetes.io should be able to browse org resources
     ensure_org_role_binding "group:k8s-infra-prow-oncall@kubernetes.io" "roles/browser"
@@ -65,42 +79,30 @@ color 6 "Ensuring org-level IAM bindings exist"
 
     # k8s-infra-gcp-auditors@
     ensure_org_role_binding "group:k8s-infra-gcp-auditors@kubernetes.io" "$(custom_org_role_name "audit.viewer")"
-    # TODO(https://github.com/kubernetes/k8s.io/issues/1659): obviated by audit.viewer, remove when bindings gone
-    old_audit_roles=(
-        "$(custom_org_role_name "StorageBucketLister")"
-        roles/compute.viewer
-        roles/dns.reader
-        roles/iam.securityReviewer
-        roles/resourcemanager.organizationViewer
-        roles/serviceusage.serviceUsageConsumer
-    )
-    for role in "${old_audit_roles[@]}"; do
-        ensure_removed_org_role_binding "group:k8s-infra-gcp-auditors@kubernetes.io" "${role}"
-    done
-
-    echo "exiting early to confirm audit.viewer role migration has worked"
-    exit 0
 
     # k8s-infra-org-admins@
     # roles/owner has too many permissions to aggregate into a custom role,
     # and some services (e.g. storage) add bindings based on membership in it
     ensure_org_role_binding "group:k8s-infra-gcp-org-admins@kubernetes.io" "roles/owner"
     # everything org admins need beyond roles/owner to manage the org
     ensure_org_role_binding "group:k8s-infra-gcp-org-admins@kubernetes.io" "$(custom_org_role_name "organization.admin")"
-    # TODO(https://github.com/kubernetes/k8s.io/issues/1659): obviated by organization.admin, remove when bindings gone
-    old_org_admin_roles=(
-        roles/billing.user
-        roles/iam.organizationRoleAdmin
-        roles/resourcemanager.organizationAdmin
-        roles/resourcemanager.projectCreator
-        roles/resourcemanager.projectDeleter
-        roles/servicemanagement.quotaAdmin
-    )
-    for role in "${old_audit_roles[@]}"; do
-        # TODO(spiffxp): remove the extra super duper paranoia once we verify
-        #                I haven't locked myself out via group membership
-        ensure_org_role_binding "user:thockin@google.com" "${role}"
-        ensure_org_role_binding "user:davanum@gmail.com" "${role}"
-        ensure_removed_org_role_binding "group:k8s-infra-gcp-org-admins@kubernetes.io" "${role}"
+) 2>&1 | indent
+
+color 6 "Ensuring removed organization IAM bindings do not exist"
+(
+    # TODO(spiffxp): remove this once the old bindings are confirmed gone
+    for role in "${old_org_admin_roles[@]}"; do
+        ensure_removed_org_role_binding "user:thockin@google.com" "${role}"
+        ensure_removed_org_role_binding "user:davanum@gmail.com" "${role}"
     done
 ) 2>&1 | indent
+
+color 6 "Ensuring removed organization custom roles do not exist"
+(
+    for role in "${old_org_roles[@]}"; do
+      color 6 "Ensuring removed organization custom role ${role}"
+      ensure_removed_custom_org_iam_role "${role}"
+    done
+) 2>&1 | indent
+
+color 6 "All done!"