Istio prow: enable secret managed by kubernetes external secrets

prow/README.md

@@ -182,3 +182,10 @@ repos:
             contexts:
             - "merges-blocked-needs-admin"
 ```
+
+## Prow Secrets
+
+Some of the prow secrets are managed by kubernetes external secrets, which
+allows prow cluster creating secrets based on values from google secret manager
+(Not necessarily the same GCP project where prow is located). See more detailed
+instruction at [Prow Secret](https://github.com/kubernetes/test-infra/blob/master/prow/prow_secrets.md).

prow/cluster/kubernetes-external-secrets_crd.yaml

@@ -0,0 +1,140 @@
+---
+# From https://github.com/external-secrets/kubernetes-external-secrets/tree/f866e34e0319d9c54ea17d07f5d5818a28d9d0f6
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: externalsecrets.kubernetes-client.io
+  labels:
+    app.kubernetes.io/part-of: prow
+  annotations:
+    # for helm v2 backwards compatibility
+    helm.sh/hook: crd-install
+    # used in e2e testing
+    app.kubernetes.io/managed-by: helm
+spec:
+  group: kubernetes-client.io
+  version: v1
+  scope: Namespaced
+
+  names:
+    shortNames:
+      - es
+    kind: ExternalSecret
+    plural: externalsecrets
+    singular: externalsecret
+
+  additionalPrinterColumns:
+    - JSONPath: .status.lastSync
+      name: Last Sync
+      type: date
+    - JSONPath: .status.status
+      name: status
+      type: string
+    - JSONPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          type: object
+          properties:
+            template:
+              description: Template which will be deep merged without mutating
+                any existing fields. into generated secret, can be used to
+                set for example annotations or type on the generated secret
+              type: object
+            backendType:
+              type: string
+              enum:
+                - secretsManager
+                - systemManager
+                - vault
+                - azureKeyVault
+                - gcpSecretsManager
+                - alicloudSecretsManager
+            vaultRole:
+              type: string
+            vaultMountPoint:
+              type: string
+            kvVersion:
+              description: Vault K/V version either 1 or 2, default = 2
+              type: integer
+              minimum: 1
+              maximum: 2
+            keyVaultName:
+              type: string
+            key:
+              type: string
+            dataFrom:
+              type: array
+              items:
+                type: string
+            data:
+              type: array
+              items:
+                type: object
+                anyOf:
+                  - properties:
+                      key:
+                        description: Secret key in backend
+                        type: string
+                      name:
+                        description: Name set for this key in the generated secret
+                        type: string
+                      property:
+                        description: Property to extract if secret in backend is a JSON object
+                      isBinary:
+                        description: >-
+                          Whether the backend secret shall be treated as binary data
+                          represented by a base64-encoded string. You must set this to true
+                          for any base64-encoded binary data in the backend - to ensure it
+                          is not encoded in base64 again. Default is false.
+                        type: boolean
+                    required:
+                      - key
+                      - name
+                  - properties:
+                      path:
+                        description: >-
+                          Path from SSM to scrape secrets
+                          This will fetch all secrets and use the key from the secret as variable name
+                      recursive:
+                        description: Allow to recurse thru all child keys on a given path
+                        type: boolean
+                    required:
+                      - path
+            roleArn:
+              type: string
+          oneOf:
+            - properties:
+                backendType:
+                  enum:
+                    - secretsManager
+                    - systemManager
+            - properties:
+                backendType:
+                  enum:
+                    - vault
+            - properties:
+                backendType:
+                  enum:
+                    - azureKeyVault
+              required:
+                - keyVaultName
+            - properties:
+                backendType:
+                  enum:
+                    - gcpSecretsManager
+            - properties:
+                backendType:
+                  enum:
+                    - alicloudSecretsManager
+          anyOf:
+            - required:
+                - data
+            - required:
+                - dataFrom
+  subresources:
+    status: {}

prow/cluster/kubernetes-external-secrets_deployment.yaml

@@ -0,0 +1,42 @@
+---
+# Source: kubernetes-external-secrets/templates/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kubernetes-external-secrets
+  namespace: "default"
+  labels:
+    app.kubernetes.io/part-of: prow
+    app.kubernetes.io/name: kubernetes-external-secrets
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kubernetes-external-secrets
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kubernetes-external-secrets
+    spec:
+      serviceAccountName: kubernetes-external-secrets-sa
+      containers:
+        - name: kubernetes-external-secrets
+          image: "ghcr.io/external-secrets/kubernetes-external-secrets:6.4.0"
+          ports:
+          - name: prometheus
+            containerPort: 3001
+          imagePullPolicy: IfNotPresent
+          resources:
+            {}
+          env:
+          - name: "LOG_LEVEL"
+            value: "info"
+          - name: "METRICS_PORT"
+            value: "3001"
+          - name: "POLLER_INTERVAL_MILLISECONDS"
+            value: "10000"
+          - name: "WATCH_TIMEOUT"
+            value: "60000"
+          # Params for env vars populated from k8s secrets
+      securityContext:
+        runAsNonRoot: true

prow/cluster/kubernetes-external-secrets_rbac.yaml

@@ -0,0 +1,72 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  annotations:
+    iam.gke.io/gcp-service-account: kubernetes-external-secrets-sa@istio-testing.iam.gserviceaccount.com
+  labels:
+    app.kubernetes.io/part-of: prow
+  namespace: default
+  name: "kubernetes-external-secrets-sa"
+---
+# Source: kubernetes-external-secrets/templates/rbac.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubernetes-external-secrets
+  labels:
+    app.kubernetes.io/part-of: prow
+    app.kubernetes.io/name: kubernetes-external-secrets
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["create", "update"]
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["get", "watch", "list"]
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    resourceNames: ["externalsecrets.kubernetes-client.io"]
+    verbs: ["get", "update"]
+  - apiGroups: ["kubernetes-client.io"]
+    resources: ["externalsecrets"]
+    verbs: ["get", "watch", "list"]
+  - apiGroups: ["kubernetes-client.io"]
+    resources: ["externalsecrets/status"]
+    verbs: ["get", "update"]
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["create"]
+---
+# Source: kubernetes-external-secrets/templates/rbac.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubernetes-external-secrets
+  labels:
+    app.kubernetes.io/part-of: prow
+    app.kubernetes.io/name: kubernetes-external-secrets
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubernetes-external-secrets
+subjects:
+  - name: kubernetes-external-secrets-sa
+    namespace: "default"
+    kind: ServiceAccount
+---
+# Source: kubernetes-external-secrets/templates/rbac.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubernetes-external-secrets-auth
+  labels:
+    app.kubernetes.io/part-of: prow
+    app.kubernetes.io/name: kubernetes-external-secrets
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- name: kubernetes-external-secrets-sa
+  namespace: "default"
+  kind: ServiceAccount

prow/cluster/kubernetes-external-secrets_service.yaml

@@ -0,0 +1,18 @@
+---
+# Source: kubernetes-external-secrets/templates/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: kubernetes-external-secrets
+  namespace: "default"
+  labels:
+    app.kubernetes.io/part-of: prow
+    app.kubernetes.io/name: kubernetes-external-secrets
+spec:
+  selector:
+    app.kubernetes.io/name: kubernetes-external-secrets
+  ports:
+    - protocol: TCP
+      port: 3001
+      name: prometheus
+      targetPort: prometheus

prow/cluster/monitoring/mixins/lib/config.libsonnet

@@ -44,6 +44,8 @@ local config = {
 
   // How many days prow hasn't been bumped.
   prowImageStaleByDays: {daysStale: 7, eventDuration: '24h'},
+
+  kubernetesExternalSecretServiceAccount: 'kubernetes-external-secrets-sa@istio-testing.iam.gserviceaccount.com',
 };
 
 // Generate the real config by adding in constant fields and defaulting where needed.

prow/cluster/monitoring/mixins/prometheus/external_secret_alerts.libsonnet

@@ -0,0 +1,26 @@
+{
+  prometheusAlerts+:: {
+    groups+: [
+      {
+        name: 'external-secret-sync',
+        rules: [
+          {
+            # https://github.com/external-secrets/kubernetes-external-secrets/blob/master/README.md#metrics
+            alert: 'Failed-syncing-external-secret',
+            # Prometheus scrapes kubernetes external secrets every 30 seconds as defined in servicemonitor, so this counts failures between scrape intervals.
+            # Since kubernetes secret manager runs every 10 seconds, there should be at least 2 runs in every 30s, so this will only report consecutive failures.
+            expr: |||
+              increase(kubernetes_external_secrets_sync_calls_count{job="kubernetes-external-secrets",status!="success"}[1m]) > 1.5
+            |||,
+            labels: {
+              severity: 'user-warning',
+            },
+            annotations: {
+              message: 'ExternalSecret {{ $labels.namespace }}/{{ $labels.name }} failed to be synced. does %s have `roles/secretmanager.viewer` and `roles/secretmanager.secretAccessor` permissions on the google secret manager secret used for this cluster secret?' % $._config.kubernetesExternalSecretServiceAccount,
+            },
+          }
+        ],
+      },
+    ],
+  },
+}

prow/cluster/monitoring/mixins/prometheus/prometheus.libsonnet

@@ -10,4 +10,5 @@
 (import 'stale_alerts.libsonnet') +
 (import 'tide_alerts.libsonnet') +
 (import 'prober_alerts.libsonnet') +
-(import 'slo_recordrules.libsonnet')
+(import 'slo_recordrules.libsonnet') +
+(import 'external_secret_alerts.libsonnet')

prow/cluster/monitoring/prow_servicemonitors.yaml

@@ -207,3 +207,23 @@ spec:
   selector:
     matchLabels:
       app: crier
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/name: kubernetes-external-secrets
+    app: kubernetes-external-secrets
+  name: kubernetes-external-secrets
+  namespace: prow-monitoring
+spec:
+  endpoints:
+  - interval: 30s
+    port: prometheus
+    scheme: http
+  namespaceSelector:
+    matchNames:
+    - default
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kubernetes-external-secrets