diff --git a/prow/Makefile b/prow/Makefile index bde0d46508f..cce625efcaa 100644 --- a/prow/Makefile +++ b/prow/Makefile @@ -7,12 +7,17 @@ include Makefile.gcloud.mk # GKE variables. PROJECT ?= istio-testing +PROJECT_ARM ?= istio-prow-build PROJECT_BUILD ?= istio-prow-build PROJECT_PRIVATE ?= istio-prow-build CLUSTER ?= prow +CLUSTER_ARM ?= prow-arm CLUSTER_BUILD ?= prow CLUSTER_PRIVATE ?= prow-private ZONE ?= us-west1-a +ZONE_ARM ?= us-central1-f +ZONE_BUILD ?= us-west1-a +ZONE_PRIVATE ?= us-west1-a update-config-dry-run: get-cluster-credentials ./recreate_prow_configmaps.py \ @@ -46,22 +51,40 @@ deploy-monitoring: get-cluster-credentials get-build-cluster-credentials: PROJECT=$(PROJECT_BUILD) get-build-cluster-credentials: CLUSTER=$(CLUSTER_BUILD) +get-build-cluster-credentials: ZONE=$(ZONE_BUILD) deploy-build: get-build-cluster-credentials kubectl apply -f ./cluster/build/ +query-build: get-build-cluster-credentials + kubectl cluster-info + +get-arm-cluster-credentials: PROJECT=$(PROJECT_ARM) +get-arm-cluster-credentials: CLUSTER=$(CLUSTER_ARM) +get-arm-cluster-credentials: ZONE=$(ZONE_ARM) + +deploy-arm: get-arm-cluster-credentials + kubectl apply -f ./cluster/arm + +query-arm: get-arm-cluster-credentials + kubectl cluster-info + get-private-cluster-credentials: PROJECT=$(PROJECT_PRIVATE) get-private-cluster-credentials: CLUSTER=$(CLUSTER_PRIVATE) +get-private-cluster-credentials: ZONE=$(ZONE_PRIVATE) deploy-private: get-private-cluster-credentials kubectl apply -f ./cluster/private/ +query-private: get-private-cluster-credentials + kubectl cluster-info + create-istio-deps-configmap: branch ?= master create-istio-deps-configmap: get-private-cluster-credentials @bash ./create-deps-cm.sh \ --local \ --branch="$(branch)" \ - --namespace=test-pods \ + --namespace=test-pods \ --key=dependencies \ $(if $(filter %,$(dry_run)),--dry-run,) \ "$(branch)-istio-deps" @@ -70,7 +93,7 @@ create-release-deps-configmap: branch ?= master create-release-deps-configmap: get-private-cluster-credentials @bash ./create-deps-cm.sh \ --branch="$(branch)" \ - --namespace=test-pods \ + --namespace=test-pods \ --key=dependencies \ $(if $(filter %,$(dry_run)),--dry-run,) \ "$(branch)-release-deps" diff --git a/prow/cluster/arm/kubernetes-external-secrets_crd.yaml b/prow/cluster/arm/kubernetes-external-secrets_crd.yaml new file mode 100644 index 00000000000..4517c263878 --- /dev/null +++ b/prow/cluster/arm/kubernetes-external-secrets_crd.yaml @@ -0,0 +1,238 @@ +--- +# From https://github.com/external-secrets/kubernetes-external-secrets/blob/master/charts/kubernetes-external-secrets/crds/kubernetes-client.io_externalsecrets_crd.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: externalsecrets.kubernetes-client.io + annotations: + # for helm v2 backwards compatibility + helm.sh/hook: crd-install + # used in e2e testing + app.kubernetes.io/managed-by: helm +spec: + group: kubernetes-client.io + scope: Namespaced + + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + required: + - spec + type: object + properties: + spec: + type: object + properties: + controllerId: + description: The ID of controller instance that manages this ExternalSecret. + This is needed in case there is more than a KES controller instances within the cluster. + type: string + type: + type: string + description: >- + DEPRECATED: Use spec.template.type + template: + description: Template which will be deep merged without mutating + any existing fields. into generated secret, can be used to + set for example annotations or type on the generated secret + type: object + x-kubernetes-preserve-unknown-fields: true + backendType: + description: >- + Determines which backend to use for fetching secrets + type: string + enum: + - secretsManager + - systemManager + - vault + - azureKeyVault + - gcpSecretsManager + - alicloudSecretsManager + - ibmcloudSecretsManager + - akeyless + vaultRole: + description: >- + Used by: vault + type: string + vaultMountPoint: + description: >- + Used by: vault + type: string + kvVersion: + description: Vault K/V version either 1 or 2, default = 2 + type: integer + minimum: 1 + maximum: 2 + keyVaultName: + description: >- + Used by: azureKeyVault + type: string + dataFrom: + type: array + items: + type: string + dataFromWithOptions: + type: array + items: + type: object + properties: + key: + description: Secret key in backend + type: string + isBinary: + description: >- + Whether the backend secret shall be treated as binary data + represented by a base64-encoded string. You must set this to true + for any base64-encoded binary data in the backend - to ensure it + is not encoded in base64 again. Default is false. + type: boolean + versionStage: + description: >- + Used by: alicloudSecretsManager, secretsManager + type: string + versionId: + description: >- + Used by: secretsManager + type: string + required: + - key + data: + type: array + items: + type: object + properties: + key: + description: Secret key in backend + type: string + name: + description: Name set for this key in the generated secret + type: string + property: + description: Property to extract if secret in backend is a JSON object + type: string + isBinary: + description: >- + Whether the backend secret shall be treated as binary data + represented by a base64-encoded string. You must set this to true + for any base64-encoded binary data in the backend - to ensure it + is not encoded in base64 again. Default is false. + type: boolean + path: + description: >- + Path from SSM to scrape secrets + This will fetch all secrets and use the key from the secret as variable name + type: string + recursive: + description: Allow to recurse thru all child keys on a given path, default false + type: boolean + secretType: + description: >- + Used by: ibmcloudSecretsManager + Type of secret - one of username_password, iam_credentials or arbitrary + type: string + version: + description: >- + Used by: gcpSecretsManager + type: string + x-kubernetes-int-or-string: true + versionStage: + description: >- + Used by: alicloudSecretsManager, secretsManager + type: string + versionId: + description: >- + Used by: secretsManager + type: string + oneOf: + - required: + - key + - name + - required: + - path + roleArn: + type: string + description: >- + Used by: alicloudSecretsManager, secretsManager, systemManager + region: + type: string + description: >- + Used by: secretsManager, systemManager + projectId: + type: string + description: >- + Used by: gcpSecretsManager + keyByName: + type: boolean + description: >- + Whether to interpret the key as a secret name (if true) or ID (the default). + Used by: ibmcloudSecretsManager + oneOf: + - properties: + backendType: + enum: + - secretsManager + - systemManager + - properties: + backendType: + enum: + - vault + - properties: + backendType: + enum: + - azureKeyVault + required: + - keyVaultName + - properties: + backendType: + enum: + - gcpSecretsManager + - properties: + backendType: + enum: + - alicloudSecretsManager + - properties: + backendType: + enum: + - ibmcloudSecretsManager + - properties: + backendType: + enum: + - akeyless + anyOf: + - required: + - data + - required: + - dataFrom + - required: + - dataFromWithOptions + status: + type: object + properties: + lastSync: + type: string + status: + type: string + observedGeneration: + type: number + additionalPrinterColumns: + - jsonPath: .status.lastSync + name: Last Sync + type: date + - jsonPath: .status.status + name: status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + + names: + shortNames: + - es + kind: ExternalSecret + plural: externalsecrets + singular: externalsecret \ No newline at end of file diff --git a/prow/cluster/arm/kubernetes-external-secrets_deployment.yaml b/prow/cluster/arm/kubernetes-external-secrets_deployment.yaml new file mode 100644 index 00000000000..aae8aac5e50 --- /dev/null +++ b/prow/cluster/arm/kubernetes-external-secrets_deployment.yaml @@ -0,0 +1,41 @@ +--- +# Source: kubernetes-external-secrets/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kubernetes-external-secrets + namespace: "default" + labels: + app.kubernetes.io/name: kubernetes-external-secrets +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kubernetes-external-secrets + template: + metadata: + labels: + app.kubernetes.io/name: kubernetes-external-secrets + spec: + serviceAccountName: kubernetes-external-secrets-sa + containers: + - name: kubernetes-external-secrets + image: "ghcr.io/external-secrets/kubernetes-external-secrets:8.5.1" + ports: + - name: prometheus + containerPort: 3001 + imagePullPolicy: IfNotPresent + resources: + {} + env: + - name: "LOG_LEVEL" + value: "info" + - name: "METRICS_PORT" + value: "3001" + - name: "POLLER_INTERVAL_MILLISECONDS" + value: "10000" + - name: "WATCH_TIMEOUT" + value: "60000" + # Params for env vars populated from k8s secrets + securityContext: + runAsNonRoot: true diff --git a/prow/cluster/arm/kubernetes-external-secrets_rbac.yaml b/prow/cluster/arm/kubernetes-external-secrets_rbac.yaml new file mode 100644 index 00000000000..88a918eeb2a --- /dev/null +++ b/prow/cluster/arm/kubernetes-external-secrets_rbac.yaml @@ -0,0 +1,67 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + iam.gke.io/gcp-service-account: kubernetes-external-secrets-sa@istio-prow-build.iam.gserviceaccount.com + namespace: default + name: "kubernetes-external-secrets-sa" +--- +# Source: kubernetes-external-secrets/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubernetes-external-secrets + labels: + app.kubernetes.io/name: kubernetes-external-secrets +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "update", "get"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "watch", "list"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + resourceNames: ["externalsecrets.kubernetes-client.io"] + verbs: ["get", "update"] + - apiGroups: ["kubernetes-client.io"] + resources: ["externalsecrets"] + verbs: ["get", "watch", "list"] + - apiGroups: ["kubernetes-client.io"] + resources: ["externalsecrets/status"] + verbs: ["get", "update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create"] +--- +# Source: kubernetes-external-secrets/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubernetes-external-secrets + labels: + app.kubernetes.io/name: kubernetes-external-secrets +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubernetes-external-secrets +subjects: + - name: kubernetes-external-secrets-sa + namespace: "default" + kind: ServiceAccount +--- +# Source: kubernetes-external-secrets/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubernetes-external-secrets-auth + labels: + app.kubernetes.io/name: kubernetes-external-secrets +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- name: kubernetes-external-secrets-sa + namespace: "default" + kind: ServiceAccount diff --git a/prow/cluster/arm/kubernetes-external-secrets_service.yaml b/prow/cluster/arm/kubernetes-external-secrets_service.yaml new file mode 100644 index 00000000000..adcd019ec7c --- /dev/null +++ b/prow/cluster/arm/kubernetes-external-secrets_service.yaml @@ -0,0 +1,17 @@ +--- +# Source: kubernetes-external-secrets/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: kubernetes-external-secrets + namespace: "default" + labels: + app.kubernetes.io/name: kubernetes-external-secrets +spec: + selector: + app.kubernetes.io/name: kubernetes-external-secrets + ports: + - protocol: TCP + port: 3001 + name: prometheus + targetPort: prometheus diff --git a/prow/cluster/arm/kubernetes_external_secrets.yaml b/prow/cluster/arm/kubernetes_external_secrets.yaml new file mode 100644 index 00000000000..09aea85672c --- /dev/null +++ b/prow/cluster/arm/kubernetes_external_secrets.yaml @@ -0,0 +1,6 @@ +# Empty +# +# We have installed no external secrets in the arm build cluster, since +# it should not be involved in any release activities. +# +# Feel free to copy them over from the build cluster if needed. diff --git a/prow/cluster/arm/prowjob_service_accounts.yaml b/prow/cluster/arm/prowjob_service_accounts.yaml new file mode 100644 index 00000000000..c713b684244 --- /dev/null +++ b/prow/cluster/arm/prowjob_service_accounts.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + iam.gke.io/gcp-service-account: istio-prow-test-job-default@istio-testing.iam.gserviceaccount.com + namespace: test-pods + # Default service account that only has permissions to access the GCS bucket + # for logging. + name: prowjob-default-sa +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + iam.gke.io/gcp-service-account: istio-prow-test-job@istio-testing.iam.gserviceaccount.com + namespace: test-pods + # Service account that has more permissions on the shared GCP projects, check + # with a Googler on what permissions it has. + # Please only use it when you do need them. + name: prowjob-advanced-sa diff --git a/prow/cluster/arm/test_pod_namespace.yaml b/prow/cluster/arm/test_pod_namespace.yaml new file mode 100644 index 00000000000..eff8a8612f0 --- /dev/null +++ b/prow/cluster/arm/test_pod_namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: test-pods diff --git a/prow/cluster/arm/tune-sysctls_daemonset.yaml b/prow/cluster/arm/tune-sysctls_daemonset.yaml new file mode 100644 index 00000000000..6cd7f677bfd --- /dev/null +++ b/prow/cluster/arm/tune-sysctls_daemonset.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: tune-sysctls + namespace: kube-system + labels: + app: tune-sysctls +spec: + selector: + matchLabels: + name: tune-sysctls + template: + metadata: + labels: + name: tune-sysctls + spec: + hostNetwork: true + hostPID: true + hostIPC: true + initContainers: + - name: setsysctls + command: + - sh + - -c + - sysctl -w fs.inotify.max_user_watches=524288; + image: alpine:3.6 + imagePullPolicy: IfNotPresent + resources: {} + securityContext: + privileged: true + volumeMounts: + - name: sys + mountPath: /sys + containers: + - name: sleepforever + resources: + requests: + cpu: 0.01 + image: alpine:3.6 + command: ["tail"] + args: ["-f", "/dev/null"] + volumes: + - name: sys + hostPath: + path: /sys diff --git a/prow/config.yaml b/prow/config.yaml index 5f7981f3661..230c96766d0 100644 --- a/prow/config.yaml +++ b/prow/config.yaml @@ -35,6 +35,10 @@ plank: path_strategy: "explicit" ssh_key_secrets: - ssh-key-secret + - cluster: prow-arm + config: + default_service_account_name: "prowjob-default-sa" # Use workload identity + gcs_credentials_secret: "" # rather than service account key secret sinker: resync_period: 1m