From febebbf62e401a05038d85a7568f71bb750e2166 Mon Sep 17 00:00:00 2001 From: Casey Callendrello Date: Thu, 8 Feb 2018 14:51:58 +0100 Subject: [PATCH 1/2] operator: switch to tectonic-ingress-controller-operator This moves the ingress controller in to its own namespace. --- config.tf | 33 ++++++------ .../manifests/02-ingress-namespace.yaml | 6 --- modules/tectonic/assets.tf | 17 +++--- .../resources/manifests/ingress/README.md | 2 + .../manifests/ingress/cluster-config.yaml | 10 ++++ .../manifests/ingress/namespace.yaml | 9 ++++ .../resources/manifests/ingress/pull.json | 12 +++++ .../manifests/ingress/svc-account.yaml | 5 ++ .../manifests/rbac/binding-admin.yaml | 3 ++ .../app-version-tectonic-ingress.yaml | 14 +++++ .../tectonic-ingress-controller-operator.yaml | 52 +++++++++++++++++++ modules/tectonic/resources/tectonic.sh | 8 +++ 12 files changed, 141 insertions(+), 30 deletions(-) delete mode 100644 modules/bootkube/resources/manifests/02-ingress-namespace.yaml create mode 100644 modules/tectonic/resources/manifests/ingress/README.md create mode 100644 modules/tectonic/resources/manifests/ingress/cluster-config.yaml create mode 100644 modules/tectonic/resources/manifests/ingress/namespace.yaml create mode 100644 modules/tectonic/resources/manifests/ingress/pull.json create mode 100644 modules/tectonic/resources/manifests/ingress/svc-account.yaml create mode 100644 modules/tectonic/resources/manifests/updater/app_versions/app-version-tectonic-ingress.yaml create mode 100644 modules/tectonic/resources/manifests/updater/operators/tectonic-ingress-controller-operator.yaml diff --git a/config.tf b/config.tf index 9daa82eaff..b0c3d164dd 100644 --- a/config.tf +++ b/config.tf @@ -67,22 +67,23 @@ variable "tectonic_container_images" { type = "map" default = { - addon_resizer = "gcr.io/google_containers/addon-resizer:2.1" - awscli = "quay.io/coreos/awscli:025a357f05242fdad6a81e8a6b520098aa65a600" - gcloudsdk = "google/cloud-sdk:178.0.0-alpine" - bootkube = "quay.io/coreos/bootkube:v0.10.0" - etcd = "quay.io/coreos/etcd:v3.2.14" - hyperkube = "quay.io/coreos/hyperkube:v1.9.1_coreos.0" - kube_core_renderer = "quay.io/coreos/kube-core-renderer-dev:6c49ce4da9fc36966812381891b4f558aa53097b" - kube_core_operator = "quay.io/coreos/kube-core-operator:beryllium-m1" - tectonic_channel_operator = "quay.io/coreos/tectonic-channel-operator:0.6.2" - tectonic_prometheus_operator = "quay.io/coreos/tectonic-prometheus-operator:v1.9.3" - tectonic_cluo_operator = "quay.io/coreos/tectonic-cluo-operator:v0.3.1" - tectonic_torcx = "quay.io/coreos/tectonic-torcx:v0.2.1" - kubernetes_addon_operator = "quay.io/coreos/kubernetes-addon-operator:beryllium-m1" - tectonic_alm_operator = "quay.io/coreos/tectonic-alm-operator:v0.3.1" - tectonic_utility_operator = "quay.io/coreos/tectonic-utility-operator:beryllium-m1" - tectonic_network_operator = "quay.io/coreos/tectonic-network-operator:beryllium-m1" + addon_resizer = "gcr.io/google_containers/addon-resizer:2.1" + awscli = "quay.io/coreos/awscli:025a357f05242fdad6a81e8a6b520098aa65a600" + gcloudsdk = "google/cloud-sdk:178.0.0-alpine" + bootkube = "quay.io/coreos/bootkube:v0.10.0" + etcd = "quay.io/coreos/etcd:v3.2.14" + hyperkube = "quay.io/coreos/hyperkube:v1.9.1_coreos.0" + kube_core_renderer = "quay.io/coreos/kube-core-renderer-dev:6c49ce4da9fc36966812381891b4f558aa53097b" + kube_core_operator = "quay.io/coreos/kube-core-operator:beryllium-m1" + tectonic_channel_operator = "quay.io/coreos/tectonic-channel-operator:0.6.2" + tectonic_prometheus_operator = "quay.io/coreos/tectonic-prometheus-operator:v1.9.3" + tectonic_cluo_operator = "quay.io/coreos/tectonic-cluo-operator:v0.3.1" + tectonic_torcx = "quay.io/coreos/tectonic-torcx:v0.2.1" + kubernetes_addon_operator = "quay.io/coreos/kubernetes-addon-operator:beryllium-m1" + tectonic_alm_operator = "quay.io/coreos/tectonic-alm-operator:v0.3.1" + tectonic_ingress_controller_operator = "quay.io/coreos/tectonic-ingress-controller-operator:f96287f555b7366af14dfcbb02f9a6529dd24b99" + tectonic_utility_operator = "quay.io/coreos/tectonic-utility-operator:7884c5c9b6cf738e3bda2731449c5c2ead54b390" + tectonic_network_operator = "quay.io/coreos/tectonic-network-operator:beryllium-m1" } } diff --git a/modules/bootkube/resources/manifests/02-ingress-namespace.yaml b/modules/bootkube/resources/manifests/02-ingress-namespace.yaml deleted file mode 100644 index 3adc00bf11..0000000000 --- a/modules/bootkube/resources/manifests/02-ingress-namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: tectonic-ingress # Create the namespace first. - labels: # network policy can only select by labels - name: tectonic-ingress diff --git a/modules/tectonic/assets.tf b/modules/tectonic/assets.tf index a697364118..fa1368dd3d 100644 --- a/modules/tectonic/assets.tf +++ b/modules/tectonic/assets.tf @@ -9,14 +9,15 @@ resource "template_dir" "tectonic" { destination_dir = "./generated/tectonic" vars { - addon_resizer_image = "${var.container_images["addon_resizer"]}" - kube_core_operator_image = "${var.container_images["kube_core_operator"]}" - kubernetes_addon_operator_image = "${var.container_images["kubernetes_addon_operator"]}" - tectonic_channel_operator_image = "${var.container_images["tectonic_channel_operator"]}" - tectonic_prometheus_operator_image = "${var.container_images["tectonic_prometheus_operator"]}" - tectonic_cluo_operator_image = "${var.container_images["tectonic_cluo_operator"]}" - tectonic_alm_operator_image = "${var.container_images["tectonic_alm_operator"]}" - tectonic_utility_operator_image = "${var.container_images["tectonic_utility_operator"]}" + addon_resizer_image = "${var.container_images["addon_resizer"]}" + kube_core_operator_image = "${var.container_images["kube_core_operator"]}" + kubernetes_addon_operator_image = "${var.container_images["kubernetes_addon_operator"]}" + tectonic_channel_operator_image = "${var.container_images["tectonic_channel_operator"]}" + tectonic_prometheus_operator_image = "${var.container_images["tectonic_prometheus_operator"]}" + tectonic_cluo_operator_image = "${var.container_images["tectonic_cluo_operator"]}" + tectonic_alm_operator_image = "${var.container_images["tectonic_alm_operator"]}" + tectonic_ingress_controller_operator_image = "${var.container_images["tectonic_ingress_controller_operator"]}" + tectonic_utility_operator_image = "${var.container_images["tectonic_utility_operator"]}" tectonic_monitoring_auth_base_image = "${var.container_base_images["tectonic_monitoring_auth"]}" config_reload_base_image = "${var.container_base_images["config_reload"]}" diff --git a/modules/tectonic/resources/manifests/ingress/README.md b/modules/tectonic/resources/manifests/ingress/README.md new file mode 100644 index 0000000000..60ef4987e9 --- /dev/null +++ b/modules/tectonic/resources/manifests/ingress/README.md @@ -0,0 +1,2 @@ +tectonic-ingress-controller-operator is a special case, since it is in its own +namespace and reads its own config. diff --git a/modules/tectonic/resources/manifests/ingress/cluster-config.yaml b/modules/tectonic/resources/manifests/ingress/cluster-config.yaml new file mode 100644 index 0000000000..3fc7027853 --- /dev/null +++ b/modules/tectonic/resources/manifests/ingress/cluster-config.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-config-v1 + namespace: tectonic-ingress +data: + ingress-config: | + apiVersion: v1 + kind: TectonicIngressOperatorConfig + installerPlatform: ${platform} diff --git a/modules/tectonic/resources/manifests/ingress/namespace.yaml b/modules/tectonic/resources/manifests/ingress/namespace.yaml new file mode 100644 index 0000000000..aed8a26db9 --- /dev/null +++ b/modules/tectonic/resources/manifests/ingress/namespace.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Namespace +metadata: + # This is the namespace used to hold the tectonic ingress controllers + name: tectonic-ingress + # Give the namespace a label, so we can select for it in networkpolicy + labels: + kubernetes.io/ingress.class: tectonic + name: tectonic-ingress diff --git a/modules/tectonic/resources/manifests/ingress/pull.json b/modules/tectonic/resources/manifests/ingress/pull.json new file mode 100644 index 0000000000..e1909447fb --- /dev/null +++ b/modules/tectonic/resources/manifests/ingress/pull.json @@ -0,0 +1,12 @@ +{ + "apiVersion": "v1", + "kind": "Secret", + "type": "kubernetes.io/dockerconfigjson", + "metadata": { + "namespace": "tectonic-ingress", + "name": "coreos-pull-secret" + }, + "data": { + ".dockerconfigjson": "${pull_secret}" + } +} diff --git a/modules/tectonic/resources/manifests/ingress/svc-account.yaml b/modules/tectonic/resources/manifests/ingress/svc-account.yaml new file mode 100644 index 0000000000..6403ef7236 --- /dev/null +++ b/modules/tectonic/resources/manifests/ingress/svc-account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tectonic-ingress-controller-operator + namespace: tectonic-ingress diff --git a/modules/tectonic/resources/manifests/rbac/binding-admin.yaml b/modules/tectonic/resources/manifests/rbac/binding-admin.yaml index 821ebe7454..6056ed9a02 100644 --- a/modules/tectonic/resources/manifests/rbac/binding-admin.yaml +++ b/modules/tectonic/resources/manifests/rbac/binding-admin.yaml @@ -8,6 +8,9 @@ subjects: - kind: ServiceAccount namespace: tectonic-system name: default + - kind: ServiceAccount + namespace: tectonic-ingress + name: tectonic-ingress-controller-operator roleRef: kind: ClusterRole name: cluster-admin diff --git a/modules/tectonic/resources/manifests/updater/app_versions/app-version-tectonic-ingress.yaml b/modules/tectonic/resources/manifests/updater/app_versions/app-version-tectonic-ingress.yaml new file mode 100644 index 0000000000..bb97f74a80 --- /dev/null +++ b/modules/tectonic/resources/manifests/updater/app_versions/app-version-tectonic-ingress.yaml @@ -0,0 +1,14 @@ +apiVersion: tco.coreos.com/v1 +kind: AppVersion +metadata: + name: tectonic-ingress + namespace: tectonic-system + labels: + managed-by-channel-operator: "true" +spec: + desiredVersion: + paused: false +status: + paused: false +upgradereq: 1 +upgradecomp: 0 diff --git a/modules/tectonic/resources/manifests/updater/operators/tectonic-ingress-controller-operator.yaml b/modules/tectonic/resources/manifests/updater/operators/tectonic-ingress-controller-operator.yaml new file mode 100644 index 0000000000..533aa2b4c3 --- /dev/null +++ b/modules/tectonic/resources/manifests/updater/operators/tectonic-ingress-controller-operator.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: tectonic-ingress-controller-operator + namespace: tectonic-ingress + labels: + k8s-app: tectonic-ingress-controller-operator + managed-by-channel-operator: "true" +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: tectonic-ingress-controller-operator + template: + metadata: + labels: + k8s-app: tectonic-ingress-controller-operator + tectonic-app-version-name: tectonic-ingress + spec: + containers: + - name: tectonic-ingress-controller-operator + image: ${tectonic_ingress_controller_operator_image} + resources: + limits: + cpu: 20m + memory: 50Mi + requests: + cpu: 20m + memory: 50Mi + volumeMounts: + - name: cluster-config + mountPath: /etc/cluster-config + imagePullSecrets: + - name: coreos-pull-secret + nodeSelector: + node-role.kubernetes.io/master: "" + restartPolicy: Always + securityContext: + runAsNonRoot: true + runAsUser: 65534 + serviceAccount: tectonic-ingress-controller-operator + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + volumes: + - name: cluster-config + configMap: + name: cluster-config-v1 + items: + - key: ingress-config + path: ingress-config diff --git a/modules/tectonic/resources/tectonic.sh b/modules/tectonic/resources/tectonic.sh index 3e9d911175..4e85b4aee6 100755 --- a/modules/tectonic/resources/tectonic.sh +++ b/modules/tectonic/resources/tectonic.sh @@ -127,9 +127,13 @@ set -e # wait for Kubernetes pods wait_for_pods kube-system +echo "creating namespaces" +kubectl create -f ingress/namespace.yaml + echo "Creating Initial Roles" kubectl delete -f rbac/role-admin.yaml +kubectl create -f ingress/svc-account.yaml kubectl create -f rbac/role-admin.yaml kubectl create -f rbac/role-user.yaml kubectl create -f rbac/binding-admin.yaml @@ -137,6 +141,7 @@ kubectl create -f rbac/binding-discovery.yaml echo "Creating Cluster Config For Tectonic" kubectl create -f cluster-config.yaml +kubectl create -f ingress/cluster-config.yaml echo "Creating Tectonic Secrets" kubectl create -f secrets/pull.json @@ -145,6 +150,7 @@ kubectl create -f secrets/ingress-tls.yaml kubectl create -f secrets/ca-cert.yaml kubectl create -f secrets/identity-grpc-client.yaml kubectl create -f secrets/identity-grpc-server.yaml +kubectl create -f ingress/pull.json echo "Creating Operators" kubectl create -f updater/tectonic-channel-operator-kind.yaml @@ -162,6 +168,7 @@ kubectl create -f updater/operators/tectonic-cluo-operator.yaml kubectl create -f updater/operators/kubernetes-addon-operator.yaml kubectl create -f updater/operators/tectonic-alm-operator.yaml kubectl create -f updater/operators/tectonic-utility-operator.yaml +kubectl create -f updater/operators/tectonic-ingress-controller-operator.yaml wait_for_crd tectonic-system appversions.tco.coreos.com kubectl create -f updater/app_versions/app-version-tectonic-cluster.yaml @@ -171,6 +178,7 @@ kubectl create -f updater/app_versions/app-version-tectonic-cluo.yaml kubectl create -f updater/app_versions/app-version-kubernetes-addon.yaml kubectl create -f updater/app_versions/app-version-tectonic-alm.yaml kubectl create -f updater/app_versions/app-version-tectonic-utility.yaml +kubectl create -f updater/app_versions/app-version-tectonic-ingress.yaml # wait for Tectonic pods wait_for_pods tectonic-system From 00ef75abec1ba65b5244cb1b7f4c1d4be3b92259 Mon Sep 17 00:00:00 2001 From: Casey Callendrello Date: Mon, 19 Feb 2018 18:38:09 +0100 Subject: [PATCH 2/2] modules/tectonic: set correct config kinds --- modules/tectonic/resources/manifests/cluster-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/tectonic/resources/manifests/cluster-config.yaml b/modules/tectonic/resources/manifests/cluster-config.yaml index c0be2aaf4b..81751f910b 100644 --- a/modules/tectonic/resources/manifests/cluster-config.yaml +++ b/modules/tectonic/resources/manifests/cluster-config.yaml @@ -6,14 +6,14 @@ metadata: data: addon-config: | apiVersion: v1 - kind: AddonConfig + kind: KubeAddonOperatorConfig heapsterConfig: dnsConfig: clusterIP: ${kube_dns_service_ip} cloudProvider: ${platform} utility-config: | apiVersion: v1 - kind: UtilityConfig + kind: TectonicUtilityOperatorConfig identityConfig: adminEmail: ${admin_email} adminPasswordHash: ${admin_password_hash}