diff --git a/HACKING.md b/HACKING.md index 63c0294d7c..34761ab770 100644 --- a/HACKING.md +++ b/HACKING.md @@ -57,6 +57,9 @@ To run the operator binary in the cluster from your local machine (as opposed to $ make run-local ``` +Set `ENABLE_CANARY=true` in your environment (or inline with the `run-local` command) to enable the ingress canary. + + Note, to rescale the operator on the cluster after local testing is complete, scale the CVO back up with: ``` diff --git a/assets/canary/daemonset.yaml b/assets/canary/daemonset.yaml new file mode 100644 index 0000000000..d04c632c8f --- /dev/null +++ b/assets/canary/daemonset.yaml @@ -0,0 +1,23 @@ +# Hello Openshift Ingress Canary daemonset +# Specific values are set at runtime +kind: DaemonSet +apiVersion: apps/v1 +# name and namespace are set at runtime. +spec: + progressDeadlineSeconds: 600 + template: + spec: + containers: + - name: hello-openshift-canary + # Image is set at runtime + imagePullPolicy: IfNotPresent + terminationMessagePolicy: FallbackToLogsOnError + ports: + - containerPort: 8080 + protocol: TCP + - containerPort: 8888 + protocol: TCP + resources: + requests: + cpu: 10m + memory: 20Mi diff --git a/assets/canary/namespace.yaml b/assets/canary/namespace.yaml new file mode 100644 index 0000000000..b3bb8fbf99 --- /dev/null +++ b/assets/canary/namespace.yaml @@ -0,0 +1,4 @@ +kind: Namespace +apiVersion: v1 +metadata: + name: openshift-ingress-canary diff --git a/assets/canary/route.yaml b/assets/canary/route.yaml new file mode 100644 index 0000000000..f6dd042de2 --- /dev/null +++ b/assets/canary/route.yaml @@ -0,0 +1,16 @@ +# Hello Openshift Ingress Canary route +# Specific values are applied at runtime +kind: Route +apiVersion: route.openshift.io/v1 +# name and namespace are set at runtime. +metadata: + annotations: + haproxy.router.openshift.io/balance: "roundrobin" +spec: + port: + targetPort: 8080-tcp + to: + kind: Service + # Service name set at run time + weight: 100 + wildcardPolicy: None diff --git a/assets/canary/service.yaml b/assets/canary/service.yaml new file mode 100644 index 0000000000..7d16afee0d --- /dev/null +++ b/assets/canary/service.yaml @@ -0,0 +1,16 @@ +# Hello Openshift Ingress Canary service +# Specific values are applied at runtime +kind: Service +apiVersion: v1 +# name and namespace are set at runtime. +spec: + type: ClusterIP + ports: + - name: 8080-tcp + port: 8080 + protocol: TCP + targetPort: 8080 + - name: 8888-tcp + port: 8888 + protocol: TCP + targetPort: 8888 diff --git a/cmd/ingress-operator/start.go b/cmd/ingress-operator/start.go index 44d3e89f01..b3c70607c1 100644 --- a/cmd/ingress-operator/start.go +++ b/cmd/ingress-operator/start.go @@ -37,6 +37,9 @@ type StartOptions struct { // IngressControllerImage is the pullspec of the ingress controller image to // be managed. IngressControllerImage string + // CanaryImage is the pullspec of the canary tester server image to + // be managed. + CanaryImage string // ReleaseVersion is the cluster version which the operator will converge to. ReleaseVersion string } @@ -58,6 +61,7 @@ func NewStartCommand() *cobra.Command { cmd.Flags().StringVarP(&options.OperatorNamespace, "namespace", "n", manifests.DefaultOperatorNamespace, "namespace the operator is deployed to (required)") cmd.Flags().StringVarP(&options.IngressControllerImage, "image", "i", "", "image of the ingress controller the operator will manage (required)") + cmd.Flags().StringVarP(&options.CanaryImage, "canary-image", "c", "", "image of the canary container that the operator will manage (optional)") cmd.Flags().StringVarP(&options.ReleaseVersion, "release-version", "", statuscontroller.UnknownVersionValue, "the release version the operator should converge to (required)") cmd.Flags().StringVarP(&options.MetricsListenAddr, "metrics-listen-addr", "", ":60000", "metrics endpoint listen address (required)") cmd.Flags().StringVarP(&options.ShutdownFile, "shutdown-file", "s", defaultTrustedCABundle, "if provided, shut down the operator when this file changes") @@ -90,6 +94,7 @@ func start(opts *StartOptions) error { OperatorReleaseVersion: opts.ReleaseVersion, Namespace: opts.OperatorNamespace, IngressControllerImage: opts.IngressControllerImage, + CanaryImage: opts.CanaryImage, } // Set up the channels for the watcher and operator. diff --git a/hack/run-local.sh b/hack/run-local.sh index 83c022d6cc..c3e883bad8 100755 --- a/hack/run-local.sh +++ b/hack/run-local.sh @@ -14,5 +14,10 @@ echo "Image: ${IMAGE}" echo "Release version: ${RELEASE_VERSION}" echo "Namespace: ${NAMESPACE}" -${DELVE:-} ./ingress-operator start --image "${IMAGE}" --release-version "${RELEASE_VERSION}" \ +if [[ ! -z ${ENABLE_CANARY:-} ]]; then + CANARY_IMAGE=$(oc get -n openshift-ingress-operator deployments/ingress-operator -o json | jq -r '.spec.template.spec.containers[0].env[] | select(.name=="CANARY_IMAGE").value') + echo "Canary Image: ${CANARY_IMAGE}" +fi + +${DELVE:-} ./ingress-operator start --image "${IMAGE}" --canary-image=${CANARY_IMAGE:-} --release-version "${RELEASE_VERSION}" \ --namespace "${NAMESPACE}" --shutdown-file "${SHUTDOWN_FILE}" "$@" diff --git a/hack/uninstall.sh b/hack/uninstall.sh index 51470be58a..a0d9958f80 100755 --- a/hack/uninstall.sh +++ b/hack/uninstall.sh @@ -22,6 +22,7 @@ else fi oc delete namespaces/openshift-ingress +oc delete namespaces/openshift-ingress-canary if [ "$WHAT" == "all" ]; then oc delete clusterroles/openshift-ingress-operator diff --git a/manifests/00-cluster-role.yaml b/manifests/00-cluster-role.yaml index dbda953cc1..c236cc3cc1 100644 --- a/manifests/00-cluster-role.yaml +++ b/manifests/00-cluster-role.yaml @@ -26,6 +26,7 @@ rules: - apps resources: - deployments + - daemonsets verbs: - "*" @@ -136,13 +137,13 @@ rules: - create # Mirrored from assets/router/cluster-role.yaml +# and expanded for the canary-controller. - apiGroups: - route.openshift.io resources: - routes verbs: - - list - - watch + - '*' # Mirrored from assets/router/cluster-role.yaml - apiGroups: diff --git a/manifests/02-deployment.yaml b/manifests/02-deployment.yaml index e035e7f674..b1f5285a15 100644 --- a/manifests/02-deployment.yaml +++ b/manifests/02-deployment.yaml @@ -47,6 +47,8 @@ spec: - "$(WATCH_NAMESPACE)" - --image - "$(IMAGE)" + - --canary-image + - "$(CANARY_IMAGE)" - --release-version - "$(RELEASE_VERSION)" env: @@ -58,6 +60,8 @@ spec: fieldPath: metadata.namespace - name: IMAGE value: openshift/origin-haproxy-router:v4.0 + - name: CANARY_IMAGE + value: openshift/hello-openshift:latest resources: requests: cpu: 10m diff --git a/pkg/manifests/bindata.go b/pkg/manifests/bindata.go index ab782d32d6..124f80a53e 100644 --- a/pkg/manifests/bindata.go +++ b/pkg/manifests/bindata.go @@ -1,5 +1,9 @@ // Code generated by go-bindata. DO NOT EDIT. // sources: +// assets/canary/daemonset.yaml (633B) +// assets/canary/namespace.yaml (74B) +// assets/canary/route.yaml (383B) +// assets/canary/service.yaml (331B) // assets/router/cluster-role-binding.yaml (329B) // assets/router/cluster-role.yaml (883B) // assets/router/deployment.yaml (2.206kB) @@ -11,7 +15,7 @@ // assets/router/service-account.yaml (213B) // assets/router/service-cloud.yaml (631B) // assets/router/service-internal.yaml (429B) -// manifests/00-cluster-role.yaml (2.682kB) +// manifests/00-cluster-role.yaml (2.728kB) // manifests/00-custom-resource-definition-internal.yaml (6.458kB) // manifests/00-custom-resource-definition.yaml (62.643kB) // manifests/00-ingress-credentials-request.yaml (1.725kB) @@ -26,7 +30,7 @@ // manifests/01-service-account.yaml (283B) // manifests/01-service.yaml (416B) // manifests/01-trusted-ca-configmap.yaml (395B) -// manifests/02-deployment.yaml (3.067kB) +// manifests/02-deployment.yaml (3.211kB) // manifests/03-cluster-operator.yaml (444B) // manifests/image-references (540B) @@ -97,6 +101,86 @@ func (fi bindataFileInfo) Sys() interface{} { return nil } +var _assetsCanaryDaemonsetYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\x90\x41\x6b\x1b\x31\x10\x85\xef\xfb\x2b\x1e\xf8\xec\x76\xd3\x43\x59\x74\x4d\x5a\x6a\x68\x9a\x05\x87\xde\xa7\xda\xb1\x23\x22\x69\xd4\x99\xd9\x80\xff\x7d\xd9\xa5\x4e\x5d\x53\x4a\x74\x12\x4f\xef\xd3\xbc\x37\x1b\x7c\xe1\x9c\x05\x0f\x8d\xab\x3d\xa5\x83\x63\x57\x8f\xca\x66\xb8\xa5\x4a\x7a\xc2\x44\x5c\xa4\x1a\x7b\xb7\xc1\xbe\x71\x4c\x87\x14\xf1\x42\x79\x66\x03\x29\xc3\xd8\x41\x0e\x9d\xab\xa7\xc2\xdd\x73\xaa\x53\xc0\xdd\x0a\xed\xd9\x3b\x6a\xe9\x3b\xab\x25\xa9\x01\xd4\x9a\xbd\x7f\xb9\xe9\x36\xa8\x54\x18\x54\xa7\xf5\x62\x8d\x22\xff\xe3\xaf\x77\x9d\x35\x8e\xa1\x03\x9a\xca\x9a\xe9\x8e\x69\xca\xa9\xf2\x9e\xa3\xd4\xc9\x02\x3e\xf6\x7d\x07\x38\x97\x96\xc9\x79\xb1\x02\x67\x68\x39\x51\xaa\x53\xaa\xac\x76\x56\x80\xed\x3a\x34\xe0\x69\xe9\xbd\x95\x73\xef\x6d\x5c\xfb\xbe\xda\x80\x0d\x76\x85\x8e\x8c\x64\xd7\x25\xff\x78\xd2\xe2\x18\xe7\x9c\x47\xc9\x29\x9e\x02\x76\x87\x6f\xe2\xa3\xb2\x71\xf5\x0b\x9f\xb3\x96\x54\xc9\x93\xd4\x7b\x36\x5b\xa0\xdf\xc0\x67\xca\xf9\x07\xc5\xe7\x47\xf9\x2a\x47\x7b\xa8\x9f\x54\x45\x2f\xc8\x26\xea\x17\xe9\x97\xfc\xaf\xad\x46\x51\x0f\x18\xfa\xa1\xbf\x78\x5f\xd7\xe5\x12\x25\x07\x3c\xde\x8e\xff\x25\x87\x61\x78\x13\xa9\x6c\x32\x6b\xe4\xbf\x82\x2c\xf2\xcf\x99\xcd\xaf\x54\x20\xb6\x39\xe0\xa6\x2f\x57\x72\xe1\x22\x7a\x0a\xf8\xd0\xdf\xa7\xee\x57\x00\x00\x00\xff\xff\xf4\x58\x1d\x74\x79\x02\x00\x00") + +func assetsCanaryDaemonsetYamlBytes() ([]byte, error) { + return bindataRead( + _assetsCanaryDaemonsetYaml, + "assets/canary/daemonset.yaml", + ) +} + +func assetsCanaryDaemonsetYaml() (*asset, error) { + bytes, err := assetsCanaryDaemonsetYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "assets/canary/daemonset.yaml", size: 633, mode: os.FileMode(420), modTime: time.Unix(1, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xa6, 0x9a, 0xa8, 0xca, 0x18, 0xb4, 0x43, 0x9a, 0x7b, 0x28, 0xbe, 0xcf, 0x7f, 0x46, 0xb3, 0xd5, 0xbe, 0x2b, 0xa4, 0x5b, 0x99, 0xe, 0x1e, 0xa2, 0x31, 0x72, 0xe9, 0xb1, 0x97, 0x66, 0x32, 0x94}} + return a, nil +} + +var _assetsCanaryNamespaceYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x04\xc0\xb1\x0d\xc4\x20\x0c\x05\xd0\xde\x53\x78\x01\x8a\x6b\x3d\xc4\x95\xd7\x7f\xc1\xbf\xc4\x8a\x30\x08\xa3\x48\xd9\x3e\xef\xf2\x68\xa6\x5f\x74\xe6\x44\xa5\x60\xfa\x8f\x2b\x7d\x84\xe9\xfd\x91\xce\x8d\x86\x0d\x13\xd5\x40\xa7\xe9\x98\x8c\x3c\xfd\xbf\x8b\xc7\xb1\x98\x59\x2a\x02\xeb\x91\x37\x00\x00\xff\xff\xa2\x0f\x7e\x71\x4a\x00\x00\x00") + +func assetsCanaryNamespaceYamlBytes() ([]byte, error) { + return bindataRead( + _assetsCanaryNamespaceYaml, + "assets/canary/namespace.yaml", + ) +} + +func assetsCanaryNamespaceYaml() (*asset, error) { + bytes, err := assetsCanaryNamespaceYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "assets/canary/namespace.yaml", size: 74, mode: os.FileMode(420), modTime: time.Unix(1, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xa, 0xb8, 0xeb, 0xfb, 0xd2, 0x1, 0x66, 0x41, 0x91, 0xc, 0x7c, 0x4, 0x28, 0x23, 0xd9, 0x68, 0x49, 0x55, 0xcb, 0x65, 0x92, 0xaa, 0x8d, 0x46, 0x67, 0xbd, 0xf8, 0x27, 0x93, 0x32, 0x6e, 0x4c}} + return a, nil +} + +var _assetsCanaryRouteYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x54\x90\x31\x6f\xe3\x30\x0c\x85\x77\xfd\x8a\x87\x78\x3e\x9f\xb3\x05\x5e\x6f\xb9\x5b\xae\x41\x03\x74\x67\x64\x26\x21\xea\x90\x02\x45\x27\xcd\xbf\x2f\x6c\x23\x68\xbb\xe9\x49\x7a\xdf\xe3\x63\x83\xbf\x3c\x8e\x86\x97\xc2\x5a\x2f\x72\x0a\xfc\xd3\xb3\x73\xad\xf8\x43\x4a\xfe\x80\xdb\x14\x9c\x1a\x1c\x0a\x67\x39\x49\xc6\x8d\xc6\x89\x2b\xc8\x19\x54\xca\x28\x3c\x80\x02\x3e\x69\xc8\x95\xd3\xbb\xe8\xd0\xe3\x75\x31\x51\x91\x37\xf6\x2a\xa6\xfd\x8a\x69\xed\x99\xd2\x8a\xfd\xbe\x6d\x53\x03\xa5\x2b\x83\x74\x58\x0e\xb5\x50\xe6\x85\x5c\x39\xbe\x51\xdb\x74\xe5\xa0\x81\x82\xfa\x04\x90\xaa\x05\x85\x98\xd6\x59\x02\x17\x2a\x6e\x1f\x8f\x76\xc9\xf0\x9f\x21\x47\x1a\x49\x33\xf7\xd8\xb8\x4d\x3a\xb8\x1d\x45\x37\xa9\x16\xce\xb3\xb7\x98\xc7\xca\x08\xf2\x33\xc7\x7e\xd6\xd8\x75\xbb\xee\x57\xe4\x92\x80\xb0\xf5\x79\xad\x75\x60\xbf\x49\xe6\xe5\xa6\x79\xaa\xb5\xc2\xd7\xc0\x58\xf6\x30\x7f\xb9\xb3\x9c\x2f\xd1\x63\xdb\x75\x09\xb8\xcb\x38\x64\xf2\x61\x6f\xa3\xe4\x47\x8f\xff\xa6\x9c\x3e\x03\x00\x00\xff\xff\xd8\xf4\x34\x5c\x7f\x01\x00\x00") + +func assetsCanaryRouteYamlBytes() ([]byte, error) { + return bindataRead( + _assetsCanaryRouteYaml, + "assets/canary/route.yaml", + ) +} + +func assetsCanaryRouteYaml() (*asset, error) { + bytes, err := assetsCanaryRouteYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "assets/canary/route.yaml", size: 383, mode: os.FileMode(420), modTime: time.Unix(1, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xae, 0x63, 0x3c, 0x6e, 0x43, 0x81, 0x26, 0xa8, 0x73, 0xd7, 0x70, 0x41, 0x40, 0x4e, 0x20, 0x41, 0xef, 0x8c, 0x86, 0xef, 0xa7, 0x32, 0xa1, 0x1a, 0x9e, 0xe, 0xdf, 0xe6, 0x39, 0x8d, 0x46, 0x8b}} + return a, nil +} + +var _assetsCanaryServiceYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\xcc\xb1\x6a\x33\x31\x10\x04\xe0\x5e\x4f\x31\x70\xb5\x7f\xfc\x77\x42\xed\x35\x71\x95\x03\x87\xf4\x8b\x6e\xed\x88\xc8\xd2\xb2\xbb\x77\xe0\xb7\x0f\xbe\x0b\xc1\xa4\x4a\x27\xcd\xcc\x7e\x03\x5e\xb8\xd6\x8e\x57\xe1\x66\x1f\xe5\xe2\x38\xb5\xab\xb2\x19\x46\x6a\xa4\x77\x18\xeb\x5a\x32\x87\x01\x67\xe1\x5c\x2e\x25\x63\xa5\xba\xb0\x81\x94\x41\x22\xb5\xf0\x0c\x72\xe8\xd2\xbc\xdc\x38\x7c\x96\x36\x27\x9c\xbf\xcf\x48\xca\x3b\xab\x95\xde\x12\xd6\xff\x61\x40\xa3\x1b\x83\xda\xbc\x3d\x4c\x28\xf3\x06\x19\xfb\x13\xf2\x2f\x98\x70\x4e\x01\xf0\xbb\x70\xc2\x58\x17\x73\xd6\xd3\x14\x00\xe9\xea\xf6\xa8\x0e\x1b\x91\x10\x8f\xf1\x78\xf0\x2c\x01\xd8\xdb\x3d\xda\xbf\xda\xbd\xe7\x5e\x13\xde\xc6\x69\x4b\x9c\xf4\xca\x3e\x3d\xcf\x7e\xa0\x18\xe3\x6f\x28\xc6\xf8\x17\xe8\x31\xfb\x0a\x00\x00\xff\xff\x41\xbe\x5b\x7c\x4b\x01\x00\x00") + +func assetsCanaryServiceYamlBytes() ([]byte, error) { + return bindataRead( + _assetsCanaryServiceYaml, + "assets/canary/service.yaml", + ) +} + +func assetsCanaryServiceYaml() (*asset, error) { + bytes, err := assetsCanaryServiceYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "assets/canary/service.yaml", size: 331, mode: os.FileMode(420), modTime: time.Unix(1, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x45, 0xde, 0xe2, 0x1e, 0x7a, 0x6d, 0xaf, 0xcf, 0xb3, 0xf8, 0x47, 0x40, 0xf, 0xe4, 0x19, 0xcd, 0x25, 0x2b, 0x90, 0x14, 0x3, 0x97, 0xc7, 0x48, 0x2d, 0xb, 0xbb, 0x7e, 0x99, 0xde, 0x85, 0xd2}} + return a, nil +} + var _assetsRouterClusterRoleBindingYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\x8f\x31\x4e\xc4\x40\x0c\x45\xfb\x39\x85\x25\xea\x0c\xa2\x43\xd3\x01\x37\x58\x24\x7a\xef\xc4\xbb\x31\x49\xec\xc8\xf6\xa4\xe0\xf4\x28\x4a\x44\xc3\x4a\x29\x2d\xf9\xbf\xff\xfe\x13\xbc\xb3\xf4\x0e\x31\x10\x98\xb6\x20\x03\xd3\x89\x20\x14\x38\x1c\x3e\xc9\x56\xae\x04\x6f\xb5\x6a\x93\xc8\x69\x64\xe9\x0b\x7c\x4c\xcd\x83\xec\xa2\x13\x6d\x71\x96\x7b\xc2\x85\xbf\xc8\x9c\x55\x0a\xd8\x15\x6b\xc6\x16\x83\x1a\xff\x60\xb0\x4a\x1e\x5f\x3d\xb3\x3e\xaf\x2f\x69\xa6\xc0\x1e\x03\x4b\x02\x10\x9c\xa9\x80\x2e\x24\x3e\xf0\x2d\x3a\x96\xbb\x91\x7b\xb7\x9b\x24\x6f\xd7\x6f\xaa\xe1\x25\x75\xb0\x17\x1f\x3e\x87\xce\x1f\xe1\xf8\xdf\x4f\x5f\xb0\x3e\xa2\xa6\x6d\xd8\x85\x6e\x5b\xf1\xbf\x19\xe7\x32\x27\xf0\xdf\x00\x00\x00\xff\xff\x83\x13\xa9\xa6\x49\x01\x00\x00") func assetsRouterClusterRoleBindingYamlBytes() ([]byte, error) { @@ -317,7 +401,7 @@ func assetsRouterServiceInternalYaml() (*asset, error) { return a, nil } -var _manifests00ClusterRoleYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xbc\x96\x4d\x8f\xf4\x34\x0c\xc7\xef\xfd\x14\xd1\xec\x0d\x69\x3a\xe2\x86\xe6\x86\x40\xe2\x04\x8f\x84\x10\x77\x37\xf1\xb4\x66\xd3\x38\xb2\x9d\x59\xca\xa7\x47\x7d\xdb\x7d\x76\xde\x59\x16\x4e\xd3\xb8\xce\xdf\xbf\x89\xff\x49\xf3\xe4\x7e\x88\x45\x0d\xc5\x09\x47\x74\x07\x16\x67\x1d\x3a\xce\x28\x60\x2c\x8e\x4c\x31\x1e\xea\xea\xc9\xfd\xf6\xe5\xc7\x2f\x7b\xf7\xbd\x8b\x6c\x8e\x0f\x63\x96\xa2\xd3\x8e\x4b\x0c\xae\x41\x27\x98\x23\x78\x0c\xae\x19\x26\x29\x75\x94\x26\xa9\x04\x3d\x6a\x06\x8f\x3a\xa9\xbf\x74\xe4\xbb\xea\xe9\x7d\x15\xf0\x56\x20\xc6\xc1\x25\xc4\xa0\x0e\xbc\x47\xd5\xba\x7a\xa6\x14\xf6\x2b\xe0\xaf\x1c\xb1\x82\x4c\xbf\xa3\x28\x71\xda\x3b\x69\xc0\xd7\x50\xac\x63\xa1\xbf\xc0\x88\x53\xfd\xfc\x9d\xd6\xc4\xbb\xe3\xb7\x55\x8f\x06\x01\x0c\xf6\x95\x9b\x08\xf6\x63\xb1\xa4\x1d\x1d\x6c\x4b\xa9\x15\x54\xdd\xae\xe5\x2b\xe7\x20\x25\xb6\x49\x43\xc7\x19\xce\x51\xf2\xb1\x04\xac\x05\x23\x82\x62\xfd\x3a\x7b\xd4\x1f\x97\x64\xdb\x43\x82\x16\xc3\xb6\xa3\xb6\xdb\xc2\x11\x28\x42\x43\x91\x6c\xd8\xbb\x8d\x49\xc1\x4d\x25\x25\xa2\xee\xab\xad\x83\x4c\x3f\x09\x97\x3c\x69\x6f\xdd\x66\x53\x39\x27\xa8\x5c\xc4\xe3\x12\xf3\x9c\x0e\xd4\xf6\x90\x75\x1a\xbe\x2d\xda\x34\x54\x94\x23\x79\x04\xef\xb9\x24\x9b\x63\x98\x42\x66\x5a\x47\x4b\xc6\x3a\xf0\x82\xcb\x8b\xcc\x61\xc9\x3f\xe2\x9c\x7c\x44\x69\x56\x92\x6f\x36\xd5\x39\x1f\xe4\x89\xe2\x84\x30\x60\x8e\x3c\xf4\x8f\x8a\x64\x8e\xe4\x87\x73\x99\xcc\x21\x90\x4a\xc9\xe3\x62\x37\x25\xb4\xf8\x98\x5e\xcf\x89\x8c\x85\x52\x5b\x7b\x16\x64\xad\x3d\xf7\xe7\xf2\xcb\x3a\x2c\xd9\x27\xca\x5e\x10\x0c\xa7\xc7\x16\x6d\xfa\x2d\x39\x8c\xa1\xf3\x7a\x57\xdd\x75\xa1\x77\xb3\x41\x27\xd7\x9f\x06\x1a\x4a\x81\x52\x3b\xc7\xdf\x32\x4e\x5e\xdd\x66\x8c\xa4\xf3\xc3\x0b\x98\xef\x6e\x63\xaf\x9e\x7e\x67\xd8\x73\xe4\x65\x0b\x78\x4e\x26\x1c\x23\x8a\x5e\x09\xef\xd4\xc0\xca\x43\x1d\x5a\x26\xd7\x0f\x22\x84\xa4\x82\x9e\x65\xb1\xe7\xdb\xf0\x1f\x94\x9c\x77\xcd\xdd\xff\x7a\x10\x50\x93\xe2\xad\x08\xbe\xfb\xa3\xf8\x5a\x7b\x79\x82\x4c\xa3\x83\xd6\xf5\x48\x68\x2f\x2c\xcf\x27\x2c\x63\x5f\x3e\xc8\xf2\x56\xe9\x1e\xd5\x57\xf5\x4e\xfa\xff\xc1\xd2\x8b\x29\xd7\xee\xdc\xb6\xdd\x27\xd5\xb8\xd8\xca\xd5\xbb\x4f\xee\x67\x12\x61\xc1\xe0\x0e\xc2\xbd\x03\x55\x34\xdd\x09\x17\x43\xd9\xf5\x68\x42\x5e\x77\x8b\xe4\x76\xdc\x31\xf5\x00\x7d\xbc\xb0\x55\xc7\x19\x77\xc8\x66\x55\x5d\x65\x2f\x34\xf4\x36\xce\x03\x18\xe3\x61\x81\xc9\xc8\xdf\x3e\x2d\x8c\x9f\x31\x09\x1e\x09\x5f\x2e\xf7\xe0\x73\x48\xee\x1f\x5b\x5a\x9a\x3f\xd0\xdb\xfc\xb1\xfd\x6f\x81\x1e\xee\xd0\x1d\xe7\xff\x8f\x20\xff\xc2\xba\x0f\x70\x28\xfa\x22\x64\xc3\x1d\x94\x35\x6d\x3c\x90\xf1\x4f\xf3\x9c\xd4\x04\xe8\xec\x13\x5c\x14\xbf\x9a\xfc\xcb\x78\x79\x98\x5f\x74\xac\xb6\x1c\x62\x9f\x40\x1d\x48\x3d\x1f\x51\x86\xab\x9e\x7a\xbd\x94\xc4\xe5\x32\x72\xbd\x99\x7f\x07\x00\x00\xff\xff\x1c\x48\xd8\x67\x7a\x0a\x00\x00") +var _manifests00ClusterRoleYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x56\x4f\x8f\xe3\xb6\x0f\xbd\xfb\x53\x08\xc9\x61\x81\x05\xec\xe0\x77\xfb\x21\xb7\xa2\x05\x7a\x6a\x17\x28\x8a\xde\x19\x89\x89\xd9\x91\x45\x81\xa4\x32\xeb\x7e\xfa\xc2\xff\x26\x3b\x49\x26\x49\x77\x73\x8a\x44\x91\xef\xbd\x90\x14\xad\xb5\xfb\x39\x16\x35\x14\x27\x1c\xd1\xed\x59\x9c\xb5\xe8\x38\xa3\x80\xb1\x38\x32\xc5\xb8\x6f\xaa\xb5\xfb\xf3\xcb\x2f\x5f\xb6\xee\x27\x17\xd9\x1c\xef\x07\x2f\x45\xa7\x2d\x97\x18\xdc\x0e\x9d\x60\x8e\xe0\x31\xb8\x5d\x3f\x42\xa9\xa3\x34\x42\x25\xe8\x50\x33\x78\xd4\x11\xfd\xb5\x25\xdf\x56\xeb\xf7\x2c\xe0\xad\x40\x8c\xbd\x4b\x88\x41\x1d\x78\x8f\xaa\x4d\xf5\x42\x29\x6c\x17\x81\x7f\x70\xc4\x0a\x32\xfd\x85\xa2\xc4\x69\xeb\x64\x07\xbe\x81\x62\x2d\x0b\xfd\x03\x46\x9c\x9a\x97\xff\x6b\x43\xbc\x39\xfe\xaf\xea\xd0\x20\x80\xc1\xb6\x72\xa3\x82\xed\x40\x96\xb4\xa5\xbd\xd5\x94\x0e\x82\xaa\xf5\x42\x5f\x39\x07\x29\xb1\x8d\x18\x3a\x44\x38\x47\xc9\xc7\x12\xb0\x11\x8c\x08\x8a\xcd\x5b\xf4\x80\x3f\xa4\xa4\xee\x20\xc1\x01\x43\xdd\xd2\xa1\xad\xe1\x08\x14\x61\x47\x91\xac\xdf\xba\x95\x49\xc1\x55\x25\x25\xa2\x6e\xab\xda\x41\xa6\x5f\x85\x4b\x1e\xb1\x6b\xb7\x5a\x55\xce\x09\x2a\x17\xf1\x38\xdb\x3c\xa7\x3d\x1d\x3a\xc8\x3a\x6e\x4f\x49\x1b\xb7\x8a\x72\x24\x8f\xe0\x3d\x97\x64\x93\x0d\x53\xc8\x4c\xcb\x6e\xf6\x58\x36\x5e\x70\x3e\xc8\x1c\x66\xff\x23\x4e\xce\x47\x94\xdd\xa2\xe4\xf3\xaa\xba\xd4\x07\x79\x54\x71\xa6\x30\x60\x8e\xdc\x77\xb8\x30\x06\xc0\x8e\x93\xe2\x63\x98\x99\x23\xf9\xfe\x12\x35\x73\x08\xa4\x52\xf2\x90\xfb\x5d\x09\x87\x07\xf1\x3a\x4e\x64\x2c\x94\x0e\x8d\x67\x41\xd6\xc6\x73\x77\x09\x3f\xa7\x65\xf6\x3e\x43\xf6\x82\x60\x38\x2e\x0f\x68\xe3\x6f\xc9\x61\x30\x5d\xf2\x7d\xd8\x6c\x57\x4a\x39\xf5\xeb\x78\x09\xce\x0d\x3b\x4a\x81\xd2\x61\xb2\x9f\x3c\xce\x8e\x6e\x6b\x8c\xa4\xd3\xe2\x15\xcc\xb7\xb7\x65\x2f\x2d\xfe\xae\x7f\x2f\x25\xcf\x37\xc2\x73\x32\xe1\x18\x51\xf4\x03\xf3\x46\x0d\xac\x3c\x54\xa1\x39\xb8\x79\x50\x42\x48\x2a\xe8\x59\xe6\x6e\x3d\x6d\xff\x03\xe5\x74\x89\xee\xfe\xd7\xbd\x80\x9a\x14\x6f\x45\xf0\xdd\x1f\xc5\x37\xee\x79\x05\x99\x86\x0e\x5a\xf2\x91\xd0\x5e\x59\x5e\xce\xb4\x0c\x75\xf9\x4e\x2d\x27\xa6\x7b\xaa\xbe\xe1\x3b\xab\xff\x77\x52\xcf\x4d\xb9\x54\xe7\x76\xdb\x3d\x89\xe3\x6a\x29\x97\xde\x5d\xbb\xdf\x48\x84\x05\x83\xdb\x0b\x77\x0e\x74\x98\x2d\x1b\xe1\x62\x28\x9b\x0e\x4d\xc8\xeb\x66\x86\xac\x87\x1b\xd3\xf4\xd0\xc5\x2b\x57\x75\x88\xb8\xa3\x6c\x42\xd5\x05\xf6\x4a\x41\x6f\xcb\x79\x40\xc6\x30\x2c\x30\x19\xf9\xdb\xd3\xc2\xf8\x05\x93\xe0\x91\xf0\xf5\x7a\x0d\x9e\xa3\xe4\xfe\xd8\xd2\xb2\xfb\x1b\xbd\x4d\xdf\xde\xa7\x0a\x5a\x3b\x48\xc1\xe1\xd7\x0c\x29\x0c\x31\xf3\x1b\xc3\x43\x02\xe9\xeb\xd3\x74\x69\x7e\xa0\x96\x67\x52\x3f\x7d\xfe\xf4\x84\xc4\x3d\xce\xfe\x03\x9d\xfd\x80\x0e\x45\x5f\x84\xac\xbf\x23\x65\x71\x1b\x32\x8a\x5f\xcd\x73\x52\x13\xa0\x8b\xaf\x7e\x51\xfc\x26\xf8\xf7\xe1\xa9\x31\x1d\xb4\xac\x36\xcf\xb8\x27\xa8\x0e\xa4\x9e\x8f\x28\xfd\x87\x2d\xf7\xf6\x84\x89\xf3\xd3\xe5\xe3\x29\xf7\x6f\x00\x00\x00\xff\xff\x29\x6b\xd2\x4d\xa8\x0a\x00\x00") func manifests00ClusterRoleYamlBytes() ([]byte, error) { return bindataRead( @@ -332,8 +416,8 @@ func manifests00ClusterRoleYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "manifests/00-cluster-role.yaml", size: 2682, mode: os.FileMode(420), modTime: time.Unix(1, 0)} - a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xdd, 0x74, 0x39, 0xd4, 0x72, 0x20, 0xa4, 0x28, 0x36, 0xd3, 0x69, 0x5, 0x65, 0x5c, 0x6a, 0x5e, 0x79, 0x95, 0x1e, 0x9a, 0x57, 0x3b, 0xea, 0x3, 0x41, 0x15, 0x7a, 0x35, 0x85, 0x9, 0xb9, 0x7b}} + info := bindataFileInfo{name: "manifests/00-cluster-role.yaml", size: 2728, mode: os.FileMode(420), modTime: time.Unix(1, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x2f, 0x12, 0xf7, 0x2d, 0xa, 0x6a, 0x65, 0x14, 0x2b, 0x3e, 0xac, 0xd6, 0x54, 0xf6, 0xbc, 0xbc, 0x60, 0x54, 0xc0, 0x12, 0xf4, 0x43, 0x57, 0xf2, 0xd, 0x2b, 0x48, 0xf9, 0x87, 0x2a, 0x9e, 0x47}} return a, nil } @@ -617,7 +701,7 @@ func manifests01TrustedCaConfigmapYaml() (*asset, error) { return a, nil } -var _manifests02DeploymentYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x56\x51\x6f\xdb\x36\x10\x7e\xcf\xaf\x20\x8c\x3d\x6c\xc0\x68\x3b\x69\xd7\xad\x04\xf2\xe0\xb9\x6a\x13\x20\x4e\x83\x38\x68\x1f\x8d\x33\x75\xb6\x39\x53\x24\x7b\x3c\x19\xd1\xbf\x1f\x28\x3b\xb6\x24\x3b\x6e\x1e\x36\x3d\x09\xbc\xef\x3b\x7e\x77\xbc\x3b\x12\x82\xf9\x86\x14\x8d\x77\x4a\x40\x08\x71\xb0\xb9\xbc\x58\x1b\x97\x2b\xf1\x09\x83\xf5\x55\x81\x8e\x2f\x0a\x64\xc8\x81\x41\x5d\x08\xe1\xa0\x40\x25\x8c\x5b\x12\xc6\x28\x7d\x40\x02\xf6\xb4\x33\xc4\x00\x1a\x95\xf0\x01\x5d\x5c\x99\x05\xcb\x13\x38\x70\xce\x33\xb0\xf1\x2e\x26\x7f\x42\x68\xef\x16\x66\xd9\xdf\x93\xfa\xc6\x0f\x8c\xfb\x07\x35\xcb\x40\xfe\xb9\x3a\xb9\x9b\x10\xc6\x69\x5b\xe6\xd8\x27\xb4\x08\x11\xdb\xfc\x88\x76\x21\x0b\x70\xb0\xc4\x5c\xae\xcc\x72\x25\x61\x03\xc6\xc2\xdc\x58\xc3\x95\x12\x3d\xa6\x12\x7b\x17\x31\xa0\x4e\x22\x08\x83\x35\x1a\xa2\x12\x97\x17\x42\x44\x26\x60\x5c\x56\x5b\x79\x5c\x05\x54\xe2\x11\x35\x21\x30\x26\x33\x5a\xd4\xec\x69\x6b\x2e\x80\xf5\xea\x0e\xe6\x68\x77\xe1\x9c\x49\x11\x63\x11\x2c\x30\xee\x98\x8d\xac\xa6\xcf\xb6\x9c\x9c\x71\x23\xc4\x8b\xee\x1a\xe6\x73\x9c\xb6\x24\xa5\x6f\x5d\xce\x91\x1c\x32\xc6\x94\x0e\x1f\x95\xb0\xc6\x95\xcf\x07\xe7\x3e\x47\x49\xde\x62\xbf\x8d\x2c\x20\x32\x92\x12\xbd\xde\x0e\xca\xde\xa6\x8d\x0f\xc7\x25\x84\x14\x6b\x4c\x39\x3c\xef\xa3\xb7\xdf\xeb\x45\xba\x12\xbd\xec\xd9\x44\x8e\x07\x13\x2e\x16\xa8\x59\x89\xde\xbd\x9f\xea\x15\xe6\xa5\xc5\xde\x89\x5d\x3a\x1b\x94\x8e\x10\xf4\x0a\xe6\x07\xf4\x5b\x77\xc9\x9e\x51\x97\xdc\xa0\x1d\xe2\x9b\xa2\xf6\x2e\x4f\x35\x70\x35\xfc\xb9\x06\xe7\x59\x12\x42\x5e\xfd\xbf\x0a\x22\xd2\xc6\x68\x1c\x69\xed\x4b\xc7\xf7\xaf\x97\x84\x10\x81\x8c\x27\xc3\xd5\xd8\x42\x8c\x5b\x64\xac\x22\x63\x21\xb5\x2d\xd3\x89\x48\x4d\x86\x8d\x06\xbb\x23\x68\xef\x18\x8c\x43\x6a\x14\x9d\x3c\x57\x76\x3b\xbd\x48\x85\x71\xb5\xe0\x09\xc6\x08\x4b\x7c\xf0\xd6\xe8\x4a\x89\xcf\x60\xed\x1c\xf4\xfa\xc9\xdf\xf9\x65\xfc\xea\x32\xa2\x16\xd3\x14\x09\x5c\x5a\xfb\x42\xb8\x5d\xdc\x7b\x7e\x20\x8c\x69\xca\x74\x70\x8d\x31\x32\xf0\x64\x96\xc6\xed\xe3\xe8\x8a\x53\xa9\xa9\x62\xd3\x83\xf6\x45\x01\x2e\x57\x8d\x25\x79\x2e\x26\x29\x22\x03\x71\x6b\x45\xca\xfd\x48\x6b\xad\xf7\x7e\xf9\xf5\xfb\xe8\x69\x7c\x33\xbb\x1f\x4d\xb2\xe9\xc3\x68\x9c\xfd\xd6\xeb\x10\xeb\x00\xba\xa4\xdb\xc9\xe8\xcb\x31\x74\x37\xbe\xe4\x66\x3b\x85\xbb\xa4\xc7\xec\x2e\x1b\x4d\xb3\xd9\xb7\xec\x71\x7a\xfb\xf5\xbe\x45\x47\xb7\x69\x06\x78\x38\xbc\x0e\xa9\x85\x11\x62\x03\xb6\x44\x25\x7a\xc3\xfe\xb0\x7f\x29\xa3\x83\x10\x57\x9e\x7b\x27\x3d\x75\x02\x3d\xe5\xe9\x33\xf9\x42\x75\x0c\x42\x2c\x0c\xda\xfc\x11\x17\xc7\x96\x9d\xed\x01\x78\xa5\xf6\x33\xb0\x7f\x2a\xd5\x07\x19\x75\xea\x4e\x87\x71\x54\x24\x2b\xa8\x6f\x0d\x49\xbe\x4c\x93\x6c\xf3\xbe\x3f\x6c\x30\x09\xa3\x2f\x49\x63\x6c\x0b\x23\xfc\x51\x62\xe4\xd8\x95\xab\x43\xa9\xc4\xe5\xb0\x68\x2c\x6f\xbc\x2d\x0b\x9c\xa4\x76\x8c\xed\xfa\xda\x6a\x65\x4a\x45\x9a\x4b\x0d\x2d\x5f\x45\x22\x6c\x83\x1e\x20\xeb\x41\x58\x9b\x81\x06\x59\xa3\x07\xf8\xcc\x04\x9a\x31\x1f\x04\x2c\x3a\xc2\x20\xff\xea\x6c\x55\xfb\xc5\xa3\x36\x4d\x33\x49\xd2\x1c\xf4\xf6\xaa\x3c\xee\xa1\x1f\x25\x54\xf5\x0d\xd0\x4d\x53\x87\x79\xdc\x42\x40\xcb\x4e\x7c\x52\x5a\xbf\x64\x1f\x39\x47\xa2\x8e\x25\xa2\x2e\x09\xa5\x35\x91\xd1\x49\xc8\xf3\xd4\x69\xd7\xea\xe3\xbb\x8f\xef\x3a\x48\xb6\x51\x6a\x13\x56\x48\x32\x96\x86\x31\x5e\x3f\xdd\x4d\x67\xd9\xf8\xd3\x4d\x36\x7b\x9c\x8e\x66\xdf\x6f\x9f\x6e\x66\xa3\x6c\x3a\xbb\xbc\xfa\x6b\xf6\x65\x3c\x99\x4d\x6f\x46\x57\x7f\x7c\xf8\xfd\x80\xca\xc6\x9f\x7e\x82\x3b\xf2\x33\xfe\x7b\xfc\x26\x3f\x27\x71\x67\xbc\x75\x62\x2b\x43\x64\x42\x28\xae\x57\xcc\x41\x0d\x06\x97\x57\x7f\xf6\xeb\x36\x53\x1f\x86\xc3\xe1\x70\x70\x2a\x15\x48\x2c\x17\xc6\xe2\x75\x5d\x16\x6c\xe3\x20\x90\xd9\x00\x63\xfa\xef\xeb\xa3\xa1\x94\x48\x3b\x84\x5c\x63\x75\x86\xbb\xc6\x66\x41\x04\x4f\xdd\x82\xdd\x5f\x00\x0f\x9e\x58\x89\xce\x61\xbd\xbc\x40\x0a\x64\x32\x3a\xfe\xa7\x4d\x24\xea\xf7\x4f\xe1\xa9\x52\xe2\xfd\x70\x62\xde\xd4\x5f\xdd\x16\x6a\xc4\xfb\xba\xec\x94\xaf\x9f\x77\xd4\x76\xd3\xc6\x0b\xe7\x75\x1f\x31\xbd\x04\xb9\xa9\x6c\xbb\x72\xff\x0a\xe3\xcc\x5c\xd8\xbe\x7c\x27\x10\x9a\xde\xce\x4c\x11\xc3\x58\xc4\xee\xc8\xaf\x5f\x28\x1a\xe4\xbc\x74\xb9\xc5\x4e\xc1\xa4\x2f\xd4\x19\xab\x6b\x6d\x8f\x4a\x63\xe6\xdf\x00\x00\x00\xff\xff\xde\x8e\x77\xad\xfb\x0b\x00\x00") +var _manifests02DeploymentYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x56\x41\x6f\x22\x39\x13\xbd\xe7\x57\x58\xe8\x3b\x7c\x2b\xad\x81\x64\x66\x67\x77\x2c\xe5\xc0\x32\x3d\x93\x48\x81\x89\x20\x9a\xd1\x9e\x50\xe1\x2e\xc0\x8b\xdb\xf6\x94\xab\x51\xfa\xdf\xaf\xdc\x10\x68\x1a\x42\xe6\xb0\xdb\x27\x70\xbd\xf7\x5c\x55\xae\x2a\x1b\x82\xf9\x86\x14\x8d\x77\x4a\x40\x08\xb1\xb7\xb9\xbe\x5a\x1b\x97\x2b\xf1\x09\x83\xf5\x55\x81\x8e\xaf\x0a\x64\xc8\x81\x41\x5d\x09\xe1\xa0\x40\x25\x8c\x5b\x12\xc6\x28\x7d\x40\x02\xf6\xb4\x33\xc4\x00\x1a\x95\xf0\x01\x5d\x5c\x99\x05\xcb\x33\x38\x70\xce\x33\xb0\xf1\x2e\x26\x3d\x21\xb4\x77\x0b\xb3\xec\xee\x49\x5d\xe3\x7b\xc6\xfd\x8d\x9a\x65\x20\xff\x5c\x9d\xdd\x4d\x08\xe3\xb4\x2d\x73\xec\x12\x5a\x84\x88\xc7\xfc\x88\x76\x21\x0b\x70\xb0\xc4\x5c\xae\xcc\x72\x25\x61\x03\xc6\xc2\xdc\x58\xc3\x95\x12\x1d\xa6\x12\x3b\x57\x31\xa0\x4e\x4e\x10\x06\x6b\x34\x44\x25\xae\xaf\x84\x88\x4c\xc0\xb8\xac\xb6\xee\x71\x15\x50\x89\x09\x6a\x42\x60\x4c\x66\xb4\xa8\xd9\xd3\xd6\x5c\x00\xeb\xd5\x03\xcc\xd1\xee\xc2\xb9\x90\x22\xc6\x22\x58\x60\xdc\x31\x1b\x59\x4d\x9f\x3d\x12\xb9\x20\x23\xc4\x8b\xdf\x35\xcc\xe7\x38\x3d\x72\x29\x7d\xeb\x72\x8e\xe4\x90\x31\xa6\x74\xf8\xa8\x84\x35\xae\x7c\x3e\x88\xfb\x1c\x25\x79\x8b\xdd\x63\x64\x01\x91\x91\x94\xe8\x74\x76\x50\xf6\x36\x6d\x7c\x38\x2e\x21\xa4\x58\x63\xca\xe1\x65\x8d\xce\x7e\xaf\x17\xd7\x95\xe8\x64\xcf\x26\x72\x3c\x98\x70\xb1\x40\xcd\x4a\x74\xc6\x7e\xaa\x57\x98\x97\x16\x3b\x67\x76\x69\x6d\x50\x3a\x42\xd0\x2b\x98\x1f\xd0\x3f\xbb\x4b\xf6\x8c\xba\xe4\x06\xed\x10\xdf\x14\xb5\x77\x79\xaa\x81\x9b\xfe\xdb\x3e\x38\xcf\x92\x10\xf2\xea\xbf\xf5\x20\x22\x6d\x8c\xc6\x81\xd6\xbe\x74\x3c\x7e\xbd\x24\x84\x08\x64\x3c\x19\xae\x86\x16\x62\xdc\x22\x63\x15\x19\x0b\xa9\x6d\x99\x4e\x44\x6a\x32\x6c\x34\xd8\x1d\x41\x7b\xc7\x60\x1c\x52\xa3\xe8\xe4\xa5\xb2\xdb\xf9\x8b\x54\x18\x57\x3b\x3c\xc2\x18\x61\x89\x8f\xde\x1a\x5d\x29\xf1\x19\xac\x9d\x83\x5e\x3f\xf9\x07\xbf\x8c\x5f\x5d\x46\x74\xc4\x34\x45\x02\x97\xd6\xbe\x10\xee\x17\x63\xcf\x8f\x84\x31\x4d\x99\x16\xae\x31\x46\x7a\x9e\xcc\xd2\xb8\x7d\x1c\x6d\xe7\x54\x6a\xaa\xd8\x54\xd0\xbe\x28\xc0\xe5\xaa\xb1\x24\x2f\xc5\x24\x45\x64\x20\x3e\x5a\x91\x72\x3f\xd2\x8e\xd6\x3b\xff\xfb\xff\xf7\xc1\xd3\xf0\x6e\x36\x1e\x8c\xb2\xe9\xe3\x60\x98\xfd\xd2\x69\x11\xeb\x00\xda\xa4\xfb\xd1\xe0\xcb\x29\x54\x83\x03\xaa\xce\x33\x86\x83\xf1\x60\xf2\xd7\xec\x3c\x71\x37\xf7\xe4\x66\x3b\xbe\xdb\xdc\x49\xf6\x90\x0d\xa6\xd9\xec\x5b\x36\x99\xde\x7f\x1d\x1f\xd1\xd1\x6d\x9a\x99\x39\x9c\x7a\x8b\x74\x84\x11\x62\x03\xb6\x44\x25\x3a\xfd\x6e\xbf\x7b\x2d\xa3\x83\x10\x57\x9e\x3b\x67\x95\x5a\x19\x3a\xa7\xf4\x99\x7c\xa1\x5a\x06\x21\x16\x06\x6d\x3e\xc1\xc5\xa9\x65\x67\x7b\x04\x5e\xa9\xfd\xf0\xec\x9e\x3b\xa3\x83\x1b\x75\xea\xce\x87\x71\x52\x5d\x2b\xa8\xaf\x1b\x49\xbe\x4c\x23\x70\xf3\xbe\xdb\x3f\xab\xd9\x3c\x95\xb7\xa4\x57\x68\xad\x97\xfb\xff\xa7\x85\x4a\x18\x7d\x49\x1a\xe3\x71\xbc\x84\x3f\x4a\x8c\x1c\xdb\x59\xd0\xa1\x54\xe2\xba\x5f\x34\x96\x37\xde\x96\x05\x8e\xd2\x78\x88\xc7\xf5\xbe\x75\x97\x29\x35\x4d\x2e\x35\x1c\x69\x15\x89\xb0\xcd\x65\x0f\x59\xf7\xc2\xda\xf4\x34\xc8\x1a\xdd\xc3\x67\x26\xd0\x8c\x79\x2f\x60\xd1\x72\x0c\xf2\xaf\xce\x56\xb5\x2e\x9e\x8c\x8d\x34\x23\x25\xcd\x41\x6f\xaf\xee\xd3\x9e\xfe\x51\x42\x55\xdf\x48\xed\xec\xb7\x98\xa7\x99\x02\x5a\xb6\xe2\x93\xd2\xfa\x25\xfb\xc8\x39\x12\xb5\x2c\x11\x75\x49\x28\xad\x89\x8c\x4e\x42\x9e\xa7\xce\xbf\x55\x1f\xdf\x7d\x7c\xd7\x42\xb2\x8d\x52\x9b\xb0\x42\x92\xb1\x34\x8c\xf1\xf6\xe9\x61\x3a\xcb\x86\x9f\xee\xb2\xd9\x64\x3a\x98\x7d\xbf\x7f\xba\x9b\x0d\xb2\xe9\xec\xfa\xe6\x8f\xd9\x97\xe1\x68\x36\xbd\x1b\xdc\xfc\xf6\xe1\xd7\x03\x2a\x1b\x7e\x7a\x03\x77\xa2\x33\xfc\x73\xf8\x53\x3a\x67\x71\x17\xd4\x5a\xb1\x95\x21\x32\x21\x14\xb7\x2b\xe6\xa0\x7a\xbd\xeb\x9b\xdf\xbb\x75\xf7\xaa\x0f\xfd\x7e\xbf\xdf\x3b\x97\x0a\x24\x96\x0b\x63\xf1\xb6\x2e\x0b\xb6\xb1\x17\xc8\x6c\x80\x31\xfd\xee\xea\x93\x21\x99\x48\x3b\x84\x5c\x63\x75\x81\xbb\xc6\x66\x41\x04\x4f\xed\x82\xdd\x5f\x48\x8f\x9e\x58\x89\xd6\x61\xbd\xbc\x88\x0a\x64\x32\x3a\xfe\xab\x4d\x24\xea\xf7\x58\xe1\xa9\x52\xe2\x7d\x7f\x64\x7e\xaa\xbf\xda\x2d\xd4\x88\xf7\x75\xb7\x53\xbe\xde\xee\xa8\xed\xa6\x8d\x17\xd7\xeb\x1a\x31\xbd\x4c\xb9\xe9\xd9\x76\x65\xfc\x0a\xe3\xc2\x5c\xd8\xbe\xc4\x47\x10\x9a\x6a\x17\xa6\x88\x61\x2c\x62\xfb\x26\xa9\x5f\x4c\x1a\xe4\xbc\x74\xb9\xc5\x56\xc1\xa4\x2f\xd4\x19\xab\x6b\x6d\x8f\x4a\x63\xe6\x9f\x00\x00\x00\xff\xff\x3b\xd2\x6c\x8e\x8b\x0c\x00\x00") func manifests02DeploymentYamlBytes() ([]byte, error) { return bindataRead( @@ -632,8 +716,8 @@ func manifests02DeploymentYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "manifests/02-deployment.yaml", size: 3067, mode: os.FileMode(420), modTime: time.Unix(1, 0)} - a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x68, 0xd2, 0x49, 0xb7, 0xca, 0xac, 0x91, 0x26, 0x96, 0x12, 0x34, 0xe, 0x2d, 0x81, 0x5f, 0x72, 0x3a, 0x9, 0x3d, 0xe5, 0xbb, 0xac, 0x3d, 0xf1, 0x6, 0x69, 0x32, 0x3c, 0x5, 0xc3, 0x58, 0x84}} + info := bindataFileInfo{name: "manifests/02-deployment.yaml", size: 3211, mode: os.FileMode(420), modTime: time.Unix(1, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xf6, 0xb6, 0xe0, 0x1e, 0x9d, 0x13, 0x89, 0xcf, 0x1a, 0xa1, 0x2d, 0x6c, 0xc4, 0xdd, 0xfd, 0x8c, 0x4, 0xcb, 0xf3, 0xdc, 0xa7, 0x60, 0x6d, 0x2b, 0xa5, 0x3, 0x6c, 0x84, 0x98, 0x5a, 0xba, 0x47}} return a, nil } @@ -768,6 +852,14 @@ func AssetNames() []string { // _bindata is a table, holding each asset generator, mapped to its name. var _bindata = map[string]func() (*asset, error){ + "assets/canary/daemonset.yaml": assetsCanaryDaemonsetYaml, + + "assets/canary/namespace.yaml": assetsCanaryNamespaceYaml, + + "assets/canary/route.yaml": assetsCanaryRouteYaml, + + "assets/canary/service.yaml": assetsCanaryServiceYaml, + "assets/router/cluster-role-binding.yaml": assetsRouterClusterRoleBindingYaml, "assets/router/cluster-role.yaml": assetsRouterClusterRoleYaml, @@ -869,6 +961,12 @@ type bintree struct { var _bintree = &bintree{nil, map[string]*bintree{ "assets": {nil, map[string]*bintree{ + "canary": {nil, map[string]*bintree{ + "daemonset.yaml": {assetsCanaryDaemonsetYaml, map[string]*bintree{}}, + "namespace.yaml": {assetsCanaryNamespaceYaml, map[string]*bintree{}}, + "route.yaml": {assetsCanaryRouteYaml, map[string]*bintree{}}, + "service.yaml": {assetsCanaryServiceYaml, map[string]*bintree{}}, + }}, "router": {nil, map[string]*bintree{ "cluster-role-binding.yaml": {assetsRouterClusterRoleBindingYaml, map[string]*bintree{}}, "cluster-role.yaml": {assetsRouterClusterRoleYaml, map[string]*bintree{}}, diff --git a/pkg/manifests/manifests.go b/pkg/manifests/manifests.go index c40df41246..ad5764ab49 100644 --- a/pkg/manifests/manifests.go +++ b/pkg/manifests/manifests.go @@ -33,6 +33,11 @@ const ( MetricsRoleAsset = "assets/router/metrics/role.yaml" MetricsRoleBindingAsset = "assets/router/metrics/role-binding.yaml" + CanaryNamespaceAsset = "assets/canary/namespace.yaml" + CanaryDaemonSetAsset = "assets/canary/daemonset.yaml" + CanaryServiceAsset = "assets/canary/service.yaml" + CanaryRouteAsset = "assets/canary/route.yaml" + // Annotation used to inform the certificate generation service to // generate a cluster-signed certificate and populate the secret. ServingCertSecretAnnotation = "service.alpha.openshift.io/serving-cert-secret-name" @@ -42,6 +47,10 @@ const ( // can't be established due to namespace boundaries). OwningIngressControllerLabel = "ingresscontroller.operator.openshift.io/owning-ingresscontroller" + // OwningIngressCanaryCheckLabel should be applied to any objects "owned by" the + // ingress operator's canary end-to-end check controller. + OwningIngressCanaryCheckLabel = "ingress.openshift.io/canary" + // IngressControllerFinalizer is used to block deletion of ingresscontrollers // until the operator has ensured it's safe for deletion to proceed. IngressControllerFinalizer = "ingresscontroller.operator.openshift.io/finalizer-ingresscontroller" @@ -57,6 +66,10 @@ const ( DefaultOperatorNamespace = "openshift-ingress-operator" DefaultOperandNamespace = "openshift-ingress" + // DefaultCanaryNamespace is the default namespace for + // the ingress canary check resources. + DefaultCanaryNamespace = "openshift-ingress-canary" + // DefaultIngressControllerName is the name of the default IngressController // instance. DefaultIngressControllerName = "default" @@ -174,6 +187,38 @@ func MetricsRoleBinding() *rbacv1.RoleBinding { return rb } +func CanaryNamespace() *corev1.Namespace { + ns, err := NewNamespace(MustAssetReader(CanaryNamespaceAsset)) + if err != nil { + panic(err) + } + return ns +} + +func CanaryDaemonSet() *appsv1.DaemonSet { + daemonset, err := NewDaemonSet(MustAssetReader(CanaryDaemonSetAsset)) + if err != nil { + panic(err) + } + return daemonset +} + +func CanaryService() *corev1.Service { + service, err := NewService(MustAssetReader(CanaryServiceAsset)) + if err != nil { + panic(err) + } + return service +} + +func CanaryRoute() *routev1.Route { + route, err := NewRoute(MustAssetReader(CanaryRouteAsset)) + if err != nil { + panic(err) + } + return route +} + func NewServiceAccount(manifest io.Reader) (*corev1.ServiceAccount, error) { sa := corev1.ServiceAccount{} if err := yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&sa); err != nil { @@ -246,6 +291,15 @@ func NewDeployment(manifest io.Reader) (*appsv1.Deployment, error) { return &o, nil } +func NewDaemonSet(manifest io.Reader) (*appsv1.DaemonSet, error) { + o := appsv1.DaemonSet{} + if err := yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&o); err != nil { + return nil, err + } + + return &o, nil +} + func NewRoute(manifest io.Reader) (*routev1.Route, error) { o := routev1.Route{} if err := yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&o); err != nil { diff --git a/pkg/operator/config/config.go b/pkg/operator/config/config.go index 38aa69e37d..e482f8f899 100644 --- a/pkg/operator/config/config.go +++ b/pkg/operator/config/config.go @@ -11,4 +11,7 @@ type Config struct { // IngressControllerImage is the ingress controller image to manage. IngressControllerImage string + + // CanaryImage is the ingress canary image to manage. + CanaryImage string } diff --git a/pkg/operator/controller/canary/controller.go b/pkg/operator/controller/canary/controller.go new file mode 100644 index 0000000000..1ef19f5923 --- /dev/null +++ b/pkg/operator/controller/canary/controller.go @@ -0,0 +1,172 @@ +package canary + +import ( + "fmt" + + logf "github.com/openshift/cluster-ingress-operator/pkg/log" + "github.com/openshift/cluster-ingress-operator/pkg/manifests" + + "github.com/google/go-cmp/cmp" + + operatorv1 "github.com/openshift/api/operator/v1" + routev1 "github.com/openshift/api/route/v1" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +const ( + canaryControllerName = "canary_controller" +) + +var ( + log = logf.Logger.WithName(canaryControllerName) +) + +// New creates the canary controller. +// +// The canary controller will watch the Default IngressController, as well as +// the canary service, daemonset, and route resources. +func New(mgr manager.Manager, config Config) (controller.Controller, error) { + reconciler := &reconciler{ + Config: config, + client: mgr.GetClient(), + } + c, err := controller.New(canaryControllerName, mgr, controller.Options{Reconciler: reconciler}) + if err != nil { + return nil, err + } + + // Only trigger a reconcile request for the canary controller via events for the default ingress controller. + defaultIcPredicate := predicate.NewPredicateFuncs(func(meta metav1.Object, object runtime.Object) bool { + return meta.GetName() == manifests.DefaultIngressControllerName + }) + + if err := c.Watch(&source.Kind{Type: &operatorv1.IngressController{}}, &handler.EnqueueRequestForObject{}, defaultIcPredicate); err != nil { + return nil, err + } + + return c, nil +} + +// Reconcile ensures that the canary controller's resources +// are in the desired state. +func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, error) { + result := reconcile.Result{} + errors := []error{} + + if err := r.ensureCanaryNamespace(); err != nil { + // Return if the canary namespace cannot be created since + // resource creation in a namespace that does not exist will fail. + return result, fmt.Errorf("failed to ensure canary namespace: %v", err) + } + + haveDs, daemonset, err := r.ensureCanaryDaemonSet() + if err != nil { + errors = append(errors, fmt.Errorf("failed to ensure canary daemonset: %v", err)) + } else if !haveDs { + errors = append(errors, fmt.Errorf("failed to get canary daemonset: %v", err)) + } + + trueVar := true + daemonsetRef := metav1.OwnerReference{ + APIVersion: "apps/v1", + Kind: "daemonset", + Name: daemonset.Name, + UID: daemonset.UID, + Controller: &trueVar, + } + + haveService, service, err := r.ensureCanaryService(daemonsetRef) + if err != nil { + errors = append(errors, fmt.Errorf("failed to ensure canary service: %v", err)) + } else if !haveService { + errors = append(errors, fmt.Errorf("failed to get canary service: %v", err)) + } + + if haveRoute, _, err := r.ensureCanaryRoute(service); err != nil { + errors = append(errors, fmt.Errorf("failed to ensure canary route: %v", err)) + } else if !haveRoute { + errors = append(errors, fmt.Errorf("failed to get canary route: %v", err)) + } + + return result, utilerrors.NewAggregate(errors) +} + +// Config holds all the things necessary for the controller to run. +type Config struct { + Namespace string + CanaryImage string +} + +// reconciler handles the actual canary reconciliation logic in response to +// events. +type reconciler struct { + Config + + client client.Client +} + +// TODO: Canary Controller Phase 2 +// Add callers for these 2 functions +// +// Switch the current RoutePort that the route points to. +// Use this function to periodically update the canary route endpoint +// to verify if the router has wedged. +func (r *reconciler) rotateRouteEndpoint(service *corev1.Service, current *routev1.Route) (*routev1.Route, error) { + updated, err := cycleServicePort(service, current) + if err != nil { + return nil, fmt.Errorf("failed to rotate route port: %v", err) + } + + _, err = r.updateCanaryRoute(current, updated) + if err != nil { + return current, err + } + + return updated, nil +} + +// cycleServicePort returns a route resource with Spec.Port set to the +// next available port in service.Spec.Ports that is not the current route.Spec.Port. +func cycleServicePort(service *corev1.Service, route *routev1.Route) (*routev1.Route, error) { + servicePorts := service.Spec.Ports + currentPort := route.Spec.Port + + if currentPort == nil { + return nil, fmt.Errorf("route does not have Spec.Port set") + } + + switch len(servicePorts) { + case 0: + return nil, fmt.Errorf("service has no ports") + case 1: + return nil, fmt.Errorf("service has only one port, no change possible") + } + + updated := route.DeepCopy() + currentIndex := 0 + + // Find the current port index in the service ports slice + for i, port := range servicePorts { + if cmp.Equal(port.TargetPort, currentPort.TargetPort) { + currentIndex = i + } + } + + updated.Spec.Port = &routev1.RoutePort{ + TargetPort: servicePorts[(currentIndex+1)%len(servicePorts)].TargetPort, + } + + return updated, nil +} diff --git a/pkg/operator/controller/canary/controller_test.go b/pkg/operator/controller/canary/controller_test.go new file mode 100644 index 0000000000..ed3ff7ae7b --- /dev/null +++ b/pkg/operator/controller/canary/controller_test.go @@ -0,0 +1,135 @@ +package canary + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + + routev1 "github.com/openshift/api/route/v1" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func TestCycleServicePort(t *testing.T) { + tPort1 := intstr.IntOrString{ + StrVal: "80", + } + tPort2 := intstr.IntOrString{ + StrVal: "8080", + } + tPort3 := intstr.IntOrString{ + StrVal: "8888", + } + testCases := []struct { + description string + route *routev1.Route + service *corev1.Service + success bool + index int + }{ + { + description: "service with no ports", + route: &routev1.Route{ + Spec: routev1.RouteSpec{ + Port: &routev1.RoutePort{}, + }, + }, + service: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{}, + }, + }, + success: false, + }, + { + description: "service has one port", + route: &routev1.Route{ + Spec: routev1.RouteSpec{ + Port: &routev1.RoutePort{}, + }, + }, + service: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + TargetPort: tPort1, + }, + }, + }, + }, + success: false, + }, + { + description: "service has two ports", + route: &routev1.Route{ + Spec: routev1.RouteSpec{ + Port: &routev1.RoutePort{ + TargetPort: tPort1, + }, + }, + }, + service: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + TargetPort: tPort1, + }, + { + TargetPort: tPort2, + }, + }, + }, + }, + success: true, + index: 0, + }, + { + description: "service has three ports", + route: &routev1.Route{ + Spec: routev1.RouteSpec{ + Port: &routev1.RoutePort{ + TargetPort: tPort3, + }, + }, + }, + service: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + TargetPort: tPort1, + }, + { + TargetPort: tPort2, + }, + { + TargetPort: tPort3, + }, + }, + }, + }, + success: true, + index: 2, + }, + } + + for _, tc := range testCases { + route, err := cycleServicePort(tc.service, tc.route) + if tc.success { + if err != nil { + t.Errorf("expected test case %s to not return an err, but got err %v", tc.description, err) + } + + routeTargetPort := route.Spec.Port.TargetPort + cycledIndex := (tc.index + 1) % len(tc.service.Spec.Ports) + expectedPort := tc.service.Spec.Ports[cycledIndex].TargetPort + if !cmp.Equal(expectedPort, routeTargetPort) { + t.Errorf("expected route to have port %s, but has port %s", expectedPort.String(), routeTargetPort.String()) + } + } else { + if err == nil { + t.Errorf("expected test case %s to return an err, but it did not", tc.description) + } + } + } +} diff --git a/pkg/operator/controller/canary/daemonset.go b/pkg/operator/controller/canary/daemonset.go new file mode 100644 index 0000000000..c1389167aa --- /dev/null +++ b/pkg/operator/controller/canary/daemonset.go @@ -0,0 +1,75 @@ +package canary + +import ( + "context" + "fmt" + + "github.com/openshift/cluster-ingress-operator/pkg/manifests" + "github.com/openshift/cluster-ingress-operator/pkg/operator/controller" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/errors" +) + +// ensureCanaryDaemonSet ensures the canary daemonset exists +func (r *reconciler) ensureCanaryDaemonSet() (bool, *appsv1.DaemonSet, error) { + desired := desiredCanaryDaemonSet(r.Config.CanaryImage) + haveDs, current, err := r.currentCanaryDaemonSet() + if err != nil { + return false, nil, err + } + + if haveDs { + return true, current, nil + } + + err = r.createCanaryDaemonSet(desired) + if err != nil { + return false, nil, err + } + + return true, desired, nil +} + +// currentCanaryDaemonSet returns the current canary daemonset +func (r *reconciler) currentCanaryDaemonSet() (bool, *appsv1.DaemonSet, error) { + daemonset := &appsv1.DaemonSet{} + if err := r.client.Get(context.TODO(), controller.CanaryDaemonSetName(), daemonset); err != nil { + if errors.IsNotFound(err) { + return false, nil, nil + } + return false, nil, err + } + return true, daemonset, nil +} + +// createCanaryDaemonSet creates the given daemonset resource +func (r *reconciler) createCanaryDaemonSet(daemonset *appsv1.DaemonSet) error { + if err := r.client.Create(context.TODO(), daemonset); err != nil { + return fmt.Errorf("failed to create canary daemonset %s/%s: %v", daemonset.Namespace, daemonset.Name, err) + } + + log.Info("created canary daemonset", "namespace", daemonset.Namespace, "name", daemonset.Name) + return nil +} + +// desiredCanaryDaemonSet returns the desired canary daemonset read in +// from manifests +func desiredCanaryDaemonSet(canaryImage string) *appsv1.DaemonSet { + daemonset := manifests.CanaryDaemonSet() + name := controller.CanaryDaemonSetName() + daemonset.Name = name.Name + daemonset.Namespace = name.Namespace + + daemonset.Labels = map[string]string{ + // associate the daemonset with the ingress canary controller + manifests.OwningIngressCanaryCheckLabel: canaryControllerName, + } + + daemonset.Spec.Selector = controller.CanaryDaemonSetPodSelector(canaryControllerName) + daemonset.Spec.Template.Labels = controller.CanaryDaemonSetPodSelector(canaryControllerName).MatchLabels + + daemonset.Spec.Template.Spec.Containers[0].Image = canaryImage + + return daemonset +} diff --git a/pkg/operator/controller/canary/daemonset_test.go b/pkg/operator/controller/canary/daemonset_test.go new file mode 100644 index 0000000000..d60ba7df23 --- /dev/null +++ b/pkg/operator/controller/canary/daemonset_test.go @@ -0,0 +1,59 @@ +package canary + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/openshift/cluster-ingress-operator/pkg/manifests" + "github.com/openshift/cluster-ingress-operator/pkg/operator/controller" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestDesiredCanaryDaemonSet(t *testing.T) { + canaryImage := "openshift/hello-openshift:latest" + + daemonset := desiredCanaryDaemonSet(canaryImage) + + expectedDaemonSetName := controller.CanaryDaemonSetName() + + if !cmp.Equal(daemonset.Name, expectedDaemonSetName.Name) { + t.Errorf("expected daemonset name to be %s, but got %s", expectedDaemonSetName.Name, daemonset.Name) + } + + if !cmp.Equal(daemonset.Namespace, expectedDaemonSetName.Namespace) { + t.Errorf("expected daemonset namespace to be %s, but got %s", expectedDaemonSetName.Namespace, daemonset.Namespace) + } + + expectedLabels := map[string]string{ + manifests.OwningIngressCanaryCheckLabel: canaryControllerName, + } + + if !cmp.Equal(daemonset.Labels, expectedLabels) { + t.Errorf("expected daemonset labels to be %q, but got %q", expectedLabels, daemonset.Labels) + } + + labelSelector := &metav1.LabelSelector{ + MatchLabels: map[string]string{ + controller.CanaryDaemonSetLabel: canaryControllerName, + }, + } + + if !cmp.Equal(daemonset.Spec.Selector, labelSelector) { + t.Errorf("expected daemonset selector to be %q, but got %q", labelSelector, daemonset.Spec.Selector) + } + + if !cmp.Equal(daemonset.Spec.Template.Labels, labelSelector.MatchLabels) { + t.Errorf("expected daemonset template labels to be %q, but got %q", labelSelector.MatchLabels, daemonset.Spec.Template.Labels) + } + + containers := daemonset.Spec.Template.Spec.Containers + if len(containers) != 1 { + t.Errorf("expected daemonset to have 1 container, but found %d", len(containers)) + } + + if !cmp.Equal(containers[0].Image, canaryImage) { + t.Errorf("expected daemonset container image to be %q, but got %q", canaryImage, containers[0].Image) + } +} diff --git a/pkg/operator/controller/canary/namespace.go b/pkg/operator/controller/canary/namespace.go new file mode 100644 index 0000000000..1bb53d15bf --- /dev/null +++ b/pkg/operator/controller/canary/namespace.go @@ -0,0 +1,27 @@ +package canary + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + + "github.com/openshift/cluster-ingress-operator/pkg/manifests" +) + +// ensureCanaryNamespace ensures that the ingress-canary namespace exists +func (r *reconciler) ensureCanaryNamespace() error { + ns := manifests.CanaryNamespace() + if err := r.client.Get(context.TODO(), types.NamespacedName{Name: ns.Name}, ns); err != nil { + if !errors.IsNotFound(err) { + return fmt.Errorf("failed to get ingress canary namespace %q: %v", ns.Name, err) + } + if err := r.client.Create(context.TODO(), ns); err != nil { + return fmt.Errorf("failed to create ingress canary namespace %q: %v", ns.Name, err) + } + log.Info("created ingress canary namespace", "name", ns.Name) + } + + return nil +} diff --git a/pkg/operator/controller/canary/route.go b/pkg/operator/controller/canary/route.go new file mode 100644 index 0000000000..450d4fe846 --- /dev/null +++ b/pkg/operator/controller/canary/route.go @@ -0,0 +1,141 @@ +package canary + +import ( + "context" + "fmt" + + "github.com/openshift/cluster-ingress-operator/pkg/manifests" + "github.com/openshift/cluster-ingress-operator/pkg/operator/controller" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + routev1 "github.com/openshift/api/route/v1" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" +) + +// ensureCanaryRoute ensures the canary route exists +func (r *reconciler) ensureCanaryRoute(service *corev1.Service) (bool, *routev1.Route, error) { + desired := desiredCanaryRoute(service) + haveRoute, current, err := r.currentCanaryRoute() + if err != nil { + return false, nil, err + } + + switch { + case !haveRoute: + if err := r.createCanaryRoute(desired); err != nil { + return false, nil, err + } + return r.currentCanaryRoute() + case haveRoute: + if updated, err := r.updateCanaryRoute(current, desired); err != nil { + return true, current, err + } else if updated { + return r.currentCanaryRoute() + } + } + + return true, current, nil +} + +// currentCanaryRoute gets the current canary route resource +func (r *reconciler) currentCanaryRoute() (bool, *routev1.Route, error) { + route := &routev1.Route{} + if err := r.client.Get(context.TODO(), controller.CanaryRouteName(), route); err != nil { + if errors.IsNotFound(err) { + return false, nil, nil + } + return false, nil, err + } + return true, route, nil +} + +// createCanaryRoute creates the given route +func (r *reconciler) createCanaryRoute(route *routev1.Route) error { + if err := r.client.Create(context.TODO(), route); err != nil { + return fmt.Errorf("failed to create canary route %s/%s: %v", route.Namespace, route.Name, err) + } + + log.Info("created canary route", "namespace", route.Namespace, "name", route.Name) + return nil +} + +// updateCanaryRoute updates the canary route if an appropriate change +// has been detected +func (r *reconciler) updateCanaryRoute(current, desired *routev1.Route) (bool, error) { + changed, updated := canaryRouteChanged(current, desired) + if !changed { + return false, nil + } + + if err := r.client.Update(context.TODO(), updated); err != nil { + return false, fmt.Errorf("failed to update canary route %s/%s: %v", updated.Namespace, updated.Name, err) + } + log.Info("updated canary route", "namespace", updated.Namespace, "name", updated.Name) + return true, nil +} + +// deleteCanaryRoute deletes a given route +func (r *reconciler) deleteCanaryRoute(route *routev1.Route) (bool, error) { + + if err := r.client.Delete(context.TODO(), route); err != nil { + return false, fmt.Errorf("failed to delete canary route %s/%s: %v", route.Namespace, route.Name, err) + } + + log.Info("deleted canary route", "namespace", route.Namespace, "name", route.Name) + return true, nil +} + +// canaryRouteChanged returns true if current and expected differ by Spec.Port +// or Spec.To +func canaryRouteChanged(current, expected *routev1.Route) (bool, *routev1.Route) { + changed := false + updated := current.DeepCopy() + + if !cmp.Equal(current.Spec.Port, expected.Spec.Port, cmpopts.EquateEmpty()) { + updated.Spec.Port = expected.Spec.Port + changed = true + } + + if !cmp.Equal(current.Spec.To, expected.Spec.To, cmpopts.EquateEmpty()) { + updated.Spec.To = expected.Spec.To + changed = true + } + + if !changed { + return false, nil + } + return true, updated +} + +// desiredCanaryRoute returns the desired canary route read in +// from manifests +func desiredCanaryRoute(service *corev1.Service) *routev1.Route { + route := manifests.CanaryRoute() + + name := controller.CanaryRouteName() + + route.Namespace = name.Namespace + route.Name = name.Name + + route.Labels = map[string]string{ + // associate the route with the canary controller + manifests.OwningIngressCanaryCheckLabel: canaryControllerName, + } + + route.Spec.To.Name = controller.CanaryServiceName().Name + + // Set spec.port.targetPort to the first port available in the canary service. + // The canary controller will toggle which targetPort the route targets + // to test > 1 endpoint, so it does not matter which port is selected as long + // as the canary service has > 1 ports available. If the canary service only has one + // available port, then route.Spec.Port.TargetPort will remain unchanged. + route.Spec.Port.TargetPort = service.Spec.Ports[0].TargetPort + + route.SetOwnerReferences(service.OwnerReferences) + + return route +} diff --git a/pkg/operator/controller/canary/route_test.go b/pkg/operator/controller/canary/route_test.go new file mode 100644 index 0000000000..b95b5d6148 --- /dev/null +++ b/pkg/operator/controller/canary/route_test.go @@ -0,0 +1,120 @@ +package canary + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + + routev1 "github.com/openshift/api/route/v1" + + "github.com/openshift/cluster-ingress-operator/pkg/manifests" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func TestDesiredCanaryRoute(t *testing.T) { + daemonsetRef := metav1.OwnerReference{ + Name: "test", + } + service := desiredCanaryService(daemonsetRef) + route := desiredCanaryRoute(service) + + expectedRouteName := types.NamespacedName{ + Namespace: "openshift-ingress-canary", + Name: "ingress-canary-route", + } + + if !cmp.Equal(route.Name, expectedRouteName.Name) { + t.Errorf("expected route name to be %s, but got %s", expectedRouteName.Name, route.Name) + } + + if !cmp.Equal(route.Namespace, expectedRouteName.Namespace) { + t.Errorf("expected route namespace to be %s, but got %s", expectedRouteName.Namespace, route.Namespace) + } + + expectedAnnotations := map[string]string{ + "haproxy.router.openshift.io/balance": "roundrobin", + } + + if !cmp.Equal(route.Annotations, expectedAnnotations) { + t.Errorf("expected route annotations to be %s, but got %s", expectedAnnotations, route.Annotations) + } + + expectedLabels := map[string]string{ + manifests.OwningIngressCanaryCheckLabel: canaryControllerName, + } + + if !cmp.Equal(route.Labels, expectedLabels) { + t.Errorf("expected route labels to be %q, but got %q", expectedLabels, route.Labels) + } + + routeToName := route.Spec.To.Name + if !cmp.Equal(routeToName, service.Name) { + t.Errorf("expected route.Spec.To.Name to be %q, but got %q", service.Name, routeToName) + } + + routeTarget := route.Spec.Port.TargetPort + validTarget := false + for _, port := range service.Spec.Ports { + if cmp.Equal(routeTarget, port.TargetPort) { + validTarget = true + } + } + + if !validTarget { + t.Errorf("expected %v to be a port in the %v. Route targetPort not in service targetPort list", route.Spec.Port.TargetPort, service.Spec.Ports) + } + + expectedOwnerRefs := []metav1.OwnerReference{daemonsetRef} + if !cmp.Equal(route.OwnerReferences, expectedOwnerRefs) { + t.Errorf("expected service owner references %#v, but got %#v", expectedOwnerRefs, route.OwnerReferences) + } +} + +func TestCanaryRouteChanged(t *testing.T) { + testCases := []struct { + description string + mutate func(*routev1.Route) + expect bool + }{ + { + description: "if nothing changes", + mutate: func(_ *routev1.Route) {}, + expect: false, + }, + { + description: "if route spec.To changes", + mutate: func(route *routev1.Route) { + route.Spec.To.Name = "test" + }, + expect: true, + }, + { + description: "if route spec.Port changes", + mutate: func(route *routev1.Route) { + route.Spec.Port.TargetPort = intstr.IntOrString{} + }, + expect: true, + }, + } + + daemonsetRef := metav1.OwnerReference{ + Name: "test", + } + service := desiredCanaryService(daemonsetRef) + + for _, tc := range testCases { + original := desiredCanaryRoute(service) + mutated := original.DeepCopy() + tc.mutate(mutated) + if changed, updated := canaryRouteChanged(original, mutated); changed != tc.expect { + t.Errorf("%s, expect canaryRouteChanged to be %t, got %t", tc.description, tc.expect, changed) + } else if changed { + if changedAgain, _ := canaryRouteChanged(mutated, updated); changedAgain { + t.Errorf("%s, canaryRouteChanged does not behave as a fixed point function", tc.description) + } + } + } +} diff --git a/pkg/operator/controller/canary/service.go b/pkg/operator/controller/canary/service.go new file mode 100644 index 0000000000..7fa21ebf30 --- /dev/null +++ b/pkg/operator/controller/canary/service.go @@ -0,0 +1,72 @@ +package canary + +import ( + "context" + "fmt" + + "github.com/openshift/cluster-ingress-operator/pkg/manifests" + "github.com/openshift/cluster-ingress-operator/pkg/operator/controller" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ensureCanaryService ensures the ingress canary service exists +func (r *reconciler) ensureCanaryService(daemonsetRef metav1.OwnerReference) (bool, *corev1.Service, error) { + desired := desiredCanaryService(daemonsetRef) + haveService, current, err := r.currentCanaryService() + if err != nil { + return false, nil, err + } + if haveService { + return true, current, nil + } + if err := r.createCanaryService(desired); err != nil { + return false, nil, err + } + return true, desired, nil +} + +// currentCanaryService gets the current ingress canary service resource +func (r *reconciler) currentCanaryService() (bool, *corev1.Service, error) { + current := &corev1.Service{} + err := r.client.Get(context.TODO(), controller.CanaryServiceName(), current) + if err != nil { + if errors.IsNotFound(err) { + return false, nil, nil + } + return false, nil, err + } + return true, current, nil +} + +// createCanaryService creates the given service resource +func (r *reconciler) createCanaryService(service *corev1.Service) error { + if err := r.client.Create(context.TODO(), service); err != nil { + return fmt.Errorf("failed to create canary service %s/%s: %v", service.Namespace, service.Name, err) + } + + log.Info("created canary service", "namespace", service.Namespace, "name", service.Name) + return nil +} + +// desiredCanaryService returns the desired canary service read in from manifests +func desiredCanaryService(daemonsetRef metav1.OwnerReference) *corev1.Service { + s := manifests.CanaryService() + + name := controller.CanaryServiceName() + s.Namespace = name.Namespace + s.Name = name.Name + + s.Labels = map[string]string{ + // associate the daemonset with the ingress canary controller + manifests.OwningIngressCanaryCheckLabel: canaryControllerName, + } + + s.Spec.Selector = controller.CanaryDaemonSetPodSelector(canaryControllerName).MatchLabels + + s.SetOwnerReferences([]metav1.OwnerReference{daemonsetRef}) + + return s +} diff --git a/pkg/operator/controller/canary/service_test.go b/pkg/operator/controller/canary/service_test.go new file mode 100644 index 0000000000..9dfb6d6aed --- /dev/null +++ b/pkg/operator/controller/canary/service_test.go @@ -0,0 +1,53 @@ +package canary + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/openshift/cluster-ingress-operator/pkg/manifests" + "github.com/openshift/cluster-ingress-operator/pkg/operator/controller" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +func TestDesiredCanaryService(t *testing.T) { + daemonsetRef := metav1.OwnerReference{ + Name: "test", + } + service := desiredCanaryService(daemonsetRef) + + expectedServiceName := types.NamespacedName{ + Namespace: "openshift-ingress-canary", + Name: "ingress-canary", + } + + if !cmp.Equal(service.Name, expectedServiceName.Name) { + t.Errorf("expected service name to be %s, but got %s", expectedServiceName.Name, service.Name) + } + + if !cmp.Equal(service.Namespace, expectedServiceName.Namespace) { + t.Errorf("expected service namespace to be %s, but got %s", expectedServiceName.Namespace, service.Namespace) + } + + expectedLabels := map[string]string{ + manifests.OwningIngressCanaryCheckLabel: canaryControllerName, + } + if !cmp.Equal(service.Labels, expectedLabels) { + t.Errorf("expected service labels to be %q, but got %q", expectedLabels, service.Labels) + } + + expectedSelector := map[string]string{ + controller.CanaryDaemonSetLabel: canaryControllerName, + } + + if !cmp.Equal(service.Spec.Selector, expectedSelector) { + t.Errorf("expected service selector to be %q, but got %q", expectedSelector, service.Spec.Selector) + } + + expectedOwnerRefs := []metav1.OwnerReference{daemonsetRef} + if !cmp.Equal(service.OwnerReferences, expectedOwnerRefs) { + t.Errorf("expected service owner references %#v, but got %#v", expectedOwnerRefs, service.OwnerReferences) + } +} diff --git a/pkg/operator/controller/names.go b/pkg/operator/controller/names.go index 3c22b94efc..ceef2feb73 100644 --- a/pkg/operator/controller/names.go +++ b/pkg/operator/controller/names.go @@ -25,6 +25,10 @@ const ( // controller, and for anti-affinity, to prevent colocation of replicas // of the same generation of the same ingress controller. ControllerDeploymentHashLabel = "ingresscontroller.operator.openshift.io/hash" + + // CanaryDaemonsetLabel identifies a daemonset as an ingress canary daemonset, and + // the value is the name of the owning canary controller. + CanaryDaemonSetLabel = "ingresscanary.operator.openshift.io/daemonset-ingresscanary" ) // IngressClusterOperatorName returns the namespaced name of the ClusterOperator @@ -156,3 +160,32 @@ func WildcardDNSRecordName(ic *operatorv1.IngressController) types.NamespacedNam Name: fmt.Sprintf("%s-wildcard", ic.Name), } } + +func CanaryDaemonSetName() types.NamespacedName { + return types.NamespacedName{ + Namespace: "openshift-ingress-canary", + Name: "ingress-canary", + } +} + +func CanaryDaemonSetPodSelector(canaryControllerName string) *metav1.LabelSelector { + return &metav1.LabelSelector{ + MatchLabels: map[string]string{ + CanaryDaemonSetLabel: canaryControllerName, + }, + } +} + +func CanaryServiceName() types.NamespacedName { + return types.NamespacedName{ + Namespace: "openshift-ingress-canary", + Name: "ingress-canary", + } +} + +func CanaryRouteName() types.NamespacedName { + return types.NamespacedName{ + Namespace: "openshift-ingress-canary", + Name: "ingress-canary-route", + } +} diff --git a/pkg/operator/controller/status/controller.go b/pkg/operator/controller/status/controller.go index b822c50a72..96b0eaf746 100644 --- a/pkg/operator/controller/status/controller.go +++ b/pkg/operator/controller/status/controller.go @@ -91,7 +91,8 @@ type reconciler struct { func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, error) { log.Info("Reconciling", "request", request) - nsManifest := manifests.RouterNamespace() + ingressNamespace := manifests.RouterNamespace().Name + canaryNamespace := manifests.CanaryNamespace().Name co := &configv1.ClusterOperator{ObjectMeta: metav1.ObjectMeta{Name: operatorcontroller.IngressClusterOperatorName().Name}} if err := r.client.Get(context.TODO(), operatorcontroller.IngressClusterOperatorName(), co); err != nil { @@ -107,7 +108,7 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err } oldStatus := co.Status.DeepCopy() - state, err := r.getOperatorState(nsManifest.Name) + state, err := r.getOperatorState(ingressNamespace, canaryNamespace) if err != nil { return reconcile.Result{}, fmt.Errorf("failed to get operator state: %v", err) } @@ -128,12 +129,19 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err Namespace: r.Namespace, }, } - if state.Namespace != nil { + if state.IngressNamespace != nil { related = append(related, configv1.ObjectReference{ Resource: "namespaces", - Name: state.Namespace.Name, + Name: state.IngressNamespace.Name, }) } + if state.CanaryNamespace != nil { + related = append(related, configv1.ObjectReference{ + Resource: "namespaces", + Name: state.CanaryNamespace.Name, + }) + } + co.Status.RelatedObjects = related allIngressesAvailable := checkAllIngressesAvailable(state.IngressControllers) @@ -182,23 +190,33 @@ func initializeClusterOperator(co *configv1.ClusterOperator) { } type operatorState struct { - Namespace *corev1.Namespace + IngressNamespace *corev1.Namespace + CanaryNamespace *corev1.Namespace IngressControllers []operatorv1.IngressController DNSRecords []iov1.DNSRecord } // getOperatorState gets and returns the resources necessary to compute the // operator's current state. -func (r *reconciler) getOperatorState(nsName string) (operatorState, error) { +func (r *reconciler) getOperatorState(ingressNamespace, canaryNamespace string) (operatorState, error) { state := operatorState{} - ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: nsName}} - if err := r.client.Get(context.TODO(), types.NamespacedName{Name: nsName}, ns); err != nil { + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ingressNamespace}} + if err := r.client.Get(context.TODO(), types.NamespacedName{Name: ingressNamespace}, ns); err != nil { + if !errors.IsNotFound(err) { + return state, fmt.Errorf("failed to get namespace %q: %v", ingressNamespace, err) + } + } else { + state.IngressNamespace = ns + } + + ns = &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: canaryNamespace}} + if err := r.client.Get(context.TODO(), types.NamespacedName{Name: canaryNamespace}, ns); err != nil { if !errors.IsNotFound(err) { - return state, fmt.Errorf("failed to get namespace %q: %v", nsName, err) + return state, fmt.Errorf("failed to get namespace %q: %v", canaryNamespace, err) } } else { - state.Namespace = ns + state.CanaryNamespace = ns } ingressList := &operatorv1.IngressControllerList{} diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 761fa3ed05..d771916ec7 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -10,6 +10,7 @@ import ( operatorclient "github.com/openshift/cluster-ingress-operator/pkg/operator/client" operatorconfig "github.com/openshift/cluster-ingress-operator/pkg/operator/config" operatorcontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller" + canarycontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/canary" certcontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/certificate" certpublishercontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/certificate-publisher" dnscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/dns" @@ -61,6 +62,7 @@ func New(config operatorconfig.Config, kubeConfig *rest.Config) (*Operator, erro NewCache: cache.MultiNamespacedCacheBuilder([]string{ config.Namespace, manifests.DefaultOperandNamespace, + manifests.DefaultCanaryNamespace, operatorcontroller.GlobalMachineSpecifiedConfigNamespace, }), // Use a non-caching client everywhere. The default split client does not @@ -113,6 +115,16 @@ func New(config operatorconfig.Config, kubeConfig *rest.Config) (*Operator, erro return nil, fmt.Errorf("failed to create dns controller: %v", err) } + // Set up the canary controller when the config.CanaryImage is not empty + if len(config.CanaryImage) != 0 { + if _, err := canarycontroller.New(mgr, canarycontroller.Config{ + Namespace: config.Namespace, + CanaryImage: config.CanaryImage, + }); err != nil { + return nil, fmt.Errorf("failed to create canary controller: %v", err) + } + } + return &Operator{ manager: mgr, // TODO: These are only needed for the default ingress controller stuff, which