diff --git a/Makefile b/Makefile index 9198f76915..f1988f1757 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ MCO_COMPONENTS = daemon controller server operator -EXTRA_COMPONENTS = gcp-routes-controller +EXTRA_COMPONENTS = apiserver-watcher ALL_COMPONENTS = $(patsubst %,machine-config-%,$(MCO_COMPONENTS)) $(EXTRA_COMPONENTS) PREFIX ?= /usr GO111MODULE?=on @@ -29,7 +29,7 @@ clean: # Build machine configs. Intended to be called via another target. # Example: -# make _build-gcp-routes-controller +# make _build-machine-config-operator _build-%: WHAT=$* hack/build-go.sh diff --git a/cmd/apiserver-watcher/README.md b/cmd/apiserver-watcher/README.md new file mode 100644 index 0000000000..801b8293ee --- /dev/null +++ b/cmd/apiserver-watcher/README.md @@ -0,0 +1,75 @@ +# apiserver-watcher + +## Background + +Some cloud provider load balancers need special handling for hairpin scenarios. +Because default OpenShift installations are "self-driving", i.e. the control +plane is hosted as part of the cluster, we rely on hairpin extensively. + + +``` + +---------------+ + | | +-----------------+ + | +---------+ | | | + | | kubelet +-------------> layer-3 | + | +---------+ | | load balancer | + | | +----+ | + | +---------+ | | +-----------------+ + | |apiserver+<-------+ + | +---------+ | + | | + +---------------+ +``` + +We have iptables workarounds to fix these scenarios, but they need to know when +the local apiserver is up or down. Hence, the apiserver-watcher. + +### GCP + +Google cloud load balancer is a L3LB that is special. It doesn't do DNAT; instead, it +just redirects traffic to backends and preserves the VIP as the destination IP. + +So, an agent exists on the node. It programs the node (either via iptables or routing tables) to +accept traffic destined for the VIP. However, this has a problem: all hairpin traffic +to the balanced servce is *always* handled by that backend, even if it is down +or otherwise out of rotation. + +We want to withdraw the internal API service from google-routes redirection when +it's down, or else the node (i.e. kubelet) loses access to the apiserver VIP +and becomes unmanagable. + + +See `templates/master/00-master/gcp/files/opt-libexec-openshift-gcp-routes-sh.yaml` + +### Azure + +Azure L3LB does do DNAT, which presents a different problem: we can never reply +to hairpinned traffic. The problem looks something like this: + +``` +TCP SYN master-1 -> vip outgoing +(load balance happens) +TCP SYN master1 -> master1 incoming + +(server socket accepts, reply generated) +TCP SYN, ACK master1 -> master1 +``` + +This last packet is dropped, because the client socket is expecting a SYN,ACK with +a source IP of the VIP, not master1. + +So, when the apiserver is up, we want to direct all local traffic to ourselves. +When it is down, we would like it to go over the load balancer. + +See `templates/master/00-master/azure/files/opt-libexec-openshift-azure-routes-sh.yaml` + +## Functionality + +The apiserver-watcher is installed on all the masters and monitors the +apiserver process /readyz. + +When /readyz fails, write `/run/cloud-routes/$VIP.down`, which tells the +provider-specific service to update iptables rules. When it is up, write `$VIP.up`. + +Separately, a provider-specific process watches that directory and, as necessary, +updates iptables rules accordingly. diff --git a/cmd/gcp-routes-controller/main.go b/cmd/apiserver-watcher/main.go similarity index 76% rename from cmd/gcp-routes-controller/main.go rename to cmd/apiserver-watcher/main.go index 1f5e574d81..163e09cf98 100644 --- a/cmd/gcp-routes-controller/main.go +++ b/cmd/apiserver-watcher/main.go @@ -8,13 +8,13 @@ import ( ) const ( - componentName = "gcp-routes-controller" + componentName = "apisever-watcher" ) var ( rootCmd = &cobra.Command{ Use: componentName, - Short: "Controls the gcp-routes.service on RHCOS hosts based on health checks", + Short: "Monitors the local apiserver and writes cloud-routes downfiles", Long: "", SilenceErrors: true, SilenceUsage: true, diff --git a/cmd/gcp-routes-controller/run.go b/cmd/apiserver-watcher/run.go similarity index 63% rename from cmd/gcp-routes-controller/run.go rename to cmd/apiserver-watcher/run.go index cda8ea9c6b..755f4e624a 100644 --- a/cmd/gcp-routes-controller/run.go +++ b/cmd/apiserver-watcher/run.go @@ -4,15 +4,14 @@ import ( "crypto/tls" "flag" "fmt" + "io/ioutil" "net" "net/http" "net/url" "os" - "os/exec" "os/signal" "path" "sync" - "syscall" "time" health "github.com/InVisionApp/go-health" @@ -26,41 +25,31 @@ import ( var ( runCmd = &cobra.Command{ Use: "run", - Short: "Runs the gcp-routes-controller", + Short: "Runs the apiserver-watcher", Long: "", RunE: runRunCmd, } runOpts struct { - gcpRoutesService string - rootMount string - healthCheckURL string - vip string + rootMount string + healthCheckURL string + vip string } ) // downFileDir is the directory in which gcp-routes will look for a flag-file that // indicates the route to the VIP should be withdrawn. -const downFileDir = "/run/gcp-routes" +const downFileDir = "/run/cloud-routes" func init() { rootCmd.AddCommand(runCmd) - runCmd.PersistentFlags().StringVar(&runOpts.gcpRoutesService, "gcp-routes-service", "openshift-gcp-routes.service", "The name for the service controlling gcp routes on host") runCmd.PersistentFlags().StringVar(&runOpts.rootMount, "root-mount", "/rootfs", "where the nodes root filesystem is mounted for writing down files or chrooting.") runCmd.PersistentFlags().StringVar(&runOpts.healthCheckURL, "health-check-url", "", "HTTP(s) URL for the health check") runCmd.PersistentFlags().StringVar(&runOpts.vip, "vip", "", "The VIP to remove if the health check fails. Determined from URL if not provided") } -type downMode int - -const ( - modeStopService = iota - modeDownFile -) - type handler struct { - mode downMode - vip string + vip string } func runRunCmd(cmd *cobra.Command, args []string) error { @@ -104,6 +93,7 @@ func runRunCmd(cmd *cobra.Command, args []string) error { // as a backend in the load-balancer, and add routes before we've been // re-added. // see openshift/installer/data/data/gcp/network/lb-private.tf + // see openshift/installer/data/data/azure/vnet/internal-lb.tf tracker := &healthTracker{ state: unknownTrackerState, ErrCh: errCh, @@ -130,7 +120,7 @@ func runRunCmd(cmd *cobra.Command, args []string) error { signal.Notify(c, os.Interrupt) go func() { for sig := range c { - glog.Infof("Signal %s received: shutting down gcp routes service", sig) + glog.Infof("Signal %s received: treating service as down", sig) if err := handler.onFailure(); err != nil { glog.Infof("Failed to mark service down on signal: %s", err) } @@ -151,45 +141,18 @@ func runRunCmd(cmd *cobra.Command, args []string) error { func newHandler(uri *url.URL) (*handler, error) { h := handler{} - // determine mode: if /run/gcp-routes exists, we can us the downfile mode - realPath := path.Join(runOpts.rootMount, downFileDir) - fi, err := os.Stat(realPath) - if err == nil && fi.IsDir() { - glog.Infof("%s exists, starting in downfile mode", realPath) - h.mode = modeDownFile + if runOpts.vip != "" { + h.vip = runOpts.vip } else { - glog.Infof("%s not accessible, will stop gcp-routes.service on health failure", realPath) - h.mode = modeStopService - } - - // if StopService mode and rootfs specified, chroot - if h.mode == modeStopService && runOpts.rootMount != "" { - glog.Infof(`Calling chroot("%s")`, runOpts.rootMount) - if err := syscall.Chroot(runOpts.rootMount); err != nil { - return nil, fmt.Errorf("unable to chroot to %s: %s", runOpts.rootMount, err) - } - - glog.V(2).Infof("Moving to / inside the chroot") - if err := os.Chdir("/"); err != nil { - return nil, fmt.Errorf("unable to change directory to /: %s", err) + addrs, err := net.LookupHost(uri.Hostname()) + if err != nil { + return nil, fmt.Errorf("failed to lookup host %s: %v", uri.Hostname(), err) } - } - - // otherwise, resolve vip - if h.mode == modeDownFile { - if runOpts.vip != "" { - h.vip = runOpts.vip - } else { - addrs, err := net.LookupHost(uri.Hostname()) - if err != nil { - return nil, fmt.Errorf("failed to lookup host %s: %v", uri.Hostname(), err) - } - if len(addrs) != 1 { - return nil, fmt.Errorf("hostname %s has %d addresses, expected 1 - aborting", uri.Hostname(), len(addrs)) - } - h.vip = addrs[0] - glog.Infof("Using VIP %s", h.vip) + if len(addrs) != 1 { + return nil, fmt.Errorf("hostname %s has %d addresses, expected 1 - aborting", uri.Hostname(), len(addrs)) } + h.vip = addrs[0] + glog.Infof("Using VIP %s", h.vip) } return &h, nil @@ -197,37 +160,42 @@ func newHandler(uri *url.URL) (*handler, error) { // onFailure: either stop the routes service, or write downfile func (h *handler) onFailure() error { - if h.mode == modeDownFile { - downFile := path.Join(runOpts.rootMount, downFileDir, fmt.Sprintf("%s.down", h.vip)) - fp, err := os.OpenFile(downFile, os.O_CREATE, 0644) - if err != nil { - return fmt.Errorf("failed to create downfile (%s): %v", downFile, err) - } - _ = fp.Close() - glog.Infof("healthcheck failed, created downfile %s", downFile) - } else { - if err := exec.Command("systemctl", "stop", runOpts.gcpRoutesService).Run(); err != nil { - return fmt.Errorf("Failed to terminate gcp routes service %v", err) - } - glog.Infof("healthcheck failed, stopped %s", runOpts.gcpRoutesService) + if err := writeVipStateFile(h.vip, "down"); err != nil { + return err + } + glog.Infof("healthcheck failed, created downfile %s.down", h.vip) + if err := removeVipStateFile(h.vip, "up"); err != nil { + return err } return nil } // onSuccess: either start routes service, or remove down file func (h *handler) onSuccess() error { - if h.mode == modeDownFile { - downFile := path.Join(runOpts.rootMount, downFileDir, fmt.Sprintf("%s.down", h.vip)) - err := os.Remove(downFile) - if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to remove downfile (%s): %v", downFile, err) - } - glog.Infof("healthcheck succeeded, removed downfile %s", downFile) - } else { - if err := exec.Command("systemctl", "start", runOpts.gcpRoutesService).Run(); err != nil { - return fmt.Errorf("Failed to terminate gcp routes service %v", err) - } - glog.Infof("healthcheck succeeded, started %s", runOpts.gcpRoutesService) + if err := removeVipStateFile(h.vip, "down"); err != nil { + return err + } + glog.Infof("healthcheck succeeded, removed downfile %s.down", h.vip) + if err := writeVipStateFile(h.vip, "up"); err != nil { + return err + } + return nil +} + +func writeVipStateFile(vip, state string) error { + file := path.Join(runOpts.rootMount, downFileDir, fmt.Sprintf("%s.%s", vip, state)) + err := ioutil.WriteFile(file, nil, 0644) + if err != nil { + return fmt.Errorf("failed to create file (%s): %v", file, err) + } + return nil +} + +func removeVipStateFile(vip, state string) error { + file := path.Join(runOpts.rootMount, downFileDir, fmt.Sprintf("%s.%s", vip, state)) + err := os.Remove(file) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove file (%s): %v", file, err) } return nil } diff --git a/cmd/gcp-routes-controller/version.go b/cmd/apiserver-watcher/version.go similarity index 100% rename from cmd/gcp-routes-controller/version.go rename to cmd/apiserver-watcher/version.go diff --git a/cmd/gcp-routes-controller/README.md b/cmd/gcp-routes-controller/README.md deleted file mode 100644 index 86db108248..0000000000 --- a/cmd/gcp-routes-controller/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# gcp-routes-controller - -## Background - -Google cloud load balancer is a L3LB that is special. It doesn't do DNAT; instead, it -just redirects traffic to backends and preserves the VIP as the destination IP. - -So, an agent exists on the node. It programs the node (either via iptables or routing tables) to -accept traffic destined for the VIP. However, this has a problem: all hairpin traffic -to the balanced servce is *always* handled by that backend, even if it is down -or otherwise out of rotation. - -We want to withdraw the internal API service from google-routes redirection when -it's down, or else the node (i.e. kubelet) loses access to the apiserver VIP -and becomes unmanagable. - -## Functionality - -The gcp-routes-controller is installed on all the masters and monitors the -apiserver process /readyz. - -When /readyz fails, stops the VIP routing by writing `/run/gcp-routes/VIP.down`, -which tells openshift-gcp-routes to skip that vip diff --git a/pkg/controller/template/constants.go b/pkg/controller/template/constants.go index b278fef9e2..1b9674d17c 100644 --- a/pkg/controller/template/constants.go +++ b/pkg/controller/template/constants.go @@ -4,8 +4,8 @@ const ( // MachineConfigOperatorKey is our own image used by e.g. machine-config-daemon-pull.service MachineConfigOperatorKey string = "machineConfigOperator" - // GCPRoutesControllerKey is the key that references the gcp-routes-controller image in the controller - GCPRoutesControllerKey string = "gcpRoutesControllerKey" + // APIServerWatcherKey is the key that references the apiserver-watcher image + APIServerWatcherKey string = "apiServerWatcherKey" // InfraImageKey is the key that references the infra image in the controller for crio.conf InfraImageKey string = "infraImageKey" diff --git a/pkg/operator/bootstrap.go b/pkg/operator/bootstrap.go index b519bc2535..e2559c58b9 100644 --- a/pkg/operator/bootstrap.go +++ b/pkg/operator/bootstrap.go @@ -134,7 +134,7 @@ func RenderBootstrap( spec.Images = map[string]string{ templatectrl.MachineConfigOperatorKey: imgs.MachineConfigOperator, - templatectrl.GCPRoutesControllerKey: imgs.MachineConfigOperator, + templatectrl.APIServerWatcherKey: imgs.MachineConfigOperator, templatectrl.InfraImageKey: imgs.InfraImage, templatectrl.KeepalivedKey: imgs.Keepalived, templatectrl.CorednsKey: imgs.Coredns, diff --git a/pkg/operator/sync.go b/pkg/operator/sync.go index 2cb601daea..a2fd774ef7 100644 --- a/pkg/operator/sync.go +++ b/pkg/operator/sync.go @@ -270,7 +270,7 @@ func (optr *Operator) syncRenderConfig(_ *renderConfig) error { spec.Images = map[string]string{ templatectrl.MachineConfigOperatorKey: imgs.MachineConfigOperator, - templatectrl.GCPRoutesControllerKey: imgs.MachineConfigOperator, + templatectrl.APIServerWatcherKey: imgs.MachineConfigOperator, templatectrl.InfraImageKey: imgs.InfraImage, templatectrl.KeepalivedKey: imgs.Keepalived, templatectrl.CorednsKey: imgs.Coredns, diff --git a/templates/master/00-master/gcp/files/etc-kubernetes-manifests-gcp-routes-controller.yaml b/templates/master/00-master/azure/files/etc-kubernetes-manifests-apiserver-watcher.yaml similarity index 76% rename from templates/master/00-master/gcp/files/etc-kubernetes-manifests-gcp-routes-controller.yaml rename to templates/master/00-master/azure/files/etc-kubernetes-manifests-apiserver-watcher.yaml index 20c7d74df9..2e819c9103 100644 --- a/templates/master/00-master/gcp/files/etc-kubernetes-manifests-gcp-routes-controller.yaml +++ b/templates/master/00-master/azure/files/etc-kubernetes-manifests-apiserver-watcher.yaml @@ -1,17 +1,17 @@ mode: 0644 -path: "/etc/kubernetes/manifests/gcp-routes-controller.yaml" +path: "/etc/kubernetes/manifests/apiserver-watcher.yaml" contents: inline: | apiVersion: v1 kind: Pod metadata: - name: gcp-routes-controller + name: apiserver-watcher namespace: kube-system spec: containers: - - name: gcp-routes-controller - image: "{{.Images.gcpRoutesControllerKey}}" - command: ["gcp-routes-controller"] + - name: apiserver-watcher + image: "{{.Images.apiServerWatcherKey}}" + command: ["apiserver-watcher"] args: - "run" - "--health-check-url={{.Infra.Status.APIServerInternalURL}}/readyz" diff --git a/templates/master/00-master/azure/files/opt-libexec-openshift-azure-routes-sh.yaml b/templates/master/00-master/azure/files/opt-libexec-openshift-azure-routes-sh.yaml new file mode 100644 index 0000000000..65f3f6fc40 --- /dev/null +++ b/templates/master/00-master/azure/files/opt-libexec-openshift-azure-routes-sh.yaml @@ -0,0 +1,180 @@ +mode: 0755 +path: "/opt/libexec/openshift-azure-routes.sh" +contents: + inline: | + #!/bin/bash + + # Prevent hairpin traffic when the apiserver is up + + # As per the Azure documentation (https://docs.microsoft.com/en-us/azure/load-balancer/concepts#limitations), + # if a backend is load-balanced to itself, then the traffic will be dropped. + # + # This is because the L3LB does DNAT, so while the outgoing packet has a destination + # IP of the VIP, the incoming load-balanced packet has a destination IP of the + # host. That means that it "sees" a syn with the source and destination + # IPs of itself, and duly replies wit a syn-ack back to itself. However, the client + # socket expects a syn-ack with a source IP of the VIP, so it drops the packet. + # + # The solution is to redirect traffic destined to the lb vip back to ourselves. + # + # We check /run/cloud-routes/ for files $VIP.up and $VIP.down. If the .up file + # exists, then we redirect traffic destined for that vip to ourselves via iptables. + # A systemd unit watches the directory for changes. + + set -euo pipefail + + # the list of load balancer IPs that are assigned to this node + declare -A v4vips + declare -A v6vips + + CHAIN_NAME="azure-vips" + RUN_DIR="/run/cloud-routes" + + # Create a chan if it doesn't exist + ensure_chain4() { + local table="${1}" + local chain="${2}" + + if ! iptables -w -t "${table}" -S "${chain}" &> /dev/null ; then + iptables -w -t "${table}" -N "${chain}"; + fi; + } + + # Create a chain if it doesn't exist + ensure_chain6() { + if [ ! -f /proc/net/if_inet6 ]; then + return + fi + local table="${1}" + local chain="${2}" + + if ! ip6tables -w -t "${table}" -S "${chain}" &> /dev/null ; then + ip6tables -w -t "${table}" -N "${chain}"; + fi; + } + + + ensure_rule4() { + local table="${1}" + local chain="${2}" + shift 2 + + if ! iptables -w -t "${table}" -C "${chain}" "$@" &> /dev/null; then + iptables -w -t "${table}" -A "${chain}" "$@" + fi + } + + ensure_rule6() { + if [ ! -f /proc/net/if_inet6 ]; then + return + fi + + local table="${1}" + local chain="${2}" + shift 2 + + if ! ip6tables -w -t "${table}" -C "${chain}" "$@" &> /dev/null; then + ip6tables -w -t "${table}" -A "${chain}" "$@" + fi + } + + # set the chain, ensure entry rules, ensure ESTABLISHED rule + initialize() { + ensure_chain4 nat "${CHAIN_NAME}-local" + ensure_chain6 nat "${CHAIN_NAME}-local" + + ensure_rule4 nat OUTPUT -m comment --comment 'azure LB vip overriding for local clients' -j ${CHAIN_NAME}-local + ensure_rule6 nat OUTPUT -m comment --comment 'azure LB vip overriding for local clients' -j ${CHAIN_NAME}-local + + # Need this so that existing flows (with an entry in conntrack) continue, + # even if the iptables rule is removed + ensure_rule4 filter OUTPUT -m comment --comment 'azure LB vip existing' -m addrtype ! --dst-type LOCAL -m state --state ESTABLISHED,RELATED -j ACCEPT + ensure_rule6 filter OUTPUT -m comment --comment 'azure LB vip existing' -m addrtype ! --dst-type LOCAL -m state --state ESTABLISHED,RELATED -j ACCEPT + } + + remove_stale() { + ## find extra iptables rules + for ipt_vip in $(iptables -w -t nat -S "${CHAIN_NAME}-local" | awk '$4{print $4}' | awk -F/ '{print $1}'); do + if [[ ! -v v4vips[${ipt_vip}] ]] || [[ "${v4vips[${ipt_vip}]}" = down ]]; then + echo removing stale vip "${ipt_vip}" for local clients + iptables -w -t nat -D "${CHAIN_NAME}-local" --dst "${ipt_vip}" -j REDIRECT + fi + done + + if [ ! -f /proc/net/if_inet6 ]; then + return + fi + + for ipt_vip in $(ip6tables -w -t nat -S "${CHAIN_NAME}-local" | awk '$4{print $4}' | awk -F/ '{print $1}'); do + if [[ ! -v v6vips[${ipt_vip}] ]] || [[ "${v6vips[${ipt_vip}]}" = down ]]; then + echo removing stale vip "${ipt_vip}" for local clients + ip6tables -w -t nat -D "${CHAIN_NAME}-local" --dst "${ipt_vip}" -j REDIRECT + fi + done + + } + + add_rules() { + for vip in "${!v4vips[@]}"; do + if [[ "${v4vips[${vip}]}" != down ]]; then + echo "ensuring rule for ${vip} for internal clients" + ensure_rule4 nat "${CHAIN_NAME}-local" --dst "${vip}" -j REDIRECT + fi + done + + for vip in "${!v6vips[@]}"; do + if [[ "${v6vips[${vip}]}" != down ]]; then + echo "ensuring rule for ${vip} for internal clients" + ensure_rule6 nat "${CHAIN_NAME}-local" --dst "${vip}" -j REDIRECT + fi + done + } + + clear_rules() { + iptables -t nat -F "${CHAIN_NAME}-local" || true + } + + # out paramaters: v4vips v6vips + list_lb_ips() { + for k in "${!v4vips[@]}"; do + unset v4vips["${k}"] + done + for k in "${!v6vips[@]}"; do + unset v6vips["${k}"] + done + + + shopt -s nullglob + for file in "${RUN_DIR}"/*.up ; do + vip=$(basename "${file}" .up) + if [[ -e "${RUN_DIR}/${vip}.down" ]]; then + echo "${vip} has upfile and downfile, marking as down" + else + if [[ ${vip} =~ : ]]; then + echo "processing v6 vip ${vip}" + v6vips[${vip}]="${vip}" + else + echo "processing v4 vip ${vip}" + v4vips[${vip}]="${vip}" + fi + fi + done + } + + + case "$1" in + start) + initialize + list_lb_ips + remove_stale + add_rules + echo "done applying vip rules" + ;; + cleanup) + clear_rules + ;; + *) + echo $"Usage: $0 {start|cleanup}" + exit 1 + esac + diff --git a/templates/master/00-master/azure/units/openshift-azure-routes.path.yaml b/templates/master/00-master/azure/units/openshift-azure-routes.path.yaml new file mode 100644 index 0000000000..90933b60b0 --- /dev/null +++ b/templates/master/00-master/azure/units/openshift-azure-routes.path.yaml @@ -0,0 +1,15 @@ +name: openshift-azure-routes.path +enabled: true +contents: | + [Unit] + Description=Watch for downfile changes + Before=kubelet.service + ConditionPathExists=!/etc/ignition-machine-config-encapsulated.json + + [Path] + PathExistsGlob=/run/cloud-routes/* + PathChanged=/run/cloud-routes/ + MakeDirectory=true + + [Install] + RequiredBy=kubelet.service diff --git a/templates/master/00-master/azure/units/openshift-azure-routes.service.yaml b/templates/master/00-master/azure/units/openshift-azure-routes.service.yaml new file mode 100644 index 0000000000..4cc32d7eb0 --- /dev/null +++ b/templates/master/00-master/azure/units/openshift-azure-routes.service.yaml @@ -0,0 +1,11 @@ +name: openshift-azure-routes.service +enabled: false +contents: | + [Unit] + Description=Work around Azure load balancer hairpin + + [Service] + Type=simple + ExecStart=/bin/bash /opt/libexec/openshift-azure-routes.sh start + User=root + SyslogIdentifier=openshift-azure-routes diff --git a/templates/master/00-master/gcp/files/etc-kubernetes-manifests-apiserver-watcher.yaml b/templates/master/00-master/gcp/files/etc-kubernetes-manifests-apiserver-watcher.yaml new file mode 100644 index 0000000000..2e819c9103 --- /dev/null +++ b/templates/master/00-master/gcp/files/etc-kubernetes-manifests-apiserver-watcher.yaml @@ -0,0 +1,38 @@ +mode: 0644 +path: "/etc/kubernetes/manifests/apiserver-watcher.yaml" +contents: + inline: | + apiVersion: v1 + kind: Pod + metadata: + name: apiserver-watcher + namespace: kube-system + spec: + containers: + - name: apiserver-watcher + image: "{{.Images.apiServerWatcherKey}}" + command: ["apiserver-watcher"] + args: + - "run" + - "--health-check-url={{.Infra.Status.APIServerInternalURL}}/readyz" + resources: + requests: + cpu: 20m + memory: 50Mi + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + privileged: true + volumeMounts: + - mountPath: /rootfs + name: rootfs + hostNetwork: true + hostPID: true + priorityClassName: system-node-critical + tolerations: + - operator: "Exists" + restartPolicy: Always + volumes: + - name: rootfs + hostPath: + path: / + diff --git a/templates/master/00-master/gcp/files/opt-libexec-openshift-gcp-routes-sh.yaml b/templates/master/00-master/gcp/files/opt-libexec-openshift-gcp-routes-sh.yaml index b3a83aee2d..73fa587de1 100644 --- a/templates/master/00-master/gcp/files/opt-libexec-openshift-gcp-routes-sh.yaml +++ b/templates/master/00-master/gcp/files/opt-libexec-openshift-gcp-routes-sh.yaml @@ -16,7 +16,7 @@ contents: # existing connections when the vip is removed. This is useful for draining # connections - take ourselves out of the vip, but service existing conns. # - # Additionally, clients can write a file to /run/gcp-routes/$IP.down to force + # Additionally, clients can write a file to /run/cloud-routes/$IP.down to force # a VIP as down. This is useful for graceful shutdown / upgrade. # # ~cdc~ @@ -31,7 +31,7 @@ contents: } CHAIN_NAME="gcp-vips" - RUN_DIR="/run/gcp-routes" + RUN_DIR="/run/cloud-routes" # Create a chan if it doesn't exist ensure_chain() {