From a24b4f9ec1b1b562b108d02cafd3e7736c9a1b00 Mon Sep 17 00:00:00 2001 From: Yossi Boaron Date: Mon, 31 Aug 2020 00:48:18 +0300 Subject: [PATCH 1/2] [baremetal] Stop Keepalived on bootstrap after bootstrap completed When OPENSHIFT_INSTALL_PRESERVE_BOOTSTRAP deployment configuration parameter is set to true the bootstrap node won't be destroyed and infrastructure components including Keepalived on the bootstrap continue to run. With this PR the Keepalived process is stopped when a stop message is received in Keepalived control socket. This change is important for platforms that run Keepalived in unicast mode (baremetal). --- manifests/baremetal/keepalived.yaml | 19 ++++++++++++++++++- pkg/operator/assets/bindata.go | 19 ++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/manifests/baremetal/keepalived.yaml b/manifests/baremetal/keepalived.yaml index 9dbf91be17..5f03f7e788 100644 --- a/manifests/baremetal/keepalived.yaml +++ b/manifests/baremetal/keepalived.yaml @@ -52,13 +52,23 @@ spec: /usr/sbin/keepalived -f /etc/keepalived/keepalived.conf --dont-fork --vrrp --log-detail --log-console & fi } + stop_keepalived() + { + echo "Keepalived process stopped" >> /var/run/keepalived/stopped + if pid=$(pgrep -o keepalived); then + kill -s TERM "$pid" + fi + } + msg_handler() { while read -r line; do echo "The client sent: $line" >&2 - # currently only 'reload' msg is supported + # currently only 'reload' and 'stop' msgs are supported if [ "$line" = reload ]; then reload_keepalived + elif [ "$line" = stop ]; then + stop_keepalived fi done } @@ -66,6 +76,12 @@ spec: declare -r keepalived_sock="/var/run/keepalived/keepalived.sock" export -f msg_handler export -f reload_keepalived + export -f stop_keepalived + + while [ -s "/var/run/keepalived/stopped" ]; do + echo "Container stopped" + sleep 60 + done if [ -s "/etc/keepalived/keepalived.conf" ]; then /usr/sbin/keepalived -f /etc/keepalived/keepalived.conf --dont-fork --vrrp --log-detail --log-console & fi @@ -87,6 +103,7 @@ spec: - -c - | [[ -s /etc/keepalived/keepalived.conf ]] || \ + [[ -s /var/run/keepalived/stopped ]] || \ kill -s SIGUSR1 "$(pgrep -o keepalived)" && ! grep -q "State = FAULT" /tmp/keepalived.data initialDelaySeconds: 20 terminationMessagePolicy: FallbackToLogsOnError diff --git a/pkg/operator/assets/bindata.go b/pkg/operator/assets/bindata.go index 670364e9b6..0e6945f1a6 100644 --- a/pkg/operator/assets/bindata.go +++ b/pkg/operator/assets/bindata.go @@ -360,13 +360,23 @@ spec: /usr/sbin/keepalived -f /etc/keepalived/keepalived.conf --dont-fork --vrrp --log-detail --log-console & fi } + stop_keepalived() + { + echo "Keepalived process stopped" >> /var/run/keepalived/stopped + if pid=$(pgrep -o keepalived); then + kill -s TERM "$pid" + fi + } + msg_handler() { while read -r line; do echo "The client sent: $line" >&2 - # currently only 'reload' msg is supported + # currently only 'reload' and 'stop' msgs are supported if [ "$line" = reload ]; then reload_keepalived + elif [ "$line" = stop ]; then + stop_keepalived fi done } @@ -374,6 +384,12 @@ spec: declare -r keepalived_sock="/var/run/keepalived/keepalived.sock" export -f msg_handler export -f reload_keepalived + export -f stop_keepalived + + while [ -s "/var/run/keepalived/stopped" ]; do + echo "Container stopped" + sleep 60 + done if [ -s "/etc/keepalived/keepalived.conf" ]; then /usr/sbin/keepalived -f /etc/keepalived/keepalived.conf --dont-fork --vrrp --log-detail --log-console & fi @@ -395,6 +411,7 @@ spec: - -c - | [[ -s /etc/keepalived/keepalived.conf ]] || \ + [[ -s /var/run/keepalived/stopped ]] || \ kill -s SIGUSR1 "$(pgrep -o keepalived)" && ! grep -q "State = FAULT" /tmp/keepalived.data initialDelaySeconds: 20 terminationMessagePolicy: FallbackToLogsOnError From 1537a2234580fea0d4e17bbb5b2e998c09cfca4f Mon Sep 17 00:00:00 2001 From: Yossi Boaron Date: Mon, 21 Sep 2020 12:46:10 +0300 Subject: [PATCH 2/2] =?UTF-8?q?keep=20API=20VIP=20in=20the=20bootstrap=20n?= =?UTF-8?q?ode=20until=20the=20bootstrap=E2=80=99s=20node=20API=20goes=20a?= =?UTF-8?q?way?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keeping the API VIP in the bootstrap node until the bootstrap’s node API goes away eliminates the need for master nodes to specify the bootstrap as one of their peers in the unicast Keepalived case, see [1]. Additionally, it makes the deployment flow more deterministic so other components (e.g: remove requirement for provisioning static IP , see [2] ) can benefit from it. [1] https://docs.google.com/document/d/1jPIOYuK1J3PKYS8ff8waHWeHqzgl5OZeOkWngUxHtR8/edit?usp=sharing [2] https://docs.google.com/document/d/18DVr7Ikw_gSJsiQvG8zV2oBEqlG070MzyIElcK94LJI/edit# --- manifests/baremetal/keepalived.conf.tmpl | 2 +- manifests/baremetal/keepalived.yaml | 8 -------- pkg/operator/assets/bindata.go | 10 +--------- 3 files changed, 2 insertions(+), 18 deletions(-) diff --git a/manifests/baremetal/keepalived.conf.tmpl b/manifests/baremetal/keepalived.conf.tmpl index 115143cec9..3105741939 100644 --- a/manifests/baremetal/keepalived.conf.tmpl +++ b/manifests/baremetal/keepalived.conf.tmpl @@ -10,7 +10,7 @@ state BACKUP interface {{.VRRPInterface}} virtual_router_id {{.Cluster.APIVirtualRouterID }} - priority 50 + priority 70 advert_int 1 {{ if .EnableUnicast }} unicast_src_ip {{.NonVirtualIP}} diff --git a/manifests/baremetal/keepalived.yaml b/manifests/baremetal/keepalived.yaml index 5f03f7e788..46a2637bce 100644 --- a/manifests/baremetal/keepalived.yaml +++ b/manifests/baremetal/keepalived.yaml @@ -24,14 +24,6 @@ spec: - name: run-dir empty-dir: {} containers: - - name: keepalived-unicast - image: {{ .Images.BaremetalRuntimeCfgBootstrap }} - command: - - unicastipserver - - "--api-vip" - - "{{ .ControllerConfig.Infra.Status.PlatformStatus.BareMetal.APIServerInternalIP }}" - - "--ingress-vip" - - "{{ .ControllerConfig.Infra.Status.PlatformStatus.BareMetal.IngressIP }}" - name: keepalived securityContext: privileged: true diff --git a/pkg/operator/assets/bindata.go b/pkg/operator/assets/bindata.go index 0e6945f1a6..d41dd44cd0 100644 --- a/pkg/operator/assets/bindata.go +++ b/pkg/operator/assets/bindata.go @@ -269,7 +269,7 @@ var _manifestsBaremetalKeepalivedConfTmpl = []byte(`# Configuration template for state BACKUP interface {{.VRRPInterface}} virtual_router_id {{.Cluster.APIVirtualRouterID }} - priority 50 + priority 70 advert_int 1 {{ if .EnableUnicast }} unicast_src_ip {{.NonVirtualIP}} @@ -332,14 +332,6 @@ spec: - name: run-dir empty-dir: {} containers: - - name: keepalived-unicast - image: {{ .Images.BaremetalRuntimeCfgBootstrap }} - command: - - unicastipserver - - "--api-vip" - - "{{ .ControllerConfig.Infra.Status.PlatformStatus.BareMetal.APIServerInternalIP }}" - - "--ingress-vip" - - "{{ .ControllerConfig.Infra.Status.PlatformStatus.BareMetal.IngressIP }}" - name: keepalived securityContext: privileged: true