Skip to content

Commit d9f2ea6

Browse files
UltraInstinct14UltraInstinct14
authored
Merge pull request #523 from nik-netlox/main
BFD failover detection in L2 with connection sync CICD test case
2 parents a956622 + cbe35ed commit d9f2ea6

24 files changed

+1103
-2
lines changed

‎cicd/k8s-calico-ipvs2-ha-ka-sync/README

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
## Test Case Description
2+
3+
This scenario will demonstrate LoxiLB in L2 HA mode(clustering). The setup will have 2 LoxiLB nodes, K8s(1 Master Nodes & 2 Worker Nodes) cluster with Calico CNI in ipvs mode. LoxiLB will be running as external Service LB. Workloads will be spawned in all the cluster nodes.
4+
5+
Highlights of this demo:
6+
1) HA in L2 Mode
7+
2) Sub-second HA failover detection
8+
3) Connection sync
9+
10+
Client will be connected to the LoxiLB with L2 network. Master LoxiLB will announce the Service IP to the client and cluster nodes.
11+
12+
Service CIDR will also be a Virtual IP, a subnet of the K8s cluster network.
13+
14+
In scenarios where LoxiLB runs outside of the cluster in HA mode, it is advised to create LB services in fullnat mode for ease of connectivity.
15+
16+
If you wish to create this scenario in your lab then install Vagrant and follow the steps below:
17+
18+
1. Run ./config.sh to setup the K8s cluster, client and LoxiLB nodes
19+
20+
2. Run ./validation.sh to run the TCP HA test or ./validation_with_sctp.sh to run TCP & SCTP HA Test. Test Results will be displayed at the end.
21+
22+
3. Run ./rmconfig.sh to cleanup the setup.

‎cicd/k8s-calico-ipvs2-ha-ka-sync/Vagrantfile

+95
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
# -*- mode: ruby -*-
2+
# vi: set ft=ruby :
3+
4+
require "yaml"
5+
settings = YAML.load_file "yaml/settings.yaml"
6+
7+
workers = settings["nodes"]["workers"]["count"]
8+
loxilbs = (ENV['LOXILBS'] || "2").to_i
9+
10+
Vagrant.configure("2") do |config|
11+
12+
if Vagrant.has_plugin?("vagrant-vbguest")
13+
config.vbguest.auto_update = false
14+
end
15+
config.vm.define "host" do |host|
16+
host.vm.hostname = 'host'
17+
host.vm.box = settings["software"]["cluster"]["box"]
18+
host.vm.network :private_network, ip: "192.168.80.9", :netmask => "255.255.255.0"
19+
host.vm.provision :shell, :path => "node_scripts/host.sh"
20+
host.vm.provider :virtualbox do |vbox|
21+
vbox.customize ["modifyvm", :id, "--memory", 2048]
22+
vbox.customize ["modifyvm", :id, "--cpus", 2]
23+
end
24+
end
25+
26+
27+
28+
(1..loxilbs).each do |node_number|
29+
config.vm.define "llb#{node_number}" do |loxilb|
30+
loxilb.vm.box = settings["software"]["loxilb"]["box"]["name"]
31+
loxilb.vm.box_version = settings["software"]["loxilb"]["box"]["version"]
32+
loxilb.vm.hostname = "llb#{node_number}"
33+
ip = node_number + 251
34+
loxilb.vm.network :private_network, ip: "192.168.80.#{ip}", :netmask => "255.255.255.0"
35+
loxilb.vm.provision :shell, :path => "node_scripts/loxilb#{node_number}.sh"
36+
loxilb.vm.provider :virtualbox do |vbox|
37+
vbox.customize ["modifyvm", :id, "--memory", 6000]
38+
vbox.customize ["modifyvm", :id, "--cpus", 4]
39+
vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
40+
end
41+
end
42+
end
43+
44+
config.vm.define "master" do |master|
45+
master.vm.box = settings["software"]["cluster"]["box"]
46+
master.vm.hostname = 'master'
47+
master.vm.network :private_network, ip: settings["network"]["control_ip"], :netmask => "255.255.255.0"
48+
master.vm.provision "shell",
49+
env: {
50+
"DNS_SERVERS" => settings["network"]["dns_servers"].join(" "),
51+
"ENVIRONMENT" => settings["environment"],
52+
"KUBERNETES_VERSION" => settings["software"]["kubernetes"],
53+
"OS" => settings["software"]["os"]
54+
},
55+
path: "node_scripts/common.sh"
56+
master.vm.provision "shell",
57+
env: {
58+
"CALICO_VERSION" => settings["software"]["calico"],
59+
"CONTROL_IP" => settings["network"]["control_ip"],
60+
"POD_CIDR" => settings["network"]["pod_cidr"],
61+
"SERVICE_CIDR" => settings["network"]["service_cidr"]
62+
},
63+
path: "node_scripts/master.sh"
64+
65+
master.vm.provider :virtualbox do |vbox|
66+
vbox.customize ["modifyvm", :id, "--memory", 4096]
67+
vbox.customize ["modifyvm", :id, "--cpus", 2]
68+
vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
69+
end
70+
end
71+
72+
(1..workers).each do |node_number|
73+
config.vm.define "worker#{node_number}" do |worker|
74+
worker.vm.box = settings["software"]["cluster"]["box"]
75+
worker.vm.hostname = "worker#{node_number}"
76+
ip = node_number + 200
77+
worker.vm.network :private_network, ip: "192.168.80.#{ip}", :netmask => "255.255.255.0"
78+
worker.vm.provision "shell",
79+
env: {
80+
"DNS_SERVERS" => settings["network"]["dns_servers"].join(" "),
81+
"ENVIRONMENT" => settings["environment"],
82+
"KUBERNETES_VERSION" => settings["software"]["kubernetes"],
83+
"OS" => settings["software"]["os"]
84+
},
85+
path: "node_scripts/common.sh"
86+
worker.vm.provision "shell", path: "node_scripts/worker.sh"
87+
88+
worker.vm.provider :virtualbox do |vbox|
89+
vbox.customize ["modifyvm", :id, "--memory", 4096]
90+
vbox.customize ["modifyvm", :id, "--cpus", 2]
91+
vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
92+
end
93+
end
94+
end
95+
end

‎cicd/k8s-calico-ipvs2-ha-ka-sync/config.sh

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/bin/bash
2+
VMs=$(vagrant global-status | grep -i virtualbox)
3+
while IFS= read -a VMs; do
4+
read -a vm <<< "$VMs"
5+
cd ${vm[4]} 2>&1>/dev/null
6+
echo "Destroying ${vm[1]}"
7+
vagrant destroy -f ${vm[1]}
8+
cd - 2>&1>/dev/null
9+
done <<< "$VMs"
10+
11+
vagrant up
12+
13+
for((i=1; i<=60; i++))
14+
do
15+
fin=1
16+
pods=$(vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null | grep -v "NAMESPACE")
17+
18+
while IFS= read -a pods; do
19+
read -a pod <<< "$pods"
20+
if [[ ${pod[3]} != *"Running"* ]]; then
21+
echo "${pod[1]} is not UP yet"
22+
fin=0
23+
fi
24+
done <<< "$pods"
25+
if [ $fin == 1 ];
26+
then
27+
break;
28+
fi
29+
echo "Will try after 10s"
30+
sleep 10
31+
done
32+
33+
sudo sysctl net.ipv4.conf.vboxnet1.arp_accept=1
34+
35+
#Create fullnat Service
36+
vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/tcp_fullnat.yml' 2> /dev/null
37+
vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/sctp_fullnat.yml' 2> /dev/null
38+
#vagrant ssh master -c 'kubectl apply -f https://raw.githubusercontent.com/loxilb-io/kube-loxilb/main/manifest/mesh/loxilb-mesh.yml' 2> /dev/null

‎cicd/k8s-calico-ipvs2-ha-ka-sync/host_validation.sh

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
extIP=$(cat /vagrant/extIP)
3+
4+
code=0
5+
6+
echo Service IP: $extIP
7+
8+
neigh=$( ip neigh | grep $extIP )
9+
10+
if [[ ! -z $neigh && $neigh != *"FAILED"* ]]; then
11+
echo "Host route [OK]"
12+
else
13+
echo "Host route [NOK]"
14+
fi
15+
echo -e "\n*********************************************"
16+
echo "Testing Service"
17+
echo "*********************************************"
18+
19+
# iperf client accessing fullnat service
20+
stdbuf -oL nohup iperf -c 192.168.80.5 -p 56002 -t 60 -i 1 -b 100M &> iperff.out &
21+
echo "iperf client started"
22+
echo "phase-1 done"

‎cicd/k8s-calico-ipvs2-ha-ka-sync/host_validation2.sh

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
3+
pkill iperf
4+
5+
iperff_res=$(tail -n 1 iperff.out | xargs | cut -d ' ' -f 7)
6+
7+
if [[ $iperff_res != 0 ]]; then
8+
echo -e "K8s-calico-ipvs2-ha-ka-sync TCP\t\t(fullnat)\t[OK]"
9+
else
10+
echo -e "K8s-calico-ipvs2-ha-ka-sync TCP\t\t(fullnat)\t[FAILED]"
11+
code=1
12+
fi
13+
14+
rm *.out
15+
exit $code

‎cicd/k8s-calico-ipvs2-ha-ka-sync/host_validation2_with_sctp.sh

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
for((i=0;i<50;i++))
3+
do
4+
echo "snd=100" 1> sd1.pipe
5+
sleep 1
6+
done
7+
8+
9+
echo "stats" 1> sd1.pipe
10+
11+
echo "shutdown" 1> sd1.pipe
12+
13+
pkill iperf
14+
pkill sctp_darn
15+
16+
iperff_res=$(tail -n 1 iperff.out | xargs | cut -d ' ' -f 7)
17+
18+
sdf_res=$(grep -i "Client: Sending packets.(100000/100000)" sdf.out)
19+
20+
sdf_res1=$(grep -i "packets sent" sdf.out | xargs | cut -d ' ' -f 3)
21+
sdf_res2=$(grep -i "packets rec" sdf.out | xargs | cut -d ' ' -f 3)
22+
23+
if [[ $iperff_res != 0 ]]; then
24+
echo -e "K8s-calico-ipvs2-ha-ka-sync TCP\t\t(fullnat)\t[OK]"
25+
else
26+
echo -e "K8s-calico-ipvs2-ha-ka-sync TCP\t\t(fullnat)\t[FAILED]"
27+
code=1
28+
fi
29+
30+
if [[ $sdf_res1 != 0 && $sdf_res2 != 0 && $sdf_res1 == $sdf_res2 ]]; then
31+
echo -e "K8s-calico-ipvs2-ha-ka-sync SCTP\t(fullnat)\t[OK]"
32+
else
33+
echo -e "K8s-calico-ipvs2-ha-ka-sync SCTP\t(fullnat)\t[FAILED]"
34+
code=1
35+
fi
36+
37+
rm *.pipe
38+
rm *.out
39+
exit $code

‎cicd/k8s-calico-ipvs2-ha-ka-sync/host_validation_with_sctp.sh

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/bin/bash
2+
extIP=$(cat /vagrant/extIP)
3+
4+
code=0
5+
6+
echo Service IP: $extIP
7+
8+
neigh=$( ip neigh | grep $extIP )
9+
10+
if [[ ! -z $neigh && $neigh != *"FAILED"* ]]; then
11+
echo "Host route [OK]"
12+
else
13+
echo "Host route [NOK]"
14+
fi
15+
echo -e "\n*********************************************"
16+
echo "Testing Service"
17+
echo "*********************************************"
18+
19+
# iperf client accessing fullnat service
20+
stdbuf -oL nohup iperf -c 192.168.80.5 -p 56002 -t 100 -i 1 -b 100M &> iperff.out &
21+
22+
echo "iperf client started"
23+
24+
sleep 1
25+
26+
mkfifo sd1.pipe
27+
28+
sleep infinity > sd1.pipe &
29+
30+
stdbuf -oL nohup sctp_darn -H 192.168.80.9 -h 192.168.80.5 -p 56004 -s -I < sd1.pipe &> sdf.out &
31+
32+
echo "sctp_test client started"
33+
34+
sleep 2
35+
for((i=0;i<30;i++))
36+
do
37+
echo "snd=100" 1> sd1.pipe
38+
sleep 1
39+
done
40+
echo "phase-1 done"
41+
exit 0

‎cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/common.sh

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
#!/bin/bash
2+
#
3+
# Common setup for all servers (Control Plane and Nodes)
4+
5+
set -euxo pipefail
6+
7+
# Variable Declaration
8+
9+
# DNS Setting
10+
if [ ! -d /etc/systemd/resolved.conf.d ]; then
11+
sudo mkdir /etc/systemd/resolved.conf.d/
12+
fi
13+
cat <<EOF | sudo tee /etc/systemd/resolved.conf.d/dns_servers.conf
14+
[Resolve]
15+
DNS=${DNS_SERVERS}
16+
EOF
17+
18+
sudo systemctl restart systemd-resolved
19+
20+
# disable swap
21+
sudo swapoff -a
22+
23+
# keeps the swap off during reboot
24+
(crontab -l 2>/dev/null; echo "@reboot /sbin/swapoff -a") | crontab - || true
25+
sudo apt-get update -y
26+
# Install CRI-O Runtime
27+
28+
VERSION="$(echo ${KUBERNETES_VERSION} | grep -oE '[0-9]+\.[0-9]+')"
29+
30+
# Create the .conf file to load the modules at bootup
31+
cat <<EOF | sudo tee /etc/modules-load.d/crio.conf
32+
overlay
33+
br_netfilter
34+
EOF
35+
36+
sudo modprobe overlay
37+
sudo modprobe br_netfilter
38+
39+
# Install ipvs related modules
40+
sudo modprobe ip_vs
41+
sudo modprobe ip_vs_rr
42+
sudo modprobe ip_vs_wrr
43+
sudo modprobe ip_vs_sh
44+
#sudo modprobe nf_conntrack_ipv4
45+
46+
sudo sysctl net.ipv4.vs.sloppy_sctp=1
47+
48+
# Set up required sysctl params, these persist across reboots.
49+
cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
50+
net.bridge.bridge-nf-call-iptables = 1
51+
net.ipv4.ip_forward = 1
52+
net.bridge.bridge-nf-call-ip6tables = 1
53+
EOF
54+
55+
sudo sysctl --system
56+
57+
cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
58+
deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /
59+
EOF
60+
cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.list
61+
deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/ /
62+
EOF
63+
64+
curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
65+
curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
66+
67+
sudo apt-get update
68+
sudo apt-get install cri-o cri-o-runc -y
69+
70+
cat >> /etc/default/crio << EOF
71+
${ENVIRONMENT}
72+
EOF
73+
sudo systemctl daemon-reload
74+
sudo systemctl enable crio --now
75+
76+
echo "CRI runtime installed successfully"
77+
78+
sudo apt-get update
79+
sudo apt-get install -y apt-transport-https ca-certificates curl
80+
curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg
81+
82+
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
83+
sudo apt-get update -y
84+
sudo apt-get install -y kubelet="$KUBERNETES_VERSION" kubectl="$KUBERNETES_VERSION" kubeadm="$KUBERNETES_VERSION"
85+
sudo apt-get update -y
86+
sudo apt-get install -y jq
87+
sudo apt-get install -y ipvsadm
88+
89+
local_ip="$(ip --json a s | jq -r '.[] | if .ifname == "eth1" then .addr_info[] | if .family == "inet" then .local else empty end else empty end')"
90+
cat > /etc/default/kubelet << EOF
91+
KUBELET_EXTRA_ARGS=--node-ip=$local_ip
92+
${ENVIRONMENT}
93+
EOF

‎cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/host.sh

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Install Bird to work with k3s
2+
sudo apt-get update
3+
sudo apt-get -y install lksctp-tools iperf
4+
5+
echo "Host is up"

‎cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/loxilb1.sh

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
export LOXILB_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.80' | awk '{print $2}' | cut -f1 -d '/')
2+
3+
apt-get update
4+
apt-get install -y software-properties-common
5+
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
6+
add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
7+
apt-get update
8+
apt-get install -y docker-ce
9+
docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest --cluster=192.168.80.253 --self=0 --ka=192.168.80.253

‎cicd/k8s-calico-ipvs2-ha-ka-sync/node_scripts/loxilb2.sh

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
export LOXILB_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.80' | awk '{print $2}' | cut -f1 -d '/')
2+
3+
apt-get update
4+
apt-get install -y software-properties-common
5+
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
6+
add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
7+
apt-get update
8+
apt-get install -y docker-ce
9+
docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest --cluster=192.168.80.252 --self=1 --ka=192.168.80.252

0 commit comments

Comments
 (0)