From 416fd00de393d6457f31a93ab70067b179fab1db Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Fri, 3 May 2024 16:29:14 +0900 Subject: [PATCH] cicd: updated k3s-multi-master scenario --- .../iperf-onearm-ds-all.yml | 39 +++++++++++++++++++ .../iperf-onearm-ds.yml | 19 ++++----- cicd/k3s-multi-master-service-proxy/k3s.yaml | 19 --------- .../lbconfig.txt | 2 +- .../k3s-multi-master-service-proxy/master1.sh | 4 +- .../k3s-multi-master-service-proxy/master2.sh | 3 +- .../k3s-multi-master-service-proxy/master3.sh | 3 +- 7 files changed, 56 insertions(+), 33 deletions(-) create mode 100644 cicd/k3s-multi-master-service-proxy/iperf-onearm-ds-all.yml delete mode 100644 cicd/k3s-multi-master-service-proxy/k3s.yaml diff --git a/cicd/k3s-multi-master-service-proxy/iperf-onearm-ds-all.yml b/cicd/k3s-multi-master-service-proxy/iperf-onearm-ds-all.yml new file mode 100644 index 00000000..15e719ba --- /dev/null +++ b/cicd/k3s-multi-master-service-proxy/iperf-onearm-ds-all.yml @@ -0,0 +1,39 @@ +apiVersion: v1 +kind: Service +metadata: + name: iperf-service + annotations: + loxilb.io/lbmode: "onearm" + loxilb.io/prefLocalPod: "yes" +spec: + externalTrafficPolicy: Local + loadBalancerClass: loxilb.io/loxilb + selector: + app: iperf-app + ports: + - port: 55001 + targetPort: 5001 + type: LoadBalancer +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: iperf-daemonset + labels: + app: iperf-app +spec: + selector: + matchLabels: + app: iperf-app + template: + metadata: + labels: + app: iperf-app + spec: + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: iperf + image: eyes852/ubuntu-iperf-test:0.5 + command: [ "iperf", "-s"] + ports: + - containerPort: 5001 diff --git a/cicd/k3s-multi-master-service-proxy/iperf-onearm-ds.yml b/cicd/k3s-multi-master-service-proxy/iperf-onearm-ds.yml index 52ee91d6..da244cb3 100644 --- a/cicd/k3s-multi-master-service-proxy/iperf-onearm-ds.yml +++ b/cicd/k3s-multi-master-service-proxy/iperf-onearm-ds.yml @@ -4,6 +4,7 @@ metadata: name: iperf-service annotations: loxilb.io/lbmode: "onearm" + #loxilb.io/prefLocalPod: "yes" spec: externalTrafficPolicy: Local loadBalancerClass: loxilb.io/loxilb @@ -30,15 +31,15 @@ spec: app: iperf-app spec: dnsPolicy: ClusterFirstWithHostNet - #affinity: - # nodeAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # nodeSelectorTerms: - # - matchExpressions: - # - key: "node-role.kubernetes.io/master" - # operator: DoesNotExist - # - key: "node-role.kubernetes.io/control-plane" - # operator: DoesNotExist + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: DoesNotExist + - key: "node-role.kubernetes.io/control-plane" + operator: DoesNotExist containers: - name: iperf image: eyes852/ubuntu-iperf-test:0.5 diff --git a/cicd/k3s-multi-master-service-proxy/k3s.yaml b/cicd/k3s-multi-master-service-proxy/k3s.yaml deleted file mode 100644 index 5efd01aa..00000000 --- a/cicd/k3s-multi-master-service-proxy/k3s.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdGMyVnkKZG1WeUxXTmhRREUzTVRRek9EWTBPRGt3SGhjTk1qUXdOREk1TVRBeU9EQTVXaGNOTXpRd05ESTNNVEF5T0RBNQpXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUzTVRRek9EWTBPRGt3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFTU29Zc0RSbkVnNGdpeFVDcEY3aWZpUWVWVU5QMDJidmc5bUJ6VmUwbUcKRDZmZjVWV1h2Yk1JTXM4UWdzUDlkMVMrUWNab3JyUGEvZHo3NXJDRnZrM2tvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVVZLSEQwQ0prd2k3SFQ3ejJPUUNSCmZZTC83Y1F3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUlnWmp1aENUek5sYzMyc1NTa3hTVndEaUxEQ2hHenpaRUkKaGxGSFJtNFIrbllDSVFDaHJja0kzY3F6djB6UTN2VFNjSHM5MzRYSkF6S0dnb0hQZmx4b0tkbVM4QT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K - server: https://192.168.80.80:6443 - name: default -contexts: -- context: - cluster: default - user: default - name: default -current-context: default -kind: Config -preferences: {} -users: -- name: default - user: - client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrRENDQVRlZ0F3SUJBZ0lJYUR3NFdwek1wRmd3Q2dZSUtvWkl6ajBFQXdJd0l6RWhNQjhHQTFVRUF3d1kKYXpOekxXTnNhV1Z1ZEMxallVQXhOekUwTXpnMk5EZzVNQjRYRFRJME1EUXlPVEV3TWpnd09Wb1hEVEkxTURReQpPVEV3TWpnd09Wb3dNREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhGVEFUQmdOVkJBTVRESE41CmMzUmxiVHBoWkcxcGJqQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJDdUdLQlZNSUJNYVVrZEcKQlNCalBGNzNWMmVaNHZCaGl0ejUrYWtqRUprNnBWSW1yQzdPTkNVVytjOWowZnFJS0xZVGJyblI3cWZiT0FiMgp2UU9GWTZ1alNEQkdNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBakFmCkJnTlZIU01FR0RBV2dCU21QOXBVR0FoNlpKOFZONUhnUzdHMEd6Y092akFLQmdncWhrak9QUVFEQWdOSEFEQkUKQWlBaEVNcWs3TUtPTXd6TmM4aEc3L04wR3RZZzIzQVgvdmFsdmNEOFAycitqZ0lnZHJIbDZZdmNTbWFoVG1GYgowQTJwWkpxK2hvNkg2ZThBVCtRRTJGSHVKZXM9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KLS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdFkyeHAKWlc1MExXTmhRREUzTVRRek9EWTBPRGt3SGhjTk1qUXdOREk1TVRBeU9EQTVXaGNOTXpRd05ESTNNVEF5T0RBNQpXakFqTVNFd0h3WURWUVFEREJock0zTXRZMnhwWlc1MExXTmhRREUzTVRRek9EWTBPRGt3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFTNVZyeHFpaE4yMktlcCtwSTFWalBna21hV09tUXBBek9wY0VrQzNrRVAKTDB3cS9CalVGaVo2ckNxZklLSndzckhnbGgxemtJQzhuRjdhbzU0SjdzbS9vMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVXBqL2FWQmdJZW1TZkZUZVI0RXV4CnRCczNEcjR3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUlnSFVIM3UzR2NsWmxPQWFNMEpNOVRrMTVISUhYUVJ2VVIKcmUzNksvSkNmTWNDSVFEdk5pWmdFaGVmRytrZkdoOHBHK080REFHVVA1ZG5QTXNMU2JvZVlWelhTUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K - client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUJNL1lLR0svWkkzc3IrUmZjNURQWkF3YzBtY3gwZFBlNFkwdmxpdGhMS3JvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFSzRZb0ZVd2dFeHBTUjBZRklHTThYdmRYWjVuaThHR0szUG41cVNNUW1UcWxVaWFzTHM0MApKUmI1ejJQUitvZ290aE51dWRIdXA5czRCdmE5QTRWanF3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= diff --git a/cicd/k3s-multi-master-service-proxy/lbconfig.txt b/cicd/k3s-multi-master-service-proxy/lbconfig.txt index f9609405..968df7ab 100644 --- a/cicd/k3s-multi-master-service-proxy/lbconfig.txt +++ b/cicd/k3s-multi-master-service-proxy/lbconfig.txt @@ -6,7 +6,7 @@ "port":6443, "protocol":"tcp", "sel":0, - "mode":2, + "mode":1, "BGP":false, "Monitor":true, "inactiveTimeOut":240, diff --git a/cicd/k3s-multi-master-service-proxy/master1.sh b/cicd/k3s-multi-master-service-proxy/master1.sh index 05f6c214..eff8110c 100755 --- a/cicd/k3s-multi-master-service-proxy/master1.sh +++ b/cicd/k3s-multi-master-service-proxy/master1.sh @@ -3,14 +3,14 @@ sudo su ip addr add 192.168.80.80/32 dev lo apt-get update && apt-get install ipvsadm ipset -y export MASTER_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.90' | grep '192.168.80' | awk '{print $2}' | cut -f1 -d '/') -curl -fL https://get.k3s.io | sh -s - server --node-ip=192.168.80.10 --disable servicelb --disable traefik --cluster-init --node-external-ip=192.168.80.80 --disable-cloud-controller --flannel-iface=eth1 --kube-proxy-arg proxy-mode=ipvs +curl -fL https://get.k3s.io | sh -s - server --node-ip=192.168.80.10 --disable servicelb --disable traefik --cluster-init --node-external-ip=192.168.80.80 --disable-cloud-controller --flannel-iface=eth2 --kube-proxy-arg proxy-mode=ipvs kubectl taint nodes master1 node.cloudprovider.kubernetes.io/uninitialized:NoSchedule- curl -sfL https://github.com/loxilb-io/loxilb-ebpf/raw/main/kprobe/install.sh | sh - sleep 60 echo $MASTER_IP > /vagrant/master-ip cp /var/lib/rancher/k3s/server/node-token /vagrant/node-token +sed -i -e "s/127.0.0.1/192.168.80.80/g" /etc/rancher/k3s/k3s.yaml cp /etc/rancher/k3s/k3s.yaml /vagrant/k3s.yaml -sed -i -e "s/127.0.0.1/192.168.80.80/g" /vagrant/k3s.yaml sudo mkdir -p /etc/loxilb sudo cp /vagrant/lbconfig.txt /etc/loxilb/ sudo cp /vagrant/EPconfig.txt /etc/loxilb/ diff --git a/cicd/k3s-multi-master-service-proxy/master2.sh b/cicd/k3s-multi-master-service-proxy/master2.sh index 066a2efe..afd16239 100755 --- a/cicd/k3s-multi-master-service-proxy/master2.sh +++ b/cicd/k3s-multi-master-service-proxy/master2.sh @@ -7,7 +7,8 @@ sudo mkdir -p /etc/loxilb sudo cp /vagrant/lbconfig.txt /etc/loxilb/ sudo cp /vagrant/EPconfig.txt /etc/loxilb/ curl -sfL https://github.com/loxilb-io/loxilb-ebpf/raw/main/kprobe/install.sh | sh - -curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.11 --node-external-ip=192.168.80.80 --disable-cloud-controller -t ${NODE_TOKEN} --flannel-iface=eth1 --kube-proxy-arg proxy-mode=ipvs +curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.11 --node-external-ip=192.168.80.80 --disable-cloud-controller -t ${NODE_TOKEN} --flannel-iface=eth2 --kube-proxy-arg proxy-mode=ipvs +sed -i -e "s/127.0.0.1/192.168.80.80/g" /etc/rancher/k3s/k3s.yaml kubectl taint nodes master2 node.cloudprovider.kubernetes.io/uninitialized:NoSchedule- /vagrant/wait_ready.sh sysctl net.core.netdev_max_backlog=10000 diff --git a/cicd/k3s-multi-master-service-proxy/master3.sh b/cicd/k3s-multi-master-service-proxy/master3.sh index 9d1218a6..77bf8871 100755 --- a/cicd/k3s-multi-master-service-proxy/master3.sh +++ b/cicd/k3s-multi-master-service-proxy/master3.sh @@ -7,8 +7,9 @@ sudo mkdir -p /etc/loxilb sudo cp /vagrant/lbconfig.txt /etc/loxilb/ sudo cp /vagrant/EPconfig.txt /etc/loxilb/ curl -sfL https://github.com/loxilb-io/loxilb-ebpf/raw/main/kprobe/install.sh | sh - -curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.12 --node-external-ip=192.168.80.80 --disable-cloud-controller -t ${NODE_TOKEN} --flannel-iface=eth1 --kube-proxy-arg proxy-mode=ipvs +curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.12 --node-external-ip=192.168.80.80 --disable-cloud-controller -t ${NODE_TOKEN} --flannel-iface=eth2 --kube-proxy-arg proxy-mode=ipvs kubectl taint nodes master2 node.cloudprovider.kubernetes.io/uninitialized:NoSchedule- +sed -i -e "s/127.0.0.1/192.168.80.80/g" /etc/rancher/k3s/k3s.yaml sudo kubectl apply -f /vagrant/kube-loxilb.yml /vagrant/wait_ready.sh sysctl net.core.netdev_max_backlog=10000