Merge pull request #663 from TrekkieCoder/main

cicd: updated k3s-multi-master scenario
loxilb-io · May 3, 2024 · 8366e2f · 8366e2f
2 parents f3770c6 + 416fd00
commit 8366e2f
Show file tree

Hide file tree

Showing 7 changed files with 56 additions and 33 deletions.
diff --git a/cicd/k3s-multi-master-service-proxy/iperf-onearm-ds-all.yml b/cicd/k3s-multi-master-service-proxy/iperf-onearm-ds-all.yml
@@ -0,0 +1,39 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: iperf-service
+  annotations:
+    loxilb.io/lbmode: "onearm" 
+    loxilb.io/prefLocalPod: "yes"
+spec:
+  externalTrafficPolicy: Local
+  loadBalancerClass: loxilb.io/loxilb
+  selector:
+    app: iperf-app
+  ports:
+    - port: 55001
+      targetPort: 5001
+  type: LoadBalancer
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: iperf-daemonset
+  labels:
+    app: iperf-app
+spec:
+  selector:
+    matchLabels:
+      app: iperf-app
+  template:
+    metadata:
+      labels:
+        app: iperf-app
+    spec:
+      dnsPolicy: ClusterFirstWithHostNet
+      containers:
+      - name: iperf
+        image: eyes852/ubuntu-iperf-test:0.5
+        command: [ "iperf", "-s"]
+        ports:
+        - containerPort: 5001
diff --git a/cicd/k3s-multi-master-service-proxy/iperf-onearm-ds.yml b/cicd/k3s-multi-master-service-proxy/iperf-onearm-ds.yml
@@ -4,6 +4,7 @@ metadata:
   name: iperf-service
   annotations:
     loxilb.io/lbmode: "onearm" 
+    #loxilb.io/prefLocalPod: "yes"
 spec:
   externalTrafficPolicy: Local
   loadBalancerClass: loxilb.io/loxilb
@@ -30,15 +31,15 @@ spec:
         app: iperf-app
     spec:
       dnsPolicy: ClusterFirstWithHostNet
-      #affinity:
-      #  nodeAffinity:
-      #    requiredDuringSchedulingIgnoredDuringExecution:
-      #      nodeSelectorTerms:
-      #      - matchExpressions:
-      #        - key: "node-role.kubernetes.io/master"
-      #          operator: DoesNotExist
-      #        - key: "node-role.kubernetes.io/control-plane"
-      #          operator: DoesNotExist
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: "node-role.kubernetes.io/master"
+                operator: DoesNotExist
+              - key: "node-role.kubernetes.io/control-plane"
+                operator: DoesNotExist
       containers:
       - name: iperf
         image: eyes852/ubuntu-iperf-test:0.5

diff --git a/cicd/k3s-multi-master-service-proxy/k3s.yaml b/cicd/k3s-multi-master-service-proxy/k3s.yaml
diff --git a/cicd/k3s-multi-master-service-proxy/lbconfig.txt b/cicd/k3s-multi-master-service-proxy/lbconfig.txt
@@ -6,7 +6,7 @@
             "port":6443,
             "protocol":"tcp",
             "sel":0,
-            "mode":2,
+            "mode":1,
             "BGP":false,
             "Monitor":true,
             "inactiveTimeOut":240,

diff --git a/cicd/k3s-multi-master-service-proxy/master1.sh b/cicd/k3s-multi-master-service-proxy/master1.sh
@@ -3,14 +3,14 @@ sudo su
 ip addr add 192.168.80.80/32 dev lo
 apt-get update && apt-get install ipvsadm ipset -y
 export MASTER_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.90' | grep '192.168.80' | awk '{print $2}' | cut -f1 -d '/')
-curl -fL https://get.k3s.io | sh -s - server --node-ip=192.168.80.10 --disable servicelb --disable traefik --cluster-init --node-external-ip=192.168.80.80 --disable-cloud-controller --flannel-iface=eth1 --kube-proxy-arg proxy-mode=ipvs
+curl -fL https://get.k3s.io | sh -s - server --node-ip=192.168.80.10 --disable servicelb --disable traefik --cluster-init --node-external-ip=192.168.80.80 --disable-cloud-controller --flannel-iface=eth2 --kube-proxy-arg proxy-mode=ipvs
 kubectl taint nodes master1  node.cloudprovider.kubernetes.io/uninitialized:NoSchedule-
 curl -sfL https://github.com/loxilb-io/loxilb-ebpf/raw/main/kprobe/install.sh | sh -
 sleep 60
 echo $MASTER_IP > /vagrant/master-ip
 cp /var/lib/rancher/k3s/server/node-token /vagrant/node-token
+sed -i -e "s/127.0.0.1/192.168.80.80/g" /etc/rancher/k3s/k3s.yaml
 cp /etc/rancher/k3s/k3s.yaml /vagrant/k3s.yaml
-sed -i -e "s/127.0.0.1/192.168.80.80/g" /vagrant/k3s.yaml
 sudo mkdir -p /etc/loxilb
 sudo cp /vagrant/lbconfig.txt /etc/loxilb/
 sudo cp /vagrant/EPconfig.txt /etc/loxilb/

diff --git a/cicd/k3s-multi-master-service-proxy/master2.sh b/cicd/k3s-multi-master-service-proxy/master2.sh
@@ -7,7 +7,8 @@ sudo mkdir -p /etc/loxilb
 sudo cp /vagrant/lbconfig.txt /etc/loxilb/
 sudo cp /vagrant/EPconfig.txt /etc/loxilb/
 curl -sfL https://github.com/loxilb-io/loxilb-ebpf/raw/main/kprobe/install.sh | sh -
-curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.11 --node-external-ip=192.168.80.80 --disable-cloud-controller -t ${NODE_TOKEN} --flannel-iface=eth1 --kube-proxy-arg proxy-mode=ipvs
+curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.11 --node-external-ip=192.168.80.80 --disable-cloud-controller -t ${NODE_TOKEN} --flannel-iface=eth2 --kube-proxy-arg proxy-mode=ipvs
+sed -i -e "s/127.0.0.1/192.168.80.80/g" /etc/rancher/k3s/k3s.yaml
 kubectl taint nodes master2  node.cloudprovider.kubernetes.io/uninitialized:NoSchedule-
 /vagrant/wait_ready.sh
 sysctl net.core.netdev_max_backlog=10000
diff --git a/cicd/k3s-multi-master-service-proxy/master3.sh b/cicd/k3s-multi-master-service-proxy/master3.sh
@@ -7,8 +7,9 @@ sudo mkdir -p /etc/loxilb
 sudo cp /vagrant/lbconfig.txt /etc/loxilb/
 sudo cp /vagrant/EPconfig.txt /etc/loxilb/
 curl -sfL https://github.com/loxilb-io/loxilb-ebpf/raw/main/kprobe/install.sh | sh -
-curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.12 --node-external-ip=192.168.80.80 --disable-cloud-controller -t ${NODE_TOKEN} --flannel-iface=eth1 --kube-proxy-arg proxy-mode=ipvs
+curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.12 --node-external-ip=192.168.80.80 --disable-cloud-controller -t ${NODE_TOKEN} --flannel-iface=eth2 --kube-proxy-arg proxy-mode=ipvs
 kubectl taint nodes master2  node.cloudprovider.kubernetes.io/uninitialized:NoSchedule-
+sed -i -e "s/127.0.0.1/192.168.80.80/g" /etc/rancher/k3s/k3s.yaml
 sudo kubectl apply -f /vagrant/kube-loxilb.yml
 /vagrant/wait_ready.sh
 sysctl net.core.netdev_max_backlog=10000