test: Add Calico Inbound and Outbound policies to LKE nodes for E2E (#525)

* Add calico rules script and update workflow files

* add condition always

* revert ci.yml

Author: ykim-akamai

.github/workflows/ci.yml

@@ -76,4 +76,4 @@ jobs:
           python tod_scripts/test_report_upload_script.py "$REPORT_FILENAME"
         env:
           LINODE_CLI_OBJ_ACCESS_KEY: ${{ secrets.LINODE_CLI_OBJ_ACCESS_KEY }}
-          LINODE_CLI_OBJ_SECRET_KEY: ${{ secrets.LINODE_CLI_OBJ_SECRET_KEY }}
+          LINODE_CLI_OBJ_SECRET_KEY: ${{ secrets.LINODE_CLI_OBJ_SECRET_KEY }}

.github/workflows/integration_tests_pr.yml

@@ -36,6 +36,14 @@ jobs:
         with:
           ref: ${{ inputs.sha }}
 
+      - name: Download kubectl and calicoctl for LKE clusters
+        run: |
+          curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl"
+          curl -LO "https://github.com/projectcalico/calico/releases/download/v3.25.0/calicoctl-linux-amd64"
+          chmod +x calicoctl-linux-amd64 kubectl
+          mv calicoctl-linux-amd64 /usr/local/bin/calicoctl
+          mv kubectl /usr/local/bin/kubectl
+
       - run: make ARGS="-run ${{ inputs.module }}" fixtures
         if: ${{ inputs.module != '' && steps.disallowed-char-check.outputs.match == '' }}
         env:
@@ -44,6 +52,13 @@ jobs:
         if: ${{ inputs.module == '' }}
         env:
           LINODE_TOKEN: ${{ secrets.DX_LINODE_TOKEN }}
+
+      - name: Apply Calico Rules to LKE
+        if: always()
+        run: |
+          cd scripts && ./lke_calico_rules_e2e.sh
+        env:
+          LINODE_TOKEN: ${{ secrets.DX_LINODE_TOKEN }}
 
       - name: Get the hash value of the latest commit from the PR branch
         uses: octokit/[email protected]

scripts/lke-policy.yaml

@@ -0,0 +1,78 @@
+apiVersion: projectcalico.org/v3
+kind: GlobalNetworkPolicy
+metadata:
+  name: lke-rules
+spec:
+  preDNAT: true
+  applyOnForward: true
+  order: 100
+  # Remember to run calicoctl patch command for this to work
+  selector: ""
+  ingress:
+    # Allow ICMP
+    - action: Allow
+      protocol: ICMP
+    - action: Allow
+      protocol: ICMPv6
+
+    # Allow LKE-required ports
+    - action: Allow
+      protocol: TCP
+      destination:
+        nets:
+          - 192.168.128.0/17
+          - 10.0.0.0/8
+        ports:
+          - 10250
+          - 10256
+          - 179
+    - action: Allow
+      protocol: UDP
+      destination:
+        nets:
+          - 192.168.128.0/17
+          - 10.2.0.0/16
+        ports:
+          - 51820
+
+    # Allow NodeBalancer ingress to the Node Ports & Allow DNS
+    - action: Allow
+      protocol: TCP
+      source:
+        nets:
+          - 192.168.255.0/24
+          - 10.0.0.0/8
+      destination:
+        ports:
+          - 53
+          - 30000:32767
+    - action: Allow
+      protocol: UDP
+      source:
+        nets:
+          - 192.168.255.0/24
+          - 10.0.0.0/8
+      destination:
+        ports:
+          - 53
+          - 30000:32767
+
+    # Allow cluster internal communication
+    - action: Allow
+      destination:
+        nets:
+          - 10.0.0.0/8
+    - action: Allow
+      source:
+        nets:
+          - 10.0.0.0/8
+
+    # 127.0.0.1/32 is needed for kubectl exec and node-shell
+    - action: Allow
+      destination:
+        nets:
+          - 127.0.0.1/32
+
+    # Block everything else
+    - action: Deny
+    - action: Log

scripts/lke_calico_rules_e2e.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+RETRIES=3
+DELAY=30
+
+# Function to retry a command with exponential backoff
+retry_command() {
+    local retries=$1
+    local wait_time=60
+    shift
+    until "$@"; do
+        if ((retries == 0)); then
+            echo "Command failed after multiple retries. Exiting."
+            exit 1
+        fi
+        echo "Command failed. Retrying in $wait_time seconds..."
+        sleep $wait_time
+        ((retries--))
+        wait_time=$((wait_time * 2))
+    done
+}
+
+# Fetch the list of LKE cluster IDs
+CLUSTER_IDS=$(curl -s -H "Authorization: Bearer $LINODE_TOKEN" \
+    -H "Content-Type: application/json" \
+    "https://api.linode.com/v4/lke/clusters" | jq -r '.data[].id')
+
+# Check if CLUSTER_IDS is empty
+if [ -z "$CLUSTER_IDS" ]; then
+    echo "All clusters have been cleaned and properly destroyed. No need to apply inbound or outbound rules"
+    exit 0
+fi
+
+for ID in $CLUSTER_IDS; do
+    echo "Applying Calico rules to nodes in Cluster ID: $ID"
+
+    # Download cluster configuration file with retry
+    for ((i=1; i<=RETRIES; i++)); do
+        config_response=$(curl -sH "Authorization: Bearer $LINODE_TOKEN" "https://api.linode.com/v4/lke/clusters/$ID/kubeconfig")
+        if [[ $config_response != *"kubeconfig is not yet available"* ]]; then
+            echo $config_response | jq -r '.[] | @base64d' > "/tmp/${ID}_config.yaml"
+            break
+        fi
+        echo "Attempt $i to download kubeconfig for cluster $ID failed. Retrying in $DELAY seconds..."
+        sleep $DELAY
+    done
+
+    if [[ $config_response == *"kubeconfig is not yet available"* ]]; then
+        echo "kubeconfig for cluster id:$ID not available after $RETRIES attempts, mostly likely it is an empty cluster. Skipping..."
+    else
+        # Export downloaded config file
+        export KUBECONFIG="/tmp/${ID}_config.yaml"
+
+        retry_command $RETRIES kubectl get nodes
+
+        retry_command $RETRIES calicoctl patch kubecontrollersconfiguration default --allow-version-mismatch --patch='{"spec": {"controllers": {"node": {"hostEndpoint": {"autoCreate": "Enabled"}}}}}'
+
+        retry_command $RETRIES calicoctl apply --allow-version-mismatch -f "$(pwd)/lke-policy.yaml"
+    fi
+done