Add conformance e2e workflow for Cilium on Talos.

Signed-off-by: Tom Hadlaw <tom.hadlaw@isovalent.com>

.github/workflows/conformance.yml

@@ -0,0 +1,289 @@
+name: Talos Conformance
+on:
+  pull_request_target:
+    types:
+      - opened
+      - synchronize
+      - reopened
+  push:
+    branches:
+      - main
+  pull_request:
+jobs:
+  setup-and-test:
+    runs-on: ubuntu-22.04
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        include:
+          # --- Cilium v1.15 ---
+
+          - name: 'v1.15'
+            cilium-version: "1.15.0-rc.0"
+            talos-version: 'v1.6.2'
+            kube-proxy: false
+            kube-proxy-replacement: "true"
+            socketlb: false 
+            bpf-masquerade: true
+            ipam-mode: 'kubernetes'
+            ipv4: true
+            ipv6: false
+            encryption-enabled: false
+            encryption-type: ipsec
+            tunnel-mode: vxlan
+            nodeport: true
+
+          - name: 'v1.15 w wireguard'
+            cilium-version: "1.15.0-rc.0"
+            talos-version: 'v1.6.2'
+            kube-proxy: true 
+            kube-proxy-replacement: "false"
+            socketlb: false 
+            bpf-masquerade: true
+            ipam-mode: 'kubernetes'
+            ipv4: true
+            ipv6: false
+            encryption-enabled: true
+            encryption-type: wireguard
+            tunnel-mode: vxlan
+            nodeport: false
+
+          # TODO: fix ipsec failing tests due to nodeport not ready.
+          - name: 'v1.15 w ipsec'
+            cilium-version: "1.15.0-rc.0"
+            talos-version: 'v1.6.2'
+            kube-proxy: true 
+            kube-proxy-replacement: "false"
+            socketlb: true
+            bpf-masquerade: false
+            ipam-mode: 'kubernetes'
+            ipv4: true
+            ipv6: false
+            encryption-enabled: true
+            encryption-type: ipsec 
+            tunnel-mode: vxlan
+            nodeport: false
+
+          - name: 'v1.15 w no kpr & bpf-masq'
+            cilium-version: "1.15.0-rc.0"
+            talos-version: 'v1.6.2'
+            kube-proxy: true
+            kube-proxy-replacement: "false"
+            socketlb: true
+            bpf-masquerade: true
+            ipam-mode: 'kubernetes'
+            ipv4: true
+            ipv6: false
+            encryption-enabled: false
+            tunnel-mode: vxlan
+            nodeport: false
+
+          - name: 'v1.15 clusterpool ipam mode'
+            cilium-version: "1.15.0-rc.0"
+            talos-version: 'v1.6.2'
+            kube-proxy: false
+            kube-proxy-replacement: "true"
+            socketlb: false 
+            bpf-masquerade: true
+            ipam-mode: 'cluster-pool'
+            ipv4: true
+            ipv6: false
+            encryption-enabled: false
+            encryption-type: ipsec
+            tunnel-mode: vxlan
+            nodeport: true
+
+          - name: 'v1.15 w geneve tunnel'
+            cilium-version: "1.15.0-rc.0"
+            talos-version: 'v1.6.2'
+            kube-proxy: false
+            kube-proxy-replacement: "true"
+            socketlb: false 
+            bpf-masquerade: true
+            ipam-mode: 'kubernetes'
+            ipv4: true
+            ipv6: false
+            encryption-enabled: false
+            encryption-type: ipsec
+            tunnel-mode: geneve 
+            nodeport: true
+
+          # --- Cilium v1.14 --- 
+
+          - name: 'v1.14'
+            cilium-version: "1.14.7"
+            talos-version: 'v1.6.2'
+            kube-proxy: false 
+            kube-proxy-replacement: "strict"
+            socketlb: false 
+            bpf-masquerade: true
+            ipam-mode: 'kubernetes'
+            ipv4: true
+            ipv6: false
+            encryption-enabled: false
+            encryption-type: ipsec
+            tunnel-mode: vxlan
+            nodeport: true
+
+          - name: 'v1.14 w ipsec'
+            talos-version: 'v1.6.2'
+            kube-proxy: false
+            kube-proxy-replacement: "false"
+            socketlb: true
+            bpf-masquerade: false
+            ipam-mode: 'kubernetes'
+            ipv4: true
+            ipv6: false
+            encryption-enabled: true
+            encryption-type: ipsec 
+            tunnel-mode: vxlan
+            nodeport: false
+
+          - name: 'v1.14 w wireguard'
+            cilium-version: "1.14.7"
+            talos-version: 'v1.6.2'
+            kube-proxy: false
+            kube-proxy-replacement: "strict"
+            socketlb: false 
+            bpf-masquerade: true
+            ipam-mode: 'kubernetes'
+            ipv4: true
+            ipv6: false
+            encryption-enabled: false
+            encryption-type: wireguard
+            tunnel-mode: vxlan
+            nodeport: false
+
+          - name: 'v1.14 no KPR'
+            cilium-version: "1.14.7"
+            talos-version: 'v1.6.2'
+            kube-proxy: true
+            kube-proxy-replacement: "disabled"
+            socketlb: false
+            bpf-masquerade: true
+            ipam-mode: 'kubernetes'
+            ipv4: true
+            ipv6: false
+            encryption-enabled: false
+            encryption-type: ipsec
+            tunnel-mode: vxlan
+            nodeport: true
+
+          - name: 'v1.14 w clusterpool ipam mode'
+            cilium-version: "1.14.7"
+            talos-version: 'v1.6.2'
+            kube-proxy: false
+            kube-proxy-replacement: "strict"
+            socketlb: false 
+            bpf-masquerade: true
+            ipam-mode: 'cluster-pool'
+            ipv4: true
+            ipv6: false
+            encryption-enabled: false
+            encryption-type: ipsec
+            tunnel-mode: vxlan
+            nodeport: true
+
+          - name: 'v1.14 w geneve tunnel'
+            cilium-version: "1.14.7"
+            talos-version: 'v1.6.2'
+            kube-proxy: false
+            kube-proxy-replacement: "strict"
+            socketlb: false 
+            bpf-masquerade: true
+            ipam-mode: 'kubernetes'
+            ipv4: true
+            ipv6: false
+            encryption-enabled: false
+            encryption-type: ipsec
+            tunnel-mode: geneve
+            nodeport: true
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Configure AWS credentials from shared services account
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: arn:aws:iam::478566851380:role/TalosConformanceCI
+          aws-region: us-east-2
+      - uses: hashicorp/setup-terraform@v3
+      - name: Create Talos Cluster 
+        run: |
+          cd test/conformance
+          ./create-ci-env.sh \
+            --kube-proxy ${{ matrix.kube-proxy}} \
+            --talos-version ${{ matrix.talos-version }} \
+            --owner "isovalent/terraform-aws-talos"
+          make apply
+      - name: Install Cilium CLI
+        uses: cilium/cilium-cli@4aa6347c532075df28027772fa1e4ec2f7415341 # v0.15.20
+        with:
+          repository: cilium/cilium-cli
+          release-version: v0.15.20
+          ci-version: ""
+          binary-name: cilium-cli
+          binary-dir: /usr/local/bin
+      - name: Install Cilium
+        run: |
+          cd test/conformance
+          export $(make print-kubeconfig)
+          kubectl create -n kube-system secret generic cilium-ipsec-keys \
+            --from-literal=keys="3 rfc4106(gcm(aes)) $(echo $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64)) 128"
+          kubectl create -n kube-system -f ipmasq-config.yaml
+          cilium-cli install --version="v1.15.0-rc.0" \
+            --values=values.yaml \
+            --set ipv4.enabled=${{ matrix.ipv4 }} \
+            --set ipv6.enabled=${{ matrix.ipv6 }} \
+            --set bpf.masquerade=${{ matrix.bpf-masquerade }} \
+            --set kubeProxyReplacement=${{ matrix.kube-proxy }} \
+            --set socketLB.enabled=${{ matrix.socketlb }} \
+            --set ipam.mode=${{ matrix.ipam-mode }} \
+            --set ingressController.enabled=true \
+            --set encryption.enabled=${{ matrix.encryption-enabled }} \
+            --set encryption.type=${{ matrix.encryption-type }} \
+            --set tunnelProtocol=${{ matrix.tunnel-mode }} \
+            --set nodePort.enabled=${{ matrix.nodeport }}
+          cilium-cli status --wait
+
+      - name: Run E2E Connectivity Tests 
+        run: |
+          cd test/conformance
+          export $(make print-kubeconfig)
+          ./wait
+          kubectl create ns cilium-test 
+          kubectl label ns cilium-test pod-security.kubernetes.io/enforce=privileged
+          kubectl label ns cilium-test pod-security.kubernetes.io/warn=privileged
+          cilium-cli connectivity test --debug
+
+      - name: Fetch artifacts
+        if: ${{ !success() && steps.run-tests.outcome != 'skipped' }}
+        shell: bash
+        run: |
+          cd test/conformance
+          export $(make print-kubeconfig)
+          kubectl get svc -o wide -A
+          kubectl get pods --all-namespaces -o wide
+          cilium-cli status
+          mkdir -p cilium-sysdumps
+          cilium-cli sysdump --output-filename cilium-sysdump-${{ github.run_id }}-${{ github.run_number }}
+
+      - name: Upload artifacts
+        if: ${{ !success() }}
+        uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
+        with:
+          name: cilium-sysdumps-${{ github.run_id }}-${{ github.run_number }}
+          path: ./test/conformance/cilium-sysdump-*.zip
+
+      - name: Cleanup
+        if: always()
+        run: |
+          cd test/conformance
+          make destroy 
+

.gitignore

@@ -1,10 +1,16 @@
 /.workspace-*/
 .terraform/
+*/.terraform/*
+*/.terraform.lock.hcl
 .terraform.lock.hcl
+*/.terraform.tfstate.lock.info
 .terraform.tfstate.lock.info
 .vscode/
+*/terraform.tfstate*
 terraform.tfstate*
+*/terraform.tfvars
 terraform.tfvars
 tf/
 *.DS_Store*
-.timestamp
+.timestamp
+test/conformance/env.tfvars

example/02-talos.tf

@@ -10,11 +10,11 @@ module "talos" {
   region             = var.region
   tags               = local.tags
   # For single-node cluster support:
-  #allow_workload_on_cp_nodes = true
-  #controlplane_count         = 1
-  #workers_count              = 0
+  allow_workload_on_cp_nodes = true
+  controlplane_count         = 1
+  workers_count              = 1
   // VPC needs to be created in advance via https://github.com/isovalent/terraform-aws-vpc
   vpc_id       = module.vpc.id
   pod_cidr     = var.pod_cidr
   service_cidr = var.service_cidr
-}
+}

test/conformance/00-locals.tf

@@ -0,0 +1,21 @@
+locals {
+  expiry = file("${path.module}/.timestamp")
+  # The default tags defined here are merged with extra tags defined via var.tags in 00-variables.tf.
+  tags = merge(
+    tomap({
+      "expiry" : local.expiry,
+      "owner" : var.owner
+    }),
+    var.tags
+  )
+  extra_provisioner_environment_variables = {
+    CLUSTER_NAME = var.cluster_name
+    CLUSTER_ID   = var.cluster_id
+    POD_CIDR     = var.pod_cidr
+    SERVICE_CIDR = var.service_cidr
+    KUBECONFIG   = module.talos.path_to_kubeconfig_file
+    # See https://www.talos.dev/v1.5/kubernetes-guides/network/deploying-cilium/ 
+    KUBE_APISERVER_HOST = "localhost"
+    KUBE_APISERVER_PORT = "7445"
+  }
+}

test/conformance/00-outputs.tf

@@ -0,0 +1,25 @@
+output "cluster_name" {
+  description = "Cluster name"
+  value       = var.cluster_name
+}
+
+output "path_to_kubeconfig_file" {
+  description = "Path to the kubeconfig of the Talos Linux cluster"
+  value       = module.talos.path_to_kubeconfig_file
+}
+
+output "path_to_talosconfig_file" {
+  description = "Path to the talosconfig of the Talos Linux cluster"
+  value       = module.talos.path_to_talosconfig_file
+}
+
+output "elb_dns_name" {
+  description = "Public ELB DNS name."
+  value       = module.talos.elb_dns_name
+}
+
+output "region" {
+  description = "AWS region used for the infra"
+  value       = var.region
+}
+

test/conformance/00-providers.tf

@@ -0,0 +1,19 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.5"
+    }
+  }
+}
+
+provider "aws" {
+  region = var.region
+  default_tags {
+    tags = local.tags
+  }
+}