[wip]

.github/workflows/conformance.yml

@@ -0,0 +1,147 @@
+name: Talos Conformance
+on:
+  pull_request_target:
+    types:
+      - opened
+      - synchronize
+      - reopened
+  push:
+    branches:
+      - main
+  pull_request:
+jobs:
+  setup-and-test:
+    runs-on: ubuntu-22.04
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        include:
+          - name: 'v1.15 on Talos v1.5 w KPR'
+            cilium-version: "1.15.0-rc.0"
+            talos-version: 'v1.5.3'
+            kube-proxy: true 
+            kube-proxy-replacement: "strict"
+            socketlb: false 
+            bpf-masquerade: true
+            ipam-mode: 'kubernetes'
+            ipv4: true
+            ipv6: false
+
+          - name: 'v1.15 on Talos v1.5 w KPR'
+            cilium-version: "1.15.0-rc.0"
+            talos-version: 'v1.5.3'
+            kube-proxy: true 
+            kube-proxy-replacement: "disabled"
+            socketlb: false 
+            bpf-masquerade: true
+            ipam-mode: 'kubernetes'
+            ipv4: true
+            ipv6: false
+
+          - name: 'v1.14 on Talos v1.5 w KPR'
+            cilium-version: "1.14.7"
+            talos-version: 'v1.5.3'
+            kube-proxy: true 
+            kube-proxy-replacement: "strict"
+            socketlb: false 
+            bpf-masquerade: true
+            ipam-mode: 'kubernetes'
+            ipv4: true
+            ipv6: false
+
+          - name: 'v1.14 on Talos v1.5 no KPR'
+            cilium-version: "1.14.7"
+            talos-version: 'v1.5.3'
+            kube-proxy: true
+            kube-proxy-replacement: "disabled"
+            socketlb: false
+            bpf-masquerade: true
+            ipam-mode: 'kubernetes'
+            ipv4: true
+            ipv6: false
+    steps:
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Configure AWS credentials from shared services account
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: arn:aws:iam::478566851380:role/TalosConformanceCI
+          aws-region: us-east-2
+      - uses: hashicorp/setup-terraform@v3
+      - name: Create Talos Cluster 
+        run: |
+          cd test/conformance
+          ./create-ci-env.sh \
+            --kube-proxy ${{ matrix.kube-proxy}} \
+            --talos-version ${{ matrix.talos-version }} \
+            --owner "isovalent/terraform-aws-talos"
+          make apply
+      - name: Install Cilium CLI
+        uses: cilium/cilium-cli@4aa6347c532075df28027772fa1e4ec2f7415341 # v0.15.20
+        with:
+          repository: cilium/cilium-cli
+          release-version: v0.15.20
+          ci-version: ""
+          binary-name: cilium-cli
+          binary-dir: /usr/local/bin
+      - name: Install Cilium Enterprise
+        run: |
+          cd test/conformance
+          export $(make print-kubeconfig)
+          helm repo add isovalent https://helm.isovalent.com
+          helm repo update
+          helm install cilium isovalent/cilium-enterprise \
+            --namespace=kube-system \
+            --version ${{ matrix.cilium-version }} \
+            --values v2.yaml \
+            --set cilium.cluster.name=$(terraform output --raw cluster_name) \
+            --values=values.yaml \
+            --set ipv4.enabled=${{ matrix.ipv4 }} \
+            --set ipv6.enabled=${{ matrix.ipv6 }} \
+            --set bpf.masquerade=${{ matrix.bpf-masquerade }} \
+            --set kubeProxyReplacement=${{ matrix.kube-proxy-replacement }} \
+            --set socketLB.enabled=${{ matrix.socketlb }} \
+            --set ipam.mode=${{ matrix.ipam-mode }} \
+            --set ingressController.enabled=true
+          cilium-cli status --wait
+
+      - name: Run E2E Connectivity Tests 
+        run: |
+          cd test/conformance
+          export $(make print-kubeconfig)
+          kubectl create ns cilium-test 
+          kubectl label ns cilium-test pod-security.kubernetes.io/enforce=privileged
+          kubectl label ns cilium-test pod-security.kubernetes.io/warn=privileged
+          cilium-cli connectivity test
+
+      - name: Fetch artifacts
+        if: ${{ !success() && steps.run-tests.outcome != 'skipped' }}
+        shell: bash
+        run: |
+          cd test/conformance
+          export $(make print-kubeconfig)
+          kubectl logs -n kube-system ds/cilium
+          kubectl get pods --all-namespaces -o wide
+          cilium-cli status
+          mkdir -p cilium-sysdumps
+          cilium-cli sysdump --output-filename cilium-sysdump-${{ matrix.name }}-final
+
+      - name: Upload artifacts
+        if: ${{ !success() }}
+        uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
+        with:
+          name: cilium-sysdumps-${{ matrix.name }}
+          path: cilium-sysdump-*.zip
+
+      - name: Cleanup
+        if: always()
+        run: |
+          cd test/conformance
+          make destroy 
+

.gitignore

@@ -1,10 +1,16 @@
 /.workspace-*/
 .terraform/
+*/.terraform/*
+*/.terraform.lock.hcl
 .terraform.lock.hcl
+*/.terraform.tfstate.lock.info
 .terraform.tfstate.lock.info
 .vscode/
+*/terraform.tfstate*
 terraform.tfstate*
+*/terraform.tfvars
 terraform.tfvars
 tf/
 *.DS_Store*
-.timestamp
+.timestamp
+test/conformance/env.tfvars

README.md

@@ -49,10 +49,10 @@ module "talos" {
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | ~> 5.0 |
-| <a name="provider_local"></a> [local](#provider\_local) | n/a |
-| <a name="provider_null"></a> [null](#provider\_null) | n/a |
-| <a name="provider_random"></a> [random](#provider\_random) | ~> 3.5 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | 5.32.1 |
+| <a name="provider_local"></a> [local](#provider\_local) | 2.4.1 |
+| <a name="provider_null"></a> [null](#provider\_null) | 3.2.2 |
+| <a name="provider_random"></a> [random](#provider\_random) | 3.6.0 |
 | <a name="provider_talos"></a> [talos](#provider\_talos) | 0.4.0-alpha.0 |
 
 ### Modules

example/common.tfvars

@@ -0,0 +1,2 @@
+owner = "tom.hadlaw"
+region = "us-west-2"

test/conformance/00-locals.tf

@@ -0,0 +1,21 @@
+locals {
+  expiry = file("${path.module}/.timestamp")
+  # The default tags defined here are merged with extra tags defined via var.tags in 00-variables.tf.
+  tags = merge(
+    tomap({
+      "expiry" : local.expiry,
+      "owner" : var.owner
+    }),
+    var.tags
+  )
+  extra_provisioner_environment_variables = {
+    CLUSTER_NAME = var.cluster_name
+    CLUSTER_ID   = var.cluster_id
+    POD_CIDR     = var.pod_cidr
+    SERVICE_CIDR = var.service_cidr
+    KUBECONFIG   = module.talos.path_to_kubeconfig_file
+    # See https://www.talos.dev/v1.5/kubernetes-guides/network/deploying-cilium/ 
+    KUBE_APISERVER_HOST = "localhost"
+    KUBE_APISERVER_PORT = "7445"
+  }
+}

test/conformance/00-outputs.tf

@@ -0,0 +1,25 @@
+output "cluster_name" {
+  description = "Cluster name"
+  value       = var.cluster_name
+}
+
+output "path_to_kubeconfig_file" {
+  description = "Path to the kubeconfig of the Talos Linux cluster"
+  value       = module.talos.path_to_kubeconfig_file
+}
+
+output "path_to_talosconfig_file" {
+  description = "Path to the talosconfig of the Talos Linux cluster"
+  value       = module.talos.path_to_talosconfig_file
+}
+
+output "elb_dns_name" {
+  description = "Public ELB DNS name."
+  value       = module.talos.elb_dns_name
+}
+
+output "region" {
+  description = "AWS region used for the infra"
+  value       = var.region
+}
+

test/conformance/00-providers.tf

@@ -0,0 +1,19 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.5"
+    }
+  }
+}
+
+provider "aws" {
+  region = var.region
+  default_tags {
+    tags = local.tags
+  }
+}

test/conformance/00-variables.tf

@@ -0,0 +1,71 @@
+# vpc module & general
+variable "cluster_name" {
+  default     = "talos-cute"
+  description = "The name of the cluster."
+  type        = string
+}
+
+variable "cluster_id" {
+  default     = "1"
+  description = "The (Cilium) ID of the cluster. Must be unique for Cilium ClusterMesh and between 0-255."
+  type        = number
+}
+
+variable "region" {
+  description = "The region in which to create the cluster."
+  type        = string
+}
+
+variable "owner" {
+  description = "Owner for resource tagging"
+  type        = string
+}
+
+variable "vpc_cidr" {
+  default     = "10.0.0.0/16"
+  description = "The CIDR to use for the VPC. Currently it must be a /16 or /24."
+  type        = string
+}
+
+variable "tags" {
+  default = {
+    usage    = "cute",
+    platform = "talos"
+  }
+  description = "The set of tags to place on the created resources. These will be merged with the default tags defined via local.tags in 00-locals.tf."
+  type        = map(string)
+}
+
+# talos module
+variable "talos_version" {
+  type        = string
+  description = "Talos version to use for the cluster, if not set the newest Talos version. Check https://github.com/siderolabs/talos/releases for available releases."
+}
+
+variable "kubernetes_version" {
+  default     = "1.27.3"
+  type        = string
+  description = "Kubernetes version to use for the Talos cluster, if not set, the K8s version shipped with the selected Talos version will be used. Check https://www.talos.dev/v1.5/introduction/support-matrix/."
+}
+
+variable "service_cidr" {
+  default     = "100.68.0.0/16"
+  description = "The CIDR to use for K8s Services"
+  type        = string
+}
+
+variable "allocate_node_cidrs" {
+  description = "Whether to assign PodCIDRs to Node resources or not. Only needed in case Cilium runs in 'kubernetes' IPAM mode."
+  type        = bool
+  default     = false
+}
+
+variable "pod_cidr" {
+  default     = "100.64.0.0/14"
+  description = "The CIDR to use for K8s Pods. Depending on if allocate_node_cidrs is set or not, it will either be configured on the controllerManager and assigned to Node resources or to CiliumNode CRs (in case Cilium runs with 'cluster-pool' IPAM mode)."
+  type        = string
+}
+
+variable "disable_kube_proxy" {
+  type = bool
+}

test/conformance/01-vpc.tf

@@ -0,0 +1,17 @@
+// Create the VPC.
+
+resource "random_id" "cluster" {
+  byte_length = 4
+}
+
+module "vpc" {
+  source = "git::https://github.com/isovalent/terraform-aws-vpc.git?ref=v1.7"
+
+  # TODO: Do we need to support beta features, like ipv6 masquerade?
+  #enable_ipv6         = true
+  #vpc_ipv6_cidr_block = "2600:1f18:abcd:1234::/56"
+  cidr   = var.vpc_cidr
+  name   = "${var.cluster_name}-${random_id.cluster.dec}"
+  region = var.region
+  tags   = local.tags
+}

test/conformance/02-talos.tf

@@ -0,0 +1,21 @@
+module "talos" {
+  #source = "git::https://github.com/isovalent/terraform-aws-talos?ref=<RELEASE_TAG>"
+  source = "../../"
+
+  // Supported Talos versions (and therefore K8s versions) can be found here: https://github.com/siderolabs/talos/releases
+  #talos_version      = var.talos_version
+  kubernetes_version = var.kubernetes_version
+  cluster_name       = var.cluster_name
+  cluster_id         = var.cluster_id
+  region             = var.region
+  tags               = local.tags
+  # For single-node cluster support:
+  #allow_workload_on_cp_nodes = true
+  #controlplane_count         = 1
+  #workers_count              = 0
+  // VPC needs to be created in advance via https://github.com/isovalent/terraform-aws-vpc
+  vpc_id             = module.vpc.id
+  pod_cidr           = var.pod_cidr
+  service_cidr       = var.service_cidr
+  disable_kube_proxy = var.disable_kube_proxy
+}