From a742a7fa4da998d4cbc8b588743120684417573a Mon Sep 17 00:00:00 2001
From: Tom Hadlaw <tom.hadlaw@isovalent.com>
Date: Wed, 17 Jan 2024 16:24:48 -0800
Subject: [PATCH] Add conformance e2e workflow for Cilium on Talos.
Signed-off-by: Tom Hadlaw <tom.hadlaw@isovalent.com>
---
.github/workflows/conformance.yml | 289 ++++++++++++++++++
.gitignore | 8 +-
example/02-talos.tf | 8 +-
test/conformance/00-locals.tf | 21 ++
test/conformance/00-outputs.tf | 25 ++
test/conformance/00-providers.tf | 19 ++
test/conformance/00-variables.tf | 71 +++++
test/conformance/01-vpc.tf | 11 +
test/conformance/02-talos.tf | 18 ++
test/conformance/Makefile | 71 +++++
test/conformance/common.tfvars | 1 +
test/conformance/create-ci-env.sh | 102 +++++++
test/conformance/ipmasq-config.yaml | 10 +
.../scripts/post-cilium-install-script.sh | 5 +
.../scripts/pre-cilium-install-script.sh | 11 +
.../tests/cilium-connectivity-test.sh | 21 ++
test/conformance/values.yaml | 97 ++++++
test/conformance/wait | 14 +
18 files changed, 797 insertions(+), 5 deletions(-)
create mode 100644 .github/workflows/conformance.yml
create mode 100644 test/conformance/00-locals.tf
create mode 100644 test/conformance/00-outputs.tf
create mode 100644 test/conformance/00-providers.tf
create mode 100644 test/conformance/00-variables.tf
create mode 100644 test/conformance/01-vpc.tf
create mode 100644 test/conformance/02-talos.tf
create mode 100644 test/conformance/Makefile
create mode 100644 test/conformance/common.tfvars
create mode 100755 test/conformance/create-ci-env.sh
create mode 100644 test/conformance/ipmasq-config.yaml
create mode 100644 test/conformance/scripts/post-cilium-install-script.sh
create mode 100644 test/conformance/scripts/pre-cilium-install-script.sh
create mode 100755 test/conformance/tests/cilium-connectivity-test.sh
create mode 100644 test/conformance/values.yaml
create mode 100755 test/conformance/wait
diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml
new file mode 100644
index 0000000..fbe70d2
--- /dev/null
+++ b/.github/workflows/conformance.yml
@@ -0,0 +1,289 @@
+name: Talos Conformance
+on:
+ pull_request_target:
+ types:
+ - opened
+ - synchronize
+ - reopened
+ push:
+ branches:
+ - main
+ pull_request:
+jobs:
+ setup-and-test:
+ runs-on: ubuntu-22.04
+ permissions:
+ id-token: write
+ contents: read
+ strategy:
+ fail-fast: false
+ max-parallel: 4
+ matrix:
+ include:
+ # --- Cilium v1.15 ---
+
+ - name: 'v1.15'
+ cilium-version: "1.15.0-rc.0"
+ talos-version: 'v1.6.2'
+ kube-proxy: false
+ kube-proxy-replacement: "true"
+ socketlb: false
+ bpf-masquerade: true
+ ipam-mode: 'kubernetes'
+ ipv4: true
+ ipv6: false
+ encryption-enabled: false
+ encryption-type: ipsec
+ tunnel-mode: vxlan
+ nodeport: true
+
+ - name: 'v1.15 w wireguard'
+ cilium-version: "1.15.0-rc.0"
+ talos-version: 'v1.6.2'
+ kube-proxy: true
+ kube-proxy-replacement: "false"
+ socketlb: false
+ bpf-masquerade: true
+ ipam-mode: 'kubernetes'
+ ipv4: true
+ ipv6: false
+ encryption-enabled: true
+ encryption-type: wireguard
+ tunnel-mode: vxlan
+ nodeport: false
+
+ # TODO: fix ipsec failing tests due to nodeport not ready.
+ - name: 'v1.15 w ipsec'
+ cilium-version: "1.15.0-rc.0"
+ talos-version: 'v1.6.2'
+ kube-proxy: true
+ kube-proxy-replacement: "false"
+ socketlb: true
+ bpf-masquerade: false
+ ipam-mode: 'kubernetes'
+ ipv4: true
+ ipv6: false
+ encryption-enabled: true
+ encryption-type: ipsec
+ tunnel-mode: vxlan
+ nodeport: false
+
+ - name: 'v1.15 w no kpr & bpf-masq'
+ cilium-version: "1.15.0-rc.0"
+ talos-version: 'v1.6.2'
+ kube-proxy: true
+ kube-proxy-replacement: "false"
+ socketlb: true
+ bpf-masquerade: true
+ ipam-mode: 'kubernetes'
+ ipv4: true
+ ipv6: false
+ encryption-enabled: false
+ tunnel-mode: vxlan
+ nodeport: false
+
+ - name: 'v1.15 clusterpool ipam mode'
+ cilium-version: "1.15.0-rc.0"
+ talos-version: 'v1.6.2'
+ kube-proxy: false
+ kube-proxy-replacement: "true"
+ socketlb: false
+ bpf-masquerade: true
+ ipam-mode: 'cluster-pool'
+ ipv4: true
+ ipv6: false
+ encryption-enabled: false
+ encryption-type: ipsec
+ tunnel-mode: vxlan
+ nodeport: true
+
+ - name: 'v1.15 w geneve tunnel'
+ cilium-version: "1.15.0-rc.0"
+ talos-version: 'v1.6.2'
+ kube-proxy: false
+ kube-proxy-replacement: "true"
+ socketlb: false
+ bpf-masquerade: true
+ ipam-mode: 'kubernetes'
+ ipv4: true
+ ipv6: false
+ encryption-enabled: false
+ encryption-type: ipsec
+ tunnel-mode: geneve
+ nodeport: true
+
+ # --- Cilium v1.14 ---
+
+ - name: 'v1.14'
+ cilium-version: "1.14.7"
+ talos-version: 'v1.6.2'
+ kube-proxy: false
+ kube-proxy-replacement: "strict"
+ socketlb: false
+ bpf-masquerade: true
+ ipam-mode: 'kubernetes'
+ ipv4: true
+ ipv6: false
+ encryption-enabled: false
+ encryption-type: ipsec
+ tunnel-mode: vxlan
+ nodeport: true
+
+ - name: 'v1.14 w ipsec'
+ talos-version: 'v1.6.2'
+ kube-proxy: false
+ kube-proxy-replacement: "false"
+ socketlb: true
+ bpf-masquerade: false
+ ipam-mode: 'kubernetes'
+ ipv4: true
+ ipv6: false
+ encryption-enabled: true
+ encryption-type: ipsec
+ tunnel-mode: vxlan
+ nodeport: false
+
+ - name: 'v1.14 w wireguard'
+ cilium-version: "1.14.7"
+ talos-version: 'v1.6.2'
+ kube-proxy: false
+ kube-proxy-replacement: "strict"
+ socketlb: false
+ bpf-masquerade: true
+ ipam-mode: 'kubernetes'
+ ipv4: true
+ ipv6: false
+ encryption-enabled: false
+ encryption-type: wireguard
+ tunnel-mode: vxlan
+ nodeport: false
+
+ - name: 'v1.14 no KPR'
+ cilium-version: "1.14.7"
+ talos-version: 'v1.6.2'
+ kube-proxy: true
+ kube-proxy-replacement: "disabled"
+ socketlb: false
+ bpf-masquerade: true
+ ipam-mode: 'kubernetes'
+ ipv4: true
+ ipv6: false
+ encryption-enabled: false
+ encryption-type: ipsec
+ tunnel-mode: vxlan
+ nodeport: true
+
+ - name: 'v1.14 w clusterpool ipam mode'
+ cilium-version: "1.14.7"
+ talos-version: 'v1.6.2'
+ kube-proxy: false
+ kube-proxy-replacement: "strict"
+ socketlb: false
+ bpf-masquerade: true
+ ipam-mode: 'cluster-pool'
+ ipv4: true
+ ipv6: false
+ encryption-enabled: false
+ encryption-type: ipsec
+ tunnel-mode: vxlan
+ nodeport: true
+
+ - name: 'v1.14 w geneve tunnel'
+ cilium-version: "1.14.7"
+ talos-version: 'v1.6.2'
+ kube-proxy: false
+ kube-proxy-replacement: "strict"
+ socketlb: false
+ bpf-masquerade: true
+ ipam-mode: 'kubernetes'
+ ipv4: true
+ ipv6: false
+ encryption-enabled: false
+ encryption-type: ipsec
+ tunnel-mode: geneve
+ nodeport: true
+
+ steps:
+ - name: Checkout
+ uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+ with:
+ ref: ${{ github.event.pull_request.head.sha }}
+ - name: Configure AWS credentials from shared services account
+ uses: aws-actions/configure-aws-credentials@v2
+ with:
+ role-to-assume: arn:aws:iam::478566851380:role/TalosConformanceCI
+ aws-region: us-east-2
+ - uses: hashicorp/setup-terraform@v3
+ - name: Create Talos Cluster
+ run: |
+ cd test/conformance
+ ./create-ci-env.sh \
+ --kube-proxy ${{ matrix.kube-proxy}} \
+ --talos-version ${{ matrix.talos-version }} \
+ --owner "isovalent/terraform-aws-talos"
+ make apply
+ - name: Install Cilium CLI
+ uses: cilium/cilium-cli@4aa6347c532075df28027772fa1e4ec2f7415341 # v0.15.20
+ with:
+ repository: cilium/cilium-cli
+ release-version: v0.15.20
+ ci-version: ""
+ binary-name: cilium-cli
+ binary-dir: /usr/local/bin
+ - name: Install Cilium
+ run: |
+ cd test/conformance
+ export $(make print-kubeconfig)
+ kubectl create -n kube-system secret generic cilium-ipsec-keys \
+ --from-literal=keys="3 rfc4106(gcm(aes)) $(echo $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64)) 128"
+ kubectl create -n kube-system -f ipmasq-config.yaml
+ cilium-cli install --version="v1.15.0-rc.0" \
+ --values=values.yaml \
+ --set ipv4.enabled=${{ matrix.ipv4 }} \
+ --set ipv6.enabled=${{ matrix.ipv6 }} \
+ --set bpf.masquerade=${{ matrix.bpf-masquerade }} \
+ --set kubeProxyReplacement=${{ matrix.kube-proxy }} \
+ --set socketLB.enabled=${{ matrix.socketlb }} \
+ --set ipam.mode=${{ matrix.ipam-mode }} \
+ --set ingressController.enabled=true \
+ --set encryption.enabled=${{ matrix.encryption-enabled }} \
+ --set encryption.type=${{ matrix.encryption-type }} \
+ --set tunnelProtocol=${{ matrix.tunnel-mode }} \
+ --set nodePort.enabled=${{ matrix.nodeport }}
+ cilium-cli status --wait
+
+ - name: Run E2E Connectivity Tests
+ run: |
+ cd test/conformance
+ export $(make print-kubeconfig)
+ ./wait
+ kubectl create ns cilium-test
+ kubectl label ns cilium-test pod-security.kubernetes.io/enforce=privileged
+ kubectl label ns cilium-test pod-security.kubernetes.io/warn=privileged
+ cilium-cli connectivity test --debug
+
+ - name: Fetch artifacts
+ if: ${{ !success() && steps.run-tests.outcome != 'skipped' }}
+ shell: bash
+ run: |
+ cd test/conformance
+ export $(make print-kubeconfig)
+ kubectl get svc -o wide -A
+ kubectl get pods --all-namespaces -o wide
+ cilium-cli status
+ mkdir -p cilium-sysdumps
+ cilium-cli sysdump --output-filename cilium-sysdump-${{ github.run_id }}-${{ github.run_number }}
+
+ - name: Upload artifacts
+ if: ${{ !success() }}
+ uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
+ with:
+ name: cilium-sysdumps-${{ github.run_id }}-${{ github.run_number }}
+ path: ./test/conformance/cilium-sysdump-*.zip
+
+ - name: Cleanup
+ if: always()
+ run: |
+ cd test/conformance
+ make destroy
+
diff --git a/.gitignore b/.gitignore
index 4c0fc1b..78453ce 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,10 +1,16 @@
/.workspace-*/
.terraform/
+*/.terraform/*
+*/.terraform.lock.hcl
.terraform.lock.hcl
+*/.terraform.tfstate.lock.info
.terraform.tfstate.lock.info
.vscode/
+*/terraform.tfstate*
terraform.tfstate*
+*/terraform.tfvars
terraform.tfvars
tf/
*.DS_Store*
-.timestamp
\ No newline at end of file
+.timestamp
+test/conformance/env.tfvars
diff --git a/example/02-talos.tf b/example/02-talos.tf
index 38a4dd0..9b71d90 100644
--- a/example/02-talos.tf
+++ b/example/02-talos.tf
@@ -10,11 +10,11 @@ module "talos" {
region = var.region
tags = local.tags
# For single-node cluster support:
- #allow_workload_on_cp_nodes = true
- #controlplane_count = 1
- #workers_count = 0
+ allow_workload_on_cp_nodes = true
+ controlplane_count = 1
+ workers_count = 1
// VPC needs to be created in advance via https://github.com/isovalent/terraform-aws-vpc
vpc_id = module.vpc.id
pod_cidr = var.pod_cidr
service_cidr = var.service_cidr
-}
\ No newline at end of file
+}
diff --git a/test/conformance/00-locals.tf b/test/conformance/00-locals.tf
new file mode 100644
index 0000000..4354e24
--- /dev/null
+++ b/test/conformance/00-locals.tf
@@ -0,0 +1,21 @@
+locals {
+ expiry = file("${path.module}/.timestamp")
+ # The default tags defined here are merged with extra tags defined via var.tags in 00-variables.tf.
+ tags = merge(
+ tomap({
+ "expiry" : local.expiry,
+ "owner" : var.owner
+ }),
+ var.tags
+ )
+ extra_provisioner_environment_variables = {
+ CLUSTER_NAME = var.cluster_name
+ CLUSTER_ID = var.cluster_id
+ POD_CIDR = var.pod_cidr
+ SERVICE_CIDR = var.service_cidr
+ KUBECONFIG = module.talos.path_to_kubeconfig_file
+ # See https://www.talos.dev/v1.5/kubernetes-guides/network/deploying-cilium/
+ KUBE_APISERVER_HOST = "localhost"
+ KUBE_APISERVER_PORT = "7445"
+ }
+}
\ No newline at end of file
diff --git a/test/conformance/00-outputs.tf b/test/conformance/00-outputs.tf
new file mode 100644
index 0000000..7b044ca
--- /dev/null
+++ b/test/conformance/00-outputs.tf
@@ -0,0 +1,25 @@
+output "cluster_name" {
+ description = "Cluster name"
+ value = var.cluster_name
+}
+
+output "path_to_kubeconfig_file" {
+ description = "Path to the kubeconfig of the Talos Linux cluster"
+ value = module.talos.path_to_kubeconfig_file
+}
+
+output "path_to_talosconfig_file" {
+ description = "Path to the talosconfig of the Talos Linux cluster"
+ value = module.talos.path_to_talosconfig_file
+}
+
+output "elb_dns_name" {
+ description = "Public ELB DNS name."
+ value = module.talos.elb_dns_name
+}
+
+output "region" {
+ description = "AWS region used for the infra"
+ value = var.region
+}
+
diff --git a/test/conformance/00-providers.tf b/test/conformance/00-providers.tf
new file mode 100644
index 0000000..9d180c4
--- /dev/null
+++ b/test/conformance/00-providers.tf
@@ -0,0 +1,19 @@
+terraform {
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = ">= 5.0"
+ }
+ random = {
+ source = "hashicorp/random"
+ version = "~> 3.5"
+ }
+ }
+}
+
+provider "aws" {
+ region = var.region
+ default_tags {
+ tags = local.tags
+ }
+}
\ No newline at end of file
diff --git a/test/conformance/00-variables.tf b/test/conformance/00-variables.tf
new file mode 100644
index 0000000..a350497
--- /dev/null
+++ b/test/conformance/00-variables.tf
@@ -0,0 +1,71 @@
+# vpc module & general
+variable "cluster_name" {
+ default = "talos-cute"
+ description = "The name of the cluster."
+ type = string
+}
+
+variable "cluster_id" {
+ default = "1"
+ description = "The (Cilium) ID of the cluster. Must be unique for Cilium ClusterMesh and between 0-255."
+ type = number
+}
+
+variable "region" {
+ description = "The region in which to create the cluster."
+ type = string
+}
+
+variable "owner" {
+ description = "Owner for resource tagging"
+ type = string
+}
+
+variable "vpc_cidr" {
+ default = "10.0.0.0/16"
+ description = "The CIDR to use for the VPC. Currently it must be a /16 or /24."
+ type = string
+}
+
+variable "tags" {
+ default = {
+ usage = "cute",
+ platform = "talos"
+ }
+ description = "The set of tags to place on the created resources. These will be merged with the default tags defined via local.tags in 00-locals.tf."
+ type = map(string)
+}
+
+# talos module
+variable "talos_version" {
+ type = string
+ description = "Talos version to use for the cluster, if not set the newest Talos version. Check https://github.com/siderolabs/talos/releases for available releases."
+}
+
+variable "kubernetes_version" {
+ default = "1.27.3"
+ type = string
+ description = "Kubernetes version to use for the Talos cluster, if not set, the K8s version shipped with the selected Talos version will be used. Check https://www.talos.dev/v1.5/introduction/support-matrix/."
+}
+
+variable "service_cidr" {
+ default = "100.68.0.0/16"
+ description = "The CIDR to use for K8s Services"
+ type = string
+}
+
+variable "allocate_node_cidrs" {
+ description = "Whether to assign PodCIDRs to Node resources or not. Only needed in case Cilium runs in 'kubernetes' IPAM mode."
+ type = bool
+ default = false
+}
+
+variable "pod_cidr" {
+ default = "100.64.0.0/14"
+ description = "The CIDR to use for K8s Pods. Depending on if allocate_node_cidrs is set or not, it will either be configured on the controllerManager and assigned to Node resources or to CiliumNode CRs (in case Cilium runs with 'cluster-pool' IPAM mode)."
+ type = string
+}
+
+variable "disable_kube_proxy" {
+ type = bool
+}
diff --git a/test/conformance/01-vpc.tf b/test/conformance/01-vpc.tf
new file mode 100644
index 0000000..1f5dfd7
--- /dev/null
+++ b/test/conformance/01-vpc.tf
@@ -0,0 +1,11 @@
+resource "random_id" "cluster" {
+ byte_length = 4
+}
+
+module "vpc" {
+ source = "git::https://github.com/isovalent/terraform-aws-vpc.git?ref=v1.7"
+ cidr = var.vpc_cidr
+ name = "${var.cluster_name}-${random_id.cluster.dec}"
+ region = var.region
+ tags = local.tags
+}
diff --git a/test/conformance/02-talos.tf b/test/conformance/02-talos.tf
new file mode 100644
index 0000000..de1e476
--- /dev/null
+++ b/test/conformance/02-talos.tf
@@ -0,0 +1,18 @@
+module "talos" {
+ source = "../../"
+
+ // Supported Talos versions (and therefore K8s versions) can be found here: https://github.com/siderolabs/talos/releases
+ talos_version = var.talos_version
+ kubernetes_version = var.kubernetes_version
+ cluster_name = var.cluster_name
+ cluster_id = var.cluster_id
+ region = var.region
+ tags = local.tags
+ workers_count = 2
+
+ // VPC needs to be created in advance via https://github.com/isovalent/terraform-aws-vpc
+ vpc_id = module.vpc.id
+ pod_cidr = var.pod_cidr
+ service_cidr = var.service_cidr
+ disable_kube_proxy = var.disable_kube_proxy
+}
diff --git a/test/conformance/Makefile b/test/conformance/Makefile
new file mode 100644
index 0000000..fa240df
--- /dev/null
+++ b/test/conformance/Makefile
@@ -0,0 +1,71 @@
+SHELL := /bin/bash
+
+.PHONY: apply
+.DEFAULT_GOAL := help
+
+apply: AUTO_APPROVE ?= true
+apply: init
+apply:
+ terraform apply --auto-approve=$(AUTO_APPROVE) --var-file=common.tfvars --var-file=env.tfvars -input=false
+ @chmod 600 $$(terraform output --raw path_to_kubeconfig_file)
+
+## Destroys all infra originally created by the Terraform code
+destroy: AUTO_APPROVE ?= true
+destroy:
+ terraform destroy --auto-approve=$(AUTO_APPROVE) --var-file=common.tfvars --var-file=env.tfvars
+ @rm -f .timestamp
+
+## Initialize Terraform (installs modules, plugins, etc.)
+init:
+ @if [ ! -f .timestamp ]; then\
+ if [ "$$(uname)" == "Darwin" ]; then\
+ out=$$(date -u -v+3d '+%Y-%m-%d') && echo -n "$$out" > .timestamp;\
+ else\
+ out=$$(date -u -d '+3 days' '+%Y-%m-%d') && echo -n "$$out" > .timestamp;\
+ fi;\
+ fi
+ terraform init --upgrade
+
+## Runs Terraform plan
+plan:
+ terraform plan --var-file=common.tfvars --var-file=env.tfvars
+
+## Prints the export KUBECONFIG command for convenience
+print-kubeconfig:
+ @echo 'KUBECONFIG='$$(terraform output --raw path_to_kubeconfig_file)
+
+## Prints the export TALOSCONFIG command for convenience
+print-talosconfig:
+ @echo 'TALOSCONFIG='$$(terraform output --raw path_to_talosconfig_file)
+
+## Run infra tests from ./tests/* on cluster
+test:
+ # Wait script waits for all workloads on the cluster to stabilize, Talos has several CNI based workloads
+ # that are required for things such as kubectl exec/logs to work. So we want to ensure everything, not
+ # just Cilium, has stabilized before proceeding.
+ ./wait
+ ./tests/cilium-connectivity-test.sh
+
+# COLORS
+GREEN := $(shell tput -Txterm setaf 2)
+YELLOW := $(shell tput -Txterm setaf 3)
+WHITE := $(shell tput -Txterm setaf 7)
+RESET := $(shell tput -Txterm sgr0)
+
+TARGET_MAX_CHAR_NUM=20
+## Show this help
+help:
+ @echo ''
+ @echo 'Usage:'
+ @echo ' ${YELLOW}make${RESET} ${GREEN}<target>${RESET}'
+ @echo ''
+ @echo 'Targets:'
+ @awk '/^[a-zA-Z\-\_0-9]+:/ { \
+ helpMessage = match(lastLine, /^## (.*)/); \
+ if (helpMessage) { \
+ helpCommand = substr($$1, 0, index($$1, ":")-1); \
+ helpMessage = substr(lastLine, RSTART + 3, RLENGTH); \
+ printf " ${YELLOW}%-$(TARGET_MAX_CHAR_NUM)s${RESET} ${GREEN}%s${RESET}\n", helpCommand, helpMessage; \
+ } \
+ } \
+ { lastLine = $$0 }' $(MAKEFILE_LIST)
diff --git a/test/conformance/common.tfvars b/test/conformance/common.tfvars
new file mode 100644
index 0000000..917c3d1
--- /dev/null
+++ b/test/conformance/common.tfvars
@@ -0,0 +1 @@
+owner = "isovalent/terraform-aws-talos"
diff --git a/test/conformance/create-ci-env.sh b/test/conformance/create-ci-env.sh
new file mode 100755
index 0000000..5d40c0a
--- /dev/null
+++ b/test/conformance/create-ci-env.sh
@@ -0,0 +1,102 @@
+#!/bin/bash
+
+if [ -f env.tfvars ]; then
+ echo "env.tfvars already exists, please cleanup cluster and remove this file"
+fi
+
+pr_name=""
+kube_proxy=""
+talos_version=""
+owner=""
+
+# Function to display usage
+usage() {
+ echo "Usage: $0 [--kube-proxy <value>] [--version <value>] [--owner <owner_name>]"
+ exit 1
+}
+
+# Parse command line options
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --pr)
+ shift
+ if [ -n "$1" ] && [ ${1:0:1} != "-" ]; then
+ pr_name="$1"
+ else
+ echo "Error: Missing or invalid argument for --pr" >&2
+ usage
+ fi
+ ;;
+ --kube-proxy)
+ shift
+ if [ -n "$1" ] && [ ${1:0:1} != "-" ]; then
+ kube_proxy="$1"
+ else
+ echo "Error: Missing or invalid argument for --kube-proxy" >&2
+ usage
+ fi
+ ;;
+ --talos-version)
+ shift
+ if [ -n "$1" ] && [ ${1:0:1} != "-" ]; then
+ talos_version="$1"
+ else
+ echo "Error: Missing or invalid argument for --version" >&2
+ usage
+ fi
+ ;;
+ --owner)
+ shift
+ if [ -n "$1" ] && [ ${1:0:1} != "-" ]; then
+ owner="$1"
+ else
+ echo "Error: Missing or invalid argument for --owner" >&2
+ usage
+ fi
+ ;;
+ *)
+ echo "Error: Invalid option: $1" >&2
+ usage
+ ;;
+ esac
+ shift
+done
+
+# Display information based on flags
+if [ -n "$kube_proxy" ]; then
+ echo "Kube Proxy flag is set with value: $kube_proxy"
+fi
+
+if [ -n "$talos_version" ]; then
+ echo "Version flag is set with value: $talos_version"
+fi
+
+if [ -n "$owner" ]; then
+ echo "Owner flag is set with owner name: $owner"
+fi
+
+# If no flags are provided, display usage
+if [ -z "$kube_proxy" ] && [ -z "$talos_version" ] && [ -z "$owner" ]; then
+ usage
+fi
+
+
+pr_name=$(head -c 15 <<< ${pr_name})
+num=$(head -c 3 <<< ${RANDOM})
+
+disable_kube_proxy=false
+if [ ${kube_proxy} == "false" ]; then
+ disable_kube_proxy=true
+fi
+
+cat > env.tfvars << EOF
+cluster_name = "talos-e2e-${pr_name}-${num}"
+region = "us-east-2"
+owner = "${owner}"
+talos_version = "${talos_version}"
+disable_kube_proxy = ${disable_kube_proxy}
+EOF
+
+cat env.tfvars
+
+
diff --git a/test/conformance/ipmasq-config.yaml b/test/conformance/ipmasq-config.yaml
new file mode 100644
index 0000000..176fcfb
--- /dev/null
+++ b/test/conformance/ipmasq-config.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+data:
+ config: |
+ nonMasqueradeCIDRs:
+ - 10.0.0.0/16
+ masqLinkLocal: true
+kind: ConfigMap
+metadata:
+ name: ip-masq-agent
+ namespace: kube-system
diff --git a/test/conformance/scripts/post-cilium-install-script.sh b/test/conformance/scripts/post-cilium-install-script.sh
new file mode 100644
index 0000000..b0807ce
--- /dev/null
+++ b/test/conformance/scripts/post-cilium-install-script.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# TBD
\ No newline at end of file
diff --git a/test/conformance/scripts/pre-cilium-install-script.sh b/test/conformance/scripts/pre-cilium-install-script.sh
new file mode 100644
index 0000000..a125e1e
--- /dev/null
+++ b/test/conformance/scripts/pre-cilium-install-script.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+# Wait for the Kubernetes API server to be reachable.
+while ! kubectl get namespace > /dev/null 2>&1;
+do
+ sleep 10
+done
+
+# TBD
\ No newline at end of file
diff --git a/test/conformance/tests/cilium-connectivity-test.sh b/test/conformance/tests/cilium-connectivity-test.sh
new file mode 100755
index 0000000..5ef2ae3
--- /dev/null
+++ b/test/conformance/tests/cilium-connectivity-test.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+ns=cilium-test
+
+# Grab the path to the kubeconfig file.
+export KUBECONFIG=$(terraform output --raw path_to_kubeconfig_file)
+
+# Run the connectivity tests.
+kubectl -n kube-system port-forward svc/hubble-relay 4245:80 &
+# NS precreation is required because of https://www.talos.dev/v1.5/kubernetes-guides/configuration/pod-security/
+kubectl create ns ${ns}
+kubectl label ns ${ns} pod-security.kubernetes.io/enforce=privileged
+kubectl label ns ${ns} pod-security.kubernetes.io/warn=privileged
+PID=$!
+set +e
+cilium connectivity test --test-namespace=${ns}
+set -e
+kill -9 "${PID}"
+kubectl delete ns ${ns}
diff --git a/test/conformance/values.yaml b/test/conformance/values.yaml
new file mode 100644
index 0000000..9e02148
--- /dev/null
+++ b/test/conformance/values.yaml
@@ -0,0 +1,97 @@
+# Base set of values common across e2e test scnearios.
+# Please see .github/workflows/conformance.yaml for specific installation
+# input values.
+image:
+ pullPolicy: Always
+
+k8s:
+ requireIPv4PodCIDR: true
+ requireIPv6PodCIDR: false
+
+
+devices:
+ - eth0
+
+enableIPv4Masquerade: true
+enableIPv6Masquerade: true
+
+debug:
+ enabled: true
+
+# Configure unique cluster name & ID (used for ClusterMesh in the future)
+cluster:
+ id: 1
+
+# Routing/encapsulation mode
+tunnelProtocol: vxlan
+tunnel: vxlan
+routingMode: tunnel
+
+k8sServiceHost: "localhost"
+k8sServicePort: "7445"
+
+# -- Monitoring and Flow Visibility
+
+# Enable Cilium Hubble to gain visibility
+hubble:
+ enabled: true
+ metrics:
+ enabled:
+ # https://docs.cilium.io/en/stable/observability/metrics/#hubble-exported-metrics
+ # Remove `;query` from the `dns` line for production -> bad metrics cardinalitreproy
+ - dns:labelsContext=source_namespace,destination_namespace;query
+ - drop:labelsContext=source_namespace,destination_namespace
+ - tcp:labelsContext=source_namespace,destination_namespace
+ - port-distribution:labelsContext=source_namespace,destination_namespace
+ - icmp:labelsContext=source_namespace,destination_namespace;sourceContext=workload-name|reserved-identity;destinationContext=workload-name|reserved-identity
+ - flow:sourceContext=workload-name|reserved-identity;destinationContext=workload-name|reserved-identity;labelsContext=source_namespace,destination_namespace
+ - "httpV2:exemplars=true;labelsContext=source_ip,source_namespace,source_workload,destination_ip,destination_namespace,destination_workload,traffic_direction;sourceContext=workload-name|reserved-identity;destinationContext=workload-name|reserved-identity"
+ - "policy:sourceContext=app|workload-name|pod|reserved-identity;destinationContext=app|workload-name|pod|dns|reserved-identity;labelsContext=source_namespace,destination_namespace"
+ relay:
+ enabled: true
+
+# Enable Cilium Operator metrics
+operator:
+ prometheus:
+ enabled: true
+
+# Enable Cilium Agent metrics
+prometheus:
+ enabled: true
+
+# -- Talos Linux Specifics
+# https://www.talos.dev/v1.5/kubernetes-guides/network/deploying-cilium/
+
+# Configure IPAM/PodCIDR
+ipam:
+ # Ensure to not assign PodCIDRs to Nodes via "controllerManager.extraArgs.allocate-node-cidrs" in case you want to use Cilium's default IPAM mode, cluster-pool, instead of 'kubernetes'. Use the provided 'allocate_node_cidrs' Terraform variable to steer this behaviour.
+ operator:
+ clusterPoolIPv4PodCIDRList:
+ - "100.64.0.0/14"
+
+# Configure cgroup related configuration
+cgroup:
+ # Cilium doesn't need to mount the cgroup2 filesystem, as it's already there on Talos Linux. Let's simply mount the correct path from Talos.
+ autoMount:
+ enabled: false
+ hostRoot: /sys/fs/cgroup
+
+# Remove "SYS_MODULE" because of https://www.talos.dev/v1.5/learn-more/process-capabilities/
+securityContext:
+ capabilities:
+ ciliumAgent:
+ - CHOWN
+ - KILL
+ - NET_ADMIN
+ - NET_RAW
+ - IPC_LOCK
+ - SYS_ADMIN
+ - SYS_RESOURCE
+ - DAC_OVERRIDE
+ - FOWNER
+ - SETGID
+ - SETUID
+ cleanCiliumState:
+ - NET_ADMIN
+ - SYS_ADMIN
+ - SYS_RESOURCE
diff --git a/test/conformance/wait b/test/conformance/wait
new file mode 100755
index 0000000..6b280d4
--- /dev/null
+++ b/test/conformance/wait
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+for i in {1..10}; do
+ echo "Checking for non running Pods"
+ count=$(kubectl get pod --field-selector status.phase!=Running -A -o json | jq '.items | length')
+ if [ ${count} == "0" ]; then
+ echo "Done (count=${count})!"
+ exit 0
+ fi
+ echo "Not ready (count=${count}), waiting 30s..."
+ sleep 30
+done
+echo "Timed out waiting"
+exit 1