Skip to content

Commit

Permalink
Cilium bigbang (#1057)
Browse files Browse the repository at this point in the history
* fix(vpn-gateway): updated secret name on envfrom

* feat(cilium): switch calico+metallb with cilium
  • Loading branch information
thiagoalmeidasa authored Sep 23, 2024
1 parent 8d1ea44 commit c8c07fd
Show file tree
Hide file tree
Showing 47 changed files with 682 additions and 176 deletions.
5 changes: 4 additions & 1 deletion .gitleaks.toml.disabled → .gitleaks.toml
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
[rules.allowlist]
[extend]
useDefault = true

[allowlist]
description = "Skip encrypted secrets"
regexes = [
'''(?i)!!enveloped:''', # Encrypted secrets
Expand Down
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ repos:
- .yamllint.yaml
id: yamllint
- repo: https://github.com/tarioch/flux-check-hook
rev: v0.5.0
rev: v0.4.0
hooks:
- id: check-flux-helm-values
- repo: https://github.com/pre-commit/pre-commit-hooks
Expand Down
16 changes: 12 additions & 4 deletions ansible/inventory/group_vars/kubernetes/k3s.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,23 @@ k3s_use_unsupported_config: true
# (string) Control Plane registration address
k3s_registration_address: "{{ kubevip_address }}"

k3s_registries:
mirrors:
docker.io:
gcr.io:
ghcr.io:
k8s.gcr.io:
lscr.io:
mcr.microsoft.com:
public.ecr.aws:
quay.io:
registry.k8s.io:

# (list) A list of URLs to deploy on the primary control plane. Read notes below.
k3s_server_manifests_urls:
# Kube-vip
- url: https://kube-vip.io/manifests/rbac.yaml
filename: custom-kube-vip-rbac.yaml
# Tigera Operator
- url: https://raw.githubusercontent.com/projectcalico/calico/v3.25.1/manifests/tigera-operator.yaml
filename: custom-calico-tigera-operator.yaml
# Prometheus Operator
- url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml
filename: custom-prometheus-alertmanagerconfigs.yaml
Expand All @@ -59,5 +68,4 @@ k3s_server_manifests_urls:
# (list) A flat list of templates to deploy on the primary control plane
# /var/lib/rancher/k3s/server/manifests
k3s_server_manifests_templates:
- custom-calico-installation.yaml.j2
- custom-kube-vip-daemonset.yaml.j2
12 changes: 11 additions & 1 deletion ansible/inventory/group_vars/master/k3s.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@ k3s_server:
node-ip: "{{ ansible_host }}"
tls-san:
- "{{ kubevip_address }}"
embedded-registry: true
# Disable Docker - this will use the default containerd CRI
docker: false
flannel-backend: "none" # This needs to be in quotes
flannel-backend: "none" # This needs to be in quotes
disable:
# Disable flannel - replaced with Calico
- flannel
Expand All @@ -24,9 +25,15 @@ k3s_server:
- servicelb
# Disable traefik - replaced with ingress-nginx and installed with Flux
- traefik
# Disable coredns - replaced with helmfile coredns
- coredns
# Disable kube-proxy - replaced with cilium
- kube-proxy
disable-network-policy: true
disable-cloud-controller: true
write-kubeconfig-mode: "644"
disable-kube-proxy: true

# Network CIDR to use for pod IPs
cluster-cidr: "10.42.0.0/16"
# Network CIDR to use for service IPs
Expand All @@ -45,3 +52,6 @@ k3s_server:
kube-apiserver-arg:
# Required for HAProxy health-checks
- "anonymous-auth=true"
kubelet-arg:
- "kube-reserved=cpu=200m,memory=256Mi"
- "system-reserved=cpu=200m,memory=256Mi"
24 changes: 0 additions & 24 deletions ansible/playbooks/templates/custom-calico-installation.yaml.j2

This file was deleted.

2 changes: 2 additions & 0 deletions ansible/playbooks/templates/custom-kube-vip-daemonset.yaml.j2
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ spec:
value: "10"
- name: vip_retryperiod
value: "2"
- name: lb_enable
value: "true"
- name: address
value: "{{ k3s_registration_address }}"
securityContext:
Expand Down
113 changes: 69 additions & 44 deletions kubernetes/apps/home/mosquitto/app/helmrelease.yaml
Original file line number Diff line number Diff line change
@@ -1,75 +1,100 @@
---
# yaml-language-server: $schema=https://kubernetes-schemas.devbu.io/helm.toolkit.fluxcd.io/helmrelease_v2beta1.json
# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2.schema.json
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: mosquitto
namespace: home
name: &app mosquitto
spec:
interval: 15m
interval: 30m
chart:
spec:
chart: app-template
version: 1.5.1
version: 3.4.0
sourceRef:
kind: HelmRepository
name: bjw-s
namespace: flux-system
maxHistory: 2
install:
createNamespace: true
remediation:
retries: 3
upgrade:
cleanupOnFail: false
cleanupOnFail: true
remediation:
strategy: rollback
retries: 3
uninstall:
keepHistory: false
dependsOn:
- name: openebs
namespace: storage
values:
controller:
type: statefulset
annotations:
reloader.stakater.com/auto: "true"
image:
repository: public.ecr.aws/docker/library/eclipse-mosquitto
tag: 2.0.18
controllers:
mosquitto:
annotations:
reloader.stakater.com/auto: "true"
initContainers:
init-config:
image:
repository: public.ecr.aws/docker/library/eclipse-mosquitto
tag: 2.0.18
pullPolicy: IfNotPresent
command: ["/bin/sh", "-c"]
args: ["cp /tmp/secret/* /mosquitto/external_config/ && mosquitto_passwd -U /mosquitto/external_config/mosquitto_pwd"]
containers:
app:
image:
repository: public.ecr.aws/docker/library/eclipse-mosquitto
tag: 2.0.18
resources:
requests:
cpu: 5m
limits:
memory: 16Mi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: { drop: ["ALL"] }
defaultPodOptions:
securityContext:
runAsNonRoot: true
runAsUser: 568
runAsGroup: 568
fsGroup: 568
fsGroupChangePolicy: OnRootMismatch
seccompProfile: { type: RuntimeDefault }
service:
main:
app:
controller: *app
type: LoadBalancer
annotations:
metallb.universe.tf/allow-shared-ip: shared
externalIPs: ["${SVC_MQTT_ADDR}"]
externalTrafficPolicy: Local
io.cilium/lb-ipam-ips: ${SVC_MQTT_ADDR:=temp}
ports:
http:
enabled: false
mqtt:
enabled: true
port: 1883
podSecurityContext:
runAsUser: 568
runAsGroup: 568
fsGroup: 568
fsGroupChangePolicy: OnRootMismatch
persistence:
config-file:
data:
accessMode: ReadWriteOnce
enabled: true
type: persistentVolumeClaim
storageClass: longhorn
size: 1Gi
retain: true
globalMounts:
- path: /data
config-file:
type: configMap
name: mosquitto-configmap
subPath: mosquitto.conf
mountPath: /mosquitto/config/mosquitto.conf
readOnly: true
name: mosquitto-configmap # overriden by kustomizeconfig
advancedMounts:
mosquitto:
app:
- path: /mosquitto/config/mosquitto.conf
subPath: mosquitto.conf
secret-file:
enabled: true
type: secret
name: mosquitto-secret
subPath: mosquitto.pwd
mountPath: /mosquitto/config/mosquitto.pwd
readOnly: true
resources:
requests:
cpu: 5m
memory: 15Mi
limits:
memory: 50Mi
advancedMounts:
mosquitto:
init-config:
- path: /tmp/secret
external-config:
type: emptyDir
globalMounts:
- path: /mosquitto/external_config
20 changes: 11 additions & 9 deletions kubernetes/apps/home/mosquitto/app/secret.sops.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ metadata:
name: mosquitto-secret
namespace: home
stringData:
mosquitto.pwd: ENC[AES256_GCM,data:14ZmSNSnzwQqJb6ikL149ePPf8x4U8UHvVjrfGh0LLhBB7xroih80vr+b8QMVaZPmOfs06hDUjz0tt3LNDiYln86INxZ7LFT0DQPmTq9mys95V0911or4gEVilK/tq6zItU3C8HM7mEyIitY4/iK3hnKLS7D0TZesnI=,iv:BXxVxspgeE9WdlG9RR3ycLG8XPO9WaI3Bb5FfvOTWFM=,tag:pCXeW8m0Y+BvBWwrnj4roA==,type:str]
username: ENC[AES256_GCM,data:32kHpYoA843cilmp9w==,iv:xjLdXcZKP9mMFpR1doHfoU70upns5SH8koWNL7eGMWE=,tag:avY6dMWL0Vk0obNMKn4bFQ==,type:str]
password: ENC[AES256_GCM,data:Nee9cP1mgExuKUI9TMvhCcQtOPwhq/wIv5/QhJriwMx7u4/JIBUILHfXSQG4g+ylpkNa1fG0LnTjOmldzJmNXgCfP9cU9yVDIyHjhUN4tPTT2Xw71IapYdSKe9aSsyanlJ9RF6IlDwOuruIh,iv:o9deLrnrixxFeLokCdi0UV50phun32r3PSLhJ7JjKH0=,tag:3yBxiF6oCKMpGFHGsDUQuw==,type:str]
mosquitto.pwd: ENC[AES256_GCM,data:oHiT4UwkRjYXDJo+Jh5qQ3BhebuxedVrVCeubq8XY7y6tynNHft6yCUeQh8b6awJ3iapUwqBF/dfsCUbsxOVh6OqtcUBlgGbINFBRK6nmDzOPsadxpBTIB4BUudwNsQBlKCcoEpk4xdEiTyCAKMe/Yz7kegNIL2qLec=,iv:v/VbwMb9XJt/F538CyNvMgFqdTKernTTTIqKZKErRNw=,tag:RDe0caPf2M5WzHu9c7rVuQ==,type:str]
sops:
kms: []
gcp_kms: []
Expand All @@ -18,14 +20,14 @@ sops:
- recipient: age1w02zzfg0y4ast9mgnd9w0yuym0wqx6q967kmrmq355w4cnw0xytq2x369r
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvYmZQOUNQdmcyTTFxc05v
b2I5Z0xyandOV2lKeUo5YWRMTXpFWlhqbFd3CmVRSHZJRkNtZzRaZTJPL1ZXdDJv
eFh6TW5TMGFKaEc5TW9MbE82ZXRjTlEKLS0tIGZXMFVmZEZQZEtmRXdVMWZVTXRp
M2VhNkxaVERtb3BqaE1rVDhsc2NOa3MKMWqHaQqj2bTKXfjwhh6LuDPkb63smWxT
o5veJ0JsWR997zGhv+pgkjmKx9SKlLFh+rOC5TtzClVrpNCUA+P/lQ==
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVQW42Z2E0VFo0cTEwRzdw
cHBzS1JzUVZDTGsyTlZCY3pOYnNZOEF0TGdrCjE1cG1sUGtEYWFEWGNXS2FPMXVZ
UHpLTlN2NjNORHpqVXhUb1Y5TjZEYVEKLS0tIHBkSUZ6UFRSVHdldDc3dDh6TUkx
N1BqYVc1OFFuMDdEV0Z2amw2M2U3SGcKNzpgeGgJ55c/rxVbOTLGas0/lghAOyC2
dvbhJMndeVFp17BxcCCqhVH81DimSB2ur5HPoEeyxAgNRjk41SPf+Q==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2023-05-25T21:17:00Z"
mac: ENC[AES256_GCM,data:AWalySC7G7QN4U4m8Tq3zrv0KNIlSVA5JFuzuiR+xUfrOHGQOoOSLbpRibo4BudnTjfN9JRIQymhtGvurIJSPwpiK3a8LNR/6amty05lql1rDc+Sa3ESmPVb6afp/AYwKFB3fkBGTs7a4ZsUeClv83ZDUBr8QPXXn1YokUEQkS8=,iv:8/RK6Qkq4D4uVO3ovg2x+RXC+vnk/V84LD+/ILou63U=,tag:0RawHRNBvFbMtl9YC97oAw==,type:str]
lastmodified: "2024-09-23T21:45:20Z"
mac: ENC[AES256_GCM,data:sJQq+h4oQw2MbfPufXEffygvDeT3v0jEngCAPZnpBvo6MHoX3X6abLjdIkKQNvn0hi13L2qeZz7PdgY4iZLq8/tuEG8gH+MTeQb49vGTpgykbkTNQOFFoAd/NAbsqUc/2e6ivWToTJu3/0v0FRNP0MXoffdco9A6VCRTYqkZgE4=,iv:1C4seGEAEJ3qoWRUDfpc9rGrD53vWw/L/6LdftU61Rg=,tag:4B0Kd1VD36i49WBvVrukdA==,type:str]
pgp: []
encrypted_regex: ^(data|stringData)$
version: 3.7.3
version: 3.8.1
57 changes: 57 additions & 0 deletions kubernetes/apps/kube-system/cilium/app/helm-values.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
---
autoDirectNodeRoutes: true
bpf:
masquerade: false # Required for Talos `.machine.features.hostDNS.forwardKubeDNSToHost`
cgroup:
automount:
enabled: false
hostRoot: /sys/fs/cgroup
cluster:
id: 1
name: "homelab"
cni:
exclusive: false
# NOTE: devices might need to be set if you have more than one active NIC on your hosts
# devices: eno+ eth+
endpointRoutes:
enabled: true
envoy:
enabled: false
hubble:
enabled: false
ipam:
mode: kubernetes
ipv4NativeRoutingCIDR: "10.42.0.0/16"
k8sServiceHost: 127.0.0.1
k8sServicePort: 6444
kubeProxyReplacement: true
kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
l2announcements:
enabled: true
loadBalancer:
algorithm: maglev
mode: dsr
localRedirectPolicy: true
operator:
replicas: 1
rollOutPods: true
rollOutCiliumPods: true
routingMode: native
securityContext:
capabilities:
ciliumAgent:
- CHOWN
- KILL
- NET_ADMIN
- NET_RAW
- IPC_LOCK
- SYS_ADMIN
- SYS_RESOURCE
- DAC_OVERRIDE
- FOWNER
- SETGID
- SETUID
cleanCiliumState:
- NET_ADMIN
- SYS_ADMIN
- SYS_RESOURCE
Loading

0 comments on commit c8c07fd

Please sign in to comment.