From 67c624425ba5ec9ba72e424adee5a9ea5f56c9a2 Mon Sep 17 00:00:00 2001 From: Dimitri Koshkin Date: Wed, 14 Aug 2024 03:41:36 -0700 Subject: [PATCH] fix: Cilium-Istio compatibility fixes (#856) **What problem does this PR solve?**: Setting `cni.exclusive: false` and `socketLB.hostNamespaceOnly: true`. See Cilium's docs https://docs.cilium.io/en/latest/network/servicemesh/istio/. Without these value Cilium can interfere with Istio functionality, by always cleaning up cni config directory on the host, thus preventing Istio Pods from coming up, and interfere with Istio load-balancing once they do come up. It's safe to always set these value, because it is not Cilium's responsibility to prevent other applications from acting like a network plugin. **Which issue(s) this PR fixes**: Fixes # **How Has This Been Tested?**: **Special notes for your reviewer**: --------- Co-authored-by: Jimmi Dyson --- .../templates/cni/cilium/manifests/cilium-configmap.yaml | 2 +- .../cni/cilium/manifests/helm-addon-installation.yaml | 3 +++ hack/addons/kustomize/cilium/helm-values.yaml | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/cilium-configmap.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/cilium-configmap.yaml index b6fe39b14..cd4e8623c 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/cilium-configmap.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/cilium-configmap.yaml @@ -8,7 +8,7 @@ apiVersion: v1 data: cilium.json: | - [{"apiVersion":"v1","kind":"ServiceAccount","metadata":{"name":"cilium","namespace":"kube-system"}},{"apiVersion":"v1","kind":"ServiceAccount","metadata":{"name":"cilium-operator","namespace":"kube-system"}},{"apiVersion":"v1","data":{"agent-not-ready-taint-key":"node.cilium.io/agent-not-ready","arping-refresh-period":"30s","auto-direct-node-routes":"false","bpf-lb-acceleration":"disabled","bpf-lb-external-clusterip":"false","bpf-lb-map-max":"65536","bpf-lb-sock":"false","bpf-map-dynamic-size-ratio":"0.0025","bpf-policy-map-max":"16384","bpf-root":"/sys/fs/bpf","cgroup-root":"/run/cilium/cgroupv2","cilium-endpoint-gc-interval":"5m0s","cluster-id":"0","cluster-name":"default","cni-chaining-mode":"portmap","cni-exclusive":"true","cni-log-file":"/var/run/cilium/cilium-cni.log","custom-cni-conf":"false","debug":"false","debug-verbose":"","egress-gateway-reconciliation-trigger-interval":"1s","enable-auto-protect-node-port-range":"true","enable-bgp-control-plane":"false","enable-bpf-clock-probe":"false","enable-endpoint-health-checking":"true","enable-external-ips":"false","enable-health-check-loadbalancer-ip":"false","enable-health-check-nodeport":"true","enable-health-checking":"true","enable-host-legacy-routing":"true","enable-host-port":"false","enable-ipv4":"true","enable-ipv4-big-tcp":"false","enable-ipv4-masquerade":"true","enable-ipv6":"false","enable-ipv6-big-tcp":"false","enable-ipv6-masquerade":"true","enable-k8s-networkpolicy":"true","enable-k8s-terminating-endpoint":"true","enable-l2-neigh-discovery":"true","enable-l7-proxy":"true","enable-local-redirect-policy":"false","enable-masquerade-to-route-source":"false","enable-metrics":"true","enable-node-port":"false","enable-policy":"default","enable-remote-node-identity":"true","enable-sctp":"false","enable-svc-source-range-check":"true","enable-vtep":"false","enable-well-known-identities":"false","enable-xt-socket-fallback":"true","external-envoy-proxy":"false","identity-allocation-mode":"crd","identity-gc-interval":"15m0s","identity-heartbeat-timeout":"30m0s","install-no-conntrack-iptables-rules":"false","ipam":"kubernetes","ipam-cilium-node-update-rate":"15s","k8s-client-burst":"20","k8s-client-qps":"10","kube-proxy-replacement":"false","kube-proxy-replacement-healthz-bind-address":"","max-connected-clusters":"255","mesh-auth-enabled":"true","mesh-auth-gc-interval":"5m0s","mesh-auth-queue-size":"1024","mesh-auth-rotated-identities-queue-size":"1024","monitor-aggregation":"medium","monitor-aggregation-flags":"all","monitor-aggregation-interval":"5s","node-port-bind-protection":"true","nodes-gc-interval":"5m0s","operator-api-serve-addr":"127.0.0.1:9234","operator-prometheus-serve-addr":":9963","policy-cidr-match-mode":"","preallocate-bpf-maps":"false","procfs":"/host/proc","proxy-connect-timeout":"2","proxy-idle-timeout-seconds":"60","proxy-max-connection-duration-seconds":"0","proxy-max-requests-per-connection":"0","proxy-prometheus-port":"9964","proxy-xff-num-trusted-hops-egress":"0","proxy-xff-num-trusted-hops-ingress":"0","remove-cilium-node-taints":"true","routing-mode":"tunnel","service-no-backend-response":"reject","set-cilium-is-up-condition":"true","set-cilium-node-taints":"true","sidecar-istio-proxy-image":"cilium/istio_proxy","skip-cnp-status-startup-clean":"false","synchronize-k8s-nodes":"true","tofqdns-dns-reject-response-code":"refused","tofqdns-enable-dns-compression":"true","tofqdns-endpoint-max-ip-per-hostname":"50","tofqdns-idle-connection-grace-period":"0s","tofqdns-max-deferred-connection-deletes":"10000","tofqdns-proxy-response-max-delay":"100ms","tunnel-protocol":"vxlan","unmanaged-pod-watcher-interval":"15","vtep-cidr":"","vtep-endpoint":"","vtep-mac":"","vtep-mask":"","write-cni-conf-when-ready":"/host/etc/cni/net.d/05-cilium.conflist"},"kind":"ConfigMap","metadata":{"name":"cilium-config","namespace":"kube-system"}},{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRole","metadata":{"labels":{"app.kubernetes.io/part-of":"cilium"},"name":"cilium"},"rules":[{"apiGroups":["networking.k8s.io"],"resources":["networkpolicies"],"verbs":["get","list","watch"]},{"apiGroups":["discovery.k8s.io"],"resources":["endpointslices"],"verbs":["get","list","watch"]},{"apiGroups":[""],"resources":["namespaces","services","pods","endpoints","nodes"],"verbs":["get","list","watch"]},{"apiGroups":["apiextensions.k8s.io"],"resources":["customresourcedefinitions"],"verbs":["list","watch","get"]},{"apiGroups":["cilium.io"],"resources":["ciliumloadbalancerippools","ciliumbgppeeringpolicies","ciliumbgpnodeconfigs","ciliumbgpadvertisements","ciliumbgppeerconfigs","ciliumclusterwideenvoyconfigs","ciliumclusterwidenetworkpolicies","ciliumegressgatewaypolicies","ciliumendpoints","ciliumendpointslices","ciliumenvoyconfigs","ciliumidentities","ciliumlocalredirectpolicies","ciliumnetworkpolicies","ciliumnodes","ciliumnodeconfigs","ciliumcidrgroups","ciliuml2announcementpolicies","ciliumpodippools"],"verbs":["list","watch"]},{"apiGroups":["cilium.io"],"resources":["ciliumidentities","ciliumendpoints","ciliumnodes"],"verbs":["create"]},{"apiGroups":["cilium.io"],"resources":["ciliumidentities"],"verbs":["update"]},{"apiGroups":["cilium.io"],"resources":["ciliumendpoints"],"verbs":["delete","get"]},{"apiGroups":["cilium.io"],"resources":["ciliumnodes","ciliumnodes/status"],"verbs":["get","update"]},{"apiGroups":["cilium.io"],"resources":["ciliumnetworkpolicies/status","ciliumclusterwidenetworkpolicies/status","ciliumendpoints/status","ciliumendpoints","ciliuml2announcementpolicies/status","ciliumbgpnodeconfigs/status"],"verbs":["patch"]}]},{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRole","metadata":{"labels":{"app.kubernetes.io/part-of":"cilium"},"name":"cilium-operator"},"rules":[{"apiGroups":[""],"resources":["pods"],"verbs":["get","list","watch","delete"]},{"apiGroups":[""],"resources":["nodes"],"verbs":["list","watch"]},{"apiGroups":[""],"resources":["nodes","nodes/status"],"verbs":["patch"]},{"apiGroups":["discovery.k8s.io"],"resources":["endpointslices"],"verbs":["get","list","watch"]},{"apiGroups":[""],"resources":["services/status"],"verbs":["update","patch"]},{"apiGroups":[""],"resources":["namespaces"],"verbs":["get","list","watch"]},{"apiGroups":[""],"resources":["services","endpoints"],"verbs":["get","list","watch"]},{"apiGroups":["cilium.io"],"resources":["ciliumnetworkpolicies","ciliumclusterwidenetworkpolicies"],"verbs":["create","update","deletecollection","patch","get","list","watch"]},{"apiGroups":["cilium.io"],"resources":["ciliumnetworkpolicies/status","ciliumclusterwidenetworkpolicies/status"],"verbs":["patch","update"]},{"apiGroups":["cilium.io"],"resources":["ciliumendpoints","ciliumidentities"],"verbs":["delete","list","watch"]},{"apiGroups":["cilium.io"],"resources":["ciliumidentities"],"verbs":["update"]},{"apiGroups":["cilium.io"],"resources":["ciliumnodes"],"verbs":["create","update","get","list","watch","delete"]},{"apiGroups":["cilium.io"],"resources":["ciliumnodes/status"],"verbs":["update"]},{"apiGroups":["cilium.io"],"resources":["ciliumendpointslices","ciliumenvoyconfigs","ciliumbgppeerconfigs","ciliumbgpadvertisements","ciliumbgpnodeconfigs"],"verbs":["create","update","get","list","watch","delete","patch"]},{"apiGroups":["apiextensions.k8s.io"],"resources":["customresourcedefinitions"],"verbs":["create","get","list","watch"]},{"apiGroups":["apiextensions.k8s.io"],"resourceNames":["ciliumloadbalancerippools.cilium.io","ciliumbgppeeringpolicies.cilium.io","ciliumbgpclusterconfigs.cilium.io","ciliumbgppeerconfigs.cilium.io","ciliumbgpadvertisements.cilium.io","ciliumbgpnodeconfigs.cilium.io","ciliumbgpnodeconfigoverrides.cilium.io","ciliumclusterwideenvoyconfigs.cilium.io","ciliumclusterwidenetworkpolicies.cilium.io","ciliumegressgatewaypolicies.cilium.io","ciliumendpoints.cilium.io","ciliumendpointslices.cilium.io","ciliumenvoyconfigs.cilium.io","ciliumexternalworkloads.cilium.io","ciliumidentities.cilium.io","ciliumlocalredirectpolicies.cilium.io","ciliumnetworkpolicies.cilium.io","ciliumnodes.cilium.io","ciliumnodeconfigs.cilium.io","ciliumcidrgroups.cilium.io","ciliuml2announcementpolicies.cilium.io","ciliumpodippools.cilium.io"],"resources":["customresourcedefinitions"],"verbs":["update"]},{"apiGroups":["cilium.io"],"resources":["ciliumloadbalancerippools","ciliumpodippools","ciliumbgpclusterconfigs","ciliumbgpnodeconfigoverrides"],"verbs":["get","list","watch"]},{"apiGroups":["cilium.io"],"resources":["ciliumpodippools"],"verbs":["create"]},{"apiGroups":["cilium.io"],"resources":["ciliumloadbalancerippools/status"],"verbs":["patch"]},{"apiGroups":["coordination.k8s.io"],"resources":["leases"],"verbs":["create","get","update"]}]},{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRoleBinding","metadata":{"labels":{"app.kubernetes.io/part-of":"cilium"},"name":"cilium"},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":"cilium"},"subjects":[{"kind":"ServiceAccount","name":"cilium","namespace":"kube-system"}]},{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRoleBinding","metadata":{"labels":{"app.kubernetes.io/part-of":"cilium"},"name":"cilium-operator"},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":"cilium-operator"},"subjects":[{"kind":"ServiceAccount","name":"cilium-operator","namespace":"kube-system"}]},{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"Role","metadata":{"labels":{"app.kubernetes.io/part-of":"cilium"},"name":"cilium-config-agent","namespace":"kube-system"},"rules":[{"apiGroups":[""],"resources":["configmaps"],"verbs":["get","list","watch"]}]},{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"RoleBinding","metadata":{"labels":{"app.kubernetes.io/part-of":"cilium"},"name":"cilium-config-agent","namespace":"kube-system"},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"cilium-config-agent"},"subjects":[{"kind":"ServiceAccount","name":"cilium","namespace":"kube-system"}]},{"apiVersion":"apps/v1","kind":"DaemonSet","metadata":{"labels":{"app.kubernetes.io/name":"cilium-agent","app.kubernetes.io/part-of":"cilium","k8s-app":"cilium"},"name":"cilium","namespace":"kube-system"},"spec":{"selector":{"matchLabels":{"k8s-app":"cilium"}},"template":{"metadata":{"annotations":{"container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites":"unconfined","container.apparmor.security.beta.kubernetes.io/cilium-agent":"unconfined","container.apparmor.security.beta.kubernetes.io/clean-cilium-state":"unconfined","container.apparmor.security.beta.kubernetes.io/mount-cgroup":"unconfined"},"labels":{"app.kubernetes.io/name":"cilium-agent","app.kubernetes.io/part-of":"cilium","k8s-app":"cilium"}},"spec":{"affinity":{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"cilium"}},"topologyKey":"kubernetes.io/hostname"}]}},"automountServiceAccountToken":true,"containers":[{"args":["--config-dir=/tmp/cilium/config-map"],"command":["cilium-agent"],"env":[{"name":"K8S_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"CILIUM_K8S_NAMESPACE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}},{"name":"CILIUM_CLUSTERMESH_CONFIG","value":"/var/lib/cilium/clustermesh/"},{"name":"GOMEMLIMIT","valueFrom":{"resourceFieldRef":{"divisor":"1","resource":"limits.memory"}}}],"image":"quay.io/cilium/cilium:v1.15.6","imagePullPolicy":"IfNotPresent","lifecycle":{"postStart":{"exec":{"command":["bash","-c","set -o errexit\nset -o pipefail\nset -o nounset\n\n# When running in AWS ENI mode, it's likely that 'aws-node' has\n# had a chance to install SNAT iptables rules. These can result\n# in dropped traffic, so we should attempt to remove them.\n# We do it using a 'postStart' hook since this may need to run\n# for nodes which might have already been init'ed but may still\n# have dangling rules. This is safe because there are no\n# dependencies on anything that is part of the startup script\n# itself, and can be safely run multiple times per node (e.g. in\n# case of a restart).\nif [[ \"$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')\" != \"0\" ]];\nthen\n echo 'Deleting iptables rules created by the AWS CNI VPC plugin'\n iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore\nfi\necho 'Done!'\n"]}},"preStop":{"exec":{"command":["/cni-uninstall.sh"]}}},"livenessProbe":{"failureThreshold":10,"httpGet":{"host":"127.0.0.1","httpHeaders":[{"name":"brief","value":"true"}],"path":"/healthz","port":9879,"scheme":"HTTP"},"periodSeconds":30,"successThreshold":1,"timeoutSeconds":5},"name":"cilium-agent","readinessProbe":{"failureThreshold":3,"httpGet":{"host":"127.0.0.1","httpHeaders":[{"name":"brief","value":"true"}],"path":"/healthz","port":9879,"scheme":"HTTP"},"periodSeconds":30,"successThreshold":1,"timeoutSeconds":5},"securityContext":{"capabilities":{"add":["CHOWN","KILL","NET_ADMIN","NET_RAW","IPC_LOCK","SYS_MODULE","SYS_ADMIN","SYS_RESOURCE","DAC_OVERRIDE","FOWNER","SETGID","SETUID"],"drop":["ALL"]},"seLinuxOptions":{"level":"s0","type":"spc_t"}},"startupProbe":{"failureThreshold":105,"httpGet":{"host":"127.0.0.1","httpHeaders":[{"name":"brief","value":"true"}],"path":"/healthz","port":9879,"scheme":"HTTP"},"initialDelaySeconds":5,"periodSeconds":2,"successThreshold":1},"terminationMessagePolicy":"FallbackToLogsOnError","volumeMounts":[{"mountPath":"/host/proc/sys/net","name":"host-proc-sys-net"},{"mountPath":"/host/proc/sys/kernel","name":"host-proc-sys-kernel"},{"mountPath":"/sys/fs/bpf","mountPropagation":"HostToContainer","name":"bpf-maps"},{"mountPath":"/var/run/cilium","name":"cilium-run"},{"mountPath":"/host/etc/cni/net.d","name":"etc-cni-netd"},{"mountPath":"/var/lib/cilium/clustermesh","name":"clustermesh-secrets","readOnly":true},{"mountPath":"/lib/modules","name":"lib-modules","readOnly":true},{"mountPath":"/run/xtables.lock","name":"xtables-lock"},{"mountPath":"/tmp","name":"tmp"}]}],"hostNetwork":true,"initContainers":[{"command":["cilium-dbg","build-config"],"env":[{"name":"K8S_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"CILIUM_K8S_NAMESPACE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}}],"image":"quay.io/cilium/cilium:v1.15.6","imagePullPolicy":"IfNotPresent","name":"config","terminationMessagePolicy":"FallbackToLogsOnError","volumeMounts":[{"mountPath":"/tmp","name":"tmp"}]},{"command":["sh","-ec","cp /usr/bin/cilium-mount /hostbin/cilium-mount;\nnsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt \"${BIN_PATH}/cilium-mount\" $CGROUP_ROOT;\nrm /hostbin/cilium-mount\n"],"env":[{"name":"CGROUP_ROOT","value":"/run/cilium/cgroupv2"},{"name":"BIN_PATH","value":"/opt/cni/bin"}],"image":"quay.io/cilium/cilium:v1.15.6","imagePullPolicy":"IfNotPresent","name":"mount-cgroup","securityContext":{"capabilities":{"add":["SYS_ADMIN","SYS_CHROOT","SYS_PTRACE"],"drop":["ALL"]},"seLinuxOptions":{"level":"s0","type":"spc_t"}},"terminationMessagePolicy":"FallbackToLogsOnError","volumeMounts":[{"mountPath":"/hostproc","name":"hostproc"},{"mountPath":"/hostbin","name":"cni-path"}]},{"command":["sh","-ec","cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix;\nnsenter --mount=/hostproc/1/ns/mnt \"${BIN_PATH}/cilium-sysctlfix\";\nrm /hostbin/cilium-sysctlfix\n"],"env":[{"name":"BIN_PATH","value":"/opt/cni/bin"}],"image":"quay.io/cilium/cilium:v1.15.6","imagePullPolicy":"IfNotPresent","name":"apply-sysctl-overwrites","securityContext":{"capabilities":{"add":["SYS_ADMIN","SYS_CHROOT","SYS_PTRACE"],"drop":["ALL"]},"seLinuxOptions":{"level":"s0","type":"spc_t"}},"terminationMessagePolicy":"FallbackToLogsOnError","volumeMounts":[{"mountPath":"/hostproc","name":"hostproc"},{"mountPath":"/hostbin","name":"cni-path"}]},{"args":["mount | grep \"/sys/fs/bpf type bpf\" || mount -t bpf bpf /sys/fs/bpf"],"command":["/bin/bash","-c","--"],"image":"quay.io/cilium/cilium:v1.15.6","imagePullPolicy":"IfNotPresent","name":"mount-bpf-fs","securityContext":{"privileged":true},"terminationMessagePolicy":"FallbackToLogsOnError","volumeMounts":[{"mountPath":"/sys/fs/bpf","mountPropagation":"Bidirectional","name":"bpf-maps"}]},{"command":["/init-container.sh"],"env":[{"name":"CILIUM_ALL_STATE","valueFrom":{"configMapKeyRef":{"key":"clean-cilium-state","name":"cilium-config","optional":true}}},{"name":"CILIUM_BPF_STATE","valueFrom":{"configMapKeyRef":{"key":"clean-cilium-bpf-state","name":"cilium-config","optional":true}}},{"name":"WRITE_CNI_CONF_WHEN_READY","valueFrom":{"configMapKeyRef":{"key":"write-cni-conf-when-ready","name":"cilium-config","optional":true}}}],"image":"quay.io/cilium/cilium:v1.15.6","imagePullPolicy":"IfNotPresent","name":"clean-cilium-state","securityContext":{"capabilities":{"add":["NET_ADMIN","SYS_MODULE","SYS_ADMIN","SYS_RESOURCE"],"drop":["ALL"]},"seLinuxOptions":{"level":"s0","type":"spc_t"}},"terminationMessagePolicy":"FallbackToLogsOnError","volumeMounts":[{"mountPath":"/sys/fs/bpf","name":"bpf-maps"},{"mountPath":"/run/cilium/cgroupv2","mountPropagation":"HostToContainer","name":"cilium-cgroup"},{"mountPath":"/var/run/cilium","name":"cilium-run"}]},{"command":["/install-plugin.sh"],"image":"quay.io/cilium/cilium:v1.15.6","imagePullPolicy":"IfNotPresent","name":"install-cni-binaries","resources":{"requests":{"cpu":"100m","memory":"10Mi"}},"securityContext":{"capabilities":{"drop":["ALL"]},"seLinuxOptions":{"level":"s0","type":"spc_t"}},"terminationMessagePolicy":"FallbackToLogsOnError","volumeMounts":[{"mountPath":"/host/opt/cni/bin","name":"cni-path"}]}],"nodeSelector":{"kubernetes.io/os":"linux"},"priorityClassName":"system-node-critical","restartPolicy":"Always","serviceAccount":"cilium","serviceAccountName":"cilium","terminationGracePeriodSeconds":1,"tolerations":[{"operator":"Exists"}],"volumes":[{"emptyDir":{},"name":"tmp"},{"hostPath":{"path":"/var/run/cilium","type":"DirectoryOrCreate"},"name":"cilium-run"},{"hostPath":{"path":"/sys/fs/bpf","type":"DirectoryOrCreate"},"name":"bpf-maps"},{"hostPath":{"path":"/proc","type":"Directory"},"name":"hostproc"},{"hostPath":{"path":"/run/cilium/cgroupv2","type":"DirectoryOrCreate"},"name":"cilium-cgroup"},{"hostPath":{"path":"/opt/cni/bin","type":"DirectoryOrCreate"},"name":"cni-path"},{"hostPath":{"path":"/etc/cni/net.d","type":"DirectoryOrCreate"},"name":"etc-cni-netd"},{"hostPath":{"path":"/lib/modules"},"name":"lib-modules"},{"hostPath":{"path":"/run/xtables.lock","type":"FileOrCreate"},"name":"xtables-lock"},{"name":"clustermesh-secrets","projected":{"defaultMode":256,"sources":[{"secret":{"name":"cilium-clustermesh","optional":true}},{"secret":{"items":[{"key":"tls.key","path":"common-etcd-client.key"},{"key":"tls.crt","path":"common-etcd-client.crt"},{"key":"ca.crt","path":"common-etcd-client-ca.crt"}],"name":"clustermesh-apiserver-remote-cert","optional":true}}]}},{"hostPath":{"path":"/proc/sys/net","type":"Directory"},"name":"host-proc-sys-net"},{"hostPath":{"path":"/proc/sys/kernel","type":"Directory"},"name":"host-proc-sys-kernel"}]}},"updateStrategy":{"rollingUpdate":{"maxUnavailable":2},"type":"RollingUpdate"}}},{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"labels":{"app.kubernetes.io/name":"cilium-operator","app.kubernetes.io/part-of":"cilium","io.cilium/app":"operator","name":"cilium-operator"},"name":"cilium-operator","namespace":"kube-system"},"spec":{"replicas":2,"selector":{"matchLabels":{"io.cilium/app":"operator","name":"cilium-operator"}},"strategy":{"rollingUpdate":{"maxSurge":"25%","maxUnavailable":"50%"},"type":"RollingUpdate"},"template":{"metadata":{"annotations":{"prometheus.io/port":"9963","prometheus.io/scrape":"true"},"labels":{"app.kubernetes.io/name":"cilium-operator","app.kubernetes.io/part-of":"cilium","io.cilium/app":"operator","name":"cilium-operator"}},"spec":{"affinity":{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"io.cilium/app":"operator"}},"topologyKey":"kubernetes.io/hostname"}]}},"automountServiceAccountToken":true,"containers":[{"args":["--config-dir=/tmp/cilium/config-map","--debug=$(CILIUM_DEBUG)"],"command":["cilium-operator-generic"],"env":[{"name":"K8S_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"CILIUM_K8S_NAMESPACE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}},{"name":"CILIUM_DEBUG","valueFrom":{"configMapKeyRef":{"key":"debug","name":"cilium-config","optional":true}}}],"image":"quay.io/cilium/operator-generic:v1.15.6","imagePullPolicy":"IfNotPresent","livenessProbe":{"httpGet":{"host":"127.0.0.1","path":"/healthz","port":9234,"scheme":"HTTP"},"initialDelaySeconds":60,"periodSeconds":10,"timeoutSeconds":3},"name":"cilium-operator","ports":[{"containerPort":9963,"hostPort":9963,"name":"prometheus","protocol":"TCP"}],"readinessProbe":{"failureThreshold":5,"httpGet":{"host":"127.0.0.1","path":"/healthz","port":9234,"scheme":"HTTP"},"initialDelaySeconds":0,"periodSeconds":5,"timeoutSeconds":3},"terminationMessagePolicy":"FallbackToLogsOnError","volumeMounts":[{"mountPath":"/tmp/cilium/config-map","name":"cilium-config-path","readOnly":true}]}],"hostNetwork":true,"nodeSelector":{"kubernetes.io/os":"linux"},"priorityClassName":"system-cluster-critical","restartPolicy":"Always","serviceAccount":"cilium-operator","serviceAccountName":"cilium-operator","tolerations":[{"operator":"Exists"}],"volumes":[{"configMap":{"name":"cilium-config"},"name":"cilium-config-path"}]}}}}] + [{"apiVersion":"v1","kind":"ServiceAccount","metadata":{"name":"cilium","namespace":"kube-system"}},{"apiVersion":"v1","kind":"ServiceAccount","metadata":{"name":"cilium-operator","namespace":"kube-system"}},{"apiVersion":"v1","data":{"agent-not-ready-taint-key":"node.cilium.io/agent-not-ready","arping-refresh-period":"30s","auto-direct-node-routes":"false","bpf-lb-acceleration":"disabled","bpf-lb-external-clusterip":"false","bpf-lb-map-max":"65536","bpf-lb-sock":"false","bpf-lb-sock-hostns-only":"true","bpf-map-dynamic-size-ratio":"0.0025","bpf-policy-map-max":"16384","bpf-root":"/sys/fs/bpf","cgroup-root":"/run/cilium/cgroupv2","cilium-endpoint-gc-interval":"5m0s","cluster-id":"0","cluster-name":"default","cni-chaining-mode":"portmap","cni-exclusive":"false","cni-log-file":"/var/run/cilium/cilium-cni.log","custom-cni-conf":"false","debug":"false","debug-verbose":"","egress-gateway-reconciliation-trigger-interval":"1s","enable-auto-protect-node-port-range":"true","enable-bgp-control-plane":"false","enable-bpf-clock-probe":"false","enable-endpoint-health-checking":"true","enable-external-ips":"false","enable-health-check-loadbalancer-ip":"false","enable-health-check-nodeport":"true","enable-health-checking":"true","enable-host-legacy-routing":"true","enable-host-port":"false","enable-ipv4":"true","enable-ipv4-big-tcp":"false","enable-ipv4-masquerade":"true","enable-ipv6":"false","enable-ipv6-big-tcp":"false","enable-ipv6-masquerade":"true","enable-k8s-networkpolicy":"true","enable-k8s-terminating-endpoint":"true","enable-l2-neigh-discovery":"true","enable-l7-proxy":"true","enable-local-redirect-policy":"false","enable-masquerade-to-route-source":"false","enable-metrics":"true","enable-node-port":"false","enable-policy":"default","enable-remote-node-identity":"true","enable-sctp":"false","enable-svc-source-range-check":"true","enable-vtep":"false","enable-well-known-identities":"false","enable-xt-socket-fallback":"true","external-envoy-proxy":"false","identity-allocation-mode":"crd","identity-gc-interval":"15m0s","identity-heartbeat-timeout":"30m0s","install-no-conntrack-iptables-rules":"false","ipam":"kubernetes","ipam-cilium-node-update-rate":"15s","k8s-client-burst":"20","k8s-client-qps":"10","kube-proxy-replacement":"false","kube-proxy-replacement-healthz-bind-address":"","max-connected-clusters":"255","mesh-auth-enabled":"true","mesh-auth-gc-interval":"5m0s","mesh-auth-queue-size":"1024","mesh-auth-rotated-identities-queue-size":"1024","monitor-aggregation":"medium","monitor-aggregation-flags":"all","monitor-aggregation-interval":"5s","node-port-bind-protection":"true","nodes-gc-interval":"5m0s","operator-api-serve-addr":"127.0.0.1:9234","operator-prometheus-serve-addr":":9963","policy-cidr-match-mode":"","preallocate-bpf-maps":"false","procfs":"/host/proc","proxy-connect-timeout":"2","proxy-idle-timeout-seconds":"60","proxy-max-connection-duration-seconds":"0","proxy-max-requests-per-connection":"0","proxy-prometheus-port":"9964","proxy-xff-num-trusted-hops-egress":"0","proxy-xff-num-trusted-hops-ingress":"0","remove-cilium-node-taints":"true","routing-mode":"tunnel","service-no-backend-response":"reject","set-cilium-is-up-condition":"true","set-cilium-node-taints":"true","sidecar-istio-proxy-image":"cilium/istio_proxy","skip-cnp-status-startup-clean":"false","synchronize-k8s-nodes":"true","tofqdns-dns-reject-response-code":"refused","tofqdns-enable-dns-compression":"true","tofqdns-endpoint-max-ip-per-hostname":"50","tofqdns-idle-connection-grace-period":"0s","tofqdns-max-deferred-connection-deletes":"10000","tofqdns-proxy-response-max-delay":"100ms","tunnel-protocol":"vxlan","unmanaged-pod-watcher-interval":"15","vtep-cidr":"","vtep-endpoint":"","vtep-mac":"","vtep-mask":"","write-cni-conf-when-ready":"/host/etc/cni/net.d/05-cilium.conflist"},"kind":"ConfigMap","metadata":{"name":"cilium-config","namespace":"kube-system"}},{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRole","metadata":{"labels":{"app.kubernetes.io/part-of":"cilium"},"name":"cilium"},"rules":[{"apiGroups":["networking.k8s.io"],"resources":["networkpolicies"],"verbs":["get","list","watch"]},{"apiGroups":["discovery.k8s.io"],"resources":["endpointslices"],"verbs":["get","list","watch"]},{"apiGroups":[""],"resources":["namespaces","services","pods","endpoints","nodes"],"verbs":["get","list","watch"]},{"apiGroups":["apiextensions.k8s.io"],"resources":["customresourcedefinitions"],"verbs":["list","watch","get"]},{"apiGroups":["cilium.io"],"resources":["ciliumloadbalancerippools","ciliumbgppeeringpolicies","ciliumbgpnodeconfigs","ciliumbgpadvertisements","ciliumbgppeerconfigs","ciliumclusterwideenvoyconfigs","ciliumclusterwidenetworkpolicies","ciliumegressgatewaypolicies","ciliumendpoints","ciliumendpointslices","ciliumenvoyconfigs","ciliumidentities","ciliumlocalredirectpolicies","ciliumnetworkpolicies","ciliumnodes","ciliumnodeconfigs","ciliumcidrgroups","ciliuml2announcementpolicies","ciliumpodippools"],"verbs":["list","watch"]},{"apiGroups":["cilium.io"],"resources":["ciliumidentities","ciliumendpoints","ciliumnodes"],"verbs":["create"]},{"apiGroups":["cilium.io"],"resources":["ciliumidentities"],"verbs":["update"]},{"apiGroups":["cilium.io"],"resources":["ciliumendpoints"],"verbs":["delete","get"]},{"apiGroups":["cilium.io"],"resources":["ciliumnodes","ciliumnodes/status"],"verbs":["get","update"]},{"apiGroups":["cilium.io"],"resources":["ciliumnetworkpolicies/status","ciliumclusterwidenetworkpolicies/status","ciliumendpoints/status","ciliumendpoints","ciliuml2announcementpolicies/status","ciliumbgpnodeconfigs/status"],"verbs":["patch"]}]},{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRole","metadata":{"labels":{"app.kubernetes.io/part-of":"cilium"},"name":"cilium-operator"},"rules":[{"apiGroups":[""],"resources":["pods"],"verbs":["get","list","watch","delete"]},{"apiGroups":[""],"resources":["nodes"],"verbs":["list","watch"]},{"apiGroups":[""],"resources":["nodes","nodes/status"],"verbs":["patch"]},{"apiGroups":["discovery.k8s.io"],"resources":["endpointslices"],"verbs":["get","list","watch"]},{"apiGroups":[""],"resources":["services/status"],"verbs":["update","patch"]},{"apiGroups":[""],"resources":["namespaces"],"verbs":["get","list","watch"]},{"apiGroups":[""],"resources":["services","endpoints"],"verbs":["get","list","watch"]},{"apiGroups":["cilium.io"],"resources":["ciliumnetworkpolicies","ciliumclusterwidenetworkpolicies"],"verbs":["create","update","deletecollection","patch","get","list","watch"]},{"apiGroups":["cilium.io"],"resources":["ciliumnetworkpolicies/status","ciliumclusterwidenetworkpolicies/status"],"verbs":["patch","update"]},{"apiGroups":["cilium.io"],"resources":["ciliumendpoints","ciliumidentities"],"verbs":["delete","list","watch"]},{"apiGroups":["cilium.io"],"resources":["ciliumidentities"],"verbs":["update"]},{"apiGroups":["cilium.io"],"resources":["ciliumnodes"],"verbs":["create","update","get","list","watch","delete"]},{"apiGroups":["cilium.io"],"resources":["ciliumnodes/status"],"verbs":["update"]},{"apiGroups":["cilium.io"],"resources":["ciliumendpointslices","ciliumenvoyconfigs","ciliumbgppeerconfigs","ciliumbgpadvertisements","ciliumbgpnodeconfigs"],"verbs":["create","update","get","list","watch","delete","patch"]},{"apiGroups":["apiextensions.k8s.io"],"resources":["customresourcedefinitions"],"verbs":["create","get","list","watch"]},{"apiGroups":["apiextensions.k8s.io"],"resourceNames":["ciliumloadbalancerippools.cilium.io","ciliumbgppeeringpolicies.cilium.io","ciliumbgpclusterconfigs.cilium.io","ciliumbgppeerconfigs.cilium.io","ciliumbgpadvertisements.cilium.io","ciliumbgpnodeconfigs.cilium.io","ciliumbgpnodeconfigoverrides.cilium.io","ciliumclusterwideenvoyconfigs.cilium.io","ciliumclusterwidenetworkpolicies.cilium.io","ciliumegressgatewaypolicies.cilium.io","ciliumendpoints.cilium.io","ciliumendpointslices.cilium.io","ciliumenvoyconfigs.cilium.io","ciliumexternalworkloads.cilium.io","ciliumidentities.cilium.io","ciliumlocalredirectpolicies.cilium.io","ciliumnetworkpolicies.cilium.io","ciliumnodes.cilium.io","ciliumnodeconfigs.cilium.io","ciliumcidrgroups.cilium.io","ciliuml2announcementpolicies.cilium.io","ciliumpodippools.cilium.io"],"resources":["customresourcedefinitions"],"verbs":["update"]},{"apiGroups":["cilium.io"],"resources":["ciliumloadbalancerippools","ciliumpodippools","ciliumbgpclusterconfigs","ciliumbgpnodeconfigoverrides"],"verbs":["get","list","watch"]},{"apiGroups":["cilium.io"],"resources":["ciliumpodippools"],"verbs":["create"]},{"apiGroups":["cilium.io"],"resources":["ciliumloadbalancerippools/status"],"verbs":["patch"]},{"apiGroups":["coordination.k8s.io"],"resources":["leases"],"verbs":["create","get","update"]}]},{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRoleBinding","metadata":{"labels":{"app.kubernetes.io/part-of":"cilium"},"name":"cilium"},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":"cilium"},"subjects":[{"kind":"ServiceAccount","name":"cilium","namespace":"kube-system"}]},{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRoleBinding","metadata":{"labels":{"app.kubernetes.io/part-of":"cilium"},"name":"cilium-operator"},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":"cilium-operator"},"subjects":[{"kind":"ServiceAccount","name":"cilium-operator","namespace":"kube-system"}]},{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"Role","metadata":{"labels":{"app.kubernetes.io/part-of":"cilium"},"name":"cilium-config-agent","namespace":"kube-system"},"rules":[{"apiGroups":[""],"resources":["configmaps"],"verbs":["get","list","watch"]}]},{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"RoleBinding","metadata":{"labels":{"app.kubernetes.io/part-of":"cilium"},"name":"cilium-config-agent","namespace":"kube-system"},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"cilium-config-agent"},"subjects":[{"kind":"ServiceAccount","name":"cilium","namespace":"kube-system"}]},{"apiVersion":"apps/v1","kind":"DaemonSet","metadata":{"labels":{"app.kubernetes.io/name":"cilium-agent","app.kubernetes.io/part-of":"cilium","k8s-app":"cilium"},"name":"cilium","namespace":"kube-system"},"spec":{"selector":{"matchLabels":{"k8s-app":"cilium"}},"template":{"metadata":{"annotations":{"container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites":"unconfined","container.apparmor.security.beta.kubernetes.io/cilium-agent":"unconfined","container.apparmor.security.beta.kubernetes.io/clean-cilium-state":"unconfined","container.apparmor.security.beta.kubernetes.io/mount-cgroup":"unconfined"},"labels":{"app.kubernetes.io/name":"cilium-agent","app.kubernetes.io/part-of":"cilium","k8s-app":"cilium"}},"spec":{"affinity":{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"cilium"}},"topologyKey":"kubernetes.io/hostname"}]}},"automountServiceAccountToken":true,"containers":[{"args":["--config-dir=/tmp/cilium/config-map"],"command":["cilium-agent"],"env":[{"name":"K8S_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"CILIUM_K8S_NAMESPACE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}},{"name":"CILIUM_CLUSTERMESH_CONFIG","value":"/var/lib/cilium/clustermesh/"},{"name":"GOMEMLIMIT","valueFrom":{"resourceFieldRef":{"divisor":"1","resource":"limits.memory"}}}],"image":"quay.io/cilium/cilium:v1.15.6","imagePullPolicy":"IfNotPresent","lifecycle":{"postStart":{"exec":{"command":["bash","-c","set -o errexit\nset -o pipefail\nset -o nounset\n\n# When running in AWS ENI mode, it's likely that 'aws-node' has\n# had a chance to install SNAT iptables rules. These can result\n# in dropped traffic, so we should attempt to remove them.\n# We do it using a 'postStart' hook since this may need to run\n# for nodes which might have already been init'ed but may still\n# have dangling rules. This is safe because there are no\n# dependencies on anything that is part of the startup script\n# itself, and can be safely run multiple times per node (e.g. in\n# case of a restart).\nif [[ \"$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')\" != \"0\" ]];\nthen\n echo 'Deleting iptables rules created by the AWS CNI VPC plugin'\n iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore\nfi\necho 'Done!'\n"]}},"preStop":{"exec":{"command":["/cni-uninstall.sh"]}}},"livenessProbe":{"failureThreshold":10,"httpGet":{"host":"127.0.0.1","httpHeaders":[{"name":"brief","value":"true"}],"path":"/healthz","port":9879,"scheme":"HTTP"},"periodSeconds":30,"successThreshold":1,"timeoutSeconds":5},"name":"cilium-agent","readinessProbe":{"failureThreshold":3,"httpGet":{"host":"127.0.0.1","httpHeaders":[{"name":"brief","value":"true"}],"path":"/healthz","port":9879,"scheme":"HTTP"},"periodSeconds":30,"successThreshold":1,"timeoutSeconds":5},"securityContext":{"capabilities":{"add":["CHOWN","KILL","NET_ADMIN","NET_RAW","IPC_LOCK","SYS_MODULE","SYS_ADMIN","SYS_RESOURCE","DAC_OVERRIDE","FOWNER","SETGID","SETUID"],"drop":["ALL"]},"seLinuxOptions":{"level":"s0","type":"spc_t"}},"startupProbe":{"failureThreshold":105,"httpGet":{"host":"127.0.0.1","httpHeaders":[{"name":"brief","value":"true"}],"path":"/healthz","port":9879,"scheme":"HTTP"},"initialDelaySeconds":5,"periodSeconds":2,"successThreshold":1},"terminationMessagePolicy":"FallbackToLogsOnError","volumeMounts":[{"mountPath":"/host/proc/sys/net","name":"host-proc-sys-net"},{"mountPath":"/host/proc/sys/kernel","name":"host-proc-sys-kernel"},{"mountPath":"/sys/fs/bpf","mountPropagation":"HostToContainer","name":"bpf-maps"},{"mountPath":"/var/run/cilium","name":"cilium-run"},{"mountPath":"/host/etc/cni/net.d","name":"etc-cni-netd"},{"mountPath":"/var/lib/cilium/clustermesh","name":"clustermesh-secrets","readOnly":true},{"mountPath":"/lib/modules","name":"lib-modules","readOnly":true},{"mountPath":"/run/xtables.lock","name":"xtables-lock"},{"mountPath":"/tmp","name":"tmp"}]}],"hostNetwork":true,"initContainers":[{"command":["cilium-dbg","build-config"],"env":[{"name":"K8S_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"CILIUM_K8S_NAMESPACE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}}],"image":"quay.io/cilium/cilium:v1.15.6","imagePullPolicy":"IfNotPresent","name":"config","terminationMessagePolicy":"FallbackToLogsOnError","volumeMounts":[{"mountPath":"/tmp","name":"tmp"}]},{"command":["sh","-ec","cp /usr/bin/cilium-mount /hostbin/cilium-mount;\nnsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt \"${BIN_PATH}/cilium-mount\" $CGROUP_ROOT;\nrm /hostbin/cilium-mount\n"],"env":[{"name":"CGROUP_ROOT","value":"/run/cilium/cgroupv2"},{"name":"BIN_PATH","value":"/opt/cni/bin"}],"image":"quay.io/cilium/cilium:v1.15.6","imagePullPolicy":"IfNotPresent","name":"mount-cgroup","securityContext":{"capabilities":{"add":["SYS_ADMIN","SYS_CHROOT","SYS_PTRACE"],"drop":["ALL"]},"seLinuxOptions":{"level":"s0","type":"spc_t"}},"terminationMessagePolicy":"FallbackToLogsOnError","volumeMounts":[{"mountPath":"/hostproc","name":"hostproc"},{"mountPath":"/hostbin","name":"cni-path"}]},{"command":["sh","-ec","cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix;\nnsenter --mount=/hostproc/1/ns/mnt \"${BIN_PATH}/cilium-sysctlfix\";\nrm /hostbin/cilium-sysctlfix\n"],"env":[{"name":"BIN_PATH","value":"/opt/cni/bin"}],"image":"quay.io/cilium/cilium:v1.15.6","imagePullPolicy":"IfNotPresent","name":"apply-sysctl-overwrites","securityContext":{"capabilities":{"add":["SYS_ADMIN","SYS_CHROOT","SYS_PTRACE"],"drop":["ALL"]},"seLinuxOptions":{"level":"s0","type":"spc_t"}},"terminationMessagePolicy":"FallbackToLogsOnError","volumeMounts":[{"mountPath":"/hostproc","name":"hostproc"},{"mountPath":"/hostbin","name":"cni-path"}]},{"args":["mount | grep \"/sys/fs/bpf type bpf\" || mount -t bpf bpf /sys/fs/bpf"],"command":["/bin/bash","-c","--"],"image":"quay.io/cilium/cilium:v1.15.6","imagePullPolicy":"IfNotPresent","name":"mount-bpf-fs","securityContext":{"privileged":true},"terminationMessagePolicy":"FallbackToLogsOnError","volumeMounts":[{"mountPath":"/sys/fs/bpf","mountPropagation":"Bidirectional","name":"bpf-maps"}]},{"command":["/init-container.sh"],"env":[{"name":"CILIUM_ALL_STATE","valueFrom":{"configMapKeyRef":{"key":"clean-cilium-state","name":"cilium-config","optional":true}}},{"name":"CILIUM_BPF_STATE","valueFrom":{"configMapKeyRef":{"key":"clean-cilium-bpf-state","name":"cilium-config","optional":true}}},{"name":"WRITE_CNI_CONF_WHEN_READY","valueFrom":{"configMapKeyRef":{"key":"write-cni-conf-when-ready","name":"cilium-config","optional":true}}}],"image":"quay.io/cilium/cilium:v1.15.6","imagePullPolicy":"IfNotPresent","name":"clean-cilium-state","securityContext":{"capabilities":{"add":["NET_ADMIN","SYS_MODULE","SYS_ADMIN","SYS_RESOURCE"],"drop":["ALL"]},"seLinuxOptions":{"level":"s0","type":"spc_t"}},"terminationMessagePolicy":"FallbackToLogsOnError","volumeMounts":[{"mountPath":"/sys/fs/bpf","name":"bpf-maps"},{"mountPath":"/run/cilium/cgroupv2","mountPropagation":"HostToContainer","name":"cilium-cgroup"},{"mountPath":"/var/run/cilium","name":"cilium-run"}]},{"command":["/install-plugin.sh"],"image":"quay.io/cilium/cilium:v1.15.6","imagePullPolicy":"IfNotPresent","name":"install-cni-binaries","resources":{"requests":{"cpu":"100m","memory":"10Mi"}},"securityContext":{"capabilities":{"drop":["ALL"]},"seLinuxOptions":{"level":"s0","type":"spc_t"}},"terminationMessagePolicy":"FallbackToLogsOnError","volumeMounts":[{"mountPath":"/host/opt/cni/bin","name":"cni-path"}]}],"nodeSelector":{"kubernetes.io/os":"linux"},"priorityClassName":"system-node-critical","restartPolicy":"Always","serviceAccount":"cilium","serviceAccountName":"cilium","terminationGracePeriodSeconds":1,"tolerations":[{"operator":"Exists"}],"volumes":[{"emptyDir":{},"name":"tmp"},{"hostPath":{"path":"/var/run/cilium","type":"DirectoryOrCreate"},"name":"cilium-run"},{"hostPath":{"path":"/sys/fs/bpf","type":"DirectoryOrCreate"},"name":"bpf-maps"},{"hostPath":{"path":"/proc","type":"Directory"},"name":"hostproc"},{"hostPath":{"path":"/run/cilium/cgroupv2","type":"DirectoryOrCreate"},"name":"cilium-cgroup"},{"hostPath":{"path":"/opt/cni/bin","type":"DirectoryOrCreate"},"name":"cni-path"},{"hostPath":{"path":"/etc/cni/net.d","type":"DirectoryOrCreate"},"name":"etc-cni-netd"},{"hostPath":{"path":"/lib/modules"},"name":"lib-modules"},{"hostPath":{"path":"/run/xtables.lock","type":"FileOrCreate"},"name":"xtables-lock"},{"name":"clustermesh-secrets","projected":{"defaultMode":256,"sources":[{"secret":{"name":"cilium-clustermesh","optional":true}},{"secret":{"items":[{"key":"tls.key","path":"common-etcd-client.key"},{"key":"tls.crt","path":"common-etcd-client.crt"},{"key":"ca.crt","path":"common-etcd-client-ca.crt"}],"name":"clustermesh-apiserver-remote-cert","optional":true}}]}},{"hostPath":{"path":"/proc/sys/net","type":"Directory"},"name":"host-proc-sys-net"},{"hostPath":{"path":"/proc/sys/kernel","type":"Directory"},"name":"host-proc-sys-kernel"}]}},"updateStrategy":{"rollingUpdate":{"maxUnavailable":2},"type":"RollingUpdate"}}},{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"labels":{"app.kubernetes.io/name":"cilium-operator","app.kubernetes.io/part-of":"cilium","io.cilium/app":"operator","name":"cilium-operator"},"name":"cilium-operator","namespace":"kube-system"},"spec":{"replicas":2,"selector":{"matchLabels":{"io.cilium/app":"operator","name":"cilium-operator"}},"strategy":{"rollingUpdate":{"maxSurge":"25%","maxUnavailable":"50%"},"type":"RollingUpdate"},"template":{"metadata":{"annotations":{"prometheus.io/port":"9963","prometheus.io/scrape":"true"},"labels":{"app.kubernetes.io/name":"cilium-operator","app.kubernetes.io/part-of":"cilium","io.cilium/app":"operator","name":"cilium-operator"}},"spec":{"affinity":{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"io.cilium/app":"operator"}},"topologyKey":"kubernetes.io/hostname"}]}},"automountServiceAccountToken":true,"containers":[{"args":["--config-dir=/tmp/cilium/config-map","--debug=$(CILIUM_DEBUG)"],"command":["cilium-operator-generic"],"env":[{"name":"K8S_NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"CILIUM_K8S_NAMESPACE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}},{"name":"CILIUM_DEBUG","valueFrom":{"configMapKeyRef":{"key":"debug","name":"cilium-config","optional":true}}}],"image":"quay.io/cilium/operator-generic:v1.15.6","imagePullPolicy":"IfNotPresent","livenessProbe":{"httpGet":{"host":"127.0.0.1","path":"/healthz","port":9234,"scheme":"HTTP"},"initialDelaySeconds":60,"periodSeconds":10,"timeoutSeconds":3},"name":"cilium-operator","ports":[{"containerPort":9963,"hostPort":9963,"name":"prometheus","protocol":"TCP"}],"readinessProbe":{"failureThreshold":5,"httpGet":{"host":"127.0.0.1","path":"/healthz","port":9234,"scheme":"HTTP"},"initialDelaySeconds":0,"periodSeconds":5,"timeoutSeconds":3},"terminationMessagePolicy":"FallbackToLogsOnError","volumeMounts":[{"mountPath":"/tmp/cilium/config-map","name":"cilium-config-path","readOnly":true}]}],"hostNetwork":true,"nodeSelector":{"kubernetes.io/os":"linux"},"priorityClassName":"system-cluster-critical","restartPolicy":"Always","serviceAccount":"cilium-operator","serviceAccountName":"cilium-operator","tolerations":[{"operator":"Exists"}],"volumes":[{"configMap":{"name":"cilium-config"},"name":"cilium-config-path"}]}}}}] kind: ConfigMap metadata: creationTimestamp: null diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml index 1d9a1fdda..4f1ab90d4 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/cni/cilium/manifests/helm-addon-installation.yaml @@ -10,6 +10,7 @@ data: values.yaml: |- cni: chainingMode: portmap + exclusive: false hubble: tls: auto: @@ -27,4 +28,6 @@ data: certgen: image: useDigest: false + socketLB: + hostNamespaceOnly: true {{- end -}} diff --git a/hack/addons/kustomize/cilium/helm-values.yaml b/hack/addons/kustomize/cilium/helm-values.yaml index f35336294..b1de61e45 100644 --- a/hack/addons/kustomize/cilium/helm-values.yaml +++ b/hack/addons/kustomize/cilium/helm-values.yaml @@ -4,6 +4,7 @@ --- cni: chainingMode: portmap + exclusive: false hubble: enabled: false ipam: @@ -16,3 +17,5 @@ operator: certgen: image: useDigest: false +socketLB: + hostNamespaceOnly: true