From 28c1f78efef29a69982f252a8c2a26c3a5802c10 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Thu, 5 Dec 2024 14:45:18 +0100 Subject: [PATCH] Update Kube-OVN v0.13.0 --- packages/system/kubeovn/Makefile | 5 +- .../charts/kube-ovn/templates/_helpers.tpl | 8 ++ .../kube-ovn/templates/central-deploy.yaml | 44 ++++--- .../kube-ovn/templates/controller-deploy.yaml | 49 +++++++- .../templates/ic-controller-deploy.yaml | 35 ++++-- .../kube-ovn/templates/kube-ovn-crd.yaml | 64 +++++++++++ .../kube-ovn/templates/monitor-deploy.yaml | 64 ++++++++--- .../charts/kube-ovn/templates/ovn-CR.yaml | 85 +++++++++++++- .../charts/kube-ovn/templates/ovn-CRB.yaml | 44 ++++++- .../charts/kube-ovn/templates/ovn-sa.yaml | 24 ++++ .../charts/kube-ovn/templates/ovncni-ds.yaml | 107 +++++++++++++++--- .../charts/kube-ovn/templates/ovsovn-ds.yaml | 81 +++++++++---- .../charts/kube-ovn/templates/pinger-ds.yaml | 26 ++++- .../kubeovn/charts/kube-ovn/values.yaml | 16 ++- .../system/kubeovn/images/kubeovn/Dockerfile | 48 +------- packages/system/kubeovn/patches/mtu.diff | 12 +- packages/system/kubeovn/values.yaml | 2 +- 17 files changed, 567 insertions(+), 147 deletions(-) diff --git a/packages/system/kubeovn/Makefile b/packages/system/kubeovn/Makefile index 9f1a6e55c..ed802848e 100644 --- a/packages/system/kubeovn/Makefile +++ b/packages/system/kubeovn/Makefile @@ -8,8 +8,9 @@ include ../../../scripts/package.mk update: rm -rf charts && mkdir -p charts/kube-ovn - curl -sSL https://github.com/kubeovn/kube-ovn/archive/refs/heads/master.tar.gz | \ - tar xzvf - --strip 1 kube-ovn-master/charts + tag=$$(git ls-remote --tags --sort="v:refname" https://github.com/kubeovn/kube-ovn | awk -F'[/^]' 'END{print $$3}') && \ + curl -sSL https://github.com/kubeovn/kube-ovn/archive/refs/tags/$${tag}.tar.gz | \ + tar xzvf - --strip 1 kube-ovn-$${tag#*v}/charts patch --no-backup-if-mismatch -p4 < patches/cozyconfig.diff patch --no-backup-if-mismatch -p4 < patches/mtu.diff diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/_helpers.tpl b/packages/system/kubeovn/charts/kube-ovn/templates/_helpers.tpl index 18b5c5440..1b9a05754 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/_helpers.tpl +++ b/packages/system/kubeovn/charts/kube-ovn/templates/_helpers.tpl @@ -75,3 +75,11 @@ Number of master nodes {{- end -}} {{- end -}} {{- end -}} + +{{- define "kubeovn.runAsUser" -}} + {{- if $.Values.func.ENABLE_OVN_IPSEC -}} + 0 + {{- else -}} + 65534 + {{- end -}} +{{- end -}} diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/central-deploy.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/central-deploy.yaml index 03a1bab7a..bbc1e09d3 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/central-deploy.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/central-deploy.yaml @@ -40,15 +40,42 @@ spec: priorityClassName: system-cluster-critical serviceAccountName: ovn-ovs hostNetwork: true + initContainers: + - name: hostpath-init + image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - "chown -R nobody: /var/run/ovn /etc/ovn /var/log/ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/ovn + name: host-log-ovn containers: - name: ovn-central image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} command: + - bash - /kube-ovn/start-db.sh securityContext: + runAsUser: {{ include "kubeovn.runAsUser" . }} + privileged: false capabilities: - add: ["SYS_NICE"] + add: + - NET_BIND_SERVICE + - SYS_NICE env: - name: ENABLE_SSL value: "{{ .Values.networking.ENABLE_SSL }}" @@ -92,16 +119,10 @@ spec: cpu: {{ index .Values "ovn-central" "limits" "cpu" }} memory: {{ index .Values "ovn-central" "limits" "memory" }} volumeMounts: - - mountPath: /var/run/openvswitch - name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - - mountPath: /etc/openvswitch - name: host-config-openvswitch - mountPath: /etc/ovn name: host-config-ovn - - mountPath: /var/log/openvswitch - name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn - mountPath: /etc/localtime @@ -131,21 +152,12 @@ spec: {{ index . 0 }}: "{{ if eq (len .) 2 }}{{ index . 1 }}{{ end }}" {{- end }} volumes: - - name: host-run-ovs - hostPath: - path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - - name: host-config-openvswitch - hostPath: - path: {{ .Values.OPENVSWITCH_DIR }} - name: host-config-ovn hostPath: path: {{ .Values.OVN_DIR }} - - name: host-log-ovs - hostPath: - path: {{ .Values.log_conf.LOG_DIR }}/openvswitch - name: host-log-ovn hostPath: path: {{ .Values.log_conf.LOG_DIR }}/ovn diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/controller-deploy.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/controller-deploy.yaml index d9f2262e5..e5217c6a0 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/controller-deploy.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/controller-deploy.yaml @@ -47,6 +47,24 @@ spec: priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true + initContainers: + - name: hostpath-init + image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - "chown -R nobody: /var/log/kube-ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - name: kube-ovn-log + mountPath: /var/log/kube-ovn containers: - name: kube-ovn-controller image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} @@ -89,6 +107,17 @@ spec: - --keep-vm-ip={{- .Values.func.ENABLE_KEEP_VM_IP }} - --enable-metrics={{- .Values.networking.ENABLE_METRICS }} - --node-local-dns-ip={{- .Values.networking.NODE_LOCAL_DNS_IP }} + - --secure-serving={{- .Values.func.SECURE_SERVING }} + - --enable-ovn-ipsec={{- .Values.func.ENABLE_OVN_IPSEC }} + - --enable-anp={{- .Values.func.ENABLE_ANP }} + - --ovsdb-con-timeout={{- .Values.func.OVSDB_CON_TIMEOUT }} + - --ovsdb-inactivity-timeout={{- .Values.func.OVSDB_INACTIVITY_TIMEOUT }} + securityContext: + runAsUser: {{ include "kubeovn.runAsUser" . }} + privileged: false + capabilities: + add: + - NET_BIND_SERVICE env: - name: ENABLE_SSL value: "{{ .Values.networking.ENABLE_SSL }}" @@ -96,6 +125,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: KUBE_NAMESPACE valueFrom: fieldRef: @@ -106,6 +139,10 @@ spec: fieldPath: spec.nodeName - name: OVN_DB_IPS value: "{{ .Values.MASTER_NODES | default (include "kubeovn.nodeIPs" .) }}" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: POD_IPS valueFrom: fieldRef: @@ -126,17 +163,21 @@ spec: readinessProbe: exec: command: - - /kube-ovn/kube-ovn-controller-healthcheck + - /kube-ovn/kube-ovn-healthcheck + - --port=10660 + - --tls={{- .Values.func.SECURE_SERVING }} periodSeconds: 3 - timeoutSeconds: 45 + timeoutSeconds: 5 livenessProbe: exec: command: - - /kube-ovn/kube-ovn-controller-healthcheck + - /kube-ovn/kube-ovn-healthcheck + - --port=10660 + - --tls={{- .Values.func.SECURE_SERVING }} initialDelaySeconds: 300 periodSeconds: 7 failureThreshold: 5 - timeoutSeconds: 45 + timeoutSeconds: 5 resources: requests: cpu: {{ index .Values "kube-ovn-controller" "requests" "cpu" }} diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/ic-controller-deploy.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/ic-controller-deploy.yaml index 40a314d13..4dec76ee1 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/ic-controller-deploy.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/ic-controller-deploy.yaml @@ -41,6 +41,28 @@ spec: priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true + initContainers: + - name: hostpath-init + image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - "chown -R nobody: /var/run/ovn /var/log/ovn /var/log/kube-ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /var/log/ovn + name: host-log-ovn + - name: kube-ovn-log + mountPath: /var/log/kube-ovn containers: - name: ovn-ic-controller image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} @@ -52,8 +74,12 @@ spec: - --logtostderr=false - --alsologtostderr=true securityContext: + runAsUser: {{ include "kubeovn.runAsUser" . }} + privileged: false capabilities: - add: ["SYS_NICE"] + add: + - NET_BIND_SERVICE + - SYS_NICE env: - name: ENABLE_SSL value: "{{ .Values.networking.ENABLE_SSL }}" @@ -62,7 +88,7 @@ spec: fieldRef: fieldPath: metadata.namespace - name: OVN_DB_IPS - value: "{{ .Values.MASTER_NODES }}" + value: "{{ .Values.MASTER_NODES | default (include "kubeovn.nodeIPs" .) }}" resources: requests: cpu: 300m @@ -73,8 +99,6 @@ spec: volumeMounts: - mountPath: /var/run/ovn name: host-run-ovn - - mountPath: /etc/ovn - name: host-config-ovn - mountPath: /var/log/ovn name: host-log-ovn - mountPath: /etc/localtime @@ -90,9 +114,6 @@ spec: - name: host-run-ovn hostPath: path: /run/ovn - - name: host-config-ovn - hostPath: - path: /etc/origin/ovn - name: host-log-ovn hostPath: path: /var/log/ovn diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/kube-ovn-crd.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/kube-ovn-crd.yaml index 565f54ebd..1c8587348 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -503,6 +503,31 @@ spec: type: string qosPolicy: type: string + bgpSpeaker: + type: object + properties: + enabled: + type: boolean + asn: + type: integer + remoteAsn: + type: integer + neighbors: + type: array + items: + type: string + holdTime: + type: string + routerId: + type: string + password: + type: string + enableGracefulRestart: + type: boolean + extraArgs: + type: array + items: + type: string tolerations: type: array items: @@ -1300,8 +1325,12 @@ spec: type: boolean v4Eip: type: string + v6Eip: + type: string v4Ip: type: string + v6Ip: + type: string vpc: type: string conditions: @@ -1493,8 +1522,12 @@ spec: type: boolean v4Eip: type: string + v6Eip: + type: string v4Ip: type: string + v6Ip: + type: string vpc: type: string externalPort: @@ -1570,12 +1603,17 @@ spec: - jsonPath: .spec.namespaces name: Namespaces type: string + - jsonPath: .status.defaultLogicalSwitch + name: DefaultSubnet + type: string name: v1 schema: openAPIV3Schema: properties: spec: properties: + defaultSubnet: + type: string enableExternal: type: boolean enableBfd: @@ -1976,6 +2014,10 @@ spec: type: string u2oInterconnectionVPC: type: string + mcastQuerierIP: + type: string + mcastQuerierMAC: + type: string v4usingIPrange: type: string v4availableIPrange: @@ -2156,6 +2198,28 @@ spec: type: boolean routeTable: type: string + namespaceSelectors: + type: array + items: + type: object + properties: + matchLabels: + type: object + additionalProperties: + type: string + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string scope: Cluster names: plural: subnets diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/monitor-deploy.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/monitor-deploy.yaml index 5099feb2b..0bb2f4d4b 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/monitor-deploy.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/monitor-deploy.yaml @@ -38,19 +38,41 @@ spec: priorityClassName: system-cluster-critical serviceAccountName: kube-ovn-app hostNetwork: true + initContainers: + - name: hostpath-init + image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - "chown -R nobody: /var/log/kube-ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - name: kube-ovn-log + mountPath: /var/log/kube-ovn containers: - name: kube-ovn-monitor image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} command: ["/kube-ovn/start-ovn-monitor.sh"] args: + - --secure-serving={{- .Values.func.SECURE_SERVING }} - --log_file=/var/log/kube-ovn/kube-ovn-monitor.log - --logtostderr=false - --alsologtostderr=true - --log_file_max_size=200 securityContext: - runAsUser: 0 + runAsUser: {{ include "kubeovn.runAsUser" . }} privileged: false + capabilities: + add: + - NET_BIND_SERVICE env: - name: ENABLE_SSL value: "{{ .Values.networking.ENABLE_SSL }}" @@ -58,6 +80,18 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: POD_IPS valueFrom: fieldRef: @@ -72,12 +106,8 @@ spec: cpu: {{ index .Values "kube-ovn-monitor" "limits" "cpu" }} memory: {{ index .Values "kube-ovn-monitor" "limits" "memory" }} volumeMounts: - - mountPath: /var/run/openvswitch - name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - - mountPath: /etc/openvswitch - name: host-config-openvswitch - mountPath: /etc/ovn name: host-config-ovn - mountPath: /var/log/ovn @@ -95,32 +125,32 @@ spec: initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10661 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10661 + - --tls={{- .Values.func.SECURE_SERVING }} + timeoutSeconds: 5 readinessProbe: failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10661 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10661 + - --tls={{- .Values.func.SECURE_SERVING }} + timeoutSeconds: 5 nodeSelector: kubernetes.io/os: "linux" {{- with splitList "=" .Values.MASTER_NODES_LABEL }} {{ index . 0 }}: "{{ if eq (len .) 2 }}{{ index . 1 }}{{ end }}" {{- end }} volumes: - - name: host-run-ovs - hostPath: - path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - - name: host-config-openvswitch - hostPath: - path: {{ .Values.OPENVSWITCH_DIR }} - name: host-config-ovn hostPath: path: {{ .Values.OVN_DIR }} diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/ovn-CR.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/ovn-CR.yaml index 23dac1573..856c9cd5b 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/ovn-CR.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/ovn-CR.yaml @@ -163,7 +163,49 @@ rules: - get - list - watch - + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create + - apiGroups: + - "certificates.k8s.io" + resources: + - "certificatesigningrequests" + verbs: + - "get" + - "list" + - "watch" + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/status + - certificatesigningrequests/approval + verbs: + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - apiGroups: + - certificates.k8s.io + resourceNames: + - kubeovn.io/signer + resources: + - signers + verbs: + - approve + - sign --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -248,7 +290,34 @@ rules: - get - list - watch - + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create + - apiGroups: + - "certificates.k8s.io" + resources: + - "certificatesigningrequests" + verbs: + - "create" + - "get" + - "list" + - "watch" + - "delete" + - apiGroups: + - "" + resources: + - "secrets" + verbs: + - "get" --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -271,3 +340,15 @@ rules: - daemonsets verbs: - get + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/ovn-CRB.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/ovn-CRB.yaml index 9230d9003..7cc43d84e 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/ovn-CRB.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/ovn-CRB.yaml @@ -10,7 +10,20 @@ subjects: - kind: ServiceAccount name: ovn namespace: {{ .Values.namespace }} - +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ovn + namespace: {{ .Values.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: ovn + namespace: {{ .Values.namespace }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -38,7 +51,20 @@ subjects: - kind: ServiceAccount name: kube-ovn-cni namespace: {{ .Values.namespace }} - +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kube-ovn-cni + namespace: {{ .Values.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: kube-ovn-cni + namespace: {{ .Values.namespace }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -52,3 +78,17 @@ subjects: - kind: ServiceAccount name: kube-ovn-app namespace: {{ .Values.namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kube-ovn-app + namespace: {{ .Values.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: kube-ovn-app + namespace: {{ .Values.namespace }} diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/ovn-sa.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/ovn-sa.yaml index 17b4a92f7..1e5e9b5c8 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/ovn-sa.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/ovn-sa.yaml @@ -18,6 +18,14 @@ kind: ServiceAccount metadata: name: ovn-ovs namespace: {{ .Values.namespace }} +{{- if .Values.global.registry.imagePullSecrets }} +imagePullSecrets: +{{- range $index, $secret := .Values.global.registry.imagePullSecrets }} +{{- if $secret }} +- name: {{ $secret | quote}} +{{- end }} +{{- end }} +{{- end }} --- apiVersion: v1 @@ -25,6 +33,14 @@ kind: ServiceAccount metadata: name: kube-ovn-cni namespace: {{ .Values.namespace }} +{{- if .Values.global.registry.imagePullSecrets }} +imagePullSecrets: +{{- range $index, $secret := .Values.global.registry.imagePullSecrets }} +{{- if $secret }} +- name: {{ $secret | quote}} +{{- end }} +{{- end }} +{{- end }} --- apiVersion: v1 @@ -32,3 +48,11 @@ kind: ServiceAccount metadata: name: kube-ovn-app namespace: {{ .Values.namespace }} +{{- if .Values.global.registry.imagePullSecrets }} +imagePullSecrets: +{{- range $index, $secret := .Values.global.registry.imagePullSecrets }} +{{- if $secret }} +- name: {{ $secret | quote}} +{{- end }} +{{- end }} +{{- end }} diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml index 423f66bd0..dafe1fd99 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml @@ -29,16 +29,52 @@ spec: hostNetwork: true hostPID: true initContainers: + - name: hostpath-init + image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -xec + - {{ if not .Values.DISABLE_MODULES_MANAGEMENT -}} + iptables -V + {{- else -}} + echo "nothing to do" + {{- end }} + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + runAsGroup: 0 + volumeMounts: + - name: usr-local-sbin + mountPath: /usr/local/sbin + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/netns + name: host-ns + readOnly: false + - name: kube-ovn-log + mountPath: /var/log/kube-ovn - name: install-cni image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} - command: ["/kube-ovn/install-cni.sh"] + command: + - /kube-ovn/install-cni.sh + - --cni-conf-dir={{ .Values.cni_conf.CNI_CONF_DIR }} + - --cni-conf-file={{ .Values.cni_conf.CNI_CONF_FILE }} + - --cni-conf-name={{- .Values.cni_conf.CNI_CONFIG_PRIORITY -}}-kube-ovn.conflist securityContext: runAsUser: 0 privileged: true volumeMounts: - mountPath: /opt/cni/bin name: cni-bin + - mountPath: /etc/cni/net.d + name: cni-conf {{- if .Values.cni_conf.MOUNT_LOCAL_BIN_DIR }} - mountPath: /usr/local/bin name: local-bin @@ -65,9 +101,6 @@ spec: - --dpdk-tunnel-iface={{- .Values.networking.DPDK_TUNNEL_IFACE }} - --network-type={{- .Values.networking.TUNNEL_TYPE }} - --default-interface-name={{- .Values.networking.vlan.VLAN_INTERFACE_NAME }} - - --cni-conf-dir={{ .Values.cni_conf.CNI_CONF_DIR }} - - --cni-conf-file={{ .Values.cni_conf.CNI_CONF_FILE }} - - --cni-conf-name={{- .Values.cni_conf.CNI_CONFIG_PRIORITY -}}-kube-ovn.conflist - --logtostderr=false - --alsologtostderr=true - --log_file=/var/log/kube-ovn/kube-ovn-cni.log @@ -76,12 +109,26 @@ spec: - --kubelet-dir={{ .Values.kubelet_conf.KUBELET_DIR }} - --enable-tproxy={{ .Values.func.ENABLE_TPROXY }} - --ovs-vsctl-concurrency={{ .Values.performance.OVS_VSCTL_CONCURRENCY }} + - --secure-serving={{- .Values.func.SECURE_SERVING }} + - --enable-ovn-ipsec={{- .Values.func.ENABLE_OVN_IPSEC }} + - --set-vxlan-tx-off={{- .Values.func.SET_VXLAN_TX_OFF }} {{- with .Values.mtu }} - --mtu={{ . }} {{- end }} securityContext: runAsUser: 0 - privileged: true + privileged: false + capabilities: + add: + - NET_ADMIN + - NET_BIND_SERVICE + - NET_RAW + - SYS_ADMIN + - SYS_PTRACE + {{- if not .Values.DISABLE_MODULES_MANAGEMENT }} + - SYS_MODULE + {{- end }} + - SYS_NICE env: - name: ENABLE_SSL value: "{{ .Values.networking.ENABLE_SSL }}" @@ -93,6 +140,14 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: POD_IPS valueFrom: fieldRef: @@ -102,19 +157,22 @@ spec: - name: DBUS_SYSTEM_BUS_ADDRESS value: "unix:path=/host/var/run/dbus/system_bus_socket" volumeMounts: + - name: usr-local-sbin + mountPath: /usr/local/sbin - name: host-modules mountPath: /lib/modules readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false - name: shared-dir mountPath: {{ .Values.kubelet_conf.KUBELET_DIR }}/pods - mountPath: /etc/openvswitch name: systemid readOnly: true - - mountPath: /etc/cni/net.d - name: cni-conf - mountPath: /run/openvswitch name: host-run-ovs - mountPropagation: Bidirectional + mountPropagation: HostToContainer - mountPath: /run/ovn name: host-run-ovn - mountPath: /host/var/run/dbus @@ -132,21 +190,31 @@ spec: - mountPath: /etc/localtime name: localtime readOnly: true + {{- if .Values.func.ENABLE_OVN_IPSEC }} + - mountPath: /etc/ovs_ipsec_keys + name: ovs-ipsec-keys + {{- end }} readinessProbe: failureThreshold: 3 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10665 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10665 + - --tls={{- .Values.func.SECURE_SERVING }} + timeoutSeconds: 5 livenessProbe: failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10665 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10665 + - --tls={{- .Values.func.SECURE_SERVING }} + timeoutSeconds: 5 resources: requests: cpu: {{ index .Values "kube-ovn-cni" "requests" "cpu" }} @@ -157,9 +225,15 @@ spec: nodeSelector: kubernetes.io/os: "linux" volumes: + - name: usr-local-sbin + emptyDir: {} - name: host-modules hostPath: path: /lib/modules + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate - name: shared-dir hostPath: path: {{ .Values.kubelet_conf.KUBELET_DIR }}/pods @@ -201,3 +275,8 @@ spec: hostPath: path: {{ .Values.cni_conf.MOUNT_LOCAL_BIN_DIR }} {{- end }} + {{- if .Values.func.ENABLE_OVN_IPSEC }} + - name: ovs-ipsec-keys + hostPath: + path: /etc/origin/ovs_ipsec_keys + {{- end }} diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/ovsovn-ds.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/ovsovn-ds.yaml index 8e11d767a..33fa06d36 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/ovsovn-ds.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/ovsovn-ds.yaml @@ -36,6 +36,46 @@ spec: serviceAccountName: ovn-ovs hostNetwork: true hostPID: true + initContainers: + - name: hostpath-init + {{- if .Values.DPDK }} + image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.dpdkRepository }}:{{ .Values.DPDK_VERSION }}-{{ .Values.global.images.kubeovn.tag }} + {{- else }} + image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -xec + - | + chown -R nobody: /var/run/ovn /var/log/ovn /etc/openvswitch /var/run/openvswitch /var/log/openvswitch + {{- if not .Values.DISABLE_MODULES_MANAGEMENT }} + iptables -V + {{- else }} + ln -sf /bin/true /usr/local/sbin/modprobe + ln -sf /bin/true /usr/local/sbin/modinfo + ln -sf /bin/true /usr/local/sbin/rmmod + {{- end }} + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - mountPath: /usr/local/sbin + name: usr-local-sbin + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /etc/openvswitch + name: host-config-openvswitch + - mountPath: /var/run/openvswitch + name: host-run-ovs + - mountPath: /var/log/openvswitch + name: host-log-ovs containers: - name: openvswitch {{- if .Values.DPDK }} @@ -47,22 +87,20 @@ spec: {{- if .Values.DPDK }} command: ["/kube-ovn/start-ovs-dpdk.sh"] {{- else }} - command: - {{- if .Values.DISABLE_MODULES_MANAGEMENT }} - - /bin/sh - - -ec - - | - ln -sf /bin/true /usr/sbin/modprobe - ln -sf /bin/true /usr/sbin/modinfo - ln -sf /bin/true /usr/sbin/rmmod - exec /kube-ovn/start-ovs.sh - {{- else }} - - /kube-ovn/start-ovs.sh - {{- end }} + command: ["/kube-ovn/start-ovs.sh"] {{- end }} securityContext: - runAsUser: 0 - privileged: true + runAsUser: {{ include "kubeovn.runAsUser" . }} + privileged: false + capabilities: + add: + - NET_ADMIN + - NET_BIND_SERVICE + {{- if not .Values.DISABLE_MODULES_MANAGEMENT }} + - SYS_MODULE + {{- end }} + - SYS_NICE + - SYS_ADMIN env: - name: ENABLE_SSL value: "{{ .Values.networking.ENABLE_SSL }}" @@ -93,9 +131,8 @@ spec: - name: OVN_REMOTE_OPENFLOW_INTERVAL value: "{{ .Values.networking.OVN_REMOTE_OPENFLOW_INTERVAL }}" volumeMounts: - - mountPath: /var/run/netns - name: host-ns - mountPropagation: HostToContainer + - mountPath: /usr/local/sbin + name: usr-local-sbin - mountPath: /lib/modules name: host-modules readOnly: true @@ -105,8 +142,6 @@ spec: name: host-run-ovn - mountPath: /etc/openvswitch name: host-config-openvswitch - - mountPath: /etc/ovn - name: host-config-ovn - mountPath: /var/log/openvswitch name: host-log-ovs - mountPath: /var/log/ovn @@ -175,6 +210,8 @@ spec: nodeSelector: kubernetes.io/os: "linux" volumes: + - name: usr-local-sbin + emptyDir: {} - name: host-modules hostPath: path: /lib/modules @@ -187,9 +224,6 @@ spec: - name: host-config-openvswitch hostPath: path: {{ .Values.OPENVSWITCH_DIR }} - - name: host-config-ovn - hostPath: - path: {{ .Values.OVN_DIR }} - name: host-log-ovs hostPath: path: {{ .Values.log_conf.LOG_DIR }}/openvswitch @@ -203,9 +237,6 @@ spec: secret: optional: true secretName: kube-ovn-tls - - name: host-ns - hostPath: - path: /var/run/netns - hostPath: path: /var/run/containerd name: cruntime diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/pinger-ds.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/pinger-ds.yaml index 8ce9e3fbd..a69a13ffb 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/pinger-ds.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/pinger-ds.yaml @@ -29,6 +29,24 @@ spec: operator: Exists serviceAccountName: kube-ovn-app hostPID: true + initContainers: + - name: hostpath-init + image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - "chown -R nobody: /var/log/kube-ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - name: kube-ovn-log + mountPath: /var/log/kube-ovn containers: - name: pinger image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} @@ -37,7 +55,7 @@ spec: args: - --external-address= {{- if eq .Values.networking.NET_STACK "dual_stack" -}} - {{ .Values.dual_stack.PINGER_EXTERNAL_ADDRESS }} + {{ .Values.dual_stack.PINGER_EXTERNAL_ADDRESS }} {{- else if eq .Values.networking.NET_STACK "ipv4" -}} {{ .Values.ipv4.PINGER_EXTERNAL_ADDRESS }} {{- else if eq .Values.networking.NET_STACK "ipv6" -}} @@ -59,8 +77,12 @@ spec: - --enable-metrics={{- .Values.networking.ENABLE_METRICS }} imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: - runAsUser: 0 + runAsUser: {{ include "kubeovn.runAsUser" . }} privileged: false + capabilities: + add: + - NET_BIND_SERVICE + - NET_RAW env: - name: ENABLE_SSL value: "{{ .Values.networking.ENABLE_SSL }}" diff --git a/packages/system/kubeovn/charts/kube-ovn/values.yaml b/packages/system/kubeovn/charts/kube-ovn/values.yaml index 333cb1b9f..1bc87e8bd 100644 --- a/packages/system/kubeovn/charts/kube-ovn/values.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/values.yaml @@ -58,7 +58,6 @@ networking: func: ENABLE_LB: true ENABLE_NP: true - ENABLE_EIP_SNAT: true ENABLE_EXTERNAL_VPC: true HW_OFFLOAD: false ENABLE_LB_SVC: false @@ -68,10 +67,16 @@ func: CHECK_GATEWAY: true LOGICAL_GATEWAY: false ENABLE_BIND_LOCAL_IP: true + SECURE_SERVING: false U2O_INTERCONNECTION: false ENABLE_TPROXY: false ENABLE_IC: false ENABLE_NAT_GW: true + ENABLE_OVN_IPSEC: false + ENABLE_ANP: false + SET_VXLAN_TX_OFF: false + OVSDB_CON_TIMEOUT: 3 + OVSDB_INACTIVITY_TIMEOUT: 10 ipv4: PINGER_EXTERNAL_ADDRESS: "1.1.1.1" @@ -119,21 +124,20 @@ log_conf: OPENVSWITCH_DIR: "/etc/origin/openvswitch" OVN_DIR: "/etc/origin/ovn" DISABLE_MODULES_MANAGEMENT: false - -imagePullSecrets: [] + nameOverride: "" fullnameOverride: "" # hybrid dpdk HYBRID_DPDK: false -HUGEPAGE_SIZE_TYPE: hugepages-2Mi # Default +HUGEPAGE_SIZE_TYPE: hugepages-2Mi # Default HUGEPAGES: 1Gi # DPDK DPDK: false DPDK_VERSION: "19.11" -DPDK_CPU: "1000m" # Default CPU configuration -DPDK_MEMORY: "2Gi" # Default Memory configuration +DPDK_CPU: "1000m" # Default CPU configuration +DPDK_MEMORY: "2Gi" # Default Memory configuration ovn-central: requests: diff --git a/packages/system/kubeovn/images/kubeovn/Dockerfile b/packages/system/kubeovn/images/kubeovn/Dockerfile index 5466865ba..56a281681 100644 --- a/packages/system/kubeovn/images/kubeovn/Dockerfile +++ b/packages/system/kubeovn/images/kubeovn/Dockerfile @@ -1,45 +1,7 @@ -ARG VERSION=v1.12.19 -ARG BASE_TAG=$VERSION +FROM docker.io/kubeovn/kube-ovn:v1.13.0 -FROM golang:1.22-bookworm as builder +# Fix https://github.com/kubeovn/kube-ovn/issues/4526 +RUN setcap CAP_NET_ADMIN,CAP_NET_BIND_SERVICE,CAP_SYS_ADMIN+eip /usr/lib/openvswitch-switch/ovs-vswitchd \ + && setcap CAP_NET_ADMIN,CAP_NET_BIND_SERVICE,CAP_SYS_ADMIN+eip /usr/sbin/xtables-legacy-multi \ + && setcap CAP_NET_ADMIN,CAP_NET_BIND_SERVICE,CAP_SYS_ADMIN+eip /usr/sbin/xtables-nft-multi -ARG COMMIT_REF=e1310e1 - -WORKDIR /source - -COPY patches /patches -RUN wget -O- https://github.com/kubeovn/kube-ovn/archive/${COMMIT_REF}.tar.gz | tar xzf - --strip-components=1 -RUN git apply /patches/*.diff -RUN sed -i 's|-z now|-z now -static|' Makefile -RUN make build-go - -WORKDIR /source/dist/images - -# imported from https://github.com/kubeovn/kube-ovn/blob/master/dist/images/Dockerfile -FROM kubeovn/kube-ovn-base:$BASE_TAG - -COPY --from=builder /source/dist/images/*.sh /kube-ovn/ -COPY --from=builder /source/dist/images/kubectl-ko /kube-ovn/kubectl-ko -COPY --from=builder /source/dist/images/01-kube-ovn.conflist /kube-ovn/01-kube-ovn.conflist -COPY --from=builder /source/dist/images/logrotate/* /etc/logrotate.d/ -COPY --from=builder /source/dist/images/grace_stop_ovn_controller /usr/share/ovn/scripts/grace_stop_ovn_controller - -WORKDIR /kube-ovn - -RUN /kube-ovn/iptables-wrapper-installer.sh --no-sanity-check -RUN rm -f /usr/bin/nc &&\ - rm -f /usr/bin/netcat &&\ - rm -f /usr/lib/apt/methods/mirror -RUN deluser sync - -COPY --from=builder /source/dist/images/kube-ovn /kube-ovn/kube-ovn -COPY --from=builder /source/dist/images/kube-ovn-cmd /kube-ovn/kube-ovn-cmd -COPY --from=builder /source/dist/images/kube-ovn-webhook /kube-ovn/kube-ovn-webhook -RUN ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller && \ - ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-daemon && \ - ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-monitor && \ - ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-pinger && \ - ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-speaker && \ - ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller-healthcheck && \ - ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker && \ - ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-ic-controller diff --git a/packages/system/kubeovn/patches/mtu.diff b/packages/system/kubeovn/patches/mtu.diff index 81353943b..da3de3aa1 100644 --- a/packages/system/kubeovn/patches/mtu.diff +++ b/packages/system/kubeovn/patches/mtu.diff @@ -1,14 +1,14 @@ diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml -index c6834ef..423f66b 100644 +index 63f4258..dafe1fd 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml -@@ -76,6 +76,9 @@ spec: - - --kubelet-dir={{ .Values.kubelet_conf.KUBELET_DIR }} - - --enable-tproxy={{ .Values.func.ENABLE_TPROXY }} - - --ovs-vsctl-concurrency={{ .Values.performance.OVS_VSCTL_CONCURRENCY }} +@@ -112,6 +112,9 @@ spec: + - --secure-serving={{- .Values.func.SECURE_SERVING }} + - --enable-ovn-ipsec={{- .Values.func.ENABLE_OVN_IPSEC }} + - --set-vxlan-tx-off={{- .Values.func.SET_VXLAN_TX_OFF }} + {{- with .Values.mtu }} + - --mtu={{ . }} + {{- end }} securityContext: runAsUser: 0 - privileged: true + privileged: false diff --git a/packages/system/kubeovn/values.yaml b/packages/system/kubeovn/values.yaml index 0fcb05ce8..92f680749 100644 --- a/packages/system/kubeovn/values.yaml +++ b/packages/system/kubeovn/values.yaml @@ -22,4 +22,4 @@ global: images: kubeovn: repository: kubeovn - tag: v1.13.0@sha256:f8b1a3d3459bf896b3e2122fd6856b790ab6919dba1d22395eeb63f4af63d16c + tag: latest@sha256:d2ecc447d4a21db222bec60702435cabec5d41f49faa85f31e63faa72d23725b