Add anti-affinity for envoy deployed by provisioner

changelogs/unreleased/6148-lubronzhan-minor.md

@@ -0,0 +1,3 @@
+## Add anti-affinity rule for envoy deployed by provisioner
+
+The envoy deployment created by the gateway provisioner now includes a default anti-affinity rule. The anti-affinity rule in the [example envoy deployment manifest](https://github.com/projectcontour/contour/blob/main/examples/deployment/03-envoy-deployment.yaml) is also updated to `preferredDuringSchedulingIgnoredDuringExecution` to be consistent with the contour deployment and the gateway provisioner anti-affinity rule.

examples/deployment/03-envoy-deployment.yaml

@@ -28,14 +28,13 @@ spec:
     spec:
       affinity:
         podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchExpressions:
-                  - key: app
-                    operator: In
-                    values:
-                      - envoy
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchLabels:
+                  app: envoy
               topologyKey: "kubernetes.io/hostname"
+            weight: 100
       containers:
         - command:
             - /bin/contour

examples/render/contour-deployment.yaml

@@ -9142,14 +9142,13 @@ spec:
     spec:
       affinity:
         podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchExpressions:
-                  - key: app
-                    operator: In
-                    values:
-                      - envoy
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchLabels:
+                  app: envoy
               topologyKey: "kubernetes.io/hostname"
+            weight: 100
       containers:
         - command:
             - /bin/contour

internal/provisioner/objects/dataplane/dataplane.go

@@ -431,8 +431,19 @@ func desiredDeployment(contour *model.Contour, contourImage, envoyImage string)
 					Labels:      envoyPodLabels(contour),
 				},
 				Spec: corev1.PodSpec{
-					// TODO anti-affinity
-					Affinity:       nil,
+					Affinity: &corev1.Affinity{
+						PodAntiAffinity: &corev1.PodAntiAffinity{
+							PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{
+								{
+									Weight: int32(100),
+									PodAffinityTerm: corev1.PodAffinityTerm{
+										LabelSelector: EnvoyPodSelector(contour),
+										TopologyKey:   "kubernetes.io/hostname",
+									},
+								},
+							},
+						},
+					},
 					Containers:     containers,
 					InitContainers: initContainers,
 					Volumes: []corev1.Volume{

internal/provisioner/objects/dataplane/dataplane_test.go

@@ -260,6 +260,27 @@ func checkDaemonSetHasMetricsPort(t *testing.T, ds *appsv1.DaemonSet, port int32
 	t.Errorf("container has unexpected metrics port %d", port)
 }
 
+func checkEnvoyDeploymentHasAffinity(t *testing.T, d *appsv1.Deployment, contour *model.Contour) {
+	t.Helper()
+	if apiequality.Semantic.DeepEqual(*d.Spec.Template.Spec.Affinity,
+		corev1.Affinity{
+			PodAntiAffinity: &corev1.PodAntiAffinity{
+				PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{
+					{
+						Weight: int32(100),
+						PodAffinityTerm: corev1.PodAffinityTerm{
+							LabelSelector: EnvoyPodSelector(contour),
+							TopologyKey:   "kubernetes.io/hostname",
+						},
+					},
+				},
+			},
+		}) {
+		return
+	}
+	t.Errorf("container has unexpected affinity %v", d.Spec.Template.Spec.Affinity)
+}
+
 func TestDesiredDaemonSet(t *testing.T) {
 	name := "ds-test"
 	cntr := model.Default(fmt.Sprintf("%s-ns", name), name)
@@ -360,6 +381,7 @@ func TestDesiredDeployment(t *testing.T) {
 	testEnvoyImage := "docker.io/envoyproxy/envoy:test"
 	deploy := desiredDeployment(cntr, testContourImage, testEnvoyImage)
 	checkDeploymentHasStrategy(t, deploy, cntr.Spec.EnvoyDeploymentStrategy)
+	checkEnvoyDeploymentHasAffinity(t, deploy, cntr)
 }
 
 func TestNodePlacementDaemonSet(t *testing.T) {

site/content/resources/upgrading.md

@@ -15,6 +15,14 @@ Contour currently only tests sequential upgrades, i.e. without skipping any mino
 This approach is recommended for users in order to minimize downtime and avoid any potential issues.
 If you choose to skip versions while upgrading, please note that this may lead to additional downtime.
 
+# Known issues
+
+1. Envoy pod stuck in pending state
+
+    If Envoy is deployed with a Deployment and the number of envoy instances is not less than number of kubernetes nodes in the clusters, during rolling upgrade, new envoy pod will be stuck in pending stage because old envoy pod is occupying host port.
+
+    Workaround: Delete the envoy instance of older version manually. This will cause a little bit of downtime but it's pretty short.
+
 # The easy way to upgrade
 
 If the following are true for you:
@@ -525,7 +533,7 @@ If your version of Contour is older than v1.22.0, please upgrade to v1.22.0 firs
 1. Update the Contour RBAC resources:
 
     ```bash
-    $ kubectl apply -f examples/contour/02-rbac.yaml 
+    $ kubectl apply -f examples/contour/02-rbac.yaml
     $ kubectl apply -f examples/contour/02-role-contour.yaml
     ```
 
@@ -591,7 +599,7 @@ If your version of Contour is older than v1.21.3, please upgrade to v1.21.3 firs
 1. Update the Contour RBAC resources:
 
     ```bash
-    $ kubectl apply -f examples/contour/02-rbac.yaml 
+    $ kubectl apply -f examples/contour/02-rbac.yaml
     $ kubectl apply -f examples/contour/02-role-contour.yaml
     ```
 
@@ -790,7 +798,7 @@ If your version of Contour is older than v1.21.0, please upgrade to v1.21.0 firs
 1. Update the Contour RBAC resources:
 
     ```bash
-    $ kubectl apply -f examples/contour/02-rbac.yaml 
+    $ kubectl apply -f examples/contour/02-rbac.yaml
     $ kubectl apply -f examples/contour/02-role-contour.yaml
     ```
 
@@ -856,7 +864,7 @@ If your version of Contour is older than v1.20.2, please upgrade to v1.20.2 firs
 1. Update the Contour RBAC resources:
 
     ```bash
-    $ kubectl apply -f examples/contour/02-rbac.yaml 
+    $ kubectl apply -f examples/contour/02-rbac.yaml
     $ kubectl apply -f examples/contour/02-role-contour.yaml
     ```