Support OpenShift (#303)

1. Allow rabbitmq-cluster-operator-role to update
rabbitmqclusters/finalizers

2. Do not BlockOwnerDeletion for PVCs

3. Change group owner of mnesia dir to 999
Otherwise, the RabbitMQ process can't write the pid file into the
/var/lib/rabbitmq/mnesia/ directory on OpenShift due to permissions
denied.
Before this commit, mnesia dir was owned by user root and group root.
On OpenShift, mnesia does not have rwx bits for everyone due to stricter
security constraints:
drwxrwx---. 2 root     root       6 Aug 20 10:03 mnesia

Fixes #234

config/rbac/role.yaml

@@ -106,6 +106,12 @@ rules:
   - list
   - update
   - watch
+- apiGroups:
+  - rabbitmq.com
+  resources:
+  - rabbitmqclusters/finalizers
+  verbs:
+  - update
 - apiGroups:
   - rabbitmq.com
   resources:

controllers/rabbitmqcluster_controller.go

@@ -75,13 +75,13 @@ type RabbitmqClusterReconciler struct {
 // +kubebuilder:rbac:groups="",resources=pods/exec,verbs=create
 // +kubebuilder:rbac:groups="",resources=pods,verbs=update;get;list;watch
 // +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update
-// +kubebuilder:rbac:groups="",resources=endpoints,verbs=get;watch
+// +kubebuilder:rbac:groups="",resources=endpoints,verbs=get;watch;list
 // +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;delete
-// +kubebuilder:rbac:groups="",resources=endpoints,verbs=list
 // +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update
 // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update
 // +kubebuilder:rbac:groups=rabbitmq.com,resources=rabbitmqclusters,verbs=get;list;watch;create;update
 // +kubebuilder:rbac:groups=rabbitmq.com,resources=rabbitmqclusters/status,verbs=get;update
+// +kubebuilder:rbac:groups=rabbitmq.com,resources=rabbitmqclusters/finalizers,verbs=update
 // +kubebuilder:rbac:groups="",resources=events,verbs=get;create;patch
 // +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update
 // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;update

go.mod

@@ -28,5 +28,6 @@ require (
 	k8s.io/api v0.18.6
 	k8s.io/apimachinery v0.18.6
 	k8s.io/client-go v0.18.6
+	k8s.io/utils v0.0.0-20200603063816-c1c6865ac451
 	sigs.k8s.io/controller-runtime v0.6.2
 )

internal/resource/statefulset.go

@@ -21,6 +21,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/strategicpatch"
+	"k8s.io/utils/pointer"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
@@ -91,6 +92,7 @@ func (builder *StatefulSetBuilder) Build() (runtime.Object, error) {
 				if err := controllerutil.SetControllerReference(builder.Instance, &pvcList[i], builder.Scheme); err != nil {
 					return nil, fmt.Errorf("failed setting controller reference: %v", err)
 				}
+				disableBlockOwnerDeletion(pvcList[i])
 			}
 			sts.Spec.VolumeClaimTemplates = pvcList
 		}
@@ -121,10 +123,19 @@ func persistentVolumeClaim(instance *rabbitmqv1beta1.RabbitmqCluster, scheme *ru
 	if err := controllerutil.SetControllerReference(instance, &pvc, scheme); err != nil {
 		return []corev1.PersistentVolumeClaim{}, fmt.Errorf("failed setting controller reference: %v", err)
 	}
+	disableBlockOwnerDeletion(pvc)
 
 	return []corev1.PersistentVolumeClaim{pvc}, nil
 }
 
+// required for OpenShift compatibility, see https://github.com/rabbitmq/cluster-operator/issues/234
+func disableBlockOwnerDeletion(pvc corev1.PersistentVolumeClaim) {
+	refs := pvc.OwnerReferences
+	for i := range refs {
+		refs[i].BlockOwnerDeletion = pointer.BoolPtr(false)
+	}
+}
+
 func (builder *StatefulSetBuilder) Update(object runtime.Object) error {
 	sts := object.(*appsv1.StatefulSet)
 
@@ -481,17 +492,29 @@ func (builder *StatefulSetBuilder) podTemplateSpec(annotations, labels map[strin
 			Tolerations:                   builder.Instance.Spec.Tolerations,
 			InitContainers: []corev1.Container{
 				{
-					Name:  "copy-config",
+					Name:  "setup-container",
 					Image: builder.Instance.Spec.Image,
+					SecurityContext: &corev1.SecurityContext{
+						RunAsUser: pointer.Int64Ptr(0),
+						Capabilities: &corev1.Capabilities{
+							Drop: []corev1.Capability{"ALL"},
+							Add:  []corev1.Capability{"CHOWN", "FOWNER"},
+						},
+					},
 					Command: []string{
-						"sh", "-c", "cp /tmp/rabbitmq/rabbitmq.conf /etc/rabbitmq/rabbitmq.conf && echo '' >> /etc/rabbitmq/rabbitmq.conf ; " +
-							"cp /tmp/rabbitmq/advanced.config /etc/rabbitmq/advanced.config ; " +
-							"cp /tmp/rabbitmq/rabbitmq-env.conf /etc/rabbitmq/rabbitmq-env.conf ; " +
+						"sh", "-c", "cp /tmp/rabbitmq/rabbitmq.conf /etc/rabbitmq/rabbitmq.conf " +
+							"&& chown 999:999 /etc/rabbitmq/rabbitmq.conf " +
+							"&& echo '' >> /etc/rabbitmq/rabbitmq.conf ; " +
+							"cp /tmp/rabbitmq/advanced.config /etc/rabbitmq/advanced.config " +
+							"&& chown 999:999 /etc/rabbitmq/advanced.config ; " +
+							"cp /tmp/rabbitmq/rabbitmq-env.conf /etc/rabbitmq/rabbitmq-env.conf " +
+							"&& chown 999:999 /etc/rabbitmq/rabbitmq-env.conf ; " +
 							"cp /tmp/erlang-cookie-secret/.erlang.cookie /var/lib/rabbitmq/.erlang.cookie " +
 							"&& chown 999:999 /var/lib/rabbitmq/.erlang.cookie " +
 							"&& chmod 600 /var/lib/rabbitmq/.erlang.cookie ; " +
 							"cp /tmp/rabbitmq-plugins/enabled_plugins /etc/rabbitmq/enabled_plugins " +
-							"&& chown 999:999 /etc/rabbitmq/enabled_plugins",
+							"&& chown 999:999 /etc/rabbitmq/enabled_plugins ; " +
+							"chgrp 999 /var/lib/rabbitmq/mnesia/",
 					},
 					Resources: corev1.ResourceRequirements{
 						Limits: map[corev1.ResourceName]k8sresource.Quantity{
@@ -524,6 +547,10 @@ func (builder *StatefulSetBuilder) podTemplateSpec(annotations, labels map[strin
 							Name:      "erlang-cookie-secret",
 							MountPath: "/tmp/erlang-cookie-secret/",
 						},
+						{
+							Name:      "persistence",
+							MountPath: "/var/lib/rabbitmq/mnesia/",
+						},
 					},
 				},
 			},
@@ -596,8 +623,8 @@ func (builder *StatefulSetBuilder) podTemplateSpec(annotations, labels map[strin
 							Exec: &corev1.ExecAction{
 								Command: []string{"/bin/bash", "-c",
 									fmt.Sprintf("if [ ! -z \"$(cat /etc/pod-info/%s)\" ]; then exit 0; fi;", DeletionMarker) +
-									fmt.Sprintf(" rabbitmq-upgrade await_online_quorum_plus_one -t %d;"+
-										" rabbitmq-upgrade await_online_synchronized_mirror -t %d", defaultGracePeriodTimeoutSeconds, defaultGracePeriodTimeoutSeconds),
+										fmt.Sprintf(" rabbitmq-upgrade await_online_quorum_plus_one -t %d;"+
+											" rabbitmq-upgrade await_online_synchronized_mirror -t %d", defaultGracePeriodTimeoutSeconds, defaultGracePeriodTimeoutSeconds),
 								},
 							},
 						},

internal/resource/statefulset_test.go

@@ -13,6 +13,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/ginkgo/extensions/table"
 	. "github.com/onsi/gomega"
+	. "github.com/onsi/gomega/gstruct"
 	rabbitmqv1beta1 "github.com/rabbitmq/cluster-operator/api/v1beta1"
 	"github.com/rabbitmq/cluster-operator/internal/resource"
 	appsv1 "k8s.io/api/apps/v1"
@@ -22,6 +23,7 @@ import (
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	defaultscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/utils/pointer"
 )
 
 var _ = Describe("StatefulSet", func() {
@@ -95,7 +97,6 @@ var _ = Describe("StatefulSet", func() {
 		})
 		Context("PVC template", func() {
 			It("creates the required PersistentVolumeClaim", func() {
-				truth := true
 				q, _ := k8sresource.ParseQuantity("10Gi")
 
 				obj, err := stsBuilder.Build()
@@ -117,8 +118,8 @@ var _ = Describe("StatefulSet", func() {
 								Kind:               "RabbitmqCluster",
 								Name:               instance.Name,
 								UID:                "",
-								Controller:         &truth,
-								BlockOwnerDeletion: &truth,
+								Controller:         pointer.BoolPtr(true),
+								BlockOwnerDeletion: pointer.BoolPtr(false),
 							},
 						},
 						Annotations: map[string]string{},
@@ -170,7 +171,6 @@ var _ = Describe("StatefulSet", func() {
 
 			It("overrides the PVC list", func() {
 				storageClass := "my-storage-class"
-				truth := true
 				builder.Instance.Spec.Override.StatefulSet = &rabbitmqv1beta1.StatefulSet{
 					Spec: &rabbitmqv1beta1.StatefulSetSpec{
 						VolumeClaimTemplates: []rabbitmqv1beta1.PersistentVolumeClaim{
@@ -221,8 +221,8 @@ var _ = Describe("StatefulSet", func() {
 									Kind:               "RabbitmqCluster",
 									Name:               instance.Name,
 									UID:                "",
-									Controller:         &truth,
-									BlockOwnerDeletion: &truth,
+									Controller:         pointer.BoolPtr(true),
+									BlockOwnerDeletion: pointer.BoolPtr(false),
 								},
 							},
 						},
@@ -245,8 +245,8 @@ var _ = Describe("StatefulSet", func() {
 									Kind:               "RabbitmqCluster",
 									Name:               instance.Name,
 									UID:                "",
-									Controller:         &truth,
-									BlockOwnerDeletion: &truth,
+									Controller:         pointer.BoolPtr(true),
+									BlockOwnerDeletion: pointer.BoolPtr(false),
 								},
 							},
 						},
@@ -942,45 +942,59 @@ var _ = Describe("StatefulSet", func() {
 			Expect(stsBuilder.Update(statefulSet)).To(Succeed())
 
 			initContainers := statefulSet.Spec.Template.Spec.InitContainers
-			Expect(len(initContainers)).To(Equal(1))
-
-			container := extractContainer(initContainers, "copy-config")
-			Expect(container.Command).To(Equal([]string{
-				"sh", "-c", "cp /tmp/rabbitmq/rabbitmq.conf /etc/rabbitmq/rabbitmq.conf && echo '' >> /etc/rabbitmq/rabbitmq.conf ; " +
-					"cp /tmp/rabbitmq/advanced.config /etc/rabbitmq/advanced.config ; " +
-					"cp /tmp/rabbitmq/rabbitmq-env.conf /etc/rabbitmq/rabbitmq-env.conf ; " +
-					"cp /tmp/erlang-cookie-secret/.erlang.cookie /var/lib/rabbitmq/.erlang.cookie " +
-					"&& chown 999:999 /var/lib/rabbitmq/.erlang.cookie " +
-					"&& chmod 600 /var/lib/rabbitmq/.erlang.cookie ; " +
-					"cp /tmp/rabbitmq-plugins/enabled_plugins /etc/rabbitmq/enabled_plugins " +
-					"&& chown 999:999 /etc/rabbitmq/enabled_plugins",
+			Expect(initContainers).To(HaveLen(1))
+
+			initContainer := extractContainer(initContainers, "setup-container")
+			Expect(initContainer).To(MatchFields(IgnoreExtras, Fields{
+				"Image": Equal("rabbitmq-image-from-cr"),
+				"SecurityContext": PointTo(MatchFields(IgnoreExtras, Fields{
+					"Capabilities": PointTo(MatchAllFields(Fields{
+						"Drop": ConsistOf([]corev1.Capability{"ALL"}),
+						"Add":  ConsistOf([]corev1.Capability{"CHOWN", "FOWNER"}),
+					})),
+				})),
+				"Command": ConsistOf(
+					"sh", "-c", "cp /tmp/rabbitmq/rabbitmq.conf /etc/rabbitmq/rabbitmq.conf "+
+						"&& chown 999:999 /etc/rabbitmq/rabbitmq.conf "+
+						"&& echo '' >> /etc/rabbitmq/rabbitmq.conf ; "+
+						"cp /tmp/rabbitmq/advanced.config /etc/rabbitmq/advanced.config "+
+						"&& chown 999:999 /etc/rabbitmq/advanced.config ; "+
+						"cp /tmp/rabbitmq/rabbitmq-env.conf /etc/rabbitmq/rabbitmq-env.conf "+
+						"&& chown 999:999 /etc/rabbitmq/rabbitmq-env.conf ; "+
+						"cp /tmp/erlang-cookie-secret/.erlang.cookie /var/lib/rabbitmq/.erlang.cookie "+
+						"&& chown 999:999 /var/lib/rabbitmq/.erlang.cookie "+
+						"&& chmod 600 /var/lib/rabbitmq/.erlang.cookie ; "+
+						"cp /tmp/rabbitmq-plugins/enabled_plugins /etc/rabbitmq/enabled_plugins "+
+						"&& chown 999:999 /etc/rabbitmq/enabled_plugins ; "+
+						"chgrp 999 /var/lib/rabbitmq/mnesia/",
+				),
+				"VolumeMounts": ConsistOf(
+					corev1.VolumeMount{
+						Name:      "server-conf",
+						MountPath: "/tmp/rabbitmq/",
+					},
+					corev1.VolumeMount{
+						Name:      "plugins-conf",
+						MountPath: "/tmp/rabbitmq-plugins/",
+					},
+					corev1.VolumeMount{
+						Name:      "rabbitmq-etc",
+						MountPath: "/etc/rabbitmq/",
+					},
+					corev1.VolumeMount{
+						Name:      "rabbitmq-erlang-cookie",
+						MountPath: "/var/lib/rabbitmq/",
+					},
+					corev1.VolumeMount{
+						Name:      "erlang-cookie-secret",
+						MountPath: "/tmp/erlang-cookie-secret/",
+					},
+					corev1.VolumeMount{
+						Name:      "persistence",
+						MountPath: "/var/lib/rabbitmq/mnesia/",
+					},
+				),
 			}))
-
-			Expect(container.VolumeMounts).To(ConsistOf(
-				corev1.VolumeMount{
-					Name:      "server-conf",
-					MountPath: "/tmp/rabbitmq/",
-				},
-				corev1.VolumeMount{
-					Name:      "plugins-conf",
-					MountPath: "/tmp/rabbitmq-plugins/",
-				},
-
-				corev1.VolumeMount{
-					Name:      "rabbitmq-etc",
-					MountPath: "/etc/rabbitmq/",
-				},
-				corev1.VolumeMount{
-					Name:      "rabbitmq-erlang-cookie",
-					MountPath: "/var/lib/rabbitmq/",
-				},
-				corev1.VolumeMount{
-					Name:      "erlang-cookie-secret",
-					MountPath: "/tmp/erlang-cookie-secret/",
-				},
-			))
-
-			Expect(container.Image).To(Equal("rabbitmq-image-from-cr"))
 		})
 
 		It("adds the required terminationGracePeriodSeconds", func() {

system_tests/system_tests.go

@@ -23,7 +23,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-const statefulSetSuffix   = "server"
+const statefulSetSuffix = "server"
 
 var _ = Describe("Operator", func() {
 	var (