Skip to content

Commit 585c609

Browse files
ansdansd
authored
Support OpenShift (#303)
1. Allow rabbitmq-cluster-operator-role to update rabbitmqclusters/finalizers 2. Do not BlockOwnerDeletion for PVCs 3. Change group owner of mnesia dir to 999 Otherwise, the RabbitMQ process can't write the pid file into the /var/lib/rabbitmq/mnesia/ directory on OpenShift due to permissions denied. Before this commit, mnesia dir was owned by user root and group root. On OpenShift, mnesia does not have rwx bits for everyone due to stricter security constraints: drwxrwx---. 2 root root 6 Aug 20 10:03 mnesia Fixes #234
1 parent d841f07 commit 585c609

File tree

6 files changed

+104
-56
lines changed

6 files changed

+104
-56
lines changed

‎config/rbac/role.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,12 @@ rules:
106106
- list
107107
- update
108108
- watch
109+
- apiGroups:
110+
- rabbitmq.com
111+
resources:
112+
- rabbitmqclusters/finalizers
113+
verbs:
114+
- update
109115
- apiGroups:
110116
- rabbitmq.com
111117
resources:

‎controllers/rabbitmqcluster_controller.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,13 +75,13 @@ type RabbitmqClusterReconciler struct {
7575
// +kubebuilder:rbac:groups="",resources=pods/exec,verbs=create
7676
// +kubebuilder:rbac:groups="",resources=pods,verbs=update;get;list;watch
7777
// +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update
78-
// +kubebuilder:rbac:groups="",resources=endpoints,verbs=get;watch
78+
// +kubebuilder:rbac:groups="",resources=endpoints,verbs=get;watch;list
7979
// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;delete
80-
// +kubebuilder:rbac:groups="",resources=endpoints,verbs=list
8180
// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update
8281
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update
8382
// +kubebuilder:rbac:groups=rabbitmq.com,resources=rabbitmqclusters,verbs=get;list;watch;create;update
8483
// +kubebuilder:rbac:groups=rabbitmq.com,resources=rabbitmqclusters/status,verbs=get;update
84+
// +kubebuilder:rbac:groups=rabbitmq.com,resources=rabbitmqclusters/finalizers,verbs=update
8585
// +kubebuilder:rbac:groups="",resources=events,verbs=get;create;patch
8686
// +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update
8787
// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;update

‎go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,5 +28,6 @@ require (
2828
k8s.io/api v0.18.6
2929
k8s.io/apimachinery v0.18.6
3030
k8s.io/client-go v0.18.6
31+
k8s.io/utils v0.0.0-20200603063816-c1c6865ac451
3132
sigs.k8s.io/controller-runtime v0.6.2
3233
)

‎internal/resource/statefulset.go

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2222
"k8s.io/apimachinery/pkg/runtime"
2323
"k8s.io/apimachinery/pkg/util/strategicpatch"
24+
"k8s.io/utils/pointer"
2425
ctrl "sigs.k8s.io/controller-runtime"
2526
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
2627
)
@@ -91,6 +92,7 @@ func (builder *StatefulSetBuilder) Build() (runtime.Object, error) {
9192
if err := controllerutil.SetControllerReference(builder.Instance, &pvcList[i], builder.Scheme); err != nil {
9293
return nil, fmt.Errorf("failed setting controller reference: %v", err)
9394
}
95+
disableBlockOwnerDeletion(pvcList[i])
9496
}
9597
sts.Spec.VolumeClaimTemplates = pvcList
9698
}
@@ -121,10 +123,19 @@ func persistentVolumeClaim(instance *rabbitmqv1beta1.RabbitmqCluster, scheme *ru
121123
if err := controllerutil.SetControllerReference(instance, &pvc, scheme); err != nil {
122124
return []corev1.PersistentVolumeClaim{}, fmt.Errorf("failed setting controller reference: %v", err)
123125
}
126+
disableBlockOwnerDeletion(pvc)
124127

125128
return []corev1.PersistentVolumeClaim{pvc}, nil
126129
}
127130

131+
// required for OpenShift compatibility, see https://github.com/rabbitmq/cluster-operator/issues/234
132+
func disableBlockOwnerDeletion(pvc corev1.PersistentVolumeClaim) {
133+
refs := pvc.OwnerReferences
134+
for i := range refs {
135+
refs[i].BlockOwnerDeletion = pointer.BoolPtr(false)
136+
}
137+
}
138+
128139
func (builder *StatefulSetBuilder) Update(object runtime.Object) error {
129140
sts := object.(*appsv1.StatefulSet)
130141

@@ -481,17 +492,29 @@ func (builder *StatefulSetBuilder) podTemplateSpec(annotations, labels map[strin
481492
Tolerations: builder.Instance.Spec.Tolerations,
482493
InitContainers: []corev1.Container{
483494
{
484-
Name: "copy-config",
495+
Name: "setup-container",
485496
Image: builder.Instance.Spec.Image,
497+
SecurityContext: &corev1.SecurityContext{
498+
RunAsUser: pointer.Int64Ptr(0),
499+
Capabilities: &corev1.Capabilities{
500+
Drop: []corev1.Capability{"ALL"},
501+
Add: []corev1.Capability{"CHOWN", "FOWNER"},
502+
},
503+
},
486504
Command: []string{
487-
"sh", "-c", "cp /tmp/rabbitmq/rabbitmq.conf /etc/rabbitmq/rabbitmq.conf && echo '' >> /etc/rabbitmq/rabbitmq.conf ; " +
488-
"cp /tmp/rabbitmq/advanced.config /etc/rabbitmq/advanced.config ; " +
489-
"cp /tmp/rabbitmq/rabbitmq-env.conf /etc/rabbitmq/rabbitmq-env.conf ; " +
505+
"sh", "-c", "cp /tmp/rabbitmq/rabbitmq.conf /etc/rabbitmq/rabbitmq.conf " +
506+
"&& chown 999:999 /etc/rabbitmq/rabbitmq.conf " +
507+
"&& echo '' >> /etc/rabbitmq/rabbitmq.conf ; " +
508+
"cp /tmp/rabbitmq/advanced.config /etc/rabbitmq/advanced.config " +
509+
"&& chown 999:999 /etc/rabbitmq/advanced.config ; " +
510+
"cp /tmp/rabbitmq/rabbitmq-env.conf /etc/rabbitmq/rabbitmq-env.conf " +
511+
"&& chown 999:999 /etc/rabbitmq/rabbitmq-env.conf ; " +
490512
"cp /tmp/erlang-cookie-secret/.erlang.cookie /var/lib/rabbitmq/.erlang.cookie " +
491513
"&& chown 999:999 /var/lib/rabbitmq/.erlang.cookie " +
492514
"&& chmod 600 /var/lib/rabbitmq/.erlang.cookie ; " +
493515
"cp /tmp/rabbitmq-plugins/enabled_plugins /etc/rabbitmq/enabled_plugins " +
494-
"&& chown 999:999 /etc/rabbitmq/enabled_plugins",
516+
"&& chown 999:999 /etc/rabbitmq/enabled_plugins ; " +
517+
"chgrp 999 /var/lib/rabbitmq/mnesia/",
495518
},
496519
Resources: corev1.ResourceRequirements{
497520
Limits: map[corev1.ResourceName]k8sresource.Quantity{
@@ -524,6 +547,10 @@ func (builder *StatefulSetBuilder) podTemplateSpec(annotations, labels map[strin
524547
Name: "erlang-cookie-secret",
525548
MountPath: "/tmp/erlang-cookie-secret/",
526549
},
550+
{
551+
Name: "persistence",
552+
MountPath: "/var/lib/rabbitmq/mnesia/",
553+
},
527554
},
528555
},
529556
},
@@ -596,8 +623,8 @@ func (builder *StatefulSetBuilder) podTemplateSpec(annotations, labels map[strin
596623
Exec: &corev1.ExecAction{
597624
Command: []string{"/bin/bash", "-c",
598625
fmt.Sprintf("if [ ! -z \"$(cat /etc/pod-info/%s)\" ]; then exit 0; fi;", DeletionMarker) +
599-
fmt.Sprintf(" rabbitmq-upgrade await_online_quorum_plus_one -t %d;"+
600-
" rabbitmq-upgrade await_online_synchronized_mirror -t %d", defaultGracePeriodTimeoutSeconds, defaultGracePeriodTimeoutSeconds),
626+
fmt.Sprintf(" rabbitmq-upgrade await_online_quorum_plus_one -t %d;"+
627+
" rabbitmq-upgrade await_online_synchronized_mirror -t %d", defaultGracePeriodTimeoutSeconds, defaultGracePeriodTimeoutSeconds),
601628
},
602629
},
603630
},

‎internal/resource/statefulset_test.go

Lines changed: 60 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
. "github.com/onsi/ginkgo"
1414
. "github.com/onsi/ginkgo/extensions/table"
1515
. "github.com/onsi/gomega"
16+
. "github.com/onsi/gomega/gstruct"
1617
rabbitmqv1beta1 "github.com/rabbitmq/cluster-operator/api/v1beta1"
1718
"github.com/rabbitmq/cluster-operator/internal/resource"
1819
appsv1 "k8s.io/api/apps/v1"
@@ -22,6 +23,7 @@ import (
2223
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2324
"k8s.io/apimachinery/pkg/runtime"
2425
defaultscheme "k8s.io/client-go/kubernetes/scheme"
26+
"k8s.io/utils/pointer"
2527
)
2628

2729
var _ = Describe("StatefulSet", func() {
@@ -95,7 +97,6 @@ var _ = Describe("StatefulSet", func() {
9597
})
9698
Context("PVC template", func() {
9799
It("creates the required PersistentVolumeClaim", func() {
98-
truth := true
99100
q, _ := k8sresource.ParseQuantity("10Gi")
100101

101102
obj, err := stsBuilder.Build()
@@ -117,8 +118,8 @@ var _ = Describe("StatefulSet", func() {
117118
Kind: "RabbitmqCluster",
118119
Name: instance.Name,
119120
UID: "",
120-
Controller: &truth,
121-
BlockOwnerDeletion: &truth,
121+
Controller: pointer.BoolPtr(true),
122+
BlockOwnerDeletion: pointer.BoolPtr(false),
122123
},
123124
},
124125
Annotations: map[string]string{},
@@ -170,7 +171,6 @@ var _ = Describe("StatefulSet", func() {
170171

171172
It("overrides the PVC list", func() {
172173
storageClass := "my-storage-class"
173-
truth := true
174174
builder.Instance.Spec.Override.StatefulSet = &rabbitmqv1beta1.StatefulSet{
175175
Spec: &rabbitmqv1beta1.StatefulSetSpec{
176176
VolumeClaimTemplates: []rabbitmqv1beta1.PersistentVolumeClaim{
@@ -221,8 +221,8 @@ var _ = Describe("StatefulSet", func() {
221221
Kind: "RabbitmqCluster",
222222
Name: instance.Name,
223223
UID: "",
224-
Controller: &truth,
225-
BlockOwnerDeletion: &truth,
224+
Controller: pointer.BoolPtr(true),
225+
BlockOwnerDeletion: pointer.BoolPtr(false),
226226
},
227227
},
228228
},
@@ -245,8 +245,8 @@ var _ = Describe("StatefulSet", func() {
245245
Kind: "RabbitmqCluster",
246246
Name: instance.Name,
247247
UID: "",
248-
Controller: &truth,
249-
BlockOwnerDeletion: &truth,
248+
Controller: pointer.BoolPtr(true),
249+
BlockOwnerDeletion: pointer.BoolPtr(false),
250250
},
251251
},
252252
},
@@ -942,45 +942,59 @@ var _ = Describe("StatefulSet", func() {
942942
Expect(stsBuilder.Update(statefulSet)).To(Succeed())
943943

944944
initContainers := statefulSet.Spec.Template.Spec.InitContainers
945-
Expect(len(initContainers)).To(Equal(1))
946-
947-
container := extractContainer(initContainers, "copy-config")
948-
Expect(container.Command).To(Equal([]string{
949-
"sh", "-c", "cp /tmp/rabbitmq/rabbitmq.conf /etc/rabbitmq/rabbitmq.conf && echo '' >> /etc/rabbitmq/rabbitmq.conf ; " +
950-
"cp /tmp/rabbitmq/advanced.config /etc/rabbitmq/advanced.config ; " +
951-
"cp /tmp/rabbitmq/rabbitmq-env.conf /etc/rabbitmq/rabbitmq-env.conf ; " +
952-
"cp /tmp/erlang-cookie-secret/.erlang.cookie /var/lib/rabbitmq/.erlang.cookie " +
953-
"&& chown 999:999 /var/lib/rabbitmq/.erlang.cookie " +
954-
"&& chmod 600 /var/lib/rabbitmq/.erlang.cookie ; " +
955-
"cp /tmp/rabbitmq-plugins/enabled_plugins /etc/rabbitmq/enabled_plugins " +
956-
"&& chown 999:999 /etc/rabbitmq/enabled_plugins",
945+
Expect(initContainers).To(HaveLen(1))
946+
947+
initContainer := extractContainer(initContainers, "setup-container")
948+
Expect(initContainer).To(MatchFields(IgnoreExtras, Fields{
949+
"Image": Equal("rabbitmq-image-from-cr"),
950+
"SecurityContext": PointTo(MatchFields(IgnoreExtras, Fields{
951+
"Capabilities": PointTo(MatchAllFields(Fields{
952+
"Drop": ConsistOf([]corev1.Capability{"ALL"}),
953+
"Add": ConsistOf([]corev1.Capability{"CHOWN", "FOWNER"}),
954+
})),
955+
})),
956+
"Command": ConsistOf(
957+
"sh", "-c", "cp /tmp/rabbitmq/rabbitmq.conf /etc/rabbitmq/rabbitmq.conf "+
958+
"&& chown 999:999 /etc/rabbitmq/rabbitmq.conf "+
959+
"&& echo '' >> /etc/rabbitmq/rabbitmq.conf ; "+
960+
"cp /tmp/rabbitmq/advanced.config /etc/rabbitmq/advanced.config "+
961+
"&& chown 999:999 /etc/rabbitmq/advanced.config ; "+
962+
"cp /tmp/rabbitmq/rabbitmq-env.conf /etc/rabbitmq/rabbitmq-env.conf "+
963+
"&& chown 999:999 /etc/rabbitmq/rabbitmq-env.conf ; "+
964+
"cp /tmp/erlang-cookie-secret/.erlang.cookie /var/lib/rabbitmq/.erlang.cookie "+
965+
"&& chown 999:999 /var/lib/rabbitmq/.erlang.cookie "+
966+
"&& chmod 600 /var/lib/rabbitmq/.erlang.cookie ; "+
967+
"cp /tmp/rabbitmq-plugins/enabled_plugins /etc/rabbitmq/enabled_plugins "+
968+
"&& chown 999:999 /etc/rabbitmq/enabled_plugins ; "+
969+
"chgrp 999 /var/lib/rabbitmq/mnesia/",
970+
),
971+
"VolumeMounts": ConsistOf(
972+
corev1.VolumeMount{
973+
Name: "server-conf",
974+
MountPath: "/tmp/rabbitmq/",
975+
},
976+
corev1.VolumeMount{
977+
Name: "plugins-conf",
978+
MountPath: "/tmp/rabbitmq-plugins/",
979+
},
980+
corev1.VolumeMount{
981+
Name: "rabbitmq-etc",
982+
MountPath: "/etc/rabbitmq/",
983+
},
984+
corev1.VolumeMount{
985+
Name: "rabbitmq-erlang-cookie",
986+
MountPath: "/var/lib/rabbitmq/",
987+
},
988+
corev1.VolumeMount{
989+
Name: "erlang-cookie-secret",
990+
MountPath: "/tmp/erlang-cookie-secret/",
991+
},
992+
corev1.VolumeMount{
993+
Name: "persistence",
994+
MountPath: "/var/lib/rabbitmq/mnesia/",
995+
},
996+
),
957997
}))
958-
959-
Expect(container.VolumeMounts).To(ConsistOf(
960-
corev1.VolumeMount{
961-
Name: "server-conf",
962-
MountPath: "/tmp/rabbitmq/",
963-
},
964-
corev1.VolumeMount{
965-
Name: "plugins-conf",
966-
MountPath: "/tmp/rabbitmq-plugins/",
967-
},
968-
969-
corev1.VolumeMount{
970-
Name: "rabbitmq-etc",
971-
MountPath: "/etc/rabbitmq/",
972-
},
973-
corev1.VolumeMount{
974-
Name: "rabbitmq-erlang-cookie",
975-
MountPath: "/var/lib/rabbitmq/",
976-
},
977-
corev1.VolumeMount{
978-
Name: "erlang-cookie-secret",
979-
MountPath: "/tmp/erlang-cookie-secret/",
980-
},
981-
))
982-
983-
Expect(container.Image).To(Equal("rabbitmq-image-from-cr"))
984998
})
985999

9861000
It("adds the required terminationGracePeriodSeconds", func() {

‎system_tests/system_tests.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import (
2323
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2424
)
2525

26-
const statefulSetSuffix = "server"
26+
const statefulSetSuffix = "server"
2727

2828
var _ = Describe("Operator", func() {
2929
var (

0 commit comments

Comments
 (0)