Update server-logger to function with multiple securityContext settin…

…gs (#555)

* Use EmptyVolume for /var/lib/vector and mount it to the system-logger image. Also, remove dockerRunsAsCassandra property handling

* Update kind and envtests to 1.27

* Use versioned logger image in the testing instead of :latest

Add missing shell directive

Fix checkout-strategy for the docker build job

.github/actions/run-integ-test/action.yml

@@ -52,12 +52,20 @@ runs:
       run: |
         docker load --input /tmp/k8ssandra-cass-operator.tar
         docker load --input /tmp/k8ssandra-system-logger.tar
+    - name: Set version value
+      id: vars
+      shell: bash
+      run: |
+        echo "version=$(make version)" >> $GITHUB_OUTPUT
     - name: Load image on the nodes of the cluster
       shell: bash
       run: |
         kind load docker-image --name=kind k8ssandra/cass-operator:latest
         kind load docker-image --name=kind k8ssandra/system-logger:latest
+
+        kind load docker-image --name=kind k8ssandra/cass-operator:v${{ steps.vars.outputs.version }}
+        kind load docker-image --name=kind k8ssandra/system-logger:v${{ steps.vars.outputs.version }}
     - name: Run integration test ( ${{ inputs.integration_test }} )
       shell: bash
       run: |
-        IMG=k8ssandra/cass-operator:latest make integ-test
+        IMG=k8ssandra/cass-operator:v${{ steps.vars.outputs.version }} LOG_IMG=k8ssandra/system-logger:v${{ steps.vars.outputs.version }} make integ-test

.github/workflows/kindIntegTest.yml

@@ -13,6 +13,12 @@ jobs:
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Set version value
+        id: vars
+        run: |
+          echo "version=$(make version)" >> $GITHUB_OUTPUT
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Build and push
@@ -22,7 +28,7 @@ jobs:
           file: Dockerfile
           context: .
           push: false
-          tags: k8ssandra/cass-operator:latest
+          tags: k8ssandra/cass-operator:latest, k8ssandra/cass-operator:v${{ steps.vars.outputs.version }}
           platforms: linux/amd64
           outputs: type=docker,dest=/tmp/k8ssandra-cass-operator.tar
           cache-from: type=local,src=/tmp/.buildx-cache
@@ -32,7 +38,7 @@ jobs:
         with:
           file: logger.Dockerfile
           push: false
-          tags: k8ssandra/system-logger:latest
+          tags: k8ssandra/system-logger:latest, k8ssandra/system-logger:v${{ steps.vars.outputs.version }}
           outputs: type=docker,dest=/tmp/k8ssandra-system-logger.tar
           platforms: linux/amd64
           cache-from: type=local,src=/tmp/.buildx-cache

.github/workflows/operatorBuildAndDeploy.yml

@@ -77,7 +77,7 @@ jobs:
           file: Dockerfile
           context: .
           push: ${{ github.event_name != 'pull_request' }}
-          tags: k8ssandra/cass-operator:${{ steps.vars.outputs.sha_short }}, k8ssandra/cass-operator:latest, k8ssandra/cass-operator:${{ steps.vars.outputs.version }}
+          tags: k8ssandra/cass-operator:${{ steps.vars.outputs.sha_short }}, k8ssandra/cass-operator:latest, k8ssandra/cass-operator:v${{ steps.vars.outputs.version }}
           platforms: linux/amd64
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
@@ -87,7 +87,7 @@ jobs:
         with:
           file: logger.Dockerfile
           push: ${{ github.event_name != 'pull_request' }}
-          tags: k8ssandra/system-logger:${{ steps.vars.outputs.sha_short }}, k8ssandra/system-logger:latest
+          tags: k8ssandra/system-logger:${{ steps.vars.outputs.sha_short }}, k8ssandra/system-logger:latest, k8ssandra/system-logger:v${{ steps.vars.outputs.version }}
           platforms: linux/amd64
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache

CHANGELOG.md

@@ -11,6 +11,9 @@ Changelog for Cass Operator, new PRs should update the `main / unreleased` secti
 
 ## unreleased
 
+* [CHANGE] [#553](https://github.com/k8ssandra/cass-operator/issues/553) dockerImageRunsAsCassandra is no longer used for anything as that's the default for current images. Use SecurityContext to override default SecurityContext (999:999)
+* [ENHANCEMENT] [#554](https://github.com/k8ssandra/cass-operator/issues/554) Add new empty directory as mount to server-system-logger (/var/lib/vector) so it works with multiple securityContexes
+
 ## v1.16.0
 
 * [CHANGE] [#542](https://github.com/k8ssandra/cass-operator/issues/542) Support 7.x.x version numbers for DSE and 5.x.x for Cassandra

Makefile

@@ -68,7 +68,7 @@ IMG_LATEST ?= $(IMAGE_TAG_BASE):latest
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:generateEmbeddedObjectMeta=true"
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
-ENVTEST_K8S_VERSION = 1.26.x
+ENVTEST_K8S_VERSION = 1.27.x
 
 # Logger image
 LOG_IMG_BASE ?= $(ORG)/system-logger
@@ -205,6 +205,7 @@ uninstall: manifests ## Uninstall CRDs from the K8s cluster specified in ~/.kube
 .PHONY: deploy
 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	LOG_IMG=${LOG_IMG} yq eval -i '.images.system-logger = env(LOG_IMG)' config/manager/image_config.yaml
 	kubectl apply --force-conflicts --server-side -k config/deployments/default
 
 .PHONY: undeploy
@@ -217,6 +218,7 @@ ifneq ($(strip $(NAMESPACE)),)
 	cd tests/kustomize && $(KUSTOMIZE) edit set namespace $(NAMESPACE)
 endif
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	LOG_IMG=${LOG_IMG} yq eval -i '.images.system-logger = env(LOG_IMG)' config/manager/image_config.yaml
 	kubectl apply --force-conflicts --server-side -k tests/$(TEST_DIR)
 
 .PHONY: undeploy-test
@@ -243,7 +245,7 @@ HELM ?= $(LOCALBIN)/helm
 OPM ?= $(LOCALBIN)/opm
 
 ## Tool Versions
-CERT_MANAGER_VERSION ?= v1.11.3
+CERT_MANAGER_VERSION ?= v1.12.2
 KUSTOMIZE_VERSION ?= v5.0.3
 CONTROLLER_TOOLS_VERSION ?= v0.12.0
 OPERATOR_SDK_VERSION ?= 1.29.0

README.md

@@ -246,7 +246,6 @@ spec:
         resources:
           requests:
             storage: 10Gi
-  dockerImageRunsAsCassandra: false
   resources:
     requests:
       memory: 2Gi

apis/cassandra/v1beta1/cassandradatacenter_types.go

@@ -104,8 +104,8 @@ type CassandraDatacenterSpec struct {
 	// +kubebuilder:validation:Enum=cassandra;dse
 	ServerType string `json:"serverType"`
 
-	// Does the Server Docker image run as the Cassandra user? Defaults to true
-	DockerImageRunsAsCassandra *bool `json:"dockerImageRunsAsCassandra,omitempty"`
+	// DEPRECATED This setting does nothing and defaults to true. Use SecurityContext instead.
+	DeprecatedDockerImageRunsAsCassandra *bool `json:"dockerImageRunsAsCassandra,omitempty"`
 
 	// Config for the server, in YAML format
 	// +kubebuilder:pruning:PreserveUnknownFields

apis/cassandra/v1beta1/cassandradatacenter_webhook.go

@@ -226,6 +226,10 @@ func ValidateDeprecatedFieldUsage(dc CassandraDatacenter) error {
 		}
 	}
 
+	if dc.Spec.DeprecatedDockerImageRunsAsCassandra != nil && !(*dc.Spec.DeprecatedDockerImageRunsAsCassandra) {
+		return attemptedTo("use removed field dockerImageRunsAsCassandra, use SecurityContext instead")
+	}
+
 	return nil
 }
 

config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml

@@ -284,8 +284,8 @@ spec:
                   container. Our default is to have it enabled.
                 type: boolean
               dockerImageRunsAsCassandra:
-                description: Does the Server Docker image run as the Cassandra user?
-                  Defaults to true
+                description: DEPRECATED This setting does nothing and defaults to
+                  true. Use SecurityContext instead.
                 type: boolean
               dseWorkloads:
                 properties:

docs/user/README.md

@@ -301,18 +301,6 @@ locally with minikube or another setup with a single Kubernetes worker node, you
 reduce the `size` value accordingly, or set the `allowMultipleNodesPerWorker`
 parameter to `true`.
 
-## The server image user
-
-If the server image runs as the "cassandra" or "dse" user, then a PodSecurityContext for that user will be defined by cass-operator. Otherwise the server image is assumed to be running as the "root" user and a PodSecurityContext is not defined.
-
-For serverType="dse", the server images run as the "dse" user.
-
-For serverType="cassandra", the cass-operator follows these steps in order to determine which user the docker image is run as:
-
-1. If the CassandraDatacenter.Spec.DockerImageRunsAsCassandra field is set, then that "true" or "false" value will be used.
-2. If the serverVersion field is set to "3.11.6", "3.11.7", or "4.0.0", cass-operator assumes the image runs as the "root" user.
-3. Otherwise, cass-operator assumes that the server is running as the "cassandra" user.
-
 ## Storage
 
 Define the storage with a combination of the previously provisioned storage

hack/cluster.sh

@@ -12,7 +12,7 @@ if [ "${running}" != 'true' ]; then
 fi
 
 # create a cluster with the local registry enabled in containerd
-cat <<EOF | kind create cluster --image kindest/node:v1.25.3 --config=-
+cat <<EOF | kind create cluster --image kindest/node:v1.27.3 --config=-
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 containerdConfigPatches:

logger.Dockerfile

@@ -3,15 +3,13 @@ ARG VERSION
 ARG TARGETPLATFORM
 
 # Install Vector
-ENV VECTOR_VERSION 0.29.1
+ENV VECTOR_VERSION 0.31.0
 RUN case ${TARGETPLATFORM} in \
          "linux/amd64")  VECTOR_ARCH=x86_64  ;; \
          "linux/arm64")  VECTOR_ARCH=aarch64  ;; \
     esac \
  && rpm -i https://packages.timber.io/vector/${VECTOR_VERSION}/vector-${VECTOR_VERSION}-1.${VECTOR_ARCH}.rpm
 
-RUN mkdir -p /var/lib/vector
-
 FROM redhat/ubi8-micro:latest
 
 ARG VERSION
@@ -29,11 +27,12 @@ LABEL description="Sidecar to output Cassandra system logs to stdout"
 COPY ./config/logger/vector_config.toml /etc/vector/vector.toml
 COPY --from=builder /usr/lib64/libstdc++.so.6 /usr/lib64/libstdc++.so.6
 COPY --from=builder /usr/bin/vector /usr/bin/vector
-COPY --from=builder --chown=999:999 /var/lib/vector /var/lib/vector
+#COPY --from=builder --chown=cassandra:root --chmod=775 /var/lib/vector /var/lib/vector
 
+#RUN chmod g+rw /var/lib/vector
 ADD https://raw.githubusercontent.com/vectordotdev/vector/master/LICENSE /licences/LICENSE
 COPY ./LICENSE.txt /licenses/
 
 # Non-root user, cassandra as default
-USER cassandra:cassandra
+USER cassandra
 ENTRYPOINT ["/usr/bin/vector"]

pkg/reconciliation/construct_podtemplatespec.go

@@ -304,7 +304,14 @@ func addVolumes(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTemplateSpe
 		},
 	}
 
-	volumeDefaults := []corev1.Volume{vServerConfig, vServerLogs}
+	vectorLib := corev1.Volume{
+		Name: "vector-lib",
+		VolumeSource: corev1.VolumeSource{
+			EmptyDir: &corev1.EmptyDirVolumeSource{},
+		},
+	}
+
+	volumeDefaults := []corev1.Volume{vServerConfig, vServerLogs, vectorLib}
 
 	if addLegacyInternodeMount {
 		vServerEncryption := corev1.Volume{
@@ -642,7 +649,12 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
 		}
 	}
 
-	volumeMounts = combineVolumeMountSlices([]corev1.VolumeMount{cassServerLogsMount}, loggerContainer.VolumeMounts)
+	vectorMount := corev1.VolumeMount{
+		Name:      "vector-lib",
+		MountPath: "/var/lib/vector",
+	}
+
+	volumeMounts = combineVolumeMountSlices([]corev1.VolumeMount{cassServerLogsMount, vectorMount}, loggerContainer.VolumeMounts)
 
 	loggerContainer.VolumeMounts = combineVolumeMountSlices(volumeMounts, generateStorageConfigVolumesMount(dc))
 
@@ -697,14 +709,11 @@ func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyI
 	}
 
 	if baseTemplate.Spec.SecurityContext == nil {
-		// workaround for https://cloud.google.com/kubernetes-engine/docs/security-bulletins#may-31-2019
-		if shouldDefineSecurityContext(dc) {
-			var userID int64 = 999
-			baseTemplate.Spec.SecurityContext = &corev1.PodSecurityContext{
-				RunAsUser:  &userID,
-				RunAsGroup: &userID,
-				FSGroup:    &userID,
-			}
+		var userID int64 = 999
+		baseTemplate.Spec.SecurityContext = &corev1.PodSecurityContext{
+			RunAsUser:  &userID,
+			RunAsGroup: &userID,
+			FSGroup:    &userID,
 		}
 	}
 

pkg/reconciliation/construct_podtemplatespec_test.go

@@ -541,6 +541,10 @@ func TestCassandraDatacenter_buildContainers_override_other_containers(t *testin
 				Name:      "server-logs",
 				MountPath: "/var/log/cassandra",
 			},
+			{
+				Name:      "vector-lib",
+				MountPath: "/var/lib/vector",
+			},
 		}) {
 		t.Errorf("Unexpected volume mounts for the logger container: %v", containers[0].VolumeMounts)
 	}
@@ -728,7 +732,7 @@ func TestCassandraDatacenter_buildPodTemplateSpec_add_initContainer_with_volumes
 	assert.True(t, volumeMountsContains(initContainers[1].VolumeMounts, volumeMountNameMatcher("server-config")))
 
 	volumes := podTemplateSpec.Spec.Volumes
-	assert.Equal(t, 4, len(volumes))
+	assert.Equal(t, 5, len(volumes))
 	// We use a contains check here because the ordering is not important
 	assert.True(t, volumesContains(volumes, volumeNameMatcher("server-config")))
 	assert.True(t, volumesContains(volumes, volumeNameMatcher("test-data")))
@@ -752,8 +756,9 @@ func TestCassandraDatacenter_buildPodTemplateSpec_add_initContainer_with_volumes
 	assert.NotNil(t, loggerContainer)
 
 	loggerVolumeMounts := loggerContainer.VolumeMounts
-	assert.Equal(t, 1, len(loggerVolumeMounts))
+	assert.Equal(t, 2, len(loggerVolumeMounts))
 	assert.True(t, volumeMountsContains(loggerVolumeMounts, volumeMountNameMatcher("server-logs")))
+	assert.True(t, volumeMountsContains(loggerVolumeMounts, volumeMountNameMatcher("vector-lib")))
 }
 
 func TestCassandraDatacenter_buildPodTemplateSpec_add_container_with_volumes(t *testing.T) {
@@ -832,7 +837,7 @@ func TestCassandraDatacenter_buildPodTemplateSpec_add_container_with_volumes(t *
 	assert.True(t, volumeMountsContains(serverConfigInitContainer.VolumeMounts, volumeMountNameMatcher("server-config")))
 
 	volumes := podTemplateSpec.Spec.Volumes
-	assert.Equal(t, 4, len(volumes))
+	assert.Equal(t, 5, len(volumes))
 	// We use a contains check here because the ordering is not important
 	assert.True(t, volumesContains(volumes, volumeNameMatcher("server-config")))
 	assert.True(t, volumesContains(volumes, volumeNameMatcher("test-data")))
@@ -866,8 +871,9 @@ func TestCassandraDatacenter_buildPodTemplateSpec_add_container_with_volumes(t *
 	assert.NotNil(t, loggerContainer)
 
 	loggerVolumeMounts := loggerContainer.VolumeMounts
-	assert.Equal(t, 1, len(loggerVolumeMounts))
+	assert.Equal(t, 2, len(loggerVolumeMounts))
 	assert.True(t, volumeMountsContains(loggerVolumeMounts, volumeMountNameMatcher("server-logs")))
+	assert.True(t, volumeMountsContains(loggerVolumeMounts, volumeMountNameMatcher("vector-lib")))
 
 	// Test the non-Legacy mode also (for new datacenters)
 	testZoneRack := dc.Spec.Racks[0]
@@ -877,7 +883,7 @@ func TestCassandraDatacenter_buildPodTemplateSpec_add_container_with_volumes(t *
 	assert.NoError(t, err, "should not have gotten error when building podTemplateSpec")
 
 	volumes = podTemplateSpec.Spec.Volumes
-	assert.Equal(t, 3, len(volumes))
+	assert.Equal(t, 4, len(volumes))
 	assert.False(t, volumesContains(volumes, volumeNameMatcher("encryption-cred-storage")))
 
 	containers = podTemplateSpec.Spec.Containers
@@ -1129,8 +1135,9 @@ func TestCassandraDatacenter_buildPodTemplateSpec_do_not_propagate_volumes(t *te
 	assert.NotNil(t, systemLoggerContainer)
 
 	systemLoggerVolumeMounts := systemLoggerContainer.VolumeMounts
-	assert.Equal(t, 1, len(systemLoggerVolumeMounts))
+	assert.Equal(t, 2, len(systemLoggerVolumeMounts))
 	assert.True(t, volumeMountsContains(systemLoggerVolumeMounts, volumeMountNameMatcher("server-logs")))
+	assert.True(t, volumeMountsContains(systemLoggerVolumeMounts, volumeMountNameMatcher("vector-lib")))
 }
 
 func TestCassandraDatacenter_buildPodTemplateSpec_openShift(t *testing.T) {

pkg/reconciliation/construct_statefulset.go

@@ -35,20 +35,6 @@ func newNamespacedNameForStatefulSet(
 	}
 }
 
-// Check if we need to define a SecurityContext.
-// If the user defines the DockerImageRunsAsCassandra field, we trust that.
-// Otherwise if ServerType is "dse", the answer is true.
-// Otherwise we use the logic in CalculateDockerImageRunsAsCassandra
-// to calculate a reasonable answer.
-func shouldDefineSecurityContext(dc *api.CassandraDatacenter) bool {
-	// The override field always wins
-	if dc.Spec.DockerImageRunsAsCassandra != nil {
-		return *dc.Spec.DockerImageRunsAsCassandra
-	}
-
-	return true
-}
-
 func rackNodeAffinitylabels(dc *api.CassandraDatacenter, rackName string) (map[string]string, error) {
 	var nodeAffinityLabels map[string]string
 	var log = logf.Log.WithName("construct_statefulset")