FEATURE: build operator image as non-root (#2171)

Signed-off-by: ImpSy <3097030+ImpSy@users.noreply.github.com>

Dockerfile

@@ -20,11 +20,23 @@ FROM golang:1.23.1 AS builder
 
 WORKDIR /workspace
 
-COPY . .
+RUN apt-get update \
+    && apt-get install -y libcap2-bin \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=bind,source=go.mod,target=go.mod \
+    --mount=type=bind,source=go.sum,target=go.sum \
+    go mod download
 
+COPY . .
+ENV GOCACHE=/root/.cache/go-build
 ARG TARGETARCH
 
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} GO111MODULE=on make build-operator
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=cache,target="/root/.cache/go-build" \
+    CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} GO111MODULE=on make build-operator
+RUN setcap 'cap_net_bind_service=+ep' /workspace/bin/spark-operator
 
 FROM ${SPARK_IMAGE}
 
@@ -34,6 +46,12 @@ RUN apt-get update \
     && apt-get install -y tini \
     && rm -rf /var/lib/apt/lists/*
 
+RUN mkdir -p /etc/k8s-webhook-server/serving-certs && \
+    chmod -R g+rw /etc/k8s-webhook-server/serving-certs && \
+    chown -R spark /etc/k8s-webhook-server/serving-certs
+
+USER spark
+
 COPY --from=builder /workspace/bin/spark-operator /usr/bin/spark-operator
 
 COPY entrypoint.sh /usr/bin/

cmd/operator/webhook/start.go

@@ -134,7 +134,8 @@ func NewStartCommand() *cobra.Command {
 	command.Flags().StringVar(&labelSelectorFilter, "label-selector-filter", "", "A comma-separated list of key=value, or key labels to filter resources during watch and list based on the specified labels.")
 	command.Flags().DurationVar(&cacheSyncTimeout, "cache-sync-timeout", 30*time.Second, "Informer cache sync timeout.")
 
-	command.Flags().StringVar(&webhookCertDir, "webhook-cert-dir", "/etc/k8s-webhook-server/serving-certs", "The directory that contains the webhook server key and certificate")
+	command.Flags().StringVar(&webhookCertDir, "webhook-cert-dir", "/etc/k8s-webhook-server/serving-certs", "The directory that contains the webhook server key and certificate. "+
+		"When running as nonRoot, you must create and own this directory before running this command.")
 	command.Flags().StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", "The file name of webhook server certificate.")
 	command.Flags().StringVar(&webhookKeyName, "webhook-key-name", "tls.key", "The file name of webhook server key.")
 	command.Flags().StringVar(&mutatingWebhookName, "mutating-webhook-name", "spark-operator-webhook", "The name of the mutating webhook.")