deploy bpfman-restricted scc with operator

As part of our default deployment in openshift we
need to deploy a custom SCC which can be used
by applications in order to receive access to their
eBPF maps without running as root.

Signed-off-by: Andrew Stoycos <astoycos@redhat.com>

Containerfile.bpfman-operator

@@ -40,6 +40,7 @@ ARG TARGETARCH
 WORKDIR /
 COPY --from=bpfman-operator-build /usr/src/bpfman-operator/config/bpfman-deployment/daemonset.yaml ./config/bpfman-deployment/daemonset.yaml
 COPY --from=bpfman-operator-build /usr/src/bpfman-operator/config/bpfman-deployment/csidriverinfo.yaml ./config/bpfman-deployment/csidriverinfo.yaml
+COPY --from=bpfman-operator-build /usr/src/bpfman-operator/config/bpfman-deployment/csidriverinfo.yaml ./config/openshift/restricted-scc.yaml
 COPY --from=bpfman-operator-build /usr/src/bpfman-operator/bpfman-operator .
 USER 65532:65532
 

cmd/bpfman-operator/main.go

@@ -28,8 +28,10 @@ import (
 	"go.uber.org/zap/zapcore"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/discovery"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/client-go/rest"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -49,6 +51,31 @@ func init() {
 	//+kubebuilder:scaffold:scheme
 }
 
+func getPlatformInfo(client discovery.DiscoveryInterface, cfg *rest.Config) (bool, error) {
+	k8sVersion, err := client.ServerVersion()
+	if err != nil {
+		setupLog.Info("issue occurred while fetching ServerVersion")
+		return false, err
+	}
+
+	setupLog.Info("detected platform version", "PlatformVersion", k8sVersion)
+
+	apiList, err := client.ServerGroups()
+	if err != nil {
+		setupLog.Info("issue occurred while fetching ServerGroups")
+		return false, err
+	}
+
+	for _, v := range apiList.Groups {
+		if v.Name == "route.openshift.io" {
+
+			setupLog.Info("route.openshift.io found in apis, platform is OpenShift")
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
 func main() {
 	var metricsAddr string
 	var enableLeaderElection bool
@@ -129,10 +156,26 @@ func main() {
 		Scheme: mgr.GetScheme(),
 	}
 
+	setupLog.Info("Discovering APIs")
+	dc, err := discovery.NewDiscoveryClientForConfig(mgr.GetConfig())
+	if err != nil {
+		setupLog.Error(err, "can't instantiate discovery client")
+		os.Exit(1)
+	}
+
+	isOpenshift, err := getPlatformInfo(dc, mgr.GetConfig())
+	if err != nil {
+		setupLog.Error(err, "unable to determine platform")
+		os.Exit(1)
+
+	}
+
 	if err = (&bpfmanoperator.BpfmanConfigReconciler{
 		ReconcilerCommon:         common,
 		BpfmanStandardDeployment: internal.BpfmanDaemonManifestPath,
 		CsiDriverDeployment:      internal.BpfmanCsiDriverPath,
+		RestrictedSCC:            internal.BpfmanRestrictedSCCPath,
+		IsOpenshift:              isOpenshift,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create bpfmanCofig controller", "controller", "BpfProgram")
 		os.Exit(1)

config/openshift/rbac.yaml

@@ -10,3 +10,17 @@ subjects:
   - kind: ServiceAccount
     name: bpfman-daemon
     namespace: bpfman
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: user
+rules:
+  - apiGroups:
+      - security.openshift.io
+    resourceNames:
+      - bpfman-restricted
+    resources:
+      - securitycontextconstraints
+    verbs:
+      - use

config/openshift/user-scc.yaml → config/openshift/restricted-scc.yaml

@@ -1,8 +1,8 @@
----
+## This is part of the payload deployed by the bpfman-operator NOT kustomize.
 apiVersion: security.openshift.io/v1
 kind: SecurityContextConstraints
 metadata:
-  name: restricted
+  name: bpfman-restricted
 allowHostDirVolumePlugin: false
 allowHostIPC: false
 allowHostNetwork: false
@@ -35,18 +35,4 @@ volumes:
   - persistentVolumeClaim
   - projected
   - secret
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: user
-rules:
-  - apiGroups:
-      - security.openshift.io
-    resourceNames:
-      - bpfman-restricted
-    resources:
-      - securitycontextconstraints
-    verbs:
-      - use
----
+

config/rbac/bpfman-operator/role.yaml

@@ -229,6 +229,16 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - security.openshift.io
+  resources:
+  - securitycontextconstraints
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
 - apiGroups:
   - storage.k8s.io
   resources:

controllers/bpfman-operator/configmap.go

@@ -33,6 +33,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 
+	osv1 "github.com/openshift/api/security/v1"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/log"
@@ -43,12 +44,15 @@ import (
 // +kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create
 // +kubebuilder:rbac:groups=storage.k8s.io,resources=csidrivers,verbs=get;list;watch;create;delete
+// +kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,verbs=get;list;watch;create;delete
 // +kubebuilder:rbac:groups=bpfman.io,resources=configmaps/finalizers,verbs=update
 
 type BpfmanConfigReconciler struct {
 	ReconcilerCommon
 	BpfmanStandardDeployment string
 	CsiDriverDeployment      string
+	RestrictedSCC            string
+	IsOpenshift              bool
 }
 
 // SetupWithManager sets up the controller with the Manager.
@@ -91,10 +95,6 @@ func (r *BpfmanConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request
 }
 
 func (r *BpfmanConfigReconciler) ReconcileBpfmanConfig(ctx context.Context, req ctrl.Request, bpfmanConfig *corev1.ConfigMap) (ctrl.Result, error) {
-	bpfmanDeployment := &appsv1.DaemonSet{}
-
-	staticBpfmanDeployment := LoadAndConfigureBpfmanDs(bpfmanConfig, r.BpfmanStandardDeployment)
-	r.Logger.V(1).Info("StaticBpfmanDeployment with CSI", "DS", staticBpfmanDeployment)
 	bpfmanCsiDriver := &storagev1.CSIDriver{}
 	// one-shot try to create bpfman's CSIDriver object if it doesn't exist, does not re-trigger reconcile.
 	if err := r.Get(ctx, types.NamespacedName{Namespace: corev1.NamespaceAll, Name: internal.BpfmanCsiDriverName}, bpfmanCsiDriver); err != nil {
@@ -109,6 +109,26 @@ func (r *BpfmanConfigReconciler) ReconcileBpfmanConfig(ctx context.Context, req
 		}
 	}
 
+	if r.IsOpenshift {
+		bpfmanRestrictedSCC := &osv1.SecurityContextConstraints{}
+		// one-shot try to create the bpfman-restricted SCC if it doesn't exist, does not re-trigger reconcile.
+		if err := r.Get(ctx, types.NamespacedName{Namespace: corev1.NamespaceAll, Name: internal.BpfmanRestrictedSccName}, bpfmanRestrictedSCC); err != nil {
+			if errors.IsNotFound(err) {
+				bpfmanRestrictedSCC = LoadRestrictedSecurityContext(r.RestrictedSCC)
+
+				r.Logger.Info("Creating Bpfman restricted scc object for unprivileged users to bind to")
+				if err := r.Create(ctx, bpfmanRestrictedSCC); err != nil {
+					r.Logger.Error(err, "Failed to create Bpfman restricted scc")
+					return ctrl.Result{Requeue: true, RequeueAfter: retryDurationOperator}, nil
+				}
+			}
+		}
+	}
+
+	bpfmanDeployment := &appsv1.DaemonSet{}
+	staticBpfmanDeployment := LoadAndConfigureBpfmanDs(bpfmanConfig, r.BpfmanStandardDeployment)
+	r.Logger.V(1).Info("StaticBpfmanDeployment with CSI", "DS", staticBpfmanDeployment)
+	// one-shot try to create the bpfman-restricted SCC if it doesn't exist, does not re-trigger reconcile.
 	if err := r.Get(ctx, types.NamespacedName{Namespace: bpfmanConfig.Namespace, Name: internal.BpfmanDsName}, bpfmanDeployment); err != nil {
 		if errors.IsNotFound(err) {
 			r.Logger.Info("Creating Bpfman Daemon")
@@ -209,8 +229,28 @@ func bpfmanConfigPredicate() predicate.Funcs {
 	}
 }
 
+// LoadRestrictedSecurityContext loads the bpfman-restricted SCC from disk which
+// users can bind to in order to utilize bpfman in an unprivileged way.
+func LoadRestrictedSecurityContext(path string) *osv1.SecurityContextConstraints {
+	// Load static SCC yaml from disk
+	file, err := os.Open(path)
+	if err != nil {
+		panic(err)
+	}
+
+	b, err := io.ReadAll(file)
+	if err != nil {
+		panic(err)
+	}
+
+	decode := scheme.Codecs.UniversalDeserializer().Decode
+	obj, _, _ := decode(b, nil, nil)
+
+	return obj.(*osv1.SecurityContextConstraints)
+}
+
 func LoadCsiDriver(path string) *storagev1.CSIDriver {
-	// Load static bpfman deployment from disk
+	// Load static CSIDriver yaml from disk
 	file, err := os.Open(path)
 	if err != nil {
 		panic(err)

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/go-logr/logr v1.4.2
 	github.com/google/go-cmp v0.6.0
 	github.com/kong/kubernetes-testing-framework v0.47.0
+	github.com/openshift/api v0.0.0-20240605201059-cefcda60d938
 	github.com/stretchr/testify v1.9.0
 	go.uber.org/zap v1.27.0
 	google.golang.org/grpc v1.64.0

go.sum

@@ -94,6 +94,8 @@ github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk=
 github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/openshift/api v0.0.0-20240605201059-cefcda60d938 h1:3Y0E8q6pFb8PgzW0N52jVZBollMeW1kI13iU/vXPCnc=
+github.com/openshift/api v0.0.0-20240605201059-cefcda60d938/go.mod h1:OOh6Qopf21pSzqNVCB5gomomBXb8o5sGKZxG2KNpaXM=
 github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

internal/constants.go

@@ -39,10 +39,12 @@ const (
 	BpfmanDsName                = "bpfman-daemon"
 	BpfmanConfigName            = "bpfman-config"
 	BpfmanCsiDriverName         = "csi.bpfman.io"
+	BpfmanRestrictedSccName     = "bpfman-restricted"
 	BpfmanContainerName         = "bpfman"
 	BpfmanAgentContainerName    = "bpfman-agent"
 	BpfmanDaemonManifestPath    = "./config/bpfman-deployment/daemonset.yaml"
 	BpfmanCsiDriverPath         = "./config/bpfman-deployment/csidriverinfo.yaml"
+	BpfmanRestrictedSCCPath     = "./config/bpfman-deployment/restricted-scc.yaml"
 	BpfmanMapFs                 = "/run/bpfman/fs/maps"
 	DefaultType                 = "tcp"
 	DefaultPath                 = "/run/bpfman-sock/bpfman.sock"