Adding sriov operator config controller finalizer, to enable generated

webhooks objects removal by sriov operator

Signed-off-by: Ido Heyvi <iheyvi@nvidia.com>

Dockerfile

@@ -2,9 +2,11 @@ FROM golang:1.22 AS builder
 WORKDIR /go/src/github.com/k8snetworkplumbingwg/sriov-network-operator
 COPY . .
 RUN make _build-manager BIN_PATH=build/_output/cmd
+RUN make _build-sriov-network-operator-config-cleanup BIN_PATH=build/_output/cmd
 
 FROM quay.io/centos/centos:stream9
 COPY --from=builder /go/src/github.com/k8snetworkplumbingwg/sriov-network-operator/build/_output/cmd/manager /usr/bin/sriov-network-operator
+COPY --from=builder /go/src/github.com/k8snetworkplumbingwg/sriov-network-operator/build/_output/cmd/sriov-network-operator-config-cleanup /usr/bin/sriov-network-operator-config-cleanup
 COPY bindata /bindata
 ENV OPERATOR_NAME=sriov-network-operator
 CMD ["/usr/bin/sriov-network-operator"]

Makefile

@@ -60,7 +60,7 @@ GOLANGCI_LINT_VER = v1.55.2
 
 all: generate lint build
 
-build: manager _build-sriov-network-config-daemon _build-webhook
+build: manager _build-sriov-network-config-daemon _build-webhook _build-sriov-network-operator-config-cleanup
 
 _build-%:
 	WHAT=$* hack/build-go.sh

api/v1/helper.go

@@ -26,11 +26,12 @@ import (
 )
 
 const (
-	LASTNETWORKNAMESPACE    = "operator.sriovnetwork.openshift.io/last-network-namespace"
-	NETATTDEFFINALIZERNAME  = "netattdef.finalizers.sriovnetwork.openshift.io"
-	POOLCONFIGFINALIZERNAME = "poolconfig.finalizers.sriovnetwork.openshift.io"
-	ESwithModeLegacy        = "legacy"
-	ESwithModeSwitchDev     = "switchdev"
+	LASTNETWORKNAMESPACE        = "operator.sriovnetwork.openshift.io/last-network-namespace"
+	NETATTDEFFINALIZERNAME      = "netattdef.finalizers.sriovnetwork.openshift.io"
+	POOLCONFIGFINALIZERNAME     = "poolconfig.finalizers.sriovnetwork.openshift.io"
+	OPERATORCONFIGFINALIZERNAME = "operatorconfig.finalizers.sriovnetwork.openshift.io"
+	ESwithModeLegacy            = "legacy"
+	ESwithModeSwitchDev         = "switchdev"
 
 	SriovCniStateEnable  = "enable"
 	SriovCniStateDisable = "disable"

bindata/manifests/operator-webhook/002-rbac.yaml

@@ -9,8 +9,6 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: operator-webhook
-  labels:
-    created-by: {{ .OperatorGeneratedResourcesLabelSelector | default "sriov-network-operator" }}
 rules:
 - apiGroups:
     - ""
@@ -34,8 +32,6 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: operator-webhook-role-binding
-  labels:
-    created-by: {{ .OperatorGeneratedResourcesLabelSelector | default "sriov-network-operator" }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole

bindata/manifests/operator-webhook/003-webhook.yaml

@@ -11,8 +11,6 @@ metadata:
     # more variables.
     cert-manager.io/inject-ca-from: {{.Namespace}}/{{.OperatorWebhookSecretName}}
     {{- end }}
-  labels:
-    created-by: {{ .OperatorGeneratedResourcesLabelSelector | default "sriov-network-operator" }}   
 webhooks:
   - name: operator-webhook.sriovnetwork.openshift.io
     sideEffects: None
@@ -45,8 +43,6 @@ metadata:
     # more variables.
     cert-manager.io/inject-ca-from: {{.Namespace}}/{{.OperatorWebhookSecretName}}
     {{- end }}
-  labels:
-    created-by: {{ .OperatorGeneratedResourcesLabelSelector | default "sriov-network-operator" }}   
 webhooks:
   - name: operator-webhook.sriovnetwork.openshift.io
     sideEffects: None

bindata/manifests/webhook/002-rbac.yaml

@@ -9,8 +9,6 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: network-resources-injector
-  labels:
-    created-by: {{ .OperatorGeneratedResourcesLabelSelector | default "sriov-network-operator" }}
 rules:
 - apiGroups:
     - k8s.cni.cncf.io
@@ -33,8 +31,6 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: network-resources-injector-role-binding
-  labels:
-    created-by: {{ .OperatorGeneratedResourcesLabelSelector | default "sriov-network-operator" }} 
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole

bindata/manifests/webhook/003-webhook.yaml

@@ -11,8 +11,6 @@ metadata:
     # more variables.
     cert-manager.io/inject-ca-from: {{.Namespace}}/{{.InjectorWebhookSecretName}}
     {{- end }}
-  labels:
-    created-by: {{ .OperatorGeneratedResourcesLabelSelector | default "sriov-network-operator" }}   
 webhooks:
   - name: network-resources-injector-config.k8s.io
     sideEffects: None

cmd/sriov-network-operator-config-cleanup/cleanup.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"context"
+	"os"
+
+	snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log"
+	"github.com/spf13/cobra"
+	"k8s.io/client-go/dynamic"
+
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned/typed/sriovnetwork/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+var (
+	namespace string
+)
+
+func init() {
+	rootCmd.Flags().StringVarP(&namespace, "namespace", "n", "", "designated SriovOperatorConfig namespace")
+}
+
+func DynamicClientFor(g schema.GroupVersionKind, obj *unstructured.Unstructured, namespace string) (dynamic.ResourceInterface, error) {
+	return nil, nil
+}
+
+func runCleanupCmd(cmd *cobra.Command, args []string) error {
+	var (
+		config *rest.Config
+		err    error
+	)
+	// init logger
+	snolog.InitLog()
+	setupLog := log.Log.WithName("sriov-network-operator-config-cleanup")
+
+	setupLog.Info("Run sriov-network-operator-config-cleanup")
+
+	// creates the in-cluster config
+	kubeconfig := os.Getenv("KUBECONFIG")
+	if kubeconfig != "" {
+		config, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
+	} else {
+		config, err = rest.InClusterConfig()
+	}
+	if err != nil {
+		setupLog.Error(err, "failed initialization k8s rest config")
+	}
+
+	sriovcs, err := sriovnetworkv1.NewForConfig(config)
+	if err != nil {
+		setupLog.Error(err, "failed to create 'sriovnetworkv1' clientset")
+	}
+
+	sriovcs.SriovOperatorConfigs(namespace).Delete(context.Background(), "default", metav1.DeleteOptions{})
+	if err != nil {
+		setupLog.Error(err, "failed to delete SriovOperatorConfig")
+		return err
+	}
+
+	return nil
+
+}

cmd/sriov-network-operator-config-cleanup/cleanup_test.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"context"
+
+	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/test/util"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/spf13/cobra"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+var (
+	testNamespace            string = "sriov-network-operator"
+	defaultSriovOperatorSpec        = sriovnetworkv1.SriovOperatorConfigSpec{
+		EnableInjector:        true,
+		EnableOperatorWebhook: true,
+		LogLevel:              2,
+		FeatureGates:          nil,
+	}
+)
+
+var _ = Describe("cleanup", Ordered, func() {
+
+	defaultSriovOperatorConfig := &sriovnetworkv1.SriovOperatorConfig{
+		ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: testNamespace},
+		Spec:       defaultSriovOperatorSpec,
+	}
+
+	BeforeEach(func() {
+		Expect(k8sClient.Create(context.Background(), defaultSriovOperatorConfig)).NotTo(HaveOccurred())
+		err := util.WaitForNamespacedObject(defaultSriovOperatorConfig, k8sClient, testNamespace, "default", util.RetryInterval, util.APITimeout*3)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("test webhook cleanup flow", func() {
+		cmd := &cobra.Command{}
+		namespace = testNamespace
+		Expect(runCleanupCmd(cmd, []string{})).Should(Succeed())
+
+		config := &sriovnetworkv1.SriovOperatorConfig{}
+		Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: "default", Namespace: testNamespace}, config)).To(MatchError(
+			ContainSubstring("sriovoperatorconfigs.sriovnetwork.openshift.io \"default\" not found")))
+
+	})
+})

cmd/sriov-network-operator-config-cleanup/main.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"flag"
+	"os"
+
+	"github.com/spf13/cobra"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log"
+)
+
+const (
+	componentName = "sriov-network-operator-config-cleanup"
+)
+
+var (
+	rootCmd = &cobra.Command{
+		Use:   componentName,
+		Short: "Removes 'default' SriovOperatorConfig",
+		Long: `Removes 'default' SriovOperatorConfig in order to cleanup non-namespaced objects e.g clusterroles/clusterrolebinding/validating/mutating webhooks
+		
+Example: sriov-network-operator-config-cleanup -n <sriov-operator ns>`,
+		RunE: runCleanupCmd,
+	}
+)
+
+func init() {
+	klog.InitFlags(nil)
+	snolog.BindFlags(flag.CommandLine)
+	rootCmd.PersistentFlags().AddGoFlagSet(flag.CommandLine)
+}
+
+func main() {
+	if err := rootCmd.Execute(); err != nil {
+		log.Log.Error(err, "Error executing", componentName)
+		os.Exit(1)
+	}
+}

cmd/sriov-network-operator-config-cleanup/suite_test.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"context"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"go.uber.org/zap/zapcore"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	//+kubebuilder:scaffold:imports
+	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/test/util"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+)
+
+var (
+	k8sClient   client.Client
+	testEnv     *envtest.Environment
+	cfg         *rest.Config
+	kubecfgPath string
+)
+
+var _ = BeforeSuite(func() {
+
+	logf.SetLogger(zap.New(
+		zap.WriteTo(GinkgoWriter),
+		zap.UseDevMode(true),
+		func(o *zap.Options) {
+			o.TimeEncoder = zapcore.RFC3339NanoTimeEncoder
+		}))
+
+	// Go to project root directory
+	err := os.Chdir("../..")
+
+	By("bootstrapping test environment")
+	testEnv = &envtest.Environment{
+		CRDDirectoryPaths:     []string{filepath.Join("config", "crd", "bases"), filepath.Join("test", "util", "crds")},
+		ErrorIfCRDPathMissing: true,
+	}
+
+	testEnv.ControlPlane.GetAPIServer().Configure().Set("disable-admission-plugins", "MutatingAdmissionWebhook", "ValidatingAdmissionWebhook")
+	cfg, err = testEnv.Start()
+	Expect(err).NotTo(HaveOccurred())
+	Expect(cfg).NotTo(BeNil())
+
+	apiserverDir := testEnv.ControlPlane.GetAPIServer().CertDir
+	println("aprserver path:", apiserverDir)
+	kubecfgPath = findKubecfg(apiserverDir, ".kubecfg")
+	println("got kubecfg:", kubecfgPath)
+	err = os.Setenv("KUBECONFIG", kubecfgPath)
+	Expect(err).NotTo(HaveOccurred())
+
+	By("registering schemes")
+	err = sriovnetworkv1.AddToScheme(scheme.Scheme)
+	Expect(err).NotTo(HaveOccurred())
+	vars.Config = cfg
+	vars.Scheme = scheme.Scheme
+	vars.Namespace = testNamespace
+
+	By("creating K8s client")
+	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
+	Expect(err).NotTo(HaveOccurred())
+	Expect(k8sClient).NotTo(BeNil())
+
+	By("creating default/common k8s objects for tests")
+	// Create test namespace
+	ns := &corev1.Namespace{
+		TypeMeta: metav1.TypeMeta{},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: testNamespace,
+		},
+		Spec:   corev1.NamespaceSpec{},
+		Status: corev1.NamespaceStatus{},
+	}
+	ctx := context.Background()
+	Expect(k8sClient.Create(ctx, ns)).Should(Succeed())
+})
+
+var _ = AfterSuite(func() {
+	By("tearing down the test environment")
+	if testEnv != nil {
+		Eventually(func() error {
+			return testEnv.Stop()
+		}, util.APITimeout, time.Second).ShouldNot(HaveOccurred())
+	}
+})
+
+func findKubecfg(path, ext string) string {
+	var cfg string
+	filepath.WalkDir(path, func(s string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if filepath.Ext(d.Name()) == ext {
+			cfg = s
+		}
+		return nil
+	})
+	return cfg
+}
+
+func TestAPIs(t *testing.T) {
+	_, reporterConfig := GinkgoConfiguration()
+
+	RegisterFailHandler(Fail)
+
+	RunSpecs(t, "operator-webhook Suite", reporterConfig)
+}