diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..80aa60dd2 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,61 @@ +version: 2 +updates: + # GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 10 + labels: + - "dependencies" + - "github-actions" + + # Docker + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 10 + labels: + - "dependencies" + - "docker" + + # Go modules - k8s dependencies + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 10 + labels: + - "dependencies" + - "go" + - "k8s" + allow: + - dependency-name: "k8s.io/*" + + # Go modules - controller-runtime dependencies + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 10 + labels: + - "dependencies" + - "go" + - "controller-runtime" + allow: + - dependency-name: "sigs.k8s.io/controller-runtime/*" + + # Go modules - general dependencies + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 10 + labels: + - "dependencies" + - "go" + - "general" + ignore: + - dependency-name: "k8s.io/*" + - dependency-name: "sigs.k8s.io/controller-runtime/*" diff --git a/.github/workflows/chart-push-release.yml b/.github/workflows/chart-push-release.yml index 912ed8fb0..851a075b7 100644 --- a/.github/workflows/chart-push-release.yml +++ b/.github/workflows/chart-push-release.yml @@ -9,7 +9,7 @@ on: - v* jobs: package-and-push-helm-chart: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: install helm uses: azure/setup-helm@v4.2.0 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 2c827b1bb..e6ba9b17e 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -11,7 +11,7 @@ on: jobs: analyze: name: Analyze - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: actions: read contents: read diff --git a/.github/workflows/image-push-master.yml b/.github/workflows/image-push-master.yml index 90ac6d48a..168d25ad1 100644 --- a/.github/workflows/image-push-master.yml +++ b/.github/workflows/image-push-master.yml @@ -11,7 +11,7 @@ on: jobs: build-and-push-image-operator: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Check out the repo uses: actions/checkout@v3 @@ -50,7 +50,7 @@ jobs: file: ./Dockerfile build-and-push-image-config-daemon: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Check out the repo uses: actions/checkout@v3 @@ -89,7 +89,7 @@ jobs: file: ./Dockerfile.sriov-network-config-daemon build-and-push-image-webhook: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Check out the repo uses: actions/checkout@v3 diff --git a/.github/workflows/image-push-release.yml b/.github/workflows/image-push-release.yml index 031a8ed52..673bb0e26 100644 --- a/.github/workflows/image-push-release.yml +++ b/.github/workflows/image-push-release.yml @@ -10,7 +10,7 @@ on: - v* jobs: build-and-push-image-operator: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Check out the repo uses: actions/checkout@v3 @@ -50,7 +50,7 @@ jobs: file: ./Dockerfile build-and-push-image-config-daemon: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Check out the repo uses: actions/checkout@v3 @@ -90,7 +90,7 @@ jobs: file: ./Dockerfile.sriov-network-config-daemon build-and-push-image-webhook: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Check out the repo uses: actions/checkout@v3 diff --git a/.github/workflows/pr-ci-triggers.yml b/.github/workflows/pr-ci-triggers.yml index 2c38f6fec..d5f243ff6 100644 --- a/.github/workflows/pr-ci-triggers.yml +++ b/.github/workflows/pr-ci-triggers.yml @@ -4,7 +4,7 @@ on: jobs: vendors-ci-triggers-list: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/github-script@v5 env: diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index 27719aa16..fa732a598 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -4,7 +4,7 @@ on: jobs: triage: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/labeler@v3 with: diff --git a/.github/workflows/pr-update.yml b/.github/workflows/pr-update.yml index 09b6e1d98..dcbe13dde 100644 --- a/.github/workflows/pr-update.yml +++ b/.github/workflows/pr-update.yml @@ -4,7 +4,7 @@ on: pull_request jobs: execute: name: "remove the lgtm label" - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: jpmcb/prow-github-actions@v1.1.2 with: diff --git a/.github/workflows/prow.yml b/.github/workflows/prow.yml index 3f25bcab8..b1eefb11a 100644 --- a/.github/workflows/prow.yml +++ b/.github/workflows/prow.yml @@ -5,7 +5,7 @@ on: jobs: execute: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: jpmcb/prow-github-actions@v1.1.2 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2fbe84c81..1c9280f20 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,13 +10,13 @@ jobs: build: name: build - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - - name: Set up Go 1.22 + - name: Set up Go 1.23 uses: actions/setup-go@v3 with: - go-version: 1.22.x + go-version: 1.23.x - name: Check out code into the Go module directory uses: actions/checkout@v2 @@ -29,27 +29,36 @@ jobs: test: name: test - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - - name: Set up Go 1.22 + - name: Set up Go 1.23 uses: actions/setup-go@v2 with: - go-version: 1.22.x + go-version: 1.23.x - name: Check out code into the Go module directory uses: actions/checkout@v2 - - name: test pkg - run: make test-pkg + - name: test pkg on kubernetes + run: CLUSTER_TYPE=kubernetes make test-pkg - - name: test cmd - run: make test-cmd + - name: test pkg on openshift + run: CLUSTER_TYPE=openshift make test-pkg - - name: test api - run: make test-api + - name: test cmd on kubernetes + run: CLUSTER_TYPE=kubernetes make test-cmd - - name: test controllers on opensfhit + - name: test cmd on openshift + run: CLUSTER_TYPE=openshift make test-cmd + + - name: test cmd on kubernetes + run: CLUSTER_TYPE=kubernetes make test-api + + - name: test api on openshift + run: CLUSTER_TYPE=openshift make test-api + + - name: test controllers on openshift run: CLUSTER_TYPE=openshift make test-controllers - name: test controllers on kubernetes @@ -60,13 +69,13 @@ jobs: modules: name: check go modules - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - - name: Set up Go 1.22 + - name: Set up Go 1.23 uses: actions/setup-go@v2 with: - go-version: 1.22.x + go-version: 1.23.x - name: Check out code into the Go module directory uses: actions/checkout@v2 @@ -76,7 +85,7 @@ jobs: manifests: name: check manifests - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Check out code uses: actions/checkout@v2 @@ -86,45 +95,64 @@ jobs: golangci: name: Golangci-lint - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - - name: Set up Go 1.22 + - name: Set up Go 1.23 uses: actions/setup-go@v2 with: - go-version: 1.22.x + go-version: 1.23.x - name: Check out code into the Go module directory uses: actions/checkout@v2 - - name: golangci-lint - uses: golangci/golangci-lint-action@v3 - with: - # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.55.2 + - name: run lint checks + run: make lint + + shellcheck: + name: Shellcheck + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - name: Run ShellCheck + uses: ludeeus/action-shellcheck@master + with: + severity: error test-coverage: name: test-coverage - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - - name: Set up Go 1.22 + - name: Set up Go 1.23 uses: actions/setup-go@v2 with: - go-version: 1.22.x + go-version: 1.23.x - name: Check out code into the Go module directory uses: actions/checkout@v2 - - name: test pkg - run: make test-pkg + - name: test pkg on kubernetes + run: CLUSTER_TYPE=kubernetes make test-pkg - - name: test cmd - run: make test-cmd + - name: test pkg on openshift + run: CLUSTER_TYPE=openshift make test-pkg - - name: test api - run: make test-api + - name: test cmd on kubernetes + run: CLUSTER_TYPE=kubernetes make test-cmd - - name: test controllers on opensfhit + - name: test cmd on openshift + run: CLUSTER_TYPE=openshift make test-cmd + + - name: test cmd on kubernetes + run: CLUSTER_TYPE=kubernetes make test-api + + - name: test api on openshift + run: CLUSTER_TYPE=openshift make test-api + + - name: test controllers on openshift run: CLUSTER_TYPE=openshift make test-controllers + - name: test controllers on kubernetes + run: CLUSTER_TYPE=kubernetes make test-controllers + - name: merge test coverage run: make merge-test-coverage @@ -148,15 +176,15 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v2 - - name: Set up Go 1.22 + - name: Set up Go 1.23 uses: actions/setup-go@v3 with: - go-version: 1.22.x + go-version: 1.23.x - name: run test run: make test-e2e-conformance-virtual-k8s-cluster-ci - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: ${{ env.TEST_REPORT_PATH }} @@ -182,15 +210,15 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v2 - - name: Set up Go 1.22 + - name: Set up Go 1.23 uses: actions/setup-go@v3 with: - go-version: 1.22.x + go-version: 1.23.x - name: run test run: make test-e2e-conformance-virtual-ocp-cluster-ci - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: ${{ env.TEST_REPORT_PATH }} diff --git a/.gitignore b/.gitignore index 6761d7ea1..8cf52d028 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,5 @@ tags .idea/ # test-environment files registry-login.conf +# go telemetry +.config/ \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index 253c4ff23..5d5c58aad 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,13 +1,6 @@ # Tested with golangci-lint ver. 1.37 run: timeout: 10m - skip-dirs: - - vendor/ - - .github/ - - deployment/ - - doc/ - - bindata/ - - pkg/client linters-settings: depguard: rules: @@ -74,11 +67,11 @@ linters: disable-all: true enable: - bodyclose + - copyloopvar - depguard - dogsled # TODO fix issues- dupl # TODO fix issues- errcheck - - exportloopref - exhaustive # TODO fix issues- funlen #- gochecknoinits @@ -107,7 +100,13 @@ linters: - whitespace issues: - # Excluding configuration per-path, per-linter, per-text and per-source + exclude-dirs: + - vendor/ + - .github/ + - deployment/ + - doc/ + - bindata/ + - pkg/client exclude-rules: - path: _test\.go linters: diff --git a/Dockerfile b/Dockerfile index 2b26247e8..a29c48d7a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,12 @@ -FROM golang:1.22 AS builder +FROM golang:1.23 AS builder WORKDIR /go/src/github.com/k8snetworkplumbingwg/sriov-network-operator COPY . . RUN make _build-manager BIN_PATH=build/_output/cmd +RUN make _build-sriov-network-operator-config-cleanup BIN_PATH=build/_output/cmd FROM quay.io/centos/centos:stream9 COPY --from=builder /go/src/github.com/k8snetworkplumbingwg/sriov-network-operator/build/_output/cmd/manager /usr/bin/sriov-network-operator +COPY --from=builder /go/src/github.com/k8snetworkplumbingwg/sriov-network-operator/build/_output/cmd/sriov-network-operator-config-cleanup /usr/bin/sriov-network-operator-config-cleanup COPY bindata /bindata ENV OPERATOR_NAME=sriov-network-operator CMD ["/usr/bin/sriov-network-operator"] diff --git a/Dockerfile.sriov-network-config-daemon b/Dockerfile.sriov-network-config-daemon index 35533448f..4ae45f817 100644 --- a/Dockerfile.sriov-network-config-daemon +++ b/Dockerfile.sriov-network-config-daemon @@ -1,4 +1,4 @@ -FROM golang:1.22 AS builder +FROM golang:1.23 AS builder WORKDIR /go/src/github.com/k8snetworkplumbingwg/sriov-network-operator COPY . . RUN make _build-sriov-network-config-daemon BIN_PATH=build/_output/cmd diff --git a/Dockerfile.webhook b/Dockerfile.webhook index 7c2e7f3d5..877cf7495 100644 --- a/Dockerfile.webhook +++ b/Dockerfile.webhook @@ -1,4 +1,4 @@ -FROM golang:1.22 AS builder +FROM golang:1.23 AS builder WORKDIR /go/src/github.com/k8snetworkplumbingwg/sriov-network-operator COPY . . RUN make _build-webhook BIN_PATH=build/_output/cmd diff --git a/Makefile b/Makefile index 3718b75bd..6e7474b85 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ export OPERATOR_EXEC?=oc BUILD_GOPATH=$(TARGET_DIR):$(TARGET_DIR)/vendor:$(CURPATH)/cmd IMAGE_BUILDER?=docker -IMAGE_BUILD_OPTS?= +IMAGE_BUILD_OPTS?=--platform linux/amd64 DOCKERFILE?=Dockerfile DOCKERFILE_CONFIG_DAEMON?=Dockerfile.sriov-network-config-daemon DOCKERFILE_WEBHOOK?=Dockerfile.webhook @@ -53,14 +53,13 @@ GOLANGCI_LINT = $(BIN_DIR)/golangci-lint # golangci-lint version should be updated periodically # we keep it fixed to avoid it from unexpectedly failing on the project # in case of a version bump -GOLANGCI_LINT_VER = v1.55.2 - +GOLANGCI_LINT_VER = v1.64.7 .PHONY: all build clean gendeepcopy test test-e2e test-e2e-k8s run image fmt sync-manifests test-e2e-conformance manifests update-codegen all: generate lint build -build: manager _build-sriov-network-config-daemon _build-webhook +build: manager _build-sriov-network-config-daemon _build-webhook _build-sriov-network-operator-config-cleanup _build-%: WHAT=$* hack/build-go.sh @@ -69,9 +68,6 @@ clean: @rm -rf $(TARGET_DIR) @rm -rf $(BIN_DIR) -update-codegen: - hack/update-codegen.sh - image: ; $(info Building images...) $(IMAGE_BUILDER) build -f $(DOCKERFILE) -t $(IMAGE_TAG) $(CURPATH) $(IMAGE_BUILD_OPTS) $(IMAGE_BUILDER) build -f $(DOCKERFILE_CONFIG_DAEMON) -t $(CONFIG_DAEMON_IMAGE_TAG) $(CURPATH) $(IMAGE_BUILD_OPTS) @@ -161,7 +157,7 @@ envtest: ## Download envtest-setup locally if necessary. GOMOCK = $(shell pwd)/bin/mockgen gomock: - $(call go-install-tool,$(GOMOCK),github.com/golang/mock/mockgen@v1.6.0) + $(call go-install-tool,$(GOMOCK),go.uber.org/mock/mockgen@v0.5.0) GINKGO = $(BIN_DIR)/ginkgo ginkgo: @@ -226,10 +222,10 @@ test-e2e-k8s: export NAMESPACE=sriov-network-operator test-e2e-k8s: test-e2e test-bindata-scripts: fakechroot - fakechroot ./test/scripts/enable-kargs_test.sh + fakechroot ./test/scripts/kargs_test.sh test-%: generate manifests envtest - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir=/tmp -p path)" HOME="$(shell pwd)" go test ./$*/... -coverprofile cover-$*.out -coverpkg ./... -v + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir=/tmp -p path)" HOME="$(shell pwd)" go test `go list ./$*/... | grep -v "/mock" | grep -v "/pkg/client"` -coverprofile cover-$*-$(CLUSTER_TYPE).out -coverpkg ./... -v GOCOVMERGE = $(BIN_DIR)/gocovmerge gocovmerge: ## Download gocovmerge locally if necessary. diff --git a/api/v1/helper.go b/api/v1/helper.go index bfdfbc473..91aee40ee 100644 --- a/api/v1/helper.go +++ b/api/v1/helper.go @@ -6,14 +6,15 @@ import ( "fmt" "os" "path/filepath" - "reflect" "regexp" "slices" "sort" "strconv" "strings" + "time" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" uns "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" intstrutil "k8s.io/apimachinery/pkg/util/intstr" @@ -26,11 +27,12 @@ import ( ) const ( - LASTNETWORKNAMESPACE = "operator.sriovnetwork.openshift.io/last-network-namespace" - NETATTDEFFINALIZERNAME = "netattdef.finalizers.sriovnetwork.openshift.io" - POOLCONFIGFINALIZERNAME = "poolconfig.finalizers.sriovnetwork.openshift.io" - ESwithModeLegacy = "legacy" - ESwithModeSwitchDev = "switchdev" + LASTNETWORKNAMESPACE = "operator.sriovnetwork.openshift.io/last-network-namespace" + NETATTDEFFINALIZERNAME = "netattdef.finalizers.sriovnetwork.openshift.io" + POOLCONFIGFINALIZERNAME = "poolconfig.finalizers.sriovnetwork.openshift.io" + OPERATORCONFIGFINALIZERNAME = "operatorconfig.finalizers.sriovnetwork.openshift.io" + ESwithModeLegacy = "legacy" + ESwithModeSwitchDev = "switchdev" SriovCniStateEnable = "enable" SriovCniStateDisable = "disable" @@ -141,53 +143,6 @@ func IsVfSupportedModel(vendorID, deviceID string) bool { return false } -func IsEnabledUnsupportedVendor(vendorID string, unsupportedNicIDMap map[string]string) bool { - for _, n := range unsupportedNicIDMap { - if IsValidPciString(n) { - ids := strings.Split(n, " ") - if vendorID == ids[0] { - return true - } - } - } - return false -} - -func IsValidPciString(nicIDString string) bool { - ids := strings.Split(nicIDString, " ") - - if len(ids) != 3 { - log.Info("IsValidPciString(): ", nicIDString) - return false - } - - if len(ids[0]) != 4 { - log.Info("IsValidPciString():", "Invalid vendor PciId ", ids[0]) - return false - } - if _, err := strconv.ParseInt(ids[0], 16, 32); err != nil { - log.Info("IsValidPciString():", "Invalid vendor PciId ", ids[0]) - } - - if len(ids[1]) != 4 { - log.Info("IsValidPciString():", "Invalid PciId of PF ", ids[1]) - return false - } - if _, err := strconv.ParseInt(ids[1], 16, 32); err != nil { - log.Info("IsValidPciString():", "Invalid PciId of PF ", ids[1]) - } - - if len(ids[2]) != 4 { - log.Info("IsValidPciString():", "Invalid PciId of VF ", ids[2]) - return false - } - if _, err := strconv.ParseInt(ids[2], 16, 32); err != nil { - log.Info("IsValidPciString():", "Invalid PciId of VF ", ids[2]) - } - - return true -} - func GetSupportedVfIds() []string { var vfIds []string for _, n := range NicIDMap { @@ -231,15 +186,6 @@ func ContainsSwitchdevInterface(interfaces []Interface) bool { return false } -func FindInterface(interfaces Interfaces, name string) (iface Interface, err error) { - for _, i := range interfaces { - if i.Name == name { - return i, nil - } - } - return Interface{}, fmt.Errorf("unable to find interface: %v", name) -} - // GetEswitchModeFromSpec returns ESwitchMode from the interface spec, returns legacy if not set func GetEswitchModeFromSpec(ifaceSpec *Interface) string { if ifaceSpec.EswitchMode == "" { @@ -260,23 +206,23 @@ func NeedToUpdateSriov(ifaceSpec *Interface, ifaceStatus *InterfaceExt) bool { if ifaceSpec.Mtu > 0 { mtu := ifaceSpec.Mtu if mtu > ifaceStatus.Mtu { - log.V(2).Info("NeedToUpdateSriov(): MTU needs update", "desired", mtu, "current", ifaceStatus.Mtu) + log.V(0).Info("NeedToUpdateSriov(): MTU needs update", "desired", mtu, "current", ifaceStatus.Mtu) return true } } currentEswitchMode := GetEswitchModeFromStatus(ifaceStatus) desiredEswitchMode := GetEswitchModeFromSpec(ifaceSpec) if currentEswitchMode != desiredEswitchMode { - log.V(2).Info("NeedToUpdateSriov(): EswitchMode needs update", "desired", desiredEswitchMode, "current", currentEswitchMode) + log.V(0).Info("NeedToUpdateSriov(): EswitchMode needs update", "desired", desiredEswitchMode, "current", currentEswitchMode) return true } if ifaceSpec.NumVfs != ifaceStatus.NumVfs { - log.V(2).Info("NeedToUpdateSriov(): NumVfs needs update", "desired", ifaceSpec.NumVfs, "current", ifaceStatus.NumVfs) + log.V(0).Info("NeedToUpdateSriov(): NumVfs needs update", "desired", ifaceSpec.NumVfs, "current", ifaceStatus.NumVfs) return true } if ifaceStatus.LinkAdminState == consts.LinkAdminStateDown { - log.V(2).Info("NeedToUpdateSriov(): PF link status needs update", "desired to include", "up", "current", ifaceStatus.LinkAdminState) + log.V(0).Info("NeedToUpdateSriov(): PF link status needs update", "desired to include", "up", "current", ifaceStatus.LinkAdminState) return true } @@ -285,24 +231,24 @@ func NeedToUpdateSriov(ifaceSpec *Interface, ifaceStatus *InterfaceExt) bool { for _, groupSpec := range ifaceSpec.VfGroups { if IndexInRange(vfStatus.VfID, groupSpec.VfRange) { if vfStatus.Driver == "" { - log.V(2).Info("NeedToUpdateSriov(): Driver needs update - has no driver", + log.V(0).Info("NeedToUpdateSriov(): Driver needs update - has no driver", "desired", groupSpec.DeviceType) return true } if groupSpec.DeviceType != "" && groupSpec.DeviceType != consts.DeviceTypeNetDevice { if groupSpec.DeviceType != vfStatus.Driver { - log.V(2).Info("NeedToUpdateSriov(): Driver needs update", + log.V(0).Info("NeedToUpdateSriov(): Driver needs update", "desired", groupSpec.DeviceType, "current", vfStatus.Driver) return true } } else { if StringInArray(vfStatus.Driver, vars.DpdkDrivers) { - log.V(2).Info("NeedToUpdateSriov(): Driver needs update", + log.V(0).Info("NeedToUpdateSriov(): Driver needs update", "desired", groupSpec.DeviceType, "current", vfStatus.Driver) return true } if vfStatus.Mtu != 0 && groupSpec.Mtu != 0 && vfStatus.Mtu != groupSpec.Mtu { - log.V(2).Info("NeedToUpdateSriov(): VF MTU needs update", + log.V(0).Info("NeedToUpdateSriov(): VF MTU needs update", "vf", vfStatus.VfID, "desired", groupSpec.Mtu, "current", vfStatus.Mtu) return true } @@ -312,20 +258,20 @@ func NeedToUpdateSriov(ifaceSpec *Interface, ifaceStatus *InterfaceExt) bool { // Node GUID. We intentionally skip empty Node GUID in vfStatus because this may happen // when the VF is allocated to a workload. if vfStatus.GUID == consts.UninitializedNodeGUID { - log.V(2).Info("NeedToUpdateSriov(): VF GUID needs update", + log.V(0).Info("NeedToUpdateSriov(): VF GUID needs update", "vf", vfStatus.VfID, "current", vfStatus.GUID) return true } } // this is needed to be sure the admin mac address is configured as expected if ifaceSpec.ExternallyManaged { - log.V(2).Info("NeedToUpdateSriov(): need to update the device as it's externally manage", + log.V(0).Info("NeedToUpdateSriov(): need to update the device as it's externally manage", "device", ifaceStatus.PciAddress) return true } } if groupSpec.VdpaType != vfStatus.VdpaType { - log.V(2).Info("NeedToUpdateSriov(): VF VdpaType mismatch", + log.V(0).Info("NeedToUpdateSriov(): VF VdpaType mismatch", "desired", groupSpec.VdpaType, "current", vfStatus.VdpaType) return true } @@ -482,6 +428,10 @@ func (p *SriovNetworkNodePolicy) ApplyBridgeConfig(state *SriovNetworkNodeState) Interface: p.Spec.Bridge.OVS.Uplink.Interface, }}, } + if p.Spec.Mtu > 0 { + mtu := p.Spec.Mtu + ovsBridge.Uplinks[0].Interface.MTURequest = &mtu + } log.Info("Update bridge for interface", "name", iface.Name, "bridge", ovsBridge.Name) // We need to keep slices with bridges ordered to avoid unnecessary updates in the K8S API. @@ -721,7 +671,13 @@ func (cr *SriovIBNetwork) RenderNetAttDef() (*uns.Unstructured, error) { data.Data["CapabilitiesConfigured"] = true data.Data["SriovCniCapabilities"] = cr.Spec.Capabilities } - + if cr.Spec.PKey == "" { + data.Data["pKeyConfigured"] = false + } else { + data.Data["pKeyConfigured"] = true + data.Data["pKey"] = cr.Spec.PKey + data.Data["GUIDSavedInUFM"] = false + } if cr.Spec.IPAM != "" { data.Data["SriovCniIpam"] = SriovCniIpam + ":" + strings.Join(strings.Fields(cr.Spec.IPAM), "") } else { @@ -764,6 +720,7 @@ func (cr *SriovNetwork) RenderNetAttDef() (*uns.Unstructured, error) { data := render.MakeRenderData() data.Data["CniType"] = "sriov" data.Data["SriovNetworkName"] = cr.Name + data.Data["pKeyConfigured"] = false if cr.Spec.NetworkNamespace == "" { data.Data["SriovNetworkNamespace"] = cr.Namespace } else { @@ -784,7 +741,6 @@ func (cr *SriovNetwork) RenderNetAttDef() (*uns.Unstructured, error) { data.Data["VlanProtoConfigured"] = true data.Data["SriovCniVlanProto"] = cr.Spec.VlanProto } - if cr.Spec.Capabilities == "" { data.Data["CapabilitiesConfigured"] = false } else { @@ -882,6 +838,7 @@ func (cr *OVSNetwork) RenderNetAttDef() (*uns.Unstructured, error) { data := render.MakeRenderData() data.Data["CniType"] = "ovs" data.Data["NetworkName"] = cr.Name + data.Data["pKeyConfigured"] = false if cr.Spec.NetworkNamespace == "" { data.Data["NetworkNamespace"] = cr.Namespace } else { @@ -1002,5 +959,45 @@ func GenerateBridgeName(iface *InterfaceExt) string { // NeedToUpdateBridges returns true if bridge for the host requires update func NeedToUpdateBridges(bridgeSpec, bridgeStatus *Bridges) bool { - return !reflect.DeepEqual(bridgeSpec, bridgeStatus) + return !equality.Semantic.DeepEqual(bridgeSpec, bridgeStatus) +} + +// SetKeepUntilTime sets an annotation to hold the "keep until time" for the node’s state. +// The "keep until time" specifies the earliest time at which the state object can be removed +// if the daemon's pod is not found on the node. +func (s *SriovNetworkNodeState) SetKeepUntilTime(t time.Time) { + ts := t.Format(time.RFC3339) + annotations := s.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[consts.NodeStateKeepUntilAnnotation] = ts + s.SetAnnotations(annotations) +} + +// GetKeepUntilTime returns the value that is stored in the "keep until time" annotation. +// The "keep until time" specifies the earliest time at which the state object can be removed +// if the daemon's pod is not found on the node. +// Return zero time instant if annotaion is not found on the object or if it has a wrong format. +func (s *SriovNetworkNodeState) GetKeepUntilTime() time.Time { + t, err := time.Parse(time.RFC3339, s.GetAnnotations()[consts.NodeStateKeepUntilAnnotation]) + if err != nil { + return time.Time{} + } + return t +} + +// ResetKeepUntilTime removes "keep until time" annotation from the state object. +// The "keep until time" specifies the earliest time at which the state object can be removed +// if the daemon's pod is not found on the node. +// Returns true if the value was removed, false otherwise. +func (s *SriovNetworkNodeState) ResetKeepUntilTime() bool { + annotations := s.GetAnnotations() + _, exist := annotations[consts.NodeStateKeepUntilAnnotation] + if !exist { + return false + } + delete(annotations, consts.NodeStateKeepUntilAnnotation) + s.SetAnnotations(annotations) + return true } diff --git a/api/v1/sriovibnetwork_types.go b/api/v1/sriovibnetwork_types.go index d8634d9ca..b54b76433 100644 --- a/api/v1/sriovibnetwork_types.go +++ b/api/v1/sriovibnetwork_types.go @@ -43,6 +43,7 @@ type SriovIBNetworkSpec struct { // MetaPluginsConfig configuration to be used in order to chain metaplugins to the sriov interface returned // by the operator. MetaPluginsConfig string `json:"metaPlugins,omitempty"` + PKey string `json:"pKey,omitempty"` } // SriovIBNetworkStatus defines the observed state of SriovIBNetwork diff --git a/api/v1/sriovnetworknodepolicy_types.go b/api/v1/sriovnetworknodepolicy_types.go index 40c53e0bf..a4417ed65 100644 --- a/api/v1/sriovnetworknodepolicy_types.go +++ b/api/v1/sriovnetworknodepolicy_types.go @@ -125,6 +125,8 @@ type OVSInterfaceConfig struct { ExternalIDs map[string]string `json:"externalIDs,omitempty"` // other_config field in the Interface table in OVSDB OtherConfig map[string]string `json:"otherConfig,omitempty"` + // mtu_request field in the Interface table in OVSDB + MTURequest *int `json:"mtuRequest,omitempty"` } // SriovNetworkNodePolicyStatus defines the observed state of SriovNetworkNodePolicy diff --git a/api/v1/sriovnetworknodestate_types.go b/api/v1/sriovnetworknodestate_types.go index 4b90d61d2..e5f59d71c 100644 --- a/api/v1/sriovnetworknodestate_types.go +++ b/api/v1/sriovnetworknodestate_types.go @@ -27,6 +27,7 @@ import ( type SriovNetworkNodeStateSpec struct { Interfaces Interfaces `json:"interfaces,omitempty"` Bridges Bridges `json:"bridges,omitempty"` + System System `json:"system,omitempty"` } type Interfaces []Interface @@ -114,10 +115,17 @@ type OVSUplinkConfigExt struct { Interface OVSInterfaceConfig `json:"interface,omitempty"` } +type System struct { + // +kubebuilder:validation:Enum=shared;exclusive + //RDMA subsystem. Allowed value "shared", "exclusive". + RdmaMode string `json:"rdmaMode,omitempty"` +} + // SriovNetworkNodeStateStatus defines the observed state of SriovNetworkNodeState type SriovNetworkNodeStateStatus struct { Interfaces InterfaceExts `json:"interfaces,omitempty"` Bridges Bridges `json:"bridges,omitempty"` + System System `json:"system,omitempty"` SyncStatus string `json:"syncStatus,omitempty"` LastSyncError string `json:"lastSyncError,omitempty"` } diff --git a/api/v1/sriovnetworkpoolconfig_types.go b/api/v1/sriovnetworkpoolconfig_types.go index c6e710a99..011ffc7d9 100644 --- a/api/v1/sriovnetworkpoolconfig_types.go +++ b/api/v1/sriovnetworkpoolconfig_types.go @@ -21,6 +21,10 @@ type SriovNetworkPoolConfigSpec struct { // Drain will respect Pod Disruption Budgets (PDBs) such as etcd quorum guards, // even if maxUnavailable is greater than one. MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` + + // +kubebuilder:validation:Enum=shared;exclusive + // RDMA subsystem. Allowed value "shared", "exclusive". + RdmaMode string `json:"rdmaMode,omitempty"` } type OvsHardwareOffloadConfig struct { diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index fc9477593..0d9d3c4cf 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -264,6 +264,11 @@ func (in *OVSInterfaceConfig) DeepCopyInto(out *OVSInterfaceConfig) { (*out)[key] = val } } + if in.MTURequest != nil { + in, out := &in.MTURequest, &out.MTURequest + *out = new(int) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OVSInterfaceConfig. @@ -783,6 +788,7 @@ func (in *SriovNetworkNodeStateSpec) DeepCopyInto(out *SriovNetworkNodeStateSpec } } in.Bridges.DeepCopyInto(&out.Bridges) + out.System = in.System } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SriovNetworkNodeStateSpec. @@ -806,6 +812,7 @@ func (in *SriovNetworkNodeStateStatus) DeepCopyInto(out *SriovNetworkNodeStateSt } } in.Bridges.DeepCopyInto(&out.Bridges) + out.System = in.System } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SriovNetworkNodeStateStatus. @@ -1066,6 +1073,21 @@ func (in *SriovOperatorConfigStatus) DeepCopy() *SriovOperatorConfigStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *System) DeepCopyInto(out *System) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new System. +func (in *System) DeepCopy() *System { + if in == nil { + return nil + } + out := new(System) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TrunkConfig) DeepCopyInto(out *TrunkConfig) { *out = *in diff --git a/bindata/manifests/cni-config/sriov/sriov-cni-config.yaml b/bindata/manifests/cni-config/sriov/sriov-cni-config.yaml index 749e326c7..0d2fe26f3 100644 --- a/bindata/manifests/cni-config/sriov/sriov-cni-config.yaml +++ b/bindata/manifests/cni-config/sriov/sriov-cni-config.yaml @@ -38,6 +38,10 @@ spec: {{- if .CapabilitiesConfigured -}} "capabilities":{{.SriovCniCapabilities}}, {{- end -}} +{{- if .pKeyConfigured -}} + "pkey":"{{.pKey}}", + "guid_saved_in_ufm": "{{.GUIDSavedInUFM}}", +{{- end -}} {{- if .StateConfigured -}} "link_state":"{{.SriovCniState}}", {{- end -}} diff --git a/bindata/manifests/daemon/daemonset.yaml b/bindata/manifests/daemon/daemonset.yaml index c9b151520..a70337f7d 100644 --- a/bindata/manifests/daemon/daemonset.yaml +++ b/bindata/manifests/daemon/daemonset.yaml @@ -116,7 +116,15 @@ spec: command: - /bin/bash - -c - - mkdir -p /host/var/lib/sriov/ && cp /usr/bin/sriov-network-config-daemon /host/var/lib/sriov/sriov-network-config-daemon && chcon -t bin_t /host/var/lib/sriov/sriov-network-config-daemon | true # Allow systemd to run the file, use pipe true to not failed if the system doesn't have selinux or apparmor enabled + - | + set -e + if [ ! -f /host/usr/share/hwdata/pci.ids ]; then # If pci.ids file is missing on the host, config daemon won't be able to discover PCI devices + mkdir -p /host/usr/share/hwdata/ + cp /usr/share/hwdata/pci.ids /host/usr/share/hwdata/pci.ids + fi + mkdir -p /host/var/lib/sriov/ + cp /usr/bin/sriov-network-config-daemon /host/var/lib/sriov/sriov-network-config-daemon + chcon -t bin_t /host/var/lib/sriov/sriov-network-config-daemon || true # Allow systemd to run the file, use pipe true to not failed if the system doesn't have selinux or apparmor enabled securityContext: privileged: true resources: diff --git a/bindata/manifests/metrics-exporter/metrics-prometheus-rule.yaml b/bindata/manifests/metrics-exporter/metrics-prometheus-rule.yaml new file mode 100644 index 000000000..a385fa677 --- /dev/null +++ b/bindata/manifests/metrics-exporter/metrics-prometheus-rule.yaml @@ -0,0 +1,38 @@ +--- +{{ if and .IsPrometheusOperatorInstalled .PrometheusOperatorDeployRules }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: sriov-vf-rules + namespace: {{.Namespace}} +spec: + groups: + - name: sriov-network-metrics-operator.rules + interval: 30s + rules: + - expr: | + sriov_vf_tx_packets * on (pciAddr,node) group_left(pod,namespace,dev_type) sriov_kubepoddevice + record: network:sriov_vf_tx_packets + - expr: | + sriov_vf_rx_packets * on (pciAddr,node) group_left(pod,namespace,dev_type) sriov_kubepoddevice + record: network:sriov_vf_rx_packets + - expr: | + sriov_vf_tx_bytes * on (pciAddr,node) group_left(pod,namespace,dev_type) sriov_kubepoddevice + record: network:sriov_vf_tx_bytes + - expr: | + sriov_vf_rx_bytes * on (pciAddr,node) group_left(pod,namespace,dev_type) sriov_kubepoddevice + record: network:sriov_vf_rx_bytes + - expr: | + sriov_vf_tx_dropped * on (pciAddr,node) group_left(pod,namespace,dev_type) sriov_kubepoddevice + record: network:sriov_vf_tx_dropped + - expr: | + sriov_vf_rx_dropped * on (pciAddr,node) group_left(pod,namespace,dev_type) sriov_kubepoddevice + record: network:sriov_vf_rx_dropped + - expr: | + sriov_vf_rx_broadcast * on (pciAddr,node) group_left(pod,namespace,dev_type) sriov_kubepoddevice + record: network:sriov_vf_rx_broadcast + - expr: | + sriov_vf_rx_multicast * on (pciAddr,node) group_left(pod,namespace,dev_type) sriov_kubepoddevice + record: network:sriov_vf_rx_multicast +{{ end }} + diff --git a/bindata/manifests/metrics-exporter/metrics-prometheus.yaml b/bindata/manifests/metrics-exporter/metrics-prometheus.yaml index 45ae7adbf..d1772a554 100644 --- a/bindata/manifests/metrics-exporter/metrics-prometheus.yaml +++ b/bindata/manifests/metrics-exporter/metrics-prometheus.yaml @@ -12,6 +12,17 @@ spec: bearerTokenFile: "/var/run/secrets/kubernetes.io/serviceaccount/token" scheme: "https" honorLabels: true + relabelings: + - action: replace + sourceLabels: + - __meta_kubernetes_endpoint_node_name + targetLabel: node + - action: labeldrop + regex: pod + - action: labeldrop + regex: container + - action: labeldrop + regex: namespace tlsConfig: serverName: sriov-network-metrics-exporter-service.{{.Namespace}}.svc caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt diff --git a/bindata/manifests/operator-webhook/003-webhook.yaml b/bindata/manifests/operator-webhook/003-webhook.yaml index 9181f1edf..a7768e1bc 100644 --- a/bindata/manifests/operator-webhook/003-webhook.yaml +++ b/bindata/manifests/operator-webhook/003-webhook.yaml @@ -69,3 +69,15 @@ webhooks: apiGroups: [ "sriovnetwork.openshift.io" ] apiVersions: [ "v1" ] resources: [ "sriovnetworkpoolconfigs" ] + - operations: [ "CREATE", "UPDATE", ] + apiGroups: [ "sriovnetwork.openshift.io" ] + apiVersions: [ "v1" ] + resources: [ "sriovnetworks" ] + - operations: [ "CREATE", "UPDATE", ] + apiGroups: [ "sriovnetwork.openshift.io" ] + apiVersions: [ "v1" ] + resources: [ "sriovibnetworks" ] + - operations: [ "CREATE", "UPDATE", ] + apiGroups: [ "sriovnetwork.openshift.io" ] + apiVersions: [ "v1" ] + resources: [ "ovsnetworks" ] diff --git a/bindata/manifests/operator-webhook/004-networkpolicy.yaml b/bindata/manifests/operator-webhook/004-networkpolicy.yaml new file mode 100644 index 000000000..5acca11fd --- /dev/null +++ b/bindata/manifests/operator-webhook/004-networkpolicy.yaml @@ -0,0 +1,20 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: operator-webhook-allow-traffic-api-server + namespace: {{.Namespace}} +spec: + podSelector: + matchLabels: + app: operator-webhook + ingress: + - ports: + - protocol: TCP + port: 6443 + egress: + - ports: + - protocol: TCP + port: 6443 + policyTypes: + - Ingress + - Egress diff --git a/bindata/manifests/plugins/sriov-device-plugin.yaml b/bindata/manifests/plugins/sriov-device-plugin.yaml index a0f433a06..3660ebf79 100644 --- a/bindata/manifests/plugins/sriov-device-plugin.yaml +++ b/bindata/manifests/plugins/sriov-device-plugin.yaml @@ -27,7 +27,7 @@ spec: hostNetwork: true nodeSelector: {{- range $key, $value := .NodeSelectorField }} - {{ $key }}: {{ $value }} + {{ $key }}: "{{ $value }}" {{- end }} tolerations: - operator: Exists diff --git a/bindata/manifests/sriov-config-service/kubernetes/sriov-config-post-network-service.yaml b/bindata/manifests/sriov-config-service/kubernetes/sriov-config-post-network-service.yaml index 08c8bce76..b0e7c12b0 100644 --- a/bindata/manifests/sriov-config-service/kubernetes/sriov-config-post-network-service.yaml +++ b/bindata/manifests/sriov-config-service/kubernetes/sriov-config-post-network-service.yaml @@ -1,12 +1,12 @@ contents: | [Unit] Description=Configures SRIOV NIC - post network configuration - After=systemd-networkd-wait-online.service NetworkManager-wait-online.service openvswitch-switch.service + After=systemd-networkd-wait-online.service NetworkManager-wait-online.service openvswitch-switch.service sriov-config.service Before=kubelet.service [Service] Type=oneshot - ExecStart=/var/lib/sriov/sriov-network-config-daemon -v 2 --zap-log-level 2 service --phase post + ExecStart=/var/lib/sriov/sriov-network-config-daemon service --phase post StandardOutput=journal+console [Install] diff --git a/bindata/manifests/sriov-config-service/kubernetes/sriov-config-service.yaml b/bindata/manifests/sriov-config-service/kubernetes/sriov-config-service.yaml index f75062dab..dce2d489b 100644 --- a/bindata/manifests/sriov-config-service/kubernetes/sriov-config-service.yaml +++ b/bindata/manifests/sriov-config-service/kubernetes/sriov-config-service.yaml @@ -7,7 +7,7 @@ contents: | [Service] Type=oneshot - ExecStart=/var/lib/sriov/sriov-network-config-daemon -v 2 --zap-log-level 2 service --phase pre + ExecStart=/var/lib/sriov/sriov-network-config-daemon service --phase pre StandardOutput=journal+console [Install] diff --git a/bindata/manifests/sriov-config-service/openshift/sriov-config-service.yaml b/bindata/manifests/sriov-config-service/openshift/sriov-config-service.yaml index 706679430..89d1b79ad 100644 --- a/bindata/manifests/sriov-config-service/openshift/sriov-config-service.yaml +++ b/bindata/manifests/sriov-config-service/openshift/sriov-config-service.yaml @@ -21,7 +21,7 @@ spec: [Service] Type=oneshot - ExecStart=/var/lib/sriov/sriov-network-config-daemon service -v {{ .LogLevel }} --zap-log-level {{ .LogLevel }} --phase pre + ExecStart=/var/lib/sriov/sriov-network-config-daemon service --phase pre StandardOutput=journal+console [Install] @@ -33,12 +33,12 @@ spec: # Removal of this file signals firstboot completion ConditionPathExists=!/etc/ignition-machine-config-encapsulated.json Description=Configures SRIOV NIC - post network configuration - After=systemd-networkd-wait-online.service NetworkManager-wait-online.service openvswitch-switch.service + After=systemd-networkd-wait-online.service NetworkManager-wait-online.service openvswitch-switch.service sriov-config.service Before=kubelet.service [Service] Type=oneshot - ExecStart=/var/lib/sriov/sriov-network-config-daemon service -v {{ .LogLevel }} --zap-log-level {{ .LogLevel }} --phase post + ExecStart=/var/lib/sriov/sriov-network-config-daemon service --phase post StandardOutput=journal+console [Install] diff --git a/bindata/manifests/webhook/002-rbac.yaml b/bindata/manifests/webhook/002-rbac.yaml index 77b2d95d7..32affca29 100644 --- a/bindata/manifests/webhook/002-rbac.yaml +++ b/bindata/manifests/webhook/002-rbac.yaml @@ -21,7 +21,7 @@ rules: - apiGroups: - "" resources: - - configmap + - configmaps verbs: - 'watch' - 'list' diff --git a/bindata/manifests/webhook/005-networkpolicy.yaml b/bindata/manifests/webhook/005-networkpolicy.yaml new file mode 100644 index 000000000..17ab13895 --- /dev/null +++ b/bindata/manifests/webhook/005-networkpolicy.yaml @@ -0,0 +1,20 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: network-resources-injector-allow-traffic-api-server + namespace: {{.Namespace}} +spec: + podSelector: + matchLabels: + app: network-resources-injector + ingress: + - ports: + - protocol: TCP + port: 6443 + egress: + - ports: + - protocol: TCP + port: 6443 + policyTypes: + - Ingress + - Egress diff --git a/bindata/scripts/enable-kargs.sh b/bindata/scripts/enable-kargs.sh deleted file mode 100755 index 0dc18c784..000000000 --- a/bindata/scripts/enable-kargs.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -set -x - -declare -a kargs=( "$@" ) -ret=0 -args=$(chroot /host/ cat /proc/cmdline) - -if chroot /host/ test -f /run/ostree-booted ; then - for t in "${kargs[@]}";do - if [[ $args != *${t}* ]];then - if chroot /host/ rpm-ostree kargs | grep -vq ${t}; then - chroot /host/ rpm-ostree kargs --append ${t} > /dev/null 2>&1 - fi - let ret++ - fi - done -else - chroot /host/ which grubby > /dev/null 2>&1 - # if grubby is not there, let's tell it - if [ $? -ne 0 ]; then - exit 127 - fi - for t in "${kargs[@]}";do - if [[ $args != *${t}* ]];then - if chroot /host/ grubby --info=DEFAULT | grep args | grep -vq ${t}; then - chroot /host/ grubby --update-kernel=DEFAULT --args=${t} > /dev/null 2>&1 - fi - let ret++ - fi - done -fi - -echo $ret diff --git a/bindata/scripts/kargs.sh b/bindata/scripts/kargs.sh new file mode 100755 index 000000000..07e283600 --- /dev/null +++ b/bindata/scripts/kargs.sh @@ -0,0 +1,63 @@ +#!/bin/bash +set -x + +command=$1 +shift +declare -a kargs=( "$@" ) +ret=0 +args=$(chroot /host/ cat /proc/cmdline) + +IS_OS_UBUNTU=true; [[ "$(chroot /host/ grep -i ubuntu /etc/os-release -c)" == "0" ]] && IS_OS_UBUNTU=false + +# Kernel args configuration isn't supported for Ubuntu now, so we shouldn't do anything here +if ${IS_OS_UBUNTU} ; then + echo $ret + exit 0 +fi + +if chroot /host/ test -f /run/ostree-booted ; then + for t in "${kargs[@]}";do + if [[ $command == "add" ]];then + if [[ $args != *${t}* ]];then + if chroot /host/ rpm-ostree kargs | grep -vq ${t}; then + chroot /host/ rpm-ostree kargs --append ${t} > /dev/null 2>&1 + fi + let ret++ + fi + fi + if [[ $command == "remove" ]];then + if [[ $args == *${t}* ]];then + if chroot /host/ rpm-ostree kargs | grep -q ${t}; then + chroot /host/ rpm-ostree kargs --delete ${t} > /dev/null 2>&1 + fi + let ret++ + fi + fi + done +else + chroot /host/ which grubby > /dev/null 2>&1 + # if grubby is not there, let's tell it + if [ $? -ne 0 ]; then + exit 127 + fi + for t in "${kargs[@]}";do + if [[ $command == "add" ]];then + if [[ $args != *${t}* ]];then + if chroot /host/ grubby --info=DEFAULT | grep args | grep -vq ${t}; then + chroot /host/ grubby --update-kernel=DEFAULT --args=${t} > /dev/null 2>&1 + fi + let ret++ + fi + fi + if [[ $command == "remove" ]];then + if [[ $args == *${t}* ]];then + if chroot /host/ grubby --info=DEFAULT | grep args | grep -q ${t}; then + chroot /host/ grubby --update-kernel=DEFAULT --remove-args=${t} > /dev/null 2>&1 + fi + let ret++ + fi + fi + done +fi + +echo $ret diff --git a/cmd/sriov-network-config-daemon/service.go b/cmd/sriov-network-config-daemon/service.go index 382ad976b..8dbb9217c 100644 --- a/cmd/sriov-network-config-daemon/service.go +++ b/cmd/sriov-network-config-daemon/service.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "os" + "time" "github.com/go-logr/logr" "github.com/spf13/cobra" @@ -27,12 +28,13 @@ import ( sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper" + hosttypes "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms" plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins/generic" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins/virtual" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/systemd" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/version" ) @@ -40,6 +42,12 @@ import ( const ( PhasePre = "pre" PhasePost = "post" + + // InitializationDeviceDiscoveryTimeoutSec constant defines the number of + // seconds to wait for devices to be registered in the system with the expected name. + InitializationDeviceDiscoveryTimeoutSec = 60 + // InitializationDeviceUdevProcessingTimeoutSec constant defines the number of seconds to wait for udev rules to process + InitializationDeviceUdevProcessingTimeoutSec = 60 ) var ( @@ -57,11 +65,32 @@ var ( newPlatformHelperFunc = platforms.NewDefaultPlatformHelper ) +// ServiceConfig is a struct that encapsulates the configuration and dependencies +// needed by the SriovNetworkConfigDaemon systemd service. +type ServiceConfig struct { + hostHelper helper.HostHelpersInterface // Provides host-specific helper functions + log logr.Logger // Handles logging for the service + sriovConfig *hosttypes.SriovConfig // Contains the SR-IOV network configuration settings +} + func init() { rootCmd.AddCommand(serviceCmd) serviceCmd.Flags().StringVarP(&phaseArg, "phase", "p", PhasePre, fmt.Sprintf("configuration phase, supported values are: %s, %s", PhasePre, PhasePost)) } +func newServiceConfig(setupLog logr.Logger) (*ServiceConfig, error) { + hostHelpers, err := newHostHelpersFunc() + if err != nil { + return nil, fmt.Errorf("failed to create host helpers: %v", err) + } + + return &ServiceConfig{ + hostHelpers, + setupLog, + nil, + }, nil +} + // The service supports two configuration phases: // * pre(default) - before the NetworkManager or systemd-networkd // * post - after the NetworkManager or systemd-networkd @@ -78,6 +107,7 @@ func runServiceCmd(cmd *cobra.Command, args []string) error { } // init logger snolog.InitLog() + snolog.SetLogLevel(2) setupLog := log.Log.WithName("sriov-config-service").WithValues("phase", phaseArg) setupLog.V(0).Info("Starting sriov-config-service", "version", version.Version) @@ -86,53 +116,58 @@ func runServiceCmd(cmd *cobra.Command, args []string) error { vars.UsingSystemdMode = true vars.InChroot = true - sriovConf, err := readConf(setupLog) + sc, err := newServiceConfig(setupLog) if err != nil { - return updateSriovResultErr(setupLog, phaseArg, err) + setupLog.Error(err, "failed to create the service configuration controller, Exiting") + return err } - setupLog.V(2).Info("sriov-config-service", "config", sriovConf) - vars.DevMode = sriovConf.UnsupportedNics - vars.ManageSoftwareBridges = sriovConf.ManageSoftwareBridges - vars.OVSDBSocketPath = sriovConf.OVSDBSocketPath - if err := initSupportedNics(); err != nil { - return updateSriovResultErr(setupLog, phaseArg, fmt.Errorf("failed to initialize list of supported NIC ids: %v", err)) + err = sc.readConf() + if err != nil { + return sc.updateSriovResultErr(phaseArg, err) } - hostHelpers, err := newHostHelpersFunc() - if err != nil { - return updateSriovResultErr(setupLog, phaseArg, fmt.Errorf("failed to create hostHelpers: %v", err)) + setupLog.V(2).Info("sriov-config-service", "config", sc.sriovConfig) + vars.DevMode = sc.sriovConfig.UnsupportedNics + vars.ManageSoftwareBridges = sc.sriovConfig.ManageSoftwareBridges + vars.OVSDBSocketPath = sc.sriovConfig.OVSDBSocketPath + + if err := sc.initSupportedNics(); err != nil { + return sc.updateSriovResultErr(phaseArg, fmt.Errorf("failed to initialize list of supported NIC ids: %v", err)) } + sc.waitForDevicesInitialization() + if phaseArg == PhasePre { - err = phasePre(setupLog, sriovConf, hostHelpers) + err = sc.phasePre() } else { - err = phasePost(setupLog, sriovConf, hostHelpers) + err = sc.phasePost() } if err != nil { - return updateSriovResultErr(setupLog, phaseArg, err) + return sc.updateSriovResultErr(phaseArg, err) } - return updateSriovResultOk(setupLog, phaseArg) + return sc.updateSriovResultOk(phaseArg) } -func readConf(setupLog logr.Logger) (*systemd.SriovConfig, error) { - nodeStateSpec, err := systemd.ReadConfFile() +func (s *ServiceConfig) readConf() error { + nodeStateSpec, err := s.hostHelper.ReadConfFile() if err != nil { - if _, err := os.Stat(systemd.SriovSystemdConfigPath); !errors.Is(err, os.ErrNotExist) { - return nil, fmt.Errorf("failed to read the sriov configuration file in path %s: %v", systemd.SriovSystemdConfigPath, err) + if _, err := os.Stat(utils.GetHostExtensionPath(consts.SriovSystemdConfigPath)); !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("failed to read the sriov configuration file in path %s: %v", utils.GetHostExtensionPath(consts.SriovSystemdConfigPath), err) } - setupLog.Info("configuration file not found, use default config") - nodeStateSpec = &systemd.SriovConfig{ + s.log.Info("configuration file not found, use default config") + nodeStateSpec = &hosttypes.SriovConfig{ Spec: sriovv1.SriovNetworkNodeStateSpec{}, UnsupportedNics: false, PlatformType: consts.Baremetal, } } - return nodeStateSpec, nil + s.sriovConfig = nodeStateSpec + return nil } -func initSupportedNics() error { - supportedNicIds, err := systemd.ReadSriovSupportedNics() +func (s *ServiceConfig) initSupportedNics() error { + supportedNicIds, err := s.hostHelper.ReadSriovSupportedNics() if err != nil { return fmt.Errorf("failed to read list of supported nic ids: %v", err) } @@ -140,49 +175,49 @@ func initSupportedNics() error { return nil } -func phasePre(setupLog logr.Logger, conf *systemd.SriovConfig, hostHelpers helper.HostHelpersInterface) error { +func (s *ServiceConfig) phasePre() error { // make sure there is no stale result file to avoid situation when we // read outdated info in the Post phase when the Pre silently failed (should not happen) - if err := systemd.RemoveSriovResult(); err != nil { + if err := s.hostHelper.RemoveSriovResult(); err != nil { return fmt.Errorf("failed to remove sriov result file: %v", err) } - _, err := hostHelpers.CheckRDMAEnabled() + _, err := s.hostHelper.CheckRDMAEnabled() if err != nil { - setupLog.Error(err, "warning, failed to check RDMA state") + s.log.Error(err, "warning, failed to check RDMA state") } - hostHelpers.TryEnableTun() - hostHelpers.TryEnableVhostNet() + s.hostHelper.TryEnableTun() + s.hostHelper.TryEnableVhostNet() - return callPlugin(setupLog, PhasePre, conf, hostHelpers) + return s.callPlugin(PhasePre) } -func phasePost(setupLog logr.Logger, conf *systemd.SriovConfig, hostHelpers helper.HostHelpersInterface) error { - setupLog.V(0).Info("check result of the Pre phase") - prePhaseResult, err := systemd.ReadSriovResult() +func (s *ServiceConfig) phasePost() error { + s.log.V(0).Info("check result of the Pre phase") + prePhaseResult, err := s.hostHelper.ReadSriovResult() if err != nil { return fmt.Errorf("failed to read result of the pre phase: %v", err) } if prePhaseResult.SyncStatus != consts.SyncStatusInProgress { return fmt.Errorf("unexpected result of the pre phase: %s, syncError: %s", prePhaseResult.SyncStatus, prePhaseResult.LastSyncError) } - setupLog.V(0).Info("Pre phase succeed, continue execution") + s.log.V(0).Info("Pre phase succeed, continue execution") - return callPlugin(setupLog, PhasePost, conf, hostHelpers) + return s.callPlugin(PhasePost) } -func callPlugin(setupLog logr.Logger, phase string, conf *systemd.SriovConfig, hostHelpers helper.HostHelpersInterface) error { - configPlugin, err := getPlugin(setupLog, phase, conf, hostHelpers) +func (s *ServiceConfig) callPlugin(phase string) error { + configPlugin, err := s.getPlugin(phase) if err != nil { return err } if configPlugin == nil { - setupLog.V(0).Info("no plugin for the platform for the current phase, skip calling", "platform", conf.PlatformType) + s.log.V(0).Info("no plugin for the platform for the current phase, skip calling", "platform", s.sriovConfig.PlatformType) return nil } - nodeState, err := getNetworkNodeState(setupLog, conf, phase, hostHelpers) + nodeState, err := s.getNetworkNodeState(phase) if err != nil { return err } @@ -194,25 +229,24 @@ func callPlugin(setupLog logr.Logger, phase string, conf *systemd.SriovConfig, h if err = configPlugin.Apply(); err != nil { return fmt.Errorf("failed to apply configuration: %v", err) } - setupLog.V(0).Info("plugin call succeed") + s.log.V(0).Info("plugin call succeed") return nil } -func getPlugin(setupLog logr.Logger, phase string, - conf *systemd.SriovConfig, hostHelpers helper.HostHelpersInterface) (plugin.VendorPlugin, error) { +func (s *ServiceConfig) getPlugin(phase string) (plugin.VendorPlugin, error) { var ( configPlugin plugin.VendorPlugin err error ) - switch conf.PlatformType { + switch s.sriovConfig.PlatformType { case consts.Baremetal: switch phase { case PhasePre: - configPlugin, err = newGenericPluginFunc(hostHelpers, + configPlugin, err = newGenericPluginFunc(s.hostHelper, generic.WithSkipVFConfiguration(), generic.WithSkipBridgeConfiguration()) case PhasePost: - configPlugin, err = newGenericPluginFunc(hostHelpers) + configPlugin, err = newGenericPluginFunc(s.hostHelper) } if err != nil { return nil, fmt.Errorf("failed to create generic plugin for %v", err) @@ -220,34 +254,33 @@ func getPlugin(setupLog logr.Logger, phase string, case consts.VirtualOpenStack: switch phase { case PhasePre: - configPlugin, err = newVirtualPluginFunc(hostHelpers) + configPlugin, err = newVirtualPluginFunc(s.hostHelper) if err != nil { return nil, fmt.Errorf("failed to create virtual plugin %v", err) } case PhasePost: - setupLog.Info("skip post configuration phase for virtual cluster") + s.log.Info("skip post configuration phase for virtual cluster") return nil, nil } } return configPlugin, nil } -func getNetworkNodeState(setupLog logr.Logger, conf *systemd.SriovConfig, phase string, - hostHelpers helper.HostHelpersInterface) (*sriovv1.SriovNetworkNodeState, error) { +func (s *ServiceConfig) getNetworkNodeState(phase string) (*sriovv1.SriovNetworkNodeState, error) { var ( ifaceStatuses []sriovv1.InterfaceExt bridges sriovv1.Bridges err error ) - switch conf.PlatformType { + switch s.sriovConfig.PlatformType { case consts.Baremetal: - ifaceStatuses, err = hostHelpers.DiscoverSriovDevices(hostHelpers) + ifaceStatuses, err = s.hostHelper.DiscoverSriovDevices(s.hostHelper) if err != nil { return nil, fmt.Errorf("failed to discover sriov devices on the host: %v", err) } if phase != PhasePre && vars.ManageSoftwareBridges { // openvswitch is not available during the pre phase - bridges, err = hostHelpers.DiscoverBridges() + bridges, err = s.hostHelper.DiscoverBridges() if err != nil { return nil, fmt.Errorf("failed to discover managed bridges on the host: %v", err) } @@ -267,39 +300,88 @@ func getNetworkNodeState(setupLog logr.Logger, conf *systemd.SriovConfig, phase } } return &sriovv1.SriovNetworkNodeState{ - Spec: conf.Spec, + Spec: s.sriovConfig.Spec, Status: sriovv1.SriovNetworkNodeStateStatus{Interfaces: ifaceStatuses, Bridges: bridges}, }, nil } -func updateSriovResultErr(setupLog logr.Logger, phase string, origErr error) error { - setupLog.Error(origErr, "service call failed") - err := updateResult(setupLog, consts.SyncStatusFailed, fmt.Sprintf("%s: %v", phase, origErr)) +func (s *ServiceConfig) updateSriovResultErr(phase string, origErr error) error { + s.log.Error(origErr, "service call failed") + err := s.updateResult(consts.SyncStatusFailed, fmt.Sprintf("%s: %v", phase, origErr)) if err != nil { return err } return origErr } -func updateSriovResultOk(setupLog logr.Logger, phase string) error { - setupLog.V(0).Info("service call succeed") +func (s *ServiceConfig) updateSriovResultOk(phase string) error { + s.log.V(0).Info("service call succeed") syncStatus := consts.SyncStatusSucceeded if phase == PhasePre { syncStatus = consts.SyncStatusInProgress } - return updateResult(setupLog, syncStatus, "") + return s.updateResult(syncStatus, "") } -func updateResult(setupLog logr.Logger, result, msg string) error { - sriovResult := &systemd.SriovResult{ +func (s *ServiceConfig) updateResult(result, msg string) error { + sriovResult := &hosttypes.SriovResult{ SyncStatus: result, LastSyncError: msg, } - err := systemd.WriteSriovResult(sriovResult) + err := s.hostHelper.WriteSriovResult(sriovResult) if err != nil { - setupLog.Error(err, "failed to write sriov result file", "content", *sriovResult) + s.log.Error(err, "failed to write sriov result file", "content", *sriovResult) return fmt.Errorf("sriov-config-service failed to write sriov result file with content %v error: %v", *sriovResult, err) } - setupLog.V(0).Info("result file updated", "SyncStatus", sriovResult.SyncStatus, "LastSyncError", msg) + s.log.V(0).Info("result file updated", "SyncStatus", sriovResult.SyncStatus, "LastSyncError", msg) return nil } + +// waitForDevicesInitialization should be executed in both the pre and post-networking stages. +// This function ensures that the network devices specified in the configuration are registered +// and handled by UDEV. Sometimes, the initialization of network devices might take a significant +// amount of time, and the sriov-config systemd service may start before the devices are fully +// processed, leading to failure. +// +// To address this, we not only check if the devices are registered with the correct name but also +// wait for the udev event queue to empty. This increases the likelihood that the service will start +// only when the devices are fully initialized. It is required to call this function in the +// "post-networking" phase as well because the OS network manager might change device configurations, +// and we need to ensure these changes are fully processed before starting the post-networking part. +// +// The timeouts used in this function are intentionally kept low to avoid blocking the OS loading +// process for too long in case of any issues. +// +// Note: Currently, this function handles only Baremetal clusters. We do not have evidence that +// this logic is required for virtual clusters. +func (s *ServiceConfig) waitForDevicesInitialization() { + if s.sriovConfig.PlatformType != consts.Baremetal { + // skip waiting on virtual cluster + return + } + // wait for devices from the spec to be registered in the system with expected names + devicesToWait := make(map[string]string, len(s.sriovConfig.Spec.Interfaces)) + for _, d := range s.sriovConfig.Spec.Interfaces { + devicesToWait[d.PciAddress] = d.Name + } + deadline := time.Now().Add(time.Second * time.Duration(InitializationDeviceDiscoveryTimeoutSec)) + for time.Now().Before(deadline) { + for pciAddr, name := range devicesToWait { + if s.hostHelper.TryGetInterfaceName(pciAddr) == name { + s.log.Info("Device ready", "pci", pciAddr, "name", name) + delete(devicesToWait, pciAddr) + } + } + if len(devicesToWait) == 0 { + break + } + time.Sleep(time.Second) + } + if len(devicesToWait) != 0 { + s.log.Info("WARNING: some devices were not initialized", "devices", devicesToWait, "timeout", InitializationDeviceDiscoveryTimeoutSec) + } + if err := s.hostHelper.WaitUdevEventsProcessed(InitializationDeviceUdevProcessingTimeoutSec); err != nil { + s.log.Info("WARNING: failed to wait for udev events processing", "reason", err.Error(), + "timeout", InitializationDeviceUdevProcessingTimeoutSec) + } +} diff --git a/cmd/sriov-network-config-daemon/service_test.go b/cmd/sriov-network-config-daemon/service_test.go index 771cc3b1c..a3fe81e41 100644 --- a/cmd/sriov-network-config-daemon/service_test.go +++ b/cmd/sriov-network-config-daemon/service_test.go @@ -3,24 +3,22 @@ package main import ( "fmt" - "github.com/golang/mock/gomock" - "github.com/spf13/cobra" - "gopkg.in/yaml.v3" - + "github.com/go-logr/logr" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/spf13/cobra" + "go.uber.org/mock/gomock" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper" helperMock "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper/mock" + hosttypes "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms" platformsMock "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/mock" plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins/generic" pluginsMock "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins/mock" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/systemd" - "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/fakefilesystem" - testHelpers "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/helpers" ) func restoreOrigFuncs() { @@ -36,39 +34,42 @@ func restoreOrigFuncs() { }) } -func getTestSriovInterfaceConfig(platform int) []byte { - return []byte(fmt.Sprintf(`spec: - interfaces: - - pciaddress: 0000:d8:00.0 - numvfs: 4 - mtu: 1500 - name: enp216s0f0np0 - linktype: "" - eswitchmode: legacy - vfgroups: - - resourcename: legacy - devicetype: "" - vfrange: 0-3 - policyname: test-legacy - mtu: 1500 - isrdma: false - vdpatype: "" - externallymanaged: false -unsupportedNics: false -platformType: %d -manageSoftwareBridges: true -`, platform)) +func getTestSriovInterfaceConfig(platform consts.PlatformTypes) *hosttypes.SriovConfig { + return &hosttypes.SriovConfig{ + Spec: sriovnetworkv1.SriovNetworkNodeStateSpec{ + Interfaces: sriovnetworkv1.Interfaces{ + { + PciAddress: "0000:d8:00.0", + NumVfs: 4, + Mtu: 1500, + Name: "enp216s0f0np0", + LinkType: "", + EswitchMode: "legacy", + VfGroups: []sriovnetworkv1.VfGroup{ + { + ResourceName: "legacy", + DeviceType: "", + VfRange: "0-3", + PolicyName: "test-legacy", + Mtu: 1500, + IsRdma: false, + VdpaType: "", + }, + }, + ExternallyManaged: false, + }, + }, + }, + PlatformType: platform, + UnsupportedNics: false, + ManageSoftwareBridges: true, + } } -var testSriovSupportedNicIDs = `8086 1583 154c -8086 0d58 154c -8086 10c9 10ca -` +var testSriovSupportedNicIDs = []string{"8086 1583 154c", "8086 0d58 154c", "8086 10c9 10ca"} -func getTestResultFileContent(syncStatus, errMsg string) []byte { - data, err := yaml.Marshal(systemd.SriovResult{SyncStatus: syncStatus, LastSyncError: errMsg}) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - return data +func getTestResultFileContent(syncStatus, errMsg string) *hosttypes.SriovResult { + return &hosttypes.SriovResult{SyncStatus: syncStatus, LastSyncError: errMsg} } // checks if NodeState contains deviceName in spec and status fields @@ -150,42 +151,35 @@ var _ = Describe("Service", func() { It("Pre phase - baremetal cluster", func() { phaseArg = PhasePre - testHelpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ - Dirs: []string{"/etc/sriov-operator"}, - Files: map[string][]byte{ - "/etc/sriov-operator/sriov-supported-nics-ids.yaml": []byte(testSriovSupportedNicIDs), - "/etc/sriov-operator/sriov-interface-config.yaml": getTestSriovInterfaceConfig(0), - "/etc/sriov-operator/sriov-interface-result.yaml": []byte("something"), - }, - }) + hostHelpers.EXPECT().TryGetInterfaceName("0000:d8:00.0").Return("enp216s0f0np0") + hostHelpers.EXPECT().WaitUdevEventsProcessed(60).Return(nil) hostHelpers.EXPECT().CheckRDMAEnabled().Return(true, nil) hostHelpers.EXPECT().TryEnableTun().Return() hostHelpers.EXPECT().TryEnableVhostNet().Return() - hostHelpers.EXPECT().DiscoverSriovDevices(hostHelpers).Return([]sriovnetworkv1.InterfaceExt{{ + hostHelpers.EXPECT().DiscoverSriovDevices(gomock.Any()).Return([]sriovnetworkv1.InterfaceExt{{ Name: "enp216s0f0np0", }}, nil) + hostHelpers.EXPECT().ReadConfFile().Return(getTestSriovInterfaceConfig(0), nil) + hostHelpers.EXPECT().ReadSriovSupportedNics().Return(testSriovSupportedNicIDs, nil) + hostHelpers.EXPECT().RemoveSriovResult().Return(nil) + hostHelpers.EXPECT().WriteSriovResult(&hosttypes.SriovResult{SyncStatus: consts.SyncStatusInProgress}) + genericPlugin.EXPECT().OnNodeStateChange(newNodeStateContainsDeviceMatcher("enp216s0f0np0")).Return(true, false, nil) genericPlugin.EXPECT().Apply().Return(nil) Expect(runServiceCmd(&cobra.Command{}, []string{})).NotTo(HaveOccurred()) - - testHelpers.GinkgoAssertFileContentsEquals("/etc/sriov-operator/sriov-interface-result.yaml", - string(getTestResultFileContent("InProgress", ""))) + Expect(testCtrl.Satisfied()).To(BeTrue()) }) It("Pre phase - virtual cluster", func() { phaseArg = PhasePre - testHelpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ - Dirs: []string{"/etc/sriov-operator"}, - Files: map[string][]byte{ - "/etc/sriov-operator/sriov-supported-nics-ids.yaml": []byte(testSriovSupportedNicIDs), - "/etc/sriov-operator/sriov-interface-config.yaml": getTestSriovInterfaceConfig(1), - "/etc/sriov-operator/sriov-interface-result.yaml": []byte("something"), - }, - }) hostHelpers.EXPECT().CheckRDMAEnabled().Return(true, nil) hostHelpers.EXPECT().TryEnableTun().Return() hostHelpers.EXPECT().TryEnableVhostNet().Return() + hostHelpers.EXPECT().ReadConfFile().Return(getTestSriovInterfaceConfig(1), nil) + hostHelpers.EXPECT().ReadSriovSupportedNics().Return(testSriovSupportedNicIDs, nil) + hostHelpers.EXPECT().RemoveSriovResult().Return(nil) + hostHelpers.EXPECT().WriteSriovResult(&hosttypes.SriovResult{SyncStatus: consts.SyncStatusInProgress}) platformHelper.EXPECT().CreateOpenstackDevicesInfo().Return(nil) platformHelper.EXPECT().DiscoverSriovDevicesVirtual().Return([]sriovnetworkv1.InterfaceExt{{ @@ -196,84 +190,84 @@ var _ = Describe("Service", func() { virtualPlugin.EXPECT().Apply().Return(nil) Expect(runServiceCmd(&cobra.Command{}, []string{})).NotTo(HaveOccurred()) - - testHelpers.GinkgoAssertFileContentsEquals("/etc/sriov-operator/sriov-interface-result.yaml", - string(getTestResultFileContent("InProgress", ""))) + Expect(testCtrl.Satisfied()).To(BeTrue()) }) It("Pre phase - apply failed", func() { phaseArg = PhasePre - testHelpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ - Dirs: []string{"/etc/sriov-operator"}, - Files: map[string][]byte{ - "/etc/sriov-operator/sriov-supported-nics-ids.yaml": []byte(testSriovSupportedNicIDs), - "/etc/sriov-operator/sriov-interface-config.yaml": getTestSriovInterfaceConfig(0), - "/etc/sriov-operator/sriov-interface-result.yaml": []byte("something"), - }, - }) + hostHelpers.EXPECT().TryGetInterfaceName("0000:d8:00.0").Return("enp216s0f0np0") + hostHelpers.EXPECT().WaitUdevEventsProcessed(60).Return(nil) hostHelpers.EXPECT().CheckRDMAEnabled().Return(true, nil) hostHelpers.EXPECT().TryEnableTun().Return() hostHelpers.EXPECT().TryEnableVhostNet().Return() - hostHelpers.EXPECT().DiscoverSriovDevices(hostHelpers).Return([]sriovnetworkv1.InterfaceExt{{ + hostHelpers.EXPECT().DiscoverSriovDevices(gomock.Any()).Return([]sriovnetworkv1.InterfaceExt{{ Name: "enp216s0f0np0", }}, nil) + hostHelpers.EXPECT().ReadConfFile().Return(getTestSriovInterfaceConfig(0), nil) + hostHelpers.EXPECT().ReadSriovSupportedNics().Return(testSriovSupportedNicIDs, nil) + hostHelpers.EXPECT().RemoveSriovResult().Return(nil) + hostHelpers.EXPECT().WriteSriovResult(&hosttypes.SriovResult{SyncStatus: consts.SyncStatusFailed, LastSyncError: "pre: failed to apply configuration: test"}) + genericPlugin.EXPECT().OnNodeStateChange(newNodeStateContainsDeviceMatcher("enp216s0f0np0")).Return(true, false, nil) genericPlugin.EXPECT().Apply().Return(testError) Expect(runServiceCmd(&cobra.Command{}, []string{})).To(MatchError(ContainSubstring("test"))) - - testHelpers.GinkgoAssertFileContentsEquals("/etc/sriov-operator/sriov-interface-result.yaml", - string(getTestResultFileContent("Failed", "pre: failed to apply configuration: test"))) + Expect(testCtrl.Satisfied()).To(BeTrue()) }) It("Post phase - baremetal cluster", func() { phaseArg = PhasePost - testHelpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ - Dirs: []string{"/etc/sriov-operator"}, - Files: map[string][]byte{ - "/etc/sriov-operator/sriov-supported-nics-ids.yaml": []byte(testSriovSupportedNicIDs), - "/etc/sriov-operator/sriov-interface-config.yaml": getTestSriovInterfaceConfig(0), - "/etc/sriov-operator/sriov-interface-result.yaml": getTestResultFileContent("InProgress", ""), - }, - }) - hostHelpers.EXPECT().DiscoverSriovDevices(hostHelpers).Return([]sriovnetworkv1.InterfaceExt{{ + hostHelpers.EXPECT().TryGetInterfaceName("0000:d8:00.0").Return("enp216s0f0np0") + hostHelpers.EXPECT().WaitUdevEventsProcessed(60).Return(nil) + hostHelpers.EXPECT().DiscoverSriovDevices(gomock.Any()).Return([]sriovnetworkv1.InterfaceExt{{ Name: "enp216s0f0np0", }}, nil) hostHelpers.EXPECT().DiscoverBridges().Return(sriovnetworkv1.Bridges{}, nil) + hostHelpers.EXPECT().ReadSriovResult().Return(getTestResultFileContent("InProgress", ""), nil) + hostHelpers.EXPECT().ReadConfFile().Return(getTestSriovInterfaceConfig(0), nil) + hostHelpers.EXPECT().ReadSriovSupportedNics().Return(testSriovSupportedNicIDs, nil) + hostHelpers.EXPECT().WriteSriovResult(&hosttypes.SriovResult{SyncStatus: consts.SyncStatusSucceeded}) + genericPlugin.EXPECT().OnNodeStateChange(newNodeStateContainsDeviceMatcher("enp216s0f0np0")).Return(true, false, nil) genericPlugin.EXPECT().Apply().Return(nil) Expect(runServiceCmd(&cobra.Command{}, []string{})).NotTo(HaveOccurred()) - testHelpers.GinkgoAssertFileContentsEquals("/etc/sriov-operator/sriov-interface-result.yaml", - string(getTestResultFileContent("Succeeded", ""))) + Expect(testCtrl.Satisfied()).To(BeTrue()) }) It("Post phase - virtual cluster", func() { phaseArg = PhasePost - testHelpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ - Dirs: []string{"/etc/sriov-operator"}, - Files: map[string][]byte{ - "/etc/sriov-operator/sriov-supported-nics-ids.yaml": []byte(testSriovSupportedNicIDs), - "/etc/sriov-operator/sriov-interface-config.yaml": getTestSriovInterfaceConfig(1), - "/etc/sriov-operator/sriov-interface-result.yaml": getTestResultFileContent("InProgress", ""), - }, - }) + hostHelpers.EXPECT().ReadConfFile().Return(getTestSriovInterfaceConfig(1), nil) + hostHelpers.EXPECT().ReadSriovSupportedNics().Return(testSriovSupportedNicIDs, nil) + hostHelpers.EXPECT().ReadSriovResult().Return(getTestResultFileContent("InProgress", ""), nil) + hostHelpers.EXPECT().WriteSriovResult(&hosttypes.SriovResult{SyncStatus: consts.SyncStatusSucceeded}) + Expect(runServiceCmd(&cobra.Command{}, []string{})).NotTo(HaveOccurred()) - testHelpers.GinkgoAssertFileContentsEquals("/etc/sriov-operator/sriov-interface-result.yaml", - string(getTestResultFileContent("Succeeded", ""))) + Expect(testCtrl.Satisfied()).To(BeTrue()) }) It("Post phase - wrong result of the pre phase", func() { phaseArg = PhasePost - testHelpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ - Dirs: []string{"/etc/sriov-operator"}, - Files: map[string][]byte{ - "/etc/sriov-operator/sriov-supported-nics-ids.yaml": []byte(testSriovSupportedNicIDs), - "/etc/sriov-operator/sriov-interface-config.yaml": getTestSriovInterfaceConfig(1), - "/etc/sriov-operator/sriov-interface-result.yaml": getTestResultFileContent("Failed", "pretest"), - }, - }) + hostHelpers.EXPECT().ReadConfFile().Return(getTestSriovInterfaceConfig(1), nil) + hostHelpers.EXPECT().ReadSriovSupportedNics().Return(testSriovSupportedNicIDs, nil) + hostHelpers.EXPECT().ReadSriovResult().Return(getTestResultFileContent("Failed", "pretest"), nil) + hostHelpers.EXPECT().WriteSriovResult(&hosttypes.SriovResult{SyncStatus: consts.SyncStatusFailed, LastSyncError: "post: unexpected result of the pre phase: Failed, syncError: pretest"}) + Expect(runServiceCmd(&cobra.Command{}, []string{})).To(HaveOccurred()) - testHelpers.GinkgoAssertFileContentsEquals("/etc/sriov-operator/sriov-interface-result.yaml", - string(getTestResultFileContent("Failed", "post: unexpected result of the pre phase: Failed, syncError: pretest"))) + }) + It("waitForDevicesInitialization", func() { + cfg := &hosttypes.SriovConfig{Spec: sriovnetworkv1.SriovNetworkNodeStateSpec{ + Interfaces: []sriovnetworkv1.Interface{ + {Name: "name1", PciAddress: "0000:d8:00.0"}, + {Name: "name2", PciAddress: "0000:d8:00.1"}}}} + hostHelpers.EXPECT().TryGetInterfaceName("0000:d8:00.0").Return("other") + hostHelpers.EXPECT().TryGetInterfaceName("0000:d8:00.1").Return("") + hostHelpers.EXPECT().TryGetInterfaceName("0000:d8:00.0").Return("name1") + hostHelpers.EXPECT().TryGetInterfaceName("0000:d8:00.1").Return("") + hostHelpers.EXPECT().TryGetInterfaceName("0000:d8:00.1").Return("name2") + hostHelpers.EXPECT().WaitUdevEventsProcessed(60).Return(nil) + sc, err := newServiceConfig(logr.Discard()) + Expect(err).ToNot(HaveOccurred()) + sc.sriovConfig = cfg + sc.waitForDevicesInitialization() }) }) diff --git a/cmd/sriov-network-config-daemon/start.go b/cmd/sriov-network-config-daemon/start.go index b1ad0f667..fcf06fb84 100644 --- a/cmd/sriov-network-config-daemon/start.go +++ b/cmd/sriov-network-config-daemon/start.go @@ -18,28 +18,28 @@ package main import ( "context" "fmt" - "net" "net/url" "os" "strings" - "time" + ocpconfigapi "github.com/openshift/api/config/v1" "github.com/spf13/cobra" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/scheme" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/util/connrotation" - "sigs.k8s.io/controller-runtime/pkg/client" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" - - configv1 "github.com/openshift/api/config/v1" - mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + "sigs.k8s.io/controller-runtime/pkg/metrics/server" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - snclientset "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/daemon" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/featuregate" @@ -89,6 +89,8 @@ var ( manageSoftwareBridges bool ovsSocketPath string } + + scheme = runtime.NewScheme() ) func init() { @@ -100,13 +102,17 @@ func init() { startCmd.PersistentFlags().BoolVar(&startOpts.parallelNicConfig, "parallel-nic-config", false, "perform NIC configuration in parallel") startCmd.PersistentFlags().BoolVar(&startOpts.manageSoftwareBridges, "manage-software-bridges", false, "enable management of software bridges") startCmd.PersistentFlags().StringVar(&startOpts.ovsSocketPath, "ovs-socket-path", vars.OVSDBSocketPath, "path for OVSDB socket") -} -func runStartCmd(cmd *cobra.Command, args []string) error { - // init logger + // Init Scheme + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(sriovnetworkv1.AddToScheme(scheme)) + utilruntime.Must(ocpconfigapi.AddToScheme(scheme)) + + // Init logger snolog.InitLog() - setupLog := log.Log.WithName("sriov-network-config-daemon") +} +func configGlobalVariables() error { // Mark that we are running inside a container vars.UsingSystemdMode = false if startOpts.systemd { @@ -132,53 +138,81 @@ func runStartCmd(cmd *cobra.Command, args []string) error { } } - // This channel is used to ensure all spawned goroutines exit when we exit. - stopCh := make(chan struct{}) - defer close(stopCh) + vars.Scheme = scheme + + return nil +} + +func useKubeletKubeConfig() { + fnLogger := log.Log.WithName("sriov-network-config-daemon") + + kubeconfig, err := clientcmd.LoadFromFile("/host/etc/kubernetes/kubeconfig") + if err != nil { + fnLogger.Error(err, "failed to load kubelet kubeconfig") + } + clusterName := kubeconfig.Contexts[kubeconfig.CurrentContext].Cluster + apiURL := kubeconfig.Clusters[clusterName].Server + + urlPath, err := url.Parse(apiURL) + if err != nil { + fnLogger.Error(err, "failed to parse api url from kubelet kubeconfig") + } + + // The kubernetes in-cluster functions don't let you override the apiserver + // directly; gotta "pass" it via environment vars. + fnLogger.V(0).Info("overriding kubernetes api", "new-url", apiURL) + err = os.Setenv("KUBERNETES_SERVICE_HOST", urlPath.Hostname()) + if err != nil { + fnLogger.Error(err, "failed to set KUBERNETES_SERVICE_HOST environment variable") + } + err = os.Setenv("KUBERNETES_SERVICE_PORT", urlPath.Port()) + if err != nil { + fnLogger.Error(err, "failed to set KUBERNETES_SERVICE_PORT environment variable") + } +} + +func getOperatorConfig(kClient runtimeclient.Client) (*sriovnetworkv1.SriovOperatorConfig, error) { + defaultConfig := &sriovnetworkv1.SriovOperatorConfig{} + err := kClient.Get(context.Background(), types.NamespacedName{Namespace: vars.Namespace, Name: consts.DefaultConfigName}, defaultConfig) + if err != nil { + return nil, err + } + return defaultConfig, nil +} + +func initFeatureGates(defaultConfig *sriovnetworkv1.SriovOperatorConfig) (featuregate.FeatureGate, error) { + fnLogger := log.Log.WithName("initFeatureGates") + featureGates := featuregate.New() + featureGates.Init(defaultConfig.Spec.FeatureGates) + fnLogger.Info("Enabled featureGates", "featureGates", featureGates.String()) - // This channel is used to signal Run() something failed and to jump ship. - // It's purely a chan<- in the Daemon struct for goroutines to write to, and - // a <-chan in Run() for the main thread to listen on. - exitCh := make(chan error) - defer close(exitCh) + return featureGates, nil +} - // This channel is to make sure main thread will wait until the writer finish - // to report lastSyncError in SriovNetworkNodeState object. - syncCh := make(chan struct{}) - defer close(syncCh) +func initLogLevel(defaultConfig *sriovnetworkv1.SriovOperatorConfig) error { + fnLogger := log.Log.WithName("initLogLevel") + snolog.SetLogLevel(defaultConfig.Spec.LogLevel) + fnLogger.V(2).Info("logLevel sets", "logLevel", defaultConfig.Spec.LogLevel) + return nil +} - refreshCh := make(chan daemon.Message) - defer close(refreshCh) +func runStartCmd(cmd *cobra.Command, args []string) error { + setupLog := log.Log.WithName("sriov-network-config-daemon") + stopSignalCh := ctrl.SetupSignalHandler() + + // Load global variables + err := configGlobalVariables() + if err != nil { + setupLog.Error(err, "unable to config global variables") + return err + } var config *rest.Config - var err error // On openshift we use the kubeconfig from kubelet on the node where the daemon is running // this allow us to improve security as every daemon has access only to its own node if vars.ClusterType == consts.ClusterTypeOpenshift { - kubeconfig, err := clientcmd.LoadFromFile("/host/etc/kubernetes/kubeconfig") - if err != nil { - setupLog.Error(err, "failed to load kubelet kubeconfig") - } - clusterName := kubeconfig.Contexts[kubeconfig.CurrentContext].Cluster - apiURL := kubeconfig.Clusters[clusterName].Server - - urlPath, err := url.Parse(apiURL) - if err != nil { - setupLog.Error(err, "failed to parse api url from kubelet kubeconfig") - } - - // The kubernetes in-cluster functions don't let you override the apiserver - // directly; gotta "pass" it via environment vars. - setupLog.V(0).Info("overriding kubernetes api", "new-url", apiURL) - err = os.Setenv("KUBERNETES_SERVICE_HOST", urlPath.Hostname()) - if err != nil { - setupLog.Error(err, "failed to set KUBERNETES_SERVICE_HOST environment variable") - } - err = os.Setenv("KUBERNETES_SERVICE_PORT", urlPath.Port()) - if err != nil { - setupLog.Error(err, "failed to set KUBERNETES_SERVICE_PORT environment variable") - } + useKubeletKubeConfig() } kubeconfig := os.Getenv("KUBECONFIG") @@ -192,136 +226,129 @@ func runStartCmd(cmd *cobra.Command, args []string) error { if err != nil { return err } - vars.Config = config - vars.Scheme = scheme.Scheme - closeAllConns, err := updateDialer(config) + // create helpers + hostHelpers, err := helper.NewDefaultHostHelpers() if err != nil { + setupLog.Error(err, "failed to create hostHelpers") return err } - err = sriovnetworkv1.AddToScheme(scheme.Scheme) + platformHelper, err := platforms.NewDefaultPlatformHelper() if err != nil { - setupLog.Error(err, "failed to load sriov network CRDs to scheme") + setupLog.Error(err, "failed to create platformHelper") return err } - err = mcfgv1.AddToScheme(scheme.Scheme) + // create clients + kubeclient := kubernetes.NewForConfigOrDie(config) + kClient, err := runtimeclient.New( + config, + runtimeclient.Options{ + Scheme: vars.Scheme}) if err != nil { - setupLog.Error(err, "failed to load machine config CRDs to scheme") - return err + setupLog.Error(err, "couldn't create generic client") + os.Exit(1) } - err = configv1.Install(scheme.Scheme) + eventRecorder := daemon.NewEventRecorder(kClient, kubeclient, scheme) + defer eventRecorder.Shutdown() + + nodeInfo, err := kubeclient.CoreV1().Nodes().Get(context.Background(), vars.NodeName, v1.GetOptions{}) if err != nil { - setupLog.Error(err, "failed to load openshift config CRDs to scheme") + setupLog.Error(err, "failed to fetch node state, exiting", "node-name", startOpts.nodeName) return err } - kClient, err := client.New(config, client.Options{Scheme: scheme.Scheme}) - if err != nil { - setupLog.Error(err, "couldn't create client") - os.Exit(1) + // check for platform + for key, pType := range vars.PlatformsMap { + if strings.Contains(strings.ToLower(nodeInfo.Spec.ProviderID), strings.ToLower(key)) { + vars.PlatformType = pType + } } + setupLog.Info("Running on", "platform", vars.PlatformType.String()) - snclient := snclientset.NewForConfigOrDie(config) - kubeclient := kubernetes.NewForConfigOrDie(config) - - hostHelpers, err := helper.NewDefaultHostHelpers() - if err != nil { - setupLog.Error(err, "failed to create hostHelpers") + // Initial supported nic IDs + if err := sriovnetworkv1.InitNicIDMapFromConfigMap(kubeclient, vars.Namespace); err != nil { + setupLog.Error(err, "failed to run init NicIdMap") return err } - platformHelper, err := platforms.NewDefaultPlatformHelper() + operatorConfig, err := getOperatorConfig(kClient) if err != nil { - setupLog.Error(err, "failed to create platformHelper") + setupLog.Error(err, "Failed to get operator config object") return err } - config.Timeout = 5 * time.Second - writerclient := snclientset.NewForConfigOrDie(config) - - eventRecorder := daemon.NewEventRecorder(writerclient, kubeclient) - defer eventRecorder.Shutdown() - - setupLog.V(0).Info("starting node writer") - nodeWriter := daemon.NewNodeStateStatusWriter(writerclient, - closeAllConns, - eventRecorder, - hostHelpers, - platformHelper) - - nodeInfo, err := kubeclient.CoreV1().Nodes().Get(context.Background(), startOpts.nodeName, v1.GetOptions{}) - if err == nil { - for key, pType := range vars.PlatformsMap { - if strings.Contains(strings.ToLower(nodeInfo.Spec.ProviderID), strings.ToLower(key)) { - vars.PlatformType = pType - } - } - } else { - setupLog.Error(err, "failed to fetch node state, exiting", "node-name", startOpts.nodeName) + // init feature gates + fg, err := initFeatureGates(operatorConfig) + if err != nil { + setupLog.Error(err, "failed to initialize feature gates") return err } - setupLog.Info("Running on", "platform", vars.PlatformType.String()) - if err := sriovnetworkv1.InitNicIDMapFromConfigMap(kubeclient, vars.Namespace); err != nil { - setupLog.Error(err, "failed to run init NicIdMap") + // init log level + if err := initLogLevel(operatorConfig); err != nil { + setupLog.Error(err, "failed to initialize log level") return err } - eventRecorder.SendEvent("ConfigDaemonStart", "Config Daemon starting") + // init disable drain + vars.DisableDrain = operatorConfig.Spec.DisableDrain - // block the deamon process until nodeWriter finish first its run - err = nodeWriter.RunOnce() + // Init manager + setupLog.V(0).Info("Starting SR-IOV Network Config Daemon") + nodeStateSelector, err := fields.ParseSelector(fmt.Sprintf("metadata.name=%s,metadata.namespace=%s", vars.NodeName, vars.Namespace)) if err != nil { - setupLog.Error(err, "failed to run writer") + setupLog.Error(err, "failed to parse sriovNetworkNodeState name selector") return err } - go nodeWriter.Run(stopCh, refreshCh, syncCh) - - // Init feature gates once to prevent race conditions. - defaultConfig := &sriovnetworkv1.SriovOperatorConfig{} - err = kClient.Get(context.Background(), types.NamespacedName{Namespace: vars.Namespace, Name: consts.DefaultConfigName}, defaultConfig) + operatorConfigSelector, err := fields.ParseSelector(fmt.Sprintf("metadata.name=%s,metadata.namespace=%s", consts.DefaultConfigName, vars.Namespace)) if err != nil { - log.Log.Error(err, "Failed to get default SriovOperatorConfig object") + setupLog.Error(err, "failed to parse sriovOperatorConfig name selector") return err } - featureGates := featuregate.New() - featureGates.Init(defaultConfig.Spec.FeatureGates) - vars.MlxPluginFwReset = featureGates.IsEnabled(consts.MellanoxFirmwareResetFeatureGate) - log.Log.Info("Enabled featureGates", "featureGates", featureGates.String()) - setupLog.V(0).Info("Starting SriovNetworkConfigDaemon") - err = daemon.New( + mgr, err := ctrl.NewManager(vars.Config, ctrl.Options{ + Scheme: vars.Scheme, + Metrics: server.Options{BindAddress: "0"}, // disable metrics server for now as the daemon runs with hostNetwork + Cache: cache.Options{ // cache only the SriovNetworkNodeState with the node name + ByObject: map[runtimeclient.Object]cache.ByObject{ + &sriovnetworkv1.SriovNetworkNodeState{}: {Field: nodeStateSelector}, + &sriovnetworkv1.SriovOperatorConfig{}: {Field: operatorConfigSelector}}}, + }) + if err != nil { + setupLog.Error(err, "unable to create manager") + os.Exit(1) + } + + dm := daemon.New( kClient, - snclient, - kubeclient, hostHelpers, platformHelper, - exitCh, - stopCh, - syncCh, - refreshCh, eventRecorder, - featureGates, - startOpts.disabledPlugins, - ).Run(stopCh, exitCh) - if err != nil { - setupLog.Error(err, "failed to run daemon") + fg, + startOpts.disabledPlugins) + + // Init Daemon configuration on the node + if err = dm.Init(); err != nil { + setupLog.Error(err, "unable to initialize daemon") + os.Exit(1) + } + + // Setup reconcile loop with manager + if err = dm.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to setup daemon with manager for SriovNetworkNodeState") + os.Exit(1) } - setupLog.V(0).Info("Shutting down SriovNetworkConfigDaemon") - return err -} -// updateDialer instruments a restconfig with a dial. the returned function allows forcefully closing all active connections. -func updateDialer(clientConfig *rest.Config) (func(), error) { - if clientConfig.Transport != nil || clientConfig.Dial != nil { - return nil, fmt.Errorf("there is already a transport or dialer configured") + // Setup reconcile loop with manager + if err = daemon.NewOperatorConfigNodeReconcile(kClient).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create setup daemon manager for OperatorConfig") + os.Exit(1) } - f := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second} - d := connrotation.NewDialer(f.DialContext) - clientConfig.Dial = d.DialContext - return d.CloseAll, nil + + setupLog.Info("Starting Manager") + return mgr.Start(stopSignalCh) } diff --git a/cmd/sriov-network-operator-config-cleanup/cleanup.go b/cmd/sriov-network-operator-config-cleanup/cleanup.go new file mode 100644 index 000000000..33c07ff38 --- /dev/null +++ b/cmd/sriov-network-operator-config-cleanup/cleanup.go @@ -0,0 +1,89 @@ +package main + +import ( + "context" + "time" + + ocpconfigapi "github.com/openshift/api/config/v1" + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" +) + +var ( + namespace string + watchTO int +) + +func init() { + rootCmd.Flags().StringVarP(&namespace, "namespace", "n", "", "designated SriovOperatorConfig namespace") + rootCmd.Flags().IntVarP(&watchTO, "watch-timeout", "w", 10, "sriov-operator config post-delete watch timeout ") + + // Init Scheme + newScheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(newScheme)) + utilruntime.Must(sriovnetworkv1.AddToScheme(newScheme)) + utilruntime.Must(ocpconfigapi.AddToScheme(newScheme)) + + vars.Scheme = newScheme +} + +func runCleanupCmd(cmd *cobra.Command, args []string) error { + // init logger + snolog.InitLog() + setupLog := log.Log.WithName("sriov-network-operator-config-cleanup") + setupLog.Info("Run sriov-network-operator-config-cleanup") + + // adding context timeout although client-go Delete should be non-blocking by default + ctx, timeoutFunc := context.WithTimeout(context.Background(), time.Second*time.Duration(watchTO)) + defer timeoutFunc() + + restConfig := ctrl.GetConfigOrDie() + c, err := client.New(restConfig, client.Options{Scheme: vars.Scheme}) + if err != nil { + setupLog.Error(err, "failed to create 'sriovnetworkv1' clientset") + } + + operatorConfig := &sriovnetworkv1.SriovOperatorConfig{} + err = c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: "default"}, operatorConfig) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + setupLog.Error(err, "failed to get SriovOperatorConfig") + return err + } + + err = c.Delete(ctx, operatorConfig) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + setupLog.Error(err, "failed to delete SriovOperatorConfig") + return err + } + + for { + err = c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: "default"}, operatorConfig) + if err != nil { + if errors.IsNotFound(err) { + break + } + setupLog.Error(err, "failed to check sriovOperatorConfig exist") + return err + } + time.Sleep(100 * time.Millisecond) + } + + setupLog.Info("'default' SriovOperatorConfig is deleted") + return nil +} diff --git a/cmd/sriov-network-operator-config-cleanup/cleanup_test.go b/cmd/sriov-network-operator-config-cleanup/cleanup_test.go new file mode 100644 index 000000000..3d7b8395c --- /dev/null +++ b/cmd/sriov-network-operator-config-cleanup/cleanup_test.go @@ -0,0 +1,178 @@ +package main + +import ( + "context" + "sync" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/spf13/cobra" + "go.uber.org/mock/gomock" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/controllers" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/featuregate" + mock_platforms "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/mock" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openshift" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util" +) + +type configController struct { + k8sManager manager.Manager + ctx context.Context + cancel context.CancelFunc + wg *sync.WaitGroup +} + +var ( + controller *configController + testNamespace string = "sriov-network-operator" + defaultSriovOperatorSpec = sriovnetworkv1.SriovOperatorConfigSpec{ + EnableInjector: true, + EnableOperatorWebhook: true, + LogLevel: 2, + FeatureGates: nil, + } +) + +var _ = Describe("cleanup", Ordered, func() { + BeforeAll(func() { + By("Create SriovOperatorConfig controller k8s objs") + config := getDefaultSriovOperatorConfig() + Expect(k8sClient.Create(context.Background(), config)).Should(Succeed()) + + somePolicy := &sriovnetworkv1.SriovNetworkNodePolicy{} + somePolicy.SetNamespace(testNamespace) + somePolicy.SetName("some-policy") + somePolicy.Spec = sriovnetworkv1.SriovNetworkNodePolicySpec{ + NumVfs: 5, + NodeSelector: map[string]string{"foo": "bar"}, + NicSelector: sriovnetworkv1.SriovNetworkNicSelector{}, + Priority: 20, + } + Expect(k8sClient.Create(context.Background(), somePolicy)).ToNot(HaveOccurred()) + DeferCleanup(func() { + err := k8sClient.Delete(context.Background(), somePolicy) + Expect(err).ToNot(HaveOccurred()) + }) + + controller = newConfigController() + + }) + + It("test webhook cleanup flow", func() { + controller.start() + defer controller.stop() + + cmd := &cobra.Command{} + namespace = testNamespace + // verify that finalizer has been added, by controller, upon object creation + config := &sriovnetworkv1.SriovOperatorConfig{} + Eventually(func() []string { + // wait for SriovOperatorConfig flags to get updated + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "default", Namespace: testNamespace}, config) + if err != nil { + return nil + } + return config.Finalizers + }, util.APITimeout, util.RetryInterval).Should(Equal([]string{sriovnetworkv1.OPERATORCONFIGFINALIZERNAME})) + + Expect(runCleanupCmd(cmd, []string{})).Should(Succeed()) + config = &sriovnetworkv1.SriovOperatorConfig{} + err := util.WaitForNamespacedObjectDeleted(config, k8sClient, testNamespace, "default", util.RetryInterval, util.APITimeout) + Expect(err).NotTo(HaveOccurred()) + + }) + + It("test 'default' config cleanup timeout", func() { + // in this test case sriov-operator controller has been scaled down. + // we are testing returned ctx timeout error, for not being able to delete 'default' config object + config := getDefaultSriovOperatorConfig() + config.Finalizers = []string{sriovnetworkv1.OPERATORCONFIGFINALIZERNAME} + Expect(k8sClient.Create(context.Background(), config)).Should(Succeed()) + + cmd := &cobra.Command{} + namespace = testNamespace + // verify that finalizer has been added, by controller, upon object creation + config = &sriovnetworkv1.SriovOperatorConfig{} + Eventually(func() []string { + // wait for SriovOperatorConfig flags to get updated + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "default", Namespace: testNamespace}, config) + if err != nil { + return nil + } + return config.Finalizers + }, util.APITimeout, util.RetryInterval).Should(Equal([]string{sriovnetworkv1.OPERATORCONFIGFINALIZERNAME})) + + watchTO = 1 + err := runCleanupCmd(cmd, []string{}) + Expect(err.Error()).To(ContainSubstring("context deadline exceeded")) + }) +}) + +func getDefaultSriovOperatorConfig() *sriovnetworkv1.SriovOperatorConfig { + return &sriovnetworkv1.SriovOperatorConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: testNamespace, + }, + Spec: defaultSriovOperatorSpec, + } +} + +func newConfigController() *configController { + // setup controller manager + By("Setup controller manager") + k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + }) + Expect(err).ToNot(HaveOccurred()) + + t := GinkgoT() + mockCtrl := gomock.NewController(t) + platformHelper := mock_platforms.NewMockInterface(mockCtrl) + platformHelper.EXPECT().GetFlavor().Return(openshift.OpenshiftFlavorDefault).AnyTimes() + platformHelper.EXPECT().IsOpenshiftCluster().Return(false).AnyTimes() + platformHelper.EXPECT().IsHypershift().Return(false).AnyTimes() + + err = (&controllers.SriovOperatorConfigReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + PlatformHelper: platformHelper, + FeatureGate: featuregate.New(), + UncachedAPIReader: k8sManager.GetAPIReader(), + }).SetupWithManager(k8sManager) + Expect(err).ToNot(HaveOccurred()) + + ctx, cancel := context.WithCancel(context.Background()) + wg := sync.WaitGroup{} + controller = &configController{ + k8sManager: k8sManager, + ctx: ctx, + cancel: cancel, + wg: &wg, + } + + return controller +} + +func (c *configController) start() { + c.wg.Add(1) + go func() { + defer c.wg.Done() + defer GinkgoRecover() + By("Start controller manager") + err := c.k8sManager.Start(c.ctx) + Expect(err).ToNot(HaveOccurred()) + }() +} + +func (c *configController) stop() { + c.cancel() + c.wg.Wait() +} diff --git a/cmd/sriov-network-operator-config-cleanup/main.go b/cmd/sriov-network-operator-config-cleanup/main.go new file mode 100644 index 000000000..51874e54e --- /dev/null +++ b/cmd/sriov-network-operator-config-cleanup/main.go @@ -0,0 +1,38 @@ +package main + +import ( + "flag" + "os" + + "github.com/spf13/cobra" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/log" + + snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" +) + +const ( + componentName = "sriov-network-operator-config-cleanup" +) + +var ( + rootCmd = &cobra.Command{ + Use: componentName, + Short: "Removes 'default' SriovOperatorConfig", + Long: `Removes 'default' SriovOperatorConfig in order to cleanup non-namespaced objects e.g clusterroles/clusterrolebinding/validating/mutating webhooks + +Example: sriov-network-operator-config-cleanup -n `, + RunE: runCleanupCmd, + } +) + +func main() { + klog.InitFlags(nil) + snolog.BindFlags(flag.CommandLine) + rootCmd.PersistentFlags().AddGoFlagSet(flag.CommandLine) + + if err := rootCmd.Execute(); err != nil { + log.Log.Error(err, "Error executing sriov-network-operator-config-cleanup") + os.Exit(1) + } +} diff --git a/cmd/sriov-network-operator-config-cleanup/suite_test.go b/cmd/sriov-network-operator-config-cleanup/suite_test.go new file mode 100644 index 000000000..ee1815ff7 --- /dev/null +++ b/cmd/sriov-network-operator-config-cleanup/suite_test.go @@ -0,0 +1,121 @@ +package main + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/zap/zapcore" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + //+kubebuilder:scaffold:imports + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +var ( + k8sClient client.Client + testEnv *envtest.Environment + cfg *rest.Config + kubecfgPath string +) + +var _ = BeforeSuite(func() { + + logf.SetLogger(zap.New( + zap.WriteTo(GinkgoWriter), + zap.UseDevMode(true), + func(o *zap.Options) { + o.TimeEncoder = zapcore.RFC3339NanoTimeEncoder + })) + + // Go to project root directory + err := os.Chdir("../..") + Expect(err).NotTo(HaveOccurred()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("config", "crd", "bases"), filepath.Join("test", "util", "crds")}, + ErrorIfCRDPathMissing: true, + } + + testEnv.ControlPlane.GetAPIServer().Configure().Set("disable-admission-plugins", "MutatingAdmissionWebhook", "ValidatingAdmissionWebhook") + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + apiserverDir := testEnv.ControlPlane.GetAPIServer().CertDir + kubecfgPath = findKubecfg(apiserverDir, ".kubecfg") + err = os.Setenv("KUBECONFIG", kubecfgPath) + Expect(err).NotTo(HaveOccurred()) + + By("registering schemes") + err = sriovnetworkv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + vars.Config = cfg + vars.Scheme = scheme.Scheme + vars.Namespace = testNamespace + + By("creating K8s client") + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + By("creating default/common k8s objects for tests") + // Create test namespace + ns := &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: testNamespace, + }, + Spec: corev1.NamespaceSpec{}, + Status: corev1.NamespaceStatus{}, + } + ctx := context.Background() + Expect(k8sClient.Create(ctx, ns)).Should(Succeed()) +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + if testEnv != nil { + Eventually(func() error { + return testEnv.Stop() + }, util.APITimeout, time.Second).ShouldNot(HaveOccurred()) + } +}) + +func findKubecfg(path, ext string) string { + var cfg string + filepath.WalkDir(path, func(s string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if filepath.Ext(d.Name()) == ext { + cfg = s + } + return nil + }) + return cfg +} + +func TestAPIs(t *testing.T) { + _, reporterConfig := GinkgoConfiguration() + + RegisterFailHandler(Fail) + + RunSpecs(t, "operator-webhook Suite", reporterConfig) +} diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index a43123cfc..a83e08326 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -5,7 +5,6 @@ import ( "os" "github.com/spf13/cobra" - "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/log" snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" @@ -24,7 +23,6 @@ var ( ) func init() { - klog.InitFlags(nil) snolog.BindFlags(flag.CommandLine) rootCmd.PersistentFlags().AddGoFlagSet(flag.CommandLine) } diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovibnetworks.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovibnetworks.yaml index 4b4b44d92..498fb6c08 100644 --- a/config/crd/bases/sriovnetwork.openshift.io_sriovibnetworks.yaml +++ b/config/crd/bases/sriovnetwork.openshift.io_sriovibnetworks.yaml @@ -62,6 +62,8 @@ spec: networkNamespace: description: Namespace of the NetworkAttachmentDefinition custom resource type: string + pKey: + type: string resourceName: description: SRIOV Network device plugin endpoint resource name type: string diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml index 36c1050ea..524c5124e 100644 --- a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml +++ b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml @@ -81,6 +81,10 @@ spec: description: external_ids field in the Interface table in OVSDB type: object + mtuRequest: + description: mtu_request field in the Interface table + in OVSDB + type: integer options: additionalProperties: type: string diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml index c5bf230c3..7535346ef 100644 --- a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml +++ b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml @@ -102,6 +102,10 @@ spec: description: external_ids field in the Interface table in OVSDB type: object + mtuRequest: + description: mtu_request field in the Interface + table in OVSDB + type: integer options: additionalProperties: type: string @@ -174,6 +178,15 @@ spec: - pciAddress type: object type: array + system: + properties: + rdmaMode: + description: RDMA subsystem. Allowed value "shared", "exclusive". + enum: + - shared + - exclusive + type: string + type: object type: object status: description: SriovNetworkNodeStateStatus defines the observed state of @@ -228,6 +241,10 @@ spec: description: external_ids field in the Interface table in OVSDB type: object + mtuRequest: + description: mtu_request field in the Interface + table in OVSDB + type: integer options: additionalProperties: type: string @@ -335,6 +352,15 @@ spec: type: string syncStatus: type: string + system: + properties: + rdmaMode: + description: RDMA subsystem. Allowed value "shared", "exclusive". + enum: + - shared + - exclusive + type: string + type: object type: object type: object served: true diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml index 2cb2ece31..9ccb580ed 100644 --- a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml +++ b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml @@ -83,11 +83,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -111,6 +113,12 @@ spec: Name is the name of MachineConfigPool to be enabled with OVS hardware offload type: string type: object + rdmaMode: + description: RDMA subsystem. Allowed value "shared", "exclusive". + enum: + - shared + - exclusive + type: string type: object status: description: SriovNetworkPoolConfigStatus defines the observed state of diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml deleted file mode 100644 index 010f1b446..000000000 --- a/config/crd/kustomization.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# This kustomization.yaml is not intended to be run by itself, -# since it depends on service name and namespace that are out of this kustomize package. -# It should be run by config/default -resources: -- bases/sriovnetwork.openshift.io_sriovnetworks.yaml -- bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml -- bases/sriovnetwork.openshift.io_sriovibnetworks.yaml -- bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml -- bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml -- bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml -- bases/sriovnetwork.openshift.io_ovsnetworks.yaml -# +kubebuilder:scaffold:crdkustomizeresource - -patchesStrategicMerge: -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. -# patches here are for enabling the conversion webhook for each CRD -#- patches/webhook_in_sriovnetworks.yaml -#- patches/webhook_in_sriovnetworknodestates.yaml -#- patches/webhook_in_sriovibnetworks.yaml -#- patches/webhook_in_sriovnetworknodepolicies.yaml -#- patches/webhook_in_sriovoperatorconfigs.yaml -#- patches/webhook_in_sriovnetworkpoolconfigs.yaml -#- patches/webhook_in_ovsnetworks.yaml -# +kubebuilder:scaffold:crdkustomizewebhookpatch - -# [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. -# patches here are for enabling the CA injection for each CRD -#- patches/cainjection_in_sriovnetworks.yaml -#- patches/cainjection_in_sriovnetworknodestates.yaml -#- patches/cainjection_in_sriovibnetworks.yaml -#- patches/cainjection_in_sriovnetworknodepolicies.yaml -#- patches/cainjection_in_sriovoperatorconfigs.yaml -#- patches/cainjection_in_sriovnetworkpoolconfigs.yaml -#- patches/cainjection_in_ovsnetworks.yaml -# +kubebuilder:scaffold:crdkustomizecainjectionpatch - -# the following config is for teaching kustomize how to do kustomization for CRDs. -configurations: -- kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml deleted file mode 100644 index ec5c150a9..000000000 --- a/config/crd/kustomizeconfig.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# This file is for teaching kustomize how to substitute name and namespace reference in CRD -nameReference: -- kind: Service - version: v1 - fieldSpecs: - - kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/name - -namespace: -- kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/namespace - create: false - -varReference: -- path: metadata/annotations diff --git a/config/crd/patches/cainjection_in_ovsnetworks.yaml b/config/crd/patches/cainjection_in_ovsnetworks.yaml deleted file mode 100644 index 2386c4543..000000000 --- a/config/crd/patches/cainjection_in_ovsnetworks.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: ovsnetworks.sriovnetwork.openshift.io diff --git a/config/crd/patches/cainjection_in_sriovibnetworks.yaml b/config/crd/patches/cainjection_in_sriovibnetworks.yaml deleted file mode 100644 index 160dedeb6..000000000 --- a/config/crd/patches/cainjection_in_sriovibnetworks.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: sriovibnetworks.sriovnetwork.openshift.io diff --git a/config/crd/patches/cainjection_in_sriovnetworknodepolicies.yaml b/config/crd/patches/cainjection_in_sriovnetworknodepolicies.yaml deleted file mode 100644 index f8c201137..000000000 --- a/config/crd/patches/cainjection_in_sriovnetworknodepolicies.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: sriovnetworknodepolicies.sriovnetwork.openshift.io diff --git a/config/crd/patches/cainjection_in_sriovnetworknodestates.yaml b/config/crd/patches/cainjection_in_sriovnetworknodestates.yaml deleted file mode 100644 index 325e81848..000000000 --- a/config/crd/patches/cainjection_in_sriovnetworknodestates.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: sriovnetworknodestates.sriovnetwork.openshift.io diff --git a/config/crd/patches/cainjection_in_sriovnetworkpoolconfigs.yaml b/config/crd/patches/cainjection_in_sriovnetworkpoolconfigs.yaml deleted file mode 100644 index b84d31733..000000000 --- a/config/crd/patches/cainjection_in_sriovnetworkpoolconfigs.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: sriovnetworkpoolconfigs.sriovnetwork.openshift.io diff --git a/config/crd/patches/cainjection_in_sriovnetworks.yaml b/config/crd/patches/cainjection_in_sriovnetworks.yaml deleted file mode 100644 index 143c322eb..000000000 --- a/config/crd/patches/cainjection_in_sriovnetworks.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: sriovnetworks.sriovnetwork.openshift.io diff --git a/config/crd/patches/cainjection_in_sriovoperatorconfigs.yaml b/config/crd/patches/cainjection_in_sriovoperatorconfigs.yaml deleted file mode 100644 index 36d8da897..000000000 --- a/config/crd/patches/cainjection_in_sriovoperatorconfigs.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: sriovoperatorconfigs.sriovnetwork.openshift.io diff --git a/config/crd/patches/webhook_in_ovsnetworks.yaml b/config/crd/patches/webhook_in_ovsnetworks.yaml deleted file mode 100644 index c38d04145..000000000 --- a/config/crd/patches/webhook_in_ovsnetworks.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ovsnetworks.sriovnetwork.openshift.io -spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - namespace: system - name: webhook-service - path: /convert - conversionReviewVersions: - - v1 diff --git a/config/crd/patches/webhook_in_sriovibnetworks.yaml b/config/crd/patches/webhook_in_sriovibnetworks.yaml deleted file mode 100644 index b698dc009..000000000 --- a/config/crd/patches/webhook_in_sriovibnetworks.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: sriovibnetworks.sriovnetwork.openshift.io -spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - namespace: system - name: webhook-service - path: /convert - conversionReviewVersions: - - v1 diff --git a/config/crd/patches/webhook_in_sriovnetworknodepolicies.yaml b/config/crd/patches/webhook_in_sriovnetworknodepolicies.yaml deleted file mode 100644 index 4c284b683..000000000 --- a/config/crd/patches/webhook_in_sriovnetworknodepolicies.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: sriovnetworknodepolicies.sriovnetwork.openshift.io -spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - namespace: system - name: webhook-service - path: /convert - conversionReviewVersions: - - v1 diff --git a/config/crd/patches/webhook_in_sriovnetworknodestates.yaml b/config/crd/patches/webhook_in_sriovnetworknodestates.yaml deleted file mode 100644 index 30a99be02..000000000 --- a/config/crd/patches/webhook_in_sriovnetworknodestates.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: sriovnetworknodestates.sriovnetwork.openshift.io -spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - namespace: system - name: webhook-service - path: /convert - conversionReviewVersions: - - v1 diff --git a/config/crd/patches/webhook_in_sriovnetworkpoolconfigs.yaml b/config/crd/patches/webhook_in_sriovnetworkpoolconfigs.yaml deleted file mode 100644 index 3a78bc4a2..000000000 --- a/config/crd/patches/webhook_in_sriovnetworkpoolconfigs.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: sriovnetworkpoolconfigs.sriovnetwork.openshift.io -spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - namespace: system - name: webhook-service - path: /convert - conversionReviewVersions: - - v1 diff --git a/config/crd/patches/webhook_in_sriovnetworks.yaml b/config/crd/patches/webhook_in_sriovnetworks.yaml deleted file mode 100644 index 03cfb9883..000000000 --- a/config/crd/patches/webhook_in_sriovnetworks.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: sriovnetworks.sriovnetwork.openshift.io -spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - namespace: system - name: webhook-service - path: /convert - conversionReviewVersions: - - v1 diff --git a/config/crd/patches/webhook_in_sriovoperatorconfigs.yaml b/config/crd/patches/webhook_in_sriovoperatorconfigs.yaml deleted file mode 100644 index 001994b8f..000000000 --- a/config/crd/patches/webhook_in_sriovoperatorconfigs.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: sriovoperatorconfigs.sriovnetwork.openshift.io -spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - namespace: system - name: webhook-service - path: /convert - conversionReviewVersions: - - v1 diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml deleted file mode 100644 index fdae18c0d..000000000 --- a/config/default/kustomization.yaml +++ /dev/null @@ -1,74 +0,0 @@ -# Adds namespace to all resources. -namespace: sriov-network-operator-system - -# Value of this field is prepended to the -# names of all resources, e.g. a deployment named -# "wordpress" becomes "alices-wordpress". -# Note that it should also match with the prefix (text before '-') of the namespace -# field above. -namePrefix: sriov-network-operator- - -# Labels to add to all resources and selectors. -#commonLabels: -# someName: someValue - -bases: -- ../crd -- ../rbac -- ../manager -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in -# crd/kustomization.yaml -#- ../webhook -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. -#- ../certmanager -# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. -#- ../prometheus - -patchesStrategicMerge: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- manager_auth_proxy_patch.yaml - -# Mount the controller config file for loading manager configurations -# through a ComponentConfig type -#- manager_config_patch.yaml - -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in -# crd/kustomization.yaml -#- manager_webhook_patch.yaml - -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. -# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. -# 'CERTMANAGER' needs to be enabled to use ca injection -#- webhookcainjection_patch.yaml - -# the following config is for teaching kustomize how to do var substitution -vars: -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. -#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldref: -# fieldpath: metadata.namespace -#- name: CERTIFICATE_NAME -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -#- name: SERVICE_NAMESPACE # namespace of the service -# objref: -# kind: Service -# version: v1 -# name: webhook-service -# fieldref: -# fieldpath: metadata.namespace -#- name: SERVICE_NAME -# objref: -# kind: Service -# version: v1 -# name: webhook-service diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index a224be19e..000000000 --- a/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=10" - ports: - - containerPort: 8443 - name: https - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml deleted file mode 100644 index 6c400155c..000000000 --- a/config/default/manager_config_patch.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager - args: - - "--config=controller_manager_config.yaml" - volumeMounts: - - name: manager-config - mountPath: /controller_manager_config.yaml - subPath: controller_manager_config.yaml - volumes: - - name: manager-config - configMap: - name: manager-config diff --git a/config/manager/controller_manager_config.yaml b/config/manager/controller_manager_config.yaml deleted file mode 100644 index 5734e8e49..000000000 --- a/config/manager/controller_manager_config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 -kind: ControllerManagerConfig -health: - healthProbeBindAddress: :8081 -metrics: - bindAddress: 127.0.0.1:8080 -webhook: - port: 9443 -leaderElection: - leaderElect: true - resourceName: a56def2a.openshift.io diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml deleted file mode 100644 index 5e793dd19..000000000 --- a/config/manager/kustomization.yaml +++ /dev/null @@ -1,16 +0,0 @@ -resources: -- manager.yaml - -generatorOptions: - disableNameSuffixHash: true - -configMapGenerator: -- files: - - controller_manager_config.yaml - name: manager-config -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -images: -- name: controller - newName: controller - newTag: latest diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml deleted file mode 100644 index 9247ac6d1..000000000 --- a/config/manager/manager.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - control-plane: controller-manager - name: system ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - control-plane: controller-manager -spec: - selector: - matchLabels: - control-plane: controller-manager - replicas: 1 - template: - metadata: - labels: - control-plane: controller-manager - annotations: - openshift.io/required-scc: restricted-v2 - spec: - securityContext: - runAsNonRoot: true - containers: - - command: - - /manager - args: - - --leader-elect - image: controller:latest - name: manager - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: 100m - memory: 30Mi - requests: - cpu: 100m - memory: 20Mi - serviceAccountName: controller-manager - terminationGracePeriodSeconds: 10 diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml deleted file mode 100644 index ed137168a..000000000 --- a/config/prometheus/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- monitor.yaml diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml deleted file mode 100644 index d19136ae7..000000000 --- a/config/prometheus/monitor.yaml +++ /dev/null @@ -1,20 +0,0 @@ - -# Prometheus Monitor Service (Metrics) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - control-plane: controller-manager - name: controller-manager-metrics-monitor - namespace: system -spec: - endpoints: - - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true - selector: - matchLabels: - control-plane: controller-manager diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/auth_proxy_client_clusterrole.yaml deleted file mode 100644 index 51a75db47..000000000 --- a/config/rbac/auth_proxy_client_clusterrole.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: metrics-reader -rules: -- nonResourceURLs: - - "/metrics" - verbs: - - get diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml deleted file mode 100644 index 80e1857c5..000000000 --- a/config/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/config/rbac/auth_proxy_role_binding.yaml b/config/rbac/auth_proxy_role_binding.yaml deleted file mode 100644 index ec7acc0a1..000000000 --- a/config/rbac/auth_proxy_role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: proxy-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml deleted file mode 100644 index 6cf656be1..000000000 --- a/config/rbac/auth_proxy_service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - control-plane: controller-manager - name: controller-manager-metrics-service - namespace: system -spec: - ports: - - name: https - port: 8443 - targetPort: https - selector: - control-plane: controller-manager diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml deleted file mode 100644 index 731832a6a..000000000 --- a/config/rbac/kustomization.yaml +++ /dev/null @@ -1,18 +0,0 @@ -resources: -# All RBAC will be applied under this service account in -# the deployment namespace. You may comment out this resource -# if your manager will use a service account that exists at -# runtime. Be sure to update RoleBinding and ClusterRoleBinding -# subjects if changing service account names. -- service_account.yaml -- role.yaml -- role_binding.yaml -- leader_election_role.yaml -- leader_election_role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml deleted file mode 100644 index 4190ec805..000000000 --- a/config/rbac/leader_election_role.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# permissions to do leader election. -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: leader-election-role -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml deleted file mode 100644 index 1d1321ed4..000000000 --- a/config/rbac/leader_election_role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: leader-election-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: leader-election-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/ovsnetwork_editor_role.yaml b/config/rbac/ovsnetwork_editor_role.yaml deleted file mode 100644 index 8e5ee66ab..000000000 --- a/config/rbac/ovsnetwork_editor_role.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# permissions for end users to edit ovsnetworks. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata:plugins: - manifests.sdk.operatorframework.io/v2: {} - scorecard.sdk.operatorframework.io/v2: {} - name: ovsnetwork-editor-role -rules: -- apiGroups: - - sriovnetwork.openshift.io - resources: - - ovsnetworks - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - sriovnetwork.openshift.io - resources: - - ovsnetworks/status - verbs: - - get diff --git a/config/rbac/ovsnetwork_viewer_role.yaml b/config/rbac/ovsnetwork_viewer_role.yaml deleted file mode 100644 index b5b0cb981..000000000 --- a/config/rbac/ovsnetwork_viewer_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# permissions for end users to view ovsnetworks. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: ovsnetwork-viewer-role -rules: -- apiGroups: - - sriovnetwork.openshift.io - resources: - - ovsnetworks - verbs: - - get - - list - - watch -- apiGroups: - - sriovnetwork.openshift.io - resources: - - ovsnetworks/status - verbs: - - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml deleted file mode 100644 index 12203cf2f..000000000 --- a/config/rbac/role.yaml +++ /dev/null @@ -1,112 +0,0 @@ - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: manager-role -rules: -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovibnetworks - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovibnetworks/finalizers - verbs: - - update -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovibnetworks/status - verbs: - - get - - patch - - update -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworknodepolicies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworknodepolicies/finalizers - verbs: - - update -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworknodepolicies/status - verbs: - - get - - patch - - update -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworks - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworks/finalizers - verbs: - - update -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworks/status - verbs: - - get - - patch - - update -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovoperatorconfigs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovoperatorconfigs/finalizers - verbs: - - update -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovoperatorconfigs/status - verbs: - - get - - patch - - update diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml deleted file mode 100644 index 2070ede44..000000000 --- a/config/rbac/role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: manager-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml deleted file mode 100644 index 7cd6025bf..000000000 --- a/config/rbac/service_account.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: controller-manager - namespace: system diff --git a/config/rbac/sriovibnetwork_editor_role.yaml b/config/rbac/sriovibnetwork_editor_role.yaml deleted file mode 100644 index 46ca9b6f1..000000000 --- a/config/rbac/sriovibnetwork_editor_role.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# permissions for end users to edit sriovibnetworks. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: sriovibnetwork-editor-role -rules: -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovibnetworks - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovibnetworks/status - verbs: - - get diff --git a/config/rbac/sriovibnetwork_viewer_role.yaml b/config/rbac/sriovibnetwork_viewer_role.yaml deleted file mode 100644 index 4d980b0e7..000000000 --- a/config/rbac/sriovibnetwork_viewer_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# permissions for end users to view sriovibnetworks. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: sriovibnetwork-viewer-role -rules: -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovibnetworks - verbs: - - get - - list - - watch -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovibnetworks/status - verbs: - - get diff --git a/config/rbac/sriovnetwork_editor_role.yaml b/config/rbac/sriovnetwork_editor_role.yaml deleted file mode 100644 index 70e9f5ebd..000000000 --- a/config/rbac/sriovnetwork_editor_role.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# permissions for end users to edit sriovnetworks. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: sriovnetwork-editor-role -rules: -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworks - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworks/status - verbs: - - get diff --git a/config/rbac/sriovnetwork_viewer_role.yaml b/config/rbac/sriovnetwork_viewer_role.yaml deleted file mode 100644 index bd034738f..000000000 --- a/config/rbac/sriovnetwork_viewer_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# permissions for end users to view sriovnetworks. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: sriovnetwork-viewer-role -rules: -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworks - verbs: - - get - - list - - watch -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworks/status - verbs: - - get diff --git a/config/rbac/sriovnetworknodepolicy_editor_role.yaml b/config/rbac/sriovnetworknodepolicy_editor_role.yaml deleted file mode 100644 index 773791952..000000000 --- a/config/rbac/sriovnetworknodepolicy_editor_role.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# permissions for end users to edit sriovnetworknodepolicies. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: sriovnetworknodepolicy-editor-role -rules: -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworknodepolicies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworknodepolicies/status - verbs: - - get diff --git a/config/rbac/sriovnetworknodepolicy_viewer_role.yaml b/config/rbac/sriovnetworknodepolicy_viewer_role.yaml deleted file mode 100644 index c2fdda90d..000000000 --- a/config/rbac/sriovnetworknodepolicy_viewer_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# permissions for end users to view sriovnetworknodepolicies. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: sriovnetworknodepolicy-viewer-role -rules: -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworknodepolicies - verbs: - - get - - list - - watch -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworknodepolicies/status - verbs: - - get diff --git a/config/rbac/sriovnetworknodestate_editor_role.yaml b/config/rbac/sriovnetworknodestate_editor_role.yaml deleted file mode 100644 index 4022a44a7..000000000 --- a/config/rbac/sriovnetworknodestate_editor_role.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# permissions for end users to edit sriovnetworknodestates. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: sriovnetworknodestate-editor-role -rules: -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworknodestates - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworknodestates/status - verbs: - - get diff --git a/config/rbac/sriovnetworknodestate_viewer_role.yaml b/config/rbac/sriovnetworknodestate_viewer_role.yaml deleted file mode 100644 index 4f8ab6fb4..000000000 --- a/config/rbac/sriovnetworknodestate_viewer_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# permissions for end users to view sriovnetworknodestates. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: sriovnetworknodestate-viewer-role -rules: -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworknodestates - verbs: - - get - - list - - watch -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworknodestates/status - verbs: - - get diff --git a/config/rbac/sriovnetworkpoolconfig_editor_role.yaml b/config/rbac/sriovnetworkpoolconfig_editor_role.yaml deleted file mode 100644 index 342506ecf..000000000 --- a/config/rbac/sriovnetworkpoolconfig_editor_role.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# permissions for end users to edit sriovnetworkpoolconfigs. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: sriovnetworkpoolconfig-editor-role -rules: -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworkpoolconfigs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworkpoolconfigs/status - verbs: - - get diff --git a/config/rbac/sriovnetworkpoolconfig_viewer_role.yaml b/config/rbac/sriovnetworkpoolconfig_viewer_role.yaml deleted file mode 100644 index d669d324e..000000000 --- a/config/rbac/sriovnetworkpoolconfig_viewer_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# permissions for end users to view sriovnetworkpoolconfigs. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: sriovnetworkpoolconfig-viewer-role -rules: -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworkpoolconfigs - verbs: - - get - - list - - watch -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovnetworkpoolconfigs/status - verbs: - - get diff --git a/config/rbac/sriovoperatorconfig_editor_role.yaml b/config/rbac/sriovoperatorconfig_editor_role.yaml deleted file mode 100644 index 8cc180c21..000000000 --- a/config/rbac/sriovoperatorconfig_editor_role.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# permissions for end users to edit sriovoperatorconfigs. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: sriovoperatorconfig-editor-role -rules: -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovoperatorconfigs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovoperatorconfigs/status - verbs: - - get diff --git a/config/rbac/sriovoperatorconfig_viewer_role.yaml b/config/rbac/sriovoperatorconfig_viewer_role.yaml deleted file mode 100644 index bdf45064b..000000000 --- a/config/rbac/sriovoperatorconfig_viewer_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# permissions for end users to view sriovoperatorconfigs. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: sriovoperatorconfig-viewer-role -rules: -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovoperatorconfigs - verbs: - - get - - list - - watch -- apiGroups: - - sriovnetwork.openshift.io - resources: - - sriovoperatorconfigs/status - verbs: - - get diff --git a/config/samples/sriovnetwork_v1_ovsnetwork.yaml b/config/samples/sriovnetwork_v1_ovsnetwork.yaml deleted file mode 100644 index 6a81880c0..000000000 --- a/config/samples/sriovnetwork_v1_ovsnetwork.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: sriovnetwork.openshift.io/v1 -kind: OVSNetwork -metadata: - name: example-ovsnetwork -spec: - ipam: | - { - "type": "host-local", - "ranges": [[{"subnet": "192.178.0.0/16"}]] - } - networkNamespace: default - resourceName: switchdev-vfs - vlan: 100 - mtu: 1500 - diff --git a/config/samples/sriovnetwork_v1_sriovibnetwork.yaml b/config/samples/sriovnetwork_v1_sriovibnetwork.yaml deleted file mode 100644 index 6a2a9240e..000000000 --- a/config/samples/sriovnetwork_v1_sriovibnetwork.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: sriovnetwork.openshift.io/v1 -kind: SriovIBNetwork -metadata: - name: sriovibnetwork-sample -spec: - # Add fields here - foo: bar diff --git a/config/samples/sriovnetwork_v1_sriovnetwork.yaml b/config/samples/sriovnetwork_v1_sriovnetwork.yaml deleted file mode 100644 index 7d931a79c..000000000 --- a/config/samples/sriovnetwork_v1_sriovnetwork.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: sriovnetwork.openshift.io/v1 -kind: SriovNetwork -metadata: - name: sriovnetwork-sample -spec: - # Add fields here - foo: bar diff --git a/config/samples/sriovnetwork_v1_sriovnetworknodepolicy.yaml b/config/samples/sriovnetwork_v1_sriovnetworknodepolicy.yaml deleted file mode 100644 index 5fafad497..000000000 --- a/config/samples/sriovnetwork_v1_sriovnetworknodepolicy.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: sriovnetwork.openshift.io/v1 -kind: SriovNetworkNodePolicy -metadata: - name: sriovnetworknodepolicy-sample -spec: - # Add fields here - foo: bar diff --git a/config/samples/sriovnetwork_v1_sriovnetworknodestate.yaml b/config/samples/sriovnetwork_v1_sriovnetworknodestate.yaml deleted file mode 100644 index e01099c9c..000000000 --- a/config/samples/sriovnetwork_v1_sriovnetworknodestate.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: sriovnetwork.openshift.io/v1 -kind: SriovNetworkNodeState -metadata: - name: sriovnetworknodestate-sample -spec: - # Add fields here - foo: bar diff --git a/config/samples/sriovnetwork_v1_sriovnetworkpoolconfig.yaml b/config/samples/sriovnetwork_v1_sriovnetworkpoolconfig.yaml deleted file mode 100644 index c02325e9a..000000000 --- a/config/samples/sriovnetwork_v1_sriovnetworkpoolconfig.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: sriovnetwork.openshift.io/v1 -kind: SriovNetworkPoolConfig -metadata: - name: sriovnetworkpoolconfig-sample -spec: - # Add fields here - foo: bar diff --git a/config/samples/sriovnetwork_v1_sriovoperatorconfig.yaml b/config/samples/sriovnetwork_v1_sriovoperatorconfig.yaml deleted file mode 100644 index 14e83c09b..000000000 --- a/config/samples/sriovnetwork_v1_sriovoperatorconfig.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: sriovnetwork.openshift.io/v1 -kind: SriovOperatorConfig -metadata: - name: sriovoperatorconfig-sample -spec: - # Add fields here - foo: bar diff --git a/controllers/drain_controller.go b/controllers/drain_controller.go index 86da909d8..088d90360 100644 --- a/controllers/drain_controller.go +++ b/controllers/drain_controller.go @@ -20,15 +20,12 @@ import ( "context" "fmt" "sync" - "time" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" @@ -48,13 +45,6 @@ import ( "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) -var ( - oneNode = intstr.FromInt32(1) - defaultNpcl = &sriovnetworkv1.SriovNetworkPoolConfig{Spec: sriovnetworkv1.SriovNetworkPoolConfigSpec{ - MaxUnavailable: &oneNode, - NodeSelector: &metav1.LabelSelector{}}} -) - type DrainReconcile struct { client.Client Scheme *runtime.Scheme @@ -87,8 +77,8 @@ func NewDrainReconcileController(client client.Client, Scheme *runtime.Scheme, r // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.8.3/pkg/reconcile func (dr *DrainReconcile) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - reqLogger := log.FromContext(ctx) - reqLogger.Info("Reconciling Drain") + ctx = context.WithValue(ctx, "logger", log.FromContext(ctx)) + reqLogger := log.FromContext(ctx).WithName("Drain Reconcile") req.Namespace = vars.Namespace @@ -117,19 +107,28 @@ func (dr *DrainReconcile) Reconcile(ctx context.Context, req ctrl.Request) (ctrl } // create the drain state annotation if it doesn't exist in the sriovNetworkNodeState object - nodeStateDrainAnnotationCurrent, err := dr.ensureAnnotationExists(ctx, nodeNetworkState, constants.NodeStateDrainAnnotationCurrent) + nodeStateDrainAnnotationCurrent, currentNodeStateExist, err := dr.ensureAnnotationExists(ctx, nodeNetworkState, constants.NodeStateDrainAnnotationCurrent) + if err != nil { + reqLogger.Error(err, "failed to ensure nodeStateDrainAnnotationCurrent") + return ctrl.Result{}, err + } + _, desireNodeStateExist, err := dr.ensureAnnotationExists(ctx, nodeNetworkState, constants.NodeStateDrainAnnotation) if err != nil { reqLogger.Error(err, "failed to ensure nodeStateDrainAnnotation") return ctrl.Result{}, err } // create the drain state annotation if it doesn't exist in the node object - nodeDrainAnnotation, err := dr.ensureAnnotationExists(ctx, node, constants.NodeDrainAnnotation) + nodeDrainAnnotation, nodeExist, err := dr.ensureAnnotationExists(ctx, node, constants.NodeDrainAnnotation) if err != nil { reqLogger.Error(err, "failed to ensure nodeStateDrainAnnotation") return ctrl.Result{}, err } + // requeue the request if we needed to add any of the annotations + if !nodeExist || !currentNodeStateExist || !desireNodeStateExist { + return ctrl.Result{Requeue: true}, nil + } reqLogger.V(2).Info("Drain annotations", "nodeAnnotation", nodeDrainAnnotation, "nodeStateAnnotation", nodeStateDrainAnnotationCurrent) // Check the node request @@ -151,98 +150,14 @@ func (dr *DrainReconcile) Reconcile(ctx context.Context, req ctrl.Request) (ctrl // doesn't need to drain anymore, so we can stop the drain if nodeStateDrainAnnotationCurrent == constants.DrainComplete || nodeStateDrainAnnotationCurrent == constants.Draining { - completed, err := dr.drainer.CompleteDrainNode(ctx, node) - if err != nil { - reqLogger.Error(err, "failed to complete drain on node") - dr.recorder.Event(nodeNetworkState, - corev1.EventTypeWarning, - "DrainController", - "failed to drain node") - return ctrl.Result{}, err - } - - // if we didn't manage to complete the un drain of the node we retry - if !completed { - reqLogger.Info("complete drain was not completed re queueing the request") - dr.recorder.Event(nodeNetworkState, - corev1.EventTypeWarning, - "DrainController", - "node complete drain was not completed") - // TODO: make this time configurable - return reconcile.Result{RequeueAfter: 5 * time.Second}, nil - } - - // move the node state back to idle - err = utils.AnnotateObject(ctx, nodeNetworkState, constants.NodeStateDrainAnnotationCurrent, constants.DrainIdle, dr.Client) - if err != nil { - reqLogger.Error(err, "failed to annotate node with annotation", "annotation", constants.DrainIdle) - return ctrl.Result{}, err - } - - reqLogger.Info("completed the un drain for node") - dr.recorder.Event(nodeNetworkState, - corev1.EventTypeWarning, - "DrainController", - "node un drain completed") - return ctrl.Result{}, nil - } - } else if nodeDrainAnnotation == constants.DrainRequired || nodeDrainAnnotation == constants.RebootRequired { - // this cover the case a node request to drain or reboot - - // nothing to do here we need to wait for the node to move back to idle - if nodeStateDrainAnnotationCurrent == constants.DrainComplete { - reqLogger.Info("node requested a drain and nodeState is on drain completed nothing todo") - return ctrl.Result{}, nil - } - - // we need to start the drain, but first we need to check that we can drain the node - if nodeStateDrainAnnotationCurrent == constants.DrainIdle { - result, err := dr.tryDrainNode(ctx, node) - if err != nil { - reqLogger.Error(err, "failed to check if we can drain the node") - return ctrl.Result{}, err - } - - // in case we need to wait because we just to the max number of draining nodes - if result != nil { - return *result, nil - } - } - - // class the drain function that will also call drain to other platform providers like openshift - drained, err := dr.drainer.DrainNode(ctx, node, nodeDrainAnnotation == constants.RebootRequired) - if err != nil { - reqLogger.Error(err, "error trying to drain the node") - dr.recorder.Event(nodeNetworkState, - corev1.EventTypeWarning, - "DrainController", - "failed to drain node") - return reconcile.Result{}, err - } - - // if we didn't manage to complete the drain of the node we retry - if !drained { - reqLogger.Info("the nodes was not drained re queueing the request") - dr.recorder.Event(nodeNetworkState, - corev1.EventTypeWarning, - "DrainController", - "node drain operation was not completed") - return reconcile.Result{RequeueAfter: 5 * time.Second}, nil - } - - // if we manage to drain we label the node state with drain completed and finish - err = utils.AnnotateObject(ctx, nodeNetworkState, constants.NodeStateDrainAnnotationCurrent, constants.DrainComplete, dr.Client) - if err != nil { - reqLogger.Error(err, "failed to annotate node with annotation", "annotation", constants.DrainComplete) - return ctrl.Result{}, err + return dr.handleNodeIdleNodeStateDrainingOrCompleted(ctx, node, nodeNetworkState) } + } - reqLogger.Info("node drained successfully") - dr.recorder.Event(nodeNetworkState, - corev1.EventTypeWarning, - "DrainController", - "node drain completed") - return ctrl.Result{}, nil + // this cover the case a node request to drain or reboot + if nodeDrainAnnotation == constants.DrainRequired || + nodeDrainAnnotation == constants.RebootRequired { + return dr.handleNodeDrainOrReboot(ctx, node, nodeNetworkState, nodeDrainAnnotation, nodeStateDrainAnnotationCurrent) } reqLogger.Error(nil, "unexpected node drain annotation") @@ -260,193 +175,30 @@ func (dr *DrainReconcile) getObject(ctx context.Context, req ctrl.Request, objec return true, nil } -func (dr *DrainReconcile) ensureAnnotationExists(ctx context.Context, object client.Object, key string) (string, error) { +func (dr *DrainReconcile) ensureAnnotationExists(ctx context.Context, object client.Object, key string) (string, bool, error) { value, exist := object.GetAnnotations()[key] if !exist { - err := utils.AnnotateObject(ctx, object, constants.NodeStateDrainAnnotationCurrent, constants.DrainIdle, dr.Client) - if err != nil { - return "", err - } - return constants.DrainIdle, nil - } - - return value, nil -} - -func (dr *DrainReconcile) tryDrainNode(ctx context.Context, node *corev1.Node) (*reconcile.Result, error) { - // configure logs - reqLogger := log.FromContext(ctx) - reqLogger.Info("checkForNodeDrain():") - - //critical section we need to check if we can start the draining - dr.drainCheckMutex.Lock() - defer dr.drainCheckMutex.Unlock() - - // find the relevant node pool - nodePool, nodeList, err := dr.findNodePoolConfig(ctx, node) - if err != nil { - reqLogger.Error(err, "failed to find the pool for the requested node") - return nil, err - } - - // check how many nodes we can drain in parallel for the specific pool - maxUnv, err := nodePool.MaxUnavailable(len(nodeList)) - if err != nil { - reqLogger.Error(err, "failed to calculate max unavailable") - return nil, err - } - - current := 0 - snns := &sriovnetworkv1.SriovNetworkNodeState{} - - var currentSnns *sriovnetworkv1.SriovNetworkNodeState - for _, nodeObj := range nodeList { - err = dr.Get(ctx, client.ObjectKey{Name: nodeObj.GetName(), Namespace: vars.Namespace}, snns) - if err != nil { - if errors.IsNotFound(err) { - reqLogger.V(2).Info("node doesn't have a sriovNetworkNodePolicy") - continue - } - return nil, err - } - - if snns.GetName() == node.GetName() { - currentSnns = snns.DeepCopy() - } - - if utils.ObjectHasAnnotation(snns, constants.NodeStateDrainAnnotationCurrent, constants.Draining) || - utils.ObjectHasAnnotation(snns, constants.NodeStateDrainAnnotationCurrent, constants.DrainComplete) { - current++ - } - } - reqLogger.Info("Max node allowed to be draining at the same time", "MaxParallelNodeConfiguration", maxUnv) - reqLogger.Info("Count of draining", "drainingNodes", current) - - // if maxUnv is zero this means we drain all the nodes in parallel without a limit - if maxUnv == -1 { - reqLogger.Info("draining all the nodes in parallel") - } else if current >= maxUnv { - // the node requested to be drained, but we are at the limit so we re-enqueue the request - reqLogger.Info("MaxParallelNodeConfiguration limit reached for draining nodes re-enqueue the request") - // TODO: make this time configurable - return &reconcile.Result{RequeueAfter: 5 * time.Second}, nil - } - - if currentSnns == nil { - return nil, fmt.Errorf("failed to find sriov network node state for requested node") - } - - err = utils.AnnotateObject(ctx, currentSnns, constants.NodeStateDrainAnnotationCurrent, constants.Draining, dr.Client) - if err != nil { - reqLogger.Error(err, "failed to annotate node with annotation", "annotation", constants.Draining) - return nil, err - } - - return nil, nil -} - -func (dr *DrainReconcile) findNodePoolConfig(ctx context.Context, node *corev1.Node) (*sriovnetworkv1.SriovNetworkPoolConfig, []corev1.Node, error) { - logger := log.FromContext(ctx) - logger.Info("findNodePoolConfig():") - // get all the sriov network pool configs - npcl := &sriovnetworkv1.SriovNetworkPoolConfigList{} - err := dr.List(ctx, npcl) - if err != nil { - logger.Error(err, "failed to list sriovNetworkPoolConfig") - return nil, nil, err - } - - selectedNpcl := []*sriovnetworkv1.SriovNetworkPoolConfig{} - nodesInPools := map[string]interface{}{} - - for _, npc := range npcl.Items { - // we skip hw offload objects - if npc.Spec.OvsHardwareOffloadConfig.Name != "" { - continue - } - - if npc.Spec.NodeSelector == nil { - npc.Spec.NodeSelector = &metav1.LabelSelector{} - } - - selector, err := metav1.LabelSelectorAsSelector(npc.Spec.NodeSelector) + err := utils.AnnotateObject(ctx, object, key, constants.DrainIdle, dr.Client) if err != nil { - logger.Error(err, "failed to create label selector from nodeSelector", "nodeSelector", npc.Spec.NodeSelector) - return nil, nil, err - } - - if selector.Matches(labels.Set(node.Labels)) { - selectedNpcl = append(selectedNpcl, npc.DeepCopy()) - } - - nodeList := &corev1.NodeList{} - err = dr.List(ctx, nodeList, &client.ListOptions{LabelSelector: selector}) - if err != nil { - logger.Error(err, "failed to list all the nodes matching the pool with label selector from nodeSelector", - "machineConfigPoolName", npc, - "nodeSelector", npc.Spec.NodeSelector) - return nil, nil, err - } - - for _, nodeName := range nodeList.Items { - nodesInPools[nodeName.Name] = nil + return "", false, err } + return constants.DrainIdle, false, nil } - if len(selectedNpcl) > 1 { - // don't allow the node to be part of multiple pools - err = fmt.Errorf("node is part of more then one pool") - logger.Error(err, "multiple pools founded for a specific node", "numberOfPools", len(selectedNpcl), "pools", selectedNpcl) - return nil, nil, err - } else if len(selectedNpcl) == 1 { - // found one pool for our node - logger.V(2).Info("found sriovNetworkPool", "pool", *selectedNpcl[0]) - selector, err := metav1.LabelSelectorAsSelector(selectedNpcl[0].Spec.NodeSelector) - if err != nil { - logger.Error(err, "failed to create label selector from nodeSelector", "nodeSelector", selectedNpcl[0].Spec.NodeSelector) - return nil, nil, err - } - - // list all the nodes that are also part of this pool and return them - nodeList := &corev1.NodeList{} - err = dr.List(ctx, nodeList, &client.ListOptions{LabelSelector: selector}) - if err != nil { - logger.Error(err, "failed to list nodes using with label selector", "labelSelector", selector) - return nil, nil, err - } - - return selectedNpcl[0], nodeList.Items, nil - } else { - // in this case we get all the nodes and remove the ones that already part of any pool - logger.V(1).Info("node doesn't belong to any pool, using default drain configuration with MaxUnavailable of one", "pool", *defaultNpcl) - nodeList := &corev1.NodeList{} - err = dr.List(ctx, nodeList) - if err != nil { - logger.Error(err, "failed to list all the nodes") - return nil, nil, err - } - - defaultNodeLists := []corev1.Node{} - for _, nodeObj := range nodeList.Items { - if _, exist := nodesInPools[nodeObj.Name]; !exist { - defaultNodeLists = append(defaultNodeLists, nodeObj) - } - } - return defaultNpcl, defaultNodeLists, nil - } + return value, true, nil } // SetupWithManager sets up the controller with the Manager. func (dr *DrainReconcile) SetupWithManager(mgr ctrl.Manager) error { createUpdateEnqueue := handler.Funcs{ - CreateFunc: func(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { - q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + CreateFunc: func(c context.Context, e event.TypedCreateEvent[client.Object], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { + w.Add(reconcile.Request{NamespacedName: types.NamespacedName{ Namespace: vars.Namespace, Name: e.Object.GetName(), }}) }, - UpdateFunc: func(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { - q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + UpdateFunc: func(ctx context.Context, e event.TypedUpdateEvent[client.Object], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { + w.Add(reconcile.Request{NamespacedName: types.NamespacedName{ Namespace: vars.Namespace, Name: e.ObjectNew.GetName(), }}) @@ -458,7 +210,17 @@ func (dr *DrainReconcile) SetupWithManager(mgr ctrl.Manager) error { nodeStatePredicates := builder.WithPredicates(DrainStateAnnotationPredicate{}) return ctrl.NewControllerManagedBy(mgr). - WithOptions(controller.Options{MaxConcurrentReconciles: 50}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: 50, + LogConstructor: func(request *reconcile.Request) logr.Logger { + logger := mgr.GetLogger().WithValues("Function", "Drain") + // Inspired by https://github.com/kubernetes-sigs/controller-runtime/blob/52b17917caa97ec546423867d9637f1787830f3e/pkg/builder/controller.go#L447 + if req, ok := any(request).(*reconcile.Request); ok && req != nil { + logger = logger.WithValues("node", request.Name) + } + return logger + }, + }). For(&corev1.Node{}, nodePredicates). Watches(&sriovnetworkv1.SriovNetworkNodeState{}, createUpdateEnqueue, nodeStatePredicates). Complete(dr) diff --git a/controllers/drain_controller_helper.go b/controllers/drain_controller_helper.go new file mode 100644 index 000000000..e284c9daa --- /dev/null +++ b/controllers/drain_controller_helper.go @@ -0,0 +1,298 @@ +package controllers + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" +) + +func (dr *DrainReconcile) handleNodeIdleNodeStateDrainingOrCompleted(ctx context.Context, + node *corev1.Node, + nodeNetworkState *sriovnetworkv1.SriovNetworkNodeState) (ctrl.Result, error) { + reqLogger := ctx.Value("logger").(logr.Logger).WithName("handleNodeIdleNodeStateDrainingOrCompleted") + completed, err := dr.drainer.CompleteDrainNode(ctx, node) + if err != nil { + reqLogger.Error(err, "failed to complete drain on node") + dr.recorder.Event(nodeNetworkState, + corev1.EventTypeWarning, + "DrainController", + "failed to drain node") + return ctrl.Result{}, err + } + + // if we didn't manage to complete the un drain of the node we retry + if !completed { + reqLogger.Info("complete drain was not completed re queueing the request") + dr.recorder.Event(nodeNetworkState, + corev1.EventTypeWarning, + "DrainController", + "node complete drain was not completed") + // TODO: make this time configurable + return reconcile.Result{RequeueAfter: constants.DrainControllerRequeueTime}, nil + } + + // move the node state back to idle + err = utils.AnnotateObject(ctx, nodeNetworkState, constants.NodeStateDrainAnnotationCurrent, constants.DrainIdle, dr.Client) + if err != nil { + reqLogger.Error(err, "failed to annotate node with annotation", "annotation", constants.DrainIdle) + return ctrl.Result{}, err + } + + reqLogger.Info("completed the un drain for node") + dr.recorder.Event(nodeNetworkState, + corev1.EventTypeWarning, + "DrainController", + "node un drain completed") + return ctrl.Result{}, nil +} + +func (dr *DrainReconcile) handleNodeDrainOrReboot(ctx context.Context, + node *corev1.Node, + nodeNetworkState *sriovnetworkv1.SriovNetworkNodeState, + nodeDrainAnnotation, + nodeStateDrainAnnotationCurrent string) (ctrl.Result, error) { + reqLogger := ctx.Value("logger").(logr.Logger).WithName("handleNodeDrainOrReboot") + // nothing to do here we need to wait for the node to move back to idle + if nodeStateDrainAnnotationCurrent == constants.DrainComplete { + reqLogger.Info("node requested a drain and nodeState is on drain completed nothing todo") + return ctrl.Result{}, nil + } + + // we need to start the drain, but first we need to check that we can drain the node + if nodeStateDrainAnnotationCurrent == constants.DrainIdle { + result, err := dr.tryDrainNode(ctx, node) + if err != nil { + reqLogger.Error(err, "failed to check if we can drain the node") + return ctrl.Result{}, err + } + + // in case we need to wait because we just to the max number of draining nodes + if result != nil { + return *result, nil + } + } + + // Check if we are on a single node, and we require a reboot/full-drain we just return + fullNodeDrain := nodeDrainAnnotation == constants.RebootRequired + singleNode := false + if fullNodeDrain { + nodeList := &corev1.NodeList{} + err := dr.Client.List(ctx, nodeList) + if err != nil { + reqLogger.Error(err, "failed to list nodes") + return ctrl.Result{}, err + } + if len(nodeList.Items) == 1 { + reqLogger.Info("drainNode(): FullNodeDrain requested and we are on Single node") + singleNode = true + } + } + + // call the drain function that will also call drain to other platform providers like openshift + drained, err := dr.drainer.DrainNode(ctx, node, fullNodeDrain, singleNode) + if err != nil { + reqLogger.Error(err, "error trying to drain the node") + dr.recorder.Event(nodeNetworkState, + corev1.EventTypeWarning, + "DrainController", + "failed to drain node") + return reconcile.Result{}, err + } + + // if we didn't manage to complete the drain of the node we retry + if !drained { + reqLogger.Info("the nodes was not drained re queueing the request") + dr.recorder.Event(nodeNetworkState, + corev1.EventTypeWarning, + "DrainController", + "node drain operation was not completed") + return reconcile.Result{RequeueAfter: constants.DrainControllerRequeueTime}, nil + } + + // if we manage to drain we label the node state with drain completed and finish + err = utils.AnnotateObject(ctx, nodeNetworkState, constants.NodeStateDrainAnnotationCurrent, constants.DrainComplete, dr.Client) + if err != nil { + reqLogger.Error(err, "failed to annotate node with annotation", "annotation", constants.DrainComplete) + return ctrl.Result{}, err + } + + reqLogger.Info("node drained successfully") + dr.recorder.Event(nodeNetworkState, + corev1.EventTypeWarning, + "DrainController", + "node drain completed") + return ctrl.Result{}, nil +} + +func (dr *DrainReconcile) tryDrainNode(ctx context.Context, node *corev1.Node) (*reconcile.Result, error) { + reqLogger := ctx.Value("logger").(logr.Logger).WithName("tryDrainNode") + + //critical section we need to check if we can start the draining + dr.drainCheckMutex.Lock() + defer dr.drainCheckMutex.Unlock() + + // find the relevant node pool + nodePool, nodeList, err := dr.findNodePoolConfig(ctx, node) + if err != nil { + reqLogger.Error(err, "failed to find the pool for the requested node") + return nil, err + } + + // check how many nodes we can drain in parallel for the specific pool + maxUnv, err := nodePool.MaxUnavailable(len(nodeList)) + if err != nil { + reqLogger.Error(err, "failed to calculate max unavailable") + return nil, err + } + + current := 0 + snns := &sriovnetworkv1.SriovNetworkNodeState{} + + var currentSnns *sriovnetworkv1.SriovNetworkNodeState + for _, nodeObj := range nodeList { + err = dr.Get(ctx, client.ObjectKey{Name: nodeObj.GetName(), Namespace: vars.Namespace}, snns) + if err != nil { + if errors.IsNotFound(err) { + reqLogger.V(2).Info("node doesn't have a sriovNetworkNodeState, skipping") + continue + } + return nil, err + } + + if snns.GetName() == node.GetName() { + currentSnns = snns.DeepCopy() + } + + if utils.ObjectHasAnnotation(snns, constants.NodeStateDrainAnnotationCurrent, constants.Draining) || + utils.ObjectHasAnnotation(snns, constants.NodeStateDrainAnnotationCurrent, constants.DrainComplete) { + current++ + } + } + reqLogger.Info("Max node allowed to be draining at the same time", "MaxParallelNodeConfiguration", maxUnv) + reqLogger.Info("Count of draining", "drainingNodes", current) + + // if maxUnv is zero this means we drain all the nodes in parallel without a limit + if maxUnv == -1 { + reqLogger.Info("draining all the nodes in parallel") + } else if current >= maxUnv { + // the node requested to be drained, but we are at the limit so we re-enqueue the request + reqLogger.Info("MaxParallelNodeConfiguration limit reached for draining nodes re-enqueue the request") + // TODO: make this time configurable + return &reconcile.Result{RequeueAfter: constants.DrainControllerRequeueTime}, nil + } + + if currentSnns == nil { + return nil, fmt.Errorf("failed to find sriov network node state for requested node") + } + + err = utils.AnnotateObject(ctx, currentSnns, constants.NodeStateDrainAnnotationCurrent, constants.Draining, dr.Client) + if err != nil { + reqLogger.Error(err, "failed to annotate node with annotation", "annotation", constants.Draining) + return nil, err + } + + return nil, nil +} + +func (dr *DrainReconcile) findNodePoolConfig(ctx context.Context, node *corev1.Node) (*sriovnetworkv1.SriovNetworkPoolConfig, []corev1.Node, error) { + logger := ctx.Value("logger").(logr.Logger).WithName("findNodePoolConfig") + // get all the sriov network pool configs + npcl := &sriovnetworkv1.SriovNetworkPoolConfigList{} + err := dr.List(ctx, npcl) + if err != nil { + logger.Error(err, "failed to list sriovNetworkPoolConfig") + return nil, nil, err + } + + selectedNpcl := []*sriovnetworkv1.SriovNetworkPoolConfig{} + nodesInPools := map[string]interface{}{} + + for _, npc := range npcl.Items { + // we skip hw offload objects + if npc.Spec.OvsHardwareOffloadConfig.Name != "" { + continue + } + + if npc.Spec.NodeSelector == nil { + npc.Spec.NodeSelector = &metav1.LabelSelector{} + } + + selector, err := metav1.LabelSelectorAsSelector(npc.Spec.NodeSelector) + if err != nil { + logger.Error(err, "failed to create label selector from nodeSelector", "nodeSelector", npc.Spec.NodeSelector) + return nil, nil, err + } + + if selector.Matches(labels.Set(node.Labels)) { + selectedNpcl = append(selectedNpcl, npc.DeepCopy()) + } + + nodeList := &corev1.NodeList{} + err = dr.List(ctx, nodeList, &client.ListOptions{LabelSelector: selector}) + if err != nil { + logger.Error(err, "failed to list all the nodes matching the pool with label selector from nodeSelector", + "machineConfigPoolName", npc, + "nodeSelector", npc.Spec.NodeSelector) + return nil, nil, err + } + + for _, nodeName := range nodeList.Items { + nodesInPools[nodeName.Name] = nil + } + } + + if len(selectedNpcl) > 1 { + // don't allow the node to be part of multiple pools + err = fmt.Errorf("node is part of more then one pool") + logger.Error(err, "multiple pools founded for a specific node", "numberOfPools", len(selectedNpcl), "pools", selectedNpcl) + return nil, nil, err + } else if len(selectedNpcl) == 1 { + // found one pool for our node + logger.V(2).Info("found sriovNetworkPool", "pool", *selectedNpcl[0]) + selector, err := metav1.LabelSelectorAsSelector(selectedNpcl[0].Spec.NodeSelector) + if err != nil { + logger.Error(err, "failed to create label selector from nodeSelector", "nodeSelector", selectedNpcl[0].Spec.NodeSelector) + return nil, nil, err + } + + // list all the nodes that are also part of this pool and return them + nodeList := &corev1.NodeList{} + err = dr.List(ctx, nodeList, &client.ListOptions{LabelSelector: selector}) + if err != nil { + logger.Error(err, "failed to list nodes using with label selector", "labelSelector", selector) + return nil, nil, err + } + + return selectedNpcl[0], nodeList.Items, nil + } else { + // in this case we get all the nodes and remove the ones that already part of any pool + logger.V(1).Info("node doesn't belong to any pool, using default drain configuration with MaxUnavailable of one", "pool", *defaultPoolConfig) + nodeList := &corev1.NodeList{} + err = dr.List(ctx, nodeList) + if err != nil { + logger.Error(err, "failed to list all the nodes") + return nil, nil, err + } + + defaultNodeLists := []corev1.Node{} + for _, nodeObj := range nodeList.Items { + if _, exist := nodesInPools[nodeObj.Name]; !exist { + defaultNodeLists = append(defaultNodeLists, nodeObj) + } + } + return defaultPoolConfig, defaultNodeLists, nil + } +} diff --git a/controllers/drain_controller_test.go b/controllers/drain_controller_test.go index de3fe0884..921ab5c1f 100644 --- a/controllers/drain_controller_test.go +++ b/controllers/drain_controller_test.go @@ -7,7 +7,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/golang/mock/gomock" + "go.uber.org/mock/gomock" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -17,7 +17,7 @@ import ( "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" - mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + mcfgv1 "github.com/openshift/api/machineconfiguration/v1" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" @@ -90,8 +90,9 @@ var _ = Describe("Drain Controller", Ordered, func() { }) BeforeEach(func() { - Expect(k8sClient.DeleteAllOf(context.Background(), &corev1.Node{})).ToNot(HaveOccurred()) - Expect(k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovNetworkNodeState{}, client.InNamespace(vars.Namespace))).ToNot(HaveOccurred()) + Expect(k8sClient.DeleteAllOf(context.Background(), &corev1.Node{}, &client.DeleteAllOfOptions{DeleteOptions: client.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)}})).ToNot(HaveOccurred()) + Expect(k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovNetworkNodeState{}, client.InNamespace(vars.Namespace), &client.DeleteAllOfOptions{DeleteOptions: client.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)}})).ToNot(HaveOccurred()) + Expect(k8sClient.DeleteAllOf(context.Background(), &corev1.Pod{}, client.InNamespace(testNamespace), &client.DeleteAllOfOptions{DeleteOptions: client.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)}})).ToNot(HaveOccurred()) poolConfig := &sriovnetworkv1.SriovNetworkPoolConfig{} poolConfig.SetNamespace(testNamespace) @@ -113,7 +114,7 @@ var _ = Describe("Drain Controller", Ordered, func() { Context("when there is only one node", func() { - It("should drain", func(ctx context.Context) { + It("should drain single node on drain require", func(ctx context.Context) { node, nodeState := createNode(ctx, "node1") simulateDaemonSetAnnotation(node, constants.DrainRequired) @@ -126,6 +127,33 @@ var _ = Describe("Drain Controller", Ordered, func() { expectNodeStateAnnotation(nodeState, constants.DrainIdle) expectNodeIsSchedulable(node) }) + + It("should not drain on reboot for single node", func(ctx context.Context) { + node, nodeState := createNode(ctx, "node1") + + simulateDaemonSetAnnotation(node, constants.RebootRequired) + + expectNodeStateAnnotation(nodeState, constants.DrainComplete) + expectNodeIsSchedulable(node) + + simulateDaemonSetAnnotation(node, constants.DrainIdle) + expectNodeStateAnnotation(nodeState, constants.DrainIdle) + expectNodeIsSchedulable(node) + }) + + It("should drain on reboot for multiple node", func(ctx context.Context) { + node, nodeState := createNode(ctx, "node1") + createNode(ctx, "node2") + + simulateDaemonSetAnnotation(node, constants.RebootRequired) + + expectNodeStateAnnotation(nodeState, constants.DrainComplete) + expectNodeIsNotSchedulable(node) + + simulateDaemonSetAnnotation(node, constants.DrainIdle) + expectNodeStateAnnotation(nodeState, constants.DrainIdle) + expectNodeIsSchedulable(node) + }) }) Context("when there are multiple nodes", func() { @@ -428,7 +456,7 @@ func createNodeWithLabel(ctx context.Context, nodeName string, label string) (*c ObjectMeta: metav1.ObjectMeta{ Name: nodeName, Namespace: vars.Namespace, - Labels: map[string]string{ + Annotations: map[string]string{ constants.NodeStateDrainAnnotationCurrent: constants.DrainIdle, }, }, diff --git a/controllers/generic_network_controller.go b/controllers/generic_network_controller.go index e6b84d3aa..e31254769 100644 --- a/controllers/generic_network_controller.go +++ b/controllers/generic_network_controller.go @@ -18,10 +18,12 @@ package controllers import ( "context" - "reflect" + "fmt" + "strings" netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" uns "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -30,6 +32,7 @@ import ( "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" @@ -40,7 +43,7 @@ import ( "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) -type networkCRInstance interface { +type NetworkCRInstance interface { client.Object // renders NetAttDef from the network instance RenderNetAttDef() (*uns.Unstructured, error) @@ -53,7 +56,7 @@ type networkController interface { reconcile.Reconciler // GetObject should return CR type which implements networkCRInstance // interface - GetObject() networkCRInstance + GetObject() NetworkCRInstance // should return CR list type GetObjectList() client.ObjectList // should return name of the controller @@ -72,7 +75,6 @@ type genericNetworkReconciler struct { } func (r *genericNetworkReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - req.Namespace = vars.Namespace reqLogger := log.FromContext(ctx).WithValues(r.controller.Name(), req.NamespacedName) reqLogger.Info("Reconciling " + r.controller.Name()) @@ -91,37 +93,37 @@ func (r *genericNetworkReconciler) Reconcile(ctx context.Context, req ctrl.Reque // Error reading the object - requeue the request. return reconcile.Result{}, err } - instanceFinalizers := instance.GetFinalizers() + + if instance == nil { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. + // Return and don't requeue + return reconcile.Result{}, nil + } + + if instance.NetworkNamespace() != "" && instance.GetNamespace() != vars.Namespace { + reqLogger.Error( + fmt.Errorf("bad value for NetworkNamespace"), + ".spec.networkNamespace can't be specified if the resource belongs to a namespace other than the operator's", + "operatorNamespace", vars.Namespace, + ".metadata.namespace", instance.GetNamespace(), + ".spec.networkNamespace", instance.NetworkNamespace(), + ) + return reconcile.Result{}, nil + } + // examine DeletionTimestamp to determine if object is under deletion if instance.GetDeletionTimestamp().IsZero() { // The object is not being deleted, so if it does not have our finalizer, // then lets add the finalizer and update the object. This is equivalent // registering our finalizer. - if !sriovnetworkv1.StringInArray(sriovnetworkv1.NETATTDEFFINALIZERNAME, instanceFinalizers) { - instance.SetFinalizers(append(instanceFinalizers, sriovnetworkv1.NETATTDEFFINALIZERNAME)) - if err := r.Update(ctx, instance); err != nil { - return reconcile.Result{}, err - } + err = r.updateFinalizers(ctx, instance) + if err != nil { + return reconcile.Result{}, err } } else { // The object is being deleted - if sriovnetworkv1.StringInArray(sriovnetworkv1.NETATTDEFFINALIZERNAME, instanceFinalizers) { - // our finalizer is present, so lets handle any external dependency - reqLogger.Info("delete NetworkAttachmentDefinition CR", "Namespace", instance.NetworkNamespace(), "Name", instance.GetName()) - if err := r.deleteNetAttDef(ctx, instance); err != nil { - // if fail to delete the external dependency here, return with error - // so that it can be retried - return reconcile.Result{}, err - } - // remove our finalizer from the list and update it. - newFinalizers, found := sriovnetworkv1.RemoveString(sriovnetworkv1.NETATTDEFFINALIZERNAME, instanceFinalizers) - if found { - instance.SetFinalizers(newFinalizers) - if err := r.Update(ctx, instance); err != nil { - return reconcile.Result{}, err - } - } - } + err = r.cleanResourcesAndFinalizers(ctx, instance) return reconcile.Result{}, err } raw, err := instance.RenderNetAttDef() @@ -151,6 +153,14 @@ func (r *genericNetworkReconciler) Reconcile(ctx context.Context, req ctrl.Reque return reconcile.Result{}, err } } + + if instance.GetNamespace() == netAttDef.Namespace { + // If the NetAttachDef is in the same namespace of the resource, then we can leverage the OwnerReference field for garbage collector + if err := controllerutil.SetOwnerReference(instance, netAttDef, r.Scheme); err != nil { + return reconcile.Result{}, err + } + } + // Check if this NetworkAttachmentDefinition already exists found := &netattdefv1.NetworkAttachmentDefinition{} err = r.Get(ctx, types.NamespacedName{Name: netAttDef.Name, Namespace: netAttDef.Namespace}, found) @@ -180,9 +190,10 @@ func (r *genericNetworkReconciler) Reconcile(ctx context.Context, req ctrl.Reque } } else { reqLogger.Info("NetworkAttachmentDefinition CR already exist") - if !reflect.DeepEqual(found.Spec, netAttDef.Spec) || !reflect.DeepEqual(found.GetAnnotations(), netAttDef.GetAnnotations()) { + if !equality.Semantic.DeepEqual(found.Spec, netAttDef.Spec) || !equality.Semantic.DeepEqual(found.GetAnnotations(), netAttDef.GetAnnotations()) { reqLogger.Info("Update NetworkAttachmentDefinition CR", "Namespace", netAttDef.Namespace, "Name", netAttDef.Name) netAttDef.SetResourceVersion(found.GetResourceVersion()) + err = r.Update(ctx, netAttDef) if err != nil { reqLogger.Error(err, "Couldn't update NetworkAttachmentDefinition CR", "Namespace", netAttDef.Namespace, "Name", netAttDef.Name) @@ -202,12 +213,38 @@ func (r *genericNetworkReconciler) SetupWithManager(mgr ctrl.Manager) error { } return ctrl.NewControllerManagedBy(mgr). For(r.controller.GetObject()). - Watches(&netattdefv1.NetworkAttachmentDefinition{}, &handler.EnqueueRequestForObject{}). + Watches(&netattdefv1.NetworkAttachmentDefinition{}, handler.EnqueueRequestsFromMapFunc(r.handleNetAttDef)). Watches(&corev1.Namespace{}, &namespaceHandler). Complete(r.controller) } -func (r *genericNetworkReconciler) namespaceHandlerCreate(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { +func (r *genericNetworkReconciler) handleNetAttDef(ctx context.Context, obj client.Object) []reconcile.Request { + ret := []reconcile.Request{} + instance := r.controller.GetObject() + nadNamespacedName := types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()} + + err := r.Get(ctx, nadNamespacedName, instance) + if err == nil { + // Found a NetworkObject in the same namespace as the NetworkAttachmentDefinition, reconcile it + ret = append(ret, reconcile.Request{NamespacedName: nadNamespacedName}) + } else if !errors.IsNotFound(err) { + log.Log.WithName(r.controller.Name()+" handleNetAttDef").Error(err, "can't get object", "object", nadNamespacedName) + } + + // Not found, try to find the NetworkObject in the operator's namespace + operatorNamespacedName := types.NamespacedName{Namespace: vars.Namespace, Name: obj.GetName()} + err = r.Get(ctx, operatorNamespacedName, instance) + if err == nil { + // Found a NetworkObject in the operator's namespace, reconcile it + ret = append(ret, reconcile.Request{NamespacedName: operatorNamespacedName}) + } else if !errors.IsNotFound(err) { + log.Log.WithName(r.controller.Name()+" handleNetAttDef").Error(err, "can't get object", "object", operatorNamespacedName) + } + + return ret +} + +func (r *genericNetworkReconciler) namespaceHandlerCreate(ctx context.Context, e event.TypedCreateEvent[client.Object], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { networkList := r.controller.GetObjectList() err := r.List(ctx, networkList, @@ -227,7 +264,7 @@ func (r *genericNetworkReconciler) namespaceHandlerCreate(ctx context.Context, e unsList.SetUnstructuredContent(unsContent) _ = unsList.EachListItem(func(o runtime.Object) error { unsObj := o.(*uns.Unstructured) - q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + w.Add(reconcile.Request{NamespacedName: types.NamespacedName{ Namespace: unsObj.GetNamespace(), Name: unsObj.GetName(), }}) @@ -236,19 +273,84 @@ func (r *genericNetworkReconciler) namespaceHandlerCreate(ctx context.Context, e } // deleteNetAttDef deletes the generated net-att-def CR -func (r *genericNetworkReconciler) deleteNetAttDef(ctx context.Context, cr networkCRInstance) error { +func (r *genericNetworkReconciler) deleteNetAttDef(ctx context.Context, cr NetworkCRInstance) error { // Fetch the NetworkAttachmentDefinition instance namespace := cr.NetworkNamespace() if namespace == "" { namespace = cr.GetNamespace() } - instance := &netattdefv1.NetworkAttachmentDefinition{ObjectMeta: metav1.ObjectMeta{Name: cr.GetName(), Namespace: namespace}} - err := r.Delete(ctx, instance) + + // First get the NetworkAttachmentDefinition to check if GUIDSavedInUFM is True + instance := &netattdefv1.NetworkAttachmentDefinition{} + err := r.Get(ctx, types.NamespacedName{Name: cr.GetName(), Namespace: namespace}, instance) if err != nil { if errors.IsNotFound(err) { return nil } return err } + + // Check the GUIDSavedInUFM in the spec's config JSON + // The CNI config is a JSON string in the spec + if instance.Spec.Config != "" { + // Check if the config contains "GUIDSavedInUFM": "true" + if strings.Contains(instance.Spec.Config, `"GUIDSavedInUFM": "true"`) { + // Skip deletion because GUIDSavedInUFM is True + logger := log.Log.WithName("deleteNetAttDef") + logger.Info("Skipping NetworkAttachmentDefinition deletion because GUIDSavedInUFM is true in spec", + "Namespace", namespace, "Name", cr.GetName()) + return nil + } + } + + // Proceed with deletion since GUIDSavedInUFM is not True + err = r.Delete(ctx, instance) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + return nil +} + +func (r *genericNetworkReconciler) updateFinalizers(ctx context.Context, instance NetworkCRInstance) error { + if instance.GetNamespace() != vars.Namespace { + // If the resource is in a namespace different than the operator one, then the NetworkAttachmentDefinition will + // be created in the same namespace and its deletion can be handled by OwnerReferences. There is no need for finalizers + return nil + } + + instanceFinalizers := instance.GetFinalizers() + if !sriovnetworkv1.StringInArray(sriovnetworkv1.NETATTDEFFINALIZERNAME, instanceFinalizers) { + instance.SetFinalizers(append(instanceFinalizers, sriovnetworkv1.NETATTDEFFINALIZERNAME)) + if err := r.Update(ctx, instance); err != nil { + return err + } + } + + return nil +} + +func (r *genericNetworkReconciler) cleanResourcesAndFinalizers(ctx context.Context, instance NetworkCRInstance) error { + instanceFinalizers := instance.GetFinalizers() + + if sriovnetworkv1.StringInArray(sriovnetworkv1.NETATTDEFFINALIZERNAME, instanceFinalizers) { + // our finalizer is present, so lets handle any external dependency + log.FromContext(ctx).Info("delete NetworkAttachmentDefinition CR", "Namespace", instance.NetworkNamespace(), "Name", instance.GetName()) + if err := r.deleteNetAttDef(ctx, instance); err != nil { + // if fail to delete the external dependency here, return with error + // so that it can be retried + return err + } + // remove our finalizer from the list and update it. + newFinalizers, found := sriovnetworkv1.RemoveString(sriovnetworkv1.NETATTDEFFINALIZERNAME, instanceFinalizers) + if found { + instance.SetFinalizers(newFinalizers) + if err := r.Update(ctx, instance); err != nil { + return err + } + } + } return nil } diff --git a/controllers/helper.go b/controllers/helper.go index 9ff735473..fec7f3d9d 100644 --- a/controllers/helper.go +++ b/controllers/helper.go @@ -22,7 +22,6 @@ import ( "encoding/json" "fmt" "os" - "sort" "strings" errs "github.com/pkg/errors" @@ -30,9 +29,12 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" uns "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" kscheme "k8s.io/client-go/kubernetes/scheme" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -47,10 +49,17 @@ import ( "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) -var webhooks = map[string](string){ - constants.InjectorWebHookName: constants.InjectorWebHookPath, - constants.OperatorWebHookName: constants.OperatorWebHookPath, -} +var ( + webhooks = map[string]string{ + constants.InjectorWebHookName: constants.InjectorWebHookPath, + constants.OperatorWebHookName: constants.OperatorWebHookPath, + } + oneNode = intstr.FromInt32(1) + defaultPoolConfig = &sriovnetworkv1.SriovNetworkPoolConfig{Spec: sriovnetworkv1.SriovNetworkPoolConfigSpec{ + MaxUnavailable: &oneNode, + NodeSelector: &metav1.LabelSelector{}, + RdmaMode: ""}} +) const ( clusterRoleResourceName = "ClusterRole" @@ -91,11 +100,7 @@ func (DrainAnnotationPredicate) Update(e event.UpdateEvent) bool { return true } - if oldAnno != newAnno { - return true - } - - return false + return oldAnno != newAnno } type DrainStateAnnotationPredicate struct { @@ -103,14 +108,7 @@ type DrainStateAnnotationPredicate struct { } func (DrainStateAnnotationPredicate) Create(e event.CreateEvent) bool { - if e.Object == nil { - return false - } - - if _, hasAnno := e.Object.GetLabels()[constants.NodeStateDrainAnnotationCurrent]; hasAnno { - return true - } - return false + return e.Object != nil } func (DrainStateAnnotationPredicate) Update(e event.UpdateEvent) bool { @@ -128,10 +126,6 @@ func (DrainStateAnnotationPredicate) Update(e event.UpdateEvent) bool { return true } - if oldAnno != newAnno { - return true - } - return oldAnno != newAnno } @@ -152,29 +146,33 @@ func formatJSON(str string) (string, error) { return prettyJSON.String(), nil } +// GetDefaultNodeSelector return a nodeSelector with worker and linux os func GetDefaultNodeSelector() map[string]string { - return map[string]string{"node-role.kubernetes.io/worker": "", - "kubernetes.io/os": "linux"} + return map[string]string{ + "node-role.kubernetes.io/worker": "", + "kubernetes.io/os": "linux", + } } -// hasNoValidPolicy returns true if no SriovNetworkNodePolicy -// or only the (deprecated) "default" policy is present -func hasNoValidPolicy(pl []sriovnetworkv1.SriovNetworkNodePolicy) bool { - switch len(pl) { - case 0: - return true - case 1: - return pl[0].Name == constants.DefaultPolicyName - default: - return false +// GetDefaultNodeSelectorForDevicePlugin return a nodeSelector with worker linux os +// and the enabled sriov device plugin +func GetNodeSelectorForDevicePlugin(dc *sriovnetworkv1.SriovOperatorConfig) map[string]string { + if len(dc.Spec.ConfigDaemonNodeSelector) == 0 { + return map[string]string{ + "kubernetes.io/os": "linux", + constants.SriovDevicePluginLabel: constants.SriovDevicePluginLabelEnabled, + } } + + tmp := dc.Spec.DeepCopy() + tmp.ConfigDaemonNodeSelector[constants.SriovDevicePluginLabel] = constants.SriovDevicePluginLabelEnabled + return tmp.ConfigDaemonNodeSelector } func syncPluginDaemonObjs(ctx context.Context, client k8sclient.Client, scheme *runtime.Scheme, - dc *sriovnetworkv1.SriovOperatorConfig, - pl *sriovnetworkv1.SriovNetworkNodePolicyList) error { + dc *sriovnetworkv1.SriovOperatorConfig) error { logger := log.Log.WithName("syncPluginDaemonObjs") logger.V(1).Info("Start to sync sriov daemons objects") @@ -185,7 +183,7 @@ func syncPluginDaemonObjs(ctx context.Context, data.Data["ReleaseVersion"] = os.Getenv("RELEASEVERSION") data.Data["ResourcePrefix"] = vars.ResourcePrefix data.Data["ImagePullSecrets"] = GetImagePullSecrets() - data.Data["NodeSelectorField"] = GetDefaultNodeSelector() + data.Data["NodeSelectorField"] = GetNodeSelectorForDevicePlugin(dc) data.Data["UseCDI"] = dc.Spec.UseCDI objs, err := renderDsForCR(constants.PluginPath, &data) if err != nil { @@ -193,34 +191,9 @@ func syncPluginDaemonObjs(ctx context.Context, return err } - if hasNoValidPolicy(pl.Items) { - for _, obj := range objs { - err := deleteK8sResource(ctx, client, obj) - if err != nil { - return err - } - } - return nil - } - // Sync DaemonSets for _, obj := range objs { - if obj.GetKind() == constants.DaemonSet && len(dc.Spec.ConfigDaemonNodeSelector) > 0 { - scheme := kscheme.Scheme - ds := &appsv1.DaemonSet{} - err = scheme.Convert(obj, ds, nil) - if err != nil { - logger.Error(err, "Fail to convert to DaemonSet") - return err - } - ds.Spec.Template.Spec.NodeSelector = dc.Spec.ConfigDaemonNodeSelector - err = scheme.Convert(ds, obj, nil) - if err != nil { - logger.Error(err, "Fail to convert to Unstructured") - return err - } - } - err = syncDsObject(ctx, client, scheme, dc, pl, obj) + err = syncDsObject(ctx, client, scheme, dc, obj) if err != nil { logger.Error(err, "Couldn't sync SR-IoV daemons objects") return err @@ -230,14 +203,7 @@ func syncPluginDaemonObjs(ctx context.Context, return nil } -func deleteK8sResource(ctx context.Context, client k8sclient.Client, in *uns.Unstructured) error { - if err := apply.DeleteObject(ctx, client, in); err != nil { - return fmt.Errorf("failed to delete object %v with err: %v", in, err) - } - return nil -} - -func syncDsObject(ctx context.Context, client k8sclient.Client, scheme *runtime.Scheme, dc *sriovnetworkv1.SriovOperatorConfig, pl *sriovnetworkv1.SriovNetworkNodePolicyList, obj *uns.Unstructured) error { +func syncDsObject(ctx context.Context, client k8sclient.Client, scheme *runtime.Scheme, dc *sriovnetworkv1.SriovOperatorConfig, obj *uns.Unstructured) error { logger := log.Log.WithName("syncDsObject") kind := obj.GetKind() logger.V(1).Info("Start to sync Objects", "Kind", kind) @@ -257,7 +223,7 @@ func syncDsObject(ctx context.Context, client k8sclient.Client, scheme *runtime. logger.Error(err, "Fail to convert to DaemonSet") return err } - err = syncDaemonSet(ctx, client, scheme, dc, pl, ds) + err = syncDaemonSet(ctx, client, scheme, dc, ds) if err != nil { logger.Error(err, "Fail to sync DaemonSet", "Namespace", ds.Namespace, "Name", ds.Name) return err @@ -266,54 +232,6 @@ func syncDsObject(ctx context.Context, client k8sclient.Client, scheme *runtime. return nil } -func setDsNodeAffinity(pl *sriovnetworkv1.SriovNetworkNodePolicyList, ds *appsv1.DaemonSet) error { - terms := nodeSelectorTermsForPolicyList(pl.Items) - if len(terms) > 0 { - ds.Spec.Template.Spec.Affinity = &corev1.Affinity{ - NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: terms, - }, - }, - } - } - return nil -} - -func nodeSelectorTermsForPolicyList(policies []sriovnetworkv1.SriovNetworkNodePolicy) []corev1.NodeSelectorTerm { - terms := []corev1.NodeSelectorTerm{} - for _, p := range policies { - // Note(adrianc): default policy is deprecated and ignored. - if p.Name == constants.DefaultPolicyName { - continue - } - - if len(p.Spec.NodeSelector) == 0 { - continue - } - expressions := []corev1.NodeSelectorRequirement{} - for k, v := range p.Spec.NodeSelector { - exp := corev1.NodeSelectorRequirement{ - Operator: corev1.NodeSelectorOpIn, - Key: k, - Values: []string{v}, - } - expressions = append(expressions, exp) - } - // sorting is needed to keep the daemon spec stable. - // the items are popped in a random order from the map - sort.Slice(expressions, func(i, j int) bool { - return expressions[i].Key < expressions[j].Key - }) - nodeSelector := corev1.NodeSelectorTerm{ - MatchExpressions: expressions, - } - terms = append(terms, nodeSelector) - } - - return terms -} - // renderDsForCR returns a busybox pod with the same name/namespace as the cr func renderDsForCR(path string, data *render.RenderData) ([]*uns.Unstructured, error) { logger := log.Log.WithName("renderDsForCR") @@ -326,16 +244,11 @@ func renderDsForCR(path string, data *render.RenderData) ([]*uns.Unstructured, e return objs, nil } -func syncDaemonSet(ctx context.Context, client k8sclient.Client, scheme *runtime.Scheme, dc *sriovnetworkv1.SriovOperatorConfig, pl *sriovnetworkv1.SriovNetworkNodePolicyList, in *appsv1.DaemonSet) error { +func syncDaemonSet(ctx context.Context, client k8sclient.Client, scheme *runtime.Scheme, dc *sriovnetworkv1.SriovOperatorConfig, in *appsv1.DaemonSet) error { logger := log.Log.WithName("syncDaemonSet") logger.V(1).Info("Start to sync DaemonSet", "Namespace", in.Namespace, "Name", in.Name) var err error - if pl != nil { - if err = setDsNodeAffinity(pl, in); err != nil { - return err - } - } if err = controllerutil.SetControllerReference(dc, in, scheme); err != nil { return err } @@ -397,3 +310,94 @@ func updateDaemonsetNodeSelector(obj *uns.Unstructured, nodeSelector map[string] } return nil } + +func findNodePoolConfig(ctx context.Context, node *corev1.Node, c k8sclient.Client) (*sriovnetworkv1.SriovNetworkPoolConfig, []corev1.Node, error) { + logger := log.FromContext(ctx) + logger.Info("FindNodePoolConfig():") + // get all the sriov network pool configs + npcl := &sriovnetworkv1.SriovNetworkPoolConfigList{} + err := c.List(ctx, npcl) + if err != nil { + logger.Error(err, "failed to list sriovNetworkPoolConfig") + return nil, nil, err + } + + selectedNpcl := []*sriovnetworkv1.SriovNetworkPoolConfig{} + nodesInPools := map[string]interface{}{} + + for _, npc := range npcl.Items { + // we skip hw offload objects + if npc.Spec.OvsHardwareOffloadConfig.Name != "" { + continue + } + + if npc.Spec.NodeSelector == nil { + npc.Spec.NodeSelector = &metav1.LabelSelector{} + } + + selector, err := metav1.LabelSelectorAsSelector(npc.Spec.NodeSelector) + if err != nil { + logger.Error(err, "failed to create label selector from nodeSelector", "nodeSelector", npc.Spec.NodeSelector) + return nil, nil, err + } + + if selector.Matches(labels.Set(node.Labels)) { + selectedNpcl = append(selectedNpcl, npc.DeepCopy()) + } + + nodeList := &corev1.NodeList{} + err = c.List(ctx, nodeList, &k8sclient.ListOptions{LabelSelector: selector}) + if err != nil { + logger.Error(err, "failed to list all the nodes matching the pool with label selector from nodeSelector", + "machineConfigPoolName", npc, + "nodeSelector", npc.Spec.NodeSelector) + return nil, nil, err + } + + for _, nodeName := range nodeList.Items { + nodesInPools[nodeName.Name] = nil + } + } + + if len(selectedNpcl) > 1 { + // don't allow the node to be part of multiple pools + err = fmt.Errorf("node is part of more then one pool") + logger.Error(err, "multiple pools founded for a specific node", "numberOfPools", len(selectedNpcl), "pools", selectedNpcl) + return nil, nil, err + } else if len(selectedNpcl) == 1 { + // found one pool for our node + logger.V(2).Info("found sriovNetworkPool", "pool", *selectedNpcl[0]) + selector, err := metav1.LabelSelectorAsSelector(selectedNpcl[0].Spec.NodeSelector) + if err != nil { + logger.Error(err, "failed to create label selector from nodeSelector", "nodeSelector", selectedNpcl[0].Spec.NodeSelector) + return nil, nil, err + } + + // list all the nodes that are also part of this pool and return them + nodeList := &corev1.NodeList{} + err = c.List(ctx, nodeList, &k8sclient.ListOptions{LabelSelector: selector}) + if err != nil { + logger.Error(err, "failed to list nodes using with label selector", "labelSelector", selector) + return nil, nil, err + } + + return selectedNpcl[0], nodeList.Items, nil + } else { + // in this case we get all the nodes and remove the ones that already part of any pool + logger.V(1).Info("node doesn't belong to any pool, using default drain configuration with MaxUnavailable of one", "pool", *defaultPoolConfig) + nodeList := &corev1.NodeList{} + err = c.List(ctx, nodeList) + if err != nil { + logger.Error(err, "failed to list all the nodes") + return nil, nil, err + } + + defaultNodeLists := []corev1.Node{} + for _, nodeObj := range nodeList.Items { + if _, exist := nodesInPools[nodeObj.Name]; !exist { + defaultNodeLists = append(defaultNodeLists, nodeObj) + } + } + return defaultPoolConfig, defaultNodeLists, nil + } +} diff --git a/controllers/helper_test.go b/controllers/helper_test.go deleted file mode 100644 index d998cf0da..000000000 --- a/controllers/helper_test.go +++ /dev/null @@ -1,330 +0,0 @@ -/* - - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "context" - "sync" - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/google/go-cmp/cmp" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - controllerruntime "sigs.k8s.io/controller-runtime" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" -) - -func TestNodeSelectorMerge(t *testing.T) { - table := []struct { - tname string - policies []sriovnetworkv1.SriovNetworkNodePolicy - expected []corev1.NodeSelectorTerm - }{ - { - tname: "testoneselector", - policies: []sriovnetworkv1.SriovNetworkNodePolicy{ - { - Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{ - "foo": "bar", - }, - }, - }, - { - Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{ - "bb": "cc", - }, - }, - }, - }, - expected: []corev1.NodeSelectorTerm{ - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Operator: corev1.NodeSelectorOpIn, - Key: "foo", - Values: []string{"bar"}, - }, - }, - }, - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Operator: corev1.NodeSelectorOpIn, - Key: "bb", - Values: []string{"cc"}, - }, - }, - }, - }, - }, - { - tname: "testtwoselectors", - policies: []sriovnetworkv1.SriovNetworkNodePolicy{ - { - Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{ - "foo": "bar", - "foo1": "bar1", - }, - }, - }, - { - Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{ - "bb": "cc", - "bb1": "cc1", - "bb2": "cc2", - }, - }, - }, - }, - expected: []corev1.NodeSelectorTerm{ - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Operator: corev1.NodeSelectorOpIn, - Key: "foo", - Values: []string{"bar"}, - }, - { - Operator: corev1.NodeSelectorOpIn, - Key: "foo1", - Values: []string{"bar1"}, - }, - }, - }, - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Operator: corev1.NodeSelectorOpIn, - Key: "bb", - Values: []string{"cc"}, - }, - { - Operator: corev1.NodeSelectorOpIn, - Key: "bb1", - Values: []string{"cc1"}, - }, - { - Operator: corev1.NodeSelectorOpIn, - Key: "bb2", - Values: []string{"cc2"}, - }, - }, - }, - }, - }, - { - tname: "testemptyselector", - policies: []sriovnetworkv1.SriovNetworkNodePolicy{ - { - Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{}, - }, - }, - }, - expected: []corev1.NodeSelectorTerm{}, - }, - } - - for _, tc := range table { - t.Run(tc.tname, func(t *testing.T) { - selectors := nodeSelectorTermsForPolicyList(tc.policies) - if !cmp.Equal(selectors, tc.expected) { - t.Error(tc.tname, "Selectors not as expected", cmp.Diff(selectors, tc.expected)) - } - }) - } -} - -var _ = Describe("Helper Validation", Ordered, func() { - - var cancel context.CancelFunc - var ctx context.Context - var dc *sriovnetworkv1.SriovOperatorConfig - var in *appsv1.DaemonSet - - BeforeAll(func() { - By("Setup controller manager") - k8sManager, err := setupK8sManagerForTest() - Expect(err).ToNot(HaveOccurred()) - - ctx, cancel = context.WithCancel(context.Background()) - - wg := sync.WaitGroup{} - wg.Add(1) - go func() { - defer wg.Done() - defer GinkgoRecover() - By("Start controller manager") - err := k8sManager.Start(ctx) - Expect(err).ToNot(HaveOccurred()) - }() - - DeferCleanup(func() { - By("Shutdown controller manager") - cancel() - wg.Wait() - }) - }) - - BeforeEach(func() { - dc = &sriovnetworkv1.SriovOperatorConfig{ - ObjectMeta: controllerruntime.ObjectMeta{ - Name: "default", - Namespace: vars.Namespace, - UID: "12312312"}} - in = &appsv1.DaemonSet{ - ObjectMeta: controllerruntime.ObjectMeta{ - Name: "sriov-device-plugin", - Namespace: vars.Namespace}, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": "sriov-device-plugin"}}, - Template: corev1.PodTemplateSpec{ - ObjectMeta: controllerruntime.ObjectMeta{ - Labels: map[string]string{"app": "sriov-device-plugin"}}, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Image: "test:latest", - Name: "test", - }, - }, - }, - }}} - - err := k8sClient.Delete(ctx, in) - if err != nil { - Expect(errors.IsNotFound(err)).To(BeTrue()) - } - }) - - Context("syncDaemonSet", func() { - It("should create a new daemon", func() { - pl := &sriovnetworkv1.SriovNetworkNodePolicyList{Items: []sriovnetworkv1.SriovNetworkNodePolicy{ - {ObjectMeta: controllerruntime.ObjectMeta{Name: "test", Namespace: vars.Namespace}}, - }} - err := syncDaemonSet(ctx, k8sClient, vars.Scheme, dc, pl, in) - Expect(err).ToNot(HaveOccurred()) - Expect(in.Spec.Template.Spec.Affinity).To(BeNil()) - }) - It("should update affinity", func() { - pl := &sriovnetworkv1.SriovNetworkNodePolicyList{Items: []sriovnetworkv1.SriovNetworkNodePolicy{ - { - ObjectMeta: controllerruntime.ObjectMeta{ - Name: "test", - Namespace: vars.Namespace, - }, - Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{"test": "test"}, - }, - }, - }} - - err := k8sClient.Create(ctx, in) - Expect(err).ToNot(HaveOccurred()) - - err = syncDaemonSet(ctx, k8sClient, vars.Scheme, dc, pl, in) - Expect(err).ToNot(HaveOccurred()) - Expect(in.Spec.Template.Spec.Affinity).ToNot(BeNil()) - Expect(in.Spec.Template.Spec.Affinity.NodeAffinity).ToNot(BeNil()) - Expect(in.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution).ToNot(BeNil()) - Expect(len(in.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms)).To(Equal(1)) - }) - It("should update affinity with multiple", func() { - pl := &sriovnetworkv1.SriovNetworkNodePolicyList{Items: []sriovnetworkv1.SriovNetworkNodePolicy{ - { - ObjectMeta: controllerruntime.ObjectMeta{ - Name: "test", - Namespace: vars.Namespace, - }, - Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{"test": "test"}, - }, - }, - { - ObjectMeta: controllerruntime.ObjectMeta{ - Name: "test1", - Namespace: vars.Namespace, - }, - Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{"test1": "test"}, - }, - }, - }} - - err := k8sClient.Create(ctx, in) - Expect(err).ToNot(HaveOccurred()) - - err = syncDaemonSet(ctx, k8sClient, vars.Scheme, dc, pl, in) - Expect(err).ToNot(HaveOccurred()) - Expect(in.Spec.Template.Spec.Affinity).ToNot(BeNil()) - Expect(in.Spec.Template.Spec.Affinity.NodeAffinity).ToNot(BeNil()) - Expect(in.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution).ToNot(BeNil()) - Expect(len(in.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms)).To(Equal(2)) - }) - It("should switch affinity", func() { - pl := &sriovnetworkv1.SriovNetworkNodePolicyList{Items: []sriovnetworkv1.SriovNetworkNodePolicy{ - { - ObjectMeta: controllerruntime.ObjectMeta{ - Name: "test1", - Namespace: vars.Namespace, - }, - Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{"test1": "test"}, - }, - }, - }} - - in.Spec.Template.Spec.Affinity = &corev1.Affinity{ - NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{{ - MatchExpressions: []corev1.NodeSelectorRequirement{{ - Operator: corev1.NodeSelectorOpIn, - Key: "test", - Values: []string{"test"}, - }}, - }}, - }, - }, - } - - err := k8sClient.Create(ctx, in) - Expect(err).ToNot(HaveOccurred()) - - err = syncDaemonSet(ctx, k8sClient, vars.Scheme, dc, pl, in) - Expect(err).ToNot(HaveOccurred()) - Expect(in.Spec.Template.Spec.Affinity).ToNot(BeNil()) - Expect(in.Spec.Template.Spec.Affinity.NodeAffinity).ToNot(BeNil()) - Expect(in.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution).ToNot(BeNil()) - Expect(len(in.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms)).To(Equal(1)) - Expect(len(in.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions)).To(Equal(1)) - Expect(in.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Key).To(Equal("test1")) - }) - }) -}) diff --git a/controllers/ovsnetwork_controller.go b/controllers/ovsnetwork_controller.go index 2f2eaee1e..53dbe3fa6 100644 --- a/controllers/ovsnetwork_controller.go +++ b/controllers/ovsnetwork_controller.go @@ -48,7 +48,7 @@ func (r *OVSNetworkReconciler) Name() string { } // return empty instance of the OVSNetwork CR -func (r *OVSNetworkReconciler) GetObject() networkCRInstance { +func (r *OVSNetworkReconciler) GetObject() NetworkCRInstance { return &sriovnetworkv1.OVSNetwork{} } diff --git a/controllers/sriovibnetwork_controller.go b/controllers/sriovibnetwork_controller.go index ee311f32a..6c0c24534 100644 --- a/controllers/sriovibnetwork_controller.go +++ b/controllers/sriovibnetwork_controller.go @@ -48,7 +48,7 @@ func (r *SriovIBNetworkReconciler) Name() string { } // return empty instance of the SriovIBNetwork CR -func (r *SriovIBNetworkReconciler) GetObject() networkCRInstance { +func (r *SriovIBNetworkReconciler) GetObject() NetworkCRInstance { return &sriovnetworkv1.SriovIBNetwork{} } diff --git a/controllers/sriovnetwork_controller.go b/controllers/sriovnetwork_controller.go index 1e3dd044e..1ef6b5cb8 100644 --- a/controllers/sriovnetwork_controller.go +++ b/controllers/sriovnetwork_controller.go @@ -48,7 +48,7 @@ func (r *SriovNetworkReconciler) Name() string { } // return empty instance of the SriovIBNetwork CR -func (r *SriovNetworkReconciler) GetObject() networkCRInstance { +func (r *SriovNetworkReconciler) GetObject() NetworkCRInstance { return &sriovnetworkv1.SriovNetwork{} } diff --git a/controllers/sriovnetwork_controller_test.go b/controllers/sriovnetwork_controller_test.go index f2f7d2a07..df146ccea 100644 --- a/controllers/sriovnetwork_controller_test.go +++ b/controllers/sriovnetwork_controller_test.go @@ -12,8 +12,10 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" dynclient "sigs.k8s.io/controller-runtime/pkg/client" . "github.com/onsi/ginkgo/v2" @@ -342,6 +344,89 @@ var _ = Describe("SriovNetwork Controller", Ordered, func() { MustPassRepeatedly(10). Should(Succeed()) }) + + Context("When the SriovNetwork namespace is not equal to the operator one", func() { + BeforeAll(func() { + nsBlue := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns-blue"}} + Expect(k8sClient.Create(context.Background(), nsBlue)).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + cleanNetworksInNamespace("ns-blue") + }) + + It("should create the NetAttachDefinition in the same namespace", func() { + cr := sriovnetworkv1.SriovNetwork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sriovnet-blue", + Namespace: "ns-blue", + }, + Spec: sriovnetworkv1.SriovNetworkSpec{ + ResourceName: "resource_x", + }, + } + + err := k8sClient.Create(ctx, &cr) + Expect(err).NotTo(HaveOccurred()) + + netAttDef := &netattdefv1.NetworkAttachmentDefinition{} + err = util.WaitForNamespacedObject(netAttDef, k8sClient, "ns-blue", cr.GetName(), util.RetryInterval, util.Timeout) + Expect(err).NotTo(HaveOccurred()) + expectedOwnerReference := metav1.OwnerReference{ + Kind: "SriovNetwork", + APIVersion: sriovnetworkv1.GroupVersion.String(), + UID: cr.UID, + Name: cr.Name, + } + Expect(netAttDef.GetAnnotations()["k8s.v1.cni.cncf.io/resourceName"]).To(Equal("openshift.io/resource_x")) + + Expect(netAttDef.ObjectMeta.OwnerReferences).To(ContainElement(expectedOwnerReference)) + + // Patch the SriovNetwork + original := cr.DeepCopy() + cr.Spec.ResourceName = "resource_y" + err = k8sClient.Patch(ctx, &cr, dynclient.MergeFrom(original)) + Expect(err).NotTo(HaveOccurred()) + + // Check that the OwnerReference persists + netAttDef = &netattdefv1.NetworkAttachmentDefinition{} + + Eventually(func(g Gomega) { + netAttDef = &netattdefv1.NetworkAttachmentDefinition{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: cr.GetName(), Namespace: "ns-blue"}, netAttDef)).To(Succeed()) + g.Expect(netAttDef.GetAnnotations()["k8s.v1.cni.cncf.io/resourceName"]).To(Equal("openshift.io/resource_y")) + g.Expect(netAttDef.ObjectMeta.OwnerReferences).To(ContainElement(expectedOwnerReference)) + }).WithPolling(100 * time.Millisecond).WithTimeout(5 * time.Second).Should(Succeed()) + + // Delete the SriovNetwork + err = k8sClient.Delete(ctx, &cr) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should not create the NetAttachDefinition if the NetworkNamespace field is not empty", func() { + cr := sriovnetworkv1.SriovNetwork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sriovnet-blue", + Namespace: "ns-blue", + }, + Spec: sriovnetworkv1.SriovNetworkSpec{ + NetworkNamespace: "default", + ResourceName: "resource_x", + }, + } + + err := k8sClient.Create(ctx, &cr) + Expect(err).NotTo(HaveOccurred()) + + Consistently(func(g Gomega) { + netAttDef := &netattdefv1.NetworkAttachmentDefinition{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: cr.GetName(), Namespace: "default"}, netAttDef) + g.Expect(err).To(HaveOccurred()) + g.Expect(errors.IsNotFound(err)).To(BeTrue()) + }).WithPolling(100 * time.Millisecond).WithTimeout(1 * time.Second).Should(Succeed()) + }) + + }) }) }) @@ -390,3 +475,24 @@ func generateExpectedNetConfig(cr *sriovnetworkv1.SriovNetwork) string { } return configStr } + +func cleanNetworksInNamespace(namespace string) { + ctx := context.Background() + EventuallyWithOffset(1, func(g Gomega) { + err := k8sClient.DeleteAllOf(ctx, &sriovnetworkv1.SriovNetwork{}, client.InNamespace(namespace)) + g.Expect(err).NotTo(HaveOccurred()) + + k8sClient.DeleteAllOf(ctx, &netattdefv1.NetworkAttachmentDefinition{}, client.InNamespace(namespace)) + g.Expect(err).NotTo(HaveOccurred()) + + sriovNetworks := &sriovnetworkv1.SriovNetworkList{} + err = k8sClient.List(ctx, sriovNetworks, client.InNamespace(namespace)) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(sriovNetworks.Items).To(BeEmpty()) + + netAttachDefs := &netattdefv1.NetworkAttachmentDefinitionList{} + err = k8sClient.List(ctx, netAttachDefs, client.InNamespace(namespace)) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(netAttachDefs.Items).To(BeEmpty()) + }).WithPolling(100 * time.Millisecond).WithTimeout(10 * time.Second).Should(Succeed()) +} diff --git a/controllers/sriovnetworknodepolicy_controller.go b/controllers/sriovnetworknodepolicy_controller.go index be46880b7..96e69c764 100644 --- a/controllers/sriovnetworknodepolicy_controller.go +++ b/controllers/sriovnetworknodepolicy_controller.go @@ -20,8 +20,9 @@ import ( "context" "encoding/json" "fmt" - "reflect" + "os" "sort" + "strconv" "strings" "time" @@ -46,6 +47,7 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/featuregate" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) @@ -85,7 +87,7 @@ func (r *SriovNetworkNodePolicyReconciler) Reconcile(ctx context.Context, req ct if err := r.Get(ctx, types.NamespacedName{Namespace: vars.Namespace, Name: constants.DefaultConfigName}, defaultOpConf); err != nil { if errors.IsNotFound(err) { reqLogger.Info("default SriovOperatorConfig object not found, cannot reconcile SriovNetworkNodePolicies. Requeue.") - return reconcile.Result{RequeueAfter: 5 * time.Second}, nil + return reconcile.Result{RequeueAfter: constants.DrainControllerRequeueTime}, nil } return reconcile.Result{}, err } @@ -133,10 +135,6 @@ func (r *SriovNetworkNodePolicyReconciler) Reconcile(ctx context.Context, req ct if err = r.syncDevicePluginConfigMap(ctx, defaultOpConf, policyList, nodeList); err != nil { return reconcile.Result{}, err } - // Render and sync Daemon objects - if err = syncPluginDaemonObjs(ctx, r.Client, r.Scheme, defaultOpConf, policyList); err != nil { - return reconcile.Result{}, err - } // All was successful. Request that this be re-triggered after ResyncPeriod, // so we can reconcile state again. @@ -145,7 +143,7 @@ func (r *SriovNetworkNodePolicyReconciler) Reconcile(ctx context.Context, req ct // SetupWithManager sets up the controller with the Manager. func (r *SriovNetworkNodePolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { - qHandler := func(q workqueue.RateLimitingInterface) { + qHandler := func(q workqueue.TypedRateLimitingInterface[reconcile.Request]) { q.AddAfter(reconcile.Request{NamespacedName: types.NamespacedName{ Namespace: "", Name: nodePolicySyncEventName, @@ -153,39 +151,47 @@ func (r *SriovNetworkNodePolicyReconciler) SetupWithManager(mgr ctrl.Manager) er } delayedEventHandler := handler.Funcs{ - CreateFunc: func(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { + CreateFunc: func(c context.Context, e event.TypedCreateEvent[client.Object], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { log.Log.WithName("SriovNetworkNodePolicy"). - Info("Enqueuing sync for create event", "resource", e.Object.GetName()) - qHandler(q) + Info("Enqueuing sync for create event", "resource", e.Object.GetName(), "type", e.Object.GetObjectKind().GroupVersionKind().String()) + qHandler(w) }, - UpdateFunc: func(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { + UpdateFunc: func(c context.Context, e event.TypedUpdateEvent[client.Object], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { log.Log.WithName("SriovNetworkNodePolicy"). - Info("Enqueuing sync for update event", "resource", e.ObjectNew.GetName()) - qHandler(q) + Info("Enqueuing sync for update event", "resource", e.ObjectNew.GetName(), "type", e.ObjectNew.GetObjectKind().GroupVersionKind().String()) + qHandler(w) }, - DeleteFunc: func(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { + DeleteFunc: func(c context.Context, e event.TypedDeleteEvent[client.Object], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { log.Log.WithName("SriovNetworkNodePolicy"). - Info("Enqueuing sync for delete event", "resource", e.Object.GetName()) - qHandler(q) + Info("Enqueuing sync for delete event", "resource", e.Object.GetName(), "type", e.Object.GetObjectKind().GroupVersionKind().String()) + qHandler(w) }, - GenericFunc: func(ctx context.Context, e event.GenericEvent, q workqueue.RateLimitingInterface) { + GenericFunc: func(c context.Context, e event.TypedGenericEvent[client.Object], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { log.Log.WithName("SriovNetworkNodePolicy"). - Info("Enqueuing sync for generic event", "resource", e.Object.GetName()) - qHandler(q) + Info("Enqueuing sync for generic event", "resource", e.Object.GetName(), "type", e.Object.GetObjectKind().GroupVersionKind().String()) + qHandler(w) }, } // we want to act fast on new or deleted nodes nodeEvenHandler := handler.Funcs{ - CreateFunc: func(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { + CreateFunc: func(c context.Context, e event.TypedCreateEvent[client.Object], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { + log.Log.WithName("SriovNetworkNodePolicy"). + Info("Enqueuing sync for create event", "resource", e.Object.GetName(), "type", e.Object.GetObjectKind().GroupVersionKind().String()) + qHandler(w) + }, + UpdateFunc: func(c context.Context, e event.TypedUpdateEvent[client.Object], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { + if equality.Semantic.DeepEqual(e.ObjectOld.GetLabels(), e.ObjectNew.GetLabels()) { + return + } log.Log.WithName("SriovNetworkNodePolicy"). - Info("Enqueuing sync for create event", "resource", e.Object.GetName()) - qHandler(q) + Info("Enqueuing sync for create event", "resource", e.ObjectNew.GetName(), "type", e.ObjectNew.GetObjectKind().GroupVersionKind().String()) + qHandler(w) }, - DeleteFunc: func(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { + DeleteFunc: func(c context.Context, e event.TypedDeleteEvent[client.Object], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { log.Log.WithName("SriovNetworkNodePolicy"). - Info("Enqueuing sync for delete event", "resource", e.Object.GetName()) - qHandler(q) + Info("Enqueuing sync for delete event", "resource", e.Object.GetName(), "type", e.Object.GetObjectKind().GroupVersionKind().String()) + qHandler(w) }, } @@ -199,7 +205,8 @@ func (r *SriovNetworkNodePolicyReconciler) SetupWithManager(mgr ctrl.Manager) er For(&sriovnetworkv1.SriovNetworkNodePolicy{}). Watches(&corev1.Node{}, nodeEvenHandler). Watches(&sriovnetworkv1.SriovNetworkNodePolicy{}, delayedEventHandler). - WatchesRawSource(&source.Channel{Source: eventChan}, delayedEventHandler). + Watches(&sriovnetworkv1.SriovNetworkPoolConfig{}, delayedEventHandler). + WatchesRawSource(source.Channel(eventChan, &handler.EnqueueRequestForObject{})). Complete(r) } @@ -219,6 +226,30 @@ func (r *SriovNetworkNodePolicyReconciler) syncDevicePluginConfigMap(ctx context return err } configData[node.Name] = string(config) + + if len(data.ResourceList) == 0 { + // if we don't have policies we should add the disabled label for the device plugin + err = utils.LabelNode(ctx, node.Name, constants.SriovDevicePluginLabel, constants.SriovDevicePluginLabelDisabled, r.Client) + if err != nil { + logger.Error(err, "failed to label node for device plugin label", + "labelKey", + constants.SriovDevicePluginLabel, + "labelValue", + constants.SriovDevicePluginLabelDisabled) + return err + } + } else { + // if we have policies we should add the enabled label for the device plugin + err = utils.LabelNode(ctx, node.Name, constants.SriovDevicePluginLabel, constants.SriovDevicePluginLabelEnabled, r.Client) + if err != nil { + logger.Error(err, "failed to label node for device plugin label", + "labelKey", + constants.SriovDevicePluginLabel, + "labelValue", + constants.SriovDevicePluginLabelEnabled) + return err + } + } } cm := &corev1.ConfigMap{ @@ -271,6 +302,13 @@ func (r *SriovNetworkNodePolicyReconciler) syncAllSriovNetworkNodeStates(ctx con ns := &sriovnetworkv1.SriovNetworkNodeState{} ns.Name = node.Name ns.Namespace = vars.Namespace + netPoolConfig, _, err := findNodePoolConfig(ctx, &node, r.Client) + if err != nil { + logger.Error(err, "failed to get SriovNetworkPoolConfig for the current node") + } + if netPoolConfig != nil { + ns.Spec.System.RdmaMode = netPoolConfig.Spec.RdmaMode + } j, _ := json.Marshal(ns) logger.V(2).Info("SriovNetworkNodeState CR", "content", j) if err := r.syncSriovNetworkNodeState(ctx, dc, npl, ns, &node); err != nil { @@ -278,6 +316,7 @@ func (r *SriovNetworkNodePolicyReconciler) syncAllSriovNetworkNodeStates(ctx con return err } } + logger.V(1).Info("Remove SriovNetworkNodeState custom resource for unselected node") nsList := &sriovnetworkv1.SriovNetworkNodeStateList{} err := r.List(ctx, nsList, &client.ListOptions{}) @@ -296,10 +335,14 @@ func (r *SriovNetworkNodePolicyReconciler) syncAllSriovNetworkNodeStates(ctx con } } if !found { - logger.Info("Deleting SriovNetworkNodeState as node with that name doesn't exist", "nodeStateName", ns.Name) - err := r.Delete(ctx, &ns, &client.DeleteOptions{}) - if err != nil { - logger.Error(err, "Fail to Delete", "SriovNetworkNodeState CR:", ns.GetName()) + // remove device plugin labels if the node doesn't exist we continue to handle the stale nodeState + logger.Info("removing device plugin label from node as SriovNetworkNodeState doesn't exist", "nodeStateName", ns.Name) + err = utils.RemoveLabelFromNode(ctx, ns.Name, constants.SriovDevicePluginLabel, r.Client) + if err != nil && !errors.IsNotFound(err) { + logger.Error(err, "Fail to remove device plugin label from node", "node", ns.Name) + return err + } + if err := r.handleStaleNodeState(ctx, &ns); err != nil { return err } } @@ -308,6 +351,56 @@ func (r *SriovNetworkNodePolicyReconciler) syncAllSriovNetworkNodeStates(ctx con return nil } +// handleStaleNodeState handles stale SriovNetworkNodeState CR (the CR which no longer have a corresponding node with the daemon). +// If the CR has the "keep until time" annotation, indicating the earliest time the state object can be removed, +// this function will compare it to the current time to determine if deletion is permissible and do deletion if allowed. +// If the annotation is absent, the function will create one with a timestamp in future, using either the default or a configured offset. +// If STALE_NODE_STATE_CLEANUP_DELAY_MINUTES env variable is set to 0, removes the CR immediately +func (r *SriovNetworkNodePolicyReconciler) handleStaleNodeState(ctx context.Context, ns *sriovnetworkv1.SriovNetworkNodeState) error { + logger := log.Log.WithName("handleStaleNodeState") + + var delayMinutes int + var err error + + envValue, found := os.LookupEnv("STALE_NODE_STATE_CLEANUP_DELAY_MINUTES") + if found { + delayMinutes, err = strconv.Atoi(envValue) + if err != nil || delayMinutes < 0 { + delayMinutes = constants.DefaultNodeStateCleanupDelayMinutes + logger.Error(err, "invalid value in STALE_NODE_STATE_CLEANUP_DELAY_MINUTES env variable, use default delay", + "delay", delayMinutes) + } + } else { + delayMinutes = constants.DefaultNodeStateCleanupDelayMinutes + } + + if delayMinutes != 0 { + now := time.Now().UTC() + keepUntilTime := ns.GetKeepUntilTime() + if keepUntilTime.IsZero() { + keepUntilTime = now.Add(time.Minute * time.Duration(delayMinutes)) + logger.V(2).Info("SriovNetworkNodeState has no matching node, configure cleanup delay for the state object", + "nodeStateName", ns.Name, "delay", delayMinutes, "keepUntilTime", keepUntilTime.String()) + ns.SetKeepUntilTime(keepUntilTime) + if err := r.Update(ctx, ns); err != nil { + logger.Error(err, "Fail to update SriovNetworkNodeState CR", "name", ns.GetName()) + return err + } + return nil + } + if now.Before(keepUntilTime) { + return nil + } + } + // remove the object if delayMinutes is 0 or if keepUntilTime is already passed + logger.Info("Deleting SriovNetworkNodeState as node with that name doesn't exist", "nodeStateName", ns.Name) + if err := r.Delete(ctx, ns, &client.DeleteOptions{}); err != nil { + logger.Error(err, "Fail to delete SriovNetworkNodeState CR", "name", ns.GetName()) + return err + } + return nil +} + func (r *SriovNetworkNodePolicyReconciler) syncSriovNetworkNodeState(ctx context.Context, dc *sriovnetworkv1.SriovOperatorConfig, npl *sriovnetworkv1.SriovNetworkNodePolicyList, @@ -333,9 +426,16 @@ func (r *SriovNetworkNodePolicyReconciler) syncSriovNetworkNodeState(ctx context return fmt.Errorf("failed to get SriovNetworkNodeState: %v", err) } } else { + keepUntilAnnotationUpdated := found.ResetKeepUntilTime() + if len(found.Status.Interfaces) == 0 { logger.Info("SriovNetworkNodeState Status Interfaces are empty. Skip update of policies in spec", "namespace", ns.Namespace, "name", ns.Name) + if keepUntilAnnotationUpdated { + if err := r.Update(ctx, found); err != nil { + return fmt.Errorf("couldn't update SriovNetworkNodeState: %v", err) + } + } return nil } @@ -378,7 +478,7 @@ func (r *SriovNetworkNodePolicyReconciler) syncSriovNetworkNodeState(ctx context // Note(adrianc): we check same ownerReferences since SriovNetworkNodeState // was owned by a default SriovNetworkNodePolicy. if we encounter a descripancy // we need to update. - if reflect.DeepEqual(newVersion.OwnerReferences, found.OwnerReferences) && + if !keepUntilAnnotationUpdated && equality.Semantic.DeepEqual(newVersion.OwnerReferences, found.OwnerReferences) && equality.Semantic.DeepEqual(newVersion.Spec, found.Spec) { logger.V(1).Info("SriovNetworkNodeState did not change, not updating") return nil @@ -415,13 +515,13 @@ func (r *SriovNetworkNodePolicyReconciler) renderDevicePluginConfigData(ctx cont found, i := resourceNameInList(p.Spec.ResourceName, &rcl) if found { - err := updateDevicePluginResource(ctx, &rcl.ResourceList[i], &p, nodeState) + err := updateDevicePluginResource(&rcl.ResourceList[i], &p, nodeState) if err != nil { return rcl, err } logger.V(1).Info("Update resource", "Resource", rcl.ResourceList[i]) } else { - rc, err := createDevicePluginResource(ctx, &p, nodeState) + rc, err := createDevicePluginResource(&p, nodeState) if err != nil { return rcl, err } @@ -442,7 +542,6 @@ func resourceNameInList(name string, rcl *dptypes.ResourceConfList) (bool, int) } func createDevicePluginResource( - ctx context.Context, p *sriovnetworkv1.SriovNetworkNodePolicy, nodeState *sriovnetworkv1.SriovNetworkNodeState) (*dptypes.ResourceConfig, error) { netDeviceSelectors := dptypes.NetDeviceSelectors{} @@ -516,7 +615,6 @@ func createDevicePluginResource( } func updateDevicePluginResource( - ctx context.Context, rc *dptypes.ResourceConfig, p *sriovnetworkv1.SriovNetworkNodePolicy, nodeState *sriovnetworkv1.SriovNetworkNodeState) error { diff --git a/controllers/sriovnetworknodepolicy_controller_test.go b/controllers/sriovnetworknodepolicy_controller_test.go index a116efe87..c35286228 100644 --- a/controllers/sriovnetworknodepolicy_controller_test.go +++ b/controllers/sriovnetworknodepolicy_controller_test.go @@ -3,20 +3,26 @@ package controllers import ( "context" "encoding/json" + "os" + "sync" "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" "github.com/google/go-cmp/cmp" + dptypes "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/types" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - dptypes "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/types" - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/featuregate" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" @@ -42,7 +48,7 @@ func TestRenderDevicePluginConfigData(t *testing.T) { { tname: "testVirtioVdpaVirtio", policy: sriovnetworkv1.SriovNetworkNodePolicy{ - Spec: v1.SriovNetworkNodePolicySpec{ + Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ ResourceName: "resourceName", DeviceType: consts.DeviceTypeNetDevice, VdpaType: consts.VdpaTypeVirtio, @@ -61,7 +67,7 @@ func TestRenderDevicePluginConfigData(t *testing.T) { }, { tname: "testVhostVdpaVirtio", policy: sriovnetworkv1.SriovNetworkNodePolicy{ - Spec: v1.SriovNetworkNodePolicySpec{ + Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ ResourceName: "resourceName", DeviceType: consts.DeviceTypeNetDevice, VdpaType: consts.VdpaTypeVhost, @@ -81,7 +87,7 @@ func TestRenderDevicePluginConfigData(t *testing.T) { { tname: "testExcludeTopology", policy: sriovnetworkv1.SriovNetworkNodePolicy{ - Spec: v1.SriovNetworkNodePolicySpec{ + Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ ResourceName: "resourceName", ExcludeTopology: true, }, @@ -126,3 +132,414 @@ func TestRenderDevicePluginConfigData(t *testing.T) { }) } } + +var _ = Describe("SriovnetworkNodePolicy controller", Ordered, func() { + var cancel context.CancelFunc + var ctx context.Context + + BeforeAll(func() { + // disable stale state cleanup delay to check that the controller can cleanup state objects + DeferCleanup(os.Setenv, "STALE_NODE_STATE_CLEANUP_DELAY_MINUTES", os.Getenv("STALE_NODE_STATE_CLEANUP_DELAY_MINUTES")) + os.Setenv("STALE_NODE_STATE_CLEANUP_DELAY_MINUTES", "0") + + By("Create SriovOperatorConfig controller k8s objs") + config := makeDefaultSriovOpConfig() + Expect(k8sClient.Create(context.Background(), config)).Should(Succeed()) + DeferCleanup(func() { + err := k8sClient.Delete(context.Background(), config) + Expect(err).ToNot(HaveOccurred()) + }) + + // setup controller manager + By("Setup controller manager") + k8sManager, err := setupK8sManagerForTest() + Expect(err).ToNot(HaveOccurred()) + + err = (&SriovNetworkNodePolicyReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + FeatureGate: featuregate.New(), + }).SetupWithManager(k8sManager) + Expect(err).ToNot(HaveOccurred()) + + ctx, cancel = context.WithCancel(context.Background()) + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + defer GinkgoRecover() + By("Start controller manager") + err := k8sManager.Start(ctx) + Expect(err).ToNot(HaveOccurred()) + }() + + DeferCleanup(func() { + By("Shut down manager") + cancel() + wg.Wait() + }) + }) + AfterEach(func() { + err := k8sClient.DeleteAllOf(context.Background(), &corev1.Node{}, k8sclient.GracePeriodSeconds(0)) + Expect(err).ToNot(HaveOccurred()) + + err = k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovNetworkNodePolicy{}, k8sclient.InNamespace(vars.Namespace), k8sclient.GracePeriodSeconds(0)) + Expect(err).ToNot(HaveOccurred()) + + err = k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovNetworkNodeState{}, k8sclient.InNamespace(vars.Namespace), k8sclient.GracePeriodSeconds(0)) + Expect(err).ToNot(HaveOccurred()) + }) + Context("device plugin labels", func() { + It("Should add the right labels to the nodes", func() { + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + Labels: map[string]string{"kubernetes.io/os": "linux", + "node-role.kubernetes.io/worker": ""}, + }} + Expect(k8sClient.Create(ctx, node)).To(Succeed()) + + nodeState := &sriovnetworkv1.SriovNetworkNodeState{} + Eventually(func(g Gomega) { + err := k8sClient.Get(context.TODO(), k8sclient.ObjectKey{Name: "node0", Namespace: testNamespace}, nodeState) + g.Expect(err).ToNot(HaveOccurred()) + }, time.Minute, time.Second).Should(Succeed()) + + Eventually(func(g Gomega) { + err := k8sClient.Get(context.Background(), k8sclient.ObjectKey{Name: node.Name}, node) + g.Expect(err).ToNot(HaveOccurred()) + value, exist := node.Labels[consts.SriovDevicePluginLabel] + g.Expect(exist).To(BeTrue()) + g.Expect(value).To(Equal(consts.SriovDevicePluginLabelDisabled)) + }, time.Minute, time.Second).Should(Succeed()) + + nodeState.Status.Interfaces = sriovnetworkv1.InterfaceExts{ + sriovnetworkv1.InterfaceExt{ + Vendor: "8086", + Driver: "i40e", + Mtu: 1500, + Name: "ens803f0", + PciAddress: "0000:86:00.0", + NumVfs: 0, + TotalVfs: 64, + }, + } + err := k8sClient.Status().Update(context.Background(), nodeState) + Expect(err).ToNot(HaveOccurred()) + + somePolicy := &sriovnetworkv1.SriovNetworkNodePolicy{} + somePolicy.SetNamespace(testNamespace) + somePolicy.SetName("some-policy") + somePolicy.Spec = sriovnetworkv1.SriovNetworkNodePolicySpec{ + NumVfs: 5, + NodeSelector: map[string]string{"node-role.kubernetes.io/worker": ""}, + NicSelector: sriovnetworkv1.SriovNetworkNicSelector{Vendor: "8086"}, + Priority: 20, + } + Expect(k8sClient.Create(context.Background(), somePolicy)).ToNot(HaveOccurred()) + + Eventually(func(g Gomega) { + err := k8sClient.Get(context.Background(), k8sclient.ObjectKey{Name: node.Name}, node) + g.Expect(err).ToNot(HaveOccurred()) + value, exist := node.Labels[consts.SriovDevicePluginLabel] + g.Expect(exist).To(BeTrue()) + g.Expect(value).To(Equal(consts.SriovDevicePluginLabelEnabled)) + }, time.Minute, time.Second).Should(Succeed()) + + delete(node.Labels, "node-role.kubernetes.io/worker") + err = k8sClient.Update(context.Background(), node) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func(g Gomega) { + err := k8sClient.Get(context.Background(), k8sclient.ObjectKey{Name: node.Name}, node) + g.Expect(err).ToNot(HaveOccurred()) + _, exist := node.Labels[consts.SriovDevicePluginLabel] + g.Expect(exist).To(BeFalse()) + }, time.Minute, time.Second).Should(Succeed()) + + Eventually(func(g Gomega) { + err := k8sClient.Get(context.Background(), k8sclient.ObjectKey{Name: node.Name, Namespace: testNamespace}, nodeState) + Expect(err).To(HaveOccurred()) + Expect(errors.IsNotFound(err)).To(BeTrue()) + }, time.Minute, time.Second).Should(Succeed()) + }) + + It("should skip label removal for nodes that doesn't exist with no stale timer", func() { + node0 := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + Labels: map[string]string{"kubernetes.io/os": "linux", + "node-role.kubernetes.io/worker": ""}, + }} + Expect(k8sClient.Create(ctx, node0)).To(Succeed()) + + node1 := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"kubernetes.io/os": "linux", + "node-role.kubernetes.io/worker": ""}, + }} + Expect(k8sClient.Create(ctx, node1)).To(Succeed()) + + nodeState := &sriovnetworkv1.SriovNetworkNodeState{} + node := &corev1.Node{} + for _, nodeName := range []string{"node0", "node1"} { + Eventually(func(g Gomega) { + err := k8sClient.Get(context.TODO(), k8sclient.ObjectKey{Name: nodeName, Namespace: testNamespace}, nodeState) + g.Expect(err).ToNot(HaveOccurred()) + }, time.Minute, time.Second).Should(Succeed()) + + Eventually(func(g Gomega) { + err := k8sClient.Get(context.Background(), k8sclient.ObjectKey{Name: nodeName}, node) + g.Expect(err).ToNot(HaveOccurred()) + value, exist := node.Labels[consts.SriovDevicePluginLabel] + g.Expect(exist).To(BeTrue()) + g.Expect(value).To(Equal(consts.SriovDevicePluginLabelDisabled)) + }, time.Minute, time.Second).Should(Succeed()) + + nodeState.Status.Interfaces = sriovnetworkv1.InterfaceExts{ + sriovnetworkv1.InterfaceExt{ + Vendor: "8086", + Driver: "i40e", + Mtu: 1500, + Name: "ens803f0", + PciAddress: "0000:86:00.0", + NumVfs: 0, + TotalVfs: 64, + }, + } + err := k8sClient.Status().Update(context.Background(), nodeState) + Expect(err).ToNot(HaveOccurred()) + } + + err := k8sClient.Delete(context.Background(), node1, k8sclient.GracePeriodSeconds(0)) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func(g Gomega) { + err := k8sClient.Get(context.TODO(), k8sclient.ObjectKey{Name: "node1", Namespace: testNamespace}, nodeState) + g.Expect(err).To(HaveOccurred()) + g.Expect(errors.IsNotFound(err)).To(BeTrue()) + }, 30*time.Second, time.Second).Should(Succeed()) + + somePolicy := &sriovnetworkv1.SriovNetworkNodePolicy{} + somePolicy.SetNamespace(testNamespace) + somePolicy.SetName("some-policy") + somePolicy.Spec = sriovnetworkv1.SriovNetworkNodePolicySpec{ + NumVfs: 5, + NodeSelector: map[string]string{"node-role.kubernetes.io/worker": ""}, + NicSelector: sriovnetworkv1.SriovNetworkNicSelector{Vendor: "8086"}, + Priority: 20, + } + Expect(k8sClient.Create(context.Background(), somePolicy)).ToNot(HaveOccurred()) + + Eventually(func(g Gomega) { + err := k8sClient.Get(context.Background(), k8sclient.ObjectKey{Name: node0.Name}, node0) + g.Expect(err).ToNot(HaveOccurred()) + value, exist := node0.Labels[consts.SriovDevicePluginLabel] + g.Expect(exist).To(BeTrue()) + g.Expect(value).To(Equal(consts.SriovDevicePluginLabelEnabled)) + }, time.Minute, time.Second).Should(Succeed()) + }) + + It("should skip label removal for nodes that doesn't exist with stale timer", func() { + err := os.Setenv("STALE_NODE_STATE_CLEANUP_DELAY_MINUTES", "5") + Expect(err).ToNot(HaveOccurred()) + defer func() { + err = os.Unsetenv("STALE_NODE_STATE_CLEANUP_DELAY_MINUTES") + Expect(err).ToNot(HaveOccurred()) + }() + + node0 := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + Labels: map[string]string{"kubernetes.io/os": "linux", + "node-role.kubernetes.io/worker": ""}, + }} + Expect(k8sClient.Create(ctx, node0)).To(Succeed()) + + node1 := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"kubernetes.io/os": "linux", + "node-role.kubernetes.io/worker": ""}, + }} + Expect(k8sClient.Create(ctx, node1)).To(Succeed()) + + nodeState := &sriovnetworkv1.SriovNetworkNodeState{} + node := &corev1.Node{} + for _, nodeName := range []string{"node0", "node1"} { + Eventually(func(g Gomega) { + err := k8sClient.Get(context.TODO(), k8sclient.ObjectKey{Name: nodeName, Namespace: testNamespace}, nodeState) + g.Expect(err).ToNot(HaveOccurred()) + }, time.Minute, time.Second).Should(Succeed()) + + Eventually(func(g Gomega) { + err := k8sClient.Get(context.Background(), k8sclient.ObjectKey{Name: nodeName}, node) + g.Expect(err).ToNot(HaveOccurred()) + value, exist := node.Labels[consts.SriovDevicePluginLabel] + g.Expect(exist).To(BeTrue()) + g.Expect(value).To(Equal(consts.SriovDevicePluginLabelDisabled)) + }, time.Minute, time.Second).Should(Succeed()) + + nodeState.Status.Interfaces = sriovnetworkv1.InterfaceExts{ + sriovnetworkv1.InterfaceExt{ + Vendor: "8086", + Driver: "i40e", + Mtu: 1500, + Name: "ens803f0", + PciAddress: "0000:86:00.0", + NumVfs: 0, + TotalVfs: 64, + }, + } + err := k8sClient.Status().Update(context.Background(), nodeState) + Expect(err).ToNot(HaveOccurred()) + } + + err = k8sClient.Delete(context.Background(), node1, k8sclient.GracePeriodSeconds(0)) + Expect(err).ToNot(HaveOccurred()) + + Consistently(func(g Gomega) { + err := k8sClient.Get(context.TODO(), k8sclient.ObjectKey{Name: "node1", Namespace: testNamespace}, nodeState) + g.Expect(err).ToNot(HaveOccurred()) + }, 10*time.Second, time.Second).Should(Succeed()) + + somePolicy := &sriovnetworkv1.SriovNetworkNodePolicy{} + somePolicy.SetNamespace(testNamespace) + somePolicy.SetName("some-policy") + somePolicy.Spec = sriovnetworkv1.SriovNetworkNodePolicySpec{ + NumVfs: 5, + NodeSelector: map[string]string{"node-role.kubernetes.io/worker": ""}, + NicSelector: sriovnetworkv1.SriovNetworkNicSelector{Vendor: "8086"}, + Priority: 20, + } + Expect(k8sClient.Create(context.Background(), somePolicy)).ToNot(HaveOccurred()) + + Eventually(func(g Gomega) { + err := k8sClient.Get(context.Background(), k8sclient.ObjectKey{Name: node0.Name}, node0) + g.Expect(err).ToNot(HaveOccurred()) + value, exist := node0.Labels[consts.SriovDevicePluginLabel] + g.Expect(exist).To(BeTrue()) + g.Expect(value).To(Equal(consts.SriovDevicePluginLabelEnabled)) + }, time.Minute, time.Second).Should(Succeed()) + }) + }) + + Context("RdmaMode", func() { + BeforeEach(func() { + Expect( + k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovNetworkPoolConfig{}, k8sclient.InNamespace(vars.Namespace)), + ).ToNot(HaveOccurred()) + }) + + It("field is correctly written to the SriovNetworkNodeState", func() { + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + Labels: map[string]string{ + "node-role.kubernetes.io/worker": "", + "kubernetes.io/os": "linux", + "test": "", + }, + }} + Expect(k8sClient.Create(ctx, node)).To(Succeed()) + + nodeState := &sriovnetworkv1.SriovNetworkNodeState{} + Eventually(func(g Gomega) { + err := k8sClient.Get(context.TODO(), k8sclient.ObjectKey{Name: "node0", Namespace: testNamespace}, nodeState) + g.Expect(err).ToNot(HaveOccurred()) + }, time.Minute, time.Second).Should(Succeed()) + + nodeState.Status.Interfaces = sriovnetworkv1.InterfaceExts{ + sriovnetworkv1.InterfaceExt{ + Vendor: "8086", + Driver: "i40e", + Mtu: 1500, + Name: "ens803f0", + PciAddress: "0000:86:00.0", + NumVfs: 0, + TotalVfs: 64, + }, + } + err := k8sClient.Status().Update(context.Background(), nodeState) + Expect(err).ToNot(HaveOccurred()) + + poolConfig := &sriovnetworkv1.SriovNetworkPoolConfig{} + poolConfig.SetNamespace(testNamespace) + poolConfig.SetName("test-workers") + poolConfig.Spec = sriovnetworkv1.SriovNetworkPoolConfigSpec{ + NodeSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "test": "", + }, + }, + RdmaMode: "exclusive", + } + Expect(k8sClient.Create(ctx, poolConfig)).To(Succeed()) + + Eventually(func(g Gomega) { + err := k8sClient.Get(context.Background(), k8sclient.ObjectKey{Name: node.Name, Namespace: testNamespace}, nodeState) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(nodeState.Spec.System.RdmaMode).To(Equal("exclusive")) + }).WithPolling(time.Second).WithTimeout(time.Minute).Should(Succeed()) + + }) + }) +}) + +var _ = Describe("SriovNetworkNodePolicyReconciler", Ordered, func() { + Context("handleStaleNodeState", func() { + var ( + ctx context.Context + r *SriovNetworkNodePolicyReconciler + nodeState *sriovnetworkv1.SriovNetworkNodeState + ) + + BeforeEach(func() { + ctx = context.Background() + scheme := runtime.NewScheme() + utilruntime.Must(sriovnetworkv1.AddToScheme(scheme)) + nodeState = &sriovnetworkv1.SriovNetworkNodeState{ObjectMeta: metav1.ObjectMeta{Name: "node1"}} + r = &SriovNetworkNodePolicyReconciler{Client: fake.NewClientBuilder().WithObjects(nodeState).Build()} + }) + It("should set default delay", func() { + nodeState := nodeState.DeepCopy() + Expect(r.handleStaleNodeState(ctx, nodeState)).NotTo(HaveOccurred()) + Expect(r.Get(ctx, types.NamespacedName{Name: nodeState.Name}, nodeState)).NotTo(HaveOccurred()) + Expect(time.Now().UTC().Before(nodeState.GetKeepUntilTime())).To(BeTrue()) + }) + It("should remove CR if wait time expired", func() { + nodeState := nodeState.DeepCopy() + nodeState.SetKeepUntilTime(time.Now().UTC().Add(-time.Minute)) + Expect(r.handleStaleNodeState(ctx, nodeState)).NotTo(HaveOccurred()) + Expect(errors.IsNotFound(r.Get(ctx, types.NamespacedName{Name: nodeState.Name}, nodeState))).To(BeTrue()) + }) + It("should keep existing wait time if already set", func() { + nodeState := nodeState.DeepCopy() + nodeState.SetKeepUntilTime(time.Now().UTC().Add(time.Minute)) + testTime := nodeState.GetKeepUntilTime() + r.Update(ctx, nodeState) + Expect(r.handleStaleNodeState(ctx, nodeState)).NotTo(HaveOccurred()) + Expect(r.Get(ctx, types.NamespacedName{Name: nodeState.Name}, nodeState)).NotTo(HaveOccurred()) + Expect(nodeState.GetKeepUntilTime()).To(Equal(testTime)) + }) + It("non default dealy", func() { + DeferCleanup(os.Setenv, "STALE_NODE_STATE_CLEANUP_DELAY_MINUTES", os.Getenv("STALE_NODE_STATE_CLEANUP_DELAY_MINUTES")) + os.Setenv("STALE_NODE_STATE_CLEANUP_DELAY_MINUTES", "60") + nodeState := nodeState.DeepCopy() + Expect(r.handleStaleNodeState(ctx, nodeState)).NotTo(HaveOccurred()) + Expect(r.Get(ctx, types.NamespacedName{Name: nodeState.Name}, nodeState)).NotTo(HaveOccurred()) + Expect(time.Until(nodeState.GetKeepUntilTime()) > 30*time.Minute).To(BeTrue()) + }) + It("invalid non default delay - should use default", func() { + DeferCleanup(os.Setenv, "STALE_NODE_STATE_CLEANUP_DELAY_MINUTES", os.Getenv("STALE_NODE_STATE_CLEANUP_DELAY_MINUTES")) + os.Setenv("STALE_NODE_STATE_CLEANUP_DELAY_MINUTES", "-20") + nodeState := nodeState.DeepCopy() + Expect(r.handleStaleNodeState(ctx, nodeState)).NotTo(HaveOccurred()) + Expect(r.Get(ctx, types.NamespacedName{Name: nodeState.Name}, nodeState)).NotTo(HaveOccurred()) + Expect(time.Until(nodeState.GetKeepUntilTime()) > 20*time.Minute).To(BeTrue()) + }) + It("should remove CR if delay is zero", func() { + DeferCleanup(os.Setenv, "STALE_NODE_STATE_CLEANUP_DELAY_MINUTES", os.Getenv("STALE_NODE_STATE_CLEANUP_DELAY_MINUTES")) + os.Setenv("STALE_NODE_STATE_CLEANUP_DELAY_MINUTES", "0") + nodeState := nodeState.DeepCopy() + Expect(r.handleStaleNodeState(ctx, nodeState)).NotTo(HaveOccurred()) + Expect(errors.IsNotFound(r.Get(ctx, types.NamespacedName{Name: nodeState.Name}, nodeState))).To(BeTrue()) + }) + }) +}) diff --git a/controllers/sriovnetworkpoolconfig_controller.go b/controllers/sriovnetworkpoolconfig_controller.go index 43fd513c9..0ec05f625 100644 --- a/controllers/sriovnetworkpoolconfig_controller.go +++ b/controllers/sriovnetworkpoolconfig_controller.go @@ -4,10 +4,10 @@ import ( "context" "encoding/json" "fmt" - "reflect" - mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + mcfgv1 "github.com/openshift/api/machineconfiguration/v1" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -196,7 +196,7 @@ func (r *SriovNetworkPoolConfigReconciler) syncOvsHardwareOffloadMachineConfigs( // ignition and compare. json.Unmarshal(foundMC.Spec.Config.Raw, &foundIgn) json.Unmarshal(mc.Spec.Config.Raw, &renderedIgn) - if !reflect.DeepEqual(foundIgn, renderedIgn) { + if !equality.Semantic.DeepEqual(foundIgn, renderedIgn) { logger.Info("MachineConfig already exists, updating") mc.SetResourceVersion(foundMC.GetResourceVersion()) err = r.Update(ctx, mc) diff --git a/controllers/sriovnetworkpoolconfig_controller_test.go b/controllers/sriovnetworkpoolconfig_controller_test.go index f2f9d7df1..d477e0248 100644 --- a/controllers/sriovnetworkpoolconfig_controller_test.go +++ b/controllers/sriovnetworkpoolconfig_controller_test.go @@ -8,11 +8,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" - mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + mcfgv1 "github.com/openshift/api/machineconfiguration/v1" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" diff --git a/controllers/sriovoperatorconfig_controller.go b/controllers/sriovoperatorconfig_controller.go index 8d028d8eb..f3e35e183 100644 --- a/controllers/sriovoperatorconfig_controller.go +++ b/controllers/sriovoperatorconfig_controller.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "errors" "fmt" "os" "sort" @@ -28,6 +29,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" uns "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" kscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" @@ -38,24 +40,26 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - machinev1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + "github.com/go-logr/logr" + machinev1 "github.com/openshift/api/machineconfiguration/v1" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - apply "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/apply" - consts "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/apply" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/featuregate" snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms" - render "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/render" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/render" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) // SriovOperatorConfigReconciler reconciles a SriovOperatorConfig object type SriovOperatorConfigReconciler struct { client.Client - Scheme *runtime.Scheme - PlatformHelper platforms.Interface - FeatureGate featuregate.FeatureGate + Scheme *runtime.Scheme + PlatformHelper platforms.Interface + FeatureGate featuregate.FeatureGate + UncachedAPIReader client.Reader } //+kubebuilder:rbac:groups=sriovnetwork.openshift.io,resources=sriovoperatorconfigs,verbs=get;list;watch;create;update;patch;delete @@ -90,6 +94,16 @@ func (r *SriovOperatorConfigReconciler) Reconcile(ctx context.Context, req ctrl. snolog.SetLogLevel(defaultConfig.Spec.LogLevel) + // examine DeletionTimestamp to determine if object is under deletion + if !defaultConfig.ObjectMeta.DeletionTimestamp.IsZero() { + // The object is being deleted + return r.handleSriovOperatorConfigDeletion(ctx, defaultConfig, logger) + } + + if err = r.syncOperatorConfigFinalizers(ctx, defaultConfig, logger); err != nil { + return reconcile.Result{}, err + } + r.FeatureGate.Init(defaultConfig.Spec.FeatureGates) logger.Info("enabled featureGates", "featureGates", r.FeatureGate.String()) @@ -124,7 +138,7 @@ func (r *SriovOperatorConfigReconciler) Reconcile(ctx context.Context, req ctrl. return reconcile.Result{}, err } - if err = syncPluginDaemonObjs(ctx, r.Client, r.Scheme, defaultConfig, policyList); err != nil { + if err = syncPluginDaemonObjs(ctx, r.Client, r.Scheme, defaultConfig); err != nil { return reconcile.Result{}, err } @@ -241,6 +255,7 @@ func (r *SriovOperatorConfigReconciler) syncMetricsExporter(ctx context.Context, data.Data["IsOpenshift"] = r.PlatformHelper.IsOpenshiftCluster() data.Data["IsPrometheusOperatorInstalled"] = strings.ToLower(os.Getenv("METRICS_EXPORTER_PROMETHEUS_OPERATOR_ENABLED")) == trueString + data.Data["PrometheusOperatorDeployRules"] = strings.ToLower(os.Getenv("METRICS_EXPORTER_PROMETHEUS_DEPLOY_RULES")) == trueString data.Data["PrometheusOperatorServiceAccount"] = os.Getenv("METRICS_EXPORTER_PROMETHEUS_OPERATOR_SERVICE_ACCOUNT") data.Data["PrometheusOperatorNamespace"] = os.Getenv("METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE") @@ -396,7 +411,8 @@ func (r *SriovOperatorConfigReconciler) syncOpenShiftSystemdService(ctx context. if cr.Spec.ConfigurationMode != sriovnetworkv1.SystemdConfigurationMode { obj := &machinev1.MachineConfig{} - err := r.Get(context.TODO(), types.NamespacedName{Name: consts.SystemdServiceOcpMachineConfigName}, obj) + // use uncached api reader to get machineconfig to reduce memory footprint + err := r.UncachedAPIReader.Get(ctx, types.NamespacedName{Name: consts.SystemdServiceOcpMachineConfigName}, obj) if err != nil { if apierrors.IsNotFound(err) { return nil @@ -407,7 +423,7 @@ func (r *SriovOperatorConfigReconciler) syncOpenShiftSystemdService(ctx context. } logger.Info("Systemd service was deployed but the operator is now operating on daemonset mode, removing the machine config") - err = r.Delete(context.TODO(), obj) + err = r.Delete(ctx, obj) if err != nil { logger.Error(err, "failed to remove the systemd service machine config") return err @@ -429,6 +445,50 @@ func (r *SriovOperatorConfigReconciler) syncOpenShiftSystemdService(ctx context. return r.setLabelInsideObject(ctx, cr, objs) } +func (r SriovOperatorConfigReconciler) syncOperatorConfigFinalizers(ctx context.Context, defaultConfig *sriovnetworkv1.SriovOperatorConfig, logger logr.Logger) error { + if sriovnetworkv1.StringInArray(sriovnetworkv1.OPERATORCONFIGFINALIZERNAME, defaultConfig.ObjectMeta.Finalizers) { + return nil + } + + newObj := defaultConfig.DeepCopyObject().(client.Object) + newObj.SetFinalizers( + append(newObj.GetFinalizers(), sriovnetworkv1.OPERATORCONFIGFINALIZERNAME), + ) + + logger.WithName("syncOperatorConfigFinalizers"). + Info("Adding finalizer", "key", sriovnetworkv1.OPERATORCONFIGFINALIZERNAME) + + patch := client.MergeFrom(defaultConfig) + err := r.Patch(ctx, newObj, patch) + if err != nil { + return fmt.Errorf("can't patch SriovOperatorConfig to add finalizer [%s]: %w", sriovnetworkv1.OPERATORCONFIGFINALIZERNAME, err) + } + + // Refresh the defaultConfig object with the latest changes + return r.Get(ctx, types.NamespacedName{Namespace: defaultConfig.Namespace, Name: defaultConfig.Name}, defaultConfig) +} + +func (r *SriovOperatorConfigReconciler) handleSriovOperatorConfigDeletion(ctx context.Context, + defaultConfig *sriovnetworkv1.SriovOperatorConfig, logger logr.Logger) (ctrl.Result, error) { + var err error + if sriovnetworkv1.StringInArray(sriovnetworkv1.OPERATORCONFIGFINALIZERNAME, defaultConfig.ObjectMeta.Finalizers) { + // our finalizer is present, so lets handle any external dependency + logger.Info("delete SriovOperatorConfig CR", "Namespace", defaultConfig.Namespace, "Name", defaultConfig.Name) + // make sure webhooks objects are deleted prior of removing finalizer + err = r.deleteAllWebhooks(ctx) + if err != nil { + return reconcile.Result{}, err + } + // remove our finalizer from the list and update it. + defaultConfig.ObjectMeta.Finalizers, _ = sriovnetworkv1.RemoveString(sriovnetworkv1.OPERATORCONFIGFINALIZERNAME, defaultConfig.ObjectMeta.Finalizers) + if err := r.Update(ctx, defaultConfig); err != nil { + return reconcile.Result{}, err + } + } + + return reconcile.Result{}, err +} + func (r SriovOperatorConfigReconciler) setLabelInsideObject(ctx context.Context, cr *sriovnetworkv1.SriovOperatorConfig, objs []*uns.Unstructured) error { logger := log.Log.WithName("setLabelInsideObject") for _, obj := range objs { @@ -456,3 +516,29 @@ func (r SriovOperatorConfigReconciler) setLabelInsideObject(ctx context.Context, return nil } + +func (r SriovOperatorConfigReconciler) deleteAllWebhooks(ctx context.Context) error { + var err error + obj := &uns.Unstructured{} + obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "admissionregistration.k8s.io", Kind: "MutatingWebhookConfiguration", Version: "v1"}) + obj.SetName(consts.OperatorWebHookName) + err = errors.Join( + err, r.deleteWebhookObject(ctx, obj), + ) + + obj = &uns.Unstructured{} + obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "admissionregistration.k8s.io", Kind: "ValidatingWebhookConfiguration", Version: "v1"}) + obj.SetName(consts.OperatorWebHookName) + err = errors.Join( + err, r.deleteWebhookObject(ctx, obj), + ) + + obj = &uns.Unstructured{} + obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "admissionregistration.k8s.io", Kind: "MutatingWebhookConfiguration", Version: "v1"}) + obj.SetName(consts.InjectorWebHookName) + err = errors.Join( + err, r.deleteWebhookObject(ctx, obj), + ) + + return err +} diff --git a/controllers/sriovoperatorconfig_controller_test.go b/controllers/sriovoperatorconfig_controller_test.go index 9ba05490b..6a4bfbb25 100644 --- a/controllers/sriovoperatorconfig_controller_test.go +++ b/controllers/sriovoperatorconfig_controller_test.go @@ -2,14 +2,15 @@ package controllers import ( "context" - "fmt" "os" "strings" "sync" + "time" admv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + networkv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -19,9 +20,9 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" @@ -29,7 +30,9 @@ import ( mock_platforms "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/mock" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openshift" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" - util "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util" + + uns "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) var _ = Describe("SriovOperatorConfig controller", Ordered, func() { @@ -38,20 +41,8 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() { BeforeAll(func() { By("Create SriovOperatorConfig controller k8s objs") - config := &sriovnetworkv1.SriovOperatorConfig{} - config.SetNamespace(testNamespace) - config.SetName(consts.DefaultConfigName) - config.Spec = sriovnetworkv1.SriovOperatorConfigSpec{ - EnableInjector: true, - EnableOperatorWebhook: true, - ConfigDaemonNodeSelector: map[string]string{}, - LogLevel: 2, - } + config := makeDefaultSriovOpConfig() Expect(k8sClient.Create(context.Background(), config)).Should(Succeed()) - DeferCleanup(func() { - err := k8sClient.Delete(context.Background(), config) - Expect(err).ToNot(HaveOccurred()) - }) somePolicy := &sriovnetworkv1.SriovNetworkNodePolicy{} somePolicy.SetNamespace(testNamespace) @@ -63,10 +54,6 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() { Priority: 20, } Expect(k8sClient.Create(context.Background(), somePolicy)).ToNot(HaveOccurred()) - DeferCleanup(func() { - err := k8sClient.Delete(context.Background(), somePolicy) - Expect(err).ToNot(HaveOccurred()) - }) // setup controller manager By("Setup controller manager") @@ -81,10 +68,11 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() { platformHelper.EXPECT().IsHypershift().Return(false).AnyTimes() err = (&SriovOperatorConfigReconciler{ - Client: k8sManager.GetClient(), - Scheme: k8sManager.GetScheme(), - PlatformHelper: platformHelper, - FeatureGate: featuregate.New(), + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + PlatformHelper: platformHelper, + FeatureGate: featuregate.New(), + UncachedAPIReader: k8sManager.GetAPIReader(), }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) @@ -108,18 +96,47 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() { }) Context("When is up", func() { + AfterAll(func() { + err := k8sClient.DeleteAllOf(context.Background(), &corev1.Node{}) + Expect(err).ToNot(HaveOccurred()) + + err = k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovNetworkNodePolicy{}, client.InNamespace(vars.Namespace)) + Expect(err).ToNot(HaveOccurred()) + + err = k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovNetworkNodeState{}, client.InNamespace(vars.Namespace)) + Expect(err).ToNot(HaveOccurred()) + + err = k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovOperatorConfig{}, client.InNamespace(vars.Namespace)) + Expect(err).ToNot(HaveOccurred()) + + operatorConfigList := &sriovnetworkv1.SriovOperatorConfigList{} + Eventually(func(g Gomega) { + err = k8sClient.List(context.Background(), operatorConfigList, &client.ListOptions{Namespace: vars.Namespace}) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(operatorConfigList.Items)).To(Equal(0)) + }, time.Minute, time.Second).Should(Succeed()) + }) + BeforeEach(func() { + var err error config := &sriovnetworkv1.SriovOperatorConfig{} - err := util.WaitForNamespacedObject(config, k8sClient, testNamespace, "default", util.RetryInterval, util.APITimeout) - Expect(err).NotTo(HaveOccurred()) - config.Spec = sriovnetworkv1.SriovOperatorConfigSpec{ - EnableInjector: true, - EnableOperatorWebhook: true, - LogLevel: 2, - FeatureGates: map[string]bool{}, - } - err = k8sClient.Update(ctx, config) - Expect(err).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + err = util.WaitForNamespacedObject(config, k8sClient, testNamespace, "default", util.RetryInterval, util.APITimeout) + g.Expect(err).NotTo(HaveOccurred()) + // in case controller yet to add object's finalizer (e.g whenever test deferCleanup is creating new 'default' config object) + g.Expect(config.Finalizers).ToNot(BeEmpty()) + + config.Spec = sriovnetworkv1.SriovOperatorConfigSpec{ + EnableInjector: true, + EnableOperatorWebhook: true, + LogLevel: 2, + FeatureGates: map[string]bool{}, + } + err = k8sClient.Update(ctx, config) + Expect(err).NotTo(HaveOccurred()) + }).Should(Succeed()) + }) It("should have webhook enable", func() { @@ -161,6 +178,10 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() { err = util.WaitForNamespacedObjectDeleted(daemonSet, k8sClient, testNamespace, "network-resources-injector", util.RetryInterval, util.APITimeout) Expect(err).NotTo(HaveOccurred()) + networkPolicy := &networkv1.NetworkPolicy{} + err = util.WaitForNamespacedObjectDeleted(networkPolicy, k8sClient, testNamespace, "network-resources-injector-allow-traffic-api-server", util.RetryInterval, util.APITimeout) + Expect(err).NotTo(HaveOccurred()) + mutateCfg := &admv1.MutatingWebhookConfiguration{} err = util.WaitForNamespacedObjectDeleted(mutateCfg, k8sClient, testNamespace, "network-resources-injector-config", util.RetryInterval, util.APITimeout) Expect(err).NotTo(HaveOccurred()) @@ -177,6 +198,10 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() { err = util.WaitForNamespacedObject(daemonSet, k8sClient, testNamespace, "network-resources-injector", util.RetryInterval, util.APITimeout) Expect(err).NotTo(HaveOccurred()) + networkPolicy = &networkv1.NetworkPolicy{} + err = util.WaitForNamespacedObject(networkPolicy, k8sClient, testNamespace, "network-resources-injector-allow-traffic-api-server", util.RetryInterval, util.APITimeout) + Expect(err).NotTo(HaveOccurred()) + mutateCfg = &admv1.MutatingWebhookConfiguration{} err = util.WaitForNamespacedObject(mutateCfg, k8sClient, testNamespace, "network-resources-injector-config", util.RetryInterval, util.APITimeout) Expect(err).NotTo(HaveOccurred()) @@ -196,6 +221,10 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() { err = util.WaitForNamespacedObjectDeleted(daemonSet, k8sClient, testNamespace, "operator-webhook", util.RetryInterval, util.APITimeout) Expect(err).NotTo(HaveOccurred()) + networkPolicy := &networkv1.NetworkPolicy{} + err = util.WaitForNamespacedObjectDeleted(networkPolicy, k8sClient, testNamespace, "operator-webhook-allow-traffic-api-server", util.RetryInterval, util.APITimeout) + Expect(err).NotTo(HaveOccurred()) + mutateCfg := &admv1.MutatingWebhookConfiguration{} err = util.WaitForNamespacedObjectDeleted(mutateCfg, k8sClient, testNamespace, "sriov-operator-webhook-config", util.RetryInterval, util.APITimeout) Expect(err).NotTo(HaveOccurred()) @@ -204,7 +233,7 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() { err = util.WaitForNamespacedObjectDeleted(validateCfg, k8sClient, testNamespace, "sriov-operator-webhook-config", util.RetryInterval, util.APITimeout) Expect(err).NotTo(HaveOccurred()) - By("set disable to enableOperatorWebhook") + By("set enable to enableOperatorWebhook") Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: "default"}, config)).NotTo(HaveOccurred()) config.Spec.EnableOperatorWebhook = true @@ -215,6 +244,10 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() { err = util.WaitForNamespacedObject(daemonSet, k8sClient, testNamespace, "operator-webhook", util.RetryInterval, util.APITimeout) Expect(err).NotTo(HaveOccurred()) + networkPolicy = &networkv1.NetworkPolicy{} + err = util.WaitForNamespacedObject(networkPolicy, k8sClient, testNamespace, "operator-webhook-allow-traffic-api-server", util.RetryInterval, util.APITimeout) + Expect(err).NotTo(HaveOccurred()) + mutateCfg = &admv1.MutatingWebhookConfiguration{} err = util.WaitForNamespacedObject(mutateCfg, k8sClient, testNamespace, "sriov-operator-webhook-config", util.RetryInterval, util.APITimeout) Expect(err).NotTo(HaveOccurred()) @@ -224,6 +257,101 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() { Expect(err).NotTo(HaveOccurred()) }) + // Namespaced resources are deleted via the `.ObjectMeta.OwnerReference` field. That logic can't be tested here because testenv doesn't have built-in controllers + // (See https://book.kubebuilder.io/reference/envtest#testing-considerations). Since Service and DaemonSet are deleted when default/SriovOperatorConfig is no longer + // present, it's important that webhook configurations are deleted as well. + It("should delete the webhooks when SriovOperatorConfig/default is deleted", func() { + DeferCleanup(k8sClient.Create, context.Background(), makeDefaultSriovOpConfig()) + + err := k8sClient.Delete(context.Background(), &sriovnetworkv1.SriovOperatorConfig{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: "default"}, + }) + Expect(err).NotTo(HaveOccurred()) + + assertResourceDoesNotExist( + schema.GroupVersionKind{Group: "admissionregistration.k8s.io", Kind: "MutatingWebhookConfiguration", Version: "v1"}, + client.ObjectKey{Name: "sriov-operator-webhook-config"}) + assertResourceDoesNotExist( + schema.GroupVersionKind{Group: "admissionregistration.k8s.io", Kind: "ValidatingWebhookConfiguration", Version: "v1"}, + client.ObjectKey{Name: "sriov-operator-webhook-config"}) + + assertResourceDoesNotExist( + schema.GroupVersionKind{Group: "admissionregistration.k8s.io", Kind: "MutatingWebhookConfiguration", Version: "v1"}, + client.ObjectKey{Name: "network-resources-injector-config"}) + }) + + It("should add/delete finalizer 'operatorconfig' when SriovOperatorConfig/default is added/deleted", func() { + DeferCleanup(k8sClient.Create, context.Background(), makeDefaultSriovOpConfig()) + + // verify that finalizer has been added upon object creation + config := &sriovnetworkv1.SriovOperatorConfig{} + Eventually(func() []string { + // wait for SriovOperatorConfig flags to get updated + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "default", Namespace: testNamespace}, config) + if err != nil { + return nil + } + return config.Finalizers + }, util.APITimeout, util.RetryInterval).Should(Equal([]string{sriovnetworkv1.OPERATORCONFIGFINALIZERNAME})) + + err := k8sClient.Delete(context.Background(), &sriovnetworkv1.SriovOperatorConfig{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: "default"}, + }) + Expect(err).NotTo(HaveOccurred()) + + // verify that finalizer has been removed + var empty []string + config = &sriovnetworkv1.SriovOperatorConfig{} + Eventually(func() []string { + // wait for SriovOperatorConfig flags to get updated + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "default", Namespace: testNamespace}, config) + if err != nil { + return nil + } + return config.Finalizers + }, util.APITimeout, util.RetryInterval).Should(Equal(empty)) + }) + + It("should not remove fields with default values when SriovOperatorConfig is created", func() { + err := k8sClient.Delete(context.Background(), &sriovnetworkv1.SriovOperatorConfig{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: consts.DefaultConfigName}, + }) + Expect(err).NotTo(HaveOccurred()) + + config := &uns.Unstructured{} + config.SetGroupVersionKind(sriovnetworkv1.GroupVersion.WithKind("SriovOperatorConfig")) + config.SetName(consts.DefaultConfigName) + config.SetNamespace(testNamespace) + config.Object["spec"] = map[string]interface{}{ + "enableInjector": false, + "enableOperatorWebhook": false, + "logLevel": 0, + "disableDrain": false, + } + + Eventually(func() error { + return k8sClient.Create(context.Background(), config) + }).Should(Succeed()) + + By("Wait for the operator to reconcile the object") + Eventually(func(g Gomega) { + err := k8sClient.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: consts.DefaultConfigName}, config) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(config.GetFinalizers()).To(ContainElement(sriovnetworkv1.OPERATORCONFIGFINALIZERNAME)) + }, util.APITimeout, util.RetryInterval).Should(Succeed()) + + By("Verify default values have not been omitted") + obj := &uns.Unstructured{} + obj.SetGroupVersionKind(sriovnetworkv1.GroupVersion.WithKind("SriovOperatorConfig")) + err = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: consts.DefaultConfigName}, obj) + Expect(err).NotTo(HaveOccurred()) + + Expect(obj.Object["spec"]).To(HaveKeyWithValue("enableInjector", false)) + Expect(obj.Object["spec"]).To(HaveKeyWithValue("enableOperatorWebhook", false)) + Expect(obj.Object["spec"]).To(HaveKeyWithValue("logLevel", int64(0))) + Expect(obj.Object["spec"]).To(HaveKeyWithValue("disableDrain", false)) + }) + It("should be able to update the node selector of sriov-network-config-daemon", func() { By("specify the configDaemonNodeSelector") nodeSelector := map[string]string{"node-role.kubernetes.io/worker": ""} @@ -232,7 +360,6 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() { daemonSet := &appsv1.DaemonSet{} Eventually(func() map[string]string { - // By("wait for DaemonSet NodeSelector") err := k8sClient.Get(ctx, types.NamespacedName{Name: "sriov-network-config-daemon", Namespace: testNamespace}, daemonSet) if err != nil { return nil @@ -241,6 +368,32 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() { }, util.APITimeout, util.RetryInterval).Should(Equal(nodeSelector)) }) + It("should be able to update the node selector of sriov-network-device-plugin", func() { + By("specify the configDaemonNodeSelector") + daemonSet := &appsv1.DaemonSet{} + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, types.NamespacedName{Name: "sriov-device-plugin", Namespace: testNamespace}, daemonSet) + g.Expect(err).ToNot(HaveOccurred()) + _, exist := daemonSet.Spec.Template.Spec.NodeSelector["node-role.kubernetes.io/worker"] + g.Expect(exist).To(BeFalse()) + _, exist = daemonSet.Spec.Template.Spec.NodeSelector[consts.SriovDevicePluginLabel] + g.Expect(exist).To(BeTrue()) + }, util.APITimeout, util.RetryInterval).Should(Succeed()) + + nodeSelector := map[string]string{"node-role.kubernetes.io/worker": ""} + restore := updateConfigDaemonNodeSelector(nodeSelector) + DeferCleanup(restore) + + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, types.NamespacedName{Name: "sriov-device-plugin", Namespace: testNamespace}, daemonSet) + g.Expect(err).ToNot(HaveOccurred()) + _, exist := daemonSet.Spec.Template.Spec.NodeSelector["node-role.kubernetes.io/worker"] + g.Expect(exist).To(BeTrue()) + _, exist = daemonSet.Spec.Template.Spec.NodeSelector[consts.SriovDevicePluginLabel] + g.Expect(exist).To(BeTrue()) + }, util.APITimeout, util.RetryInterval).Should(Succeed()) + }) + It("should be able to do multiple updates to the node selector of sriov-network-config-daemon", func() { By("changing the configDaemonNodeSelector") firstNodeSelector := map[string]string{"labelA": "", "labelB": "", "labelC": ""} @@ -373,13 +526,15 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() { metricsDaemonset := appsv1.DaemonSet{} err := util.WaitForNamespacedObject(&metricsDaemonset, k8sClient, testNamespace, "sriov-network-metrics-exporter", util.RetryInterval, util.APITimeout) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(metricsDaemonset.Spec.Template.Spec.NodeSelector).To((Equal(nodeSelector))) - }).Should(Succeed()) + g.Expect(metricsDaemonset.Spec.Template.Spec.NodeSelector).To(Equal(nodeSelector)) + }, time.Minute, time.Second).Should(Succeed()) }) It("should deploy extra configuration when the Prometheus operator is installed", func() { DeferCleanup(os.Setenv, "METRICS_EXPORTER_PROMETHEUS_OPERATOR_ENABLED", os.Getenv("METRICS_EXPORTER_PROMETHEUS_OPERATOR_ENABLED")) os.Setenv("METRICS_EXPORTER_PROMETHEUS_OPERATOR_ENABLED", "true") + DeferCleanup(os.Setenv, "METRICS_EXPORTER_PROMETHEUS_DEPLOY_RULES", os.Getenv("METRICS_EXPORTER_PROMETHEUS_DEPLOY_RULES")) + os.Setenv("METRICS_EXPORTER_PROMETHEUS_DEPLOY_RULES", "true") err := util.WaitForNamespacedObject(&rbacv1.Role{}, k8sClient, testNamespace, "prometheus-k8s", util.RetryInterval, util.APITimeout) Expect(err).ToNot(HaveOccurred()) @@ -394,6 +549,14 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() { Version: "v1", }, client.ObjectKey{Namespace: testNamespace, Name: "sriov-network-metrics-exporter"}) + + assertResourceExists( + schema.GroupVersionKind{ + Group: "monitoring.coreos.com", + Kind: "PrometheusRule", + Version: "v1", + }, + client.ObjectKey{Namespace: testNamespace, Name: "sriov-vf-rules"}) }) }) }) @@ -457,56 +620,22 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() { g.Expect(injectorCfg.Webhooks[0].ClientConfig.CABundle).To(Equal([]byte("ca-bundle-2\n"))) }, "1s").Should(Succeed()) }) - - It("should reconcile to a converging state when multiple node policies are set", func() { - By("Creating a consistent number of node policies") - for i := 0; i < 30; i++ { - p := &sriovnetworkv1.SriovNetworkNodePolicy{ - ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: fmt.Sprintf("p%d", i)}, - Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ - Priority: 99, - NodeSelector: map[string]string{"foo": fmt.Sprintf("v%d", i)}, - }, - } - err := k8sClient.Create(context.Background(), p) - Expect(err).NotTo(HaveOccurred()) - } - - By("Triggering a the reconcile loop") - config := &sriovnetworkv1.SriovOperatorConfig{} - err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "default", Namespace: testNamespace}, config) - Expect(err).NotTo(HaveOccurred()) - if config.ObjectMeta.Labels == nil { - config.ObjectMeta.Labels = make(map[string]string) - } - config.ObjectMeta.Labels["trigger-test"] = "test-reconcile-daemonset" - err = k8sClient.Update(context.Background(), config) - Expect(err).NotTo(HaveOccurred()) - - By("Wait until device-plugin Daemonset's affinity has been calculated") - var expectedAffinity *corev1.Affinity - - Eventually(func(g Gomega) { - daemonSet := &appsv1.DaemonSet{} - err = k8sClient.Get(context.Background(), types.NamespacedName{Name: "sriov-device-plugin", Namespace: testNamespace}, daemonSet) - g.Expect(err).NotTo(HaveOccurred()) - // Wait until the last policy (with NodeSelector foo=v29) has been considered at least one time - g.Expect(daemonSet.Spec.Template.Spec.Affinity.String()).To(ContainSubstring("v29")) - expectedAffinity = daemonSet.Spec.Template.Spec.Affinity - }, "3s", "1s").Should(Succeed()) - - By("Verify device-plugin Daemonset's affinity doesn't change over time") - Consistently(func(g Gomega) { - daemonSet := &appsv1.DaemonSet{} - err = k8sClient.Get(context.Background(), types.NamespacedName{Name: "sriov-device-plugin", Namespace: testNamespace}, daemonSet) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(daemonSet.Spec.Template.Spec.Affinity). - To(Equal(expectedAffinity)) - }, "3s", "1s").Should(Succeed()) - }) }) }) +func makeDefaultSriovOpConfig() *sriovnetworkv1.SriovOperatorConfig { + config := &sriovnetworkv1.SriovOperatorConfig{} + config.SetNamespace(testNamespace) + config.SetName(consts.DefaultConfigName) + config.Spec = sriovnetworkv1.SriovOperatorConfigSpec{ + EnableInjector: true, + EnableOperatorWebhook: true, + ConfigDaemonNodeSelector: map[string]string{}, + LogLevel: 2, + } + return config +} + func assertResourceExists(gvk schema.GroupVersionKind, key client.ObjectKey) { u := &unstructured.Unstructured{} u.SetGroupVersionKind(gvk) @@ -514,6 +643,20 @@ func assertResourceExists(gvk schema.GroupVersionKind, key client.ObjectKey) { Expect(err).NotTo(HaveOccurred()) } +func assertResourceDoesNotExist(gvk schema.GroupVersionKind, key client.ObjectKey) { + Eventually(func(g Gomega) { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(gvk) + err := k8sClient.Get(context.Background(), key, u) + g.Expect(err).To(HaveOccurred()) + g.Expect(errors.IsNotFound(err)).To(BeTrue()) + }). + WithOffset(1). + WithPolling(100*time.Millisecond). + WithTimeout(2*time.Second). + Should(Succeed(), "Resource type[%s] name[%s] still present in the cluster", gvk.String(), key.String()) +} + func updateConfigDaemonNodeSelector(newValue map[string]string) func() { config := &sriovnetworkv1.SriovOperatorConfig{} err := k8sClient.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: "default"}, config) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index bc2f13b8e..e477142c8 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -37,14 +37,16 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/metrics/server" netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" openshiftconfigv1 "github.com/openshift/api/config/v1" - mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + mcfgv1 "github.com/openshift/api/machineconfiguration/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" //+kubebuilder:scaffold:imports sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util" ) @@ -63,7 +65,8 @@ const testNamespace = "openshift-sriov-network-operator" func setupK8sManagerForTest() (manager.Manager, error) { k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme.Scheme, + Scheme: scheme.Scheme, + Metrics: server.Options{BindAddress: "0"}, // we don't need metrics server for tests }) if err != nil { @@ -90,10 +93,12 @@ var _ = BeforeSuite(func() { logf.SetLogger(zap.New( zap.WriteTo(GinkgoWriter), + zap.Level(zapcore.Level(-2)), zap.UseDevMode(true), func(o *zap.Options) { o.TimeEncoder = zapcore.RFC3339NanoTimeEncoder })) + snolog.InitLog() // Go to project root directory err = os.Chdir("..") @@ -188,6 +193,13 @@ var _ = BeforeSuite(func() { } Expect(k8sClient.Create(context.Background(), ns)).Should(Succeed()) + sa := &corev1.ServiceAccount{TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: testNamespace, + }} + Expect(k8sClient.Create(context.Background(), sa)).Should(Succeed()) + // Create openshift Infrastructure infra := &openshiftconfigv1.Infrastructure{ ObjectMeta: metav1.ObjectMeta{ diff --git a/deploy/clusterrole.yaml b/deploy/clusterrole.yaml index e7a596061..e7a84394e 100644 --- a/deploy/clusterrole.yaml +++ b/deploy/clusterrole.yaml @@ -45,12 +45,6 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch", "patch", "update"] -- apiGroups: [""] - resources: ["pods"] - verbs: ["*"] -- apiGroups: ["apps"] - resources: ["daemonsets"] - verbs: ["get"] - apiGroups: [ "config.openshift.io" ] resources: [ "infrastructures" ] verbs: [ "get", "list", "watch" ] diff --git a/deploy/configmap.yaml b/deploy/configmap.yaml index b21b07ea6..6f1381d00 100644 --- a/deploy/configmap.yaml +++ b/deploy/configmap.yaml @@ -16,8 +16,16 @@ data: Intel_ice_Columbiaville_E810-CQDA2_2CQDA2: "8086 1592 1889" Intel_ice_Columbiaville_E810-XXVDA4: "8086 1593 1889" Intel_ice_Columbiaville_E810-XXVDA2: "8086 159b 1889" + Intel_ice_Columbiaville_E810-XXV_BACKPLANE: "8086 1599 1889" Intel_ice_Columbiaville_E810: "8086 1591 1889" Intel_ice_Columbiapark_E823C: "8086 188a 1889" + Intel_ice_Columbiapark_E823L_SFP: "8086 124d 1889" + Intel_ice_Columbiapark_E823L_BACKPLANE: "8086 124c 1889" + Intel_ice_Columbiapark_E825C_BACKPLANE: "8086 579c 1889" + Intel_ice_Columbiapark_E825C_QSFP: "8086 579d 1889" + Intel_ice_Columbiapark_E825C_SFP: "8086 579e 1889" + Intel_ice_Connorsville_E830_QSFP: "8086 12d2 1889" + Intel_ice_Connorsville_E830_SFP: "8086 12d3 1889" Nvidia_mlx5_ConnectX-4: "15b3 1013 1014" Nvidia_mlx5_ConnectX-4LX: "15b3 1015 1016" Nvidia_mlx5_ConnectX-5: "15b3 1017 1018" @@ -26,6 +34,7 @@ data: Nvidia_mlx5_ConnectX-6_Dx: "15b3 101d 101e" Nvidia_mlx5_ConnectX-6_Lx: "15b3 101f 101e" Nvidia_mlx5_ConnectX-7: "15b3 1021 101e" + Nvidia_mlx5_ConnectX-8: "15b3 1023 101e" Nvidia_mlx5_MT42822_BlueField-2_integrated_ConnectX-6_Dx: "15b3 a2d6 101e" Nvidia_mlx5_MT43244_BlueField-3_integrated_ConnectX-7_Dx: "15b3 a2dc 101e" Broadcom_bnxt_BCM57414_2x25G: "14e4 16d7 16dc" diff --git a/deploy/operator.yaml b/deploy/operator.yaml index b2aa302ab..99e28df31 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -7,10 +7,8 @@ spec: selector: matchLabels: name: sriov-network-operator - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 33% + strategy: + type: Recreate template: metadata: labels: @@ -20,14 +18,22 @@ spec: spec: affinity: nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + # In the context of Hypershift, the SR-IOV network + # Operator is deployed on Nodepools which are labeled + # as workers. So we relax the node affinity to prefer + # masters/control-plane when possible otherwise we + # schedule where it's possible. + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: Exists + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/control-plane" + operator: Exists tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master @@ -43,8 +49,6 @@ spec: image: $SRIOV_NETWORK_OPERATOR_IMAGE command: - sriov-network-operator - args: - - --leader-elect=$OPERATOR_LEADER_ELECTION_ENABLE resources: requests: cpu: 100m @@ -78,6 +82,8 @@ spec: value: $METRICS_EXPORTER_KUBE_RBAC_PROXY_IMAGE - name: METRICS_EXPORTER_PROMETHEUS_OPERATOR_ENABLED value: "$METRICS_EXPORTER_PROMETHEUS_OPERATOR_ENABLED" + - name: METRICS_EXPORTER_PROMETHEUS_DEPLOY_RULES + value: "$METRICS_EXPORTER_PROMETHEUS_DEPLOY_RULES" - name: METRICS_EXPORTER_PROMETHEUS_OPERATOR_SERVICE_ACCOUNT value: $METRICS_EXPORTER_PROMETHEUS_OPERATOR_SERVICE_ACCOUNT - name: METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE diff --git a/deploy/role.yaml b/deploy/role.yaml index a24f13729..990d3de85 100644 --- a/deploy/role.yaml +++ b/deploy/role.yaml @@ -1,7 +1,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - creationTimestamp: null name: sriov-network-operator rules: - apiGroups: @@ -29,9 +28,21 @@ rules: - monitoring.coreos.com resources: - servicemonitors + - prometheusrules verbs: - get - create + - update + - delete +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - create + - update + - delete - apiGroups: - apps resourceNames: @@ -73,13 +84,10 @@ rules: resources: - pods verbs: - - '*' -- apiGroups: - - apps - resources: - - daemonsets - verbs: - - '*' + - "get" + - "list" + - "watch" + - "delete" - apiGroups: - sriovnetwork.openshift.io resources: diff --git a/deployment/sriov-network-operator-chart/README.md b/deployment/sriov-network-operator-chart/README.md index 8875b36eb..d1d6a0bb4 100644 --- a/deployment/sriov-network-operator-chart/README.md +++ b/deployment/sriov-network-operator-chart/README.md @@ -41,7 +41,7 @@ For additional information and methods for installing Helm, refer to the officia #### Deploy from OCI repo ``` -$ helm install -n sriov-network-operator --create-namespace --version 1.3.0 --set sriovOperatorConfig.deploy=true sriov-network-operator oci://ghcr.io/k8snetworkplumbingwg/sriov-network-operator-chart +$ helm install -n sriov-network-operator --create-namespace --version 1.5.0 --set sriovOperatorConfig.deploy=true sriov-network-operator oci://ghcr.io/k8snetworkplumbingwg/sriov-network-operator-chart ``` #### Deploy from project sources @@ -89,6 +89,7 @@ We have introduced the following Chart parameters. | `operator.metricsExporter.prometheusOperator.enabled` | bool | false | Wheter the operator shoud configure Prometheus resources or not (e.g. `ServiceMonitors`). | | `operator.metricsExporter.prometheusOperator.serviceAccount` | string | `prometheus-k8s` | The service account used by the Prometheus Operator. This is used to give Prometheus the permission to list resource in the SR-IOV operator namespace | | `operator.metricsExporter.prometheusOperator.namespace` | string | `monitoring` | The namespace where the Prometheus Operator is installed. Setting this variable makes the operator deploy `monitoring.coreos.com` resources. | +| `operator.metricsExporter.prometheusOperator.deployRules` | bool | false | Whether the operator should deploy `PrometheusRules` to scrape namespace version of metrics. | #### Admission Controllers parameters @@ -134,6 +135,11 @@ This section contains general parameters that apply to both the operator and dae | `sriovOperatorConfig.configurationMode` | string | `daemon` | sriov-network-config-daemon configuration mode. either `daemon` or `systemd` | | `sriovOperatorConfig.featureGates` | map[string]bool | `{}` | feature gates to enable/disable | +**Note** + +When `sriovOperatorConfig.configurationMode` is configured as `systemd`, configurations files and `systemd` service files are created on the node. +Upon chart deletion, those files are not cleaned up. For cases where this is not acceptable, users should rather configured the `daemon` mode. + ### Images parameters | Name | description | diff --git a/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovibnetworks.yaml b/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovibnetworks.yaml index 4b4b44d92..498fb6c08 100644 --- a/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovibnetworks.yaml +++ b/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovibnetworks.yaml @@ -62,6 +62,8 @@ spec: networkNamespace: description: Namespace of the NetworkAttachmentDefinition custom resource type: string + pKey: + type: string resourceName: description: SRIOV Network device plugin endpoint resource name type: string diff --git a/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml b/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml index 36c1050ea..524c5124e 100644 --- a/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml +++ b/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml @@ -81,6 +81,10 @@ spec: description: external_ids field in the Interface table in OVSDB type: object + mtuRequest: + description: mtu_request field in the Interface table + in OVSDB + type: integer options: additionalProperties: type: string diff --git a/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml b/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml index c5bf230c3..7535346ef 100644 --- a/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml +++ b/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml @@ -102,6 +102,10 @@ spec: description: external_ids field in the Interface table in OVSDB type: object + mtuRequest: + description: mtu_request field in the Interface + table in OVSDB + type: integer options: additionalProperties: type: string @@ -174,6 +178,15 @@ spec: - pciAddress type: object type: array + system: + properties: + rdmaMode: + description: RDMA subsystem. Allowed value "shared", "exclusive". + enum: + - shared + - exclusive + type: string + type: object type: object status: description: SriovNetworkNodeStateStatus defines the observed state of @@ -228,6 +241,10 @@ spec: description: external_ids field in the Interface table in OVSDB type: object + mtuRequest: + description: mtu_request field in the Interface + table in OVSDB + type: integer options: additionalProperties: type: string @@ -335,6 +352,15 @@ spec: type: string syncStatus: type: string + system: + properties: + rdmaMode: + description: RDMA subsystem. Allowed value "shared", "exclusive". + enum: + - shared + - exclusive + type: string + type: object type: object type: object served: true diff --git a/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml b/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml index 2cb2ece31..9ccb580ed 100644 --- a/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml +++ b/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml @@ -83,11 +83,13 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string @@ -111,6 +113,12 @@ spec: Name is the name of MachineConfigPool to be enabled with OVS hardware offload type: string type: object + rdmaMode: + description: RDMA subsystem. Allowed value "shared", "exclusive". + enum: + - shared + - exclusive + type: string type: object status: description: SriovNetworkPoolConfigStatus defines the observed state of diff --git a/deployment/sriov-network-operator-chart/templates/clusterrole.yaml b/deployment/sriov-network-operator-chart/templates/clusterrole.yaml index 7cd8fd014..519d2c05c 100644 --- a/deployment/sriov-network-operator-chart/templates/clusterrole.yaml +++ b/deployment/sriov-network-operator-chart/templates/clusterrole.yaml @@ -49,12 +49,6 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch", "patch", "update"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["*"] - - apiGroups: ["apps"] - resources: ["daemonsets"] - verbs: ["get"] - apiGroups: [ "config.openshift.io" ] resources: [ "infrastructures" ] verbs: [ "get", "list", "watch" ] diff --git a/deployment/sriov-network-operator-chart/templates/configmap.yaml b/deployment/sriov-network-operator-chart/templates/configmap.yaml index 6f6ab3bcc..b9ae0eda8 100644 --- a/deployment/sriov-network-operator-chart/templates/configmap.yaml +++ b/deployment/sriov-network-operator-chart/templates/configmap.yaml @@ -16,8 +16,16 @@ data: Intel_ice_Columbiaville_E810-CQDA2_2CQDA2: "8086 1592 1889" Intel_ice_Columbiaville_E810-XXVDA4: "8086 1593 1889" Intel_ice_Columbiaville_E810-XXVDA2: "8086 159b 1889" + Intel_ice_Columbiaville_E810-XXV_BACKPLANE: "8086 1599 1889" Intel_ice_Columbiaville_E810: "8086 1591 1889" Intel_ice_Columbiapark_E823C: "8086 188a 1889" + Intel_ice_Columbiapark_E823L_SFP: "8086 124d 1889" + Intel_ice_Columbiapark_E823L_BACKPLANE: "8086 124c 1889" + Intel_ice_Columbiapark_E825C_BACKPLANE: "8086 579c 1889" + Intel_ice_Columbiapark_E825C_QSFP: "8086 579d 1889" + Intel_ice_Columbiapark_E825C_SFP: "8086 579e 1889" + Intel_ice_Connorsville_E830_QSFP: "8086 12d2 1889" + Intel_ice_Connorsville_E830_SFP: "8086 12d3 1889" Nvidia_mlx5_ConnectX-4: "15b3 1013 1014" Nvidia_mlx5_ConnectX-4LX: "15b3 1015 1016" Nvidia_mlx5_ConnectX-5: "15b3 1017 1018" @@ -26,6 +34,7 @@ data: Nvidia_mlx5_ConnectX-6_Dx: "15b3 101d 101e" Nvidia_mlx5_ConnectX-6_Lx: "15b3 101f 101e" Nvidia_mlx5_ConnectX-7: "15b3 1021 101e" + Nvidia_mlx5_ConnectX-8: "15b3 1023 101e" Nvidia_mlx5_MT42822_BlueField-2_integrated_ConnectX-6_Dx: "15b3 a2d6 101e" Nvidia_mlx5_MT43244_BlueField-3_integrated_ConnectX-7_Dx: "15b3 a2dc 101e" Broadcom_bnxt_BCM57414_2x25G: "14e4 16d7 16dc" diff --git a/deployment/sriov-network-operator-chart/templates/operator.yaml b/deployment/sriov-network-operator-chart/templates/operator.yaml index 12a9cc660..e9b4b6d6b 100644 --- a/deployment/sriov-network-operator-chart/templates/operator.yaml +++ b/deployment/sriov-network-operator-chart/templates/operator.yaml @@ -10,9 +10,7 @@ spec: matchLabels: name: sriov-network-operator strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 33% + type: Recreate template: metadata: annotations: @@ -83,6 +81,8 @@ spec: {{- if .Values.operator.metricsExporter.prometheusOperator.enabled }} - name: METRICS_EXPORTER_PROMETHEUS_OPERATOR_ENABLED value: {{ .Values.operator.metricsExporter.prometheusOperator.enabled | quote}} + - name: METRICS_EXPORTER_PROMETHEUS_DEPLOY_RULES + value: {{ .Values.operator.metricsExporter.prometheusOperator.deployRules | quote}} - name: METRICS_EXPORTER_PROMETHEUS_OPERATOR_SERVICE_ACCOUNT value: {{ .Values.operator.metricsExporter.prometheusOperator.serviceAccount }} - name: METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE @@ -110,6 +110,8 @@ spec: value: {{ .Values.operator.cniBinPath }} - name: CLUSTER_TYPE value: {{ .Values.operator.clusterType }} + - name: STALE_NODE_STATE_CLEANUP_DELAY_MINUTES + value: "{{ .Values.operator.staleNodeStateCleanupDelayMinutes }}" {{- if .Values.operator.admissionControllers.enabled }} - name: ADMISSION_CONTROLLERS_CERTIFICATES_OPERATOR_SECRET_NAME value: {{ .Values.operator.admissionControllers.certificates.secretNames.operator }} diff --git a/deployment/sriov-network-operator-chart/templates/pre-delete-webooks.yaml b/deployment/sriov-network-operator-chart/templates/pre-delete-webooks.yaml new file mode 100644 index 000000000..202275607 --- /dev/null +++ b/deployment/sriov-network-operator-chart/templates/pre-delete-webooks.yaml @@ -0,0 +1,33 @@ +# The following job will be used as Helm pre-delete hook. It executes a small go-client binary +# which intent to delete 'default' SriovOperatorConfig, that triggers operator removal of generated cluster objects +# e.g. mutating/validating webhooks, within operator's recoinciling loop and +# preventing operator cluster object remainings while using helm uninstall +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "sriov-network-operator.fullname" . }}-pre-delete-hook + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": hook-succeeded,hook-failed +spec: + template: + spec: + serviceAccountName: {{ include "sriov-network-operator.fullname" . }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + containers: + - name: cleanup + image: {{ .Values.images.operator }} + command: + - sriov-network-operator-config-cleanup + args: + - --namespace + - {{ .Release.Namespace }} + restartPolicy: Never + backoffLimit: 2 + diff --git a/deployment/sriov-network-operator-chart/templates/role.yaml b/deployment/sriov-network-operator-chart/templates/role.yaml index 29cf80cce..244f5c259 100644 --- a/deployment/sriov-network-operator-chart/templates/role.yaml +++ b/deployment/sriov-network-operator-chart/templates/role.yaml @@ -28,13 +28,25 @@ rules: - statefulsets verbs: - '*' + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - create + - update + - delete - apiGroups: - monitoring.coreos.com resources: - servicemonitors + - prometheusrules verbs: - get - create + - update + - delete - apiGroups: - apps resourceNames: @@ -79,13 +91,10 @@ rules: resources: - pods verbs: - - '*' - - apiGroups: - - apps - resources: - - daemonsets - verbs: - - '*' + - "get" + - "list" + - "watch" + - "delete" - apiGroups: - sriovnetwork.openshift.io resources: diff --git a/deployment/sriov-network-operator-chart/values.yaml b/deployment/sriov-network-operator-chart/values.yaml index 3e73006b7..ec9323bf7 100644 --- a/deployment/sriov-network-operator-chart/values.yaml +++ b/deployment/sriov-network-operator-chart/values.yaml @@ -27,6 +27,10 @@ operator: resourcePrefix: "openshift.io" cniBinPath: "/opt/cni/bin" clusterType: "kubernetes" + # minimal amount of time (in minutes) the operator will wait before removing + # stale SriovNetworkNodeState objects (objects that doesn't match node with the daemon) + # "0" means no extra delay, in this case the CR will be removed by the next reconcilation cycle (may take up to 5 minutes) + staleNodeStateCleanupDelayMinutes: "30" metricsExporter: port: "9110" certificates: @@ -35,6 +39,7 @@ operator: enabled: false serviceAccount: "prometheus-k8s" namespace: "monitoring" + deployRules: false admissionControllers: enabled: false certificates: diff --git a/doc/supported-hardware.md b/doc/supported-hardware.md index 446190905..b1cdc5864 100644 --- a/doc/supported-hardware.md +++ b/doc/supported-hardware.md @@ -13,7 +13,15 @@ The following SR-IOV capable hardware is supported with sriov-network-operator: | Intel E810-CQDA2/2CQDA2 Family | 8086 | 1592 | | Intel E810-XXVDA4 Family | 8086 | 1593 | | Intel E810-XXVDA2 Family | 8086 | 159b | +| Intel E810-XXV Backplane Family | 8086 | 1599 | | Intel E823-C Family | 8086 | 188a | +| Intel E823-L SFP Family | 8086 | 124d | +| Intel E823-L Backplane Family | 8086 | 124c | +| Intel E825-C Backplane Family | 8086 | 579c | +| Intel E825-C QSFP Family | 8086 | 579d | +| Intel E825-C SFP Family | 8086 | 579e | +| Intel E830-CC QSFP Family | 8086 | 12d2 | +| Intel E830-CC SFP Family | 8086 | 12d3 | | Mellanox MT27700 Family [ConnectX-4] | 15b3 | 1013 | | Mellanox MT27710 Family [ConnectX-4 Lx] | 15b3 | 1015 | | Mellanox MT27800 Family [ConnectX-5] | 15b3 | 1017 | @@ -21,7 +29,8 @@ The following SR-IOV capable hardware is supported with sriov-network-operator: | Mellanox MT28908 Family [ConnectX-6] | 15b3 | 101b | | Mellanox MT28908 Family [ConnectX-6 Dx] | 15b3 | 101d | | Mellanox MT28908 Family [ConnectX-6 Lx] | 15b3 | 101f | -| Mellanox MT2910 Family [ConnectX-7 | 15b3 | 1021 | +| Mellanox MT2910 Family [ConnectX-7] | 15b3 | 1021 | +| Mellanox CX8 Family [ConnectX-8] | 15b3 | 1023 | | Mellanox MT42822 BlueField-2 integrated ConnectX-6 Dx | 15b3 | a2d6 | | Mellanox MT43244 BlueField-3 integrated ConnectX-7 Dx | 15b3 | a2dc | | Qlogic QL45000 Series 50GbE Controller | 1077 | 1654 | @@ -53,7 +62,15 @@ The following table depicts the supported SR-IOV hardware features of each suppo | Intel E810-CQDA2/2CQDA2 Family | V | V | X | | Intel E810-XXVDA4 Family | V | V | X | | Intel E810-XXVDA2 Family | V | V | X | +| Intel E810-XXV Backplane Family | V | V | X | | Intel E823-C Family | V | V | X | +| Intel E823-L SFP Family | V | V | X | +| Intel E823-L Backplane Family | V | V | X | +| Intel E825-C Backplane | V | V | X | +| Intel E825-C QSFP Family | V | V | X | +| Intel E825-C SFP Family | V | V | X | +| Intel E830-CC QSFP Family | V | V | X | +| Intel E830-CC SFP Family | V | V | X | | Mellanox MT27700 Family [ConnectX-4] | V | V | V | | Mellanox MT27710 Family [ConnectX-4 Lx] | V | V | V | | Mellanox MT27800 Family [ConnectX-5] | V | V | V | @@ -62,6 +79,7 @@ The following table depicts the supported SR-IOV hardware features of each suppo | Mellanox MT28908 Family [ConnectX-6 Dx] | V | V | V | | Mellanox MT28908 Family [ConnectX-6 Lx] | V | V | V | | Mellanox MT28908 Family [ConnectX-7] | V | V | V | +| Mellanox CX8 Family [ConnectX-8] | V | V | V | | Mellanox MT42822 BlueField-2 integrated ConnectX-6 Dx | V | V | V | | Mellanox MT43244 BlueField-3 integrated ConnectX-6 Dx | V | V | V | | Qlogic QL45000 Series 50GbE Controller | V | X | X | diff --git a/go.mod b/go.mod index 0353c7ec1..0a85bd4ce 100644 --- a/go.mod +++ b/go.mod @@ -1,72 +1,74 @@ module github.com/k8snetworkplumbingwg/sriov-network-operator -go 1.22.4 +go 1.23.0 + +toolchain go1.23.4 require ( github.com/Masterminds/sprig/v3 v3.2.2 github.com/blang/semver v3.5.1+incompatible github.com/cenkalti/backoff v2.2.1+incompatible github.com/coreos/go-systemd/v22 v22.5.0 - github.com/fsnotify/fsnotify v1.7.0 - github.com/go-logr/logr v1.2.4 + github.com/fsnotify/fsnotify v1.8.0 + github.com/go-logr/logr v1.4.2 github.com/go-logr/stdr v1.2.2 - github.com/golang/mock v1.4.4 github.com/google/go-cmp v0.6.0 github.com/google/renameio/v2 v2.0.0 - github.com/google/uuid v1.3.1 + github.com/google/uuid v1.6.0 github.com/hashicorp/go-retryablehttp v0.7.7 - github.com/jaypipes/ghw v0.9.0 - github.com/jaypipes/pcidb v1.0.0 + github.com/jaypipes/ghw v0.13.1-0.20241024164530-c1bfc6e6cd6a + github.com/jaypipes/pcidb v1.0.1 github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 github.com/k8snetworkplumbingwg/sriov-network-device-plugin v0.0.0-20221127172732-a5a7395122e3 github.com/k8snetworkplumbingwg/sriovnet v1.2.0 - github.com/onsi/ginkgo/v2 v2.11.0 - github.com/onsi/gomega v1.27.10 + github.com/onsi/ginkgo/v2 v2.22.1 + github.com/onsi/gomega v1.36.2 github.com/openshift-kni/k8sreporter v1.0.4 - github.com/openshift/api v0.0.0-20230807132801-600991d550ac - github.com/openshift/client-go v0.0.0-20230607134213-3cd0021bbee3 - github.com/openshift/machine-config-operator v0.0.1-0.20231024085435-7e1fb719c1ba + github.com/openshift/api v0.0.0-20250227152946-1ee1ef831100 + github.com/openshift/client-go v0.0.0-20250125113824-8e1f0b8fa9a7 + github.com/openshift/machine-config-operator v0.0.1-0.20250320230514-53e78f3692ee github.com/ovn-org/libovsdb v0.6.1-0.20240125124854-03f787b1a892 github.com/pkg/errors v0.9.1 - github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.68.0 + github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.74.0 github.com/prometheus-operator/prometheus-operator/pkg/client v0.68.0 - github.com/prometheus/client_model v0.5.0 - github.com/prometheus/common v0.45.0 + github.com/prometheus/client_model v0.6.1 + github.com/prometheus/common v0.62.0 github.com/safchain/ethtool v0.3.0 - github.com/spf13/cobra v1.7.0 - github.com/stretchr/testify v1.8.4 + github.com/spf13/cobra v1.8.1 + github.com/stretchr/testify v1.10.0 github.com/vishvananda/netlink v1.2.1-beta.2.0.20240221172127-ec7bcb248e94 github.com/vishvananda/netns v0.0.4 - go.uber.org/zap v1.25.0 - golang.org/x/time v0.3.0 + go.uber.org/mock v0.5.0 + go.uber.org/zap v1.27.0 + golang.org/x/net v0.38.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.28.3 - k8s.io/apiextensions-apiserver v0.28.3 - k8s.io/apimachinery v0.28.3 - k8s.io/client-go v0.28.3 - k8s.io/code-generator v0.28.3 - k8s.io/klog/v2 v2.100.1 - k8s.io/kubectl v0.28.3 - k8s.io/utils v0.0.0-20230726121419-3b25d923346b - sigs.k8s.io/controller-runtime v0.16.3 + k8s.io/api v0.32.1 + k8s.io/apiextensions-apiserver v0.32.1 + k8s.io/apimachinery v0.32.1 + k8s.io/client-go v0.32.1 + k8s.io/code-generator v0.32.1 + k8s.io/klog/v2 v2.130.1 + k8s.io/kubectl v0.32.1 + k8s.io/utils v0.0.0-20241210054802-24370beab758 + sigs.k8s.io/controller-runtime v0.20.2 ) require ( - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect - github.com/Masterminds/semver/v3 v3.1.1 // indirect + github.com/Masterminds/semver/v3 v3.3.0 // indirect github.com/Mellanox/sriovnet v1.0.3 // indirect github.com/StackExchange/wmi v1.2.1 // indirect github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect - github.com/aws/aws-sdk-go v1.44.204 // indirect + github.com/aws/aws-sdk-go v1.55.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/hub v1.0.1 // indirect github.com/cenkalti/rpc2 v0.0.0-20210604223624-c1acbc6ec984 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/clarketm/json v1.17.1 // indirect github.com/coreos/fcct v0.5.0 // indirect @@ -78,28 +80,28 @@ require ( github.com/coreos/ignition/v2 v2.15.0 // indirect github.com/coreos/vcontext v0.0.0-20230201181013-d72178a18687 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect - github.com/evanphx/json-patch/v5 v5.7.0 // indirect - github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect - github.com/frankban/quicktest v1.14.4 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect github.com/go-errors/errors v1.4.2 // indirect - github.com/go-logr/zapr v1.2.4 // indirect + github.com/go-logr/zapr v1.3.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/go-openapi/jsonpointer v0.20.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.4 // indirect - github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/glog v1.1.0 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/btree v1.0.1 // indirect - github.com/google/gnostic-models v0.6.8 // indirect + github.com/golang/glog v1.2.4 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/gnostic-models v0.6.9 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect + github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/gorilla/websocket v1.5.0 // indirect github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/huandu/xstrings v1.3.2 // indirect @@ -108,69 +110,72 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/k8snetworkplumbingwg/govdpa v0.1.4 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/mailru/easyjson v0.9.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/moby/spdystream v0.2.0 // indirect - github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect + github.com/moby/spdystream v0.5.0 // indirect + github.com/moby/term v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/openshift/library-go v0.0.0-20231020125025-211b32f1a1f2 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/openshift/library-go v0.0.0-20250129210218-fe56c2cf5d70 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.17.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.20.5 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/robfig/cron v1.2.0 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/samber/lo v1.47.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect - github.com/spf13/afero v1.9.4 // indirect - github.com/spf13/cast v1.5.0 // indirect - github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace // indirect - github.com/stretchr/objx v0.5.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.7.0 // indirect + github.com/spf13/pflag v1.0.6 // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect + github.com/x448/float16 v0.8.4 // indirect github.com/xlab/treeprint v1.2.0 // indirect - go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect + go.opentelemetry.io/otel v1.31.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect go.uber.org/multierr v1.11.0 // indirect go4.org v0.0.0-20200104003542-c7e774b10ea0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect - golang.org/x/mod v0.13.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/oauth2 v0.13.0 // indirect - golang.org/x/sync v0.4.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.14.0 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/mod v0.22.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.9.0 // indirect + golang.org/x/tools v0.29.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.56.3 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d // indirect + google.golang.org/grpc v1.69.4 // indirect + google.golang.org/protobuf v1.36.4 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect howett.net/plist v1.0.0 // indirect - k8s.io/apiserver v0.28.3 // indirect - k8s.io/cli-runtime v0.28.3 // indirect - k8s.io/component-base v0.28.3 // indirect - k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 // indirect - k8s.io/kube-aggregator v0.27.4 // indirect - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect - k8s.io/kubelet v0.27.7 // indirect - sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + k8s.io/apiserver v0.32.1 // indirect + k8s.io/cli-runtime v0.32.1 // indirect + k8s.io/component-base v0.32.1 // indirect + k8s.io/gengo/v2 v2.0.0-20250106234829-0359904fc2a6 // indirect + k8s.io/kube-aggregator v0.32.1 // indirect + k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 // indirect + k8s.io/kubelet v0.32.1 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96 // indirect - sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect - sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect + sigs.k8s.io/kustomize/api v0.18.0 // indirect + sigs.k8s.io/kustomize/kyaml v0.18.1 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.5.0 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index 6f90a1a94..8212720e7 100644 --- a/go.sum +++ b/go.sum @@ -1,51 +1,12 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= +github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= github.com/Mellanox/sriovnet v1.0.3 h1:Nlmxr2mkp16aIP4CJcsnqCczxQQgOuzNDm/nu9qTBZ8= @@ -60,11 +21,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/ashcrow/osrelease v0.0.0-20180626175927-9b292693c55c h1:icme0QhxrgZOxTBnT6K8dfGLwbKWSOVwPB95XTbo8Ws= github.com/ashcrow/osrelease v0.0.0-20180626175927-9b292693c55c/go.mod h1:BRljTyotlu+6N+Qlu5MhjxpdmccCnp9lDvZjNNV8qr4= github.com/aws/aws-sdk-go v1.19.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.44.204 h1:7/tPUXfNOHB390A63t6fJIwmlwVQAkAwcbzKsU2/6OQ= -github.com/aws/aws-sdk-go v1.44.204/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= @@ -75,26 +33,22 @@ github.com/cenk/hub v1.0.1 h1:RBwXNOF4a8KjD8BJ08XqN8KbrqaGiQLDrgvUGJSHuPA= github.com/cenk/hub v1.0.1/go.mod h1:rJM1LNAW0ppT8FMMuPK6c2NP/R2nH/UthtuRySSaf6Y= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/hub v1.0.1 h1:UMtjc6dHSaOQTO15SVA50MBIR9zQwvsukQupDrkIRtg= github.com/cenkalti/hub v1.0.1/go.mod h1:tcYwtS3a2d9NO/0xDXVJWx3IedurUjYCqFCmpi0lpHs= github.com/cenkalti/rpc2 v0.0.0-20210604223624-c1acbc6ec984 h1:CNwZyGS6KpfaOWbh2yLkSy3rSTUh3jub9CzpFpP6PVQ= github.com/cenkalti/rpc2 v0.0.0-20210604223624-c1acbc6ec984/go.mod h1:v2npkhrXyk5BCnkNIiPdRI23Uq6uWPUQGL2hnRcRr/M= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/clarketm/json v1.17.1 h1:U1IxjqJkJ7bRK4L6dyphmoO840P6bdhPdbbLySourqI= github.com/clarketm/json v1.17.1/go.mod h1:ynr2LRfb0fQU34l07csRNBTcivjySLLiY1YzQqKVfdo= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/containers/image/v5 v5.34.1 h1:/m2bkFnuedTyNkzma8s7cFLjeefPIb4trjyafWhIlwM= +github.com/containers/image/v5 v5.34.1/go.mod h1:/WnvUSEfdqC/ahMRd4YJDBLrpYWkGl018rB77iB3FDo= +github.com/containers/storage v1.57.1 h1:hKPoFsuBcB3qTzBxa4IFpZMRzUuL5Xhv/BE44W0XHx8= +github.com/containers/storage v1.57.1/go.mod h1:i/Hb4lu7YgFr9G0K6BMjqW0BLJO1sFsnWQwj2UoWCUM= github.com/coreos/fcct v0.5.0 h1:f/z+MCoR2vULes+MyoPEApQ6iluy/JbXoRi6dahPItQ= github.com/coreos/fcct v0.5.0/go.mod h1:cbE+j77YSQwFB2fozWVB3qsI2Pi3YiVEbDz/b6Yywdo= github.com/coreos/go-json v0.0.0-20230131223807-18775e0fb4fb h1:rmqyI19j3Z/74bIRhuC59RB442rXUazKNueVpfJPxg4= @@ -115,11 +69,13 @@ github.com/coreos/ignition v0.35.0/go.mod h1:WJQapxzEn9DE0ryxsGvm8QnBajm/XsS/Pkr github.com/coreos/ignition/v2 v2.1.1/go.mod h1:RqmqU64zxarUJa3l4cHtbhcSwfQLpUhv0WVziZwoXvE= github.com/coreos/ignition/v2 v2.15.0 h1:v2fQ6QvkcAF+La5PHHpnpBS1eGZo+LYL1wTOPvDKAcs= github.com/coreos/ignition/v2 v2.15.0/go.mod h1:+7BiKurzCFg3P427Ml0wqnKzIuhLimnil6LhFV2DkJM= +github.com/coreos/rpmostree-client-go v0.0.0-20230914135003-fae0786302f7 h1:gpIn0B0F00GJPlI1iPjJbHMS01QoxDs7Bf8UgH9D3wg= +github.com/coreos/rpmostree-client-go v0.0.0-20230914135003-fae0786302f7/go.mod h1:WiAXRoGnl4Lwr7OM5izCLWMLH6Y43sYWJREoeeHEg4E= github.com/coreos/vcontext v0.0.0-20190529201340-22b159166068/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE= github.com/coreos/vcontext v0.0.0-20191017033345-260217907eb5/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE= github.com/coreos/vcontext v0.0.0-20230201181013-d72178a18687 h1:uSmlDgJGbUB0bwQBcZomBTottKwEDF5fF8UjSwKSzWM= github.com/coreos/vcontext v0.0.0-20230201181013-d72178a18687/go.mod h1:Salmysdw7DAVuobBW/LwsKKgpyCPHUhjyJoMJD+ZJiI= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -127,146 +83,75 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc= -github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= -github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= -github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= -github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= -github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 h1:Mn26/9ZMNWSw9C9ERFA1PUxfmGpolnw2v0bKOREu5ew= github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= -github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= -github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= -github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/golang/glog v1.2.4 h1:CNNw5U8lSiiBk7druxtSHHTsRWcxKoac6kZKm2peBBc= +github.com/golang/glog v1.2.4/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= @@ -275,33 +160,24 @@ github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB1 github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jaypipes/ghw v0.9.0 h1:TWF4wNIGtZcgDJaiNcFgby5BR8s2ixcUe0ydxNO2McY= -github.com/jaypipes/ghw v0.9.0/go.mod h1:dXMo19735vXOjpIBDyDYSp31sB2u4hrtRCMxInqQ64k= -github.com/jaypipes/pcidb v1.0.0 h1:vtZIfkiCUE42oYbJS0TAq9XSfSmcsgo9IdxSm9qzYU8= -github.com/jaypipes/pcidb v1.0.0/go.mod h1:TnYUvqhPBzCKnH34KrIX22kAeEbDCSRJ9cqLRCuNDfk= +github.com/jaypipes/ghw v0.13.1-0.20241024164530-c1bfc6e6cd6a h1:orxBMCkYww7RFCk3iCDP9DC3l+yKtp4VdWtctCTyjPQ= +github.com/jaypipes/ghw v0.13.1-0.20241024164530-c1bfc6e6cd6a/go.mod h1:F4UM7Ix55ONYwD3Lck2S4BI+hKezOwtizuJxXDFsioo= +github.com/jaypipes/pcidb v1.0.1 h1:WB2zh27T3nwg8AE8ei81sNRb9yWBii3JGNJtT7K9Oic= +github.com/jaypipes/pcidb v1.0.1/go.mod h1:6xYUz/yYEyOkIkUt2t2J2folIuZ4Yg6uByCGFXMCeE4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/k8snetworkplumbingwg/govdpa v0.1.4 h1:e6mM7JFZkLVJeMQw3px96EigHAhnb4VUlqhNub/2Psk= github.com/k8snetworkplumbingwg/govdpa v0.1.4/go.mod h1:UQR1xu7A+nnRK1dkLEi12OnNL0OiBPpIKOYDuaQQkck= @@ -313,9 +189,10 @@ github.com/k8snetworkplumbingwg/sriovnet v1.2.0 h1:6ELfAxCB1dvosGUy3DVRmfH+HWTzm github.com/k8snetworkplumbingwg/sriovnet v1.2.0/go.mod h1:jyWzGe6ZtYiPq6ih6aXCOy6mZ49Y9mNyBOLBBXnli+k= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -323,20 +200,19 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= @@ -344,10 +220,10 @@ github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTS github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= -github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= +github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -357,24 +233,28 @@ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= -github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/ginkgo/v2 v2.22.1 h1:QW7tbJAUDyVDVOM5dFa7qaybo+CRfR7bemlQUN6Z8aM= +github.com/onsi/ginkgo/v2 v2.22.1/go.mod h1:S6aTpoRsSq2cZOd+pssHAlKW/Q/jZt6cPrPlnj4a1xM= +github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= +github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/openshift-kni/k8sreporter v1.0.4 h1:jEwX6Pqei60kO1U0JLo+ePjQaP7DNn/M6d63KCS2tS0= github.com/openshift-kni/k8sreporter v1.0.4/go.mod h1:fg8HI9yxiKAi6UzR6NTtrmQmA2WKzUqmkRUHwQ1+Bj8= -github.com/openshift/api v0.0.0-20230807132801-600991d550ac h1:HqT8MmYGXiUGUW0BjygTGOOvqO2wIsTaG3q8nboJyPY= -github.com/openshift/api v0.0.0-20230807132801-600991d550ac/go.mod h1:yimSGmjsI+XF1mr+AKBs2//fSXIOhhetHGbMlBEfXbs= -github.com/openshift/client-go v0.0.0-20230607134213-3cd0021bbee3 h1:uVCq/Sx2y4UZh+qCsCL1BBUJpc3DULHkN4j7XHHgHtw= -github.com/openshift/client-go v0.0.0-20230607134213-3cd0021bbee3/go.mod h1:M+VUIcqx5IvgzejcbgmQnxETPrXRYlcufHpw2bAgz9Y= -github.com/openshift/library-go v0.0.0-20231020125025-211b32f1a1f2 h1:TWG/YVRhSvjYq8iIwJ2Wpoopgg0zuh+ZAl1RSm4J8Z0= -github.com/openshift/library-go v0.0.0-20231020125025-211b32f1a1f2/go.mod h1:ZFwNwC3opc/7aOvzUbU95zp33Lbxet48h80ryH3p6DY= -github.com/openshift/machine-config-operator v0.0.1-0.20231024085435-7e1fb719c1ba h1:WM6K+m2xMAwbQDetKGhV/Rd8yukF3AsU1z74cqoWrz0= -github.com/openshift/machine-config-operator v0.0.1-0.20231024085435-7e1fb719c1ba/go.mod h1:mSt3ACow31pa1hTRONn+yT5e+KFkgi7G2bFEx5Nj+n0= +github.com/openshift/api v0.0.0-20250227152946-1ee1ef831100 h1:Qg0N7NUEhnlgSKOgIdFUUy7VAC9ThuXSQPzMIPnw8aE= +github.com/openshift/api v0.0.0-20250227152946-1ee1ef831100/go.mod h1:yk60tHAmHhtVpJQo3TwVYq2zpuP70iJIFDCmeKMIzPw= +github.com/openshift/client-go v0.0.0-20250125113824-8e1f0b8fa9a7 h1:4iliLcvr1P9EUMZgIaSNEKNQQzBn+L6PSequlFOuB6Q= +github.com/openshift/client-go v0.0.0-20250125113824-8e1f0b8fa9a7/go.mod h1:2tcufBE4Cu6RNgDCxcUJepa530kGo5GFVfR9BSnndhI= +github.com/openshift/library-go v0.0.0-20250129210218-fe56c2cf5d70 h1:VLj8CU9q009xlMuR4wNcqDX4lVa2Ji3u/iYnBLHtQUc= +github.com/openshift/library-go v0.0.0-20250129210218-fe56c2cf5d70/go.mod h1:TQx0VEhZ/92qRXIMDu2Wg4bUPmw5HRNE6wpSZ+IsP0Y= +github.com/openshift/machine-config-operator v0.0.1-0.20250320230514-53e78f3692ee h1:ITOXo9zrpzl68qrzh/+1uRJlbpMl88jETfWfFyIa3uU= +github.com/openshift/machine-config-operator v0.0.1-0.20250320230514-53e78f3692ee/go.mod h1:ZA2cpM5ol3zcxRpG4gQr+XSlEQJe732noPWcMz6MQ4k= github.com/ovn-org/libovsdb v0.6.1-0.20240125124854-03f787b1a892 h1:/yg3/z+RH+iDLMxp6FTnmlk5bStK542/Rge5EBjnA9A= github.com/ovn-org/libovsdb v0.6.1-0.20240125124854-03f787b1a892/go.mod h1:LC5DOvcY58jOG3HTvDyCVidoMJDurPeu+xlxv5Krd9Q= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= @@ -385,66 +265,60 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.68.0 h1:yl9ceUSUBo9woQIO+8eoWpcxZkdZgm89g+rVvu37TUw= -github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.68.0/go.mod h1:9Uuu3pEU2jB8PwuqkHvegQ0HV/BlZRJUyfTYAqfdVF8= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.74.0 h1:AHzMWDxNiAVscJL6+4wkvFRTpMnJqiaZFEKA/osaBXE= +github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.74.0/go.mod h1:wAR5JopumPtAZnu0Cjv2PSqV4p4QB09LMhc6fZZTXuA= github.com/prometheus-operator/prometheus-operator/pkg/client v0.68.0 h1:8FS0sXpFkFPxp2gfkxyEMnhZV9yhf7xPbpsIeUZHlzM= github.com/prometheus-operator/prometheus-operator/pkg/client v0.68.0/go.mod h1:ul4ND0BMCcOX1OSZvbJA1/lh7yQ2ILHNKuZIojGISe4= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= -github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/safchain/ethtool v0.3.0 h1:gimQJpsI6sc1yIqP/y8GYgiXn/NjgvpM0RNoWLVVmP0= github.com/safchain/ethtool v0.3.0/go.mod h1:SA9BwrgyAqNo7M+uaL6IYbxpm5wk3L7Mm6ocLW+CJUs= -github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= +github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.9.4 h1:Sd43wM1IWz/s1aVXdOBkjJvuP8UdyqioeE4AmM0QsBs= -github.com/spf13/afero v1.9.4/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= -github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= -github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= +github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace h1:9PNP1jnUjRhfmGMlkXHjYPishpcw4jpSt/V/xYY3FMA= -github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= @@ -456,390 +330,108 @@ github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1Y github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= -go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= +go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= +go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= -go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go4.org v0.0.0-20200104003542-c7e774b10ea0 h1:M6XsnQeLwG+rHQ+/rrGh3puBI3WZEy9TBWmf2H+enQA= go4.org v0.0.0-20200104003542-c7e774b10ea0/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= -golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191110163157-d32e6e3b99c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= +golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= -google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d h1:xJJRGY7TJcvIlpSrN3K6LAWgNFUILlO+OMAqtg9aqnw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d/go.mod h1:3ENsm/5D1mzDyhpzeRi1NR784I0BcofWBoSc5QqqMK4= +google.golang.org/grpc v1.69.4 h1:MF5TftSMkd8GLw/m0KM6V8CMOCY6NZ1NQDPGFgbTt4A= +google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= +google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -848,61 +440,49 @@ gopkg.in/yaml.v3 v3.0.0-20191010095647-fc94e3f71652/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM= howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= -k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM= -k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc= -k8s.io/apiextensions-apiserver v0.28.3 h1:Od7DEnhXHnHPZG+W9I97/fSQkVpVPQx2diy+2EtmY08= -k8s.io/apiextensions-apiserver v0.28.3/go.mod h1:NE1XJZ4On0hS11aWWJUTNkmVB03j9LM7gJSisbRt8Lc= -k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A= -k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8= -k8s.io/apiserver v0.28.3 h1:8Ov47O1cMyeDzTXz0rwcfIIGAP/dP7L8rWbEljRcg5w= -k8s.io/apiserver v0.28.3/go.mod h1:YIpM+9wngNAv8Ctt0rHG4vQuX/I5rvkEMtZtsxW2rNM= -k8s.io/cli-runtime v0.28.3 h1:lvuJYVkwCqHEvpS6KuTZsUVwPePFjBfSGvuaLl2SxzA= -k8s.io/cli-runtime v0.28.3/go.mod h1:jeX37ZPjIcENVuXDDTskG3+FnVuZms5D9omDXS/2Jjc= -k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4= -k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo= -k8s.io/code-generator v0.28.3 h1:I847QvdpYx7xKiG2KVQeCSyNF/xU9TowaDAg601mvlw= -k8s.io/code-generator v0.28.3/go.mod h1:A2EAHTRYvCvBrb/MM2zZBNipeCk3f8NtpdNIKawC43M= -k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= -k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= -k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 h1:pWEwq4Asjm4vjW7vcsmijwBhOr1/shsbSYiWXmNGlks= -k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= -k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-aggregator v0.27.4 h1:WdK9iiBr32G8bWfpUEFVQl70RZO2dU19ZAktUXL5JFc= -k8s.io/kube-aggregator v0.27.4/go.mod h1:+eG83gkAyh0uilQEAOgheeQW4hr+PkyV+5O1nLGsjlM= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= -k8s.io/kubectl v0.28.3 h1:H1Peu1O3EbN9zHkJCcvhiJ4NUj6lb88sGPO5wrWIM6k= -k8s.io/kubectl v0.28.3/go.mod h1:RDAudrth/2wQ3Sg46fbKKl4/g+XImzvbsSRZdP2RiyE= -k8s.io/kubelet v0.27.7 h1:DiptBLFbl6nyadTP9DUfhiReasBDV1qyE1r8h2o5mXc= -k8s.io/kubelet v0.27.7/go.mod h1:WKoEgiCa6/hzmgN4UgVioEwcpLC8wg+9Xzzc8fqOCYs= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= -sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +k8s.io/api v0.32.1 h1:f562zw9cy+GvXzXf0CKlVQ7yHJVYzLfL6JAS4kOAaOc= +k8s.io/api v0.32.1/go.mod h1:/Yi/BqkuueW1BgpoePYBRdDYfjPF5sgTr5+YqDZra5k= +k8s.io/apiextensions-apiserver v0.32.1 h1:hjkALhRUeCariC8DiVmb5jj0VjIc1N0DREP32+6UXZw= +k8s.io/apiextensions-apiserver v0.32.1/go.mod h1:sxWIGuGiYov7Io1fAS2X06NjMIk5CbRHc2StSmbaQto= +k8s.io/apimachinery v0.32.1 h1:683ENpaCBjma4CYqsmZyhEzrGz6cjn1MY/X2jB2hkZs= +k8s.io/apimachinery v0.32.1/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/apiserver v0.32.1 h1:oo0OozRos66WFq87Zc5tclUX2r0mymoVHRq8JmR7Aak= +k8s.io/apiserver v0.32.1/go.mod h1:UcB9tWjBY7aryeI5zAgzVJB/6k7E97bkr1RgqDz0jPw= +k8s.io/cli-runtime v0.32.1 h1:19nwZPlYGJPUDbhAxDIS2/oydCikvKMHsxroKNGA2mM= +k8s.io/cli-runtime v0.32.1/go.mod h1:NJPbeadVFnV2E7B7vF+FvU09mpwYlZCu8PqjzfuOnkY= +k8s.io/client-go v0.32.1 h1:otM0AxdhdBIaQh7l1Q0jQpmo7WOFIk5FFa4bg6YMdUU= +k8s.io/client-go v0.32.1/go.mod h1:aTTKZY7MdxUaJ/KiUs8D+GssR9zJZi77ZqtzcGXIiDg= +k8s.io/code-generator v0.32.1 h1:4lw1kFNDuFYXquTkB7Sl5EwPMUP2yyW9hh6BnFfRZFY= +k8s.io/code-generator v0.32.1/go.mod h1:zaILfm00CVyP/6/pJMJ3zxRepXkxyDfUV5SNG4CjZI4= +k8s.io/component-base v0.32.1 h1:/5IfJ0dHIKBWysGV0yKTFfacZ5yNV1sulPh3ilJjRZk= +k8s.io/component-base v0.32.1/go.mod h1:j1iMMHi/sqAHeG5z+O9BFNCF698a1u0186zkjMZQ28w= +k8s.io/gengo/v2 v2.0.0-20250106234829-0359904fc2a6 h1:SdzkGIk4b5LFkVO36PuO0Bx4tpBDJDpNN0F1/v8JM5c= +k8s.io/gengo/v2 v2.0.0-20250106234829-0359904fc2a6/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-aggregator v0.32.1 h1:cztPyIHbo6tgrhYHDqmdmvxUufJKuxgAC/vog7yeWek= +k8s.io/kube-aggregator v0.32.1/go.mod h1:sXjL5T8FO/rlBzTbBhahw9V5Nnr1UtzZHKTj9WxQCOU= +k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 h1:hcha5B1kVACrLujCKLbr8XWMxCxzQx42DY8QKYJrDLg= +k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7/go.mod h1:GewRfANuJ70iYzvn+i4lezLDAFzvjxZYK1gn1lWcfas= +k8s.io/kubectl v0.32.1 h1:/btLtXLQUU1rWx8AEvX9jrb9LaI6yeezt3sFALhB8M8= +k8s.io/kubectl v0.32.1/go.mod h1:sezNuyWi1STk4ZNPVRIFfgjqMI6XMf+oCVLjZen/pFQ= +k8s.io/kubelet v0.32.1 h1:bB91GvMsZb+LfzBxnjPEr1Fal/sdxZtYphlfwAaRJGw= +k8s.io/kubelet v0.32.1/go.mod h1:4sAEZ6PlewD0GroV3zscY7llym6kmNNTVmUI/Qshm6w= +k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0= +k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.20.2 h1:/439OZVxoEc02psi1h4QO3bHzTgu49bb347Xp4gW1pc= +sigs.k8s.io/controller-runtime v0.20.2/go.mod h1:xg2XB0K5ShQzAgsoujxuKN4LNXR2LfwwHsPj7Iaw+XY= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96 h1:PFWFSkpArPNJxFX4ZKWAk9NSeRoZaXschn+ULa4xVek= sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96/go.mod h1:EOBQyBowOUsd7U4CJnMHNE0ri+zCXyouGdLwC/jZU+I= -sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0= -sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3/go.mod h1:9n16EZKMhXBNSiUC5kSdFQJkdH3zbxS/JoO619G1VAY= -sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 h1:W6cLQc5pnqM7vh3b7HvGNfXrJ/xL6BDMS0v1V/HHg5U= -sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3/go.mod h1:JWP1Fj0VWGHyw3YUPjXSQnRnrwezrZSrApfX5S0nIag= -sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= -sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/kustomize/api v0.18.0 h1:hTzp67k+3NEVInwz5BHyzc9rGxIauoXferXyjv5lWPo= +sigs.k8s.io/kustomize/api v0.18.0/go.mod h1:f8isXnX+8b+SGLHQ6yO4JG1rdkZlvhaCf/uZbLVMb0U= +sigs.k8s.io/kustomize/kyaml v0.18.1 h1:WvBo56Wzw3fjS+7vBjN6TeivvpbW9GmRaWZ9CIVmt4E= +sigs.k8s.io/kustomize/kyaml v0.18.1/go.mod h1:C3L2BFVU1jgcddNBE1TxuVLgS46TjObMwW5FT9FcjYo= +sigs.k8s.io/structured-merge-diff/v4 v4.5.0 h1:nbCitCK2hfnhyiKo6uf2HxUPTCodY6Qaf85SbDIaMBk= +sigs.k8s.io/structured-merge-diff/v4 v4.5.0/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/hack/deploy-setup.sh b/hack/deploy-setup.sh index 2c2fc7d8d..807479c38 100755 --- a/hack/deploy-setup.sh +++ b/hack/deploy-setup.sh @@ -22,7 +22,7 @@ load_manifest() { fi files="service_account.yaml role.yaml role_binding.yaml clusterrole.yaml clusterrolebinding.yaml configmap.yaml sriovoperatorconfig.yaml operator.yaml" for m in ${files}; do - if [ "$(echo ${EXCLUSIONS[@]} | grep -o ${m} | wc -w | xargs)" == "0" ] ; then + if [ "$(echo "${EXCLUSIONS[@]}" | grep -o ${m} | wc -w | xargs)" == "0" ] ; then envsubst< ${m} | ${OPERATOR_EXEC} apply ${namespace:-} --validate=false -f - fi done diff --git a/hack/env.sh b/hack/env.sh index 28e0007e7..fa6e28580 100755 --- a/hack/env.sh +++ b/hack/env.sh @@ -1,3 +1,5 @@ +#!/bin/bash + if [ -z $SKIP_VAR_SET ]; then export SRIOV_CNI_IMAGE=${SRIOV_CNI_IMAGE:-ghcr.io/k8snetworkplumbingwg/sriov-cni} export SRIOV_INFINIBAND_CNI_IMAGE=${SRIOV_INFINIBAND_CNI_IMAGE:-ghcr.io/k8snetworkplumbingwg/ib-sriov-cni} @@ -12,22 +14,27 @@ if [ -z $SKIP_VAR_SET ]; then export METRICS_EXPORTER_IMAGE=${METRICS_EXPORTER_IMAGE:-ghcr.io/k8snetworkplumbingwg/sriov-network-metrics-exporter} export SRIOV_NETWORK_OPERATOR_IMAGE=${SRIOV_NETWORK_OPERATOR_IMAGE:-ghcr.io/k8snetworkplumbingwg/sriov-network-operator} export METRICS_EXPORTER_KUBE_RBAC_PROXY_IMAGE=${METRICS_EXPORTER_KUBE_RBAC_PROXY_IMAGE:-gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0} + fail_msg_detect="is empty and failed to detect" else - # ensure that OVS_CNI_IMAGE is set, empty string is a valid value - OVS_CNI_IMAGE=${OVS_CNI_IMAGE:-} - # ensure that RDMA_CNI_IMAGE is set, empty string is a valid value - RDMA_CNI_IMAGE=${$RDMA_CNI_IMAGE:-} - METRICS_EXPORTER_KUBE_RBAC_PROXY_IMAGE=${METRICS_EXPORTER_KUBE_RBAC_PROXY_IMAGE:-} - [ -z $SRIOV_CNI_IMAGE ] && echo "SRIOV_CNI_IMAGE is empty but SKIP_VAR_SET is set" && exit 1 - [ -z $SRIOV_INFINIBAND_CNI_IMAGE ] && echo "SRIOV_INFINIBAND_CNI_IMAGE is empty but SKIP_VAR_SET is set" && exit 1 - [ -z $SRIOV_DEVICE_PLUGIN_IMAGE ] && echo "SRIOV_DEVICE_PLUGIN_IMAGE is empty but SKIP_VAR_SET is set" && exit 1 - [ -z $NETWORK_RESOURCES_INJECTOR_IMAGE ] && echo "NETWORK_RESOURCES_INJECTOR_IMAGE is empty but SKIP_VAR_SET is set" && exit 1 - [ -z $SRIOV_NETWORK_CONFIG_DAEMON_IMAGE ] && echo "SRIOV_NETWORK_CONFIG_DAEMON_IMAGE is empty but SKIP_VAR_SET is set" && exit 1 - [ -z $SRIOV_NETWORK_WEBHOOK_IMAGE ] && echo "SRIOV_NETWORK_WEBHOOK_IMAGE is empty but SKIP_VAR_SET is set" && exit 1 - [ -z $METRICS_EXPORTER_IMAGE ] && echo "METRICS_EXPORTER_IMAGE is empty but SKIP_VAR_SET is set" && exit 1 - [ -z $SRIOV_NETWORK_OPERATOR_IMAGE ] && echo "SRIOV_NETWORK_OPERATOR_IMAGE is empty but SKIP_VAR_SET is set" && exit 1 + fail_msg_detect="is empty but SKIP_VAR_SET is set" fi +# ensure that OVS_CNI_IMAGE is set, empty string is a valid value +OVS_CNI_IMAGE=${OVS_CNI_IMAGE:-} +# ensure that RDMA_CNI_IMAGE is set, empty string is a valid value +RDMA_CNI_IMAGE=${RDMA_CNI_IMAGE:-} +METRICS_EXPORTER_KUBE_RBAC_PROXY_IMAGE=${METRICS_EXPORTER_KUBE_RBAC_PROXY_IMAGE:-} +[ -z $SRIOV_CNI_IMAGE ] && echo "SRIOV_CNI_IMAGE $fail_msg_detect" && exit 1 +[ -z $SRIOV_INFINIBAND_CNI_IMAGE ] && echo "SRIOV_INFINIBAND_CNI_IMAGE $fail_msg_detect" && exit 1 +[ -z $SRIOV_DEVICE_PLUGIN_IMAGE ] && echo "SRIOV_DEVICE_PLUGIN_IMAGE $fail_msg_detect" && exit 1 +[ -z $NETWORK_RESOURCES_INJECTOR_IMAGE ] && echo "NETWORK_RESOURCES_INJECTOR_IMAGE $fail_msg_detect" && exit 1 +[ -z $SRIOV_NETWORK_CONFIG_DAEMON_IMAGE ] && echo "SRIOV_NETWORK_CONFIG_DAEMON_IMAGE $fail_msg_detect" && exit 1 +[ -z $SRIOV_NETWORK_WEBHOOK_IMAGE ] && echo "SRIOV_NETWORK_WEBHOOK_IMAGE $fail_msg_detect" && exit 1 +[ -z $METRICS_EXPORTER_IMAGE ] && echo "METRICS_EXPORTER_IMAGE $fail_msg_detect" && exit 1 +[ -z $SRIOV_NETWORK_OPERATOR_IMAGE ] && echo "SRIOV_NETWORK_OPERATOR_IMAGE $fail_msg_detect" && exit 1 + +unset fail_msg_detect + set -x export RELEASE_VERSION=4.7.0 @@ -42,6 +49,5 @@ export ADMISSION_CONTROLLERS_CERTIFICATES_CERT_MANAGER_ENABLED=${ADMISSION_CONTR export ADMISSION_CONTROLLERS_CERTIFICATES_OPERATOR_CA_CRT=${ADMISSION_CONTROLLERS_CERTIFICATES_OPERATOR_CA_CRT:-""} export ADMISSION_CONTROLLERS_CERTIFICATES_INJECTOR_CA_CRT=${ADMISSION_CONTROLLERS_CERTIFICATES_INJECTOR_CA_CRT:-""} export DEV_MODE=${DEV_MODE:-"FALSE"} -export OPERATOR_LEADER_ELECTION_ENABLE=${OPERATOR_LEADER_ELECTION_ENABLE:-"false"} export METRICS_EXPORTER_SECRET_NAME=${METRICS_EXPORTER_SECRET_NAME:-"metrics-exporter-cert"} export METRICS_EXPORTER_PORT=${METRICS_EXPORTER_PORT:-"9110"} diff --git a/hack/run-e2e-conformance-virtual-cluster.sh b/hack/run-e2e-conformance-virtual-cluster.sh index 1a75a280d..353c8528d 100755 --- a/hack/run-e2e-conformance-virtual-cluster.sh +++ b/hack/run-e2e-conformance-virtual-cluster.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -xeo pipefail -cluster_version=${CLUSTER_VERSION:-1.29.3} +cluster_version=${CLUSTER_VERSION:-1.32.0} cluster_name=${CLUSTER_NAME:-virtual} domain_name=$cluster_name.lab @@ -196,6 +196,22 @@ WantedBy=default.target' > /etc/systemd/system/disable-offload.service systemctl daemon-reload systemctl enable --now disable-offload +echo '[Unit] +Description=load br_netfilter +After=network.target + +[Service] +Type=oneshot +ExecStart=/usr/bin/bash -c "modprobe br_netfilter" +StandardOutput=journal+console +StandardError=journal+console + +[Install] +WantedBy=default.target' > /etc/systemd/system/load-br-netfilter.service + +systemctl daemon-reload +systemctl enable --now load-br-netfilter + systemctl restart NetworkManager EOF diff --git a/hack/run-e2e-conformance-virtual-ocp.sh b/hack/run-e2e-conformance-virtual-ocp.sh index a61906fb2..ed25ab55f 100755 --- a/hack/run-e2e-conformance-virtual-ocp.sh +++ b/hack/run-e2e-conformance-virtual-ocp.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -xeo pipefail -OCP_VERSION=${OCP_VERSION:-4.16.0} +OCP_VERSION=${OCP_VERSION:-4.18} +OCP_RELEASE_TYPE=${OCP_RELEASE_TYPE:-stable} cluster_name=${CLUSTER_NAME:-ocp-virt} domain_name=lab @@ -52,6 +53,7 @@ kcli create network -c 192.168.123.0/24 ocp kcli create network -c 192.168.${virtual_router_id}.0/24 --nodhcp -i $cluster_name cat < ./${cluster_name}-plan.yaml +version: $OCP_RELEASE_TYPE tag: $OCP_VERSION ctlplane_memory: 32768 worker_memory: 8192 @@ -189,10 +191,10 @@ export OPERATOR_EXEC=kubectl export CLUSTER_TYPE=openshift export DEV_MODE=TRUE export CLUSTER_HAS_EMULATED_PF=TRUE -export OPERATOR_LEADER_ELECTION_ENABLE=true export METRICS_EXPORTER_PROMETHEUS_OPERATOR_ENABLED=true +export METRICS_EXPORTER_PROMETHEUS_DEPLOY_RULES=true export METRICS_EXPORTER_PROMETHEUS_OPERATOR_SERVICE_ACCOUNT=${METRICS_EXPORTER_PROMETHEUS_OPERATOR_SERVICE_ACCOUNT:-"prometheus-k8s"} -export METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE=${METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE:-"openshfit-monitoring"} +export METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE=${METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE:-"openshift-monitoring"} export SRIOV_NETWORK_OPERATOR_IMAGE="$registry/$NAMESPACE/sriov-network-operator:latest" export SRIOV_NETWORK_CONFIG_DAEMON_IMAGE="$registry/$NAMESPACE/sriov-network-config-daemon:latest" @@ -215,7 +217,7 @@ DELAY_SECONDS=10 retries=0 until [ $retries -ge $MAX_RETRIES ]; do # wait for all the openshift cluster operators to be running - if [ $(kubectl get clusteroperator --no-headers | awk '{print $3}' | grep True | wc -l) -eq 33 ]; then + if [ $(kubectl get clusteroperator --no-headers | awk '{print $3}' | grep -v True | wc -l) -eq 0 ]; then break fi retries=$((retries+1)) @@ -275,7 +277,7 @@ podman rmi -fi ${SRIOV_NETWORK_WEBHOOK_IMAGE} podman logout $registry echo "## apply CRDs" -kubectl apply -k $root/config/crd +kubectl apply -f $root/config/crd/bases cat <:,. This flag can be repeated to specify done return_interfaces_to_default_namespace(){ - for netns in ${netnses[@]};do + for netns in "${netnses[@]}";do for pf in ${pfs[$netns]};do return_interface_to_default_namespace "${netns}" "${pf}" done @@ -277,19 +277,20 @@ switch_interface_vf_representors(){ return 0 fi - for interface in $(ls /sys/class/net);do - phys_switch_id=$(cat /sys/class/net/$interface/phys_switch_id) + for interface in /sys/class/net/*;do + phys_switch_id=$(cat $interface/phys_switch_id) if [[ "$phys_switch_id" != "${pf_switch_ids[$pf_name]}" ]]; then continue fi - phys_port_name=$(cat /sys/class/net/$interface/phys_port_name) + phys_port_name=$(cat $interface/phys_port_name) phys_port_name_pf_index=${phys_port_name%vf*} phys_port_name_pf_index=${phys_port_name_pf_index#pf} if [[ "$phys_port_name_pf_index" != "${pf_port_names[$pf_name]:1}" ]]; then continue fi - echo "Switching VF representor $interface of PF $pf_name to netns $worker_netns" - switch_vf $interface $worker_netns + interface_name=${interface##*/} + echo "Switching VF representor $interface_name of PF $pf_name to netns $worker_netns" + switch_vf $interface_name $worker_netns done } @@ -348,7 +349,7 @@ variables_check(){ check_empty_var(){ local var_name="$1" - if [[ -z "${!var_name[@]}" ]];then + if [[ -z "${!var_name[*]}" ]];then echo "Error: $var_name is empty..." return 1 fi @@ -360,7 +361,7 @@ main(){ trap return_interfaces_to_default_namespace INT EXIT TERM while true;do - for netns in ${netnses[@]};do + for netns in "${netnses[@]}";do switch_pfs "$netns" "${pfs[$netns]}" sleep 2 switch_netns_vfs "$netns" @@ -388,7 +389,7 @@ if [[ "$status" != "0" ]];then exit $status fi -for netns in ${netnses[@]};do +for netns in "${netnses[@]}";do netns_create "$netns" let status=$status+$? if [[ "$status" != "0" ]];then @@ -397,13 +398,13 @@ for netns in ${netnses[@]};do fi done -for netns in ${netnses[@]};do +for netns in "${netnses[@]}";do get_pcis_from_pfs "$netns" "${pfs[$netns]}" get_pf_switch_dev_info "$netns" "${pfs[$netns]}" done if [[ "${#pcis[@]}" == "0" ]];then - echo "Error: could not get pci addresses of interfaces ${pfs[@]}!!" + echo "Error: could not get pci addresses of interfaces ${pfs[*]}!!" exit 1 fi diff --git a/hack/virtual-cluster-redeploy.sh b/hack/virtual-cluster-redeploy.sh index a16aeeaf4..29ca958bc 100755 --- a/hack/virtual-cluster-redeploy.sh +++ b/hack/virtual-cluster-redeploy.sh @@ -25,9 +25,23 @@ if [ $CLUSTER_TYPE == "openshift" ]; then # dockercfg password is in the form `:password`. We need to trim the `:` prefix pass=${pass#":"} - registry="default-route-openshift-image-registry.apps.${cluster_name}.${domain_name}" + registry=`kubectl -n openshift-image-registry get route --no-headers | awk '{print $2}'` podman login -u serviceaccount -p ${pass} $registry --tls-verify=false + export ADMISSION_CONTROLLERS_ENABLED=true + export SKIP_VAR_SET="" + export NAMESPACE="openshift-sriov-network-operator" + export OPERATOR_NAMESPACE=$NAMESPACE + export MULTUS_NAMESPACE="openshift-multus" + export OPERATOR_EXEC=kubectl + export CLUSTER_TYPE=openshift + export DEV_MODE=TRUE + export CLUSTER_HAS_EMULATED_PF=TRUE + export METRICS_EXPORTER_PROMETHEUS_OPERATOR_ENABLED=true + export METRICS_EXPORTER_PROMETHEUS_DEPLOY_RULES=true + export METRICS_EXPORTER_PROMETHEUS_OPERATOR_SERVICE_ACCOUNT=${METRICS_EXPORTER_PROMETHEUS_OPERATOR_SERVICE_ACCOUNT:-"prometheus-k8s"} + export METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE=${METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE:-"openshift-monitoring"} + export SRIOV_NETWORK_OPERATOR_IMAGE="$registry/$NAMESPACE/sriov-network-operator:latest" export SRIOV_NETWORK_CONFIG_DAEMON_IMAGE="$registry/$NAMESPACE/sriov-network-config-daemon:latest" export SRIOV_NETWORK_WEBHOOK_IMAGE="$registry/$NAMESPACE/sriov-network-operator-webhook:latest" @@ -48,7 +62,6 @@ else fi export ADMISSION_CONTROLLERS_ENABLED=true -export OPERATOR_LEADER_ELECTION_ENABLE=true export SKIP_VAR_SET="" export OPERATOR_NAMESPACE=$NAMESPACE export OPERATOR_EXEC=kubectl @@ -72,6 +85,10 @@ if [ $CLUSTER_TYPE == "openshift" ]; then export SRIOV_NETWORK_OPERATOR_IMAGE="image-registry.openshift-image-registry.svc:5000/$NAMESPACE/sriov-network-operator:latest" export SRIOV_NETWORK_CONFIG_DAEMON_IMAGE="image-registry.openshift-image-registry.svc:5000/$NAMESPACE/sriov-network-config-daemon:latest" export SRIOV_NETWORK_WEBHOOK_IMAGE="image-registry.openshift-image-registry.svc:5000/$NAMESPACE/sriov-network-operator-webhook:latest" + + echo "## apply CRDs" + kubectl apply -k $root/config/crd + echo "## deploying SRIOV Network Operator" hack/deploy-setup.sh $NAMESPACE else diff --git a/main.go b/main.go index 7195c59c3..6aaa11d04 100644 --- a/main.go +++ b/main.go @@ -25,7 +25,7 @@ import ( // to ensure that exec-entrypoint and run can make use of them. netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" openshiftconfigv1 "github.com/openshift/api/config/v1" - mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + mcfgv1 "github.com/openshift/api/machineconfiguration/v1" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -48,9 +48,6 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" "github.com/k8snetworkplumbingwg/sriov-network-operator/controllers" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/featuregate" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/leaderelection" - - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" @@ -75,69 +72,43 @@ func init() { func main() { var metricsAddr string - var enableLeaderElection bool var probeAddr string flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - flag.BoolVar(&enableLeaderElection, "leader-elect", false, - "Enable leader election for controller manager. "+ - "Enabling this will ensure there is only one active controller manager.") + snolog.BindFlags(flag.CommandLine) flag.Parse() snolog.InitLog() restConfig := ctrl.GetConfigOrDie() - kubeClient, err := client.New(restConfig, client.Options{Scheme: scheme}) - if err != nil { - setupLog.Error(err, "couldn't create client") - os.Exit(1) - } - if vars.ResourcePrefix == "" { setupLog.Error(nil, "RESOURCE_PREFIX environment variable can't be empty") os.Exit(1) } - le := leaderelection.GetLeaderElectionConfig(kubeClient, enableLeaderElection) - - leaderElectionMgr, err := ctrl.NewManager(restConfig, ctrl.Options{ - Scheme: scheme, - HealthProbeBindAddress: probeAddr, - Metrics: server.Options{BindAddress: "0"}, - LeaderElection: enableLeaderElection, - LeaseDuration: &le.LeaseDuration, - LeaderElectionReleaseOnCancel: true, - RenewDeadline: &le.RenewDeadline, - RetryPeriod: &le.RetryPeriod, - LeaderElectionID: consts.LeaderElectionID, + mgr, err := ctrl.NewManager(restConfig, ctrl.Options{ + Scheme: scheme, + HealthProbeBindAddress: probeAddr, + Metrics: server.Options{BindAddress: metricsAddr}, + WebhookServer: webhook.NewServer(webhook.Options{Port: 9443}), + Cache: cache.Options{DefaultNamespaces: map[string]cache.Config{vars.Namespace: {}}}, }) if err != nil { - setupLog.Error(err, "unable to start leader election manager") + setupLog.Error(err, "unable to create manager") os.Exit(1) } - if err := leaderElectionMgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) } - if err := leaderElectionMgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up ready check") os.Exit(1) } - mgr, err := ctrl.NewManager(restConfig, ctrl.Options{ - Scheme: scheme, - Metrics: server.Options{BindAddress: metricsAddr}, - WebhookServer: webhook.NewServer(webhook.Options{Port: 9443}), - Cache: cache.Options{DefaultNamespaces: map[string]cache.Config{vars.Namespace: {}}}, - }) - if err != nil { - setupLog.Error(err, "unable to start manager") - os.Exit(1) - } - mgrGlobal, err := ctrl.NewManager(restConfig, ctrl.Options{ Scheme: scheme, Metrics: server.Options{BindAddress: "0"}, @@ -166,7 +137,6 @@ func main() { err = mgrGlobal.GetCache().IndexField(context.Background(), &sriovnetworkv1.OVSNetwork{}, "spec.networkNamespace", func(o client.Object) []string { return []string{o.(*sriovnetworkv1.OVSNetwork).Spec.NetworkNamespace} }) - if err != nil { setupLog.Error(err, "unable to create index field for cache") os.Exit(1) @@ -219,10 +189,11 @@ func main() { os.Exit(1) } if err = (&controllers.SriovOperatorConfigReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - PlatformHelper: platformsHelper, - FeatureGate: featureGate, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + PlatformHelper: platformsHelper, + FeatureGate: featureGate, + UncachedAPIReader: mgr.GetAPIReader(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "SriovOperatorConfig") os.Exit(1) @@ -267,22 +238,6 @@ func main() { } // +kubebuilder:scaffold:builder - leaderElectionErr := make(chan error) - leaderElectionContext, cancelLeaderElection := context.WithCancel(context.Background()) - go func() { - setupLog.Info("starting leader election manager") - leaderElectionErr <- leaderElectionMgr.Start(leaderElectionContext) - }() - - select { - case <-leaderElectionMgr.Elected(): - case err := <-leaderElectionErr: - setupLog.Error(err, "Leader Election Manager error") - os.Exit(1) - } - - setupLog.Info("acquired lease") - stopSignalCh := ctrl.SetupSignalHandler() globalManagerErr := make(chan error) @@ -299,53 +254,42 @@ func main() { namespacedManagerErr <- mgr.Start(namespacedManagerCtx) }() + shutdownClient, err := client.New(restConfig, client.Options{ + Scheme: vars.Scheme, + Cache: &client.CacheOptions{ + DisableFor: []client.Object{ + &sriovnetworkv1.SriovNetwork{}, + }, + }, + }) + if err != nil { + setupLog.Error(err, "unable to create generic client for shutdown process") + os.Exit(1) + } + select { // Wait for a stop signal case <-stopSignalCh.Done(): setupLog.Info("Stop signal received") - globalManagerCancel() namespacedManagerCancel() <-globalManagerErr <-namespacedManagerErr - - utils.Shutdown() - - cancelLeaderElection() - <-leaderElectionErr - - case err := <-leaderElectionErr: - setupLog.Error(err, "Leader Election Manager error") - globalManagerCancel() - namespacedManagerCancel() - <-globalManagerErr - <-namespacedManagerErr - - os.Exit(1) + utils.Shutdown(shutdownClient) case err := <-globalManagerErr: setupLog.Error(err, "Global Manager error") - namespacedManagerCancel() <-namespacedManagerErr - - utils.Shutdown() - - cancelLeaderElection() - <-leaderElectionErr + utils.Shutdown(shutdownClient) os.Exit(1) case err := <-namespacedManagerErr: setupLog.Error(err, "Namsepaced Manager error") - globalManagerCancel() <-globalManagerErr - - utils.Shutdown() - - cancelLeaderElection() - <-leaderElectionErr + utils.Shutdown(shutdownClient) os.Exit(1) } diff --git a/pkg/apply/merge.go b/pkg/apply/merge.go index d2ed6d4fb..9ee859f83 100644 --- a/pkg/apply/merge.go +++ b/pkg/apply/merge.go @@ -220,7 +220,7 @@ func mergeAnnotations(current, updated *uns.Unstructured) { for k, v := range updatedAnnotations { curAnnotations[k] = v } - if len(curAnnotations) > 1 { + if len(curAnnotations) > 0 { updated.SetAnnotations(curAnnotations) } } @@ -238,7 +238,7 @@ func mergeLabels(current, updated *uns.Unstructured) { for k, v := range updatedLabels { curLabels[k] = v } - if len(curLabels) > 1 { + if len(curLabels) > 0 { updated.SetLabels(curLabels) } } diff --git a/pkg/apply/merge_test.go b/pkg/apply/merge_test.go index f6ad89289..ecf2fd98d 100644 --- a/pkg/apply/merge_test.go +++ b/pkg/apply/merge_test.go @@ -107,6 +107,38 @@ metadata: })) } +func TestMergeOne(t *testing.T) { + g := NewGomegaWithT(t) + + cur := UnstructuredFromYaml(t, ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: d1 + labels: + label-c: cur + annotations: + annotation-c: cur`) + + upd := UnstructuredFromYaml(t, ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: d1`) + + // this mutates updated + err := MergeObjectForUpdate(cur, upd) + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(upd.GetLabels()).To(Equal(map[string]string{ + "label-c": "cur", + })) + + g.Expect(upd.GetAnnotations()).To(Equal(map[string]string{ + "annotation-c": "cur", + })) +} + func TestMergeNilCur(t *testing.T) { g := NewGomegaWithT(t) diff --git a/pkg/client/clientset/versioned/clientset.go b/pkg/client/clientset/versioned/clientset.go deleted file mode 100644 index 5b09d09c0..000000000 --- a/pkg/client/clientset/versioned/clientset.go +++ /dev/null @@ -1,81 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -package versioned - -import ( - "fmt" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned/typed/sriovnetwork/v1" - discovery "k8s.io/client-go/discovery" - rest "k8s.io/client-go/rest" - flowcontrol "k8s.io/client-go/util/flowcontrol" -) - -type Interface interface { - Discovery() discovery.DiscoveryInterface - SriovnetworkV1() sriovnetworkv1.SriovnetworkV1Interface -} - -// Clientset contains the clients for groups. Each group has exactly one -// version included in a Clientset. -type Clientset struct { - *discovery.DiscoveryClient - sriovnetworkV1 *sriovnetworkv1.SriovnetworkV1Client -} - -// SriovnetworkV1 retrieves the SriovnetworkV1Client -func (c *Clientset) SriovnetworkV1() sriovnetworkv1.SriovnetworkV1Interface { - return c.sriovnetworkV1 -} - -// Discovery retrieves the DiscoveryClient -func (c *Clientset) Discovery() discovery.DiscoveryInterface { - if c == nil { - return nil - } - return c.DiscoveryClient -} - -// NewForConfig creates a new Clientset for the given config. -// If config's RateLimiter is not set and QPS and Burst are acceptable, -// NewForConfig will generate a rate-limiter in configShallowCopy. -func NewForConfig(c *rest.Config) (*Clientset, error) { - configShallowCopy := *c - if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { - if configShallowCopy.Burst <= 0 { - return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") - } - configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) - } - var cs Clientset - var err error - cs.sriovnetworkV1, err = sriovnetworkv1.NewForConfig(&configShallowCopy) - if err != nil { - return nil, err - } - - cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy) - if err != nil { - return nil, err - } - return &cs, nil -} - -// NewForConfigOrDie creates a new Clientset for the given config and -// panics if there is an error in the config. -func NewForConfigOrDie(c *rest.Config) *Clientset { - var cs Clientset - cs.sriovnetworkV1 = sriovnetworkv1.NewForConfigOrDie(c) - - cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c) - return &cs -} - -// New creates a new Clientset for the given RESTClient. -func New(c rest.Interface) *Clientset { - var cs Clientset - cs.sriovnetworkV1 = sriovnetworkv1.New(c) - - cs.DiscoveryClient = discovery.NewDiscoveryClient(c) - return &cs -} diff --git a/pkg/client/clientset/versioned/doc.go b/pkg/client/clientset/versioned/doc.go deleted file mode 100644 index 0e0c2a890..000000000 --- a/pkg/client/clientset/versioned/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -// This package has the automatically generated clientset. -package versioned diff --git a/pkg/client/clientset/versioned/fake/clientset_generated.go b/pkg/client/clientset/versioned/fake/clientset_generated.go deleted file mode 100644 index 5747b4d6a..000000000 --- a/pkg/client/clientset/versioned/fake/clientset_generated.go +++ /dev/null @@ -1,66 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -package fake - -import ( - clientset "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned/typed/sriovnetwork/v1" - fakesriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned/typed/sriovnetwork/v1/fake" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/discovery" - fakediscovery "k8s.io/client-go/discovery/fake" - "k8s.io/client-go/testing" -) - -// NewSimpleClientset returns a clientset that will respond with the provided objects. -// It's backed by a very simple object tracker that processes creates, updates and deletions as-is, -// without applying any validations and/or defaults. It shouldn't be considered a replacement -// for a real clientset and is mostly useful in simple unit tests. -func NewSimpleClientset(objects ...runtime.Object) *Clientset { - o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) - for _, obj := range objects { - if err := o.Add(obj); err != nil { - panic(err) - } - } - - cs := &Clientset{tracker: o} - cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} - cs.AddReactor("*", "*", testing.ObjectReaction(o)) - cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { - gvr := action.GetResource() - ns := action.GetNamespace() - watch, err := o.Watch(gvr, ns) - if err != nil { - return false, nil, err - } - return true, watch, nil - }) - - return cs -} - -// Clientset implements clientset.Interface. Meant to be embedded into a -// struct to get a default implementation. This makes faking out just the method -// you want to test easier. -type Clientset struct { - testing.Fake - discovery *fakediscovery.FakeDiscovery - tracker testing.ObjectTracker -} - -func (c *Clientset) Discovery() discovery.DiscoveryInterface { - return c.discovery -} - -func (c *Clientset) Tracker() testing.ObjectTracker { - return c.tracker -} - -var _ clientset.Interface = &Clientset{} - -// SriovnetworkV1 retrieves the SriovnetworkV1Client -func (c *Clientset) SriovnetworkV1() sriovnetworkv1.SriovnetworkV1Interface { - return &fakesriovnetworkv1.FakeSriovnetworkV1{Fake: &c.Fake} -} diff --git a/pkg/client/clientset/versioned/fake/doc.go b/pkg/client/clientset/versioned/fake/doc.go deleted file mode 100644 index 3630ed1cd..000000000 --- a/pkg/client/clientset/versioned/fake/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -// This package has the automatically generated fake clientset. -package fake diff --git a/pkg/client/clientset/versioned/fake/register.go b/pkg/client/clientset/versioned/fake/register.go deleted file mode 100644 index 60ee7df8f..000000000 --- a/pkg/client/clientset/versioned/fake/register.go +++ /dev/null @@ -1,40 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -package fake - -import ( - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - schema "k8s.io/apimachinery/pkg/runtime/schema" - serializer "k8s.io/apimachinery/pkg/runtime/serializer" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" -) - -var scheme = runtime.NewScheme() -var codecs = serializer.NewCodecFactory(scheme) - -var localSchemeBuilder = runtime.SchemeBuilder{ - sriovnetworkv1.AddToScheme, -} - -// AddToScheme adds all types of this clientset into the given scheme. This allows composition -// of clientsets, like in: -// -// import ( -// "k8s.io/client-go/kubernetes" -// clientsetscheme "k8s.io/client-go/kubernetes/scheme" -// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" -// ) -// -// kclientset, _ := kubernetes.NewForConfig(c) -// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) -// -// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types -// correctly. -var AddToScheme = localSchemeBuilder.AddToScheme - -func init() { - v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) - utilruntime.Must(AddToScheme(scheme)) -} diff --git a/pkg/client/clientset/versioned/scheme/doc.go b/pkg/client/clientset/versioned/scheme/doc.go deleted file mode 100644 index 14db57a58..000000000 --- a/pkg/client/clientset/versioned/scheme/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -// This package contains the scheme of the automatically generated clientset. -package scheme diff --git a/pkg/client/clientset/versioned/scheme/register.go b/pkg/client/clientset/versioned/scheme/register.go deleted file mode 100644 index 7ac435d5c..000000000 --- a/pkg/client/clientset/versioned/scheme/register.go +++ /dev/null @@ -1,40 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -package scheme - -import ( - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - schema "k8s.io/apimachinery/pkg/runtime/schema" - serializer "k8s.io/apimachinery/pkg/runtime/serializer" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" -) - -var Scheme = runtime.NewScheme() -var Codecs = serializer.NewCodecFactory(Scheme) -var ParameterCodec = runtime.NewParameterCodec(Scheme) -var localSchemeBuilder = runtime.SchemeBuilder{ - sriovnetworkv1.AddToScheme, -} - -// AddToScheme adds all types of this clientset into the given scheme. This allows composition -// of clientsets, like in: -// -// import ( -// "k8s.io/client-go/kubernetes" -// clientsetscheme "k8s.io/client-go/kubernetes/scheme" -// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" -// ) -// -// kclientset, _ := kubernetes.NewForConfig(c) -// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) -// -// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types -// correctly. -var AddToScheme = localSchemeBuilder.AddToScheme - -func init() { - v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) - utilruntime.Must(AddToScheme(Scheme)) -} diff --git a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/doc.go b/pkg/client/clientset/versioned/typed/sriovnetwork/v1/doc.go deleted file mode 100644 index 225e6b2be..000000000 --- a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -// This package has the automatically generated typed clients. -package v1 diff --git a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/fake/doc.go b/pkg/client/clientset/versioned/typed/sriovnetwork/v1/fake/doc.go deleted file mode 100644 index 2b5ba4c8e..000000000 --- a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/fake/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -// Package fake has the automatically generated clients. -package fake diff --git a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/fake/fake_sriovnetwork.go b/pkg/client/clientset/versioned/typed/sriovnetwork/v1/fake/fake_sriovnetwork.go deleted file mode 100644 index 7c42e8a3f..000000000 --- a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/fake/fake_sriovnetwork.go +++ /dev/null @@ -1,126 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -package fake - -import ( - "context" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - labels "k8s.io/apimachinery/pkg/labels" - schema "k8s.io/apimachinery/pkg/runtime/schema" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - testing "k8s.io/client-go/testing" -) - -// FakeSriovNetworks implements SriovNetworkInterface -type FakeSriovNetworks struct { - Fake *FakeSriovnetworkV1 - ns string -} - -var sriovnetworksResource = schema.GroupVersionResource{Group: "sriovnetwork.openshift.io", Version: "v1", Resource: "sriovnetworks"} - -var sriovnetworksKind = schema.GroupVersionKind{Group: "sriovnetwork.openshift.io", Version: "v1", Kind: "SriovNetwork"} - -// Get takes name of the sriovNetwork, and returns the corresponding sriovNetwork object, and an error if there is any. -func (c *FakeSriovNetworks) Get(ctx context.Context, name string, options v1.GetOptions) (result *sriovnetworkv1.SriovNetwork, err error) { - obj, err := c.Fake. - Invokes(testing.NewGetAction(sriovnetworksResource, c.ns, name), &sriovnetworkv1.SriovNetwork{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovNetwork), err -} - -// List takes label and field selectors, and returns the list of SriovNetworks that match those selectors. -func (c *FakeSriovNetworks) List(ctx context.Context, opts v1.ListOptions) (result *sriovnetworkv1.SriovNetworkList, err error) { - obj, err := c.Fake. - Invokes(testing.NewListAction(sriovnetworksResource, sriovnetworksKind, c.ns, opts), &sriovnetworkv1.SriovNetworkList{}) - - if obj == nil { - return nil, err - } - - label, _, _ := testing.ExtractFromListOptions(opts) - if label == nil { - label = labels.Everything() - } - list := &sriovnetworkv1.SriovNetworkList{ListMeta: obj.(*sriovnetworkv1.SriovNetworkList).ListMeta} - for _, item := range obj.(*sriovnetworkv1.SriovNetworkList).Items { - if label.Matches(labels.Set(item.Labels)) { - list.Items = append(list.Items, item) - } - } - return list, err -} - -// Watch returns a watch.Interface that watches the requested sriovNetworks. -func (c *FakeSriovNetworks) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - return c.Fake. - InvokesWatch(testing.NewWatchAction(sriovnetworksResource, c.ns, opts)) - -} - -// Create takes the representation of a sriovNetwork and creates it. Returns the server's representation of the sriovNetwork, and an error, if there is any. -func (c *FakeSriovNetworks) Create(ctx context.Context, sriovNetwork *sriovnetworkv1.SriovNetwork, opts v1.CreateOptions) (result *sriovnetworkv1.SriovNetwork, err error) { - obj, err := c.Fake. - Invokes(testing.NewCreateAction(sriovnetworksResource, c.ns, sriovNetwork), &sriovnetworkv1.SriovNetwork{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovNetwork), err -} - -// Update takes the representation of a sriovNetwork and updates it. Returns the server's representation of the sriovNetwork, and an error, if there is any. -func (c *FakeSriovNetworks) Update(ctx context.Context, sriovNetwork *sriovnetworkv1.SriovNetwork, opts v1.UpdateOptions) (result *sriovnetworkv1.SriovNetwork, err error) { - obj, err := c.Fake. - Invokes(testing.NewUpdateAction(sriovnetworksResource, c.ns, sriovNetwork), &sriovnetworkv1.SriovNetwork{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovNetwork), err -} - -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *FakeSriovNetworks) UpdateStatus(ctx context.Context, sriovNetwork *sriovnetworkv1.SriovNetwork, opts v1.UpdateOptions) (*sriovnetworkv1.SriovNetwork, error) { - obj, err := c.Fake. - Invokes(testing.NewUpdateSubresourceAction(sriovnetworksResource, "status", c.ns, sriovNetwork), &sriovnetworkv1.SriovNetwork{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovNetwork), err -} - -// Delete takes name of the sriovNetwork and deletes it. Returns an error if one occurs. -func (c *FakeSriovNetworks) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - _, err := c.Fake. - Invokes(testing.NewDeleteAction(sriovnetworksResource, c.ns, name), &sriovnetworkv1.SriovNetwork{}) - - return err -} - -// DeleteCollection deletes a collection of objects. -func (c *FakeSriovNetworks) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - action := testing.NewDeleteCollectionAction(sriovnetworksResource, c.ns, listOpts) - - _, err := c.Fake.Invokes(action, &sriovnetworkv1.SriovNetworkList{}) - return err -} - -// Patch applies the patch and returns the patched sriovNetwork. -func (c *FakeSriovNetworks) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *sriovnetworkv1.SriovNetwork, err error) { - obj, err := c.Fake. - Invokes(testing.NewPatchSubresourceAction(sriovnetworksResource, c.ns, name, pt, data, subresources...), &sriovnetworkv1.SriovNetwork{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovNetwork), err -} diff --git a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/fake/fake_sriovnetwork_client.go b/pkg/client/clientset/versioned/typed/sriovnetwork/v1/fake/fake_sriovnetwork_client.go deleted file mode 100644 index a917e895f..000000000 --- a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/fake/fake_sriovnetwork_client.go +++ /dev/null @@ -1,36 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -package fake - -import ( - v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned/typed/sriovnetwork/v1" - rest "k8s.io/client-go/rest" - testing "k8s.io/client-go/testing" -) - -type FakeSriovnetworkV1 struct { - *testing.Fake -} - -func (c *FakeSriovnetworkV1) SriovNetworks(namespace string) v1.SriovNetworkInterface { - return &FakeSriovNetworks{c, namespace} -} - -func (c *FakeSriovnetworkV1) SriovNetworkNodePolicies(namespace string) v1.SriovNetworkNodePolicyInterface { - return &FakeSriovNetworkNodePolicies{c, namespace} -} - -func (c *FakeSriovnetworkV1) SriovNetworkNodeStates(namespace string) v1.SriovNetworkNodeStateInterface { - return &FakeSriovNetworkNodeStates{c, namespace} -} - -func (c *FakeSriovnetworkV1) SriovOperatorConfigs(namespace string) v1.SriovOperatorConfigInterface { - return &FakeSriovOperatorConfigs{c, namespace} -} - -// RESTClient returns a RESTClient that is used to communicate -// with API server by this client implementation. -func (c *FakeSriovnetworkV1) RESTClient() rest.Interface { - var ret *rest.RESTClient - return ret -} diff --git a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/fake/fake_sriovnetworknodepolicy.go b/pkg/client/clientset/versioned/typed/sriovnetwork/v1/fake/fake_sriovnetworknodepolicy.go deleted file mode 100644 index ab3628f34..000000000 --- a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/fake/fake_sriovnetworknodepolicy.go +++ /dev/null @@ -1,126 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -package fake - -import ( - "context" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - labels "k8s.io/apimachinery/pkg/labels" - schema "k8s.io/apimachinery/pkg/runtime/schema" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - testing "k8s.io/client-go/testing" -) - -// FakeSriovNetworkNodePolicies implements SriovNetworkNodePolicyInterface -type FakeSriovNetworkNodePolicies struct { - Fake *FakeSriovnetworkV1 - ns string -} - -var sriovnetworknodepoliciesResource = schema.GroupVersionResource{Group: "sriovnetwork.openshift.io", Version: "v1", Resource: "sriovnetworknodepolicies"} - -var sriovnetworknodepoliciesKind = schema.GroupVersionKind{Group: "sriovnetwork.openshift.io", Version: "v1", Kind: "SriovNetworkNodePolicy"} - -// Get takes name of the sriovNetworkNodePolicy, and returns the corresponding sriovNetworkNodePolicy object, and an error if there is any. -func (c *FakeSriovNetworkNodePolicies) Get(ctx context.Context, name string, options v1.GetOptions) (result *sriovnetworkv1.SriovNetworkNodePolicy, err error) { - obj, err := c.Fake. - Invokes(testing.NewGetAction(sriovnetworknodepoliciesResource, c.ns, name), &sriovnetworkv1.SriovNetworkNodePolicy{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovNetworkNodePolicy), err -} - -// List takes label and field selectors, and returns the list of SriovNetworkNodePolicies that match those selectors. -func (c *FakeSriovNetworkNodePolicies) List(ctx context.Context, opts v1.ListOptions) (result *sriovnetworkv1.SriovNetworkNodePolicyList, err error) { - obj, err := c.Fake. - Invokes(testing.NewListAction(sriovnetworknodepoliciesResource, sriovnetworknodepoliciesKind, c.ns, opts), &sriovnetworkv1.SriovNetworkNodePolicyList{}) - - if obj == nil { - return nil, err - } - - label, _, _ := testing.ExtractFromListOptions(opts) - if label == nil { - label = labels.Everything() - } - list := &sriovnetworkv1.SriovNetworkNodePolicyList{ListMeta: obj.(*sriovnetworkv1.SriovNetworkNodePolicyList).ListMeta} - for _, item := range obj.(*sriovnetworkv1.SriovNetworkNodePolicyList).Items { - if label.Matches(labels.Set(item.Labels)) { - list.Items = append(list.Items, item) - } - } - return list, err -} - -// Watch returns a watch.Interface that watches the requested sriovNetworkNodePolicies. -func (c *FakeSriovNetworkNodePolicies) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - return c.Fake. - InvokesWatch(testing.NewWatchAction(sriovnetworknodepoliciesResource, c.ns, opts)) - -} - -// Create takes the representation of a sriovNetworkNodePolicy and creates it. Returns the server's representation of the sriovNetworkNodePolicy, and an error, if there is any. -func (c *FakeSriovNetworkNodePolicies) Create(ctx context.Context, sriovNetworkNodePolicy *sriovnetworkv1.SriovNetworkNodePolicy, opts v1.CreateOptions) (result *sriovnetworkv1.SriovNetworkNodePolicy, err error) { - obj, err := c.Fake. - Invokes(testing.NewCreateAction(sriovnetworknodepoliciesResource, c.ns, sriovNetworkNodePolicy), &sriovnetworkv1.SriovNetworkNodePolicy{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovNetworkNodePolicy), err -} - -// Update takes the representation of a sriovNetworkNodePolicy and updates it. Returns the server's representation of the sriovNetworkNodePolicy, and an error, if there is any. -func (c *FakeSriovNetworkNodePolicies) Update(ctx context.Context, sriovNetworkNodePolicy *sriovnetworkv1.SriovNetworkNodePolicy, opts v1.UpdateOptions) (result *sriovnetworkv1.SriovNetworkNodePolicy, err error) { - obj, err := c.Fake. - Invokes(testing.NewUpdateAction(sriovnetworknodepoliciesResource, c.ns, sriovNetworkNodePolicy), &sriovnetworkv1.SriovNetworkNodePolicy{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovNetworkNodePolicy), err -} - -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *FakeSriovNetworkNodePolicies) UpdateStatus(ctx context.Context, sriovNetworkNodePolicy *sriovnetworkv1.SriovNetworkNodePolicy, opts v1.UpdateOptions) (*sriovnetworkv1.SriovNetworkNodePolicy, error) { - obj, err := c.Fake. - Invokes(testing.NewUpdateSubresourceAction(sriovnetworknodepoliciesResource, "status", c.ns, sriovNetworkNodePolicy), &sriovnetworkv1.SriovNetworkNodePolicy{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovNetworkNodePolicy), err -} - -// Delete takes name of the sriovNetworkNodePolicy and deletes it. Returns an error if one occurs. -func (c *FakeSriovNetworkNodePolicies) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - _, err := c.Fake. - Invokes(testing.NewDeleteAction(sriovnetworknodepoliciesResource, c.ns, name), &sriovnetworkv1.SriovNetworkNodePolicy{}) - - return err -} - -// DeleteCollection deletes a collection of objects. -func (c *FakeSriovNetworkNodePolicies) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - action := testing.NewDeleteCollectionAction(sriovnetworknodepoliciesResource, c.ns, listOpts) - - _, err := c.Fake.Invokes(action, &sriovnetworkv1.SriovNetworkNodePolicyList{}) - return err -} - -// Patch applies the patch and returns the patched sriovNetworkNodePolicy. -func (c *FakeSriovNetworkNodePolicies) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *sriovnetworkv1.SriovNetworkNodePolicy, err error) { - obj, err := c.Fake. - Invokes(testing.NewPatchSubresourceAction(sriovnetworknodepoliciesResource, c.ns, name, pt, data, subresources...), &sriovnetworkv1.SriovNetworkNodePolicy{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovNetworkNodePolicy), err -} diff --git a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/fake/fake_sriovnetworknodestate.go b/pkg/client/clientset/versioned/typed/sriovnetwork/v1/fake/fake_sriovnetworknodestate.go deleted file mode 100644 index 96491cd48..000000000 --- a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/fake/fake_sriovnetworknodestate.go +++ /dev/null @@ -1,126 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -package fake - -import ( - "context" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - labels "k8s.io/apimachinery/pkg/labels" - schema "k8s.io/apimachinery/pkg/runtime/schema" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - testing "k8s.io/client-go/testing" -) - -// FakeSriovNetworkNodeStates implements SriovNetworkNodeStateInterface -type FakeSriovNetworkNodeStates struct { - Fake *FakeSriovnetworkV1 - ns string -} - -var sriovnetworknodestatesResource = schema.GroupVersionResource{Group: "sriovnetwork.openshift.io", Version: "v1", Resource: "sriovnetworknodestates"} - -var sriovnetworknodestatesKind = schema.GroupVersionKind{Group: "sriovnetwork.openshift.io", Version: "v1", Kind: "SriovNetworkNodeState"} - -// Get takes name of the sriovNetworkNodeState, and returns the corresponding sriovNetworkNodeState object, and an error if there is any. -func (c *FakeSriovNetworkNodeStates) Get(ctx context.Context, name string, options v1.GetOptions) (result *sriovnetworkv1.SriovNetworkNodeState, err error) { - obj, err := c.Fake. - Invokes(testing.NewGetAction(sriovnetworknodestatesResource, c.ns, name), &sriovnetworkv1.SriovNetworkNodeState{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovNetworkNodeState), err -} - -// List takes label and field selectors, and returns the list of SriovNetworkNodeStates that match those selectors. -func (c *FakeSriovNetworkNodeStates) List(ctx context.Context, opts v1.ListOptions) (result *sriovnetworkv1.SriovNetworkNodeStateList, err error) { - obj, err := c.Fake. - Invokes(testing.NewListAction(sriovnetworknodestatesResource, sriovnetworknodestatesKind, c.ns, opts), &sriovnetworkv1.SriovNetworkNodeStateList{}) - - if obj == nil { - return nil, err - } - - label, _, _ := testing.ExtractFromListOptions(opts) - if label == nil { - label = labels.Everything() - } - list := &sriovnetworkv1.SriovNetworkNodeStateList{ListMeta: obj.(*sriovnetworkv1.SriovNetworkNodeStateList).ListMeta} - for _, item := range obj.(*sriovnetworkv1.SriovNetworkNodeStateList).Items { - if label.Matches(labels.Set(item.Labels)) { - list.Items = append(list.Items, item) - } - } - return list, err -} - -// Watch returns a watch.Interface that watches the requested sriovNetworkNodeStates. -func (c *FakeSriovNetworkNodeStates) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - return c.Fake. - InvokesWatch(testing.NewWatchAction(sriovnetworknodestatesResource, c.ns, opts)) - -} - -// Create takes the representation of a sriovNetworkNodeState and creates it. Returns the server's representation of the sriovNetworkNodeState, and an error, if there is any. -func (c *FakeSriovNetworkNodeStates) Create(ctx context.Context, sriovNetworkNodeState *sriovnetworkv1.SriovNetworkNodeState, opts v1.CreateOptions) (result *sriovnetworkv1.SriovNetworkNodeState, err error) { - obj, err := c.Fake. - Invokes(testing.NewCreateAction(sriovnetworknodestatesResource, c.ns, sriovNetworkNodeState), &sriovnetworkv1.SriovNetworkNodeState{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovNetworkNodeState), err -} - -// Update takes the representation of a sriovNetworkNodeState and updates it. Returns the server's representation of the sriovNetworkNodeState, and an error, if there is any. -func (c *FakeSriovNetworkNodeStates) Update(ctx context.Context, sriovNetworkNodeState *sriovnetworkv1.SriovNetworkNodeState, opts v1.UpdateOptions) (result *sriovnetworkv1.SriovNetworkNodeState, err error) { - obj, err := c.Fake. - Invokes(testing.NewUpdateAction(sriovnetworknodestatesResource, c.ns, sriovNetworkNodeState), &sriovnetworkv1.SriovNetworkNodeState{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovNetworkNodeState), err -} - -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *FakeSriovNetworkNodeStates) UpdateStatus(ctx context.Context, sriovNetworkNodeState *sriovnetworkv1.SriovNetworkNodeState, opts v1.UpdateOptions) (*sriovnetworkv1.SriovNetworkNodeState, error) { - obj, err := c.Fake. - Invokes(testing.NewUpdateSubresourceAction(sriovnetworknodestatesResource, "status", c.ns, sriovNetworkNodeState), &sriovnetworkv1.SriovNetworkNodeState{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovNetworkNodeState), err -} - -// Delete takes name of the sriovNetworkNodeState and deletes it. Returns an error if one occurs. -func (c *FakeSriovNetworkNodeStates) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - _, err := c.Fake. - Invokes(testing.NewDeleteAction(sriovnetworknodestatesResource, c.ns, name), &sriovnetworkv1.SriovNetworkNodeState{}) - - return err -} - -// DeleteCollection deletes a collection of objects. -func (c *FakeSriovNetworkNodeStates) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - action := testing.NewDeleteCollectionAction(sriovnetworknodestatesResource, c.ns, listOpts) - - _, err := c.Fake.Invokes(action, &sriovnetworkv1.SriovNetworkNodeStateList{}) - return err -} - -// Patch applies the patch and returns the patched sriovNetworkNodeState. -func (c *FakeSriovNetworkNodeStates) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *sriovnetworkv1.SriovNetworkNodeState, err error) { - obj, err := c.Fake. - Invokes(testing.NewPatchSubresourceAction(sriovnetworknodestatesResource, c.ns, name, pt, data, subresources...), &sriovnetworkv1.SriovNetworkNodeState{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovNetworkNodeState), err -} diff --git a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/fake/fake_sriovoperatorconfig.go b/pkg/client/clientset/versioned/typed/sriovnetwork/v1/fake/fake_sriovoperatorconfig.go deleted file mode 100644 index 5f2fb696a..000000000 --- a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/fake/fake_sriovoperatorconfig.go +++ /dev/null @@ -1,126 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -package fake - -import ( - "context" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - labels "k8s.io/apimachinery/pkg/labels" - schema "k8s.io/apimachinery/pkg/runtime/schema" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - testing "k8s.io/client-go/testing" -) - -// FakeSriovOperatorConfigs implements SriovOperatorConfigInterface -type FakeSriovOperatorConfigs struct { - Fake *FakeSriovnetworkV1 - ns string -} - -var sriovoperatorconfigsResource = schema.GroupVersionResource{Group: "sriovnetwork.openshift.io", Version: "v1", Resource: "sriovoperatorconfigs"} - -var sriovoperatorconfigsKind = schema.GroupVersionKind{Group: "sriovnetwork.openshift.io", Version: "v1", Kind: "SriovOperatorConfig"} - -// Get takes name of the sriovOperatorConfig, and returns the corresponding sriovOperatorConfig object, and an error if there is any. -func (c *FakeSriovOperatorConfigs) Get(ctx context.Context, name string, options v1.GetOptions) (result *sriovnetworkv1.SriovOperatorConfig, err error) { - obj, err := c.Fake. - Invokes(testing.NewGetAction(sriovoperatorconfigsResource, c.ns, name), &sriovnetworkv1.SriovOperatorConfig{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovOperatorConfig), err -} - -// List takes label and field selectors, and returns the list of SriovOperatorConfigs that match those selectors. -func (c *FakeSriovOperatorConfigs) List(ctx context.Context, opts v1.ListOptions) (result *sriovnetworkv1.SriovOperatorConfigList, err error) { - obj, err := c.Fake. - Invokes(testing.NewListAction(sriovoperatorconfigsResource, sriovoperatorconfigsKind, c.ns, opts), &sriovnetworkv1.SriovOperatorConfigList{}) - - if obj == nil { - return nil, err - } - - label, _, _ := testing.ExtractFromListOptions(opts) - if label == nil { - label = labels.Everything() - } - list := &sriovnetworkv1.SriovOperatorConfigList{ListMeta: obj.(*sriovnetworkv1.SriovOperatorConfigList).ListMeta} - for _, item := range obj.(*sriovnetworkv1.SriovOperatorConfigList).Items { - if label.Matches(labels.Set(item.Labels)) { - list.Items = append(list.Items, item) - } - } - return list, err -} - -// Watch returns a watch.Interface that watches the requested sriovOperatorConfigs. -func (c *FakeSriovOperatorConfigs) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - return c.Fake. - InvokesWatch(testing.NewWatchAction(sriovoperatorconfigsResource, c.ns, opts)) - -} - -// Create takes the representation of a sriovOperatorConfig and creates it. Returns the server's representation of the sriovOperatorConfig, and an error, if there is any. -func (c *FakeSriovOperatorConfigs) Create(ctx context.Context, sriovOperatorConfig *sriovnetworkv1.SriovOperatorConfig, opts v1.CreateOptions) (result *sriovnetworkv1.SriovOperatorConfig, err error) { - obj, err := c.Fake. - Invokes(testing.NewCreateAction(sriovoperatorconfigsResource, c.ns, sriovOperatorConfig), &sriovnetworkv1.SriovOperatorConfig{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovOperatorConfig), err -} - -// Update takes the representation of a sriovOperatorConfig and updates it. Returns the server's representation of the sriovOperatorConfig, and an error, if there is any. -func (c *FakeSriovOperatorConfigs) Update(ctx context.Context, sriovOperatorConfig *sriovnetworkv1.SriovOperatorConfig, opts v1.UpdateOptions) (result *sriovnetworkv1.SriovOperatorConfig, err error) { - obj, err := c.Fake. - Invokes(testing.NewUpdateAction(sriovoperatorconfigsResource, c.ns, sriovOperatorConfig), &sriovnetworkv1.SriovOperatorConfig{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovOperatorConfig), err -} - -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *FakeSriovOperatorConfigs) UpdateStatus(ctx context.Context, sriovOperatorConfig *sriovnetworkv1.SriovOperatorConfig, opts v1.UpdateOptions) (*sriovnetworkv1.SriovOperatorConfig, error) { - obj, err := c.Fake. - Invokes(testing.NewUpdateSubresourceAction(sriovoperatorconfigsResource, "status", c.ns, sriovOperatorConfig), &sriovnetworkv1.SriovOperatorConfig{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovOperatorConfig), err -} - -// Delete takes name of the sriovOperatorConfig and deletes it. Returns an error if one occurs. -func (c *FakeSriovOperatorConfigs) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - _, err := c.Fake. - Invokes(testing.NewDeleteAction(sriovoperatorconfigsResource, c.ns, name), &sriovnetworkv1.SriovOperatorConfig{}) - - return err -} - -// DeleteCollection deletes a collection of objects. -func (c *FakeSriovOperatorConfigs) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - action := testing.NewDeleteCollectionAction(sriovoperatorconfigsResource, c.ns, listOpts) - - _, err := c.Fake.Invokes(action, &sriovnetworkv1.SriovOperatorConfigList{}) - return err -} - -// Patch applies the patch and returns the patched sriovOperatorConfig. -func (c *FakeSriovOperatorConfigs) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *sriovnetworkv1.SriovOperatorConfig, err error) { - obj, err := c.Fake. - Invokes(testing.NewPatchSubresourceAction(sriovoperatorconfigsResource, c.ns, name, pt, data, subresources...), &sriovnetworkv1.SriovOperatorConfig{}) - - if obj == nil { - return nil, err - } - return obj.(*sriovnetworkv1.SriovOperatorConfig), err -} diff --git a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/generated_expansion.go b/pkg/client/clientset/versioned/typed/sriovnetwork/v1/generated_expansion.go deleted file mode 100644 index 8913a3742..000000000 --- a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/generated_expansion.go +++ /dev/null @@ -1,11 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -package v1 - -type SriovNetworkExpansion interface{} - -type SriovNetworkNodePolicyExpansion interface{} - -type SriovNetworkNodeStateExpansion interface{} - -type SriovOperatorConfigExpansion interface{} diff --git a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/sriovnetwork.go b/pkg/client/clientset/versioned/typed/sriovnetwork/v1/sriovnetwork.go deleted file mode 100644 index 472c83e0c..000000000 --- a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/sriovnetwork.go +++ /dev/null @@ -1,179 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -package v1 - -import ( - "context" - "time" - - v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - scheme "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned/scheme" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - rest "k8s.io/client-go/rest" -) - -// SriovNetworksGetter has a method to return a SriovNetworkInterface. -// A group's client should implement this interface. -type SriovNetworksGetter interface { - SriovNetworks(namespace string) SriovNetworkInterface -} - -// SriovNetworkInterface has methods to work with SriovNetwork resources. -type SriovNetworkInterface interface { - Create(ctx context.Context, sriovNetwork *v1.SriovNetwork, opts metav1.CreateOptions) (*v1.SriovNetwork, error) - Update(ctx context.Context, sriovNetwork *v1.SriovNetwork, opts metav1.UpdateOptions) (*v1.SriovNetwork, error) - UpdateStatus(ctx context.Context, sriovNetwork *v1.SriovNetwork, opts metav1.UpdateOptions) (*v1.SriovNetwork, error) - Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error - DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error - Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.SriovNetwork, error) - List(ctx context.Context, opts metav1.ListOptions) (*v1.SriovNetworkList, error) - Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) - Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.SriovNetwork, err error) - SriovNetworkExpansion -} - -// sriovNetworks implements SriovNetworkInterface -type sriovNetworks struct { - client rest.Interface - ns string -} - -// newSriovNetworks returns a SriovNetworks -func newSriovNetworks(c *SriovnetworkV1Client, namespace string) *sriovNetworks { - return &sriovNetworks{ - client: c.RESTClient(), - ns: namespace, - } -} - -// Get takes name of the sriovNetwork, and returns the corresponding sriovNetwork object, and an error if there is any. -func (c *sriovNetworks) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.SriovNetwork, err error) { - result = &v1.SriovNetwork{} - err = c.client.Get(). - Namespace(c.ns). - Resource("sriovnetworks"). - Name(name). - VersionedParams(&options, scheme.ParameterCodec). - Do(ctx). - Into(result) - return -} - -// List takes label and field selectors, and returns the list of SriovNetworks that match those selectors. -func (c *sriovNetworks) List(ctx context.Context, opts metav1.ListOptions) (result *v1.SriovNetworkList, err error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - result = &v1.SriovNetworkList{} - err = c.client.Get(). - Namespace(c.ns). - Resource("sriovnetworks"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Do(ctx). - Into(result) - return -} - -// Watch returns a watch.Interface that watches the requested sriovNetworks. -func (c *sriovNetworks) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - opts.Watch = true - return c.client.Get(). - Namespace(c.ns). - Resource("sriovnetworks"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Watch(ctx) -} - -// Create takes the representation of a sriovNetwork and creates it. Returns the server's representation of the sriovNetwork, and an error, if there is any. -func (c *sriovNetworks) Create(ctx context.Context, sriovNetwork *v1.SriovNetwork, opts metav1.CreateOptions) (result *v1.SriovNetwork, err error) { - result = &v1.SriovNetwork{} - err = c.client.Post(). - Namespace(c.ns). - Resource("sriovnetworks"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(sriovNetwork). - Do(ctx). - Into(result) - return -} - -// Update takes the representation of a sriovNetwork and updates it. Returns the server's representation of the sriovNetwork, and an error, if there is any. -func (c *sriovNetworks) Update(ctx context.Context, sriovNetwork *v1.SriovNetwork, opts metav1.UpdateOptions) (result *v1.SriovNetwork, err error) { - result = &v1.SriovNetwork{} - err = c.client.Put(). - Namespace(c.ns). - Resource("sriovnetworks"). - Name(sriovNetwork.Name). - VersionedParams(&opts, scheme.ParameterCodec). - Body(sriovNetwork). - Do(ctx). - Into(result) - return -} - -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *sriovNetworks) UpdateStatus(ctx context.Context, sriovNetwork *v1.SriovNetwork, opts metav1.UpdateOptions) (result *v1.SriovNetwork, err error) { - result = &v1.SriovNetwork{} - err = c.client.Put(). - Namespace(c.ns). - Resource("sriovnetworks"). - Name(sriovNetwork.Name). - SubResource("status"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(sriovNetwork). - Do(ctx). - Into(result) - return -} - -// Delete takes name of the sriovNetwork and deletes it. Returns an error if one occurs. -func (c *sriovNetworks) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { - return c.client.Delete(). - Namespace(c.ns). - Resource("sriovnetworks"). - Name(name). - Body(&opts). - Do(ctx). - Error() -} - -// DeleteCollection deletes a collection of objects. -func (c *sriovNetworks) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { - var timeout time.Duration - if listOpts.TimeoutSeconds != nil { - timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second - } - return c.client.Delete(). - Namespace(c.ns). - Resource("sriovnetworks"). - VersionedParams(&listOpts, scheme.ParameterCodec). - Timeout(timeout). - Body(&opts). - Do(ctx). - Error() -} - -// Patch applies the patch and returns the patched sriovNetwork. -func (c *sriovNetworks) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.SriovNetwork, err error) { - result = &v1.SriovNetwork{} - err = c.client.Patch(pt). - Namespace(c.ns). - Resource("sriovnetworks"). - Name(name). - SubResource(subresources...). - VersionedParams(&opts, scheme.ParameterCodec). - Body(data). - Do(ctx). - Into(result) - return -} diff --git a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/sriovnetwork_client.go b/pkg/client/clientset/versioned/typed/sriovnetwork/v1/sriovnetwork_client.go deleted file mode 100644 index 11444b70c..000000000 --- a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/sriovnetwork_client.go +++ /dev/null @@ -1,88 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -package v1 - -import ( - v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned/scheme" - rest "k8s.io/client-go/rest" -) - -type SriovnetworkV1Interface interface { - RESTClient() rest.Interface - SriovNetworksGetter - SriovNetworkNodePoliciesGetter - SriovNetworkNodeStatesGetter - SriovOperatorConfigsGetter -} - -// SriovnetworkV1Client is used to interact with features provided by the sriovnetwork group. -type SriovnetworkV1Client struct { - restClient rest.Interface -} - -func (c *SriovnetworkV1Client) SriovNetworks(namespace string) SriovNetworkInterface { - return newSriovNetworks(c, namespace) -} - -func (c *SriovnetworkV1Client) SriovNetworkNodePolicies(namespace string) SriovNetworkNodePolicyInterface { - return newSriovNetworkNodePolicies(c, namespace) -} - -func (c *SriovnetworkV1Client) SriovNetworkNodeStates(namespace string) SriovNetworkNodeStateInterface { - return newSriovNetworkNodeStates(c, namespace) -} - -func (c *SriovnetworkV1Client) SriovOperatorConfigs(namespace string) SriovOperatorConfigInterface { - return newSriovOperatorConfigs(c, namespace) -} - -// NewForConfig creates a new SriovnetworkV1Client for the given config. -func NewForConfig(c *rest.Config) (*SriovnetworkV1Client, error) { - config := *c - if err := setConfigDefaults(&config); err != nil { - return nil, err - } - client, err := rest.RESTClientFor(&config) - if err != nil { - return nil, err - } - return &SriovnetworkV1Client{client}, nil -} - -// NewForConfigOrDie creates a new SriovnetworkV1Client for the given config and -// panics if there is an error in the config. -func NewForConfigOrDie(c *rest.Config) *SriovnetworkV1Client { - client, err := NewForConfig(c) - if err != nil { - panic(err) - } - return client -} - -// New creates a new SriovnetworkV1Client for the given RESTClient. -func New(c rest.Interface) *SriovnetworkV1Client { - return &SriovnetworkV1Client{c} -} - -func setConfigDefaults(config *rest.Config) error { - gv := v1.SchemeGroupVersion - config.GroupVersion = &gv - config.APIPath = "/apis" - config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() - - if config.UserAgent == "" { - config.UserAgent = rest.DefaultKubernetesUserAgent() - } - - return nil -} - -// RESTClient returns a RESTClient that is used to communicate -// with API server by this client implementation. -func (c *SriovnetworkV1Client) RESTClient() rest.Interface { - if c == nil { - return nil - } - return c.restClient -} diff --git a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/sriovnetworknodepolicy.go b/pkg/client/clientset/versioned/typed/sriovnetwork/v1/sriovnetworknodepolicy.go deleted file mode 100644 index 9feb9606d..000000000 --- a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/sriovnetworknodepolicy.go +++ /dev/null @@ -1,179 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -package v1 - -import ( - "context" - "time" - - v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - scheme "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned/scheme" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - rest "k8s.io/client-go/rest" -) - -// SriovNetworkNodePoliciesGetter has a method to return a SriovNetworkNodePolicyInterface. -// A group's client should implement this interface. -type SriovNetworkNodePoliciesGetter interface { - SriovNetworkNodePolicies(namespace string) SriovNetworkNodePolicyInterface -} - -// SriovNetworkNodePolicyInterface has methods to work with SriovNetworkNodePolicy resources. -type SriovNetworkNodePolicyInterface interface { - Create(ctx context.Context, sriovNetworkNodePolicy *v1.SriovNetworkNodePolicy, opts metav1.CreateOptions) (*v1.SriovNetworkNodePolicy, error) - Update(ctx context.Context, sriovNetworkNodePolicy *v1.SriovNetworkNodePolicy, opts metav1.UpdateOptions) (*v1.SriovNetworkNodePolicy, error) - UpdateStatus(ctx context.Context, sriovNetworkNodePolicy *v1.SriovNetworkNodePolicy, opts metav1.UpdateOptions) (*v1.SriovNetworkNodePolicy, error) - Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error - DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error - Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.SriovNetworkNodePolicy, error) - List(ctx context.Context, opts metav1.ListOptions) (*v1.SriovNetworkNodePolicyList, error) - Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) - Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.SriovNetworkNodePolicy, err error) - SriovNetworkNodePolicyExpansion -} - -// sriovNetworkNodePolicies implements SriovNetworkNodePolicyInterface -type sriovNetworkNodePolicies struct { - client rest.Interface - ns string -} - -// newSriovNetworkNodePolicies returns a SriovNetworkNodePolicies -func newSriovNetworkNodePolicies(c *SriovnetworkV1Client, namespace string) *sriovNetworkNodePolicies { - return &sriovNetworkNodePolicies{ - client: c.RESTClient(), - ns: namespace, - } -} - -// Get takes name of the sriovNetworkNodePolicy, and returns the corresponding sriovNetworkNodePolicy object, and an error if there is any. -func (c *sriovNetworkNodePolicies) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.SriovNetworkNodePolicy, err error) { - result = &v1.SriovNetworkNodePolicy{} - err = c.client.Get(). - Namespace(c.ns). - Resource("sriovnetworknodepolicies"). - Name(name). - VersionedParams(&options, scheme.ParameterCodec). - Do(ctx). - Into(result) - return -} - -// List takes label and field selectors, and returns the list of SriovNetworkNodePolicies that match those selectors. -func (c *sriovNetworkNodePolicies) List(ctx context.Context, opts metav1.ListOptions) (result *v1.SriovNetworkNodePolicyList, err error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - result = &v1.SriovNetworkNodePolicyList{} - err = c.client.Get(). - Namespace(c.ns). - Resource("sriovnetworknodepolicies"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Do(ctx). - Into(result) - return -} - -// Watch returns a watch.Interface that watches the requested sriovNetworkNodePolicies. -func (c *sriovNetworkNodePolicies) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - opts.Watch = true - return c.client.Get(). - Namespace(c.ns). - Resource("sriovnetworknodepolicies"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Watch(ctx) -} - -// Create takes the representation of a sriovNetworkNodePolicy and creates it. Returns the server's representation of the sriovNetworkNodePolicy, and an error, if there is any. -func (c *sriovNetworkNodePolicies) Create(ctx context.Context, sriovNetworkNodePolicy *v1.SriovNetworkNodePolicy, opts metav1.CreateOptions) (result *v1.SriovNetworkNodePolicy, err error) { - result = &v1.SriovNetworkNodePolicy{} - err = c.client.Post(). - Namespace(c.ns). - Resource("sriovnetworknodepolicies"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(sriovNetworkNodePolicy). - Do(ctx). - Into(result) - return -} - -// Update takes the representation of a sriovNetworkNodePolicy and updates it. Returns the server's representation of the sriovNetworkNodePolicy, and an error, if there is any. -func (c *sriovNetworkNodePolicies) Update(ctx context.Context, sriovNetworkNodePolicy *v1.SriovNetworkNodePolicy, opts metav1.UpdateOptions) (result *v1.SriovNetworkNodePolicy, err error) { - result = &v1.SriovNetworkNodePolicy{} - err = c.client.Put(). - Namespace(c.ns). - Resource("sriovnetworknodepolicies"). - Name(sriovNetworkNodePolicy.Name). - VersionedParams(&opts, scheme.ParameterCodec). - Body(sriovNetworkNodePolicy). - Do(ctx). - Into(result) - return -} - -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *sriovNetworkNodePolicies) UpdateStatus(ctx context.Context, sriovNetworkNodePolicy *v1.SriovNetworkNodePolicy, opts metav1.UpdateOptions) (result *v1.SriovNetworkNodePolicy, err error) { - result = &v1.SriovNetworkNodePolicy{} - err = c.client.Put(). - Namespace(c.ns). - Resource("sriovnetworknodepolicies"). - Name(sriovNetworkNodePolicy.Name). - SubResource("status"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(sriovNetworkNodePolicy). - Do(ctx). - Into(result) - return -} - -// Delete takes name of the sriovNetworkNodePolicy and deletes it. Returns an error if one occurs. -func (c *sriovNetworkNodePolicies) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { - return c.client.Delete(). - Namespace(c.ns). - Resource("sriovnetworknodepolicies"). - Name(name). - Body(&opts). - Do(ctx). - Error() -} - -// DeleteCollection deletes a collection of objects. -func (c *sriovNetworkNodePolicies) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { - var timeout time.Duration - if listOpts.TimeoutSeconds != nil { - timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second - } - return c.client.Delete(). - Namespace(c.ns). - Resource("sriovnetworknodepolicies"). - VersionedParams(&listOpts, scheme.ParameterCodec). - Timeout(timeout). - Body(&opts). - Do(ctx). - Error() -} - -// Patch applies the patch and returns the patched sriovNetworkNodePolicy. -func (c *sriovNetworkNodePolicies) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.SriovNetworkNodePolicy, err error) { - result = &v1.SriovNetworkNodePolicy{} - err = c.client.Patch(pt). - Namespace(c.ns). - Resource("sriovnetworknodepolicies"). - Name(name). - SubResource(subresources...). - VersionedParams(&opts, scheme.ParameterCodec). - Body(data). - Do(ctx). - Into(result) - return -} diff --git a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/sriovnetworknodestate.go b/pkg/client/clientset/versioned/typed/sriovnetwork/v1/sriovnetworknodestate.go deleted file mode 100644 index c41c0e306..000000000 --- a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/sriovnetworknodestate.go +++ /dev/null @@ -1,179 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -package v1 - -import ( - "context" - "time" - - v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - scheme "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned/scheme" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - rest "k8s.io/client-go/rest" -) - -// SriovNetworkNodeStatesGetter has a method to return a SriovNetworkNodeStateInterface. -// A group's client should implement this interface. -type SriovNetworkNodeStatesGetter interface { - SriovNetworkNodeStates(namespace string) SriovNetworkNodeStateInterface -} - -// SriovNetworkNodeStateInterface has methods to work with SriovNetworkNodeState resources. -type SriovNetworkNodeStateInterface interface { - Create(ctx context.Context, sriovNetworkNodeState *v1.SriovNetworkNodeState, opts metav1.CreateOptions) (*v1.SriovNetworkNodeState, error) - Update(ctx context.Context, sriovNetworkNodeState *v1.SriovNetworkNodeState, opts metav1.UpdateOptions) (*v1.SriovNetworkNodeState, error) - UpdateStatus(ctx context.Context, sriovNetworkNodeState *v1.SriovNetworkNodeState, opts metav1.UpdateOptions) (*v1.SriovNetworkNodeState, error) - Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error - DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error - Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.SriovNetworkNodeState, error) - List(ctx context.Context, opts metav1.ListOptions) (*v1.SriovNetworkNodeStateList, error) - Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) - Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.SriovNetworkNodeState, err error) - SriovNetworkNodeStateExpansion -} - -// sriovNetworkNodeStates implements SriovNetworkNodeStateInterface -type sriovNetworkNodeStates struct { - client rest.Interface - ns string -} - -// newSriovNetworkNodeStates returns a SriovNetworkNodeStates -func newSriovNetworkNodeStates(c *SriovnetworkV1Client, namespace string) *sriovNetworkNodeStates { - return &sriovNetworkNodeStates{ - client: c.RESTClient(), - ns: namespace, - } -} - -// Get takes name of the sriovNetworkNodeState, and returns the corresponding sriovNetworkNodeState object, and an error if there is any. -func (c *sriovNetworkNodeStates) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.SriovNetworkNodeState, err error) { - result = &v1.SriovNetworkNodeState{} - err = c.client.Get(). - Namespace(c.ns). - Resource("sriovnetworknodestates"). - Name(name). - VersionedParams(&options, scheme.ParameterCodec). - Do(ctx). - Into(result) - return -} - -// List takes label and field selectors, and returns the list of SriovNetworkNodeStates that match those selectors. -func (c *sriovNetworkNodeStates) List(ctx context.Context, opts metav1.ListOptions) (result *v1.SriovNetworkNodeStateList, err error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - result = &v1.SriovNetworkNodeStateList{} - err = c.client.Get(). - Namespace(c.ns). - Resource("sriovnetworknodestates"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Do(ctx). - Into(result) - return -} - -// Watch returns a watch.Interface that watches the requested sriovNetworkNodeStates. -func (c *sriovNetworkNodeStates) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - opts.Watch = true - return c.client.Get(). - Namespace(c.ns). - Resource("sriovnetworknodestates"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Watch(ctx) -} - -// Create takes the representation of a sriovNetworkNodeState and creates it. Returns the server's representation of the sriovNetworkNodeState, and an error, if there is any. -func (c *sriovNetworkNodeStates) Create(ctx context.Context, sriovNetworkNodeState *v1.SriovNetworkNodeState, opts metav1.CreateOptions) (result *v1.SriovNetworkNodeState, err error) { - result = &v1.SriovNetworkNodeState{} - err = c.client.Post(). - Namespace(c.ns). - Resource("sriovnetworknodestates"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(sriovNetworkNodeState). - Do(ctx). - Into(result) - return -} - -// Update takes the representation of a sriovNetworkNodeState and updates it. Returns the server's representation of the sriovNetworkNodeState, and an error, if there is any. -func (c *sriovNetworkNodeStates) Update(ctx context.Context, sriovNetworkNodeState *v1.SriovNetworkNodeState, opts metav1.UpdateOptions) (result *v1.SriovNetworkNodeState, err error) { - result = &v1.SriovNetworkNodeState{} - err = c.client.Put(). - Namespace(c.ns). - Resource("sriovnetworknodestates"). - Name(sriovNetworkNodeState.Name). - VersionedParams(&opts, scheme.ParameterCodec). - Body(sriovNetworkNodeState). - Do(ctx). - Into(result) - return -} - -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *sriovNetworkNodeStates) UpdateStatus(ctx context.Context, sriovNetworkNodeState *v1.SriovNetworkNodeState, opts metav1.UpdateOptions) (result *v1.SriovNetworkNodeState, err error) { - result = &v1.SriovNetworkNodeState{} - err = c.client.Put(). - Namespace(c.ns). - Resource("sriovnetworknodestates"). - Name(sriovNetworkNodeState.Name). - SubResource("status"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(sriovNetworkNodeState). - Do(ctx). - Into(result) - return -} - -// Delete takes name of the sriovNetworkNodeState and deletes it. Returns an error if one occurs. -func (c *sriovNetworkNodeStates) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { - return c.client.Delete(). - Namespace(c.ns). - Resource("sriovnetworknodestates"). - Name(name). - Body(&opts). - Do(ctx). - Error() -} - -// DeleteCollection deletes a collection of objects. -func (c *sriovNetworkNodeStates) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { - var timeout time.Duration - if listOpts.TimeoutSeconds != nil { - timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second - } - return c.client.Delete(). - Namespace(c.ns). - Resource("sriovnetworknodestates"). - VersionedParams(&listOpts, scheme.ParameterCodec). - Timeout(timeout). - Body(&opts). - Do(ctx). - Error() -} - -// Patch applies the patch and returns the patched sriovNetworkNodeState. -func (c *sriovNetworkNodeStates) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.SriovNetworkNodeState, err error) { - result = &v1.SriovNetworkNodeState{} - err = c.client.Patch(pt). - Namespace(c.ns). - Resource("sriovnetworknodestates"). - Name(name). - SubResource(subresources...). - VersionedParams(&opts, scheme.ParameterCodec). - Body(data). - Do(ctx). - Into(result) - return -} diff --git a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/sriovoperatorconfig.go b/pkg/client/clientset/versioned/typed/sriovnetwork/v1/sriovoperatorconfig.go deleted file mode 100644 index 4e8b89c1b..000000000 --- a/pkg/client/clientset/versioned/typed/sriovnetwork/v1/sriovoperatorconfig.go +++ /dev/null @@ -1,179 +0,0 @@ -// Code generated by client-gen. DO NOT EDIT. - -package v1 - -import ( - "context" - "time" - - v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - scheme "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned/scheme" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - rest "k8s.io/client-go/rest" -) - -// SriovOperatorConfigsGetter has a method to return a SriovOperatorConfigInterface. -// A group's client should implement this interface. -type SriovOperatorConfigsGetter interface { - SriovOperatorConfigs(namespace string) SriovOperatorConfigInterface -} - -// SriovOperatorConfigInterface has methods to work with SriovOperatorConfig resources. -type SriovOperatorConfigInterface interface { - Create(ctx context.Context, sriovOperatorConfig *v1.SriovOperatorConfig, opts metav1.CreateOptions) (*v1.SriovOperatorConfig, error) - Update(ctx context.Context, sriovOperatorConfig *v1.SriovOperatorConfig, opts metav1.UpdateOptions) (*v1.SriovOperatorConfig, error) - UpdateStatus(ctx context.Context, sriovOperatorConfig *v1.SriovOperatorConfig, opts metav1.UpdateOptions) (*v1.SriovOperatorConfig, error) - Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error - DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error - Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.SriovOperatorConfig, error) - List(ctx context.Context, opts metav1.ListOptions) (*v1.SriovOperatorConfigList, error) - Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) - Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.SriovOperatorConfig, err error) - SriovOperatorConfigExpansion -} - -// sriovOperatorConfigs implements SriovOperatorConfigInterface -type sriovOperatorConfigs struct { - client rest.Interface - ns string -} - -// newSriovOperatorConfigs returns a SriovOperatorConfigs -func newSriovOperatorConfigs(c *SriovnetworkV1Client, namespace string) *sriovOperatorConfigs { - return &sriovOperatorConfigs{ - client: c.RESTClient(), - ns: namespace, - } -} - -// Get takes name of the sriovOperatorConfig, and returns the corresponding sriovOperatorConfig object, and an error if there is any. -func (c *sriovOperatorConfigs) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.SriovOperatorConfig, err error) { - result = &v1.SriovOperatorConfig{} - err = c.client.Get(). - Namespace(c.ns). - Resource("sriovoperatorconfigs"). - Name(name). - VersionedParams(&options, scheme.ParameterCodec). - Do(ctx). - Into(result) - return -} - -// List takes label and field selectors, and returns the list of SriovOperatorConfigs that match those selectors. -func (c *sriovOperatorConfigs) List(ctx context.Context, opts metav1.ListOptions) (result *v1.SriovOperatorConfigList, err error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - result = &v1.SriovOperatorConfigList{} - err = c.client.Get(). - Namespace(c.ns). - Resource("sriovoperatorconfigs"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Do(ctx). - Into(result) - return -} - -// Watch returns a watch.Interface that watches the requested sriovOperatorConfigs. -func (c *sriovOperatorConfigs) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - opts.Watch = true - return c.client.Get(). - Namespace(c.ns). - Resource("sriovoperatorconfigs"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Watch(ctx) -} - -// Create takes the representation of a sriovOperatorConfig and creates it. Returns the server's representation of the sriovOperatorConfig, and an error, if there is any. -func (c *sriovOperatorConfigs) Create(ctx context.Context, sriovOperatorConfig *v1.SriovOperatorConfig, opts metav1.CreateOptions) (result *v1.SriovOperatorConfig, err error) { - result = &v1.SriovOperatorConfig{} - err = c.client.Post(). - Namespace(c.ns). - Resource("sriovoperatorconfigs"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(sriovOperatorConfig). - Do(ctx). - Into(result) - return -} - -// Update takes the representation of a sriovOperatorConfig and updates it. Returns the server's representation of the sriovOperatorConfig, and an error, if there is any. -func (c *sriovOperatorConfigs) Update(ctx context.Context, sriovOperatorConfig *v1.SriovOperatorConfig, opts metav1.UpdateOptions) (result *v1.SriovOperatorConfig, err error) { - result = &v1.SriovOperatorConfig{} - err = c.client.Put(). - Namespace(c.ns). - Resource("sriovoperatorconfigs"). - Name(sriovOperatorConfig.Name). - VersionedParams(&opts, scheme.ParameterCodec). - Body(sriovOperatorConfig). - Do(ctx). - Into(result) - return -} - -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *sriovOperatorConfigs) UpdateStatus(ctx context.Context, sriovOperatorConfig *v1.SriovOperatorConfig, opts metav1.UpdateOptions) (result *v1.SriovOperatorConfig, err error) { - result = &v1.SriovOperatorConfig{} - err = c.client.Put(). - Namespace(c.ns). - Resource("sriovoperatorconfigs"). - Name(sriovOperatorConfig.Name). - SubResource("status"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(sriovOperatorConfig). - Do(ctx). - Into(result) - return -} - -// Delete takes name of the sriovOperatorConfig and deletes it. Returns an error if one occurs. -func (c *sriovOperatorConfigs) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { - return c.client.Delete(). - Namespace(c.ns). - Resource("sriovoperatorconfigs"). - Name(name). - Body(&opts). - Do(ctx). - Error() -} - -// DeleteCollection deletes a collection of objects. -func (c *sriovOperatorConfigs) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { - var timeout time.Duration - if listOpts.TimeoutSeconds != nil { - timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second - } - return c.client.Delete(). - Namespace(c.ns). - Resource("sriovoperatorconfigs"). - VersionedParams(&listOpts, scheme.ParameterCodec). - Timeout(timeout). - Body(&opts). - Do(ctx). - Error() -} - -// Patch applies the patch and returns the patched sriovOperatorConfig. -func (c *sriovOperatorConfigs) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.SriovOperatorConfig, err error) { - result = &v1.SriovOperatorConfig{} - err = c.client.Patch(pt). - Namespace(c.ns). - Resource("sriovoperatorconfigs"). - Name(name). - SubResource(subresources...). - VersionedParams(&opts, scheme.ParameterCodec). - Body(data). - Do(ctx). - Into(result) - return -} diff --git a/pkg/client/informers/externalversions/factory.go b/pkg/client/informers/externalversions/factory.go deleted file mode 100644 index 57c769d43..000000000 --- a/pkg/client/informers/externalversions/factory.go +++ /dev/null @@ -1,165 +0,0 @@ -// Code generated by informer-gen. DO NOT EDIT. - -package externalversions - -import ( - reflect "reflect" - sync "sync" - time "time" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - schema "k8s.io/apimachinery/pkg/runtime/schema" - cache "k8s.io/client-go/tools/cache" - - versioned "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" - internalinterfaces "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/informers/externalversions/internalinterfaces" - sriovnetwork "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/informers/externalversions/sriovnetwork" -) - -// SharedInformerOption defines the functional option type for SharedInformerFactory. -type SharedInformerOption func(*sharedInformerFactory) *sharedInformerFactory - -type sharedInformerFactory struct { - client versioned.Interface - namespace string - tweakListOptions internalinterfaces.TweakListOptionsFunc - lock sync.Mutex - defaultResync time.Duration - customResync map[reflect.Type]time.Duration - - informers map[reflect.Type]cache.SharedIndexInformer - // startedInformers is used for tracking which informers have been started. - // This allows Start() to be called multiple times safely. - startedInformers map[reflect.Type]bool -} - -// WithCustomResyncConfig sets a custom resync period for the specified informer types. -func WithCustomResyncConfig(resyncConfig map[v1.Object]time.Duration) SharedInformerOption { - return func(factory *sharedInformerFactory) *sharedInformerFactory { - for k, v := range resyncConfig { - factory.customResync[reflect.TypeOf(k)] = v - } - return factory - } -} - -// WithTweakListOptions sets a custom filter on all listers of the configured SharedInformerFactory. -func WithTweakListOptions(tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerOption { - return func(factory *sharedInformerFactory) *sharedInformerFactory { - factory.tweakListOptions = tweakListOptions - return factory - } -} - -// WithNamespace limits the SharedInformerFactory to the specified namespace. -func WithNamespace(namespace string) SharedInformerOption { - return func(factory *sharedInformerFactory) *sharedInformerFactory { - factory.namespace = namespace - return factory - } -} - -// NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces. -func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory { - return NewSharedInformerFactoryWithOptions(client, defaultResync) -} - -// NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory. -// Listers obtained via this SharedInformerFactory will be subject to the same filters -// as specified here. -// Deprecated: Please use NewSharedInformerFactoryWithOptions instead -func NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory { - return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions)) -} - -// NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options. -func NewSharedInformerFactoryWithOptions(client versioned.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory { - factory := &sharedInformerFactory{ - client: client, - namespace: v1.NamespaceAll, - defaultResync: defaultResync, - informers: make(map[reflect.Type]cache.SharedIndexInformer), - startedInformers: make(map[reflect.Type]bool), - customResync: make(map[reflect.Type]time.Duration), - } - - // Apply all options - for _, opt := range options { - factory = opt(factory) - } - - return factory -} - -// Start initializes all requested informers. -func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) { - f.lock.Lock() - defer f.lock.Unlock() - - for informerType, informer := range f.informers { - if !f.startedInformers[informerType] { - go informer.Run(stopCh) - f.startedInformers[informerType] = true - } - } -} - -// WaitForCacheSync waits for all started informers' cache were synced. -func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { - informers := func() map[reflect.Type]cache.SharedIndexInformer { - f.lock.Lock() - defer f.lock.Unlock() - - informers := map[reflect.Type]cache.SharedIndexInformer{} - for informerType, informer := range f.informers { - if f.startedInformers[informerType] { - informers[informerType] = informer - } - } - return informers - }() - - res := map[reflect.Type]bool{} - for informType, informer := range informers { - res[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced) - } - return res -} - -// InternalInformerFor returns the SharedIndexInformer for obj using an internal -// client. -func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer { - f.lock.Lock() - defer f.lock.Unlock() - - informerType := reflect.TypeOf(obj) - informer, exists := f.informers[informerType] - if exists { - return informer - } - - resyncPeriod, exists := f.customResync[informerType] - if !exists { - resyncPeriod = f.defaultResync - } - - informer = newFunc(f.client, resyncPeriod) - f.informers[informerType] = informer - - return informer -} - -// SharedInformerFactory provides shared informers for resources in all known -// API group versions. -type SharedInformerFactory interface { - internalinterfaces.SharedInformerFactory - ForResource(resource schema.GroupVersionResource) (GenericInformer, error) - WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool - - Sriovnetwork() sriovnetwork.Interface -} - -func (f *sharedInformerFactory) Sriovnetwork() sriovnetwork.Interface { - return sriovnetwork.New(f, f.namespace, f.tweakListOptions) -} diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go deleted file mode 100644 index f225f0d75..000000000 --- a/pkg/client/informers/externalversions/generic.go +++ /dev/null @@ -1,53 +0,0 @@ -// Code generated by informer-gen. DO NOT EDIT. - -package externalversions - -import ( - "fmt" - - schema "k8s.io/apimachinery/pkg/runtime/schema" - cache "k8s.io/client-go/tools/cache" - - v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" -) - -// GenericInformer is type of SharedIndexInformer which will locate and delegate to other -// sharedInformers based on type -type GenericInformer interface { - Informer() cache.SharedIndexInformer - Lister() cache.GenericLister -} - -type genericInformer struct { - informer cache.SharedIndexInformer - resource schema.GroupResource -} - -// Informer returns the SharedIndexInformer. -func (f *genericInformer) Informer() cache.SharedIndexInformer { - return f.informer -} - -// Lister returns the GenericLister. -func (f *genericInformer) Lister() cache.GenericLister { - return cache.NewGenericLister(f.Informer().GetIndexer(), f.resource) -} - -// ForResource gives generic access to a shared informer of the matching type -// TODO extend this to unknown resources with a client pool -func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { - switch resource { - // Group=sriovnetwork, Version=v1 - case v1.SchemeGroupVersion.WithResource("sriovnetworks"): - return &genericInformer{resource: resource.GroupResource(), informer: f.Sriovnetwork().V1().SriovNetworks().Informer()}, nil - case v1.SchemeGroupVersion.WithResource("sriovnetworknodepolicies"): - return &genericInformer{resource: resource.GroupResource(), informer: f.Sriovnetwork().V1().SriovNetworkNodePolicies().Informer()}, nil - case v1.SchemeGroupVersion.WithResource("sriovnetworknodestates"): - return &genericInformer{resource: resource.GroupResource(), informer: f.Sriovnetwork().V1().SriovNetworkNodeStates().Informer()}, nil - case v1.SchemeGroupVersion.WithResource("sriovoperatorconfigs"): - return &genericInformer{resource: resource.GroupResource(), informer: f.Sriovnetwork().V1().SriovOperatorConfigs().Informer()}, nil - - } - - return nil, fmt.Errorf("no informer found for %v", resource) -} diff --git a/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go b/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go deleted file mode 100644 index 87fe71c3c..000000000 --- a/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go +++ /dev/null @@ -1,25 +0,0 @@ -// Code generated by informer-gen. DO NOT EDIT. - -package internalinterfaces - -import ( - time "time" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - cache "k8s.io/client-go/tools/cache" - - versioned "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" -) - -// NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. -type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer - -// SharedInformerFactory a small interface to allow for adding an informer without an import cycle -type SharedInformerFactory interface { - Start(stopCh <-chan struct{}) - InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer -} - -// TweakListOptionsFunc is a function that transforms a v1.ListOptions. -type TweakListOptionsFunc func(*v1.ListOptions) diff --git a/pkg/client/informers/externalversions/sriovnetwork/interface.go b/pkg/client/informers/externalversions/sriovnetwork/interface.go deleted file mode 100644 index 5eaaab532..000000000 --- a/pkg/client/informers/externalversions/sriovnetwork/interface.go +++ /dev/null @@ -1,30 +0,0 @@ -// Code generated by informer-gen. DO NOT EDIT. - -package sriovnetwork - -import ( - internalinterfaces "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/informers/externalversions/internalinterfaces" - v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/informers/externalversions/sriovnetwork/v1" -) - -// Interface provides access to each of this group's versions. -type Interface interface { - // V1 provides access to shared informers for resources in V1. - V1() v1.Interface -} - -type group struct { - factory internalinterfaces.SharedInformerFactory - namespace string - tweakListOptions internalinterfaces.TweakListOptionsFunc -} - -// New returns a new Interface. -func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { - return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} -} - -// V1 returns a new v1.Interface. -func (g *group) V1() v1.Interface { - return v1.New(g.factory, g.namespace, g.tweakListOptions) -} diff --git a/pkg/client/informers/externalversions/sriovnetwork/v1/interface.go b/pkg/client/informers/externalversions/sriovnetwork/v1/interface.go deleted file mode 100644 index 396adb52f..000000000 --- a/pkg/client/informers/externalversions/sriovnetwork/v1/interface.go +++ /dev/null @@ -1,50 +0,0 @@ -// Code generated by informer-gen. DO NOT EDIT. - -package v1 - -import ( - internalinterfaces "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/informers/externalversions/internalinterfaces" -) - -// Interface provides access to all the informers in this group version. -type Interface interface { - // SriovNetworks returns a SriovNetworkInformer. - SriovNetworks() SriovNetworkInformer - // SriovNetworkNodePolicies returns a SriovNetworkNodePolicyInformer. - SriovNetworkNodePolicies() SriovNetworkNodePolicyInformer - // SriovNetworkNodeStates returns a SriovNetworkNodeStateInformer. - SriovNetworkNodeStates() SriovNetworkNodeStateInformer - // SriovOperatorConfigs returns a SriovOperatorConfigInformer. - SriovOperatorConfigs() SriovOperatorConfigInformer -} - -type version struct { - factory internalinterfaces.SharedInformerFactory - namespace string - tweakListOptions internalinterfaces.TweakListOptionsFunc -} - -// New returns a new Interface. -func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { - return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} -} - -// SriovNetworks returns a SriovNetworkInformer. -func (v *version) SriovNetworks() SriovNetworkInformer { - return &sriovNetworkInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} -} - -// SriovNetworkNodePolicies returns a SriovNetworkNodePolicyInformer. -func (v *version) SriovNetworkNodePolicies() SriovNetworkNodePolicyInformer { - return &sriovNetworkNodePolicyInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} -} - -// SriovNetworkNodeStates returns a SriovNetworkNodeStateInformer. -func (v *version) SriovNetworkNodeStates() SriovNetworkNodeStateInformer { - return &sriovNetworkNodeStateInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} -} - -// SriovOperatorConfigs returns a SriovOperatorConfigInformer. -func (v *version) SriovOperatorConfigs() SriovOperatorConfigInformer { - return &sriovOperatorConfigInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} -} diff --git a/pkg/client/informers/externalversions/sriovnetwork/v1/sriovnetwork.go b/pkg/client/informers/externalversions/sriovnetwork/v1/sriovnetwork.go deleted file mode 100644 index f5bed4a9a..000000000 --- a/pkg/client/informers/externalversions/sriovnetwork/v1/sriovnetwork.go +++ /dev/null @@ -1,75 +0,0 @@ -// Code generated by informer-gen. DO NOT EDIT. - -package v1 - -import ( - "context" - time "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - watch "k8s.io/apimachinery/pkg/watch" - cache "k8s.io/client-go/tools/cache" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - versioned "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" - internalinterfaces "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/informers/externalversions/internalinterfaces" - v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/listers/sriovnetwork/v1" -) - -// SriovNetworkInformer provides access to a shared informer and lister for -// SriovNetworks. -type SriovNetworkInformer interface { - Informer() cache.SharedIndexInformer - Lister() v1.SriovNetworkLister -} - -type sriovNetworkInformer struct { - factory internalinterfaces.SharedInformerFactory - tweakListOptions internalinterfaces.TweakListOptionsFunc - namespace string -} - -// NewSriovNetworkInformer constructs a new informer for SriovNetwork type. -// Always prefer using an informer factory to get a shared informer instead of getting an independent -// one. This reduces memory footprint and number of connections to the server. -func NewSriovNetworkInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { - return NewFilteredSriovNetworkInformer(client, namespace, resyncPeriod, indexers, nil) -} - -// NewFilteredSriovNetworkInformer constructs a new informer for SriovNetwork type. -// Always prefer using an informer factory to get a shared informer instead of getting an independent -// one. This reduces memory footprint and number of connections to the server. -func NewFilteredSriovNetworkInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { - return cache.NewSharedIndexInformer( - &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - if tweakListOptions != nil { - tweakListOptions(&options) - } - return client.SriovnetworkV1().SriovNetworks(namespace).List(context.TODO(), options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - if tweakListOptions != nil { - tweakListOptions(&options) - } - return client.SriovnetworkV1().SriovNetworks(namespace).Watch(context.TODO(), options) - }, - }, - &sriovnetworkv1.SriovNetwork{}, - resyncPeriod, - indexers, - ) -} - -func (f *sriovNetworkInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { - return NewFilteredSriovNetworkInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) -} - -func (f *sriovNetworkInformer) Informer() cache.SharedIndexInformer { - return f.factory.InformerFor(&sriovnetworkv1.SriovNetwork{}, f.defaultInformer) -} - -func (f *sriovNetworkInformer) Lister() v1.SriovNetworkLister { - return v1.NewSriovNetworkLister(f.Informer().GetIndexer()) -} diff --git a/pkg/client/informers/externalversions/sriovnetwork/v1/sriovnetworknodepolicy.go b/pkg/client/informers/externalversions/sriovnetwork/v1/sriovnetworknodepolicy.go deleted file mode 100644 index df6879ee9..000000000 --- a/pkg/client/informers/externalversions/sriovnetwork/v1/sriovnetworknodepolicy.go +++ /dev/null @@ -1,75 +0,0 @@ -// Code generated by informer-gen. DO NOT EDIT. - -package v1 - -import ( - "context" - time "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - watch "k8s.io/apimachinery/pkg/watch" - cache "k8s.io/client-go/tools/cache" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - versioned "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" - internalinterfaces "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/informers/externalversions/internalinterfaces" - v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/listers/sriovnetwork/v1" -) - -// SriovNetworkNodePolicyInformer provides access to a shared informer and lister for -// SriovNetworkNodePolicies. -type SriovNetworkNodePolicyInformer interface { - Informer() cache.SharedIndexInformer - Lister() v1.SriovNetworkNodePolicyLister -} - -type sriovNetworkNodePolicyInformer struct { - factory internalinterfaces.SharedInformerFactory - tweakListOptions internalinterfaces.TweakListOptionsFunc - namespace string -} - -// NewSriovNetworkNodePolicyInformer constructs a new informer for SriovNetworkNodePolicy type. -// Always prefer using an informer factory to get a shared informer instead of getting an independent -// one. This reduces memory footprint and number of connections to the server. -func NewSriovNetworkNodePolicyInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { - return NewFilteredSriovNetworkNodePolicyInformer(client, namespace, resyncPeriod, indexers, nil) -} - -// NewFilteredSriovNetworkNodePolicyInformer constructs a new informer for SriovNetworkNodePolicy type. -// Always prefer using an informer factory to get a shared informer instead of getting an independent -// one. This reduces memory footprint and number of connections to the server. -func NewFilteredSriovNetworkNodePolicyInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { - return cache.NewSharedIndexInformer( - &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - if tweakListOptions != nil { - tweakListOptions(&options) - } - return client.SriovnetworkV1().SriovNetworkNodePolicies(namespace).List(context.TODO(), options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - if tweakListOptions != nil { - tweakListOptions(&options) - } - return client.SriovnetworkV1().SriovNetworkNodePolicies(namespace).Watch(context.TODO(), options) - }, - }, - &sriovnetworkv1.SriovNetworkNodePolicy{}, - resyncPeriod, - indexers, - ) -} - -func (f *sriovNetworkNodePolicyInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { - return NewFilteredSriovNetworkNodePolicyInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) -} - -func (f *sriovNetworkNodePolicyInformer) Informer() cache.SharedIndexInformer { - return f.factory.InformerFor(&sriovnetworkv1.SriovNetworkNodePolicy{}, f.defaultInformer) -} - -func (f *sriovNetworkNodePolicyInformer) Lister() v1.SriovNetworkNodePolicyLister { - return v1.NewSriovNetworkNodePolicyLister(f.Informer().GetIndexer()) -} diff --git a/pkg/client/informers/externalversions/sriovnetwork/v1/sriovnetworknodestate.go b/pkg/client/informers/externalversions/sriovnetwork/v1/sriovnetworknodestate.go deleted file mode 100644 index 69f5d0b69..000000000 --- a/pkg/client/informers/externalversions/sriovnetwork/v1/sriovnetworknodestate.go +++ /dev/null @@ -1,75 +0,0 @@ -// Code generated by informer-gen. DO NOT EDIT. - -package v1 - -import ( - "context" - time "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - watch "k8s.io/apimachinery/pkg/watch" - cache "k8s.io/client-go/tools/cache" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - versioned "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" - internalinterfaces "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/informers/externalversions/internalinterfaces" - v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/listers/sriovnetwork/v1" -) - -// SriovNetworkNodeStateInformer provides access to a shared informer and lister for -// SriovNetworkNodeStates. -type SriovNetworkNodeStateInformer interface { - Informer() cache.SharedIndexInformer - Lister() v1.SriovNetworkNodeStateLister -} - -type sriovNetworkNodeStateInformer struct { - factory internalinterfaces.SharedInformerFactory - tweakListOptions internalinterfaces.TweakListOptionsFunc - namespace string -} - -// NewSriovNetworkNodeStateInformer constructs a new informer for SriovNetworkNodeState type. -// Always prefer using an informer factory to get a shared informer instead of getting an independent -// one. This reduces memory footprint and number of connections to the server. -func NewSriovNetworkNodeStateInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { - return NewFilteredSriovNetworkNodeStateInformer(client, namespace, resyncPeriod, indexers, nil) -} - -// NewFilteredSriovNetworkNodeStateInformer constructs a new informer for SriovNetworkNodeState type. -// Always prefer using an informer factory to get a shared informer instead of getting an independent -// one. This reduces memory footprint and number of connections to the server. -func NewFilteredSriovNetworkNodeStateInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { - return cache.NewSharedIndexInformer( - &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - if tweakListOptions != nil { - tweakListOptions(&options) - } - return client.SriovnetworkV1().SriovNetworkNodeStates(namespace).List(context.TODO(), options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - if tweakListOptions != nil { - tweakListOptions(&options) - } - return client.SriovnetworkV1().SriovNetworkNodeStates(namespace).Watch(context.TODO(), options) - }, - }, - &sriovnetworkv1.SriovNetworkNodeState{}, - resyncPeriod, - indexers, - ) -} - -func (f *sriovNetworkNodeStateInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { - return NewFilteredSriovNetworkNodeStateInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) -} - -func (f *sriovNetworkNodeStateInformer) Informer() cache.SharedIndexInformer { - return f.factory.InformerFor(&sriovnetworkv1.SriovNetworkNodeState{}, f.defaultInformer) -} - -func (f *sriovNetworkNodeStateInformer) Lister() v1.SriovNetworkNodeStateLister { - return v1.NewSriovNetworkNodeStateLister(f.Informer().GetIndexer()) -} diff --git a/pkg/client/informers/externalversions/sriovnetwork/v1/sriovoperatorconfig.go b/pkg/client/informers/externalversions/sriovnetwork/v1/sriovoperatorconfig.go deleted file mode 100644 index 142f261e2..000000000 --- a/pkg/client/informers/externalversions/sriovnetwork/v1/sriovoperatorconfig.go +++ /dev/null @@ -1,75 +0,0 @@ -// Code generated by informer-gen. DO NOT EDIT. - -package v1 - -import ( - "context" - time "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - watch "k8s.io/apimachinery/pkg/watch" - cache "k8s.io/client-go/tools/cache" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - versioned "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" - internalinterfaces "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/informers/externalversions/internalinterfaces" - v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/listers/sriovnetwork/v1" -) - -// SriovOperatorConfigInformer provides access to a shared informer and lister for -// SriovOperatorConfigs. -type SriovOperatorConfigInformer interface { - Informer() cache.SharedIndexInformer - Lister() v1.SriovOperatorConfigLister -} - -type sriovOperatorConfigInformer struct { - factory internalinterfaces.SharedInformerFactory - tweakListOptions internalinterfaces.TweakListOptionsFunc - namespace string -} - -// NewSriovOperatorConfigInformer constructs a new informer for SriovOperatorConfig type. -// Always prefer using an informer factory to get a shared informer instead of getting an independent -// one. This reduces memory footprint and number of connections to the server. -func NewSriovOperatorConfigInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { - return NewFilteredSriovOperatorConfigInformer(client, namespace, resyncPeriod, indexers, nil) -} - -// NewFilteredSriovOperatorConfigInformer constructs a new informer for SriovOperatorConfig type. -// Always prefer using an informer factory to get a shared informer instead of getting an independent -// one. This reduces memory footprint and number of connections to the server. -func NewFilteredSriovOperatorConfigInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { - return cache.NewSharedIndexInformer( - &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - if tweakListOptions != nil { - tweakListOptions(&options) - } - return client.SriovnetworkV1().SriovOperatorConfigs(namespace).List(context.TODO(), options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - if tweakListOptions != nil { - tweakListOptions(&options) - } - return client.SriovnetworkV1().SriovOperatorConfigs(namespace).Watch(context.TODO(), options) - }, - }, - &sriovnetworkv1.SriovOperatorConfig{}, - resyncPeriod, - indexers, - ) -} - -func (f *sriovOperatorConfigInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { - return NewFilteredSriovOperatorConfigInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) -} - -func (f *sriovOperatorConfigInformer) Informer() cache.SharedIndexInformer { - return f.factory.InformerFor(&sriovnetworkv1.SriovOperatorConfig{}, f.defaultInformer) -} - -func (f *sriovOperatorConfigInformer) Lister() v1.SriovOperatorConfigLister { - return v1.NewSriovOperatorConfigLister(f.Informer().GetIndexer()) -} diff --git a/pkg/client/listers/sriovnetwork/v1/expansion_generated.go b/pkg/client/listers/sriovnetwork/v1/expansion_generated.go deleted file mode 100644 index 4494d6736..000000000 --- a/pkg/client/listers/sriovnetwork/v1/expansion_generated.go +++ /dev/null @@ -1,35 +0,0 @@ -// Code generated by lister-gen. DO NOT EDIT. - -package v1 - -// SriovNetworkListerExpansion allows custom methods to be added to -// SriovNetworkLister. -type SriovNetworkListerExpansion interface{} - -// SriovNetworkNamespaceListerExpansion allows custom methods to be added to -// SriovNetworkNamespaceLister. -type SriovNetworkNamespaceListerExpansion interface{} - -// SriovNetworkNodePolicyListerExpansion allows custom methods to be added to -// SriovNetworkNodePolicyLister. -type SriovNetworkNodePolicyListerExpansion interface{} - -// SriovNetworkNodePolicyNamespaceListerExpansion allows custom methods to be added to -// SriovNetworkNodePolicyNamespaceLister. -type SriovNetworkNodePolicyNamespaceListerExpansion interface{} - -// SriovNetworkNodeStateListerExpansion allows custom methods to be added to -// SriovNetworkNodeStateLister. -type SriovNetworkNodeStateListerExpansion interface{} - -// SriovNetworkNodeStateNamespaceListerExpansion allows custom methods to be added to -// SriovNetworkNodeStateNamespaceLister. -type SriovNetworkNodeStateNamespaceListerExpansion interface{} - -// SriovOperatorConfigListerExpansion allows custom methods to be added to -// SriovOperatorConfigLister. -type SriovOperatorConfigListerExpansion interface{} - -// SriovOperatorConfigNamespaceListerExpansion allows custom methods to be added to -// SriovOperatorConfigNamespaceLister. -type SriovOperatorConfigNamespaceListerExpansion interface{} diff --git a/pkg/client/listers/sriovnetwork/v1/sriovnetwork.go b/pkg/client/listers/sriovnetwork/v1/sriovnetwork.go deleted file mode 100644 index a0eebb626..000000000 --- a/pkg/client/listers/sriovnetwork/v1/sriovnetwork.go +++ /dev/null @@ -1,84 +0,0 @@ -// Code generated by lister-gen. DO NOT EDIT. - -package v1 - -import ( - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/tools/cache" - - v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" -) - -// SriovNetworkLister helps list SriovNetworks. -// All objects returned here must be treated as read-only. -type SriovNetworkLister interface { - // List lists all SriovNetworks in the indexer. - // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1.SriovNetwork, err error) - // SriovNetworks returns an object that can list and get SriovNetworks. - SriovNetworks(namespace string) SriovNetworkNamespaceLister - SriovNetworkListerExpansion -} - -// sriovNetworkLister implements the SriovNetworkLister interface. -type sriovNetworkLister struct { - indexer cache.Indexer -} - -// NewSriovNetworkLister returns a new SriovNetworkLister. -func NewSriovNetworkLister(indexer cache.Indexer) SriovNetworkLister { - return &sriovNetworkLister{indexer: indexer} -} - -// List lists all SriovNetworks in the indexer. -func (s *sriovNetworkLister) List(selector labels.Selector) (ret []*v1.SriovNetwork, err error) { - err = cache.ListAll(s.indexer, selector, func(m interface{}) { - ret = append(ret, m.(*v1.SriovNetwork)) - }) - return ret, err -} - -// SriovNetworks returns an object that can list and get SriovNetworks. -func (s *sriovNetworkLister) SriovNetworks(namespace string) SriovNetworkNamespaceLister { - return sriovNetworkNamespaceLister{indexer: s.indexer, namespace: namespace} -} - -// SriovNetworkNamespaceLister helps list and get SriovNetworks. -// All objects returned here must be treated as read-only. -type SriovNetworkNamespaceLister interface { - // List lists all SriovNetworks in the indexer for a given namespace. - // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1.SriovNetwork, err error) - // Get retrieves the SriovNetwork from the indexer for a given namespace and name. - // Objects returned here must be treated as read-only. - Get(name string) (*v1.SriovNetwork, error) - SriovNetworkNamespaceListerExpansion -} - -// sriovNetworkNamespaceLister implements the SriovNetworkNamespaceLister -// interface. -type sriovNetworkNamespaceLister struct { - indexer cache.Indexer - namespace string -} - -// List lists all SriovNetworks in the indexer for a given namespace. -func (s sriovNetworkNamespaceLister) List(selector labels.Selector) (ret []*v1.SriovNetwork, err error) { - err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { - ret = append(ret, m.(*v1.SriovNetwork)) - }) - return ret, err -} - -// Get retrieves the SriovNetwork from the indexer for a given namespace and name. -func (s sriovNetworkNamespaceLister) Get(name string) (*v1.SriovNetwork, error) { - obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) - if err != nil { - return nil, err - } - if !exists { - return nil, errors.NewNotFound(v1.Resource("sriovnetwork"), name) - } - return obj.(*v1.SriovNetwork), nil -} diff --git a/pkg/client/listers/sriovnetwork/v1/sriovnetworknodepolicy.go b/pkg/client/listers/sriovnetwork/v1/sriovnetworknodepolicy.go deleted file mode 100644 index 8e81010a9..000000000 --- a/pkg/client/listers/sriovnetwork/v1/sriovnetworknodepolicy.go +++ /dev/null @@ -1,84 +0,0 @@ -// Code generated by lister-gen. DO NOT EDIT. - -package v1 - -import ( - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/tools/cache" - - v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" -) - -// SriovNetworkNodePolicyLister helps list SriovNetworkNodePolicies. -// All objects returned here must be treated as read-only. -type SriovNetworkNodePolicyLister interface { - // List lists all SriovNetworkNodePolicies in the indexer. - // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1.SriovNetworkNodePolicy, err error) - // SriovNetworkNodePolicies returns an object that can list and get SriovNetworkNodePolicies. - SriovNetworkNodePolicies(namespace string) SriovNetworkNodePolicyNamespaceLister - SriovNetworkNodePolicyListerExpansion -} - -// sriovNetworkNodePolicyLister implements the SriovNetworkNodePolicyLister interface. -type sriovNetworkNodePolicyLister struct { - indexer cache.Indexer -} - -// NewSriovNetworkNodePolicyLister returns a new SriovNetworkNodePolicyLister. -func NewSriovNetworkNodePolicyLister(indexer cache.Indexer) SriovNetworkNodePolicyLister { - return &sriovNetworkNodePolicyLister{indexer: indexer} -} - -// List lists all SriovNetworkNodePolicies in the indexer. -func (s *sriovNetworkNodePolicyLister) List(selector labels.Selector) (ret []*v1.SriovNetworkNodePolicy, err error) { - err = cache.ListAll(s.indexer, selector, func(m interface{}) { - ret = append(ret, m.(*v1.SriovNetworkNodePolicy)) - }) - return ret, err -} - -// SriovNetworkNodePolicies returns an object that can list and get SriovNetworkNodePolicies. -func (s *sriovNetworkNodePolicyLister) SriovNetworkNodePolicies(namespace string) SriovNetworkNodePolicyNamespaceLister { - return sriovNetworkNodePolicyNamespaceLister{indexer: s.indexer, namespace: namespace} -} - -// SriovNetworkNodePolicyNamespaceLister helps list and get SriovNetworkNodePolicies. -// All objects returned here must be treated as read-only. -type SriovNetworkNodePolicyNamespaceLister interface { - // List lists all SriovNetworkNodePolicies in the indexer for a given namespace. - // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1.SriovNetworkNodePolicy, err error) - // Get retrieves the SriovNetworkNodePolicy from the indexer for a given namespace and name. - // Objects returned here must be treated as read-only. - Get(name string) (*v1.SriovNetworkNodePolicy, error) - SriovNetworkNodePolicyNamespaceListerExpansion -} - -// sriovNetworkNodePolicyNamespaceLister implements the SriovNetworkNodePolicyNamespaceLister -// interface. -type sriovNetworkNodePolicyNamespaceLister struct { - indexer cache.Indexer - namespace string -} - -// List lists all SriovNetworkNodePolicies in the indexer for a given namespace. -func (s sriovNetworkNodePolicyNamespaceLister) List(selector labels.Selector) (ret []*v1.SriovNetworkNodePolicy, err error) { - err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { - ret = append(ret, m.(*v1.SriovNetworkNodePolicy)) - }) - return ret, err -} - -// Get retrieves the SriovNetworkNodePolicy from the indexer for a given namespace and name. -func (s sriovNetworkNodePolicyNamespaceLister) Get(name string) (*v1.SriovNetworkNodePolicy, error) { - obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) - if err != nil { - return nil, err - } - if !exists { - return nil, errors.NewNotFound(v1.Resource("sriovnetworknodepolicy"), name) - } - return obj.(*v1.SriovNetworkNodePolicy), nil -} diff --git a/pkg/client/listers/sriovnetwork/v1/sriovnetworknodestate.go b/pkg/client/listers/sriovnetwork/v1/sriovnetworknodestate.go deleted file mode 100644 index efc53d6ee..000000000 --- a/pkg/client/listers/sriovnetwork/v1/sriovnetworknodestate.go +++ /dev/null @@ -1,84 +0,0 @@ -// Code generated by lister-gen. DO NOT EDIT. - -package v1 - -import ( - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/tools/cache" - - v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" -) - -// SriovNetworkNodeStateLister helps list SriovNetworkNodeStates. -// All objects returned here must be treated as read-only. -type SriovNetworkNodeStateLister interface { - // List lists all SriovNetworkNodeStates in the indexer. - // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1.SriovNetworkNodeState, err error) - // SriovNetworkNodeStates returns an object that can list and get SriovNetworkNodeStates. - SriovNetworkNodeStates(namespace string) SriovNetworkNodeStateNamespaceLister - SriovNetworkNodeStateListerExpansion -} - -// sriovNetworkNodeStateLister implements the SriovNetworkNodeStateLister interface. -type sriovNetworkNodeStateLister struct { - indexer cache.Indexer -} - -// NewSriovNetworkNodeStateLister returns a new SriovNetworkNodeStateLister. -func NewSriovNetworkNodeStateLister(indexer cache.Indexer) SriovNetworkNodeStateLister { - return &sriovNetworkNodeStateLister{indexer: indexer} -} - -// List lists all SriovNetworkNodeStates in the indexer. -func (s *sriovNetworkNodeStateLister) List(selector labels.Selector) (ret []*v1.SriovNetworkNodeState, err error) { - err = cache.ListAll(s.indexer, selector, func(m interface{}) { - ret = append(ret, m.(*v1.SriovNetworkNodeState)) - }) - return ret, err -} - -// SriovNetworkNodeStates returns an object that can list and get SriovNetworkNodeStates. -func (s *sriovNetworkNodeStateLister) SriovNetworkNodeStates(namespace string) SriovNetworkNodeStateNamespaceLister { - return sriovNetworkNodeStateNamespaceLister{indexer: s.indexer, namespace: namespace} -} - -// SriovNetworkNodeStateNamespaceLister helps list and get SriovNetworkNodeStates. -// All objects returned here must be treated as read-only. -type SriovNetworkNodeStateNamespaceLister interface { - // List lists all SriovNetworkNodeStates in the indexer for a given namespace. - // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1.SriovNetworkNodeState, err error) - // Get retrieves the SriovNetworkNodeState from the indexer for a given namespace and name. - // Objects returned here must be treated as read-only. - Get(name string) (*v1.SriovNetworkNodeState, error) - SriovNetworkNodeStateNamespaceListerExpansion -} - -// sriovNetworkNodeStateNamespaceLister implements the SriovNetworkNodeStateNamespaceLister -// interface. -type sriovNetworkNodeStateNamespaceLister struct { - indexer cache.Indexer - namespace string -} - -// List lists all SriovNetworkNodeStates in the indexer for a given namespace. -func (s sriovNetworkNodeStateNamespaceLister) List(selector labels.Selector) (ret []*v1.SriovNetworkNodeState, err error) { - err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { - ret = append(ret, m.(*v1.SriovNetworkNodeState)) - }) - return ret, err -} - -// Get retrieves the SriovNetworkNodeState from the indexer for a given namespace and name. -func (s sriovNetworkNodeStateNamespaceLister) Get(name string) (*v1.SriovNetworkNodeState, error) { - obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) - if err != nil { - return nil, err - } - if !exists { - return nil, errors.NewNotFound(v1.Resource("sriovnetworknodestate"), name) - } - return obj.(*v1.SriovNetworkNodeState), nil -} diff --git a/pkg/client/listers/sriovnetwork/v1/sriovoperatorconfig.go b/pkg/client/listers/sriovnetwork/v1/sriovoperatorconfig.go deleted file mode 100644 index 9bad2022a..000000000 --- a/pkg/client/listers/sriovnetwork/v1/sriovoperatorconfig.go +++ /dev/null @@ -1,84 +0,0 @@ -// Code generated by lister-gen. DO NOT EDIT. - -package v1 - -import ( - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/tools/cache" - - v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" -) - -// SriovOperatorConfigLister helps list SriovOperatorConfigs. -// All objects returned here must be treated as read-only. -type SriovOperatorConfigLister interface { - // List lists all SriovOperatorConfigs in the indexer. - // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1.SriovOperatorConfig, err error) - // SriovOperatorConfigs returns an object that can list and get SriovOperatorConfigs. - SriovOperatorConfigs(namespace string) SriovOperatorConfigNamespaceLister - SriovOperatorConfigListerExpansion -} - -// sriovOperatorConfigLister implements the SriovOperatorConfigLister interface. -type sriovOperatorConfigLister struct { - indexer cache.Indexer -} - -// NewSriovOperatorConfigLister returns a new SriovOperatorConfigLister. -func NewSriovOperatorConfigLister(indexer cache.Indexer) SriovOperatorConfigLister { - return &sriovOperatorConfigLister{indexer: indexer} -} - -// List lists all SriovOperatorConfigs in the indexer. -func (s *sriovOperatorConfigLister) List(selector labels.Selector) (ret []*v1.SriovOperatorConfig, err error) { - err = cache.ListAll(s.indexer, selector, func(m interface{}) { - ret = append(ret, m.(*v1.SriovOperatorConfig)) - }) - return ret, err -} - -// SriovOperatorConfigs returns an object that can list and get SriovOperatorConfigs. -func (s *sriovOperatorConfigLister) SriovOperatorConfigs(namespace string) SriovOperatorConfigNamespaceLister { - return sriovOperatorConfigNamespaceLister{indexer: s.indexer, namespace: namespace} -} - -// SriovOperatorConfigNamespaceLister helps list and get SriovOperatorConfigs. -// All objects returned here must be treated as read-only. -type SriovOperatorConfigNamespaceLister interface { - // List lists all SriovOperatorConfigs in the indexer for a given namespace. - // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1.SriovOperatorConfig, err error) - // Get retrieves the SriovOperatorConfig from the indexer for a given namespace and name. - // Objects returned here must be treated as read-only. - Get(name string) (*v1.SriovOperatorConfig, error) - SriovOperatorConfigNamespaceListerExpansion -} - -// sriovOperatorConfigNamespaceLister implements the SriovOperatorConfigNamespaceLister -// interface. -type sriovOperatorConfigNamespaceLister struct { - indexer cache.Indexer - namespace string -} - -// List lists all SriovOperatorConfigs in the indexer for a given namespace. -func (s sriovOperatorConfigNamespaceLister) List(selector labels.Selector) (ret []*v1.SriovOperatorConfig, err error) { - err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { - ret = append(ret, m.(*v1.SriovOperatorConfig)) - }) - return ret, err -} - -// Get retrieves the SriovOperatorConfig from the indexer for a given namespace and name. -func (s sriovOperatorConfigNamespaceLister) Get(name string) (*v1.SriovOperatorConfig, error) { - obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) - if err != nil { - return nil, err - } - if !exists { - return nil, errors.NewNotFound(v1.Resource("sriovoperatorconfig"), name) - } - return obj.(*v1.SriovOperatorConfig), nil -} diff --git a/pkg/consts/constants.go b/pkg/consts/constants.go index f3c076111..2a26c784e 100644 --- a/pkg/consts/constants.go +++ b/pkg/consts/constants.go @@ -14,7 +14,10 @@ const ( Chroot = "/host" Host = "/host" - ResyncPeriod = 5 * time.Minute + ResyncPeriod = 5 * time.Minute + DaemonRequeueTime = 30 * time.Second + DrainControllerRequeueTime = 5 * time.Second + DefaultConfigName = "default" ConfigDaemonPath = "./bindata/manifests/daemon" InjectorWebHookPath = "./bindata/manifests/webhook" @@ -36,7 +39,6 @@ const ( ServiceAccount = "ServiceAccount" DPConfigFileName = "config.json" OVSHWOLMachineConfigNameSuffix = "ovs-hw-offload" - LeaderElectionID = "a56def2a.openshift.io" LinkTypeEthernet = "ether" LinkTypeInfiniband = "infiniband" @@ -54,6 +56,9 @@ const ( VdpaTypeVirtio = "virtio" VdpaTypeVhost = "vhost" + RdmaSubsystemModeShared = "shared" + RdmaSubsystemModeExclusive = "exclusive" + ClusterTypeOpenshift = "openshift" ClusterTypeKubernetes = "kubernetes" @@ -67,6 +72,10 @@ const ( MachineConfigPoolPausedAnnotationIdle = "Idle" MachineConfigPoolPausedAnnotationPaused = "Paused" + SriovDevicePluginLabel = "sriovnetwork.openshift.io/device-plugin" + SriovDevicePluginLabelEnabled = "Enabled" + SriovDevicePluginLabelDisabled = "Disabled" + NodeDrainAnnotation = "sriovnetwork.openshift.io/state" NodeStateDrainAnnotation = "sriovnetwork.openshift.io/desired-state" NodeStateDrainAnnotationCurrent = "sriovnetwork.openshift.io/current-state" @@ -81,11 +90,21 @@ const ( SyncStatusInProgress = "InProgress" DrainDeleted = "Deleted" + DrainDelete = "delete" DrainEvicted = "Evicted" + DrainEvict = "evict" MCPPauseAnnotationState = "sriovnetwork.openshift.io/state" MCPPauseAnnotationTime = "sriovnetwork.openshift.io/time" + // NodeStateKeepUntilAnnotation contains name of the "keep until time" annotation for SriovNetworkNodeState object. + // The "keep until time" specifies the earliest time at which the state object can be removed + // if the daemon's pod is not found on the node. + NodeStateKeepUntilAnnotation = "sriovnetwork.openshift.io/keep-state-until" + // DefaultNodeStateCleanupDelayMinutes contains default delay before removing stale SriovNetworkNodeState CRs + // (the CRs that no longer have a corresponding node with the daemon). + DefaultNodeStateCleanupDelayMinutes = 30 + CheckpointFileName = "sno-initial-node-state.json" Unknown = "Unknown" @@ -121,9 +140,22 @@ const ( `IMPORT{program}="/etc/udev/switchdev-vf-link-name.sh $attr{phys_port_name}", ` + `NAME="%s_$env{NUMBER}"` - KernelArgPciRealloc = "pci=realloc" - KernelArgIntelIommu = "intel_iommu=on" - KernelArgIommuPt = "iommu=pt" + KernelArgPciRealloc = "pci=realloc" + KernelArgIntelIommu = "intel_iommu=on" + KernelArgIommuPt = "iommu=pt" + KernelArgIommuOn = "iommu=on" + KernelArgRdmaShared = "ib_core.netns_mode=1" + KernelArgRdmaExclusive = "ib_core.netns_mode=0" + + // Systemd consts + SriovSystemdConfigPath = SriovConfBasePath + "/sriov-interface-config.yaml" + SriovSystemdResultPath = SriovConfBasePath + "/sriov-interface-result.yaml" + SriovSystemdSupportedNicPath = SriovConfBasePath + "/sriov-supported-nics-ids.yaml" + SriovSystemdServiceBinaryPath = "/var/lib/sriov/sriov-network-config-daemon" + + SriovServiceBasePath = "/etc/systemd/system" + SriovServicePath = SriovServiceBasePath + "/sriov-config.service" + SriovPostNetworkServicePath = SriovServiceBasePath + "/sriov-config-post-network.service" // Feature gates // ParallelNicConfigFeatureGate: allow to configure nics in parallel diff --git a/pkg/daemon/config.go b/pkg/daemon/config.go new file mode 100644 index 000000000..c7918bb9d --- /dev/null +++ b/pkg/daemon/config.go @@ -0,0 +1,65 @@ +package daemon + +import ( + "context" + + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" +) + +// OperatorConfigNodeReconcile represents the reconcile struct for the OperatorConfig. +type OperatorConfigNodeReconcile struct { + client client.Client + latestFeatureGates map[string]bool +} + +// NewOperatorConfigNodeReconcile creates a new instance of OperatorConfigNodeReconcile with the given client. +func NewOperatorConfigNodeReconcile(client client.Client) *OperatorConfigNodeReconcile { + return &OperatorConfigNodeReconcile{client: client, latestFeatureGates: make(map[string]bool)} +} + +// Reconcile reconciles the OperatorConfig resource. It updates log level and feature gates as necessary. +func (oc *OperatorConfigNodeReconcile) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + reqLogger := log.FromContext(ctx).WithName("Reconcile") + operatorConfig := &sriovnetworkv1.SriovOperatorConfig{} + err := oc.client.Get(ctx, client.ObjectKey{Namespace: req.Namespace, Name: req.Name}, operatorConfig) + if err != nil { + if errors.IsNotFound(err) { + reqLogger.Info("OperatorConfig doesn't exist", "name", req.Name, "namespace", req.Namespace) + return ctrl.Result{}, nil + } + reqLogger.Error(err, "Failed to get OperatorConfig", "name", req.Name, "namespace", req.Namespace) + return ctrl.Result{}, err + } + + // update log level + snolog.SetLogLevel(operatorConfig.Spec.LogLevel) + + newDisableDrain := operatorConfig.Spec.DisableDrain + if vars.DisableDrain != newDisableDrain { + vars.DisableDrain = newDisableDrain + log.Log.Info("Set Disable Drain", "value", vars.DisableDrain) + } + + if !equality.Semantic.DeepEqual(oc.latestFeatureGates, operatorConfig.Spec.FeatureGates) { + vars.FeatureGate.Init(operatorConfig.Spec.FeatureGates) + oc.latestFeatureGates = operatorConfig.Spec.FeatureGates + log.Log.Info("Updated featureGates", "featureGates", vars.FeatureGate.String()) + } + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the reconciliation logic for this controller using the given manager. +func (oc *OperatorConfigNodeReconcile) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&sriovnetworkv1.SriovOperatorConfig{}). + Complete(oc) +} diff --git a/pkg/daemon/config_test.go b/pkg/daemon/config_test.go new file mode 100644 index 000000000..c6095ab91 --- /dev/null +++ b/pkg/daemon/config_test.go @@ -0,0 +1,170 @@ +package daemon_test + +import ( + "context" + "sync" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/daemon" + snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" +) + +var _ = Describe("Daemon OperatorConfig Controller", Ordered, func() { + var cancel context.CancelFunc + var ctx context.Context + + BeforeAll(func() { + By("Setup controller manager") + k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + }) + Expect(err).ToNot(HaveOccurred()) + + configController := daemon.NewOperatorConfigNodeReconcile(k8sClient) + err = configController.SetupWithManager(k8sManager) + Expect(err).ToNot(HaveOccurred()) + + ctx, cancel = context.WithCancel(context.Background()) + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + defer GinkgoRecover() + By("Start controller manager") + err := k8sManager.Start(ctx) + Expect(err).ToNot(HaveOccurred()) + }() + + DeferCleanup(func() { + By("Shutdown controller manager") + cancel() + wg.Wait() + }) + + err = k8sClient.Create(ctx, &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: "default"}}) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + Expect(k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovOperatorConfig{}, client.InNamespace(testNamespace))).ToNot(HaveOccurred()) + }) + + Context("LogLevel", func() { + It("should configure the log level base on sriovOperatorConfig", func() { + soc := &sriovnetworkv1.SriovOperatorConfig{ObjectMeta: metav1.ObjectMeta{ + Name: consts.DefaultConfigName, + Namespace: testNamespace, + }, + Spec: sriovnetworkv1.SriovOperatorConfigSpec{ + LogLevel: 1, + }, + } + + err := k8sClient.Create(ctx, soc) + Expect(err).ToNot(HaveOccurred()) + validateExpectedLogLevel(1) + + }) + + It("should update the log level in runtime", func() { + soc := &sriovnetworkv1.SriovOperatorConfig{ObjectMeta: metav1.ObjectMeta{ + Name: consts.DefaultConfigName, + Namespace: testNamespace, + }, + Spec: sriovnetworkv1.SriovOperatorConfigSpec{ + LogLevel: 1, + }, + } + + err := k8sClient.Create(ctx, soc) + Expect(err).ToNot(HaveOccurred()) + validateExpectedLogLevel(1) + + soc.Spec.LogLevel = 2 + err = k8sClient.Update(ctx, soc) + Expect(err).ToNot(HaveOccurred()) + validateExpectedLogLevel(2) + }) + }) + + Context("Disable Drain", func() { + It("should update the skip drain flag", func() { + soc := &sriovnetworkv1.SriovOperatorConfig{ObjectMeta: metav1.ObjectMeta{ + Name: consts.DefaultConfigName, + Namespace: testNamespace, + }, + Spec: sriovnetworkv1.SriovOperatorConfigSpec{ + DisableDrain: true, + }, + } + + err := k8sClient.Create(ctx, soc) + Expect(err).ToNot(HaveOccurred()) + validateExpectedDrain(true) + + soc.Spec.DisableDrain = false + err = k8sClient.Update(ctx, soc) + Expect(err).ToNot(HaveOccurred()) + validateExpectedDrain(false) + }) + }) + + Context("Feature gates", func() { + It("should update the feature gates struct", func() { + soc := &sriovnetworkv1.SriovOperatorConfig{ObjectMeta: metav1.ObjectMeta{ + Name: consts.DefaultConfigName, + Namespace: testNamespace, + }, + Spec: sriovnetworkv1.SriovOperatorConfigSpec{ + FeatureGates: map[string]bool{ + "test": true, + "bla": true, + }, + }, + } + + err := k8sClient.Create(ctx, soc) + Expect(err).ToNot(HaveOccurred()) + EventuallyWithOffset(1, func(g Gomega) { + g.Expect(vars.FeatureGate.IsEnabled("test")).To(BeTrue()) + }, "15s", "3s").Should(Succeed()) + EventuallyWithOffset(1, func(g Gomega) { + g.Expect(vars.FeatureGate.IsEnabled("bla")).To(BeTrue()) + }, "15s", "3s").Should(Succeed()) + + soc.Spec.FeatureGates["test"] = false + err = k8sClient.Update(ctx, soc) + Expect(err).ToNot(HaveOccurred()) + EventuallyWithOffset(1, func(g Gomega) { + g.Expect(vars.FeatureGate.IsEnabled("test")).To(BeFalse()) + }, "15s", "3s").Should(Succeed()) + EventuallyWithOffset(1, func(g Gomega) { + g.Expect(vars.FeatureGate.IsEnabled("bla")).To(BeTrue()) + }, "15s", "3s").Should(Succeed()) + }) + }) +}) + +func validateExpectedLogLevel(level int) { + EventuallyWithOffset(1, func(g Gomega) { + g.Expect(snolog.GetLogLevel()).To(Equal(level)) + }, "15s", "3s").Should(Succeed()) +} + +func validateExpectedDrain(disableDrain bool) { + EventuallyWithOffset(1, func(g Gomega) { + g.Expect(vars.DisableDrain).To(Equal(disableDrain)) + }, "15s", "3s").Should(Succeed()) +} diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 5ed31ff85..36bf76bbe 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -3,489 +3,364 @@ package daemon import ( "context" "fmt" - "math/rand" - "os/exec" - "reflect" - "sync" "time" - "golang.org/x/time/rate" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/util/workqueue" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - snclientset "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" - sninformer "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/informers/externalversions" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/featuregate" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper" - snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" + hosttypes "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms" plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/systemd" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) -const ( - // updateDelay is the baseline speed at which we react to changes. We don't - // need to react in milliseconds as any change would involve rebooting the node. - updateDelay = 5 * time.Second - // maxUpdateBackoff is the maximum time to react to a change as we back off - // in the face of errors. - maxUpdateBackoff = 60 * time.Second -) - -type Message struct { - syncStatus string - lastSyncError string -} - -type Daemon struct { +// NodeReconciler struct holds various components necessary for reconciling an SR-IOV node. +// It includes a Kubernetes client, SR-IOV client, and other utility interfaces. +// The struct is designed to manage the lifecycle of an SR-IOV devices on a given node. +type NodeReconciler struct { client client.Client - sriovClient snclientset.Interface - // kubeClient allows interaction with Kubernetes, including the node we are running on. - kubeClient kubernetes.Interface - - desiredNodeState *sriovnetworkv1.SriovNetworkNodeState - currentNodeState *sriovnetworkv1.SriovNetworkNodeState - - // list of disabled plugins - disabledPlugins []string - - loadedPlugins map[string]plugin.VendorPlugin - HostHelpers helper.HostHelpersInterface platformHelpers platforms.Interface - // channel used by callbacks to signal Run() of an error - exitCh chan<- error - - // channel used to ensure all spawned goroutines exit when we exit. - stopCh <-chan struct{} - - syncCh <-chan struct{} - - refreshCh chan<- Message - - mu *sync.Mutex - - disableDrain bool - - workqueue workqueue.RateLimitingInterface - eventRecorder *EventRecorder featureGate featuregate.FeatureGate + + // list of disabled plugins + disabledPlugins []string + + loadedPlugins map[string]plugin.VendorPlugin + lastAppliedGeneration int64 } +// New creates a new instance of NodeReconciler. func New( client client.Client, - sriovClient snclientset.Interface, - kubeClient kubernetes.Interface, hostHelpers helper.HostHelpersInterface, platformHelper platforms.Interface, - exitCh chan<- error, - stopCh <-chan struct{}, - syncCh <-chan struct{}, - refreshCh chan<- Message, er *EventRecorder, featureGates featuregate.FeatureGate, disabledPlugins []string, -) *Daemon { - return &Daemon{ - client: client, - sriovClient: sriovClient, - kubeClient: kubeClient, - HostHelpers: hostHelpers, - platformHelpers: platformHelper, - exitCh: exitCh, - stopCh: stopCh, - syncCh: syncCh, - refreshCh: refreshCh, - desiredNodeState: &sriovnetworkv1.SriovNetworkNodeState{}, - currentNodeState: &sriovnetworkv1.SriovNetworkNodeState{}, - workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.NewMaxOfRateLimiter( - &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(updateDelay), 1)}, - workqueue.NewItemExponentialFailureRateLimiter(1*time.Second, maxUpdateBackoff)), "SriovNetworkNodeState"), - eventRecorder: er, - featureGate: featureGates, - disabledPlugins: disabledPlugins, +) *NodeReconciler { + return &NodeReconciler{ + client: client, + HostHelpers: hostHelpers, + platformHelpers: platformHelper, + + lastAppliedGeneration: 0, + eventRecorder: er, + featureGate: featureGates, + disabledPlugins: disabledPlugins, } } -// Run the config daemon -func (dn *Daemon) Run(stopCh <-chan struct{}, exitCh <-chan error) error { - log.Log.V(0).Info("Run()", "node", vars.NodeName) - - if vars.ClusterType == consts.ClusterTypeOpenshift { - log.Log.V(0).Info("Run(): start daemon.", "openshiftFlavor", dn.platformHelpers.GetFlavor()) - } else { - log.Log.V(0).Info("Run(): start daemon.") - } +// Init initializes the Sriov Network Operator daemon. +// It enables kernel modules, prepare udev rules and load the host network state +func (dn *NodeReconciler) Init() error { + funcLog := log.Log.WithName("Init") + var err error if !vars.UsingSystemdMode { - log.Log.V(0).Info("Run(): daemon running in daemon mode") - dn.HostHelpers.CheckRDMAEnabled() + funcLog.V(0).Info("daemon running in daemon mode") + _, err = dn.HostHelpers.CheckRDMAEnabled() + if err != nil { + funcLog.Error(err, "warning, failed to check RDMA state") + } dn.HostHelpers.TryEnableTun() dn.HostHelpers.TryEnableVhostNet() - err := systemd.CleanSriovFilesFromHost(vars.ClusterType == consts.ClusterTypeOpenshift) + err = dn.HostHelpers.CleanSriovFilesFromHost(vars.ClusterType == consts.ClusterTypeOpenshift) if err != nil { - log.Log.Error(err, "failed to remove all the systemd sriov files") + funcLog.Error(err, "failed to remove all the systemd sriov files") } } else { - log.Log.V(0).Info("Run(): daemon running in systemd mode") + funcLog.V(0).Info("Run(): daemon running in systemd mode") } - // Only watch own SriovNetworkNodeState CR - defer utilruntime.HandleCrash() - defer dn.workqueue.ShutDown() - if err := dn.prepareNMUdevRule(); err != nil { - log.Log.Error(err, "failed to prepare udev files to disable network manager on requested VFs") + funcLog.Error(err, "failed to prepare udev files to disable network manager on requested VFs") } if err := dn.HostHelpers.PrepareVFRepUdevRule(); err != nil { - log.Log.Error(err, "failed to prepare udev files to rename VF representors for requested VFs") - } - - var timeout int64 = 5 - var metadataKey = "metadata.name" - dn.mu = &sync.Mutex{} - informerFactory := sninformer.NewFilteredSharedInformerFactory(dn.sriovClient, - time.Second*15, - vars.Namespace, - func(lo *metav1.ListOptions) { - lo.FieldSelector = metadataKey + "=" + vars.NodeName - lo.TimeoutSeconds = &timeout - }, - ) - - informer := informerFactory.Sriovnetwork().V1().SriovNetworkNodeStates().Informer() - informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: dn.enqueueNodeState, - UpdateFunc: func(old, new interface{}) { - dn.enqueueNodeState(new) - }, - }) - - cfgInformerFactory := sninformer.NewFilteredSharedInformerFactory(dn.sriovClient, - time.Second*30, - vars.Namespace, - func(lo *metav1.ListOptions) { - lo.FieldSelector = metadataKey + "=" + "default" - }, - ) - - cfgInformer := cfgInformerFactory.Sriovnetwork().V1().SriovOperatorConfigs().Informer() - cfgInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: dn.operatorConfigAddHandler, - UpdateFunc: dn.operatorConfigChangeHandler, - }) - - rand.Seed(time.Now().UnixNano()) - go cfgInformer.Run(dn.stopCh) - time.Sleep(5 * time.Second) - go informer.Run(dn.stopCh) - if ok := cache.WaitForCacheSync(stopCh, cfgInformer.HasSynced, informer.HasSynced); !ok { - return fmt.Errorf("failed to wait for caches to sync") - } - - log.Log.Info("Starting workers") - // Launch one worker to process - go wait.Until(dn.runWorker, time.Second, stopCh) - log.Log.Info("Started workers") - - for { - select { - case <-stopCh: - log.Log.V(0).Info("Run(): stop daemon") - return nil - case err, more := <-exitCh: - log.Log.Error(err, "got an error") - if more { - dn.refreshCh <- Message{ - syncStatus: consts.SyncStatusFailed, - lastSyncError: err.Error(), - } - } - return err - } - } -} - -func (dn *Daemon) runWorker() { - for dn.processNextWorkItem() { + funcLog.Error(err, "failed to prepare udev files to rename VF representors for requested VFs") } -} -func (dn *Daemon) enqueueNodeState(obj interface{}) { - var ns *sriovnetworkv1.SriovNetworkNodeState - var ok bool - if ns, ok = obj.(*sriovnetworkv1.SriovNetworkNodeState); !ok { - utilruntime.HandleError(fmt.Errorf("expected SriovNetworkNodeState but got %#v", obj)) - return - } - key := ns.GetGeneration() - dn.workqueue.Add(key) -} - -func (dn *Daemon) processNextWorkItem() bool { - log.Log.V(2).Info("processNextWorkItem", "worker-queue-size", dn.workqueue.Len()) - obj, shutdown := dn.workqueue.Get() - if shutdown { - return false - } - - log.Log.V(2).Info("get item from queue", "item", obj.(int64)) - - // We wrap this block in a func so we can defer c.workqueue.Done. - err := func(obj interface{}) error { - // We call Done here so the workqueue knows we have finished - // processing this item. - defer dn.workqueue.Done(obj) - var key int64 - var ok bool - if key, ok = obj.(int64); !ok { - // As the item in the workqueue is actually invalid, we call - // Forget here. - dn.workqueue.Forget(obj) - utilruntime.HandleError(fmt.Errorf("expected workItem in workqueue but got %#v", obj)) - return nil + // init openstack info + if vars.PlatformType == consts.VirtualOpenStack { + ns, err := dn.HostHelpers.GetCheckPointNodeState() + if err != nil { + return err } - err := dn.nodeStateSyncHandler() - if err != nil { - // Ereport error message, and put the item back to work queue for retry. - dn.refreshCh <- Message{ - syncStatus: consts.SyncStatusFailed, - lastSyncError: err.Error(), + if ns == nil { + err = dn.platformHelpers.CreateOpenstackDevicesInfo() + if err != nil { + return err } - <-dn.syncCh - dn.workqueue.AddRateLimited(key) - return fmt.Errorf("error syncing: %s, requeuing", err.Error()) + } else { + dn.platformHelpers.CreateOpenstackDevicesInfoFromNodeStatus(ns) } - // Finally, if no error occurs we Forget this item so it does not - // get queued again until another change happens. - dn.workqueue.Forget(obj) - log.Log.Info("Successfully synced") - return nil - }(obj) + } + // get interfaces + ns := &sriovnetworkv1.SriovNetworkNodeState{} + err = dn.updateStatusFromHost(ns) if err != nil { - utilruntime.HandleError(err) + funcLog.Error(err, "failed to get host network status on init") + return err } - return true -} - -func (dn *Daemon) operatorConfigAddHandler(obj interface{}) { - dn.operatorConfigChangeHandler(&sriovnetworkv1.SriovOperatorConfig{}, obj) -} - -func (dn *Daemon) operatorConfigChangeHandler(old, new interface{}) { - oldCfg := old.(*sriovnetworkv1.SriovOperatorConfig) - newCfg := new.(*sriovnetworkv1.SriovOperatorConfig) - if newCfg.Namespace != vars.Namespace || newCfg.Name != consts.DefaultConfigName { - log.Log.V(2).Info("unsupported SriovOperatorConfig", "namespace", newCfg.Namespace, "name", newCfg.Name) - return + // init vendor plugins + dn.loadedPlugins, err = loadPlugins(ns, dn.HostHelpers, dn.disabledPlugins) + if err != nil { + funcLog.Error(err, "failed to load vendor plugins") + return err } - snolog.SetLogLevel(newCfg.Spec.LogLevel) + // save init state + err = dn.HostHelpers.WriteCheckpointFile(ns) + if err != nil { + funcLog.Error(err, "failed to write checkpoint file on host") + } + return err +} - newDisableDrain := newCfg.Spec.DisableDrain - if dn.disableDrain != newDisableDrain { - dn.disableDrain = newDisableDrain - log.Log.Info("Set Disable Drain", "value", dn.disableDrain) +// Reconcile Reconciles the nodeState object by performing the following steps: +// 1. Retrieves the latest NodeState from the API server. +// 2. Checks if the object has the required drain controller annotations for the current generation. +// 3. Updates the nodeState Status object with the existing network state (interfaces, bridges, and RDMA status). +// 4. If running in systemd mode, checks the sriov result from the config-daemon that runs in systemd. +// 5. Compares the latest generation with the last applied generation to determine if a refresh on NICs is needed. +// 6. Checks for drift between the host state and the nodeState status. +// 7. Updates the sync state of the nodeState object as per the current requirements. +// 8. Determines if a drain is required based on the current state of the nodeState. +// 9. Handles the drain if necessary, ensuring that it does not conflict with other drain requests. +// 10. Applies the changes to the nodeState if there are no issues and updates the sync status accordingly. +// 11. If a reboot is required after applying the changes, returns a result to trigger a reboot. +// +// Returns a Result indicating whether or not the controller should requeue the request for further processing. +func (dn *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + reqLogger := log.FromContext(ctx).WithName("Reconcile") + // Get the latest NodeState + desiredNodeState := &sriovnetworkv1.SriovNetworkNodeState{} + err := dn.client.Get(ctx, client.ObjectKey{Namespace: req.Namespace, Name: req.Name}, desiredNodeState) + if err != nil { + if errors.IsNotFound(err) { + reqLogger.Info("NodeState doesn't exist") + return ctrl.Result{}, nil + } + reqLogger.Error(err, "Failed to fetch node state", "name", vars.NodeName) + return ctrl.Result{}, err } - if !reflect.DeepEqual(oldCfg.Spec.FeatureGates, newCfg.Spec.FeatureGates) { - dn.featureGate.Init(newCfg.Spec.FeatureGates) - log.Log.Info("Updated featureGates", "featureGates", dn.featureGate.String()) + // Check the object as the drain controller annotations + // if not just wait for the drain controller to add them before we start taking care of the nodeState + if !utils.ObjectHasAnnotationKey(desiredNodeState, consts.NodeStateDrainAnnotationCurrent) || + !utils.ObjectHasAnnotationKey(desiredNodeState, consts.NodeStateDrainAnnotation) { + reqLogger.V(2).Info("NodeState doesn't have the current drain annotation") + return ctrl.Result{}, nil } - vars.MlxPluginFwReset = dn.featureGate.IsEnabled(consts.MellanoxFirmwareResetFeatureGate) -} + latest := desiredNodeState.GetGeneration() + current := desiredNodeState.DeepCopy() + reqLogger.V(0).Info("new generation", "generation", latest) -func (dn *Daemon) nodeStateSyncHandler() error { - var err error - // Get the latest NodeState - var sriovResult = &systemd.SriovResult{SyncStatus: consts.SyncStatusSucceeded, LastSyncError: ""} - dn.desiredNodeState, err = dn.sriovClient.SriovnetworkV1().SriovNetworkNodeStates(vars.Namespace).Get(context.Background(), vars.NodeName, metav1.GetOptions{}) + // Update the nodeState Status object with the existing network state (interfaces bridges and rdma status) + err = dn.updateStatusFromHost(desiredNodeState) if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): Failed to fetch node state", "name", vars.NodeName) - return err + reqLogger.Error(err, "failed to get host network status") + return ctrl.Result{}, err } - latest := dn.desiredNodeState.GetGeneration() - log.Log.V(0).Info("nodeStateSyncHandler(): new generation", "generation", latest) - // load plugins if it has not loaded - if len(dn.loadedPlugins) == 0 { - dn.loadedPlugins, err = loadPlugins(dn.desiredNodeState, dn.HostHelpers, dn.disabledPlugins) + // if we are running in systemd mode we want to get the sriov result from the config-daemon that runs in systemd + sriovResult, sriovResultExists, err := dn.checkSystemdStatus() + //TODO: in the case we need to think what to do if we try to apply again or not + if err != nil { + reqLogger.Error(err, "failed to check systemd status unexpected error") + err = dn.updateSyncState(ctx, desiredNodeState, consts.SyncStatusFailed, "unexpected error") if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): failed to enable vendor plugins") - return err + reqLogger.Error(err, "failed to update nodeState status") + return ctrl.Result{}, err } + + return ctrl.Result{}, nil } - skipReconciliation := true - // if the operator complete the drain operator we should continue the configuration - if !dn.isDrainCompleted() { - if vars.UsingSystemdMode && dn.currentNodeState.GetGeneration() == latest { - serviceEnabled, err := dn.HostHelpers.IsServiceEnabled(systemd.SriovServicePath) - if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): failed to check if sriov-config service exist on host") - return err - } - postNetworkServiceEnabled, err := dn.HostHelpers.IsServiceEnabled(systemd.SriovPostNetworkServicePath) - if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): failed to check if sriov-config-post-network service exist on host") - return err - } + // if we are on the latest generation make a refresh on the nics + if dn.lastAppliedGeneration == latest { + isDrifted, err := dn.checkHostStateDrift(ctx, desiredNodeState) + if err != nil { + reqLogger.Error(err, "failed to refresh host state") + return ctrl.Result{}, err + } - // if the service doesn't exist we should continue to let the k8s plugin to create the service files - // this is only for k8s base environments, for openshift the sriov-operator creates a machine config to will apply - // the system service and reboot the node the config-daemon doesn't need to do anything. - if !(serviceEnabled && postNetworkServiceEnabled) { - sriovResult = &systemd.SriovResult{SyncStatus: consts.SyncStatusFailed, - LastSyncError: fmt.Sprintf("some sriov systemd services are not available on node: "+ - "sriov-config available:%t, sriov-config-post-network available:%t", serviceEnabled, postNetworkServiceEnabled)} - } else { - sriovResult, err = systemd.ReadSriovResult() + // if there are no host state drift changes, and we are on the latest applied policy + // we check if we need to publish a new nodeState status if not we requeue + if !isDrifted { + shouldUpdate := dn.shouldUpdateStatus(current, desiredNodeState) + if shouldUpdate { + reqLogger.Info("updating nodeState with new host status") + err = dn.updateSyncState(ctx, desiredNodeState, desiredNodeState.Status.SyncStatus, desiredNodeState.Status.LastSyncError) if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): failed to load sriov result file from host") - return err + reqLogger.Error(err, "failed to update nodeState new host status") + return ctrl.Result{}, err } } - if sriovResult.LastSyncError != "" || sriovResult.SyncStatus == consts.SyncStatusFailed { - log.Log.Info("nodeStateSyncHandler(): sync failed systemd service error", "last-sync-error", sriovResult.LastSyncError) - // add the error but don't requeue - dn.refreshCh <- Message{ - syncStatus: consts.SyncStatusFailed, - lastSyncError: sriovResult.LastSyncError, - } - <-dn.syncCh - return nil - } + return ctrl.Result{RequeueAfter: consts.DaemonRequeueTime}, nil } + } - skipReconciliation, err = dn.shouldSkipReconciliation(dn.desiredNodeState) + // set sync state to inProgress, but we don't clear the failed status + err = dn.updateSyncState(ctx, desiredNodeState, consts.SyncStatusInProgress, desiredNodeState.Status.LastSyncError) + if err != nil { + reqLogger.Error(err, "failed to update sync status to inProgress") + return ctrl.Result{}, err + } + + reqReboot, reqDrain, err := dn.checkOnNodeStateChange(desiredNodeState) + if err != nil { + return ctrl.Result{}, err + } + + if vars.UsingSystemdMode { + // When running using systemd check if the applied configuration is the latest one + // or there is a new config we need to apply + // When using systemd configuration we write the file + systemdConfModified, err := dn.writeSystemdConfigFile(desiredNodeState) if err != nil { - return err + reqLogger.Error(err, "failed to write systemd config file") + return ctrl.Result{}, err } + reqDrain = reqDrain || systemdConfModified || !sriovResultExists + // require reboot if drain needed for systemd mode + reqReboot = reqReboot || reqDrain } - // we are done with the configuration just return here - if dn.currentNodeState.GetGeneration() == dn.desiredNodeState.GetGeneration() && - dn.desiredNodeState.Status.SyncStatus == consts.SyncStatusSucceeded && skipReconciliation { - log.Log.Info("Current state and desire state are equal together with sync status succeeded nothing to do") - return nil - } + reqLogger.V(0).Info("aggregated daemon node state requirement", + "drain-required", reqDrain, "reboot-required", reqReboot, "disable-drain", vars.DisableDrain) - dn.refreshCh <- Message{ - syncStatus: consts.SyncStatusInProgress, - lastSyncError: "", + // handle drain only if the plugins request drain, or we are already in a draining request state + if reqDrain || + !utils.ObjectHasAnnotation(desiredNodeState, consts.NodeStateDrainAnnotationCurrent, consts.DrainIdle) { + drainInProcess, err := dn.handleDrain(ctx, desiredNodeState, reqReboot) + if err != nil { + reqLogger.Error(err, "failed to handle drain") + return ctrl.Result{}, err + } + + // TODO: remove this after we stop using the node annotation + // drain is still in progress we will still requeue the request in case there is an un-expect state in the draining + // this will allow the daemon to try again. + if drainInProcess { + reqLogger.Info("node drain still in progress, requeue") + return ctrl.Result{RequeueAfter: consts.DaemonRequeueTime}, nil + } } - // wait for writer to refresh status then pull again the latest node state - <-dn.syncCh - // we need to load the latest status to our object - // if we don't do it we can have a race here where the user remove the virtual functions but the operator didn't - // trigger the refresh - updatedState, err := dn.sriovClient.SriovnetworkV1().SriovNetworkNodeStates(vars.Namespace).Get(context.Background(), vars.NodeName, metav1.GetOptions{}) - if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): Failed to fetch node state", "name", vars.NodeName) - return err + // if we finish the drain we should run apply here + if dn.isDrainCompleted(reqDrain, desiredNodeState) { + return dn.apply(ctx, desiredNodeState, reqReboot, sriovResult) } - dn.desiredNodeState.Status = updatedState.Status + return ctrl.Result{}, nil +} + +// checkOnNodeStateChange checks the state change required for the node based on the desired SriovNetworkNodeState. +// The function iterates over all loaded plugins and calls their OnNodeStateChange method with the desired state. +// It returns two boolean values indicating whether a reboot or drain operation is required. +func (dn *NodeReconciler) checkOnNodeStateChange(desiredNodeState *sriovnetworkv1.SriovNetworkNodeState) (bool, bool, error) { + funcLog := log.Log.WithName("checkOnNodeStateChange") reqReboot := false reqDrain := false // check if any of the plugins required to drain or reboot the node for k, p := range dn.loadedPlugins { - d, r := false, false - if dn.currentNodeState.GetName() == "" { - log.Log.V(0).Info("nodeStateSyncHandler(): calling OnNodeStateChange for a new node state") - } else { - log.Log.V(0).Info("nodeStateSyncHandler(): calling OnNodeStateChange for an updated node state") - } - d, r, err = p.OnNodeStateChange(dn.desiredNodeState) + d, r, err := p.OnNodeStateChange(desiredNodeState) if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): OnNodeStateChange plugin error", "plugin-name", k) - return err + funcLog.Error(err, "OnNodeStateChange plugin error", "plugin-name", k) + return false, false, err } - log.Log.V(0).Info("nodeStateSyncHandler(): OnNodeStateChange result", "plugin", k, "drain-required", d, "reboot-required", r) + funcLog.V(0).Info("OnNodeStateChange result", + "plugin", k, + "drain-required", d, + "reboot-required", r) reqDrain = reqDrain || d reqReboot = reqReboot || r } - // When running using systemd check if the applied configuration is the latest one - // or there is a new config we need to apply - // When using systemd configuration we write the file - if vars.UsingSystemdMode { - log.Log.V(0).Info("nodeStateSyncHandler(): writing systemd config file to host") - systemdConfModified, err := systemd.WriteConfFile(dn.desiredNodeState) - if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): failed to write configuration file for systemd mode") - return err - } - if systemdConfModified { - // remove existing result file to make sure that we will not use outdated result, e.g. in case if - // systemd service was not triggered for some reason - err = systemd.RemoveSriovResult() - if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): failed to remove result file for systemd mode") - return err - } - } - reqDrain = reqDrain || systemdConfModified - // require reboot if drain needed for systemd mode - reqReboot = reqReboot || systemdConfModified || reqDrain - log.Log.V(0).Info("nodeStateSyncHandler(): systemd mode WriteConfFile results", - "drain-required", reqDrain, "reboot-required", reqReboot, "disable-drain", dn.disableDrain) + return reqReboot, reqDrain, nil +} - err = systemd.WriteSriovSupportedNics() - if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): failed to write supported nic ids file for systemd mode") - return err - } +// checkSystemdStatus Checks the status of systemd services on the host node. +// return the sriovResult struct a boolean if the result file exist on the node +func (dn *NodeReconciler) checkSystemdStatus() (*hosttypes.SriovResult, bool, error) { + if !vars.UsingSystemdMode { + return nil, false, nil } - log.Log.V(0).Info("nodeStateSyncHandler(): aggregated daemon", - "drain-required", reqDrain, "reboot-required", reqReboot, "disable-drain", dn.disableDrain) - - // handle drain only if the plugin request drain, or we are already in a draining request state - if reqDrain || !utils.ObjectHasAnnotation(dn.desiredNodeState, - consts.NodeStateDrainAnnotationCurrent, - consts.DrainIdle) { - drainInProcess, err := dn.handleDrain(reqReboot) + funcLog := log.Log.WithName("checkSystemdStatus") + serviceEnabled, err := dn.HostHelpers.IsServiceEnabled(consts.SriovServicePath) + if err != nil { + funcLog.Error(err, "failed to check if sriov-config service exist on host") + return nil, false, err + } + postNetworkServiceEnabled, err := dn.HostHelpers.IsServiceEnabled(consts.SriovPostNetworkServicePath) + if err != nil { + funcLog.Error(err, "failed to check if sriov-config-post-network service exist on host") + return nil, false, err + } + + // if the service doesn't exist we should continue to let the k8s plugin to create the service files + // this is only for k8s base environments, for openshift the sriov-operator creates a machine config to will apply + // the system service and reboot the node the config-daemon doesn't need to do anything. + sriovResult := &hosttypes.SriovResult{SyncStatus: consts.SyncStatusFailed, + LastSyncError: fmt.Sprintf("some sriov systemd services are not available on node: "+ + "sriov-config available:%t, sriov-config-post-network available:%t", serviceEnabled, postNetworkServiceEnabled)} + exist := false + + // check if the service exist + if serviceEnabled && postNetworkServiceEnabled { + exist = true + sriovResult, err = dn.HostHelpers.ReadSriovResult() if err != nil { - log.Log.Error(err, "failed to handle drain") - return err - } - if drainInProcess { - return nil + funcLog.Error(err, "failed to load sriov result file from host") + return nil, false, err } } + return sriovResult, exist, nil +} +// apply applies the desired state of the node by: +// 1. Applying vendor plugins that have been loaded. +// 2. Depending on whether a reboot is required or if the configuration is being done via systemd, it applies the generic or virtual plugin(s). +// 3. Rebooting the node if necessary and sending an event. +// 4. Restarting the device plugin pod on the node. +// 5. Requesting annotation updates for draining the idle state of the node. +// 6. Synchronizing with the host network status and updating the sync status of the node in the nodeState object. +// 7. Updating the lastAppliedGeneration to the current generation. +func (dn *NodeReconciler) apply(ctx context.Context, desiredNodeState *sriovnetworkv1.SriovNetworkNodeState, reqReboot bool, sriovResult *hosttypes.SriovResult) (ctrl.Result, error) { + reqLogger := log.FromContext(ctx).WithName("Apply") // apply the vendor plugins after we are done with drain if needed for k, p := range dn.loadedPlugins { // Skip both the general and virtual plugin apply them last if k != GenericPluginName && k != VirtualPluginName { err := p.Apply() if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): plugin Apply failed", "plugin-name", k) - return err + reqLogger.Error(err, "plugin Apply failed", "plugin-name", k) + return ctrl.Result{}, err } } } @@ -497,10 +372,10 @@ func (dn *Daemon) nodeStateSyncHandler() error { selectedPlugin, ok := dn.loadedPlugins[GenericPluginName] if ok { // Apply generic plugin last - err = selectedPlugin.Apply() + err := selectedPlugin.Apply() if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): generic plugin fail to apply") - return err + reqLogger.Error(err, "generic plugin fail to apply") + return ctrl.Result{}, err } } @@ -508,187 +383,175 @@ func (dn *Daemon) nodeStateSyncHandler() error { selectedPlugin, ok = dn.loadedPlugins[VirtualPluginName] if ok { // Apply virtual plugin last - err = selectedPlugin.Apply() + err := selectedPlugin.Apply() if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): virtual plugin failed to apply") - return err + reqLogger.Error(err, "virtual plugin failed to apply") + return ctrl.Result{}, err } } } if reqReboot { - log.Log.Info("nodeStateSyncHandler(): reboot node") - dn.eventRecorder.SendEvent("RebootNode", "Reboot node has been initiated") - dn.rebootNode() - return nil + reqLogger.Info("reboot node") + dn.eventRecorder.SendEvent(ctx, "RebootNode", "Reboot node has been initiated") + return ctrl.Result{}, dn.rebootNode() } - // restart device plugin pod - log.Log.Info("nodeStateSyncHandler(): restart device plugin pod") - if err := dn.restartDevicePluginPod(); err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): fail to restart device plugin pod") - return err + if err := dn.restartDevicePluginPod(ctx); err != nil { + reqLogger.Error(err, "failed to restart device plugin on the node") + return ctrl.Result{}, err } - log.Log.Info("nodeStateSyncHandler(): apply 'Idle' annotation for node") - err = utils.AnnotateNode(context.Background(), vars.NodeName, consts.NodeDrainAnnotation, consts.DrainIdle, dn.client) + err := dn.annotate(ctx, desiredNodeState, consts.DrainIdle) if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): Failed to annotate node") - return err + reqLogger.Error(err, "failed to request annotation update to idle") + return ctrl.Result{}, err } - log.Log.Info("nodeStateSyncHandler(): apply 'Idle' annotation for nodeState") - if err := utils.AnnotateObject(context.Background(), dn.desiredNodeState, - consts.NodeStateDrainAnnotation, - consts.DrainIdle, dn.client); err != nil { - return err + reqLogger.Info("sync succeeded") + syncStatus := consts.SyncStatusSucceeded + lastSyncError := "" + if vars.UsingSystemdMode { + syncStatus = sriovResult.SyncStatus + lastSyncError = sriovResult.LastSyncError } - log.Log.Info("nodeStateSyncHandler(): sync succeeded") - dn.currentNodeState = dn.desiredNodeState.DeepCopy() - if vars.UsingSystemdMode { - dn.refreshCh <- Message{ - syncStatus: sriovResult.SyncStatus, - lastSyncError: sriovResult.LastSyncError, - } - } else { - dn.refreshCh <- Message{ - syncStatus: consts.SyncStatusSucceeded, - lastSyncError: "", - } + // Update the nodeState Status object with the existing network interfaces + err = dn.updateStatusFromHost(desiredNodeState) + if err != nil { + reqLogger.Error(err, "failed to get host network status") + return ctrl.Result{}, err } - // wait for writer to refresh the status - <-dn.syncCh - return nil + + err = dn.updateSyncState(ctx, desiredNodeState, syncStatus, lastSyncError) + if err != nil { + reqLogger.Error(err, "failed to update sync status") + return ctrl.Result{}, err + } + + // update the lastAppliedGeneration + dn.lastAppliedGeneration = desiredNodeState.Generation + return ctrl.Result{RequeueAfter: consts.DaemonRequeueTime}, nil } -func (dn *Daemon) shouldSkipReconciliation(latestState *sriovnetworkv1.SriovNetworkNodeState) (bool, error) { - log.Log.V(0).Info("shouldSkipReconciliation()") - var err error +// checkHostStateDrift returns true if the node state drifted from the nodeState policy +// Check if there is a change in the host network interfaces that require a reconfiguration by the daemon +func (dn *NodeReconciler) checkHostStateDrift(ctx context.Context, desiredNodeState *sriovnetworkv1.SriovNetworkNodeState) (bool, error) { + funcLog := log.Log.WithName("checkHostStateDrift()") // Skip when SriovNetworkNodeState object has just been created. - if latestState.GetGeneration() == 1 && len(latestState.Spec.Interfaces) == 0 { - err = dn.HostHelpers.ClearPCIAddressFolder() + if desiredNodeState.GetGeneration() == 1 && len(desiredNodeState.Spec.Interfaces) == 0 { + err := dn.HostHelpers.ClearPCIAddressFolder() if err != nil { - log.Log.Error(err, "failed to clear the PCI address configuration") + funcLog.Error(err, "failed to clear the PCI address configuration") return false, err } - log.Log.V(0).Info( - "shouldSkipReconciliation(): interface policy spec not yet set by controller for sriovNetworkNodeState", - "name", latestState.Name) - if latestState.Status.SyncStatus != consts.SyncStatusSucceeded { - dn.refreshCh <- Message{ - syncStatus: consts.SyncStatusSucceeded, - lastSyncError: "", - } - // wait for writer to refresh status - <-dn.syncCh + funcLog.V(0).Info("interface policy spec not yet set by controller for sriovNetworkNodeState", + "name", desiredNodeState.Name) + if desiredNodeState.Status.SyncStatus != consts.SyncStatusSucceeded || + desiredNodeState.Status.LastSyncError != "" { + err = dn.updateSyncState(ctx, desiredNodeState, consts.SyncStatusSucceeded, "") } - return true, nil + return false, err } // Verify changes in the status of the SriovNetworkNodeState CR. - if dn.currentNodeState.GetGeneration() == latestState.GetGeneration() { - log.Log.V(0).Info("shouldSkipReconciliation() verifying status change") - for _, p := range dn.loadedPlugins { - // Verify changes in the status of the SriovNetworkNodeState CR. - log.Log.V(0).Info("shouldSkipReconciliation(): verifying status change for plugin", "pluginName", p.Name()) - changed, err := p.CheckStatusChanges(latestState) - if err != nil { - return false, err - } - if changed { - log.Log.V(0).Info("shouldSkipReconciliation(): plugin require change", "pluginName", p.Name()) - return false, nil - } + log.Log.V(0).Info("verifying interfaces status change") + for _, p := range dn.loadedPlugins { + // Verify changes in the status of the SriovNetworkNodeState CR. + log.Log.V(2).Info("verifying status change for plugin", "pluginName", p.Name()) + changed, err := p.CheckStatusChanges(desiredNodeState) + if err != nil { + return false, err + } + if changed { + log.Log.V(0).Info("plugin require change", "pluginName", p.Name()) + return true, nil } + } - log.Log.V(0).Info("shouldSkipReconciliation(): Interface not changed") - if latestState.Status.LastSyncError != "" || - latestState.Status.SyncStatus != consts.SyncStatusSucceeded { - dn.refreshCh <- Message{ - syncStatus: consts.SyncStatusSucceeded, - lastSyncError: "", - } - // wait for writer to refresh the status - <-dn.syncCh + log.Log.V(0).Info("Interfaces not changed") + return false, nil +} + +// writeSystemdConfigFile Writes the systemd configuration file for the node +// and handles any necessary actions such as removing an existing result file and writing supported NIC IDs. +// +// The function first attempts to write the systemd configuration file based on the desired node state. +// If successful, it checks if the configuration file was modified. If so, it removes the existing result file (if present) to ensure that outdated results are not used. +// After writing the configuration file and potentially removing the old one, it writes a file containing supported NIC IDs. +func (dn *NodeReconciler) writeSystemdConfigFile(desiredNodeState *sriovnetworkv1.SriovNetworkNodeState) (bool, error) { + funcLog := log.Log.WithName("writeSystemdConfigFile()") + funcLog.V(0).Info("writing systemd config file to host") + systemdConfModified, err := dn.HostHelpers.WriteConfFile(desiredNodeState) + if err != nil { + funcLog.Error(err, "failed to write configuration file for systemd mode") + return false, err + } + if systemdConfModified { + // remove existing result file to make sure that we will not use outdated result, e.g. in case if + // systemd service was not triggered for some reason + err = dn.HostHelpers.RemoveSriovResult() + if err != nil { + funcLog.Error(err, "failed to remove result file for systemd mode") + return false, err } + } - return true, nil + err = dn.HostHelpers.WriteSriovSupportedNics() + if err != nil { + funcLog.Error(err, "failed to write supported nic ids file for systemd mode") + return false, err } - return false, nil + funcLog.V(0).Info("systemd mode WriteConfFile results", + "drain-required", systemdConfModified, "reboot-required", systemdConfModified) + return systemdConfModified, nil } // handleDrain: adds the right annotation to the node and nodeState object // returns true if we need to finish the reconcile loop and wait for a new object -func (dn *Daemon) handleDrain(reqReboot bool) (bool, error) { +func (dn *NodeReconciler) handleDrain(ctx context.Context, desiredNodeState *sriovnetworkv1.SriovNetworkNodeState, reqReboot bool) (bool, error) { + funcLog := log.Log.WithName("handleDrain") // done with the drain we can continue with the configuration - if utils.ObjectHasAnnotation(dn.desiredNodeState, consts.NodeStateDrainAnnotationCurrent, consts.DrainComplete) { - log.Log.Info("handleDrain(): the node complete the draining") + if utils.ObjectHasAnnotation(desiredNodeState, consts.NodeStateDrainAnnotationCurrent, consts.DrainComplete) { + funcLog.Info("the node complete the draining") return false, nil } // the operator is still draining the node so we reconcile - if utils.ObjectHasAnnotation(dn.desiredNodeState, consts.NodeStateDrainAnnotationCurrent, consts.Draining) { - log.Log.Info("handleDrain(): the node is still draining") + if utils.ObjectHasAnnotation(desiredNodeState, consts.NodeStateDrainAnnotationCurrent, consts.Draining) { + funcLog.Info("the node is still draining") return true, nil } // drain is disabled we continue with the configuration - if dn.disableDrain { - log.Log.Info("handleDrain(): drain is disabled in sriovOperatorConfig") + if vars.DisableDrain { + funcLog.Info("drain is disabled in sriovOperatorConfig") return false, nil } + // annotate both node and node state with drain or reboot + annotation := consts.DrainRequired if reqReboot { - log.Log.Info("handleDrain(): apply 'Reboot_Required' annotation for node") - err := utils.AnnotateNode(context.Background(), vars.NodeName, consts.NodeDrainAnnotation, consts.RebootRequired, dn.client) - if err != nil { - log.Log.Error(err, "applyDrainRequired(): Failed to annotate node") - return false, err - } - - log.Log.Info("handleDrain(): apply 'Reboot_Required' annotation for nodeState") - if err := utils.AnnotateObject(context.Background(), dn.desiredNodeState, - consts.NodeStateDrainAnnotation, - consts.RebootRequired, dn.client); err != nil { - return false, err - } - - // the node was annotated we need to wait for the operator to finish the drain - return true, nil - } - log.Log.Info("handleDrain(): apply 'Drain_Required' annotation for node") - err := utils.AnnotateNode(context.Background(), vars.NodeName, consts.NodeDrainAnnotation, consts.DrainRequired, dn.client) - if err != nil { - log.Log.Error(err, "handleDrain(): Failed to annotate node") - return false, err - } - - log.Log.Info("handleDrain(): apply 'Drain_Required' annotation for nodeState") - if err := utils.AnnotateObject(context.Background(), dn.desiredNodeState, - consts.NodeStateDrainAnnotation, - consts.DrainRequired, dn.client); err != nil { - return false, err + annotation = consts.RebootRequired } - - // the node was annotated we need to wait for the operator to finish the drain - return true, nil + return true, dn.annotate(ctx, desiredNodeState, annotation) } -func (dn *Daemon) restartDevicePluginPod() error { - dn.mu.Lock() - defer dn.mu.Unlock() +// restartDevicePluginPod restarts the device plugin pod on the specified node. +// +// The function checks if the pod exists, deletes it if found, and waits for it to be deleted successfully. +func (dn *NodeReconciler) restartDevicePluginPod(ctx context.Context) error { log.Log.V(2).Info("restartDevicePluginPod(): try to restart device plugin pod") - - var podToDelete string - pods, err := dn.kubeClient.CoreV1().Pods(vars.Namespace).List(context.Background(), metav1.ListOptions{ - LabelSelector: "app=sriov-device-plugin", - FieldSelector: "spec.nodeName=" + vars.NodeName, - ResourceVersion: "0", - }) + pods := &corev1.PodList{} + err := dn.client.List(ctx, pods, &client.ListOptions{ + Namespace: vars.Namespace, Raw: &metav1.ListOptions{ + LabelSelector: "app=sriov-device-plugin", + FieldSelector: "spec.nodeName=" + vars.NodeName, + ResourceVersion: "0", + }}) if err != nil { if errors.IsNotFound(err) { log.Log.Info("restartDevicePluginPod(): device plugin pod exited") @@ -702,45 +565,50 @@ func (dn *Daemon) restartDevicePluginPod() error { log.Log.Info("restartDevicePluginPod(): device plugin pod exited") return nil } - podToDelete = pods.Items[0].Name - - log.Log.V(2).Info("restartDevicePluginPod(): Found device plugin pod, deleting it", "pod-name", podToDelete) - err = dn.kubeClient.CoreV1().Pods(vars.Namespace).Delete(context.Background(), podToDelete, metav1.DeleteOptions{}) - if errors.IsNotFound(err) { - log.Log.Info("restartDevicePluginPod(): pod to delete not found") - return nil - } - if err != nil { - log.Log.Error(err, "restartDevicePluginPod(): Failed to delete device plugin pod, retrying") - return err - } - if err := wait.PollImmediateUntil(3*time.Second, func() (bool, error) { - _, err := dn.kubeClient.CoreV1().Pods(vars.Namespace).Get(context.Background(), podToDelete, metav1.GetOptions{}) + for _, pod := range pods.Items { + log.Log.V(2).Info("restartDevicePluginPod(): Found device plugin pod, deleting it", "pod-name", pod.Name) + err = dn.client.Delete(ctx, &pod) if errors.IsNotFound(err) { - log.Log.Info("restartDevicePluginPod(): device plugin pod exited") - return true, nil + log.Log.Info("restartDevicePluginPod(): pod to delete not found") + continue } - if err != nil { - log.Log.Error(err, "restartDevicePluginPod(): Failed to check for device plugin exit, retrying") - } else { - log.Log.Info("restartDevicePluginPod(): waiting for device plugin pod to exit", "pod-name", podToDelete) + log.Log.Error(err, "restartDevicePluginPod(): Failed to delete device plugin pod, retrying") + return err + } + + tmpPod := &corev1.Pod{} + if err := wait.PollUntilContextCancel(ctx, 3*time.Second, true, func(ctx context.Context) (bool, error) { + err := dn.client.Get(ctx, client.ObjectKeyFromObject(&pod), tmpPod) + if errors.IsNotFound(err) { + log.Log.Info("restartDevicePluginPod(): device plugin pod exited") + return true, nil + } + + if err != nil { + log.Log.Error(err, "restartDevicePluginPod(): Failed to check for device plugin exit, retrying") + } else { + log.Log.Info("restartDevicePluginPod(): waiting for device plugin pod to exit", "pod-name", pod.Name) + } + return false, nil + }); err != nil { + log.Log.Error(err, "restartDevicePluginPod(): failed to wait for checking pod deletion") + return err } - return false, nil - }, dn.stopCh); err != nil { - log.Log.Error(err, "restartDevicePluginPod(): failed to wait for checking pod deletion") - return err } return nil } -func (dn *Daemon) rebootNode() { - log.Log.Info("rebootNode(): trigger node reboot") +// rebootNode Reboots the node by executing a systemd-run command +func (dn *NodeReconciler) rebootNode() error { + funcLog := log.Log.WithName("rebootNode") + funcLog.Info("trigger node reboot") exit, err := dn.HostHelpers.Chroot(consts.Host) if err != nil { - log.Log.Error(err, "rebootNode(): chroot command failed") + funcLog.Error(err, "chroot command failed") + return err } defer exit() // creates a new transient systemd unit to reboot the system. @@ -750,15 +618,19 @@ func (dn *Daemon) rebootNode() { // However note we use `;` instead of `&&` so we keep rebooting even // if kubelet failed to shutdown - that way the machine will still eventually reboot // as systemd will time out the stop invocation. - cmd := exec.Command("systemd-run", "--unit", "sriov-network-config-daemon-reboot", + stdOut, StdErr, err := dn.HostHelpers.RunCommand("systemd-run", "--unit", "sriov-network-config-daemon-reboot", "--description", "sriov-network-config-daemon reboot node", "/bin/sh", "-c", "systemctl stop kubelet.service; reboot") - if err := cmd.Run(); err != nil { - log.Log.Error(err, "failed to reboot node") + if err != nil { + funcLog.Error(err, "failed to reboot node", "stdOut", stdOut, "StdErr", StdErr) + return err } + return nil } -func (dn *Daemon) prepareNMUdevRule() error { +// prepareNMUdevRule prepares/validate the status of the config-daemon custom udev rules needed to control +// the virtual functions by the operator only. +func (dn *NodeReconciler) prepareNMUdevRule() error { // we need to remove the Red Hat Virtio network device from the udev rule configuration // if we don't remove it when running the config-daemon on a virtual node it will disconnect the node after a reboot // even that the operator should not be installed on virtual environments that are not openstack @@ -775,6 +647,66 @@ func (dn *Daemon) prepareNMUdevRule() error { } // isDrainCompleted returns true if the current-state annotation is drain completed -func (dn *Daemon) isDrainCompleted() bool { - return utils.ObjectHasAnnotation(dn.desiredNodeState, consts.NodeStateDrainAnnotationCurrent, consts.DrainComplete) +func (dn *NodeReconciler) isDrainCompleted(reqDrain bool, desiredNodeState *sriovnetworkv1.SriovNetworkNodeState) bool { + if vars.DisableDrain { + return true + } + + // if we need to drain check the drain status + if reqDrain { + return utils.ObjectHasAnnotation(desiredNodeState, consts.NodeStateDrainAnnotationCurrent, consts.DrainComplete) + } + + // check in case a reboot was requested and the second run doesn't require a drain + if !utils.ObjectHasAnnotation(desiredNodeState, consts.NodeStateDrainAnnotation, consts.DrainIdle) { + return utils.ObjectHasAnnotation(desiredNodeState, consts.NodeStateDrainAnnotationCurrent, consts.DrainComplete) + } + + // if we don't need to drain at all just return true so we can apply the configuration + return true +} + +// annotate annotates the nodeState object with specified annotation. +func (dn *NodeReconciler) annotate( + ctx context.Context, + desiredNodeState *sriovnetworkv1.SriovNetworkNodeState, + annotationState string) error { + funcLog := log.Log.WithName("annotate") + + funcLog.Info(fmt.Sprintf("apply '%s' annotation for node", annotationState)) + if err := utils.AnnotateNode(ctx, + desiredNodeState.Name, + consts.NodeDrainAnnotation, + annotationState, dn.client); err != nil { + funcLog.Error(err, "Failed to annotate node") + return err + } + + funcLog.Info(fmt.Sprintf("apply '%s' annotation for nodeState", annotationState)) + if err := utils.AnnotateObject(ctx, desiredNodeState, + consts.NodeStateDrainAnnotation, + annotationState, dn.client); err != nil { + funcLog.Error(err, "Failed to annotate nodeState") + return err + } + + // the node was annotated we need to wait for the operator to finish the drain + return nil +} + +// SetupWithManager sets up the controller with the Manager. +func (dn *NodeReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&sriovnetworkv1.SriovNetworkNodeState{}). + WithEventFilter(predicate.Or(predicate.AnnotationChangedPredicate{}, predicate.GenerationChangedPredicate{})). + WithOptions(controller.Options{MaxConcurrentReconciles: 1}). + Complete(dn) +} + +// ------------------------------------- +// ---- unit tests helper function ----- +// ------------------------------------- + +func (dn *NodeReconciler) GetLastAppliedGeneration() int64 { + return dn.lastAppliedGeneration } diff --git a/pkg/daemon/daemon_suite_test.go b/pkg/daemon/daemon_suite_test.go new file mode 100644 index 000000000..f8d3103ce --- /dev/null +++ b/pkg/daemon/daemon_suite_test.go @@ -0,0 +1,135 @@ +package daemon_test + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + openshiftconfigv1 "github.com/openshift/api/config/v1" + mcfgv1 "github.com/openshift/api/machineconfiguration/v1" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "go.uber.org/zap/zapcore" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + k8sClient client.Client + testEnv *envtest.Environment + cfg *rest.Config +) + +// Define utility constants for object names and testing timeouts/durations and intervals. +const testNamespace = "openshift-sriov-network-operator" + +var _ = BeforeSuite(func() { + var err error + + logf.SetLogger(zap.New( + zap.WriteTo(GinkgoWriter), + zap.UseDevMode(true), + func(o *zap.Options) { + o.TimeEncoder = zapcore.RFC3339NanoTimeEncoder + })) + + // Go to project root directory + err = os.Chdir("../..") + Expect(err).NotTo(HaveOccurred()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("config", "crd", "bases"), filepath.Join("test", "util", "crds")}, + ErrorIfCRDPathMissing: true, + } + + testEnv.ControlPlane.GetAPIServer().Configure().Set("disable-admission-plugins", "MutatingAdmissionWebhook", "ValidatingAdmissionWebhook") + + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + By("registering schemes") + err = sriovnetworkv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = netattdefv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = mcfgv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = openshiftconfigv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = monitoringv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + vars.Config = cfg + vars.Scheme = scheme.Scheme + vars.Namespace = testNamespace + + By("creating K8s client") + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + By("creating default/common k8s objects for tests") + // Create test namespace + ns := &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: testNamespace, + }, + Spec: corev1.NamespaceSpec{}, + Status: corev1.NamespaceStatus{}, + } + Expect(k8sClient.Create(context.Background(), ns)).Should(Succeed()) + + sa := &corev1.ServiceAccount{TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: testNamespace, + }} + Expect(k8sClient.Create(context.Background(), sa)).Should(Succeed()) + + // Create openshift Infrastructure + infra := &openshiftconfigv1.Infrastructure{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + Spec: openshiftconfigv1.InfrastructureSpec{}, + Status: openshiftconfigv1.InfrastructureStatus{ + ControlPlaneTopology: openshiftconfigv1.HighlyAvailableTopologyMode, + }, + } + Expect(k8sClient.Create(context.Background(), infra)).Should(Succeed()) +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + if testEnv != nil { + Eventually(func() error { + return testEnv.Stop() + }, util.APITimeout, time.Second).ShouldNot(HaveOccurred()) + } +}) + +func TestDaemon(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Daemon Suite") +} diff --git a/pkg/daemon/daemon_test.go b/pkg/daemon/daemon_test.go index f1111810a..19a69a425 100644 --- a/pkg/daemon/daemon_test.go +++ b/pkg/daemon/daemon_test.go @@ -1,304 +1,430 @@ -package daemon +package daemon_test import ( "context" - "flag" - "testing" + "os" + "sync" + "sync/atomic" + "time" - "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "go.uber.org/zap/zapcore" + "go.uber.org/mock/gomock" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - fakek8s "k8s.io/client-go/kubernetes/fake" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" - kclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - snclient "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" - snclientset "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned/fake" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/daemon" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/featuregate" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper" mock_helper "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper/mock" + hostTypes "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" + snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms" mock_platforms "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/mock" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openshift" - plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins/fake" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins/generic" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" - "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/fakefilesystem" ) -func TestConfigDaemon(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Config Daemon Suite") -} - -var _ = BeforeSuite(func() { - // Increase verbosity to help debugging failures - flag.Set("logtostderr", "true") - flag.Set("stderrthreshold", "WARNING") - flag.Set("v", "2") - - logf.SetLogger(zap.New( - zap.WriteTo(GinkgoWriter), - zap.UseDevMode(true), - func(o *zap.Options) { - o.TimeEncoder = zapcore.RFC3339NanoTimeEncoder - })) -}) +var ( + k8sManager manager.Manager + kubeclient *kubernetes.Clientset + eventRecorder *daemon.EventRecorder + wg sync.WaitGroup + startDaemon func(dc *daemon.NodeReconciler) -var _ = Describe("Config Daemon", func() { - var stopCh chan struct{} - var syncCh chan struct{} - var exitCh chan error - var refreshCh chan Message + t FullGinkgoTInterface + mockCtrl *gomock.Controller + hostHelper *mock_helper.MockHostHelpersInterface + platformHelper *mock_platforms.MockInterface - var cleanFakeFs func() + discoverSriovReturn atomic.Pointer[[]sriovnetworkv1.InterfaceExt] + nodeState *sriovnetworkv1.SriovNetworkNodeState - var sut *Daemon + daemonReconciler *daemon.NodeReconciler +) - BeforeEach(func() { - stopCh = make(chan struct{}) - refreshCh = make(chan Message) - exitCh = make(chan error) - syncCh = make(chan struct{}, 64) +const ( + waitTime = 30 * time.Minute + retryTime = 5 * time.Second +) - // Fill syncCh with values so daemon doesn't wait for a writer - for i := 0; i < 64; i++ { - syncCh <- struct{}{} +var _ = Describe("Daemon Controller", Ordered, func() { + BeforeAll(func() { + wg = sync.WaitGroup{} + DeferCleanup(wg.Wait) + + ctx, cancel := context.WithCancel(context.Background()) + DeferCleanup(cancel) + + startDaemon = func(dc *daemon.NodeReconciler) { + By("start controller manager") + wg.Add(1) + go func() { + defer wg.Done() + defer GinkgoRecover() + By("Start controller manager") + err := k8sManager.Start(ctx) + Expect(err).ToNot(HaveOccurred()) + }() } - // Create virtual filesystem for Daemon - fakeFs := &fakefilesystem.FS{ - Dirs: []string{ - "bindata/scripts", - "host/etc/sriov-operator", - "host/etc/sriov-operator/pci", - "host/etc/udev/rules.d", - }, - Symlinks: map[string]string{}, - Files: map[string][]byte{ - "/bindata/scripts/enable-rdma.sh": []byte(""), - "/bindata/scripts/load-kmod.sh": []byte(""), + Expect(k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovOperatorConfig{}, client.InNamespace(testNamespace))).ToNot(HaveOccurred()) + soc := &sriovnetworkv1.SriovOperatorConfig{ObjectMeta: metav1.ObjectMeta{ + Name: constants.DefaultConfigName, + Namespace: testNamespace, + }, + Spec: sriovnetworkv1.SriovOperatorConfigSpec{ + LogLevel: 2, }, } - - var err error - vars.FilesystemRoot, cleanFakeFs, err = fakeFs.Use() + err := k8sClient.Create(ctx, soc) Expect(err).ToNot(HaveOccurred()) - vars.UsingSystemdMode = false - vars.NodeName = "test-node" - vars.Namespace = "sriov-network-operator" - vars.PlatformType = consts.Baremetal + kubeclient = kubernetes.NewForConfigOrDie(cfg) + eventRecorder = daemon.NewEventRecorder(k8sClient, kubeclient, scheme.Scheme) + DeferCleanup(func() { + eventRecorder.Shutdown() + }) - FakeSupportedNicIDs := corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: sriovnetworkv1.SupportedNicIDConfigmap, - Namespace: vars.Namespace, - }, - Data: map[string]string{ - "Intel_i40e_XXV710": "8086 158a 154c", - "Nvidia_mlx5_ConnectX-4": "15b3 1013 1014", - }, + snolog.SetLogLevel(2) + // Check if the environment variable CLUSTER_TYPE is set + if clusterType, ok := os.LookupEnv("CLUSTER_TYPE"); ok && clusterType == constants.ClusterTypeOpenshift { + vars.ClusterType = constants.ClusterTypeOpenshift + } else { + vars.ClusterType = constants.ClusterTypeKubernetes } - SriovDevicePluginPod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "sriov-device-plugin-xxxx", - Namespace: vars.Namespace, - Labels: map[string]string{ - "app": "sriov-device-plugin", - }, - }, - Spec: corev1.PodSpec{ - NodeName: "test-node", - }, + By("Init mock functions") + t = GinkgoT() + mockCtrl = gomock.NewController(t) + hostHelper = mock_helper.NewMockHostHelpersInterface(mockCtrl) + platformHelper = mock_platforms.NewMockInterface(mockCtrl) + + // daemon initialization default mocks + hostHelper.EXPECT().CheckRDMAEnabled().Return(true, nil) + hostHelper.EXPECT().CleanSriovFilesFromHost(vars.ClusterType == constants.ClusterTypeOpenshift).Return(nil) + hostHelper.EXPECT().TryEnableTun() + hostHelper.EXPECT().TryEnableVhostNet() + hostHelper.EXPECT().PrepareNMUdevRule([]string{}).Return(nil) + hostHelper.EXPECT().PrepareVFRepUdevRule().Return(nil) + hostHelper.EXPECT().WriteCheckpointFile(gomock.Any()).Return(nil) + + // general + hostHelper.EXPECT().Chroot(gomock.Any()).Return(func() error { return nil }, nil).AnyTimes() + hostHelper.EXPECT().RunCommand("/bin/sh", gomock.Any(), gomock.Any(), gomock.Any()).Return("", "", nil).AnyTimes() + + discoverSriovReturn.Store(&[]sriovnetworkv1.InterfaceExt{}) + + hostHelper.EXPECT().DiscoverSriovDevices(hostHelper).DoAndReturn(func(helpersInterface helper.HostHelpersInterface) ([]sriovnetworkv1.InterfaceExt, error) { + return *discoverSriovReturn.Load(), nil + }).AnyTimes() + + hostHelper.EXPECT().LoadPfsStatus("0000:16:00.0").Return(&sriovnetworkv1.Interface{ExternallyManaged: false}, true, nil).AnyTimes() + + hostHelper.EXPECT().ClearPCIAddressFolder().Return(nil).AnyTimes() + hostHelper.EXPECT().DiscoverRDMASubsystem().Return("shared", nil).AnyTimes() + hostHelper.EXPECT().GetCurrentKernelArgs().Return("", nil).AnyTimes() + hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgPciRealloc).Return(true).AnyTimes() + hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgIntelIommu).Return(true).AnyTimes() + hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgIommuPt).Return(true).AnyTimes() + hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgRdmaExclusive).Return(false).AnyTimes() + hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgRdmaShared).Return(false).AnyTimes() + hostHelper.EXPECT().SetRDMASubsystem("").Return(nil).AnyTimes() + + hostHelper.EXPECT().ConfigSriovInterfaces(gomock.Any(), gomock.Any(), gomock.Any(), false).Return(nil).AnyTimes() + + // k8s plugin for k8s cluster type + if vars.ClusterType == constants.ClusterTypeKubernetes { + hostHelper.EXPECT().ReadServiceManifestFile(gomock.Any()).Return(&hostTypes.Service{Name: "test"}, nil).AnyTimes() + hostHelper.EXPECT().ReadServiceInjectionManifestFile(gomock.Any()).Return(&hostTypes.Service{Name: "test"}, nil).AnyTimes() } - err = sriovnetworkv1.AddToScheme(scheme.Scheme) - Expect(err).ToNot(HaveOccurred()) - kClient := kclient.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(&corev1.Node{ - ObjectMeta: metav1.ObjectMeta{Name: "test-node"}}, - &sriovnetworkv1.SriovNetworkNodeState{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node", - Namespace: vars.Namespace, - }}).Build() - - kubeClient := fakek8s.NewSimpleClientset(&FakeSupportedNicIDs, &SriovDevicePluginPod) - snclient := snclientset.NewSimpleClientset() - err = sriovnetworkv1.InitNicIDMapFromConfigMap(kubeClient, vars.Namespace) - Expect(err).ToNot(HaveOccurred()) - - er := NewEventRecorder(snclient, kubeClient) - - t := GinkgoT() - mockCtrl := gomock.NewController(t) - platformHelper := mock_platforms.NewMockInterface(mockCtrl) - platformHelper.EXPECT().GetFlavor().Return(openshift.OpenshiftFlavorDefault).AnyTimes() - platformHelper.EXPECT().IsOpenshiftCluster().Return(false).AnyTimes() - platformHelper.EXPECT().IsHypershift().Return(false).AnyTimes() - - vendorHelper := mock_helper.NewMockHostHelpersInterface(mockCtrl) - vendorHelper.EXPECT().CheckRDMAEnabled().Return(true, nil).AnyTimes() - vendorHelper.EXPECT().TryEnableVhostNet().AnyTimes() - vendorHelper.EXPECT().TryEnableTun().AnyTimes() - vendorHelper.EXPECT().PrepareNMUdevRule([]string{"0x1014", "0x154c"}).Return(nil).AnyTimes() - vendorHelper.EXPECT().PrepareVFRepUdevRule().Return(nil).AnyTimes() - featureGates := featuregate.New() + featureGates.Init(map[string]bool{}) + daemonReconciler = createDaemon(hostHelper, platformHelper, featureGates, []string{}) + startDaemon(daemonReconciler) - sut = New( - kClient, - snclient, - kubeClient, - vendorHelper, - platformHelper, - exitCh, - stopCh, - syncCh, - refreshCh, - er, - featureGates, - nil, - ) - - sut.loadedPlugins = map[string]plugin.VendorPlugin{generic.PluginName: &fake.FakePlugin{PluginName: "fake"}} - - go func() { - defer GinkgoRecover() - err := sut.Run(stopCh, exitCh) - Expect(err).ToNot(HaveOccurred()) - }() + _, nodeState = createNode("node1") }) - AfterEach(func() { - close(stopCh) - close(syncCh) - close(exitCh) - close(refreshCh) + AfterAll(func() { + Expect(k8sClient.DeleteAllOf(context.Background(), &corev1.Node{})).ToNot(HaveOccurred()) - cleanFakeFs() + Expect(k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovNetworkNodeState{}, client.InNamespace(testNamespace))).ToNot(HaveOccurred()) + Expect(k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovOperatorConfig{}, client.InNamespace(testNamespace))).ToNot(HaveOccurred()) }) - Context("Should", func() { - It("restart sriov-device-plugin pod", func() { + Context("Config Daemon generic flow", func() { + + It("Should expose nodeState Status section", func(ctx context.Context) { + + discoverSriovReturn.Store(&[]sriovnetworkv1.InterfaceExt{ + { + Name: "eno1", + Driver: "ice", + PciAddress: "0000:16:00.0", + DeviceID: "1593", + Vendor: "8086", + EswitchMode: "legacy", + LinkAdminState: "up", + LinkSpeed: "10000 Mb/s", + LinkType: "ETH", + Mac: "aa:bb:cc:dd:ee:ff", + Mtu: 1500, + TotalVfs: 2, + NumVfs: 0, + }, + }) - _, err := sut.kubeClient.CoreV1().Nodes(). - Create(context.Background(), &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{Name: "test-node"}, - }, metav1.CreateOptions{}) - Expect(err).To(BeNil()) + By("waiting for state to be succeeded") + eventuallySyncStatusEqual(nodeState, constants.SyncStatusSucceeded) - nodeState := &sriovnetworkv1.SriovNetworkNodeState{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node", - Generation: 123, - Annotations: map[string]string{consts.NodeStateDrainAnnotationCurrent: consts.DrainIdle}, - }, - Spec: sriovnetworkv1.SriovNetworkNodeStateSpec{}, - Status: sriovnetworkv1.SriovNetworkNodeStateStatus{ - Interfaces: []sriovnetworkv1.InterfaceExt{ + By("add spec to node state") + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState) + Expect(err).ToNot(HaveOccurred()) + + nodeState.Spec.Interfaces = []sriovnetworkv1.Interface{ + {Name: "eno1", + PciAddress: "0000:16:00.0", + LinkType: "eth", + NumVfs: 2, + VfGroups: []sriovnetworkv1.VfGroup{ + {ResourceName: "test", + DeviceType: "netdevice", + PolicyName: "test-policy", + VfRange: "eno1#0-1"}, + }}, + } + + discoverSriovReturn.Store(&[]sriovnetworkv1.InterfaceExt{ + { + Name: "eno1", + Driver: "ice", + PciAddress: "0000:16:00.0", + DeviceID: "1593", + Vendor: "8086", + EswitchMode: "legacy", + LinkAdminState: "up", + LinkSpeed: "10000 Mb/s", + LinkType: "ETH", + Mac: "aa:bb:cc:dd:ee:ff", + Mtu: 1500, + TotalVfs: 2, + NumVfs: 2, + VFs: []sriovnetworkv1.VirtualFunction{ { - VFs: []sriovnetworkv1.VirtualFunction{ - {}, - }, - DeviceID: "158b", - Driver: "i40e", - Mtu: 1500, - Name: "ens803f0", - PciAddress: "0000:86:00.0", - Vendor: "8086", - NumVfs: 4, - TotalVfs: 64, + Name: "eno1f0", + PciAddress: "0000:16:00.1", + VfID: 0, }, - }, + { + Name: "eno1f1", + PciAddress: "0000:16:00.2", + VfID: 1, + }}, }, - } - Expect( - createSriovNetworkNodeState(sut.sriovClient, nodeState)). - To(BeNil()) + }) + + err = k8sClient.Update(ctx, nodeState) + Expect(err).ToNot(HaveOccurred()) - var msg Message - Eventually(refreshCh, "30s").Should(Receive(&msg)) - Expect(msg.syncStatus).To(Equal("InProgress")) + By("waiting to require drain") + EventuallyWithOffset(1, func(g Gomega) { + g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)). + ToNot(HaveOccurred()) + g.Expect(daemonReconciler.GetLastAppliedGeneration()).To(Equal(int64(2))) + }, waitTime, retryTime).Should(Succeed()) - Eventually(refreshCh, "30s").Should(Receive(&msg)) - Expect(msg.syncStatus).To(Equal("Succeeded")) + err = k8sClient.Get(ctx, types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState) + Expect(err).ToNot(HaveOccurred()) + nodeState.Spec.Interfaces = []sriovnetworkv1.Interface{} + err = k8sClient.Update(ctx, nodeState) + Expect(err).ToNot(HaveOccurred()) - Eventually(func() (int, error) { - podList, err := sut.kubeClient.CoreV1().Pods(vars.Namespace).List(context.Background(), metav1.ListOptions{ - LabelSelector: "app=sriov-device-plugin", - FieldSelector: "spec.nodeName=test-node", - }) + EventuallyWithOffset(1, func(g Gomega) { + g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)). + ToNot(HaveOccurred()) - if err != nil { - return 0, err - } + g.Expect(nodeState.Annotations[constants.NodeStateDrainAnnotation]).To(Equal(constants.DrainRequired)) + }, waitTime, retryTime).Should(Succeed()) - return len(podList.Items), nil - }, "10s").Should(BeZero()) + patchAnnotation(nodeState, constants.NodeStateDrainAnnotationCurrent, constants.DrainComplete) + // Validate status + EventuallyWithOffset(1, func(g Gomega) { + g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)). + ToNot(HaveOccurred()) - }) + g.Expect(nodeState.Annotations[constants.NodeStateDrainAnnotation]).To(Equal(constants.DrainIdle)) + }, waitTime, retryTime).Should(Succeed()) + patchAnnotation(nodeState, constants.NodeStateDrainAnnotationCurrent, constants.DrainIdle) - It("ignore non latest SriovNetworkNodeState generations", func() { + // Validate status + EventuallyWithOffset(1, func(g Gomega) { + g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)). + ToNot(HaveOccurred()) - _, err := sut.kubeClient.CoreV1().Nodes().Create(context.Background(), &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node", - }, - }, metav1.CreateOptions{}) - Expect(err).To(BeNil()) - - nodeState1 := &sriovnetworkv1.SriovNetworkNodeState{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node", - Generation: 123, - Annotations: map[string]string{consts.NodeStateDrainAnnotationCurrent: consts.DrainIdle}, - }, - } - Expect( - createSriovNetworkNodeState(sut.sriovClient, nodeState1)). - To(BeNil()) - - nodeState2 := &sriovnetworkv1.SriovNetworkNodeState{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node", - Generation: 777, - Annotations: map[string]string{consts.NodeStateDrainAnnotationCurrent: consts.DrainIdle}, + g.Expect(nodeState.Status.SyncStatus).To(Equal(constants.SyncStatusSucceeded)) + }, waitTime, retryTime).Should(Succeed()) + + Expect(nodeState.Status.LastSyncError).To(Equal("")) + }) + + It("Should apply the reset configuration when disableDrain is true", func(ctx context.Context) { + DeferCleanup(func(x bool) { vars.DisableDrain = x }, vars.DisableDrain) + vars.DisableDrain = true + + discoverSriovReturn.Store(&[]sriovnetworkv1.InterfaceExt{ + { + Name: "eno1", + Driver: "ice", + PciAddress: "0000:16:00.0", + DeviceID: "1593", + Vendor: "8086", + EswitchMode: "legacy", + LinkAdminState: "up", + LinkSpeed: "10000 Mb/s", + LinkType: "ETH", + Mac: "aa:bb:cc:dd:ee:ff", + Mtu: 1500, + TotalVfs: 2, + NumVfs: 2, + VFs: []sriovnetworkv1.VirtualFunction{ + { + Name: "eno1f0", + PciAddress: "0000:16:00.1", + VfID: 0, + }, + { + Name: "eno1f1", + PciAddress: "0000:16:00.2", + VfID: 1, + }}, }, + }) + + nodeState.Spec.Interfaces = []sriovnetworkv1.Interface{ + {Name: "eno1", + PciAddress: "0000:16:00.0", + LinkType: "eth", + NumVfs: 2, + VfGroups: []sriovnetworkv1.VfGroup{ + {ResourceName: "test", + DeviceType: "netdevice", + PolicyName: "test-policy", + VfRange: "eno1#0-1"}, + }}, } - Expect( - updateSriovNetworkNodeState(sut.sriovClient, nodeState2)). - To(BeNil()) + err := k8sClient.Update(ctx, nodeState) + Expect(err).ToNot(HaveOccurred()) - var msg Message - Eventually(refreshCh, "10s").Should(Receive(&msg)) - Expect(msg.syncStatus).To(Equal("InProgress")) + eventuallySyncStatusEqual(nodeState, constants.SyncStatusSucceeded) - Eventually(refreshCh, "10s").Should(Receive(&msg)) - Expect(msg.syncStatus).To(Equal("Succeeded")) + By("Simulate node policy removal") + nodeState.Spec.Interfaces = []sriovnetworkv1.Interface{} + err = k8sClient.Update(ctx, nodeState) + Expect(err).ToNot(HaveOccurred()) - Expect(sut.desiredNodeState.GetGeneration()).To(BeNumerically("==", 777)) + eventuallySyncStatusEqual(nodeState, constants.SyncStatusSucceeded) + assertLastStatusTransitionsContains(nodeState, 2, constants.SyncStatusInProgress) }) }) }) -func createSriovNetworkNodeState(c snclient.Interface, nodeState *sriovnetworkv1.SriovNetworkNodeState) error { - _, err := c.SriovnetworkV1(). - SriovNetworkNodeStates(vars.Namespace). - Create(context.Background(), nodeState, metav1.CreateOptions{}) - return err +func patchAnnotation(nodeState *sriovnetworkv1.SriovNetworkNodeState, key, value string) { + originalNodeState := nodeState.DeepCopy() + nodeState.Annotations[key] = value + err := k8sClient.Patch(context.Background(), nodeState, client.MergeFrom(originalNodeState)) + Expect(err).ToNot(HaveOccurred()) +} + +func createNode(nodeName string) (*corev1.Node, *sriovnetworkv1.SriovNetworkNodeState) { + node := corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Annotations: map[string]string{ + constants.NodeDrainAnnotation: constants.DrainIdle, + "machineconfiguration.openshift.io/desiredConfig": "worker-1", + }, + Labels: map[string]string{ + "test": "", + }, + }, + } + + nodeState := sriovnetworkv1.SriovNetworkNodeState{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Namespace: testNamespace, + Annotations: map[string]string{ + constants.NodeStateDrainAnnotation: constants.DrainIdle, + constants.NodeStateDrainAnnotationCurrent: constants.DrainIdle, + }, + }, + } + + Expect(k8sClient.Create(context.Background(), &node)).ToNot(HaveOccurred()) + Expect(k8sClient.Create(context.Background(), &nodeState)).ToNot(HaveOccurred()) + vars.NodeName = nodeName + + return &node, &nodeState +} + +func createDaemon( + hostHelper helper.HostHelpersInterface, + platformHelper platforms.Interface, + featureGates featuregate.FeatureGate, + disablePlugins []string) *daemon.NodeReconciler { + kClient, err := client.New( + cfg, + client.Options{ + Scheme: scheme.Scheme}) + Expect(err).ToNot(HaveOccurred()) + + By("Setup controller manager") + k8sManager, err = ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + }) + Expect(err).ToNot(HaveOccurred()) + + configController := daemon.New(kClient, hostHelper, platformHelper, eventRecorder, featureGates, disablePlugins) + err = configController.Init() + Expect(err).ToNot(HaveOccurred()) + err = configController.SetupWithManager(k8sManager) + Expect(err).ToNot(HaveOccurred()) + + return configController +} + +func eventuallySyncStatusEqual(nodeState *sriovnetworkv1.SriovNetworkNodeState, expectedState string) { + EventuallyWithOffset(1, func(g Gomega) { + g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)). + ToNot(HaveOccurred()) + g.Expect(nodeState.Status.SyncStatus).To(Equal(expectedState)) + }, 10*time.Second, 100*time.Millisecond).Should(Succeed()) } -func updateSriovNetworkNodeState(c snclient.Interface, nodeState *sriovnetworkv1.SriovNetworkNodeState) error { - _, err := c.SriovnetworkV1(). - SriovNetworkNodeStates(vars.Namespace). - Update(context.Background(), nodeState, metav1.UpdateOptions{}) - return err +func assertLastStatusTransitionsContains(nodeState *sriovnetworkv1.SriovNetworkNodeState, numberOfTransitions int, status string) { + events := &corev1.EventList{} + err := k8sClient.List( + context.Background(), + events, + client.MatchingFields{ + "involvedObject.name": nodeState.Name, + "reason": "SyncStatusChanged", + }, + client.Limit(numberOfTransitions), + ) + Expect(err).ToNot(HaveOccurred()) + + // Status transition events are in the form + // `Status changed from: Succeed to: InProgress` + Expect(events.Items).To(ContainElement( + HaveField("Message", ContainSubstring("to: "+status)))) } diff --git a/pkg/daemon/event_recorder.go b/pkg/daemon/event_recorder.go index 25b2d2351..80d99660d 100644 --- a/pkg/daemon/event_recorder.go +++ b/pkg/daemon/event_recorder.go @@ -4,29 +4,29 @@ import ( "context" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/scheme" typedv1core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" - snclientset "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) type EventRecorder struct { - client snclientset.Interface + client client.Client eventRecorder record.EventRecorder eventBroadcaster record.EventBroadcaster } // NewEventRecorder Create a new EventRecorder -func NewEventRecorder(c snclientset.Interface, kubeclient kubernetes.Interface) *EventRecorder { +func NewEventRecorder(c client.Client, kubeclient kubernetes.Interface, s *runtime.Scheme) *EventRecorder { eventBroadcaster := record.NewBroadcaster() eventBroadcaster.StartStructuredLogging(4) eventBroadcaster.StartRecordingToSink(&typedv1core.EventSinkImpl{Interface: kubeclient.CoreV1().Events("")}) - eventRecorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "config-daemon"}) + eventRecorder := eventBroadcaster.NewRecorder(s, corev1.EventSource{Component: "config-daemon"}) return &EventRecorder{ client: c, eventRecorder: eventRecorder, @@ -35,8 +35,9 @@ func NewEventRecorder(c snclientset.Interface, kubeclient kubernetes.Interface) } // SendEvent Send an Event on the NodeState object -func (e *EventRecorder) SendEvent(eventType string, msg string) { - nodeState, err := e.client.SriovnetworkV1().SriovNetworkNodeStates(vars.Namespace).Get(context.Background(), vars.NodeName, metav1.GetOptions{}) +func (e *EventRecorder) SendEvent(ctx context.Context, eventType string, msg string) { + nodeState := &sriovnetworkv1.SriovNetworkNodeState{} + err := e.client.Get(ctx, client.ObjectKey{Namespace: vars.Namespace, Name: vars.NodeName}, nodeState) if err != nil { log.Log.V(2).Error(err, "SendEvent(): Failed to fetch node state, skip SendEvent", "name", vars.NodeName) return diff --git a/pkg/daemon/plugin_test.go b/pkg/daemon/plugin_test.go index a13fc1f8b..9eeea7f53 100644 --- a/pkg/daemon/plugin_test.go +++ b/pkg/daemon/plugin_test.go @@ -1,16 +1,14 @@ package daemon import ( - gomock "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper" helperMocks "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper/mock" plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" - fakePlugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins/fake" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) @@ -41,11 +39,14 @@ var _ = Describe("config daemon plugin loading tests", func() { vars.ClusterType = consts.ClusterTypeKubernetes gmockController = gomock.NewController(GinkgoT()) helperMock = helperMocks.NewMockHostHelpersInterface(gmockController) - // k8s plugin is ATM the only plugin which require mocking/faking, as its New method performs additional logic - // other than simple plugin struct initialization - K8sPlugin = func(_ helper.HostHelpersInterface) (plugin.VendorPlugin, error) { - return &fakePlugin.FakePlugin{PluginName: "k8s"}, nil - } + helperMock.EXPECT().GetCurrentKernelArgs().Return("", nil).AnyTimes() + helperMock.EXPECT().IsKernelArgsSet("", consts.KernelArgIntelIommu).Return(false).AnyTimes() + helperMock.EXPECT().IsKernelArgsSet("", consts.KernelArgIommuPt).Return(false).AnyTimes() + helperMock.EXPECT().IsKernelArgsSet("", consts.KernelArgPciRealloc).Return(false).AnyTimes() + helperMock.EXPECT().IsKernelArgsSet("", consts.KernelArgRdmaExclusive).Return(false).AnyTimes() + helperMock.EXPECT().IsKernelArgsSet("", consts.KernelArgRdmaShared).Return(false).AnyTimes() + helperMock.EXPECT().ReadServiceInjectionManifestFile(gomock.Any()).Return(nil, nil).AnyTimes() + helperMock.EXPECT().ReadServiceManifestFile(gomock.Any()).Return(nil, nil).AnyTimes() }) It("loads relevant plugins Baremetal platform, kubernetes cluster type", func() { diff --git a/pkg/daemon/status.go b/pkg/daemon/status.go new file mode 100644 index 000000000..ea19d8dc1 --- /dev/null +++ b/pkg/daemon/status.go @@ -0,0 +1,144 @@ +package daemon + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" +) + +const ( + Unknown = "Unknown" +) + +func (dn *NodeReconciler) updateSyncState(ctx context.Context, desiredNodeState *sriovnetworkv1.SriovNetworkNodeState, status, failedMessage string) error { + funcLog := log.Log.WithName("updateSyncState") + currentNodeState := &sriovnetworkv1.SriovNetworkNodeState{} + desiredNodeState.Status.SyncStatus = status + desiredNodeState.Status.LastSyncError = failedMessage + + retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { + if err := dn.client.Get(ctx, client.ObjectKey{Namespace: desiredNodeState.Namespace, Name: desiredNodeState.Name}, currentNodeState); err != nil { + funcLog.Error(err, "failed to get latest node state", + "SyncStatus", status, + "LastSyncError", failedMessage) + return err + } + // update the object meta if not the patch can fail if the object did change + desiredNodeState.ObjectMeta = currentNodeState.ObjectMeta + + funcLog.V(2).Info("update nodeState status", + "CurrentSyncStatus", currentNodeState.Status.SyncStatus, + "CurrentLastSyncError", currentNodeState.Status.LastSyncError, + "NewSyncStatus", desiredNodeState.Status.SyncStatus, + "NewFailedMessage", desiredNodeState.Status.LastSyncError) + + err := dn.client.Status().Patch(ctx, desiredNodeState, client.MergeFrom(currentNodeState)) + if err != nil { + funcLog.Error(err, "failed to update node state status", + "SyncStatus", status, + "LastSyncError", failedMessage) + return err + } + return nil + }) + + if retryErr != nil { + funcLog.Error(retryErr, "failed to update node state status") + return retryErr + } + + dn.recordStatusChangeEvent(ctx, currentNodeState.Status.SyncStatus, status, failedMessage) + return nil +} + +func (dn *NodeReconciler) shouldUpdateStatus(current, desiredNodeState *sriovnetworkv1.SriovNetworkNodeState) bool { + // check number of interfaces are equal + if len(current.Status.Interfaces) != len(desiredNodeState.Status.Interfaces) { + return true + } + + // check for bridges + if !equality.Semantic.DeepEqual(current.Status.Bridges, desiredNodeState.Status.Bridges) { + return true + } + + // check for system + if !equality.Semantic.DeepEqual(current.Status.System, desiredNodeState.Status.System) { + return true + } + + // check for interfaces + // we can't use deep equal here because if we have a vf inside a pod is name will not be available for example + // we use the index for both lists + c := current.Status.DeepCopy().Interfaces + d := desiredNodeState.Status.DeepCopy().Interfaces + for idx := range d { + // check if it's a new device + if d[idx].PciAddress != c[idx].PciAddress { + return true + } + // remove all the vfs + d[idx].VFs = nil + c[idx].VFs = nil + + if !equality.Semantic.DeepEqual(d[idx], c[idx]) { + return true + } + } + + return false +} + +func (dn *NodeReconciler) updateStatusFromHost(nodeState *sriovnetworkv1.SriovNetworkNodeState) error { + log.Log.WithName("updateStatusFromHost").Info("Getting host network status") + var ifaces []sriovnetworkv1.InterfaceExt + var bridges sriovnetworkv1.Bridges + var err error + + if vars.PlatformType == consts.VirtualOpenStack { + ifaces, err = dn.platformHelpers.DiscoverSriovDevicesVirtual() + if err != nil { + return err + } + } else { + ifaces, err = dn.HostHelpers.DiscoverSriovDevices(dn.HostHelpers) + if err != nil { + return err + } + if vars.ManageSoftwareBridges { + bridges, err = dn.HostHelpers.DiscoverBridges() + if err != nil { + return err + } + } + } + + nodeState.Status.Interfaces = ifaces + nodeState.Status.Bridges = bridges + nodeState.Status.System.RdmaMode, err = dn.HostHelpers.DiscoverRDMASubsystem() + return err +} + +func (dn *NodeReconciler) recordStatusChangeEvent(ctx context.Context, oldStatus, newStatus, lastError string) { + if oldStatus != newStatus { + if oldStatus == "" { + oldStatus = Unknown + } + if newStatus == "" { + newStatus = Unknown + } + eventMsg := fmt.Sprintf("Status changed from: %s to: %s", oldStatus, newStatus) + if lastError != "" { + eventMsg = fmt.Sprintf("%s. Last Error: %s", eventMsg, lastError) + } + dn.eventRecorder.SendEvent(ctx, "SyncStatusChanged", eventMsg) + } +} diff --git a/pkg/daemon/writer.go b/pkg/daemon/writer.go deleted file mode 100644 index 09d06d8f9..000000000 --- a/pkg/daemon/writer.go +++ /dev/null @@ -1,282 +0,0 @@ -package daemon - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - "time" - - "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/util/retry" - "sigs.k8s.io/controller-runtime/pkg/log" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - snclientset "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" -) - -const ( - CheckpointFileName = "sno-initial-node-state.json" - Unknown = "Unknown" -) - -type NodeStateStatusWriter struct { - client snclientset.Interface - status sriovnetworkv1.SriovNetworkNodeStateStatus - OnHeartbeatFailure func() - platformHelper platforms.Interface - hostHelper helper.HostHelpersInterface - eventRecorder *EventRecorder -} - -// NewNodeStateStatusWriter Create a new NodeStateStatusWriter -func NewNodeStateStatusWriter(c snclientset.Interface, - f func(), er *EventRecorder, - hostHelper helper.HostHelpersInterface, - platformHelper platforms.Interface) *NodeStateStatusWriter { - return &NodeStateStatusWriter{ - client: c, - OnHeartbeatFailure: f, - eventRecorder: er, - hostHelper: hostHelper, - platformHelper: platformHelper, - } -} - -// RunOnce initial the interface status for both baremetal and virtual environments -func (w *NodeStateStatusWriter) RunOnce() error { - log.Log.V(0).Info("RunOnce()") - msg := Message{} - - if vars.PlatformType == consts.VirtualOpenStack { - ns, err := w.getCheckPointNodeState() - if err != nil { - return err - } - - if ns == nil { - err = w.platformHelper.CreateOpenstackDevicesInfo() - if err != nil { - return err - } - } else { - w.platformHelper.CreateOpenstackDevicesInfoFromNodeStatus(ns) - } - } - - log.Log.V(0).Info("RunOnce(): first poll for nic status") - if err := w.pollNicStatus(); err != nil { - log.Log.Error(err, "RunOnce(): first poll failed") - } - - ns, err := w.setNodeStateStatus(msg) - if err != nil { - log.Log.Error(err, "RunOnce(): first writing to node status failed") - } - return w.writeCheckpointFile(ns) -} - -// Run reads from the writer channel and sets the interface status. It will -// return if the stop channel is closed. Intended to be run via a goroutine. -func (w *NodeStateStatusWriter) Run(stop <-chan struct{}, refresh <-chan Message, syncCh chan<- struct{}) error { - log.Log.V(0).Info("Run(): start writer") - msg := Message{} - - for { - select { - case <-stop: - log.Log.V(0).Info("Run(): stop writer") - return nil - case msg = <-refresh: - log.Log.V(0).Info("Run(): refresh trigger") - if err := w.pollNicStatus(); err != nil { - continue - } - _, err := w.setNodeStateStatus(msg) - if err != nil { - log.Log.Error(err, "Run() refresh: writing to node status failed") - } - syncCh <- struct{}{} - case <-time.After(30 * time.Second): - log.Log.V(2).Info("Run(): period refresh") - if err := w.pollNicStatus(); err != nil { - continue - } - w.setNodeStateStatus(msg) - } - } -} - -func (w *NodeStateStatusWriter) pollNicStatus() error { - log.Log.V(2).Info("pollNicStatus()") - var iface []sriovnetworkv1.InterfaceExt - var bridges sriovnetworkv1.Bridges - var err error - - if vars.PlatformType == consts.VirtualOpenStack { - iface, err = w.platformHelper.DiscoverSriovDevicesVirtual() - if err != nil { - return err - } - } else { - iface, err = w.hostHelper.DiscoverSriovDevices(w.hostHelper) - if err != nil { - return err - } - if vars.ManageSoftwareBridges { - bridges, err = w.hostHelper.DiscoverBridges() - if err != nil { - return err - } - } - } - - w.status.Interfaces = iface - w.status.Bridges = bridges - - return nil -} - -func (w *NodeStateStatusWriter) updateNodeStateStatusRetry(f func(*sriovnetworkv1.SriovNetworkNodeState)) (*sriovnetworkv1.SriovNetworkNodeState, error) { - var nodeState *sriovnetworkv1.SriovNetworkNodeState - var oldStatus, newStatus, lastError string - - err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - n, getErr := w.getNodeState() - if getErr != nil { - return getErr - } - oldStatus = n.Status.SyncStatus - - // Call the status modifier. - f(n) - - newStatus = n.Status.SyncStatus - lastError = n.Status.LastSyncError - - var err error - nodeState, err = w.client.SriovnetworkV1().SriovNetworkNodeStates(vars.Namespace).UpdateStatus(context.Background(), n, metav1.UpdateOptions{}) - if err != nil { - log.Log.V(0).Error(err, "updateNodeStateStatusRetry(): fail to update the node status") - } - return err - }) - if err != nil { - // may be conflict if max retries were hit - return nil, fmt.Errorf("unable to update node %v: %v", nodeState, err) - } - - w.recordStatusChangeEvent(oldStatus, newStatus, lastError) - - return nodeState, nil -} - -func (w *NodeStateStatusWriter) setNodeStateStatus(msg Message) (*sriovnetworkv1.SriovNetworkNodeState, error) { - nodeState, err := w.updateNodeStateStatusRetry(func(nodeState *sriovnetworkv1.SriovNetworkNodeState) { - nodeState.Status.Interfaces = w.status.Interfaces - nodeState.Status.Bridges = w.status.Bridges - if msg.lastSyncError != "" || msg.syncStatus == consts.SyncStatusSucceeded { - // clear lastSyncError when sync Succeeded - nodeState.Status.LastSyncError = msg.lastSyncError - } - nodeState.Status.SyncStatus = msg.syncStatus - - log.Log.V(0).Info("setNodeStateStatus(): status", - "sync-status", nodeState.Status.SyncStatus, - "last-sync-error", nodeState.Status.LastSyncError) - }) - if err != nil { - return nil, err - } - return nodeState, nil -} - -// recordStatusChangeEvent sends event in case oldStatus differs from newStatus -func (w *NodeStateStatusWriter) recordStatusChangeEvent(oldStatus, newStatus, lastError string) { - if oldStatus != newStatus { - if oldStatus == "" { - oldStatus = Unknown - } - if newStatus == "" { - newStatus = Unknown - } - eventMsg := fmt.Sprintf("Status changed from: %s to: %s", oldStatus, newStatus) - if lastError != "" { - eventMsg = fmt.Sprintf("%s. Last Error: %s", eventMsg, lastError) - } - w.eventRecorder.SendEvent("SyncStatusChanged", eventMsg) - } -} - -// getNodeState queries the kube apiserver to get the SriovNetworkNodeState CR -func (w *NodeStateStatusWriter) getNodeState() (*sriovnetworkv1.SriovNetworkNodeState, error) { - var lastErr error - var n *sriovnetworkv1.SriovNetworkNodeState - err := wait.PollImmediate(10*time.Second, 5*time.Minute, func() (bool, error) { - n, lastErr = w.client.SriovnetworkV1().SriovNetworkNodeStates(vars.Namespace).Get(context.Background(), vars.NodeName, metav1.GetOptions{}) - if lastErr == nil { - return true, nil - } - log.Log.Error(lastErr, "getNodeState(): Failed to fetch node state, close all connections and retry...", "name", vars.NodeName) - // Use the Get() also as an client-go keepalive indicator for the TCP connection. - w.OnHeartbeatFailure() - return false, nil - }) - if err != nil { - if err == wait.ErrWaitTimeout { - return nil, errors.Wrapf(lastErr, "Timed out trying to fetch node %s", vars.NodeName) - } - return nil, err - } - return n, nil -} - -func (w *NodeStateStatusWriter) writeCheckpointFile(ns *sriovnetworkv1.SriovNetworkNodeState) error { - configdir := filepath.Join(vars.Destdir, CheckpointFileName) - file, err := os.OpenFile(configdir, os.O_RDWR|os.O_CREATE, 0644) - if err != nil { - return err - } - defer file.Close() - log.Log.Info("writeCheckpointFile(): try to decode the checkpoint file") - if err = json.NewDecoder(file).Decode(&sriovnetworkv1.InitialState); err != nil { - log.Log.V(2).Error(err, "writeCheckpointFile(): fail to decode, writing new file instead") - log.Log.Info("writeCheckpointFile(): write checkpoint file") - if err = file.Truncate(0); err != nil { - return err - } - if _, err = file.Seek(0, 0); err != nil { - return err - } - if err = json.NewEncoder(file).Encode(*ns); err != nil { - return err - } - sriovnetworkv1.InitialState = *ns - } - return nil -} - -func (w *NodeStateStatusWriter) getCheckPointNodeState() (*sriovnetworkv1.SriovNetworkNodeState, error) { - log.Log.Info("getCheckPointNodeState()") - configdir := filepath.Join(vars.Destdir, CheckpointFileName) - file, err := os.OpenFile(configdir, os.O_RDONLY, 0644) - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } - return nil, err - } - defer file.Close() - if err = json.NewDecoder(file).Decode(&sriovnetworkv1.InitialState); err != nil { - return nil, err - } - - return &sriovnetworkv1.InitialState, nil -} diff --git a/pkg/drain/drainer.go b/pkg/drain/drainer.go index a3500dc47..a58f922ff 100644 --- a/pkg/drain/drainer.go +++ b/pkg/drain/drainer.go @@ -6,17 +6,21 @@ import ( "strings" "time" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/kubectl/pkg/drain" - "sigs.k8s.io/controller-runtime/pkg/log" constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) +var ( + DrainTimeOut = 90 * time.Second +) + // writer implements io.Writer interface as a pass-through for log.Log. type writer struct { logFunc func(msg string, keysAndValues ...interface{}) @@ -29,7 +33,7 @@ func (w writer) Write(p []byte) (n int, err error) { } type DrainInterface interface { - DrainNode(context.Context, *corev1.Node, bool) (bool, error) + DrainNode(context.Context, *corev1.Node, bool, bool) (bool, error) CompleteDrainNode(context.Context, *corev1.Node) (bool, error) } @@ -53,9 +57,9 @@ func NewDrainer(platformHelpers platforms.Interface) (DrainInterface, error) { // DrainNode the function cordon a node and drain pods from it // if fullNodeDrain true all the pods on the system will get drained // for openshift system we also pause the machine config pool this machine is part of it -func (d *Drainer) DrainNode(ctx context.Context, node *corev1.Node, fullNodeDrain bool) (bool, error) { - reqLogger := log.FromContext(ctx).WithValues("drain node", node.Name) - reqLogger.Info("drainNode(): Node drain requested", "node", node.Name) +func (d *Drainer) DrainNode(ctx context.Context, node *corev1.Node, fullNodeDrain, singleNode bool) (bool, error) { + reqLogger := ctx.Value("logger").(logr.Logger).WithName("drainNode") + reqLogger.Info("Node drain requested") completed, err := d.platformHelpers.OpenshiftBeforeDrainNode(ctx, node) if err != nil { @@ -68,19 +72,26 @@ func (d *Drainer) DrainNode(ctx context.Context, node *corev1.Node, fullNodeDrai return false, nil } + // Check if we are on a single node, and we require a reboot/full-drain we just return + if fullNodeDrain && singleNode { + return true, nil + } + drainHelper := createDrainHelper(d.kubeClient, ctx, fullNodeDrain) backoff := wait.Backoff{ - Steps: 5, - Duration: 10 * time.Second, + Steps: 3, + Duration: 2 * time.Second, Factor: 2, } var lastErr error reqLogger.Info("drainNode(): Start draining") - if err = wait.ExponentialBackoff(backoff, func() (bool, error) { + if err = wait.ExponentialBackoffWithContext(ctx, backoff, func(ctx context.Context) (bool, error) { + nodeCopy := node.DeepCopy() err := drain.RunCordonOrUncordon(drainHelper, node, true) if err != nil { lastErr = err + node = nodeCopy reqLogger.Info("drainNode(): Cordon failed, retrying", "error", err) return false, nil } @@ -98,7 +109,7 @@ func (d *Drainer) DrainNode(ctx context.Context, node *corev1.Node, fullNodeDrai reqLogger.Info("drainNode(): failed to drain node", "error", err) return false, err } - reqLogger.Info("drainNode(): drain complete") + reqLogger.Info("drainNode(): Drain completed") return true, nil } @@ -106,8 +117,7 @@ func (d *Drainer) DrainNode(ctx context.Context, node *corev1.Node, fullNodeDrai // for openshift system we also remove the pause from the machine config pool this node is part of // only if we are the last draining node on that pool func (d *Drainer) CompleteDrainNode(ctx context.Context, node *corev1.Node) (bool, error) { - logger := log.FromContext(ctx) - logger.Info("CompleteDrainNode:()") + logger := ctx.Value("logger").(logr.Logger).WithName("CompleteDrainNode") // Create drain helper object // full drain is not important here @@ -135,24 +145,33 @@ func (d *Drainer) CompleteDrainNode(ctx context.Context, node *corev1.Node) (boo // if fullDrain is false we only remove pods that have the resourcePrefix // if not we remove all the pods in the node func createDrainHelper(kubeClient kubernetes.Interface, ctx context.Context, fullDrain bool) *drain.Helper { - logger := log.FromContext(ctx) + logger := ctx.Value("logger").(logr.Logger).WithName("createDrainHelper") + drainer := &drain.Helper{ Client: kubeClient, Force: true, IgnoreAllDaemonSets: true, DeleteEmptyDirData: true, GracePeriodSeconds: -1, - Timeout: 90 * time.Second, - OnPodDeletedOrEvicted: func(pod *corev1.Pod, usingEviction bool) { + Timeout: DrainTimeOut, + OnPodDeletionOrEvictionFinished: func(pod *corev1.Pod, usingEviction bool, err error) { + if err != nil { + verbStr := constants.DrainDelete + if usingEviction { + verbStr = constants.DrainEvict + } + logger.Error(err, fmt.Sprintf("failed to %s pod %s/%s from node", verbStr, pod.Namespace, pod.Name)) + return + } verbStr := constants.DrainDeleted if usingEviction { verbStr = constants.DrainEvicted } - log.Log.Info(fmt.Sprintf("%s pod from Node %s/%s", verbStr, pod.Namespace, pod.Name)) + logger.Info(fmt.Sprintf("%s pod %s/%s from node", verbStr, pod.Namespace, pod.Name)) }, Ctx: ctx, - Out: writer{logger.Info}, - ErrOut: writer{func(msg string, kv ...interface{}) { logger.Error(nil, msg, kv...) }}, + Out: writer{func(msg string, kv ...interface{}) { logger.Info(strings.ReplaceAll(msg, "\n", "")) }}, + ErrOut: writer{func(msg string, kv ...interface{}) { logger.Error(nil, strings.ReplaceAll(msg, "\n", ""), kv...) }}, } // when we just want to drain and not reboot we can only remove the pods using sriov devices diff --git a/pkg/drain/drainer_suite_test.go b/pkg/drain/drainer_suite_test.go new file mode 100644 index 000000000..e785b8b28 --- /dev/null +++ b/pkg/drain/drainer_suite_test.go @@ -0,0 +1,136 @@ +package drain_test + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + openshiftconfigv1 "github.com/openshift/api/config/v1" + mcfgv1 "github.com/openshift/api/machineconfiguration/v1" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "go.uber.org/zap/zapcore" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + k8sClient client.Client + testEnv *envtest.Environment + cfg *rest.Config +) + +// Define utility constants for object names and testing timeouts/durations and intervals. +const testNamespace = "openshift-sriov-network-operator" + +var _ = BeforeSuite(func() { + var err error + + logf.SetLogger(zap.New( + zap.WriteTo(GinkgoWriter), + zap.UseDevMode(true), + func(o *zap.Options) { + o.TimeEncoder = zapcore.RFC3339NanoTimeEncoder + })) + + // Go to project root directory + err = os.Chdir("../..") + Expect(err).NotTo(HaveOccurred()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("config", "crd", "bases"), filepath.Join("test", "util", "crds")}, + ErrorIfCRDPathMissing: true, + } + + testEnv.ControlPlane.GetAPIServer().Configure().Set("disable-admission-plugins", "MutatingAdmissionWebhook", "ValidatingAdmissionWebhook") + + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + By("registering schemes") + err = sriovnetworkv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = netattdefv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = mcfgv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = openshiftconfigv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = monitoringv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + vars.Config = cfg + vars.Scheme = scheme.Scheme + vars.Namespace = testNamespace + vars.ResourcePrefix = "sriovoperator.io" + + By("creating K8s client") + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + By("creating default/common k8s objects for tests") + // Create test namespace + ns := &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: testNamespace, + }, + Spec: corev1.NamespaceSpec{}, + Status: corev1.NamespaceStatus{}, + } + Expect(k8sClient.Create(context.Background(), ns)).Should(Succeed()) + + sa := &corev1.ServiceAccount{TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: testNamespace, + }} + Expect(k8sClient.Create(context.Background(), sa)).Should(Succeed()) + + // Create openshift Infrastructure + infra := &openshiftconfigv1.Infrastructure{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + Spec: openshiftconfigv1.InfrastructureSpec{}, + Status: openshiftconfigv1.InfrastructureStatus{ + ControlPlaneTopology: openshiftconfigv1.HighlyAvailableTopologyMode, + }, + } + Expect(k8sClient.Create(context.Background(), infra)).Should(Succeed()) +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + if testEnv != nil { + Eventually(func() error { + return testEnv.Stop() + }, util.APITimeout, time.Second).ShouldNot(HaveOccurred()) + } +}) + +func TestDrainer(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Drainer Suite") +} diff --git a/pkg/drain/drainer_test.go b/pkg/drain/drainer_test.go new file mode 100644 index 000000000..12f3afd41 --- /dev/null +++ b/pkg/drain/drainer_test.go @@ -0,0 +1,258 @@ +package drain_test + +import ( + "context" + "fmt" + "os" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "go.uber.org/mock/gomock" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/drain" + snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" + mock_platforms "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/mock" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" +) + +var ( + cancel context.CancelFunc + ctx context.Context + drn drain.DrainInterface + t FullGinkgoTInterface + mockCtrl *gomock.Controller + platformHelper *mock_platforms.MockInterface +) + +var _ = Describe("Drainer", Ordered, func() { + BeforeAll(func() { + Expect(k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovOperatorConfig{}, client.InNamespace(testNamespace))).ToNot(HaveOccurred()) + soc := &sriovnetworkv1.SriovOperatorConfig{ObjectMeta: metav1.ObjectMeta{ + Name: constants.DefaultConfigName, + Namespace: testNamespace, + }, + Spec: sriovnetworkv1.SriovOperatorConfigSpec{ + LogLevel: 2, + }, + } + err := k8sClient.Create(context.Background(), soc) + Expect(err).ToNot(HaveOccurred()) + + snolog.SetLogLevel(2) + // Check if the environment variable CLUSTER_TYPE is set + if clusterType, ok := os.LookupEnv("CLUSTER_TYPE"); ok && clusterType == constants.ClusterTypeOpenshift { + vars.ClusterType = constants.ClusterTypeOpenshift + } else { + vars.ClusterType = constants.ClusterTypeKubernetes + } + }) + + BeforeEach(func() { + ctx, cancel = context.WithCancel(context.Background()) + ctx = context.WithValue(ctx, "logger", log.FromContext(ctx)) + + t = GinkgoT() + mockCtrl = gomock.NewController(t) + platformHelper = mock_platforms.NewMockInterface(mockCtrl) + + // new drainer + var err error + drn, err = drain.NewDrainer(platformHelper) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + Expect(k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovNetworkNodeState{}, client.InNamespace(testNamespace))).ToNot(HaveOccurred()) + Expect(k8sClient.DeleteAllOf(context.Background(), &corev1.Pod{}, client.InNamespace(testNamespace), &client.DeleteAllOfOptions{DeleteOptions: client.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)}})).ToNot(HaveOccurred()) + Expect(k8sClient.DeleteAllOf(context.Background(), &corev1.Node{}, &client.DeleteAllOfOptions{DeleteOptions: client.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)}})).ToNot(HaveOccurred()) + + By("Shutdown controller manager") + cancel() + }) + + AfterAll(func() { + Expect(k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovOperatorConfig{}, client.InNamespace(testNamespace))).ToNot(HaveOccurred()) + }) + + Context("DrainNode", func() { + It("should failed if the platform is not able to drain", func() { + n, _ := createNode("node0") + platformHelper.EXPECT().OpenshiftBeforeDrainNode(ctx, n).Return(false, fmt.Errorf("failed")) + + completed, err := drn.DrainNode(ctx, n, false, false) + Expect(err).To(HaveOccurred()) + Expect(completed).To(BeFalse()) + }) + + It("should return not completed base on OpenshiftBeforeDrainNode call", func() { + n, _ := createNode("node0") + platformHelper.EXPECT().OpenshiftBeforeDrainNode(ctx, n).Return(false, nil) + + completed, err := drn.DrainNode(ctx, n, false, false) + Expect(err).ToNot(HaveOccurred()) + Expect(completed).To(BeFalse()) + }) + + It("should return error if not able to cordon the node", func() { + n, _ := createNode("node0") + nCopy := n.DeepCopy() + nCopy.Name = "node1" + originalDrainTimeOut := drain.DrainTimeOut + drain.DrainTimeOut = 3 * time.Second + defer func() { + drain.DrainTimeOut = originalDrainTimeOut + }() + + platformHelper.EXPECT().OpenshiftBeforeDrainNode(ctx, nCopy).Return(true, nil) + + _, err := drn.DrainNode(ctx, nCopy, false, false) + Expect(err).To(HaveOccurred()) + }) + + It("should return error if not able to drain the node", func() { + n, _ := createNode("node0") + platformHelper.EXPECT().OpenshiftBeforeDrainNode(ctx, n).Return(true, nil) + createPodWithFinalizerOnNode(ctx, "test-node-0", "node0") + originalDrainTimeOut := drain.DrainTimeOut + drain.DrainTimeOut = 3 * time.Second + defer func() { + drain.DrainTimeOut = originalDrainTimeOut + }() + + _, err := drn.DrainNode(ctx, n, true, false) + Expect(err).To(HaveOccurred()) + }) + + It("should remove pods with sriov devices only if a full drain is not requested", func() { + n, _ := createNode("node1") + resources := map[corev1.ResourceName]resource.Quantity{corev1.ResourceName(fmt.Sprintf("%s/test", vars.ResourcePrefix)): resource.MustParse("1")} + n.Status.Allocatable = resources + n.Status.Capacity = resources + err := k8sClient.Status().Update(ctx, n) + Expect(err).ToNot(HaveOccurred()) + platformHelper.EXPECT().OpenshiftBeforeDrainNode(ctx, n).Return(true, nil) + createPodOnNode(ctx, "regular-pod", "node1") + createPodWithSriovDeviceOnNode(ctx, "sriov-pod", "node1") + + go func() { + Eventually(func(g Gomega) { + podObj := &corev1.Pod{} + err = k8sClient.Get(ctx, client.ObjectKey{Name: "sriov-pod", Namespace: testNamespace}, podObj) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(podObj.DeletionTimestamp).ToNot(BeNil()) + err = k8sClient.Delete(ctx, podObj, &client.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)}) + g.Expect(err).ToNot(HaveOccurred()) + }, 2*time.Minute, time.Second).Should(Succeed()) + }() + + _, err = drn.DrainNode(ctx, n, false, false) + Expect(err).ToNot(HaveOccurred()) + pod := &corev1.Pod{} + err = k8sClient.Get(ctx, client.ObjectKey{Name: "regular-pod", Namespace: testNamespace}, pod) + Expect(err).ToNot(HaveOccurred()) + err = k8sClient.Get(ctx, client.ObjectKey{Name: "sriov-pod", Namespace: testNamespace}, pod) + Expect(err).To(HaveOccurred()) + Expect(errors.IsNotFound(err)).To(BeTrue()) + + }) + }) + + Context("CompleteDrain", func() { + It("should return error if the un cordon failed", func() { + n, _ := createNode("node0") + nCopy := n.DeepCopy() + nCopy.Name = "node1" + nCopy.Spec.Unschedulable = true + + completed, err := drn.CompleteDrainNode(ctx, nCopy) + Expect(err).To(HaveOccurred()) + Expect(completed).To(BeFalse()) + }) + + It("should return error if OpenshiftAfterCompleteDrainNode failed", func() { + n, _ := createNode("node0") + n.Spec.Unschedulable = true + platformHelper.EXPECT().OpenshiftAfterCompleteDrainNode(ctx, n).Return(false, fmt.Errorf("test")) + + completed, err := drn.CompleteDrainNode(ctx, n) + Expect(err).To(HaveOccurred()) + Expect(completed).To(BeFalse()) + }) + + It("should return completed", func() { + n, _ := createNode("node0") + n.Spec.Unschedulable = true + platformHelper.EXPECT().OpenshiftAfterCompleteDrainNode(ctx, n).Return(true, nil) + + completed, err := drn.CompleteDrainNode(ctx, n) + Expect(err).ToNot(HaveOccurred()) + Expect(completed).To(BeTrue()) + }) + }) +}) + +func createNode(nodeName string) (*corev1.Node, *sriovnetworkv1.SriovNetworkNodeState) { + node := corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Annotations: map[string]string{ + constants.NodeDrainAnnotation: constants.DrainIdle, + "machineconfiguration.openshift.io/desiredConfig": "worker-1", + }, + Labels: map[string]string{ + "test": "", + }, + }, + } + + nodeState := sriovnetworkv1.SriovNetworkNodeState{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Namespace: testNamespace, + Annotations: map[string]string{ + constants.NodeStateDrainAnnotation: constants.DrainIdle, + constants.NodeStateDrainAnnotationCurrent: constants.DrainIdle, + }, + }, + } + + Expect(k8sClient.Create(ctx, &node)).ToNot(HaveOccurred()) + Expect(k8sClient.Create(ctx, &nodeState)).ToNot(HaveOccurred()) + vars.NodeName = nodeName + + return &node, &nodeState +} + +func createPodOnNode(ctx context.Context, podName, nodeName string) { + pod := corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: podName, Namespace: testNamespace}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "test", Image: "test", Command: []string{"test"}}}, + NodeName: nodeName, TerminationGracePeriodSeconds: pointer.Int64(1)}} + Expect(k8sClient.Create(ctx, &pod)).ToNot(HaveOccurred()) +} + +func createPodWithFinalizerOnNode(ctx context.Context, podName, nodeName string) { + pod := corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: podName, Namespace: testNamespace, Finalizers: []string{"sriov-operator.io/test"}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "test", Image: "test", Command: []string{"test"}}}, + NodeName: nodeName, TerminationGracePeriodSeconds: pointer.Int64(1)}} + Expect(k8sClient.Create(ctx, &pod)).ToNot(HaveOccurred()) +} + +func createPodWithSriovDeviceOnNode(ctx context.Context, podName, nodeName string) { + resources := map[corev1.ResourceName]resource.Quantity{corev1.ResourceName(fmt.Sprintf("%s/test", vars.ResourcePrefix)): resource.MustParse("1")} + pod := corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: podName, Namespace: testNamespace}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "test", Image: "test", Command: []string{"test"}, + Resources: corev1.ResourceRequirements{Requests: resources, Limits: resources}}}, + NodeName: nodeName, TerminationGracePeriodSeconds: pointer.Int64(1)}} + Expect(k8sClient.Create(ctx, &pod)).ToNot(HaveOccurred()) +} diff --git a/pkg/helper/host.go b/pkg/helper/host.go index 2e93a30d3..3a8eebd6a 100644 --- a/pkg/helper/host.go +++ b/pkg/helper/host.go @@ -24,22 +24,14 @@ type hostHelpers struct { mlx.MellanoxInterface } -// Use for unit tests -func NewHostHelpers(utilsHelper utils.CmdInterface, - hostManager host.HostManagerInterface, - storeManager store.ManagerInterface, - mlxHelper mlx.MellanoxInterface) HostHelpersInterface { - return &hostHelpers{utilsHelper, hostManager, storeManager, mlxHelper} -} - func NewDefaultHostHelpers() (HostHelpersInterface, error) { utilsHelper := utils.New() - mlxHelper := mlx.New(utilsHelper) hostManager, err := host.NewHostManager(utilsHelper) if err != nil { log.Log.Error(err, "failed to create host manager") return nil, err } + mlxHelper := mlx.New(utilsHelper, hostManager) storeManager, err := store.NewManager() if err != nil { log.Log.Error(err, "failed to create store manager") diff --git a/pkg/helper/mock/mock_helper.go b/pkg/helper/mock/mock_helper.go index cfca2a768..f162cdf18 100644 --- a/pkg/helper/mock/mock_helper.go +++ b/pkg/helper/mock/mock_helper.go @@ -1,24 +1,31 @@ // Code generated by MockGen. DO NOT EDIT. // Source: host.go +// +// Generated by this command: +// +// mockgen -destination mock/mock_helper.go -source host.go +// // Package mock_helper is a generated GoMock package. package mock_helper import ( + "net" reflect "reflect" - gomock "github.com/golang/mock/gomock" v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" store "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/store" types "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" mlxutils "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vendors/mellanox" netlink "github.com/vishvananda/netlink" + gomock "go.uber.org/mock/gomock" ) // MockHostHelpersInterface is a mock of HostHelpersInterface interface. type MockHostHelpersInterface struct { ctrl *gomock.Controller recorder *MockHostHelpersInterfaceMockRecorder + isgomock struct{} } // MockHostHelpersInterfaceMockRecorder is the mock recorder for MockHostHelpersInterface. @@ -47,7 +54,7 @@ func (m *MockHostHelpersInterface) AddDisableNMUdevRule(pfPciAddress string) err } // AddDisableNMUdevRule indicates an expected call of AddDisableNMUdevRule. -func (mr *MockHostHelpersInterfaceMockRecorder) AddDisableNMUdevRule(pfPciAddress interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) AddDisableNMUdevRule(pfPciAddress any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddDisableNMUdevRule", reflect.TypeOf((*MockHostHelpersInterface)(nil).AddDisableNMUdevRule), pfPciAddress) } @@ -61,7 +68,7 @@ func (m *MockHostHelpersInterface) AddPersistPFNameUdevRule(pfPciAddress, pfName } // AddPersistPFNameUdevRule indicates an expected call of AddPersistPFNameUdevRule. -func (mr *MockHostHelpersInterfaceMockRecorder) AddPersistPFNameUdevRule(pfPciAddress, pfName interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) AddPersistPFNameUdevRule(pfPciAddress, pfName any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddPersistPFNameUdevRule", reflect.TypeOf((*MockHostHelpersInterface)(nil).AddPersistPFNameUdevRule), pfPciAddress, pfName) } @@ -75,7 +82,7 @@ func (m *MockHostHelpersInterface) AddVfRepresentorUdevRule(pfPciAddress, pfName } // AddVfRepresentorUdevRule indicates an expected call of AddVfRepresentorUdevRule. -func (mr *MockHostHelpersInterfaceMockRecorder) AddVfRepresentorUdevRule(pfPciAddress, pfName, pfSwitchID, pfSwitchPort interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) AddVfRepresentorUdevRule(pfPciAddress, pfName, pfSwitchID, pfSwitchPort any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddVfRepresentorUdevRule", reflect.TypeOf((*MockHostHelpersInterface)(nil).AddVfRepresentorUdevRule), pfPciAddress, pfName, pfSwitchID, pfSwitchPort) } @@ -89,7 +96,7 @@ func (m *MockHostHelpersInterface) BindDefaultDriver(pciAddr string) error { } // BindDefaultDriver indicates an expected call of BindDefaultDriver. -func (mr *MockHostHelpersInterfaceMockRecorder) BindDefaultDriver(pciAddr interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) BindDefaultDriver(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BindDefaultDriver", reflect.TypeOf((*MockHostHelpersInterface)(nil).BindDefaultDriver), pciAddr) } @@ -103,7 +110,7 @@ func (m *MockHostHelpersInterface) BindDpdkDriver(pciAddr, driver string) error } // BindDpdkDriver indicates an expected call of BindDpdkDriver. -func (mr *MockHostHelpersInterfaceMockRecorder) BindDpdkDriver(pciAddr, driver interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) BindDpdkDriver(pciAddr, driver any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BindDpdkDriver", reflect.TypeOf((*MockHostHelpersInterface)(nil).BindDpdkDriver), pciAddr, driver) } @@ -117,7 +124,7 @@ func (m *MockHostHelpersInterface) BindDriverByBusAndDevice(bus, device, driver } // BindDriverByBusAndDevice indicates an expected call of BindDriverByBusAndDevice. -func (mr *MockHostHelpersInterfaceMockRecorder) BindDriverByBusAndDevice(bus, device, driver interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) BindDriverByBusAndDevice(bus, device, driver any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BindDriverByBusAndDevice", reflect.TypeOf((*MockHostHelpersInterface)(nil).BindDriverByBusAndDevice), bus, device, driver) } @@ -147,11 +154,25 @@ func (m *MockHostHelpersInterface) Chroot(arg0 string) (func() error, error) { } // Chroot indicates an expected call of Chroot. -func (mr *MockHostHelpersInterfaceMockRecorder) Chroot(arg0 interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) Chroot(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Chroot", reflect.TypeOf((*MockHostHelpersInterface)(nil).Chroot), arg0) } +// CleanSriovFilesFromHost mocks base method. +func (m *MockHostHelpersInterface) CleanSriovFilesFromHost(isOpenShift bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CleanSriovFilesFromHost", isOpenShift) + ret0, _ := ret[0].(error) + return ret0 +} + +// CleanSriovFilesFromHost indicates an expected call of CleanSriovFilesFromHost. +func (mr *MockHostHelpersInterfaceMockRecorder) CleanSriovFilesFromHost(isOpenShift any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanSriovFilesFromHost", reflect.TypeOf((*MockHostHelpersInterface)(nil).CleanSriovFilesFromHost), isOpenShift) +} + // ClearPCIAddressFolder mocks base method. func (m *MockHostHelpersInterface) ClearPCIAddressFolder() error { m.ctrl.T.Helper() @@ -176,7 +197,7 @@ func (m *MockHostHelpersInterface) CompareServices(serviceA, serviceB *types.Ser } // CompareServices indicates an expected call of CompareServices. -func (mr *MockHostHelpersInterfaceMockRecorder) CompareServices(serviceA, serviceB interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) CompareServices(serviceA, serviceB any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompareServices", reflect.TypeOf((*MockHostHelpersInterface)(nil).CompareServices), serviceA, serviceB) } @@ -190,7 +211,7 @@ func (m *MockHostHelpersInterface) ConfigSriovDeviceVirtual(iface *v1.Interface) } // ConfigSriovDeviceVirtual indicates an expected call of ConfigSriovDeviceVirtual. -func (mr *MockHostHelpersInterfaceMockRecorder) ConfigSriovDeviceVirtual(iface interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) ConfigSriovDeviceVirtual(iface any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigSriovDeviceVirtual", reflect.TypeOf((*MockHostHelpersInterface)(nil).ConfigSriovDeviceVirtual), iface) } @@ -204,7 +225,7 @@ func (m *MockHostHelpersInterface) ConfigSriovInterfaces(storeManager store.Mana } // ConfigSriovInterfaces indicates an expected call of ConfigSriovInterfaces. -func (mr *MockHostHelpersInterfaceMockRecorder) ConfigSriovInterfaces(storeManager, interfaces, ifaceStatuses, skipVFConfiguration interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) ConfigSriovInterfaces(storeManager, interfaces, ifaceStatuses, skipVFConfiguration any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigSriovInterfaces", reflect.TypeOf((*MockHostHelpersInterface)(nil).ConfigSriovInterfaces), storeManager, interfaces, ifaceStatuses, skipVFConfiguration) } @@ -218,7 +239,7 @@ func (m *MockHostHelpersInterface) ConfigureBridges(bridgesSpec, bridgesStatus v } // ConfigureBridges indicates an expected call of ConfigureBridges. -func (mr *MockHostHelpersInterfaceMockRecorder) ConfigureBridges(bridgesSpec, bridgesStatus interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) ConfigureBridges(bridgesSpec, bridgesStatus any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigureBridges", reflect.TypeOf((*MockHostHelpersInterface)(nil).ConfigureBridges), bridgesSpec, bridgesStatus) } @@ -232,7 +253,7 @@ func (m *MockHostHelpersInterface) ConfigureVfGUID(vfAddr, pfAddr string, vfID i } // ConfigureVfGUID indicates an expected call of ConfigureVfGUID. -func (mr *MockHostHelpersInterfaceMockRecorder) ConfigureVfGUID(vfAddr, pfAddr, vfID, pfLink interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) ConfigureVfGUID(vfAddr, pfAddr, vfID, pfLink any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigureVfGUID", reflect.TypeOf((*MockHostHelpersInterface)(nil).ConfigureVfGUID), vfAddr, pfAddr, vfID, pfLink) } @@ -246,7 +267,7 @@ func (m *MockHostHelpersInterface) CreateVDPADevice(pciAddr, vdpaType string) er } // CreateVDPADevice indicates an expected call of CreateVDPADevice. -func (mr *MockHostHelpersInterfaceMockRecorder) CreateVDPADevice(pciAddr, vdpaType interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) CreateVDPADevice(pciAddr, vdpaType any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateVDPADevice", reflect.TypeOf((*MockHostHelpersInterface)(nil).CreateVDPADevice), pciAddr, vdpaType) } @@ -260,7 +281,7 @@ func (m *MockHostHelpersInterface) DeleteVDPADevice(pciAddr string) error { } // DeleteVDPADevice indicates an expected call of DeleteVDPADevice. -func (mr *MockHostHelpersInterfaceMockRecorder) DeleteVDPADevice(pciAddr interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) DeleteVDPADevice(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteVDPADevice", reflect.TypeOf((*MockHostHelpersInterface)(nil).DeleteVDPADevice), pciAddr) } @@ -274,7 +295,7 @@ func (m *MockHostHelpersInterface) DetachInterfaceFromManagedBridge(pciAddr stri } // DetachInterfaceFromManagedBridge indicates an expected call of DetachInterfaceFromManagedBridge. -func (mr *MockHostHelpersInterfaceMockRecorder) DetachInterfaceFromManagedBridge(pciAddr interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) DetachInterfaceFromManagedBridge(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DetachInterfaceFromManagedBridge", reflect.TypeOf((*MockHostHelpersInterface)(nil).DetachInterfaceFromManagedBridge), pciAddr) } @@ -294,6 +315,21 @@ func (mr *MockHostHelpersInterfaceMockRecorder) DiscoverBridges() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiscoverBridges", reflect.TypeOf((*MockHostHelpersInterface)(nil).DiscoverBridges)) } +// DiscoverRDMASubsystem mocks base method. +func (m *MockHostHelpersInterface) DiscoverRDMASubsystem() (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DiscoverRDMASubsystem") + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DiscoverRDMASubsystem indicates an expected call of DiscoverRDMASubsystem. +func (mr *MockHostHelpersInterfaceMockRecorder) DiscoverRDMASubsystem() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiscoverRDMASubsystem", reflect.TypeOf((*MockHostHelpersInterface)(nil).DiscoverRDMASubsystem)) +} + // DiscoverSriovDevices mocks base method. func (m *MockHostHelpersInterface) DiscoverSriovDevices(storeManager store.ManagerInterface) ([]v1.InterfaceExt, error) { m.ctrl.T.Helper() @@ -304,7 +340,7 @@ func (m *MockHostHelpersInterface) DiscoverSriovDevices(storeManager store.Manag } // DiscoverSriovDevices indicates an expected call of DiscoverSriovDevices. -func (mr *MockHostHelpersInterfaceMockRecorder) DiscoverSriovDevices(storeManager interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) DiscoverSriovDevices(storeManager any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiscoverSriovDevices", reflect.TypeOf((*MockHostHelpersInterface)(nil).DiscoverSriovDevices), storeManager) } @@ -318,7 +354,7 @@ func (m *MockHostHelpersInterface) DiscoverVDPAType(pciAddr string) string { } // DiscoverVDPAType indicates an expected call of DiscoverVDPAType. -func (mr *MockHostHelpersInterfaceMockRecorder) DiscoverVDPAType(pciAddr interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) DiscoverVDPAType(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiscoverVDPAType", reflect.TypeOf((*MockHostHelpersInterface)(nil).DiscoverVDPAType), pciAddr) } @@ -332,7 +368,7 @@ func (m *MockHostHelpersInterface) EnableHwTcOffload(ifaceName string) error { } // EnableHwTcOffload indicates an expected call of EnableHwTcOffload. -func (mr *MockHostHelpersInterfaceMockRecorder) EnableHwTcOffload(ifaceName interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) EnableHwTcOffload(ifaceName any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnableHwTcOffload", reflect.TypeOf((*MockHostHelpersInterface)(nil).EnableHwTcOffload), ifaceName) } @@ -346,11 +382,26 @@ func (m *MockHostHelpersInterface) EnableService(service *types.Service) error { } // EnableService indicates an expected call of EnableService. -func (mr *MockHostHelpersInterfaceMockRecorder) EnableService(service interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) EnableService(service any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnableService", reflect.TypeOf((*MockHostHelpersInterface)(nil).EnableService), service) } +// GetCPUVendor mocks base method. +func (m *MockHostHelpersInterface) GetCPUVendor() (types.CPUVendor, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCPUVendor") + ret0, _ := ret[0].(types.CPUVendor) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCPUVendor indicates an expected call of GetCPUVendor. +func (mr *MockHostHelpersInterfaceMockRecorder) GetCPUVendor() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCPUVendor", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetCPUVendor)) +} + // GetCheckPointNodeState mocks base method. func (m *MockHostHelpersInterface) GetCheckPointNodeState() (*v1.SriovNetworkNodeState, error) { m.ctrl.T.Helper() @@ -391,7 +442,7 @@ func (m *MockHostHelpersInterface) GetDevlinkDeviceParam(pciAddr, paramName stri } // GetDevlinkDeviceParam indicates an expected call of GetDevlinkDeviceParam. -func (mr *MockHostHelpersInterfaceMockRecorder) GetDevlinkDeviceParam(pciAddr, paramName interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) GetDevlinkDeviceParam(pciAddr, paramName any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDevlinkDeviceParam", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetDevlinkDeviceParam), pciAddr, paramName) } @@ -406,7 +457,7 @@ func (m *MockHostHelpersInterface) GetDriverByBusAndDevice(bus, device string) ( } // GetDriverByBusAndDevice indicates an expected call of GetDriverByBusAndDevice. -func (mr *MockHostHelpersInterfaceMockRecorder) GetDriverByBusAndDevice(bus, device interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) GetDriverByBusAndDevice(bus, device any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDriverByBusAndDevice", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetDriverByBusAndDevice), bus, device) } @@ -421,7 +472,7 @@ func (m *MockHostHelpersInterface) GetInterfaceIndex(pciAddr string) (int, error } // GetInterfaceIndex indicates an expected call of GetInterfaceIndex. -func (mr *MockHostHelpersInterfaceMockRecorder) GetInterfaceIndex(pciAddr interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) GetInterfaceIndex(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInterfaceIndex", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetInterfaceIndex), pciAddr) } @@ -435,7 +486,7 @@ func (m *MockHostHelpersInterface) GetLinkType(name string) string { } // GetLinkType indicates an expected call of GetLinkType. -func (mr *MockHostHelpersInterfaceMockRecorder) GetLinkType(name interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) GetLinkType(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLinkType", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetLinkType), name) } @@ -450,7 +501,7 @@ func (m *MockHostHelpersInterface) GetMellanoxBlueFieldMode(arg0 string) (mlxuti } // GetMellanoxBlueFieldMode indicates an expected call of GetMellanoxBlueFieldMode. -func (mr *MockHostHelpersInterfaceMockRecorder) GetMellanoxBlueFieldMode(arg0 interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) GetMellanoxBlueFieldMode(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMellanoxBlueFieldMode", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetMellanoxBlueFieldMode), arg0) } @@ -466,7 +517,7 @@ func (m *MockHostHelpersInterface) GetMlxNicFwData(pciAddress string) (*mlxutils } // GetMlxNicFwData indicates an expected call of GetMlxNicFwData. -func (mr *MockHostHelpersInterfaceMockRecorder) GetMlxNicFwData(pciAddress interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) GetMlxNicFwData(pciAddress any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMlxNicFwData", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetMlxNicFwData), pciAddress) } @@ -480,7 +531,7 @@ func (m *MockHostHelpersInterface) GetNetDevLinkAdminState(ifaceName string) str } // GetNetDevLinkAdminState indicates an expected call of GetNetDevLinkAdminState. -func (mr *MockHostHelpersInterfaceMockRecorder) GetNetDevLinkAdminState(ifaceName interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) GetNetDevLinkAdminState(ifaceName any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNetDevLinkAdminState", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetNetDevLinkAdminState), ifaceName) } @@ -494,7 +545,7 @@ func (m *MockHostHelpersInterface) GetNetDevLinkSpeed(name string) string { } // GetNetDevLinkSpeed indicates an expected call of GetNetDevLinkSpeed. -func (mr *MockHostHelpersInterfaceMockRecorder) GetNetDevLinkSpeed(name interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) GetNetDevLinkSpeed(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNetDevLinkSpeed", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetNetDevLinkSpeed), name) } @@ -508,7 +559,7 @@ func (m *MockHostHelpersInterface) GetNetDevMac(name string) string { } // GetNetDevMac indicates an expected call of GetNetDevMac. -func (mr *MockHostHelpersInterfaceMockRecorder) GetNetDevMac(name interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) GetNetDevMac(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNetDevMac", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetNetDevMac), name) } @@ -522,7 +573,7 @@ func (m *MockHostHelpersInterface) GetNetDevNodeGUID(pciAddr string) string { } // GetNetDevNodeGUID indicates an expected call of GetNetDevNodeGUID. -func (mr *MockHostHelpersInterfaceMockRecorder) GetNetDevNodeGUID(pciAddr interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) GetNetDevNodeGUID(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNetDevNodeGUID", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetNetDevNodeGUID), pciAddr) } @@ -536,7 +587,7 @@ func (m *MockHostHelpersInterface) GetNetdevMTU(pciAddr string) int { } // GetNetdevMTU indicates an expected call of GetNetdevMTU. -func (mr *MockHostHelpersInterfaceMockRecorder) GetNetdevMTU(pciAddr interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) GetNetdevMTU(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNetdevMTU", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetNetdevMTU), pciAddr) } @@ -550,7 +601,7 @@ func (m *MockHostHelpersInterface) GetNicSriovMode(pciAddr string) string { } // GetNicSriovMode indicates an expected call of GetNicSriovMode. -func (mr *MockHostHelpersInterfaceMockRecorder) GetNicSriovMode(pciAddr interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) GetNicSriovMode(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNicSriovMode", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetNicSriovMode), pciAddr) } @@ -565,7 +616,7 @@ func (m *MockHostHelpersInterface) GetPciAddressFromInterfaceName(interfaceName } // GetPciAddressFromInterfaceName indicates an expected call of GetPciAddressFromInterfaceName. -func (mr *MockHostHelpersInterfaceMockRecorder) GetPciAddressFromInterfaceName(interfaceName interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) GetPciAddressFromInterfaceName(interfaceName any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPciAddressFromInterfaceName", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetPciAddressFromInterfaceName), interfaceName) } @@ -580,7 +631,7 @@ func (m *MockHostHelpersInterface) GetPhysPortName(name string) (string, error) } // GetPhysPortName indicates an expected call of GetPhysPortName. -func (mr *MockHostHelpersInterfaceMockRecorder) GetPhysPortName(name interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) GetPhysPortName(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPhysPortName", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetPhysPortName), name) } @@ -595,7 +646,7 @@ func (m *MockHostHelpersInterface) GetPhysSwitchID(name string) (string, error) } // GetPhysSwitchID indicates an expected call of GetPhysSwitchID. -func (mr *MockHostHelpersInterfaceMockRecorder) GetPhysSwitchID(name interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) GetPhysSwitchID(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPhysSwitchID", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetPhysSwitchID), name) } @@ -610,7 +661,7 @@ func (m *MockHostHelpersInterface) HasDriver(pciAddr string) (bool, string) { } // HasDriver indicates an expected call of HasDriver. -func (mr *MockHostHelpersInterfaceMockRecorder) HasDriver(pciAddr interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) HasDriver(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasDriver", reflect.TypeOf((*MockHostHelpersInterface)(nil).HasDriver), pciAddr) } @@ -624,7 +675,7 @@ func (m *MockHostHelpersInterface) IsKernelArgsSet(cmdLine, karg string) bool { } // IsKernelArgsSet indicates an expected call of IsKernelArgsSet. -func (mr *MockHostHelpersInterfaceMockRecorder) IsKernelArgsSet(cmdLine, karg interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) IsKernelArgsSet(cmdLine, karg any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsKernelArgsSet", reflect.TypeOf((*MockHostHelpersInterface)(nil).IsKernelArgsSet), cmdLine, karg) } @@ -653,7 +704,7 @@ func (m *MockHostHelpersInterface) IsKernelModuleLoaded(name string) (bool, erro } // IsKernelModuleLoaded indicates an expected call of IsKernelModuleLoaded. -func (mr *MockHostHelpersInterfaceMockRecorder) IsKernelModuleLoaded(name interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) IsKernelModuleLoaded(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsKernelModuleLoaded", reflect.TypeOf((*MockHostHelpersInterface)(nil).IsKernelModuleLoaded), name) } @@ -668,7 +719,7 @@ func (m *MockHostHelpersInterface) IsServiceEnabled(servicePath string) (bool, e } // IsServiceEnabled indicates an expected call of IsServiceEnabled. -func (mr *MockHostHelpersInterfaceMockRecorder) IsServiceEnabled(servicePath interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) IsServiceEnabled(servicePath any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsServiceEnabled", reflect.TypeOf((*MockHostHelpersInterface)(nil).IsServiceEnabled), servicePath) } @@ -683,7 +734,7 @@ func (m *MockHostHelpersInterface) IsServiceExist(servicePath string) (bool, err } // IsServiceExist indicates an expected call of IsServiceExist. -func (mr *MockHostHelpersInterfaceMockRecorder) IsServiceExist(servicePath interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) IsServiceExist(servicePath any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsServiceExist", reflect.TypeOf((*MockHostHelpersInterface)(nil).IsServiceExist), servicePath) } @@ -697,7 +748,7 @@ func (m *MockHostHelpersInterface) IsSwitchdev(name string) bool { } // IsSwitchdev indicates an expected call of IsSwitchdev. -func (mr *MockHostHelpersInterfaceMockRecorder) IsSwitchdev(name interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) IsSwitchdev(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSwitchdev", reflect.TypeOf((*MockHostHelpersInterface)(nil).IsSwitchdev), name) } @@ -705,7 +756,7 @@ func (mr *MockHostHelpersInterfaceMockRecorder) IsSwitchdev(name interface{}) *g // LoadKernelModule mocks base method. func (m *MockHostHelpersInterface) LoadKernelModule(name string, args ...string) error { m.ctrl.T.Helper() - varargs := []interface{}{name} + varargs := []any{name} for _, a := range args { varargs = append(varargs, a) } @@ -715,9 +766,9 @@ func (m *MockHostHelpersInterface) LoadKernelModule(name string, args ...string) } // LoadKernelModule indicates an expected call of LoadKernelModule. -func (mr *MockHostHelpersInterfaceMockRecorder) LoadKernelModule(name interface{}, args ...interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) LoadKernelModule(name any, args ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{name}, args...) + varargs := append([]any{name}, args...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadKernelModule", reflect.TypeOf((*MockHostHelpersInterface)(nil).LoadKernelModule), varargs...) } @@ -732,7 +783,7 @@ func (m *MockHostHelpersInterface) LoadPfsStatus(pciAddress string) (*v1.Interfa } // LoadPfsStatus indicates an expected call of LoadPfsStatus. -func (mr *MockHostHelpersInterfaceMockRecorder) LoadPfsStatus(pciAddress interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) LoadPfsStatus(pciAddress any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadPfsStatus", reflect.TypeOf((*MockHostHelpersInterface)(nil).LoadPfsStatus), pciAddress) } @@ -760,23 +811,23 @@ func (m *MockHostHelpersInterface) MlxConfigFW(attributesToChange map[string]mlx } // MlxConfigFW indicates an expected call of MlxConfigFW. -func (mr *MockHostHelpersInterfaceMockRecorder) MlxConfigFW(attributesToChange interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) MlxConfigFW(attributesToChange any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MlxConfigFW", reflect.TypeOf((*MockHostHelpersInterface)(nil).MlxConfigFW), attributesToChange) } // MlxResetFW mocks base method. -func (m *MockHostHelpersInterface) MlxResetFW(pciAddresses []string) error { +func (m *MockHostHelpersInterface) MlxResetFW(pciAddresses []string, mellanoxNicsStatus map[string]map[string]v1.InterfaceExt) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MlxResetFW", pciAddresses) + ret := m.ctrl.Call(m, "MlxResetFW", pciAddresses, mellanoxNicsStatus) ret0, _ := ret[0].(error) return ret0 } // MlxResetFW indicates an expected call of MlxResetFW. -func (mr *MockHostHelpersInterfaceMockRecorder) MlxResetFW(pciAddresses interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) MlxResetFW(pciAddresses, mellanoxNicsStatus any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MlxResetFW", reflect.TypeOf((*MockHostHelpersInterface)(nil).MlxResetFW), pciAddresses) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MlxResetFW", reflect.TypeOf((*MockHostHelpersInterface)(nil).MlxResetFW), pciAddresses, mellanoxNicsStatus) } // MstConfigReadData mocks base method. @@ -790,7 +841,7 @@ func (m *MockHostHelpersInterface) MstConfigReadData(arg0 string) (string, strin } // MstConfigReadData indicates an expected call of MstConfigReadData. -func (mr *MockHostHelpersInterfaceMockRecorder) MstConfigReadData(arg0 interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) MstConfigReadData(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MstConfigReadData", reflect.TypeOf((*MockHostHelpersInterface)(nil).MstConfigReadData), arg0) } @@ -804,7 +855,7 @@ func (m *MockHostHelpersInterface) PrepareNMUdevRule(supportedVfIds []string) er } // PrepareNMUdevRule indicates an expected call of PrepareNMUdevRule. -func (mr *MockHostHelpersInterfaceMockRecorder) PrepareNMUdevRule(supportedVfIds interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) PrepareNMUdevRule(supportedVfIds any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PrepareNMUdevRule", reflect.TypeOf((*MockHostHelpersInterface)(nil).PrepareNMUdevRule), supportedVfIds) } @@ -823,6 +874,21 @@ func (mr *MockHostHelpersInterfaceMockRecorder) PrepareVFRepUdevRule() *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PrepareVFRepUdevRule", reflect.TypeOf((*MockHostHelpersInterface)(nil).PrepareVFRepUdevRule)) } +// ReadConfFile mocks base method. +func (m *MockHostHelpersInterface) ReadConfFile() (*types.SriovConfig, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadConfFile") + ret0, _ := ret[0].(*types.SriovConfig) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReadConfFile indicates an expected call of ReadConfFile. +func (mr *MockHostHelpersInterfaceMockRecorder) ReadConfFile() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfFile", reflect.TypeOf((*MockHostHelpersInterface)(nil).ReadConfFile)) +} + // ReadService mocks base method. func (m *MockHostHelpersInterface) ReadService(servicePath string) (*types.Service, error) { m.ctrl.T.Helper() @@ -833,7 +899,7 @@ func (m *MockHostHelpersInterface) ReadService(servicePath string) (*types.Servi } // ReadService indicates an expected call of ReadService. -func (mr *MockHostHelpersInterfaceMockRecorder) ReadService(servicePath interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) ReadService(servicePath any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadService", reflect.TypeOf((*MockHostHelpersInterface)(nil).ReadService), servicePath) } @@ -848,7 +914,7 @@ func (m *MockHostHelpersInterface) ReadServiceInjectionManifestFile(path string) } // ReadServiceInjectionManifestFile indicates an expected call of ReadServiceInjectionManifestFile. -func (mr *MockHostHelpersInterfaceMockRecorder) ReadServiceInjectionManifestFile(path interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) ReadServiceInjectionManifestFile(path any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadServiceInjectionManifestFile", reflect.TypeOf((*MockHostHelpersInterface)(nil).ReadServiceInjectionManifestFile), path) } @@ -863,11 +929,41 @@ func (m *MockHostHelpersInterface) ReadServiceManifestFile(path string) (*types. } // ReadServiceManifestFile indicates an expected call of ReadServiceManifestFile. -func (mr *MockHostHelpersInterfaceMockRecorder) ReadServiceManifestFile(path interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) ReadServiceManifestFile(path any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadServiceManifestFile", reflect.TypeOf((*MockHostHelpersInterface)(nil).ReadServiceManifestFile), path) } +// ReadSriovResult mocks base method. +func (m *MockHostHelpersInterface) ReadSriovResult() (*types.SriovResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadSriovResult") + ret0, _ := ret[0].(*types.SriovResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReadSriovResult indicates an expected call of ReadSriovResult. +func (mr *MockHostHelpersInterfaceMockRecorder) ReadSriovResult() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadSriovResult", reflect.TypeOf((*MockHostHelpersInterface)(nil).ReadSriovResult)) +} + +// ReadSriovSupportedNics mocks base method. +func (m *MockHostHelpersInterface) ReadSriovSupportedNics() ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadSriovSupportedNics") + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReadSriovSupportedNics indicates an expected call of ReadSriovSupportedNics. +func (mr *MockHostHelpersInterfaceMockRecorder) ReadSriovSupportedNics() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadSriovSupportedNics", reflect.TypeOf((*MockHostHelpersInterface)(nil).ReadSriovSupportedNics)) +} + // RebindVfToDefaultDriver mocks base method. func (m *MockHostHelpersInterface) RebindVfToDefaultDriver(pciAddr string) error { m.ctrl.T.Helper() @@ -877,7 +973,7 @@ func (m *MockHostHelpersInterface) RebindVfToDefaultDriver(pciAddr string) error } // RebindVfToDefaultDriver indicates an expected call of RebindVfToDefaultDriver. -func (mr *MockHostHelpersInterfaceMockRecorder) RebindVfToDefaultDriver(pciAddr interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) RebindVfToDefaultDriver(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RebindVfToDefaultDriver", reflect.TypeOf((*MockHostHelpersInterface)(nil).RebindVfToDefaultDriver), pciAddr) } @@ -891,7 +987,7 @@ func (m *MockHostHelpersInterface) RemoveDisableNMUdevRule(pfPciAddress string) } // RemoveDisableNMUdevRule indicates an expected call of RemoveDisableNMUdevRule. -func (mr *MockHostHelpersInterfaceMockRecorder) RemoveDisableNMUdevRule(pfPciAddress interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) RemoveDisableNMUdevRule(pfPciAddress any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveDisableNMUdevRule", reflect.TypeOf((*MockHostHelpersInterface)(nil).RemoveDisableNMUdevRule), pfPciAddress) } @@ -905,7 +1001,7 @@ func (m *MockHostHelpersInterface) RemovePersistPFNameUdevRule(pfPciAddress stri } // RemovePersistPFNameUdevRule indicates an expected call of RemovePersistPFNameUdevRule. -func (mr *MockHostHelpersInterfaceMockRecorder) RemovePersistPFNameUdevRule(pfPciAddress interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) RemovePersistPFNameUdevRule(pfPciAddress any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemovePersistPFNameUdevRule", reflect.TypeOf((*MockHostHelpersInterface)(nil).RemovePersistPFNameUdevRule), pfPciAddress) } @@ -919,11 +1015,25 @@ func (m *MockHostHelpersInterface) RemovePfAppliedStatus(pciAddress string) erro } // RemovePfAppliedStatus indicates an expected call of RemovePfAppliedStatus. -func (mr *MockHostHelpersInterfaceMockRecorder) RemovePfAppliedStatus(pciAddress interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) RemovePfAppliedStatus(pciAddress any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemovePfAppliedStatus", reflect.TypeOf((*MockHostHelpersInterface)(nil).RemovePfAppliedStatus), pciAddress) } +// RemoveSriovResult mocks base method. +func (m *MockHostHelpersInterface) RemoveSriovResult() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveSriovResult") + ret0, _ := ret[0].(error) + return ret0 +} + +// RemoveSriovResult indicates an expected call of RemoveSriovResult. +func (mr *MockHostHelpersInterfaceMockRecorder) RemoveSriovResult() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveSriovResult", reflect.TypeOf((*MockHostHelpersInterface)(nil).RemoveSriovResult)) +} + // RemoveVfRepresentorUdevRule mocks base method. func (m *MockHostHelpersInterface) RemoveVfRepresentorUdevRule(pfPciAddress string) error { m.ctrl.T.Helper() @@ -933,7 +1043,7 @@ func (m *MockHostHelpersInterface) RemoveVfRepresentorUdevRule(pfPciAddress stri } // RemoveVfRepresentorUdevRule indicates an expected call of RemoveVfRepresentorUdevRule. -func (mr *MockHostHelpersInterfaceMockRecorder) RemoveVfRepresentorUdevRule(pfPciAddress interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) RemoveVfRepresentorUdevRule(pfPciAddress any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveVfRepresentorUdevRule", reflect.TypeOf((*MockHostHelpersInterface)(nil).RemoveVfRepresentorUdevRule), pfPciAddress) } @@ -947,7 +1057,7 @@ func (m *MockHostHelpersInterface) ResetSriovDevice(ifaceStatus v1.InterfaceExt) } // ResetSriovDevice indicates an expected call of ResetSriovDevice. -func (mr *MockHostHelpersInterfaceMockRecorder) ResetSriovDevice(ifaceStatus interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) ResetSriovDevice(ifaceStatus any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetSriovDevice", reflect.TypeOf((*MockHostHelpersInterface)(nil).ResetSriovDevice), ifaceStatus) } @@ -955,7 +1065,7 @@ func (mr *MockHostHelpersInterfaceMockRecorder) ResetSriovDevice(ifaceStatus int // RunCommand mocks base method. func (m *MockHostHelpersInterface) RunCommand(arg0 string, arg1 ...string) (string, string, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0} + varargs := []any{arg0} for _, a := range arg1 { varargs = append(varargs, a) } @@ -967,9 +1077,9 @@ func (m *MockHostHelpersInterface) RunCommand(arg0 string, arg1 ...string) (stri } // RunCommand indicates an expected call of RunCommand. -func (mr *MockHostHelpersInterfaceMockRecorder) RunCommand(arg0 interface{}, arg1 ...interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) RunCommand(arg0 any, arg1 ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0}, arg1...) + varargs := append([]any{arg0}, arg1...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RunCommand", reflect.TypeOf((*MockHostHelpersInterface)(nil).RunCommand), varargs...) } @@ -982,7 +1092,7 @@ func (m *MockHostHelpersInterface) SaveLastPfAppliedStatus(PfInfo *v1.Interface) } // SaveLastPfAppliedStatus indicates an expected call of SaveLastPfAppliedStatus. -func (mr *MockHostHelpersInterfaceMockRecorder) SaveLastPfAppliedStatus(PfInfo interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) SaveLastPfAppliedStatus(PfInfo any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveLastPfAppliedStatus", reflect.TypeOf((*MockHostHelpersInterface)(nil).SaveLastPfAppliedStatus), PfInfo) } @@ -996,7 +1106,7 @@ func (m *MockHostHelpersInterface) SetDevlinkDeviceParam(pciAddr, paramName, val } // SetDevlinkDeviceParam indicates an expected call of SetDevlinkDeviceParam. -func (mr *MockHostHelpersInterfaceMockRecorder) SetDevlinkDeviceParam(pciAddr, paramName, value interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) SetDevlinkDeviceParam(pciAddr, paramName, value any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDevlinkDeviceParam", reflect.TypeOf((*MockHostHelpersInterface)(nil).SetDevlinkDeviceParam), pciAddr, paramName, value) } @@ -1010,7 +1120,7 @@ func (m *MockHostHelpersInterface) SetNetdevMTU(pciAddr string, mtu int) error { } // SetNetdevMTU indicates an expected call of SetNetdevMTU. -func (mr *MockHostHelpersInterfaceMockRecorder) SetNetdevMTU(pciAddr, mtu interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) SetNetdevMTU(pciAddr, mtu any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetNetdevMTU", reflect.TypeOf((*MockHostHelpersInterface)(nil).SetNetdevMTU), pciAddr, mtu) } @@ -1024,11 +1134,25 @@ func (m *MockHostHelpersInterface) SetNicSriovMode(pciAddr, mode string) error { } // SetNicSriovMode indicates an expected call of SetNicSriovMode. -func (mr *MockHostHelpersInterfaceMockRecorder) SetNicSriovMode(pciAddr, mode interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) SetNicSriovMode(pciAddr, mode any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetNicSriovMode", reflect.TypeOf((*MockHostHelpersInterface)(nil).SetNicSriovMode), pciAddr, mode) } +// SetRDMASubsystem mocks base method. +func (m *MockHostHelpersInterface) SetRDMASubsystem(mode string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetRDMASubsystem", mode) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetRDMASubsystem indicates an expected call of SetRDMASubsystem. +func (mr *MockHostHelpersInterfaceMockRecorder) SetRDMASubsystem(mode any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetRDMASubsystem", reflect.TypeOf((*MockHostHelpersInterface)(nil).SetRDMASubsystem), mode) +} + // SetSriovNumVfs mocks base method. func (m *MockHostHelpersInterface) SetSriovNumVfs(pciAddr string, numVfs int) error { m.ctrl.T.Helper() @@ -1038,7 +1162,7 @@ func (m *MockHostHelpersInterface) SetSriovNumVfs(pciAddr string, numVfs int) er } // SetSriovNumVfs indicates an expected call of SetSriovNumVfs. -func (mr *MockHostHelpersInterfaceMockRecorder) SetSriovNumVfs(pciAddr, numVfs interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) SetSriovNumVfs(pciAddr, numVfs any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetSriovNumVfs", reflect.TypeOf((*MockHostHelpersInterface)(nil).SetSriovNumVfs), pciAddr, numVfs) } @@ -1052,7 +1176,7 @@ func (m *MockHostHelpersInterface) SetVfAdminMac(vfAddr string, pfLink, vfLink n } // SetVfAdminMac indicates an expected call of SetVfAdminMac. -func (mr *MockHostHelpersInterfaceMockRecorder) SetVfAdminMac(vfAddr, pfLink, vfLink interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) SetVfAdminMac(vfAddr, pfLink, vfLink any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetVfAdminMac", reflect.TypeOf((*MockHostHelpersInterface)(nil).SetVfAdminMac), vfAddr, pfLink, vfLink) } @@ -1090,7 +1214,7 @@ func (m *MockHostHelpersInterface) TryGetInterfaceName(pciAddr string) string { } // TryGetInterfaceName indicates an expected call of TryGetInterfaceName. -func (mr *MockHostHelpersInterfaceMockRecorder) TryGetInterfaceName(pciAddr interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) TryGetInterfaceName(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TryGetInterfaceName", reflect.TypeOf((*MockHostHelpersInterface)(nil).TryGetInterfaceName), pciAddr) } @@ -1104,7 +1228,7 @@ func (m *MockHostHelpersInterface) TryToGetVirtualInterfaceName(pciAddr string) } // TryToGetVirtualInterfaceName indicates an expected call of TryToGetVirtualInterfaceName. -func (mr *MockHostHelpersInterfaceMockRecorder) TryToGetVirtualInterfaceName(pciAddr interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) TryToGetVirtualInterfaceName(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TryToGetVirtualInterfaceName", reflect.TypeOf((*MockHostHelpersInterface)(nil).TryToGetVirtualInterfaceName), pciAddr) } @@ -1118,7 +1242,7 @@ func (m *MockHostHelpersInterface) Unbind(pciAddr string) error { } // Unbind indicates an expected call of Unbind. -func (mr *MockHostHelpersInterfaceMockRecorder) Unbind(pciAddr interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) Unbind(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Unbind", reflect.TypeOf((*MockHostHelpersInterface)(nil).Unbind), pciAddr) } @@ -1132,7 +1256,7 @@ func (m *MockHostHelpersInterface) UnbindDriverByBusAndDevice(bus, device string } // UnbindDriverByBusAndDevice indicates an expected call of UnbindDriverByBusAndDevice. -func (mr *MockHostHelpersInterfaceMockRecorder) UnbindDriverByBusAndDevice(bus, device interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) UnbindDriverByBusAndDevice(bus, device any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnbindDriverByBusAndDevice", reflect.TypeOf((*MockHostHelpersInterface)(nil).UnbindDriverByBusAndDevice), bus, device) } @@ -1146,7 +1270,7 @@ func (m *MockHostHelpersInterface) UnbindDriverIfNeeded(pciAddr string, isRdma b } // UnbindDriverIfNeeded indicates an expected call of UnbindDriverIfNeeded. -func (mr *MockHostHelpersInterfaceMockRecorder) UnbindDriverIfNeeded(pciAddr, isRdma interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) UnbindDriverIfNeeded(pciAddr, isRdma any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnbindDriverIfNeeded", reflect.TypeOf((*MockHostHelpersInterface)(nil).UnbindDriverIfNeeded), pciAddr, isRdma) } @@ -1160,7 +1284,7 @@ func (m *MockHostHelpersInterface) UpdateSystemService(serviceObj *types.Service } // UpdateSystemService indicates an expected call of UpdateSystemService. -func (mr *MockHostHelpersInterfaceMockRecorder) UpdateSystemService(serviceObj interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) UpdateSystemService(serviceObj any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateSystemService", reflect.TypeOf((*MockHostHelpersInterface)(nil).UpdateSystemService), serviceObj) } @@ -1175,11 +1299,25 @@ func (m *MockHostHelpersInterface) VFIsReady(pciAddr string) (netlink.Link, erro } // VFIsReady indicates an expected call of VFIsReady. -func (mr *MockHostHelpersInterfaceMockRecorder) VFIsReady(pciAddr interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) VFIsReady(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VFIsReady", reflect.TypeOf((*MockHostHelpersInterface)(nil).VFIsReady), pciAddr) } +// WaitUdevEventsProcessed mocks base method. +func (m *MockHostHelpersInterface) WaitUdevEventsProcessed(timeout int) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WaitUdevEventsProcessed", timeout) + ret0, _ := ret[0].(error) + return ret0 +} + +// WaitUdevEventsProcessed indicates an expected call of WaitUdevEventsProcessed. +func (mr *MockHostHelpersInterfaceMockRecorder) WaitUdevEventsProcessed(timeout any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitUdevEventsProcessed", reflect.TypeOf((*MockHostHelpersInterface)(nil).WaitUdevEventsProcessed), timeout) +} + // WriteCheckpointFile mocks base method. func (m *MockHostHelpersInterface) WriteCheckpointFile(arg0 *v1.SriovNetworkNodeState) error { m.ctrl.T.Helper() @@ -1189,7 +1327,65 @@ func (m *MockHostHelpersInterface) WriteCheckpointFile(arg0 *v1.SriovNetworkNode } // WriteCheckpointFile indicates an expected call of WriteCheckpointFile. -func (mr *MockHostHelpersInterfaceMockRecorder) WriteCheckpointFile(arg0 interface{}) *gomock.Call { +func (mr *MockHostHelpersInterfaceMockRecorder) WriteCheckpointFile(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteCheckpointFile", reflect.TypeOf((*MockHostHelpersInterface)(nil).WriteCheckpointFile), arg0) } + +// WriteConfFile mocks base method. +func (m *MockHostHelpersInterface) WriteConfFile(newState *v1.SriovNetworkNodeState) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WriteConfFile", newState) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetVfGUID mocks base method +func (m *MockHostHelpersInterface) GetVfGUID(vfAddr string, pfAddr string, vfID int) (net.HardwareAddr, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVfGUID", vfAddr, pfAddr, vfID) + ret0, _ := ret[0].(net.HardwareAddr) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// WriteConfFile indicates an expected call of WriteConfFile. +func (mr *MockHostHelpersInterfaceMockRecorder) WriteConfFile(newState any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteConfFile", reflect.TypeOf((*MockHostHelpersInterface)(nil).WriteConfFile), newState) +} + +// WriteSriovResult mocks base method. +func (m *MockHostHelpersInterface) WriteSriovResult(result *types.SriovResult) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WriteSriovResult", result) + ret0, _ := ret[0].(error) + return ret0 +} + +// WriteSriovResult indicates an expected call of WriteSriovResult. +func (mr *MockHostHelpersInterfaceMockRecorder) WriteSriovResult(result any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteSriovResult", reflect.TypeOf((*MockHostHelpersInterface)(nil).WriteSriovResult), result) +} + +// WriteSriovSupportedNics mocks base method. +func (m *MockHostHelpersInterface) WriteSriovSupportedNics() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WriteSriovSupportedNics") + ret0, _ := ret[0].(error) + return ret0 +} + +// WriteSriovSupportedNics indicates an expected call of WriteSriovSupportedNics. +func (mr *MockHostHelpersInterfaceMockRecorder) WriteSriovSupportedNics() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteSriovSupportedNics", reflect.TypeOf((*MockHostHelpersInterface)(nil).WriteSriovSupportedNics)) +} + +// GetVfGUID indicates an expected call of GetVfGUID +func (mr *MockHostHelpersInterfaceMockRecorder) GetVfGUID(vfAddr, pfAddr string, vfID int) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVfGUID", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetVfGUID), vfAddr, pfAddr, vfID) +} diff --git a/pkg/host/internal/bridge/bridge_test.go b/pkg/host/internal/bridge/bridge_test.go index 072ca27a5..e253a93dc 100644 --- a/pkg/host/internal/bridge/bridge_test.go +++ b/pkg/host/internal/bridge/bridge_test.go @@ -3,7 +3,7 @@ package bridge import ( "fmt" - "github.com/golang/mock/gomock" + "go.uber.org/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" diff --git a/pkg/host/internal/bridge/ovs/mock/mock_ovs.go b/pkg/host/internal/bridge/ovs/mock/mock_ovs.go index ecd618a0f..28f5420a5 100644 --- a/pkg/host/internal/bridge/ovs/mock/mock_ovs.go +++ b/pkg/host/internal/bridge/ovs/mock/mock_ovs.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: ovs.go +// +// Generated by this command: +// +// mockgen -destination mock/mock_ovs.go -source ovs.go +// // Package mock_ovs is a generated GoMock package. package mock_ovs @@ -8,14 +13,15 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + gomock "go.uber.org/mock/gomock" ) // MockInterface is a mock of Interface interface. type MockInterface struct { ctrl *gomock.Controller recorder *MockInterfaceMockRecorder + isgomock struct{} } // MockInterfaceMockRecorder is the mock recorder for MockInterface. @@ -44,7 +50,7 @@ func (m *MockInterface) CreateOVSBridge(ctx context.Context, conf *v1.OVSConfigE } // CreateOVSBridge indicates an expected call of CreateOVSBridge. -func (mr *MockInterfaceMockRecorder) CreateOVSBridge(ctx, conf interface{}) *gomock.Call { +func (mr *MockInterfaceMockRecorder) CreateOVSBridge(ctx, conf any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOVSBridge", reflect.TypeOf((*MockInterface)(nil).CreateOVSBridge), ctx, conf) } @@ -59,7 +65,7 @@ func (m *MockInterface) GetOVSBridges(ctx context.Context) ([]v1.OVSConfigExt, e } // GetOVSBridges indicates an expected call of GetOVSBridges. -func (mr *MockInterfaceMockRecorder) GetOVSBridges(ctx interface{}) *gomock.Call { +func (mr *MockInterfaceMockRecorder) GetOVSBridges(ctx any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOVSBridges", reflect.TypeOf((*MockInterface)(nil).GetOVSBridges), ctx) } @@ -73,7 +79,7 @@ func (m *MockInterface) RemoveInterfaceFromOVSBridge(ctx context.Context, ifaceA } // RemoveInterfaceFromOVSBridge indicates an expected call of RemoveInterfaceFromOVSBridge. -func (mr *MockInterfaceMockRecorder) RemoveInterfaceFromOVSBridge(ctx, ifaceAddr interface{}) *gomock.Call { +func (mr *MockInterfaceMockRecorder) RemoveInterfaceFromOVSBridge(ctx, ifaceAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveInterfaceFromOVSBridge", reflect.TypeOf((*MockInterface)(nil).RemoveInterfaceFromOVSBridge), ctx, ifaceAddr) } @@ -87,7 +93,7 @@ func (m *MockInterface) RemoveOVSBridge(ctx context.Context, bridgeName string) } // RemoveOVSBridge indicates an expected call of RemoveOVSBridge. -func (mr *MockInterfaceMockRecorder) RemoveOVSBridge(ctx, bridgeName interface{}) *gomock.Call { +func (mr *MockInterfaceMockRecorder) RemoveOVSBridge(ctx, bridgeName any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveOVSBridge", reflect.TypeOf((*MockInterface)(nil).RemoveOVSBridge), ctx, bridgeName) } diff --git a/pkg/host/internal/bridge/ovs/models.go b/pkg/host/internal/bridge/ovs/models.go index 4bd356312..234a4ef32 100644 --- a/pkg/host/internal/bridge/ovs/models.go +++ b/pkg/host/internal/bridge/ovs/models.go @@ -36,6 +36,7 @@ type InterfaceEntry struct { Options map[string]string `ovsdb:"options"` ExternalIDs map[string]string `ovsdb:"external_ids"` OtherConfig map[string]string `ovsdb:"other_config"` + MTURequest *int `ovsdb:"mtu_request"` } // PortEntry represents some fields of the object in the Port table diff --git a/pkg/host/internal/bridge/ovs/ovs.go b/pkg/host/internal/bridge/ovs/ovs.go index 7ad8a3e8c..d2411f907 100644 --- a/pkg/host/internal/bridge/ovs/ovs.go +++ b/pkg/host/internal/bridge/ovs/ovs.go @@ -6,15 +6,16 @@ import ( "errors" "fmt" "os" - "reflect" "sort" "strings" "time" + "github.com/go-logr/logr" "github.com/google/uuid" "github.com/ovn-org/libovsdb/client" "github.com/ovn-org/libovsdb/model" "github.com/ovn-org/libovsdb/ovsdb" + "k8s.io/apimachinery/pkg/api/equality" "sigs.k8s.io/controller-runtime/pkg/log" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" @@ -83,7 +84,7 @@ func (o *ovs) CreateOVSBridge(ctx context.Context, conf *sriovnetworkv1.OVSConfi funcLog.Error(err, "CreateOVSBridge(): failed to read data from store") return fmt.Errorf("failed to read data from store: %v", err) } - if knownConfig == nil || !reflect.DeepEqual(conf, knownConfig) { + if knownConfig == nil || !equality.Semantic.DeepEqual(conf, knownConfig) { funcLog.V(2).Info("CreateOVSBridge(): save current configuration to the store") // config in store manager is not found or it is not the same config as passed with conf arg, // update config in the store manager @@ -102,13 +103,13 @@ func (o *ovs) CreateOVSBridge(ctx context.Context, conf *sriovnetworkv1.OVSConfi return err } if currentState != nil { - if reflect.DeepEqual(conf, currentState) { + if equality.Semantic.DeepEqual(conf, currentState) { // bridge already exist with the right config funcLog.V(2).Info("CreateOVSBridge(): bridge state already match current configuration, no actions required") return nil } funcLog.V(2).Info("CreateOVSBridge(): bridge state differs from the current configuration, reconfiguration required") - keepBridge = reflect.DeepEqual(conf.Bridge, currentState.Bridge) + keepBridge = equality.Semantic.DeepEqual(conf.Bridge, currentState.Bridge) } } else { funcLog.V(2).Info("CreateOVSBridge(): configuration for the bridge not found in the store, create the bridge") @@ -148,6 +149,10 @@ func (o *ovs) CreateOVSBridge(ctx context.Context, conf *sriovnetworkv1.OVSConfi funcLog.Error(err, "CreateOVSBridge(): failed to get bridge after creation") return err } + if err := o.ensureInternalInterface(ctx, funcLog, dbClient, bridge); err != nil { + funcLog.Error(err, "CreateOVSBridge(): failed to add internal interface to the bridge") + return err + } funcLog.V(2).Info("CreateOVSBridge(): add uplink interface to the bridge") if err := o.addInterface(ctx, dbClient, bridge, &InterfaceEntry{ Name: conf.Uplinks[0].Name, @@ -156,6 +161,7 @@ func (o *ovs) CreateOVSBridge(ctx context.Context, conf *sriovnetworkv1.OVSConfi Options: conf.Uplinks[0].Interface.Options, ExternalIDs: conf.Uplinks[0].Interface.ExternalIDs, OtherConfig: conf.Uplinks[0].Interface.OtherConfig, + MTURequest: conf.Uplinks[0].Interface.MTURequest, }); err != nil { funcLog.Error(err, "CreateOVSBridge(): failed to add uplink interface to the bridge") return err @@ -163,6 +169,40 @@ func (o *ovs) CreateOVSBridge(ctx context.Context, conf *sriovnetworkv1.OVSConfi return nil } +func (o *ovs) ensureInternalInterface(ctx context.Context, funcLog logr.Logger, dbClient client.Client, bridge *BridgeEntry) error { + funcLog.V(2).Info("CreateOVSBridge(): Check if internal interface exists in the bridge") + existingIface, err := o.getInterfaceByName(ctx, dbClient, bridge.Name) + if err != nil { + funcLog.Error(err, "CreateOVSBridge(): failed to check internal interface in the bridge") + return err + } + if existingIface != nil { + // If interface exists but is in error state, we should remove it + if existingIface.Error != nil { + funcLog.V(2).Info("CreateOVSBridge(): internal interface exists but is in error state, removing it", "error", *existingIface.Error) + if err := o.deleteInterfaceByName(ctx, dbClient, bridge.Name); err != nil { + funcLog.Error(err, "CreateOVSBridge(): failed to remove internal interface in error state") + return err + } + existingIface = nil + } else { + funcLog.V(2).Info("CreateOVSBridge(): internal interface already exists and is valid") + return nil // Interface is already in the desired state, return early + } + } + // existingIface is nil here, so we need to create it + funcLog.V(2).Info("CreateOVSBridge(): add internal interface to the bridge") + if err := o.addInterface(ctx, dbClient, bridge, &InterfaceEntry{ + Name: bridge.Name, + UUID: uuid.NewString(), + Type: "internal", + }); err != nil { + funcLog.Error(err, "CreateOVSBridge(): failed to add internal interface to the bridge") + return err + } + return nil +} + // GetOVSBridges returns configuration for all managed bridges func (o *ovs) GetOVSBridges(ctx context.Context) ([]sriovnetworkv1.OVSConfigExt, error) { ctx, cancel := setDefaultTimeout(ctx) @@ -413,7 +453,7 @@ func (o *ovs) addInterface(ctx context.Context, dbClient client.Client, br *Brid return fmt.Errorf("failed to prepare operation for bridge mutate: %v", err) } if err := o.execTransaction(ctx, dbClient, addInterfaceOPs, addPortOPs, bridgeMutateOps); err != nil { - return fmt.Errorf("bridge deletion failed: %v", err) + return fmt.Errorf("bridge add interface failed: %v", err) } // check that interface has no error right after creation for i := 0; i < interfaceErrorCheckCount; i++ { @@ -592,6 +632,10 @@ func (o *ovs) getCurrentBridgeState(ctx context.Context, dbClient client.Client, OtherConfig: updateMap(knownConfigUplink.Interface.OtherConfig, iface.OtherConfig), }, }} + if iface.MTURequest != nil { + mtu := *iface.MTURequest + currentConfig.Uplinks[0].Interface.MTURequest = &mtu + } return currentConfig, nil } @@ -707,6 +751,7 @@ func getClient(ctx context.Context) (client.Client, error) { &interfaceEntry.Options, &interfaceEntry.ExternalIDs, &interfaceEntry.OtherConfig, + &interfaceEntry.MTURequest, ), client.WithTable(portEntry, &portEntry.UUID, diff --git a/pkg/host/internal/bridge/ovs/ovs_test.go b/pkg/host/internal/bridge/ovs/ovs_test.go index 666fe9218..362a93ebb 100644 --- a/pkg/host/internal/bridge/ovs/ovs_test.go +++ b/pkg/host/internal/bridge/ovs/ovs_test.go @@ -8,13 +8,13 @@ import ( "path/filepath" "time" - "github.com/golang/mock/gomock" "github.com/google/uuid" "github.com/ovn-org/libovsdb/client" "github.com/ovn-org/libovsdb/database/inmemory" "github.com/ovn-org/libovsdb/model" "github.com/ovn-org/libovsdb/ovsdb" "github.com/ovn-org/libovsdb/server" + "go.uber.org/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -27,6 +27,7 @@ import ( ) func getManagedBridges() map[string]*sriovnetworkv1.OVSConfigExt { + mtu := 5000 return map[string]*sriovnetworkv1.OVSConfigExt{ "br-0000_d8_00.0": { Name: "br-0000_d8_00.0", @@ -43,6 +44,7 @@ func getManagedBridges() map[string]*sriovnetworkv1.OVSConfigExt { ExternalIDs: map[string]string{"iface_externalID_key": "iface_externalID_value"}, OtherConfig: map[string]string{"iface_otherConfig_key": "iface_otherConfig_value"}, Options: map[string]string{"iface_options_key": "iface_options_value"}, + MTURequest: &mtu, }, }}, }, @@ -83,6 +85,7 @@ func (t *testDBEntries) GetCreateOperations(c client.Client) []ovsdb.Operation { } func getDefaultInitialDBContent() *testDBEntries { + mtu := 5000 iface := &InterfaceEntry{ Name: "enp216s0f0np0", UUID: uuid.NewString(), @@ -90,6 +93,7 @@ func getDefaultInitialDBContent() *testDBEntries { ExternalIDs: map[string]string{"iface_externalID_key": "iface_externalID_value"}, OtherConfig: map[string]string{"iface_otherConfig_key": "iface_otherConfig_value"}, Options: map[string]string{"iface_options_key": "iface_options_value"}, + MTURequest: &mtu, } port := &PortEntry{ Name: "enp216s0f0np0", @@ -137,25 +141,43 @@ func createInitialDBContent(ctx context.Context, c client.Client, expectedState func validateDBConfig(dbContent *testDBEntries, conf *sriovnetworkv1.OVSConfigExt) { Expect(dbContent.OpenVSwitch).To(HaveLen(1)) Expect(dbContent.Bridge).To(HaveLen(1)) - Expect(dbContent.Interface).To(HaveLen(1)) - Expect(dbContent.Port).To(HaveLen(1)) + Expect(dbContent.Interface).To(HaveLen(2)) + Expect(dbContent.Port).To(HaveLen(2)) ovs := dbContent.OpenVSwitch[0] br := dbContent.Bridge[0] - port := dbContent.Port[0] - iface := dbContent.Interface[0] + ports := make(map[string]*PortEntry, 0) + interfaces := make(map[string]*InterfaceEntry, 0) + for _, p := range dbContent.Port { + ports[p.Name] = p + } + for _, ifc := range dbContent.Interface { + interfaces[ifc.Name] = ifc + } Expect(ovs.Bridges).To(ContainElement(br.UUID)) Expect(br.Name).To(Equal(conf.Name)) Expect(br.DatapathType).To(Equal(conf.Bridge.DatapathType)) Expect(br.OtherConfig).To(Equal(conf.Bridge.OtherConfig)) Expect(br.ExternalIDs).To(Equal(conf.Bridge.ExternalIDs)) + port, ok := ports[conf.Uplinks[0].Name] + Expect(ok).To(BeTrue()) Expect(br.Ports).To(ContainElement(port.UUID)) - Expect(port.Name).To(Equal(conf.Uplinks[0].Name)) + iface, ok := interfaces[conf.Uplinks[0].Name] + Expect(ok).To(BeTrue()) Expect(port.Interfaces).To(ContainElement(iface.UUID)) - Expect(iface.Name).To(Equal(conf.Uplinks[0].Name)) Expect(iface.Options).To(Equal(conf.Uplinks[0].Interface.Options)) Expect(iface.Type).To(Equal(conf.Uplinks[0].Interface.Type)) Expect(iface.OtherConfig).To(Equal(conf.Uplinks[0].Interface.OtherConfig)) Expect(iface.ExternalIDs).To(Equal(conf.Uplinks[0].Interface.ExternalIDs)) + Expect(iface.MTURequest).To(Equal(conf.Uplinks[0].Interface.MTURequest)) + internalPort, ok := ports[conf.Name] + Expect(ok).To(BeTrue()) + internalIface, ok := interfaces[conf.Name] + Expect(ok).To(BeTrue()) + Expect(internalPort.Interfaces).To(ContainElement(internalIface.UUID)) + Expect(internalIface.Options).To(BeNil()) + Expect(internalIface.Type).To(Equal("internal")) + Expect(internalIface.OtherConfig).To(BeNil()) + Expect(internalIface.ExternalIDs).To(BeNil()) } var _ = Describe("OVS", func() { @@ -316,6 +338,19 @@ var _ = Describe("OVS", func() { // dbContent should be exactly same Expect(dbContent).To(Equal(initialDBContent)) }) + It("Bridge already exists, provided config semantically the same", func() { + storedConf := getManagedBridges()["br-0000_d8_00.0"] + providedConf := storedConf.DeepCopy() + storedConf.Uplinks[0].Interface.ExternalIDs = nil + providedConf.Uplinks[0].Interface.ExternalIDs = map[string]string{} + store.EXPECT().GetManagedOVSBridge("br-0000_d8_00.0").Return(storedConf, nil) + initialDBContent := getDefaultInitialDBContent() + createInitialDBContent(ctx, ovsClient, initialDBContent) + Expect(ovs.CreateOVSBridge(ctx, providedConf)).NotTo(HaveOccurred()) + dbContent := getDBContent(ctx, ovsClient) + // dbContent should be exactly the same + Expect(dbContent).To(Equal(initialDBContent)) + }) It("No Bridge, create bridge", func() { expectedConf := getManagedBridges()["br-0000_d8_00.0"] store.EXPECT().GetManagedOVSBridge("br-0000_d8_00.0").Return(nil, nil) @@ -402,6 +437,46 @@ var _ = Describe("OVS", func() { Expect(dbContent.Bridge[0].UUID).To(Equal(initialDBContent.Bridge[0].UUID)) Expect(dbContent.Interface[0].UUID).NotTo(Equal(initialDBContent.Interface[0].UUID)) }) + It("Bridge and internal interface exist and are valid, should not recreate them", func() { + expectedConf := getManagedBridges()["br-0000_d8_00.0"] + store.EXPECT().GetManagedOVSBridge("br-0000_d8_00.0").Return(expectedConf, nil) + + initialDBContent := getDefaultInitialDBContent() + // Add valid internal interface + internalIface := &InterfaceEntry{ + Name: expectedConf.Name, + UUID: uuid.NewString(), + Type: "internal", + } + internalPort := &PortEntry{ + Name: expectedConf.Name, + UUID: uuid.NewString(), + Interfaces: []string{internalIface.UUID}, + } + initialDBContent.Interface = append(initialDBContent.Interface, internalIface) + initialDBContent.Port = append(initialDBContent.Port, internalPort) + initialDBContent.Bridge[0].Ports = append(initialDBContent.Bridge[0].Ports, internalPort.UUID) + createInitialDBContent(ctx, ovsClient, initialDBContent) + + Expect(ovs.CreateOVSBridge(ctx, expectedConf)).NotTo(HaveOccurred()) + + dbContent := getDBContent(ctx, ovsClient) + validateDBConfig(dbContent, expectedConf) + + // All components should remain unchanged + Expect(dbContent.Bridge[0].UUID).To(Equal(initialDBContent.Bridge[0].UUID)) + // Find internal interface in the result + var internalIfaceFound bool + for _, iface := range dbContent.Interface { + if iface.Name == expectedConf.Name { + internalIfaceFound = true + Expect(iface.UUID).To(Equal(internalIface.UUID)) + Expect(iface.Error).To(BeNil()) + break + } + } + Expect(internalIfaceFound).To(BeTrue()) + }) }) Context("GetOVSBridges", func() { It("Bridge exist, but no managed bridges in config", func() { @@ -457,6 +532,7 @@ var _ = Describe("OVS", func() { initialDBContent := getDefaultInitialDBContent() initialDBContent.Bridge[0].ExternalIDs = nil initialDBContent.Bridge[0].OtherConfig = nil + initialDBContent.Interface[0].MTURequest = nil createInitialDBContent(ctx, ovsClient, initialDBContent) conf := getManagedBridges() store.EXPECT().GetManagedOVSBridges().Return(conf, nil) @@ -465,6 +541,7 @@ var _ = Describe("OVS", func() { Expect(ret).To(HaveLen(1)) Expect(ret[0].Bridge.ExternalIDs).To(BeEmpty()) Expect(ret[0].Bridge.OtherConfig).To(BeEmpty()) + Expect(ret[0].Uplinks[0].Interface.MTURequest).To(BeNil()) }) }) Context("RemoveOVSBridge", func() { diff --git a/pkg/host/internal/bridge/ovs/store/mock/mock_store.go b/pkg/host/internal/bridge/ovs/store/mock/mock_store.go index 2f98b96b9..e62f4bd23 100644 --- a/pkg/host/internal/bridge/ovs/store/mock/mock_store.go +++ b/pkg/host/internal/bridge/ovs/store/mock/mock_store.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: store.go +// +// Generated by this command: +// +// mockgen -destination mock/mock_store.go -source store.go +// // Package mock_store is a generated GoMock package. package mock_store @@ -7,14 +12,15 @@ package mock_store import ( reflect "reflect" - gomock "github.com/golang/mock/gomock" v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + gomock "go.uber.org/mock/gomock" ) // MockStore is a mock of Store interface. type MockStore struct { ctrl *gomock.Controller recorder *MockStoreMockRecorder + isgomock struct{} } // MockStoreMockRecorder is the mock recorder for MockStore. @@ -43,7 +49,7 @@ func (m *MockStore) AddManagedOVSBridge(br *v1.OVSConfigExt) error { } // AddManagedOVSBridge indicates an expected call of AddManagedOVSBridge. -func (mr *MockStoreMockRecorder) AddManagedOVSBridge(br interface{}) *gomock.Call { +func (mr *MockStoreMockRecorder) AddManagedOVSBridge(br any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddManagedOVSBridge", reflect.TypeOf((*MockStore)(nil).AddManagedOVSBridge), br) } @@ -58,7 +64,7 @@ func (m *MockStore) GetManagedOVSBridge(name string) (*v1.OVSConfigExt, error) { } // GetManagedOVSBridge indicates an expected call of GetManagedOVSBridge. -func (mr *MockStoreMockRecorder) GetManagedOVSBridge(name interface{}) *gomock.Call { +func (mr *MockStoreMockRecorder) GetManagedOVSBridge(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetManagedOVSBridge", reflect.TypeOf((*MockStore)(nil).GetManagedOVSBridge), name) } @@ -87,7 +93,7 @@ func (m *MockStore) RemoveManagedOVSBridge(name string) error { } // RemoveManagedOVSBridge indicates an expected call of RemoveManagedOVSBridge. -func (mr *MockStoreMockRecorder) RemoveManagedOVSBridge(name interface{}) *gomock.Call { +func (mr *MockStoreMockRecorder) RemoveManagedOVSBridge(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveManagedOVSBridge", reflect.TypeOf((*MockStore)(nil).RemoveManagedOVSBridge), name) } diff --git a/pkg/host/internal/bridge/ovs/test_db.ovsschema b/pkg/host/internal/bridge/ovs/test_db.ovsschema index 46c59dd0c..2b7a96a9a 100644 --- a/pkg/host/internal/bridge/ovs/test_db.ovsschema +++ b/pkg/host/internal/bridge/ovs/test_db.ovsschema @@ -105,6 +105,15 @@ }, "type": { "type": "string" + }, + "mtu_request":{ + "type": { + "key": { + "minInteger":1, + "type": "integer" + }, + "min": 0 + } } }, "indexes": [ diff --git a/pkg/host/internal/cpu/cpu.go b/pkg/host/internal/cpu/cpu.go new file mode 100644 index 000000000..fd02157e6 --- /dev/null +++ b/pkg/host/internal/cpu/cpu.go @@ -0,0 +1,40 @@ +package cpu + +import ( + "fmt" + + ghwPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/ghw" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" +) + +type cpuInfoProvider struct { + ghwLib ghwPkg.GHWLib +} + +func New(ghwLib ghwPkg.GHWLib) *cpuInfoProvider { + return &cpuInfoProvider{ + ghwLib: ghwLib, + } +} + +func (c *cpuInfoProvider) GetCPUVendor() (types.CPUVendor, error) { + cpuInfo, err := c.ghwLib.CPU() + if err != nil { + return -1, fmt.Errorf("can't retrieve the CPU vendor: %w", err) + } + + if len(cpuInfo.Processors) == 0 { + return -1, fmt.Errorf("wrong CPU information retrieved: %v", cpuInfo) + } + + switch cpuInfo.Processors[0].Vendor { + case "GenuineIntel": + return types.CPUVendorIntel, nil + case "AuthenticAMD": + return types.CPUVendorAMD, nil + case "ARM": + return types.CPUVendorARM, nil + } + + return -1, fmt.Errorf("unknown CPU vendor: %s", cpuInfo.Processors[0].Vendor) +} diff --git a/pkg/host/internal/infiniband/ib_guid_config_test.go b/pkg/host/internal/infiniband/ib_guid_config_test.go index 63f6b1683..5e2845d75 100644 --- a/pkg/host/internal/infiniband/ib_guid_config_test.go +++ b/pkg/host/internal/infiniband/ib_guid_config_test.go @@ -3,10 +3,10 @@ package infiniband import ( "fmt" - "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/vishvananda/netlink" + "go.uber.org/mock/gomock" netlinkLibPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" netlinkMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink/mock" diff --git a/pkg/host/internal/infiniband/ib_guid_pool_test.go b/pkg/host/internal/infiniband/ib_guid_pool_test.go index 86878d21a..04da6edb0 100644 --- a/pkg/host/internal/infiniband/ib_guid_pool_test.go +++ b/pkg/host/internal/infiniband/ib_guid_pool_test.go @@ -1,9 +1,9 @@ package infiniband import ( - "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" netlinkLibPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" netlinkMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink/mock" diff --git a/pkg/host/internal/infiniband/infiniband.go b/pkg/host/internal/infiniband/infiniband.go index f54957bc3..a554e84ee 100644 --- a/pkg/host/internal/infiniband/infiniband.go +++ b/pkg/host/internal/infiniband/infiniband.go @@ -5,6 +5,10 @@ import ( "fmt" "io/fs" "net" + "os" + "path/filepath" + "strconv" + "strings" "github.com/vishvananda/netlink" "sigs.k8s.io/controller-runtime/pkg/log" @@ -55,6 +59,29 @@ func (i *infiniband) ConfigureVfGUID(vfAddr string, pfAddr string, vfID int, pfL return i.applyVfGUIDToInterface(guid, vfAddr, vfID, pfLink) } +// GetVfGUID gets a GUID from sysfs for an IB VF device +func (i *infiniband) GetVfGUID(vfAddr string, pfAddr string, vfID int) (net.HardwareAddr, error) { + guidPath := filepath.Join(consts.SysBusPciDevices, pfAddr, "sriov", strconv.Itoa(vfID), "node") + data, err := os.ReadFile(guidPath) + if err != nil { + if os.IsNotExist(err) { + log.Log.Info("GetVfGUID(): GUID file doesn't exist", "path", guidPath) + return nil, nil + } + return nil, fmt.Errorf("failed to read GUID file %s: %v", guidPath, err) + } + guidStr := strings.TrimSpace(string(data)) + if guidStr == "" { + return nil, nil + } + guid, err := net.ParseMAC(guidStr) + if err != nil { + return nil, fmt.Errorf("failed to parse GUID %s: %v", guidStr, err) + } + log.Log.Info("GetVfGUID(): found guid", "guid", guid) + return guid, nil +} + func (i *infiniband) applyVfGUIDToInterface(guid net.HardwareAddr, vfAddr string, vfID int, pfLink netlink.Link) error { if err := i.netlinkLib.LinkSetVfNodeGUID(pfLink, vfID, guid); err != nil { return err diff --git a/pkg/host/internal/infiniband/infiniband_test.go b/pkg/host/internal/infiniband/infiniband_test.go index 58b3a8fec..17c18effc 100644 --- a/pkg/host/internal/infiniband/infiniband_test.go +++ b/pkg/host/internal/infiniband/infiniband_test.go @@ -3,9 +3,9 @@ package infiniband import ( "net" - "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" netlinkLibPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" diff --git a/pkg/host/internal/kernel/kernel_test.go b/pkg/host/internal/kernel/kernel_test.go index c6aef5cb6..fd27945cf 100644 --- a/pkg/host/internal/kernel/kernel_test.go +++ b/pkg/host/internal/kernel/kernel_test.go @@ -1,12 +1,19 @@ package kernel import ( + "fmt" + "path" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + mock_utils "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils/mock" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/fakefilesystem" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/helpers" ) @@ -14,11 +21,25 @@ import ( var _ = Describe("Kernel", func() { Context("Drivers", func() { var ( - k types.KernelInterface + k types.KernelInterface + kMocked types.KernelInterface + u *mock_utils.MockCmdInterface + t FullGinkgoTInterface + mockCtrl *gomock.Controller ) BeforeEach(func() { k = New(utils.New()) + + t = GinkgoT() + mockCtrl = gomock.NewController(t) + u = mock_utils.NewMockCmdInterface(mockCtrl) + kMocked = New(u) }) + + AfterEach(func() { + Expect(mockCtrl.Satisfied()).To(BeTrue()) + }) + Context("Unbind, UnbindDriverByBusAndDevice", func() { It("unknown device", func() { Expect(k.UnbindDriverByBusAndDevice(consts.BusPci, "unknown-dev")).NotTo(HaveOccurred()) @@ -51,6 +72,168 @@ var _ = Describe("Kernel", func() { Expect(k.Unbind("0000:d8:00.0")).To(HaveOccurred()) }) }) + + Context("LoadKernelModule", func() { + It("should return still try to load the kernel module if not able to check if it's loaded", func() { + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s lsmod | grep \"^tun\"", getHost())).Return("", "lsmod failed", fmt.Errorf("lsmod failed")) + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s modprobe tun ", getHost())).Return("", "", nil) + err := kMocked.LoadKernelModule("tun") + Expect(err).ToNot(HaveOccurred()) + }) + + It("should not try to to load the driver if it's already loaded", func() { + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s lsmod | grep \"^tun\"", getHost())).Return("tun", "", nil) + err := kMocked.LoadKernelModule("tun") + Expect(err).ToNot(HaveOccurred()) + }) + + It("should return error if not able to load kernel module", func() { + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s lsmod | grep \"^tun\"", getHost())).Return("", "", nil) + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s modprobe tun ", getHost())).Return("", "", fmt.Errorf("failed to run modprobe")) + err := kMocked.LoadKernelModule("tun") + Expect(err).To(HaveOccurred()) + }) + + It("should pass the args to the modprobe command", func() { + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s lsmod | grep \"^tun\"", getHost())).Return("", "", nil) + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s modprobe tun test=ok", getHost())).Return("", "", nil) + err := kMocked.LoadKernelModule("tun", "test=ok") + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Context("IsKernelModuleLoaded", func() { + It("should return error if not able to run lsmod command", func() { + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s lsmod | grep \"^tun\"", getHost())).Return("", "lsmod failed", fmt.Errorf("lsmod failed")) + enabled, err := kMocked.IsKernelModuleLoaded("tun") + Expect(err).To(HaveOccurred()) + Expect(enabled).To(BeFalse()) + }) + + It("should return error if stderr is not empty", func() { + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s lsmod | grep \"^tun\"", getHost())).Return("", "lsmod failed", nil) + enabled, err := kMocked.IsKernelModuleLoaded("tun") + Expect(err).To(HaveOccurred()) + Expect(enabled).To(BeFalse()) + }) + + It("should return false if std is empty", func() { + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s lsmod | grep \"^tun\"", getHost())).Return("", "", nil) + enabled, err := kMocked.IsKernelModuleLoaded("tun") + Expect(err).ToNot(HaveOccurred()) + Expect(enabled).To(BeFalse()) + }) + + It("should return true if std is not empty", func() { + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s lsmod | grep \"^tun\"", getHost())).Return("tun", "", nil) + enabled, err := kMocked.IsKernelModuleLoaded("tun") + Expect(err).ToNot(HaveOccurred()) + Expect(enabled).To(BeTrue()) + }) + }) + + Context("GetCurrentKernelArgs", func() { + It("should return error if not able to read the cmdline file", func() { + cmdline, err := k.GetCurrentKernelArgs() + Expect(err).To(HaveOccurred()) + Expect(cmdline).To(Equal("")) + }) + + It("should return cmdline", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{ + "/host/proc"}, + Files: map[string][]byte{ + "/host/proc/cmdline": []byte("iommu=pt")}, + }) + + cmdline, err := k.GetCurrentKernelArgs() + Expect(err).ToNot(HaveOccurred()) + Expect(cmdline).To(Equal("iommu=pt")) + }) + + It("should read the file without the /host when running on systemd mode", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{ + "/proc"}, + Files: map[string][]byte{ + "/proc/cmdline": []byte("iommu=pt")}, + }) + vars.UsingSystemdMode = true + defer func() { + vars.UsingSystemdMode = false + }() + + cmdline, err := k.GetCurrentKernelArgs() + Expect(err).ToNot(HaveOccurred()) + Expect(cmdline).To(Equal("iommu=pt")) + }) + }) + + Context("IsKernelArgsSet", func() { + It("should return false if the kernel arg does not exist is cmdline", func() { + set := k.IsKernelArgsSet("iommu=pt", consts.KernelArgIntelIommu) + Expect(set).To(BeFalse()) + }) + + It("should return true if the kernel arg exist in the cmdline", func() { + set := k.IsKernelArgsSet("iommu=pt", consts.KernelArgIommuPt) + Expect(set).To(BeTrue()) + }) + }) + + Context("RebindVfToDefaultDriver", func() { + It("should failed to rebind if unbind failed the driver", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{ + "/sys/bus/pci/devices/0000:d8:00.0"}, + Symlinks: map[string]string{ + "/sys/bus/pci/devices/0000:d8:00.0/driver": "../../../../bus/pci/drivers/test-driver"}, + }) + + err := k.RebindVfToDefaultDriver("0000:d8:00.0") + Expect(err).To(HaveOccurred()) + }) + + It("should be able to rebind the vf to the default driver", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{ + "/sys/bus/pci/devices/0000:d8:00.0", + "/sys/bus/pci/drivers/test-driver"}, + Symlinks: map[string]string{}, + Files: map[string][]byte{ + "/sys/bus/pci/drivers/test-driver/unbind": {}}, + }) + + err := k.RebindVfToDefaultDriver("0000:d8:00.0") + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Context("UnbindDriverIfNeeded", func() { + It("should always return nil for non rdma", func() { + err := k.UnbindDriverIfNeeded("0000:d8:00.0", false) + Expect(err).ToNot(HaveOccurred()) + }) + It("should called unbind functions", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{ + "/sys/bus/pci/devices/0000:d8:00.0", + "/sys/bus/pci/drivers/test-driver"}, + Symlinks: map[string]string{ + "/sys/bus/pci/devices/0000:d8:00.0/driver": "../../../../bus/pci/drivers/test-driver"}, + Files: map[string][]byte{ + "/sys/bus/pci/drivers/test-driver/unbind": {}}, + }) + + err := k.UnbindDriverIfNeeded("0000:d8:00.0", true) + Expect(err).ToNot(HaveOccurred()) + + // check that echo to unbind path was done + helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/drivers/test-driver/unbind", "0000:d8:00.0") + }) + }) + Context("HasDriver", func() { It("unknown device", func() { has, driver := k.HasDriver("unknown-dev") @@ -76,6 +259,7 @@ var _ = Describe("Kernel", func() { Expect(driver).To(Equal("test-driver")) }) }) + Context("BindDefaultDriver", func() { It("unknown device", func() { Expect(k.BindDefaultDriver("unknown-dev")).To(HaveOccurred()) @@ -118,6 +302,7 @@ var _ = Describe("Kernel", func() { helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/drivers_probe", "0000:d8:00.0") }) }) + Context("BindDpdkDriver", func() { It("unknown device", func() { Expect(k.BindDpdkDriver("unknown-dev", "vfio-pci")).To(HaveOccurred()) @@ -176,6 +361,7 @@ var _ = Describe("Kernel", func() { Expect(k.BindDpdkDriver("0000:d8:00.0", "vfio-pci")).To(HaveOccurred()) }) }) + Context("BindDriverByBusAndDevice", func() { It("device doesn't support driver_override", func() { helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ @@ -196,6 +382,7 @@ var _ = Describe("Kernel", func() { helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/drivers/vfio-pci/bind", "0000:d8:00.0") }) }) + Context("GetDriverByBusAndDevice", func() { It("device has driver", func() { helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ @@ -239,6 +426,75 @@ var _ = Describe("Kernel", func() { Expect(k.IsKernelLockdownMode()).To(BeFalse()) }) + + It("should return false if there is an error running the cat command", func() { + u.EXPECT().RunCommand("cat", fmt.Sprintf("%s/host/sys/kernel/security/lockdown", vars.FilesystemRoot)).Return("", "", fmt.Errorf("file doesn't exist")) + loaded := kMocked.IsKernelLockdownMode() + Expect(loaded).To(BeFalse()) + Expect(mockCtrl.Satisfied()).To(BeTrue()) + }) + }) + + Context("TryEnableTun", func() { + It("should load tun kernel module", func() { + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s lsmod | grep \"^tun\"", getHost())).Return("", "", nil) + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s modprobe tun ", getHost())).Return("", "", nil) + kMocked.TryEnableTun() + Expect(mockCtrl.Satisfied()).To(BeTrue()) + }) + }) + + Context("TryEnableVhostNet", func() { + It("should load tun kernel module", func() { + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s lsmod | grep \"^vhost_net\"", getHost())).Return("", "", nil) + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s modprobe vhost_net ", getHost())).Return("", "", nil) + kMocked.TryEnableVhostNet() + Expect(mockCtrl.Satisfied()).To(BeTrue()) + }) + }) + + Context("CheckRDMAEnabled", func() { + It("should return error if lsmod command failed", func() { + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s lsmod | grep --quiet 'mlx5_core'", getHost())).Return("", "lsmod failed", fmt.Errorf("lsmod failed")) + enabled, err := kMocked.CheckRDMAEnabled() + Expect(err).To(HaveOccurred()) + Expect(enabled).To(BeFalse()) + }) + + It("should return false if no RDMA capable devices exist", func() { + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s lsmod | grep --quiet 'mlx5_core'", getHost())).Return("", "", fmt.Errorf("lsmod failed")) + enabled, err := kMocked.CheckRDMAEnabled() + Expect(err).ToNot(HaveOccurred()) + Expect(enabled).To(BeFalse()) + }) + + It("should return error if ib and rdma lsmod command failed", func() { + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s lsmod | grep --quiet 'mlx5_core'", getHost())).Return("", "", nil) + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s lsmod | grep --quiet '\\(^ib\\|^rdma\\)'", getHost())).Return("", "lsmod failed", fmt.Errorf("lsmod failed")) + enabled, err := kMocked.CheckRDMAEnabled() + Expect(err).To(HaveOccurred()) + Expect(enabled).To(BeFalse()) + }) + + It("should return false if ib and rdma lsmod command return empty", func() { + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s lsmod | grep --quiet 'mlx5_core'", getHost())).Return("", "", nil) + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s lsmod | grep --quiet '\\(^ib\\|^rdma\\)'", getHost())).Return("", "", fmt.Errorf("lsmod failed")) + enabled, err := kMocked.CheckRDMAEnabled() + Expect(err).ToNot(HaveOccurred()) + Expect(enabled).To(BeFalse()) + }) + + It("should return true if ib and rdma are loaded", func() { + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s lsmod | grep --quiet 'mlx5_core'", getHost())).Return("", "", nil) + u.EXPECT().RunCommand("/bin/sh", "-c", fmt.Sprintf("chroot %s lsmod | grep --quiet '\\(^ib\\|^rdma\\)'", getHost())).Return("", "", nil) + enabled, err := kMocked.CheckRDMAEnabled() + Expect(err).ToNot(HaveOccurred()) + Expect(enabled).To(BeTrue()) + }) }) }) }) + +func getHost() string { + return path.Join(vars.FilesystemRoot, "/host") +} diff --git a/pkg/host/internal/lib/dputils/mock/mock_dputils.go b/pkg/host/internal/lib/dputils/mock/mock_dputils.go index de32180f2..2b26dddb1 100644 --- a/pkg/host/internal/lib/dputils/mock/mock_dputils.go +++ b/pkg/host/internal/lib/dputils/mock/mock_dputils.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: dputils.go +// +// Generated by this command: +// +// mockgen -destination mock/mock_dputils.go -source dputils.go +// // Package mock_dputils is a generated GoMock package. package mock_dputils @@ -7,13 +12,14 @@ package mock_dputils import ( reflect "reflect" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" ) // MockDPUtilsLib is a mock of DPUtilsLib interface. type MockDPUtilsLib struct { ctrl *gomock.Controller recorder *MockDPUtilsLibMockRecorder + isgomock struct{} } // MockDPUtilsLibMockRecorder is the mock recorder for MockDPUtilsLib. @@ -43,7 +49,7 @@ func (m *MockDPUtilsLib) GetDriverName(pciAddr string) (string, error) { } // GetDriverName indicates an expected call of GetDriverName. -func (mr *MockDPUtilsLibMockRecorder) GetDriverName(pciAddr interface{}) *gomock.Call { +func (mr *MockDPUtilsLibMockRecorder) GetDriverName(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDriverName", reflect.TypeOf((*MockDPUtilsLib)(nil).GetDriverName), pciAddr) } @@ -58,7 +64,7 @@ func (m *MockDPUtilsLib) GetNetNames(pciAddr string) ([]string, error) { } // GetNetNames indicates an expected call of GetNetNames. -func (mr *MockDPUtilsLibMockRecorder) GetNetNames(pciAddr interface{}) *gomock.Call { +func (mr *MockDPUtilsLibMockRecorder) GetNetNames(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNetNames", reflect.TypeOf((*MockDPUtilsLib)(nil).GetNetNames), pciAddr) } @@ -72,7 +78,7 @@ func (m *MockDPUtilsLib) GetSriovVFcapacity(pf string) int { } // GetSriovVFcapacity indicates an expected call of GetSriovVFcapacity. -func (mr *MockDPUtilsLibMockRecorder) GetSriovVFcapacity(pf interface{}) *gomock.Call { +func (mr *MockDPUtilsLibMockRecorder) GetSriovVFcapacity(pf any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSriovVFcapacity", reflect.TypeOf((*MockDPUtilsLib)(nil).GetSriovVFcapacity), pf) } @@ -87,7 +93,7 @@ func (m *MockDPUtilsLib) GetVFID(pciAddr string) (int, error) { } // GetVFID indicates an expected call of GetVFID. -func (mr *MockDPUtilsLibMockRecorder) GetVFID(pciAddr interface{}) *gomock.Call { +func (mr *MockDPUtilsLibMockRecorder) GetVFID(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVFID", reflect.TypeOf((*MockDPUtilsLib)(nil).GetVFID), pciAddr) } @@ -102,7 +108,7 @@ func (m *MockDPUtilsLib) GetVFList(pf string) ([]string, error) { } // GetVFList indicates an expected call of GetVFList. -func (mr *MockDPUtilsLibMockRecorder) GetVFList(pf interface{}) *gomock.Call { +func (mr *MockDPUtilsLibMockRecorder) GetVFList(pf any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVFList", reflect.TypeOf((*MockDPUtilsLib)(nil).GetVFList), pf) } @@ -116,7 +122,7 @@ func (m *MockDPUtilsLib) GetVFconfigured(pf string) int { } // GetVFconfigured indicates an expected call of GetVFconfigured. -func (mr *MockDPUtilsLibMockRecorder) GetVFconfigured(pf interface{}) *gomock.Call { +func (mr *MockDPUtilsLibMockRecorder) GetVFconfigured(pf any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVFconfigured", reflect.TypeOf((*MockDPUtilsLib)(nil).GetVFconfigured), pf) } @@ -130,7 +136,7 @@ func (m *MockDPUtilsLib) IsSriovPF(pciAddr string) bool { } // IsSriovPF indicates an expected call of IsSriovPF. -func (mr *MockDPUtilsLibMockRecorder) IsSriovPF(pciAddr interface{}) *gomock.Call { +func (mr *MockDPUtilsLibMockRecorder) IsSriovPF(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSriovPF", reflect.TypeOf((*MockDPUtilsLib)(nil).IsSriovPF), pciAddr) } @@ -144,7 +150,7 @@ func (m *MockDPUtilsLib) IsSriovVF(pciAddr string) bool { } // IsSriovVF indicates an expected call of IsSriovVF. -func (mr *MockDPUtilsLibMockRecorder) IsSriovVF(pciAddr interface{}) *gomock.Call { +func (mr *MockDPUtilsLibMockRecorder) IsSriovVF(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSriovVF", reflect.TypeOf((*MockDPUtilsLib)(nil).IsSriovVF), pciAddr) } @@ -158,7 +164,7 @@ func (m *MockDPUtilsLib) SriovConfigured(addr string) bool { } // SriovConfigured indicates an expected call of SriovConfigured. -func (mr *MockDPUtilsLibMockRecorder) SriovConfigured(addr interface{}) *gomock.Call { +func (mr *MockDPUtilsLibMockRecorder) SriovConfigured(addr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SriovConfigured", reflect.TypeOf((*MockDPUtilsLib)(nil).SriovConfigured), addr) } diff --git a/pkg/host/internal/lib/ethtool/mock/mock_ethtool.go b/pkg/host/internal/lib/ethtool/mock/mock_ethtool.go index b838bc868..ce70fde07 100644 --- a/pkg/host/internal/lib/ethtool/mock/mock_ethtool.go +++ b/pkg/host/internal/lib/ethtool/mock/mock_ethtool.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: ethtool.go +// +// Generated by this command: +// +// mockgen -destination mock/mock_ethtool.go -source ethtool.go +// // Package mock_ethtool is a generated GoMock package. package mock_ethtool @@ -7,13 +12,14 @@ package mock_ethtool import ( reflect "reflect" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" ) // MockEthtoolLib is a mock of EthtoolLib interface. type MockEthtoolLib struct { ctrl *gomock.Controller recorder *MockEthtoolLibMockRecorder + isgomock struct{} } // MockEthtoolLibMockRecorder is the mock recorder for MockEthtoolLib. @@ -42,7 +48,7 @@ func (m *MockEthtoolLib) Change(ifaceName string, config map[string]bool) error } // Change indicates an expected call of Change. -func (mr *MockEthtoolLibMockRecorder) Change(ifaceName, config interface{}) *gomock.Call { +func (mr *MockEthtoolLibMockRecorder) Change(ifaceName, config any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Change", reflect.TypeOf((*MockEthtoolLib)(nil).Change), ifaceName, config) } @@ -57,7 +63,7 @@ func (m *MockEthtoolLib) FeatureNames(ifaceName string) (map[string]uint, error) } // FeatureNames indicates an expected call of FeatureNames. -func (mr *MockEthtoolLibMockRecorder) FeatureNames(ifaceName interface{}) *gomock.Call { +func (mr *MockEthtoolLibMockRecorder) FeatureNames(ifaceName any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FeatureNames", reflect.TypeOf((*MockEthtoolLib)(nil).FeatureNames), ifaceName) } @@ -72,7 +78,7 @@ func (m *MockEthtoolLib) Features(ifaceName string) (map[string]bool, error) { } // Features indicates an expected call of Features. -func (mr *MockEthtoolLibMockRecorder) Features(ifaceName interface{}) *gomock.Call { +func (mr *MockEthtoolLibMockRecorder) Features(ifaceName any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Features", reflect.TypeOf((*MockEthtoolLib)(nil).Features), ifaceName) } diff --git a/pkg/host/internal/lib/ghw/ghw.go b/pkg/host/internal/lib/ghw/ghw.go index 6a6829604..2a4ba609d 100644 --- a/pkg/host/internal/lib/ghw/ghw.go +++ b/pkg/host/internal/lib/ghw/ghw.go @@ -2,6 +2,8 @@ package ghw import ( "github.com/jaypipes/ghw" + "github.com/jaypipes/ghw/pkg/cpu" + "github.com/jaypipes/ghw/pkg/pci" ) func New() GHWLib { @@ -11,19 +13,19 @@ func New() GHWLib { //go:generate ../../../../../bin/mockgen -destination mock/mock_ghw.go -source ghw.go type GHWLib interface { // PCI returns a pointer to an Info that provide methods to access info about devices - PCI() (Info, error) -} + PCI() (*pci.Info, error) -// Info interface provide methods to access info about devices -type Info interface { - // ListDevices returns a list of pointers to Device structs present on the - // host system - ListDevices() []*ghw.PCIDevice + // CPU returns a pointer to an Info that provide methods to access info about devices + CPU() (*cpu.Info, error) } type libWrapper struct{} // PCI returns a pointer to an Info that provide methods to access info about devices -func (w *libWrapper) PCI() (Info, error) { +func (w *libWrapper) PCI() (*pci.Info, error) { return ghw.PCI() } + +func (w *libWrapper) CPU() (*cpu.Info, error) { + return ghw.CPU() +} diff --git a/pkg/host/internal/lib/ghw/mock/mock_ghw.go b/pkg/host/internal/lib/ghw/mock/mock_ghw.go index 2e2b4b5c5..7909429c6 100644 --- a/pkg/host/internal/lib/ghw/mock/mock_ghw.go +++ b/pkg/host/internal/lib/ghw/mock/mock_ghw.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: ghw.go +// +// Generated by this command: +// +// mockgen -destination mock/mock_ghw.go -source ghw.go +// // Package mock_ghw is a generated GoMock package. package mock_ghw @@ -7,15 +12,16 @@ package mock_ghw import ( reflect "reflect" - gomock "github.com/golang/mock/gomock" - ghw "github.com/jaypipes/ghw" - ghw0 "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/ghw" + cpu "github.com/jaypipes/ghw/pkg/cpu" + pci "github.com/jaypipes/ghw/pkg/pci" + gomock "go.uber.org/mock/gomock" ) // MockGHWLib is a mock of GHWLib interface. type MockGHWLib struct { ctrl *gomock.Controller recorder *MockGHWLibMockRecorder + isgomock struct{} } // MockGHWLibMockRecorder is the mock recorder for MockGHWLib. @@ -35,54 +41,32 @@ func (m *MockGHWLib) EXPECT() *MockGHWLibMockRecorder { return m.recorder } -// PCI mocks base method. -func (m *MockGHWLib) PCI() (ghw0.Info, error) { +// CPU mocks base method. +func (m *MockGHWLib) CPU() (*cpu.Info, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "PCI") - ret0, _ := ret[0].(ghw0.Info) + ret := m.ctrl.Call(m, "CPU") + ret0, _ := ret[0].(*cpu.Info) ret1, _ := ret[1].(error) return ret0, ret1 } -// PCI indicates an expected call of PCI. -func (mr *MockGHWLibMockRecorder) PCI() *gomock.Call { +// CPU indicates an expected call of CPU. +func (mr *MockGHWLibMockRecorder) CPU() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PCI", reflect.TypeOf((*MockGHWLib)(nil).PCI)) -} - -// MockInfo is a mock of Info interface. -type MockInfo struct { - ctrl *gomock.Controller - recorder *MockInfoMockRecorder -} - -// MockInfoMockRecorder is the mock recorder for MockInfo. -type MockInfoMockRecorder struct { - mock *MockInfo -} - -// NewMockInfo creates a new mock instance. -func NewMockInfo(ctrl *gomock.Controller) *MockInfo { - mock := &MockInfo{ctrl: ctrl} - mock.recorder = &MockInfoMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockInfo) EXPECT() *MockInfoMockRecorder { - return m.recorder + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CPU", reflect.TypeOf((*MockGHWLib)(nil).CPU)) } -// ListDevices mocks base method. -func (m *MockInfo) ListDevices() []*ghw.PCIDevice { +// PCI mocks base method. +func (m *MockGHWLib) PCI() (*pci.Info, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListDevices") - ret0, _ := ret[0].([]*ghw.PCIDevice) - return ret0 + ret := m.ctrl.Call(m, "PCI") + ret0, _ := ret[0].(*pci.Info) + ret1, _ := ret[1].(error) + return ret0, ret1 } -// ListDevices indicates an expected call of ListDevices. -func (mr *MockInfoMockRecorder) ListDevices() *gomock.Call { +// PCI indicates an expected call of PCI. +func (mr *MockGHWLibMockRecorder) PCI() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDevices", reflect.TypeOf((*MockInfo)(nil).ListDevices)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PCI", reflect.TypeOf((*MockGHWLib)(nil).PCI)) } diff --git a/pkg/host/internal/lib/netlink/mock/mock_netlink.go b/pkg/host/internal/lib/netlink/mock/mock_netlink.go index 5b3bcc790..8d919a0db 100644 --- a/pkg/host/internal/lib/netlink/mock/mock_netlink.go +++ b/pkg/host/internal/lib/netlink/mock/mock_netlink.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: netlink.go +// +// Generated by this command: +// +// mockgen -destination mock/mock_netlink.go -source netlink.go +// // Package mock_netlink is a generated GoMock package. package mock_netlink @@ -8,15 +13,16 @@ import ( net "net" reflect "reflect" - gomock "github.com/golang/mock/gomock" netlink "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" netlink0 "github.com/vishvananda/netlink" + gomock "go.uber.org/mock/gomock" ) // MockLink is a mock of Link interface. type MockLink struct { ctrl *gomock.Controller recorder *MockLinkMockRecorder + isgomock struct{} } // MockLinkMockRecorder is the mock recorder for MockLink. @@ -68,6 +74,7 @@ func (mr *MockLinkMockRecorder) Type() *gomock.Call { type MockNetlinkLib struct { ctrl *gomock.Controller recorder *MockNetlinkLibMockRecorder + isgomock struct{} } // MockNetlinkLibMockRecorder is the mock recorder for MockNetlinkLib. @@ -97,7 +104,7 @@ func (m *MockNetlinkLib) DevLinkGetDeviceByName(bus, device string) (*netlink0.D } // DevLinkGetDeviceByName indicates an expected call of DevLinkGetDeviceByName. -func (mr *MockNetlinkLibMockRecorder) DevLinkGetDeviceByName(bus, device interface{}) *gomock.Call { +func (mr *MockNetlinkLibMockRecorder) DevLinkGetDeviceByName(bus, device any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DevLinkGetDeviceByName", reflect.TypeOf((*MockNetlinkLib)(nil).DevLinkGetDeviceByName), bus, device) } @@ -111,7 +118,7 @@ func (m *MockNetlinkLib) DevLinkSetEswitchMode(dev *netlink0.DevlinkDevice, newM } // DevLinkSetEswitchMode indicates an expected call of DevLinkSetEswitchMode. -func (mr *MockNetlinkLibMockRecorder) DevLinkSetEswitchMode(dev, newMode interface{}) *gomock.Call { +func (mr *MockNetlinkLibMockRecorder) DevLinkSetEswitchMode(dev, newMode any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DevLinkSetEswitchMode", reflect.TypeOf((*MockNetlinkLib)(nil).DevLinkSetEswitchMode), dev, newMode) } @@ -126,13 +133,13 @@ func (m *MockNetlinkLib) DevlinkGetDeviceParamByName(bus, device, param string) } // DevlinkGetDeviceParamByName indicates an expected call of DevlinkGetDeviceParamByName. -func (mr *MockNetlinkLibMockRecorder) DevlinkGetDeviceParamByName(bus, device, param interface{}) *gomock.Call { +func (mr *MockNetlinkLibMockRecorder) DevlinkGetDeviceParamByName(bus, device, param any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DevlinkGetDeviceParamByName", reflect.TypeOf((*MockNetlinkLib)(nil).DevlinkGetDeviceParamByName), bus, device, param) } // DevlinkSetDeviceParam mocks base method. -func (m *MockNetlinkLib) DevlinkSetDeviceParam(bus, device, param string, cmode uint8, value interface{}) error { +func (m *MockNetlinkLib) DevlinkSetDeviceParam(bus, device, param string, cmode uint8, value any) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DevlinkSetDeviceParam", bus, device, param, cmode, value) ret0, _ := ret[0].(error) @@ -140,7 +147,7 @@ func (m *MockNetlinkLib) DevlinkSetDeviceParam(bus, device, param string, cmode } // DevlinkSetDeviceParam indicates an expected call of DevlinkSetDeviceParam. -func (mr *MockNetlinkLibMockRecorder) DevlinkSetDeviceParam(bus, device, param, cmode, value interface{}) *gomock.Call { +func (mr *MockNetlinkLibMockRecorder) DevlinkSetDeviceParam(bus, device, param, cmode, value any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DevlinkSetDeviceParam", reflect.TypeOf((*MockNetlinkLib)(nil).DevlinkSetDeviceParam), bus, device, param, cmode, value) } @@ -154,7 +161,7 @@ func (m *MockNetlinkLib) IsLinkAdminStateUp(link netlink.Link) bool { } // IsLinkAdminStateUp indicates an expected call of IsLinkAdminStateUp. -func (mr *MockNetlinkLibMockRecorder) IsLinkAdminStateUp(link interface{}) *gomock.Call { +func (mr *MockNetlinkLibMockRecorder) IsLinkAdminStateUp(link any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsLinkAdminStateUp", reflect.TypeOf((*MockNetlinkLib)(nil).IsLinkAdminStateUp), link) } @@ -169,7 +176,7 @@ func (m *MockNetlinkLib) LinkByIndex(index int) (netlink.Link, error) { } // LinkByIndex indicates an expected call of LinkByIndex. -func (mr *MockNetlinkLibMockRecorder) LinkByIndex(index interface{}) *gomock.Call { +func (mr *MockNetlinkLibMockRecorder) LinkByIndex(index any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LinkByIndex", reflect.TypeOf((*MockNetlinkLib)(nil).LinkByIndex), index) } @@ -184,7 +191,7 @@ func (m *MockNetlinkLib) LinkByName(name string) (netlink.Link, error) { } // LinkByName indicates an expected call of LinkByName. -func (mr *MockNetlinkLibMockRecorder) LinkByName(name interface{}) *gomock.Call { +func (mr *MockNetlinkLibMockRecorder) LinkByName(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LinkByName", reflect.TypeOf((*MockNetlinkLib)(nil).LinkByName), name) } @@ -213,7 +220,7 @@ func (m *MockNetlinkLib) LinkSetMTU(link netlink.Link, mtu int) error { } // LinkSetMTU indicates an expected call of LinkSetMTU. -func (mr *MockNetlinkLibMockRecorder) LinkSetMTU(link, mtu interface{}) *gomock.Call { +func (mr *MockNetlinkLibMockRecorder) LinkSetMTU(link, mtu any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LinkSetMTU", reflect.TypeOf((*MockNetlinkLib)(nil).LinkSetMTU), link, mtu) } @@ -227,7 +234,7 @@ func (m *MockNetlinkLib) LinkSetUp(link netlink.Link) error { } // LinkSetUp indicates an expected call of LinkSetUp. -func (mr *MockNetlinkLibMockRecorder) LinkSetUp(link interface{}) *gomock.Call { +func (mr *MockNetlinkLibMockRecorder) LinkSetUp(link any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LinkSetUp", reflect.TypeOf((*MockNetlinkLib)(nil).LinkSetUp), link) } @@ -241,7 +248,7 @@ func (m *MockNetlinkLib) LinkSetVfHardwareAddr(link netlink.Link, vf int, hwaddr } // LinkSetVfHardwareAddr indicates an expected call of LinkSetVfHardwareAddr. -func (mr *MockNetlinkLibMockRecorder) LinkSetVfHardwareAddr(link, vf, hwaddr interface{}) *gomock.Call { +func (mr *MockNetlinkLibMockRecorder) LinkSetVfHardwareAddr(link, vf, hwaddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LinkSetVfHardwareAddr", reflect.TypeOf((*MockNetlinkLib)(nil).LinkSetVfHardwareAddr), link, vf, hwaddr) } @@ -255,7 +262,7 @@ func (m *MockNetlinkLib) LinkSetVfNodeGUID(link netlink.Link, vf int, nodeguid n } // LinkSetVfNodeGUID indicates an expected call of LinkSetVfNodeGUID. -func (mr *MockNetlinkLibMockRecorder) LinkSetVfNodeGUID(link, vf, nodeguid interface{}) *gomock.Call { +func (mr *MockNetlinkLibMockRecorder) LinkSetVfNodeGUID(link, vf, nodeguid any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LinkSetVfNodeGUID", reflect.TypeOf((*MockNetlinkLib)(nil).LinkSetVfNodeGUID), link, vf, nodeguid) } @@ -269,7 +276,7 @@ func (m *MockNetlinkLib) LinkSetVfPortGUID(link netlink.Link, vf int, portguid n } // LinkSetVfPortGUID indicates an expected call of LinkSetVfPortGUID. -func (mr *MockNetlinkLibMockRecorder) LinkSetVfPortGUID(link, vf, portguid interface{}) *gomock.Call { +func (mr *MockNetlinkLibMockRecorder) LinkSetVfPortGUID(link, vf, portguid any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LinkSetVfPortGUID", reflect.TypeOf((*MockNetlinkLib)(nil).LinkSetVfPortGUID), link, vf, portguid) } @@ -284,11 +291,26 @@ func (m *MockNetlinkLib) RdmaLinkByName(name string) (*netlink0.RdmaLink, error) } // RdmaLinkByName indicates an expected call of RdmaLinkByName. -func (mr *MockNetlinkLibMockRecorder) RdmaLinkByName(name interface{}) *gomock.Call { +func (mr *MockNetlinkLibMockRecorder) RdmaLinkByName(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RdmaLinkByName", reflect.TypeOf((*MockNetlinkLib)(nil).RdmaLinkByName), name) } +// RdmaSystemGetNetnsMode mocks base method. +func (m *MockNetlinkLib) RdmaSystemGetNetnsMode() (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RdmaSystemGetNetnsMode") + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RdmaSystemGetNetnsMode indicates an expected call of RdmaSystemGetNetnsMode. +func (mr *MockNetlinkLibMockRecorder) RdmaSystemGetNetnsMode() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RdmaSystemGetNetnsMode", reflect.TypeOf((*MockNetlinkLib)(nil).RdmaSystemGetNetnsMode)) +} + // VDPADelDev mocks base method. func (m *MockNetlinkLib) VDPADelDev(name string) error { m.ctrl.T.Helper() @@ -298,7 +320,7 @@ func (m *MockNetlinkLib) VDPADelDev(name string) error { } // VDPADelDev indicates an expected call of VDPADelDev. -func (mr *MockNetlinkLibMockRecorder) VDPADelDev(name interface{}) *gomock.Call { +func (mr *MockNetlinkLibMockRecorder) VDPADelDev(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VDPADelDev", reflect.TypeOf((*MockNetlinkLib)(nil).VDPADelDev), name) } @@ -313,7 +335,7 @@ func (m *MockNetlinkLib) VDPAGetDevByName(name string) (*netlink0.VDPADev, error } // VDPAGetDevByName indicates an expected call of VDPAGetDevByName. -func (mr *MockNetlinkLibMockRecorder) VDPAGetDevByName(name interface{}) *gomock.Call { +func (mr *MockNetlinkLibMockRecorder) VDPAGetDevByName(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VDPAGetDevByName", reflect.TypeOf((*MockNetlinkLib)(nil).VDPAGetDevByName), name) } @@ -327,7 +349,7 @@ func (m *MockNetlinkLib) VDPANewDev(name, mgmtBus, mgmtName string, params netli } // VDPANewDev indicates an expected call of VDPANewDev. -func (mr *MockNetlinkLibMockRecorder) VDPANewDev(name, mgmtBus, mgmtName, params interface{}) *gomock.Call { +func (mr *MockNetlinkLibMockRecorder) VDPANewDev(name, mgmtBus, mgmtName, params any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VDPANewDev", reflect.TypeOf((*MockNetlinkLib)(nil).VDPANewDev), name, mgmtBus, mgmtName, params) } diff --git a/pkg/host/internal/lib/netlink/netlink.go b/pkg/host/internal/lib/netlink/netlink.go index ed063834e..ad6056710 100644 --- a/pkg/host/internal/lib/netlink/netlink.go +++ b/pkg/host/internal/lib/netlink/netlink.go @@ -68,6 +68,8 @@ type NetlinkLib interface { RdmaLinkByName(name string) (*netlink.RdmaLink, error) // IsLinkAdminStateUp checks if the admin state of a link is up IsLinkAdminStateUp(link Link) bool + // RdmaSystemGetNetnsMode returns RDMA subsystem mode + RdmaSystemGetNetnsMode() (string, error) } type libWrapper struct{} @@ -185,3 +187,8 @@ func (w *libWrapper) RdmaLinkByName(name string) (*netlink.RdmaLink, error) { func (w *libWrapper) IsLinkAdminStateUp(link Link) bool { return link.Attrs().Flags&net.FlagUp == 1 } + +// RdmaSystemGetNetnsMode returns RDMA subsystem mode +func (w *libWrapper) RdmaSystemGetNetnsMode() (string, error) { + return netlink.RdmaSystemGetNetnsMode() +} diff --git a/pkg/host/internal/lib/sriovnet/mock/mock_sriovnet.go b/pkg/host/internal/lib/sriovnet/mock/mock_sriovnet.go index 937ce7914..bd2ad0b40 100644 --- a/pkg/host/internal/lib/sriovnet/mock/mock_sriovnet.go +++ b/pkg/host/internal/lib/sriovnet/mock/mock_sriovnet.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: sriovnet.go +// +// Generated by this command: +// +// mockgen -destination mock/mock_sriovnet.go -source sriovnet.go +// // Package mock_sriovnet is a generated GoMock package. package mock_sriovnet @@ -7,13 +12,14 @@ package mock_sriovnet import ( reflect "reflect" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" ) // MockSriovnetLib is a mock of SriovnetLib interface. type MockSriovnetLib struct { ctrl *gomock.Controller recorder *MockSriovnetLibMockRecorder + isgomock struct{} } // MockSriovnetLibMockRecorder is the mock recorder for MockSriovnetLib. @@ -43,7 +49,7 @@ func (m *MockSriovnetLib) GetVfRepresentor(uplink string, vfIndex int) (string, } // GetVfRepresentor indicates an expected call of GetVfRepresentor. -func (mr *MockSriovnetLibMockRecorder) GetVfRepresentor(uplink, vfIndex interface{}) *gomock.Call { +func (mr *MockSriovnetLibMockRecorder) GetVfRepresentor(uplink, vfIndex any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVfRepresentor", reflect.TypeOf((*MockSriovnetLib)(nil).GetVfRepresentor), uplink, vfIndex) } diff --git a/pkg/host/internal/network/network.go b/pkg/host/internal/network/network.go index b3014f9e9..feb140a4c 100644 --- a/pkg/host/internal/network/network.go +++ b/pkg/host/internal/network/network.go @@ -75,7 +75,6 @@ func (n *network) TryToGetVirtualInterfaceName(pciAddr string) string { func (n *network) TryGetInterfaceName(pciAddr string) string { names, err := n.dputilsLib.GetNetNames(pciAddr) if err != nil || len(names) < 1 { - log.Log.Error(err, "TryGetInterfaceName(): failed to get interface name") return "" } netDevName := names[0] @@ -96,7 +95,6 @@ func (n *network) TryGetInterfaceName(pciAddr string) string { return name } - log.Log.V(2).Info("TryGetInterfaceName()", "name", netDevName) return netDevName } @@ -264,12 +262,12 @@ func (n *network) GetDevlinkDeviceParam(pciAddr, paramName string) (string, erro funcLog.Error(err, "GetDevlinkDeviceParam(): fail to get devlink device param") return "", err } - if len(param.Values) == 0 { - err = fmt.Errorf("param %s has no value", paramName) - funcLog.Error(err, "GetDevlinkDeviceParam(): error") - return "", err + if len(param.Values) == 0 || param.Values[0].Data == nil { + funcLog.Info("GetDevlinkDeviceParam(): WARNING: can't read devlink parameter from the device, an empty value received") + return "", nil } var value string + var ok bool switch param.Type { case nl.DEVLINK_PARAM_TYPE_U8, nl.DEVLINK_PARAM_TYPE_U16, nl.DEVLINK_PARAM_TYPE_U32: var valData uint64 @@ -281,14 +279,22 @@ func (n *network) GetDevlinkDeviceParam(pciAddr, paramName string) (string, erro case uint32: valData = uint64(v) default: - return "", fmt.Errorf("unexpected uint type type") + return "", fmt.Errorf("value is not uint") } value = strconv.FormatUint(valData, 10) case nl.DEVLINK_PARAM_TYPE_STRING: - value = param.Values[0].Data.(string) + value, ok = param.Values[0].Data.(string) + if !ok { + return "", fmt.Errorf("value is not a string") + } case nl.DEVLINK_PARAM_TYPE_BOOL: - value = strconv.FormatBool(param.Values[0].Data.(bool)) + var boolValue bool + boolValue, ok = param.Values[0].Data.(bool) + if !ok { + return "", fmt.Errorf("value is not a bool") + } + value = strconv.FormatBool(boolValue) default: return "", fmt.Errorf("unknown value type: %d", param.Type) } @@ -421,3 +427,42 @@ func (n *network) GetPciAddressFromInterfaceName(interfaceName string) (string, log.Log.V(2).Info("GetPciAddressFromInterfaceName(): result", "interface", interfaceName, "pci address", pciAddress) return pciAddress, nil } + +func (n *network) DiscoverRDMASubsystem() (string, error) { + subsystem, err := n.netlinkLib.RdmaSystemGetNetnsMode() + + if err != nil { + log.Log.Error(err, "DiscoverRDMASubsystem(): failed to get RDMA subsystem mode") + return "", err + } + + return subsystem, nil +} + +func (n *network) SetRDMASubsystem(mode string) error { + log.Log.Info("SetRDMASubsystem(): Updating RDMA subsystem mode", "mode", mode) + path := filepath.Join(vars.FilesystemRoot, consts.Host, "etc", "modprobe.d", "sriov_network_operator_modules_config.conf") + + if mode == "" { + err := os.Remove(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + log.Log.Error(err, "failed to remove ib_core config file") + return err + } + return nil + } + + modeValue := 1 + if mode == "exclusive" { + modeValue = 0 + } + config := fmt.Sprintf("# This file is managed by sriov-network-operator do not edit.\noptions ib_core netns_mode=%d\n", modeValue) + + err := os.WriteFile(path, []byte(config), 0644) + if err != nil { + log.Log.Error(err, "SetRDMASubsystem(): failed to write sriov_network_operator_modules_config.conf") + return fmt.Errorf("failed to write sriov_network_operator_modules_config.conf: %v", err) + } + + return nil +} diff --git a/pkg/host/internal/network/network_test.go b/pkg/host/internal/network/network_test.go index 19eb3f438..40c74eae0 100644 --- a/pkg/host/internal/network/network_test.go +++ b/pkg/host/internal/network/network_test.go @@ -2,19 +2,21 @@ package network import ( "fmt" + "net" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/vishvananda/netlink" "github.com/vishvananda/netlink/nl" + "go.uber.org/mock/gomock" - "github.com/golang/mock/gomock" - - hostMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper/mock" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" dputilsMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/dputils/mock" ethtoolMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/ethtool/mock" netlinkMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink/mock" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" + utilsMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils/mock" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/fakefilesystem" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/helpers" ) @@ -34,7 +36,7 @@ var _ = Describe("Network", func() { netlinkLibMock *netlinkMockPkg.MockNetlinkLib ethtoolLibMock *ethtoolMockPkg.MockEthtoolLib dputilsLibMock *dputilsMockPkg.MockDPUtilsLib - hostMock *hostMockPkg.MockHostHelpersInterface + utilsMock *utilsMockPkg.MockCmdInterface testCtrl *gomock.Controller testErr = fmt.Errorf("test") @@ -44,14 +46,35 @@ var _ = Describe("Network", func() { netlinkLibMock = netlinkMockPkg.NewMockNetlinkLib(testCtrl) ethtoolLibMock = ethtoolMockPkg.NewMockEthtoolLib(testCtrl) dputilsLibMock = dputilsMockPkg.NewMockDPUtilsLib(testCtrl) - hostMock = hostMockPkg.NewMockHostHelpersInterface(testCtrl) + utilsMock = utilsMockPkg.NewMockCmdInterface(testCtrl) - n = New(hostMock, dputilsLibMock, netlinkLibMock, ethtoolLibMock) + n = New(utilsMock, dputilsLibMock, netlinkLibMock, ethtoolLibMock) }) AfterEach(func() { testCtrl.Finish() }) + Context("TryToGetVirtualInterfaceName", func() { + It("should get the interface name if attached to kernel interface", func() { + dputilsLibMock.EXPECT().GetNetNames("0000:d8:00.0").Return([]string{"eno1"}, nil) + name := n.TryToGetVirtualInterfaceName("0000:d8:00.0") + Expect(name).To(Equal("eno1")) + }) + It("should get the virtio interface name via sysfs", func() { + dputilsLibMock.EXPECT().GetNetNames("0000:d8:00.0").Return([]string{""}, nil) + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{ + "/sys/bus/pci/devices/0000:d8:00.0/virtio-1/net", + "/sys/bus/pci/devices/0000:d8:00.0/virtio-2/net"}, + Files: map[string][]byte{ + "/sys/bus/pci/devices/0000:d8:00.0/virtio-1/net/eno1": []byte(""), + }, + }) + + name := n.TryToGetVirtualInterfaceName("0000:d8:00.0") + Expect(name).To(Equal("eno1")) + }) + }) Context("GetDevlinkDeviceParam", func() { It("get - string", func() { netlinkLibMock.EXPECT().DevlinkGetDeviceParamByName("pci", "0000:d8:00.1", "param_name").Return( @@ -283,4 +306,179 @@ var _ = Describe("Network", func() { Expect(pci).To(Equal("0000:3b:00.0")) }) }) + Context("DiscoverRDMASubsystem", func() { + It("Should get RDMA Subsystem using netlink", func() { + netlinkLibMock.EXPECT().RdmaSystemGetNetnsMode().Return("shared", nil) + + pci, err := n.DiscoverRDMASubsystem() + Expect(err).NotTo(HaveOccurred()) + Expect(pci).To(Equal("shared")) + }) + }) + Context("GetPhysPortName", func() { + It("should return error if phys_port_name doesn't exist", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{ + "/sys/class/net/eno1"}, + }) + name, err := n.GetPhysPortName("eno1") + Expect(err).To(HaveOccurred()) + Expect(name).To(BeEmpty()) + }) + It("should return the empty and no error if the file content is empty", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{ + "/sys/class/net/eno1"}, + Files: map[string][]byte{ + "/sys/class/net/eno1/phys_port_name": []byte(""), + }, + }) + name, err := n.GetPhysPortName("eno1") + Expect(err).ToNot(HaveOccurred()) + Expect(name).To(BeEmpty()) + }) + It("should return the physical port name without spaces", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{ + "/sys/class/net/eno1"}, + Files: map[string][]byte{ + "/sys/class/net/eno1/phys_port_name": []byte("eno1p "), + }, + }) + name, err := n.GetPhysPortName("eno1") + Expect(err).ToNot(HaveOccurred()) + Expect(name).To(Equal("eno1p")) + }) + }) + Context("GetNetdevMTU", func() { + It("should return 0 if not able to get interface name", func() { + dputilsLibMock.EXPECT().GetNetNames("0000:d8:00.0").Return([]string{""}, fmt.Errorf("failed to get interface name")) + mtu := n.GetNetdevMTU("0000:d8:00.0") + Expect(mtu).To(Equal(0)) + }) + It("should return 0 if not able to get interface by name", func() { + dputilsLibMock.EXPECT().GetNetNames("0000:d8:00.0").Return([]string{"eno1"}, nil) + netlinkLibMock.EXPECT().LinkByName("eno1").Return(nil, fmt.Errorf("failed to get interface")) + mtu := n.GetNetdevMTU("0000:d8:00.0") + Expect(mtu).To(Equal(0)) + }) + It("should return mtu for interface", func() { + dputilsLibMock.EXPECT().GetNetNames("0000:d8:00.0").Return([]string{"eno1"}, nil) + link := &netlink.GenericLink{LinkType: "PF", LinkAttrs: netlink.LinkAttrs{Name: "eno1", MTU: 1500}} + netlinkLibMock.EXPECT().LinkByName("eno1").Return(link, nil) + mtu := n.GetNetdevMTU("0000:d8:00.0") + Expect(mtu).To(Equal(1500)) + }) + }) + Context("SetNetdevMTU", func() { + It("should return no error without configuring for mtu lower or equal to 0", func() { + err := n.SetNetdevMTU("0000:d8:00.0", 0) + Expect(err).ToNot(HaveOccurred()) + }) + It("should be able to configure mtu on a nic", func() { + dputilsLibMock.EXPECT().GetNetNames("0000:d8:00.0").Return([]string{"eno1"}, nil) + link := &netlink.GenericLink{LinkType: "PF", LinkAttrs: netlink.LinkAttrs{Name: "eno1"}} + netlinkLibMock.EXPECT().LinkByName("eno1").Return(link, nil) + netlinkLibMock.EXPECT().LinkSetMTU(link, 1500).Return(nil) + err := n.SetNetdevMTU("0000:d8:00.0", 1500) + Expect(err).ToNot(HaveOccurred()) + }) + }) + Context("GetNetDevMac", func() { + It("should return empty mac address if not able to get interface by link", func() { + netlinkLibMock.EXPECT().LinkByName("eno1").Return(nil, fmt.Errorf("failed to find intreface")) + mac := n.GetNetDevMac("eno1") + Expect(mac).To(BeEmpty()) + }) + It("should return interface mac address", func() { + link := &netlink.GenericLink{LinkType: "PF", LinkAttrs: netlink.LinkAttrs{Name: "eno1", HardwareAddr: net.HardwareAddr{0x00, 0x00, 0x5e, 0x00, 0x53, 0x01}}} + netlinkLibMock.EXPECT().LinkByName("eno1").Return(link, nil) + mac := n.GetNetDevMac("eno1") + Expect(mac).To(Equal("00:00:5e:00:53:01")) + }) + }) + Context("GetNetDevLinkSpeed", func() { + It("should return empty string if the speed file doesn't exist", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{ + "/sys/class/net/eno1"}, + }) + Expect(n.GetNetDevLinkSpeed("eno1")).To(BeEmpty()) + }) + It("should return the interface speed from sysfs", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{ + "/sys/class/net/eno1"}, + Files: map[string][]byte{ + "/sys/class/net/eno1/speed": []byte("1000"), + }, + }) + Expect(n.GetNetDevLinkSpeed("eno1")).To(Equal("1000 Mb/s")) + }) + }) + Context("GetNetDevLinkAdminState", func() { + It("should return empty state if device name is empty", func() { + state := n.GetNetDevLinkAdminState("") + Expect(state).To(BeEmpty()) + }) + It("should return empty state if not able to get interface by name", func() { + netlinkLibMock.EXPECT().LinkByName("eno1").Return(nil, fmt.Errorf("failed to find intreface")) + state := n.GetNetDevLinkAdminState("eno1") + Expect(state).To(BeEmpty()) + }) + It("should return link up", func() { + link := &netlink.GenericLink{LinkType: "PF", LinkAttrs: netlink.LinkAttrs{Name: "eno1"}} + netlinkLibMock.EXPECT().LinkByName("eno1").Return(link, nil) + netlinkLibMock.EXPECT().IsLinkAdminStateUp(link).Return(true) + state := n.GetNetDevLinkAdminState("eno1") + Expect(state).To(Equal(consts.LinkAdminStateUp)) + }) + It("should return link down", func() { + link := &netlink.GenericLink{LinkType: "PF", LinkAttrs: netlink.LinkAttrs{Name: "eno1"}} + netlinkLibMock.EXPECT().LinkByName("eno1").Return(link, nil) + netlinkLibMock.EXPECT().IsLinkAdminStateUp(link).Return(false) + state := n.GetNetDevLinkAdminState("eno1") + Expect(state).To(Equal(consts.LinkAdminStateDown)) + }) + }) + Context("SetRDMASubsystem", func() { + It("Should set RDMA Subsystem shared mode", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/host/etc/modprobe.d"}, + Files: map[string][]byte{ + "/host/etc/modprobe.d/sriov_network_operator_modules_config.conf": {}, + }, + }) + Expect(n.SetRDMASubsystem("shared")).NotTo(HaveOccurred()) + helpers.GinkgoAssertFileContentsEquals("/host/etc/modprobe.d/sriov_network_operator_modules_config.conf", "# This file is managed by sriov-network-operator do not edit.\noptions ib_core netns_mode=1\n") + }) + It("Should set RDMA Subsystem exclusive mode", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/host/etc/modprobe.d"}, + Files: map[string][]byte{ + "/host/etc/modprobe.d/sriov_network_operator_modules_config.conf": {}, + }, + }) + Expect(n.SetRDMASubsystem("exclusive")).NotTo(HaveOccurred()) + helpers.GinkgoAssertFileContentsEquals("/host/etc/modprobe.d/sriov_network_operator_modules_config.conf", "# This file is managed by sriov-network-operator do not edit.\noptions ib_core netns_mode=0\n") + }) + + It("should remove the ib_core file if the mode is empty", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/host/etc/modprobe.d"}, + Files: map[string][]byte{ + "/host/etc/modprobe.d/sriov_network_operator_modules_config.conf": {}, + }, + }) + Expect(n.SetRDMASubsystem("")).NotTo(HaveOccurred()) + helpers.GinkgoAssertFileDoesNotExist("/host/etc/modprobe.d/sriov_network_operator_modules_config.conf") + }) + + It("should not return error if the files doesn't exist", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/host/etc/modprobe.d"}, + }) + Expect(n.SetRDMASubsystem("")).NotTo(HaveOccurred()) + }) + }) }) diff --git a/pkg/host/internal/sriov/sriov.go b/pkg/host/internal/sriov/sriov.go index 379cf6a70..5a0733fcf 100644 --- a/pkg/host/internal/sriov/sriov.go +++ b/pkg/host/internal/sriov/sriov.go @@ -122,24 +122,30 @@ func (s *sriov) ResetSriovDevice(ifaceStatus sriovnetworkv1.InterfaceExt) error return nil } -func (s *sriov) getVfInfo(vfAddr string, pfName string, eswitchMode string, devices []*ghw.PCIDevice) sriovnetworkv1.VirtualFunction { +func (s *sriov) getVfInfo(vfAddr string, pfAddr string, pfName string, eswitchMode string, devices []*ghw.PCIDevice) sriovnetworkv1.VirtualFunction { driver, err := s.dputilsLib.GetDriverName(vfAddr) if err != nil { log.Log.Error(err, "getVfInfo(): unable to parse device driver", "device", vfAddr) } - id, err := s.dputilsLib.GetVFID(vfAddr) + vfid, err := s.dputilsLib.GetVFID(vfAddr) if err != nil { log.Log.Error(err, "getVfInfo(): unable to get VF index", "device", vfAddr) } + guid, err := s.infinibandHelper.GetVfGUID(vfAddr, pfAddr, vfid) + if err != nil { + log.Log.Error(err, "GetVfGUID(): unable to get VF GUID", "device", vfAddr) + } + vf := sriovnetworkv1.VirtualFunction{ PciAddress: vfAddr, Driver: driver, - VfID: id, + VfID: vfid, VdpaType: s.vdpaHelper.DiscoverVDPAType(vfAddr), + GUID: guid.String(), } if eswitchMode == sriovnetworkv1.ESwithModeSwitchDev { - repName, err := s.sriovnetLib.GetVfRepresentor(pfName, id) + repName, err := s.sriovnetLib.GetVfRepresentor(pfName, vfid) if err != nil { log.Log.Error(err, "getVfInfo(): failed to get VF representor name", "device", vfAddr) } else { @@ -157,7 +163,6 @@ func (s *sriov) getVfInfo(vfAddr string, pfName string, eswitchMode string, devi vf.Mac = link.Attrs().HardwareAddr.String() } } - vf.GUID = s.networkHelper.GetNetDevNodeGUID(vfAddr) for _, device := range devices { if vfAddr == device.Address { @@ -176,7 +181,7 @@ func (s *sriov) VFIsReady(pciAddr string) (netlink.Link, error) { err = wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { vfIndex, err := s.networkHelper.GetInterfaceIndex(pciAddr) if err != nil { - log.Log.Error(err, "VFIsReady(): invalid index number") + log.Log.Error(err, "VFIsReady(): invalid index number", "device", pciAddr) return false, nil } vfLink, err = s.netlinkLib.LinkByIndex(vfIndex) @@ -217,7 +222,7 @@ func (s *sriov) DiscoverSriovDevices(storeManager store.ManagerInterface) ([]sri return nil, fmt.Errorf("DiscoverSriovDevices(): error getting PCI info: %v", err) } - devices := pci.ListDevices() + devices := pci.Devices if len(devices) == 0 { return nil, fmt.Errorf("DiscoverSriovDevices(): could not retrieve PCI devices") } @@ -300,7 +305,7 @@ func (s *sriov) DiscoverSriovDevices(storeManager store.ManagerInterface) ([]sri continue } for _, vf := range vfs { - instance := s.getVfInfo(vf, pfNetName, iface.EswitchMode, devices) + instance := s.getVfInfo(vf, iface.PciAddress, pfNetName, iface.EswitchMode, devices) iface.VFs = append(iface.VFs, instance) } } @@ -376,12 +381,21 @@ func (s *sriov) configureHWOptionsForSwitchdev(iface *sriovnetworkv1.Interface) log.Log.Error(err, "configureHWOptionsForSwitchdev(): fail to read current flow steering mode for the device", "device", iface.PciAddress) return err } + if currentFlowSteeringMode == "" { + log.Log.V(2).Info("configureHWOptionsForSwitchdev(): can't detect current flow_steering_mode mode for the device, skip", + "device", iface.PciAddress) + return nil + } if currentFlowSteeringMode == desiredFlowSteeringMode { return nil } // flow steering mode can be changed only when NIC is in legacy mode if s.GetNicSriovMode(iface.PciAddress) != sriovnetworkv1.ESwithModeLegacy { - s.setEswitchModeAndNumVFs(iface.PciAddress, sriovnetworkv1.ESwithModeLegacy, 0) + err = s.setEswitchModeAndNumVFs(iface.PciAddress, sriovnetworkv1.ESwithModeLegacy, 0) + if err != nil { + log.Log.Error(err, "falied to switch Eswitch mode to legacy and reset number of vfs to 0") + return err + } } if err := s.networkHelper.SetDevlinkDeviceParam(iface.PciAddress, "flow_steering_mode", desiredFlowSteeringMode); err != nil { if errors.Is(err, syscall.ENOTSUP) { @@ -479,7 +493,8 @@ func (s *sriov) configSriovVFDevices(iface *sriovnetworkv1.Interface) error { if err := s.infinibandHelper.ConfigureVfGUID(addr, iface.PciAddress, vfID, pfLink); err != nil { return err } - if err := s.kernelHelper.Unbind(iface.PciAddress); err != nil { + + if err := s.kernelHelper.Unbind(addr); err != nil { return err } } else { @@ -653,7 +668,7 @@ func (s *sriov) getConfigureAndReset(storeManager store.ManagerInterface, interf } } - if !configured && ifaceStatus.NumVfs > 0 { + if !configured { toBeResetted = append(toBeResetted, ifaceStatus) } } @@ -819,8 +834,10 @@ func (s *sriov) checkForConfigAndReset(ifaceStatus sriovnetworkv1.InterfaceExt, return err } - if err = s.ResetSriovDevice(ifaceStatus); err != nil { - return err + if ifaceStatus.NumVfs > 0 { + if err = s.ResetSriovDevice(ifaceStatus); err != nil { + return err + } } // remove pf status from host diff --git a/pkg/host/internal/sriov/sriov_test.go b/pkg/host/internal/sriov/sriov_test.go index f30e93773..4c61117bd 100644 --- a/pkg/host/internal/sriov/sriov_test.go +++ b/pkg/host/internal/sriov/sriov_test.go @@ -6,10 +6,10 @@ import ( "strconv" "syscall" - "github.com/golang/mock/gomock" - "github.com/jaypipes/ghw" + "github.com/jaypipes/ghw/pkg/pci" "github.com/jaypipes/pcidb" "github.com/vishvananda/netlink" + "go.uber.org/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -22,6 +22,7 @@ import ( hostMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/mock" hostStoreMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/store/mock" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/fakefilesystem" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/helpers" ) @@ -57,12 +58,7 @@ var _ = Describe("SRIOV", func() { }) Context("DiscoverSriovDevices", func() { - var ( - ghwInfoMock *ghwMockPkg.MockInfo - ) BeforeEach(func() { - ghwInfoMock = ghwMockPkg.NewMockInfo(testCtrl) - ghwLibMock.EXPECT().PCI().Return(ghwInfoMock, nil) origNicMap := sriovnetworkv1.NicIDMap sriovnetworkv1.InitNicIDMapFromList([]string{ "15b3 101d 101e", @@ -73,7 +69,7 @@ var _ = Describe("SRIOV", func() { }) It("discovered", func() { - ghwInfoMock.EXPECT().ListDevices().Return(getTestPCIDevices()) + ghwLibMock.EXPECT().PCI().Return(getTestPCIDevices(), nil) dputilsLibMock.EXPECT().IsSriovVF("0000:d8:00.0").Return(false) dputilsLibMock.EXPECT().IsSriovVF("0000:d8:00.2").Return(true) dputilsLibMock.EXPECT().IsSriovVF("0000:3b:00.0").Return(false) @@ -92,6 +88,7 @@ var _ = Describe("SRIOV", func() { hostMock.EXPECT().GetNetDevLinkSpeed("enp216s0f0np0").Return("100000 Mb/s") hostMock.EXPECT().GetNetDevLinkAdminState("enp216s0f0np0").Return("up") hostMock.EXPECT().GetNetDevNodeGUID("0000:d8:00.2").Return("guid1") + hostMock.EXPECT().GetVfGUID("enp216s0f0np0", "0000:d8:00.0", 0).Return(net.HardwareAddr{}, nil) storeManagerMode.EXPECT().LoadPfsStatus("0000:d8:00.0").Return(nil, false, nil) dputilsLibMock.EXPECT().IsSriovPF("0000:d8:00.0").Return(true) @@ -246,8 +243,13 @@ var _ = Describe("SRIOV", func() { hostMock.EXPECT().HasDriver("0000:d8:00.3").Return(true, "vfio-pci").Times(2) hostMock.EXPECT().UnbindDriverIfNeeded("0000:d8:00.3", false).Return(nil) hostMock.EXPECT().BindDpdkDriver("0000:d8:00.3", "vfio-pci").Return(nil) + hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.1").Return(nil) + hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.1").Return(nil) + hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.1").Return(nil) storeManagerMode.EXPECT().SaveLastPfAppliedStatus(gomock.Any()).Return(nil) + storeManagerMode.EXPECT().RemovePfAppliedStatus(gomock.Any()).Return(nil) + storeManagerMode.EXPECT().LoadPfsStatus("0000:d8:00.1").Return(&sriovnetworkv1.Interface{ExternallyManaged: false}, true, nil) Expect(s.ConfigSriovInterfaces(storeManagerMode, []sriovnetworkv1.Interface{{ @@ -275,6 +277,140 @@ var _ = Describe("SRIOV", func() { false)).NotTo(HaveOccurred()) helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs", "2") }) + + It("should configure in parallel", func() { + vars.ParallelNicConfig = true + defer func() { + vars.ParallelNicConfig = false + }() + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/sys/bus/pci/devices/0000:d8:00.0", "/sys/bus/pci/devices/0000:d8:00.1"}, + Files: map[string][]byte{"/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs": {}, + "/sys/bus/pci/devices/0000:d8:00.1/sriov_numvfs": {}}, + }) + + dputilsLibMock.EXPECT().GetSriovVFcapacity("0000:d8:00.0").Return(2) + dputilsLibMock.EXPECT().GetVFconfigured("0000:d8:00.0").Return(0) + dputilsLibMock.EXPECT().GetDriverName("0000:d8:00.0").Return("mlx5_core", nil) + netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return(&netlink.DevlinkDevice{ + Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, nil) + hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.0").Return(nil) + hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.0").Return(nil) + hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.0").Return(nil) + hostMock.EXPECT().AddDisableNMUdevRule("0000:d8:00.0").Return(nil) + dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2", "0000:d8:00.3"}, nil) + pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + netlinkLibMock.EXPECT().LinkByName("enp216s0f0np0").Return(pfLinkMock, nil).Times(3) + pfLinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Flags: 0, EncapType: "ether"}) + netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false) + netlinkLibMock.EXPECT().LinkSetUp(pfLinkMock).Return(nil) + + dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(2) + hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(false, "") + hostMock.EXPECT().BindDefaultDriver("0000:d8:00.2").Return(nil) + hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(true, "test") + hostMock.EXPECT().UnbindDriverIfNeeded("0000:d8:00.2", true).Return(nil) + hostMock.EXPECT().BindDefaultDriver("0000:d8:00.2").Return(nil) + hostMock.EXPECT().SetNetdevMTU("0000:d8:00.2", 2000).Return(nil) + hostMock.EXPECT().GetInterfaceIndex("0000:d8:00.2").Return(42, nil) + vf0LinkMock := netlinkMockPkg.NewMockLink(testCtrl) + vf0Mac, _ := net.ParseMAC("02:42:19:51:2f:af") + vf0LinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Name: "enp216s0f0_0", HardwareAddr: vf0Mac}).AnyTimes() + netlinkLibMock.EXPECT().LinkByIndex(42).Return(vf0LinkMock, nil) + netlinkLibMock.EXPECT().LinkSetVfHardwareAddr(vf0LinkMock, 0, vf0Mac).Return(nil) + + dputilsLibMock.EXPECT().GetVFID("0000:d8:00.3").Return(1, nil) + hostMock.EXPECT().HasDriver("0000:d8:00.3").Return(true, "vfio-pci").Times(2) + hostMock.EXPECT().UnbindDriverIfNeeded("0000:d8:00.3", false).Return(nil) + hostMock.EXPECT().BindDpdkDriver("0000:d8:00.3", "vfio-pci").Return(nil) + + dputilsLibMock.EXPECT().GetSriovVFcapacity("0000:d8:00.1").Return(2) + dputilsLibMock.EXPECT().GetVFconfigured("0000:d8:00.1").Return(0) + dputilsLibMock.EXPECT().GetDriverName("0000:d8:00.1").Return("mlx5_core", nil) + netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.1").Return(&netlink.DevlinkDevice{ + Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, nil) + hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.1").Return(nil) + hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.1").Return(nil) + hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.1").Return(nil) + hostMock.EXPECT().AddDisableNMUdevRule("0000:d8:00.1").Return(nil) + dputilsLibMock.EXPECT().GetVFList("0000:d8:00.1").Return([]string{"0000:d8:00.4", "0000:d8:00.5"}, nil) + pf1LinkMock := netlinkMockPkg.NewMockLink(testCtrl) + netlinkLibMock.EXPECT().LinkByName("enp216s0f0np1").Return(pf1LinkMock, nil).Times(3) + pf1LinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Flags: 0, EncapType: "ether"}) + netlinkLibMock.EXPECT().IsLinkAdminStateUp(pf1LinkMock).Return(false) + netlinkLibMock.EXPECT().LinkSetUp(pf1LinkMock).Return(nil) + + dputilsLibMock.EXPECT().GetVFID("0000:d8:00.4").Return(0, nil).Times(2) + hostMock.EXPECT().HasDriver("0000:d8:00.4").Return(false, "") + hostMock.EXPECT().BindDefaultDriver("0000:d8:00.4").Return(nil) + hostMock.EXPECT().HasDriver("0000:d8:00.4").Return(true, "test") + hostMock.EXPECT().UnbindDriverIfNeeded("0000:d8:00.4", true).Return(nil) + hostMock.EXPECT().BindDefaultDriver("0000:d8:00.4").Return(nil) + hostMock.EXPECT().SetNetdevMTU("0000:d8:00.4", 2000).Return(nil) + hostMock.EXPECT().GetInterfaceIndex("0000:d8:00.4").Return(43, nil) + pf1vf0LinkMock := netlinkMockPkg.NewMockLink(testCtrl) + pf1vf0Mac, _ := net.ParseMAC("02:42:19:51:2f:bf") + pf1vf0LinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Name: "enp216s0f1_0", HardwareAddr: pf1vf0Mac}).AnyTimes() + netlinkLibMock.EXPECT().LinkByIndex(43).Return(pf1vf0LinkMock, nil) + netlinkLibMock.EXPECT().LinkSetVfHardwareAddr(vf0LinkMock, 0, pf1vf0Mac).Return(nil) + + dputilsLibMock.EXPECT().GetVFID("0000:d8:00.5").Return(1, nil) + hostMock.EXPECT().HasDriver("0000:d8:00.5").Return(true, "vfio-pci").Times(2) + hostMock.EXPECT().UnbindDriverIfNeeded("0000:d8:00.5", false).Return(nil) + hostMock.EXPECT().BindDpdkDriver("0000:d8:00.5", "vfio-pci").Return(nil) + + storeManagerMode.EXPECT().SaveLastPfAppliedStatus(gomock.Any()).Return(nil).Times(2) + + defer GinkgoRecover() + Expect(s.ConfigSriovInterfaces(storeManagerMode, + []sriovnetworkv1.Interface{{ + Name: "enp216s0f0np0", + PciAddress: "0000:d8:00.0", + NumVfs: 2, + VfGroups: []sriovnetworkv1.VfGroup{ + { + VfRange: "0-0", + ResourceName: "test-resource0", + PolicyName: "test-policy0", + Mtu: 2000, + IsRdma: true, + }, + { + VfRange: "1-1", + ResourceName: "test-resource1", + PolicyName: "test-policy1", + Mtu: 1600, + IsRdma: false, + DeviceType: "vfio-pci", + }}, + }, + { + Name: "enp216s0f0np1", + PciAddress: "0000:d8:00.1", + NumVfs: 2, + VfGroups: []sriovnetworkv1.VfGroup{ + { + VfRange: "0-0", + ResourceName: "test-resource2", + PolicyName: "test-policy2", + Mtu: 2000, + IsRdma: true, + }, + { + VfRange: "1-1", + ResourceName: "test-resource3", + PolicyName: "test-policy3", + Mtu: 1600, + IsRdma: false, + DeviceType: "vfio-pci", + }}, + }}, + []sriovnetworkv1.InterfaceExt{{PciAddress: "0000:d8:00.0"}, {PciAddress: "0000:d8:00.1"}}, + false)).NotTo(HaveOccurred()) + helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs", "2") + helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/devices/0000:d8:00.1/sriov_numvfs", "2") + }) + It("should configure IB", func() { helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ Dirs: []string{"/sys/bus/pci/devices/0000:d8:00.0"}, @@ -396,6 +532,215 @@ var _ = Describe("SRIOV", func() { helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs", "1") }) + It("should configure switchdev even if steering mode is not detected", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/sys/bus/pci/devices/0000:d8:00.0"}, + Files: map[string][]byte{"/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs": {}}, + }) + + dputilsLibMock.EXPECT().GetSriovVFcapacity("0000:d8:00.0").Return(1) + dputilsLibMock.EXPECT().GetVFconfigured("0000:d8:00.0").Return(0) + dputilsLibMock.EXPECT().GetDriverName("0000:d8:00.0").Return("mlx5_core", nil) + hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.0").Return(nil) + hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.0").Return(nil) + hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.0").Return(nil) + hostMock.EXPECT().AddDisableNMUdevRule("0000:d8:00.0").Return(nil) + hostMock.EXPECT().AddPersistPFNameUdevRule("0000:d8:00.0", "enp216s0f0np0").Return(nil) + hostMock.EXPECT().EnableHwTcOffload("enp216s0f0np0").Return(nil) + hostMock.EXPECT().GetDevlinkDeviceParam("0000:d8:00.0", "flow_steering_mode").Return("", nil) + dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2"}, nil).Times(2) + pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + netlinkLibMock.EXPECT().LinkByName("enp216s0f0np0").Return(pfLinkMock, nil).Times(2) + netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false) + netlinkLibMock.EXPECT().LinkSetUp(pfLinkMock).Return(nil) + netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return(&netlink.DevlinkDevice{ + Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, nil).Times(2) + netlinkLibMock.EXPECT().DevLinkSetEswitchMode(gomock.Any(), "switchdev").Return(nil) + + dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(2) + hostMock.EXPECT().Unbind("0000:d8:00.2").Return(nil) + hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(false, "") + hostMock.EXPECT().BindDefaultDriver("0000:d8:00.2").Return(nil) + hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(true, "test") + hostMock.EXPECT().UnbindDriverIfNeeded("0000:d8:00.2", true).Return(nil) + hostMock.EXPECT().BindDefaultDriver("0000:d8:00.2").Return(nil) + hostMock.EXPECT().SetNetdevMTU("0000:d8:00.2", 2000).Return(nil) + hostMock.EXPECT().GetInterfaceIndex("0000:d8:00.2").Return(42, nil).AnyTimes() + vf0LinkMock := netlinkMockPkg.NewMockLink(testCtrl) + vf0Mac, _ := net.ParseMAC("02:42:19:51:2f:af") + vf0LinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Name: "enp216s0f0_0", HardwareAddr: vf0Mac}) + netlinkLibMock.EXPECT().LinkByIndex(42).Return(vf0LinkMock, nil).AnyTimes() + netlinkLibMock.EXPECT().LinkSetVfHardwareAddr(vf0LinkMock, 0, vf0Mac).Return(nil) + hostMock.EXPECT().GetPhysPortName("enp216s0f0np0").Return("p0", nil) + hostMock.EXPECT().GetPhysSwitchID("enp216s0f0np0").Return("7cfe90ff2cc0", nil) + hostMock.EXPECT().AddVfRepresentorUdevRule("0000:d8:00.0", "enp216s0f0np0", "7cfe90ff2cc0", "p0").Return(nil) + hostMock.EXPECT().CreateVDPADevice("0000:d8:00.2", "vhost_vdpa") + hostMock.EXPECT().LoadUdevRules().Return(nil) + + storeManagerMode.EXPECT().SaveLastPfAppliedStatus(gomock.Any()).Return(nil) + + Expect(s.ConfigSriovInterfaces(storeManagerMode, + []sriovnetworkv1.Interface{{ + Name: "enp216s0f0np0", + PciAddress: "0000:d8:00.0", + NumVfs: 1, + LinkType: "ETH", + EswitchMode: "switchdev", + VfGroups: []sriovnetworkv1.VfGroup{ + { + VfRange: "0-0", + ResourceName: "test-resource0", + PolicyName: "test-policy0", + Mtu: 2000, + IsRdma: true, + VdpaType: "vhost_vdpa", + }}, + }}, + []sriovnetworkv1.InterfaceExt{{PciAddress: "0000:d8:00.0"}}, + false)).NotTo(HaveOccurred()) + helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs", "1") + }) + + It("should configure switchdev even if steering mode is already smfs", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/sys/bus/pci/devices/0000:d8:00.0"}, + Files: map[string][]byte{"/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs": {}}, + }) + + dputilsLibMock.EXPECT().GetSriovVFcapacity("0000:d8:00.0").Return(1) + dputilsLibMock.EXPECT().GetVFconfigured("0000:d8:00.0").Return(0) + dputilsLibMock.EXPECT().GetDriverName("0000:d8:00.0").Return("mlx5_core", nil) + hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.0").Return(nil) + hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.0").Return(nil) + hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.0").Return(nil) + hostMock.EXPECT().AddDisableNMUdevRule("0000:d8:00.0").Return(nil) + hostMock.EXPECT().AddPersistPFNameUdevRule("0000:d8:00.0", "enp216s0f0np0").Return(nil) + hostMock.EXPECT().EnableHwTcOffload("enp216s0f0np0").Return(nil) + hostMock.EXPECT().GetDevlinkDeviceParam("0000:d8:00.0", "flow_steering_mode").Return("smfs", nil) + dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2"}, nil).Times(2) + pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + netlinkLibMock.EXPECT().LinkByName("enp216s0f0np0").Return(pfLinkMock, nil).Times(2) + netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false) + netlinkLibMock.EXPECT().LinkSetUp(pfLinkMock).Return(nil) + netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return(&netlink.DevlinkDevice{ + Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, nil).Times(2) + netlinkLibMock.EXPECT().DevLinkSetEswitchMode(gomock.Any(), "switchdev").Return(nil) + + dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(2) + hostMock.EXPECT().Unbind("0000:d8:00.2").Return(nil) + hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(false, "") + hostMock.EXPECT().BindDefaultDriver("0000:d8:00.2").Return(nil) + hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(true, "test") + hostMock.EXPECT().UnbindDriverIfNeeded("0000:d8:00.2", true).Return(nil) + hostMock.EXPECT().BindDefaultDriver("0000:d8:00.2").Return(nil) + hostMock.EXPECT().SetNetdevMTU("0000:d8:00.2", 2000).Return(nil) + hostMock.EXPECT().GetInterfaceIndex("0000:d8:00.2").Return(42, nil).AnyTimes() + vf0LinkMock := netlinkMockPkg.NewMockLink(testCtrl) + vf0Mac, _ := net.ParseMAC("02:42:19:51:2f:af") + vf0LinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Name: "enp216s0f0_0", HardwareAddr: vf0Mac}) + netlinkLibMock.EXPECT().LinkByIndex(42).Return(vf0LinkMock, nil).AnyTimes() + netlinkLibMock.EXPECT().LinkSetVfHardwareAddr(vf0LinkMock, 0, vf0Mac).Return(nil) + hostMock.EXPECT().GetPhysPortName("enp216s0f0np0").Return("p0", nil) + hostMock.EXPECT().GetPhysSwitchID("enp216s0f0np0").Return("7cfe90ff2cc0", nil) + hostMock.EXPECT().AddVfRepresentorUdevRule("0000:d8:00.0", "enp216s0f0np0", "7cfe90ff2cc0", "p0").Return(nil) + hostMock.EXPECT().CreateVDPADevice("0000:d8:00.2", "vhost_vdpa") + hostMock.EXPECT().LoadUdevRules().Return(nil) + + storeManagerMode.EXPECT().SaveLastPfAppliedStatus(gomock.Any()).Return(nil) + + Expect(s.ConfigSriovInterfaces(storeManagerMode, + []sriovnetworkv1.Interface{{ + Name: "enp216s0f0np0", + PciAddress: "0000:d8:00.0", + NumVfs: 1, + LinkType: "ETH", + EswitchMode: "switchdev", + VfGroups: []sriovnetworkv1.VfGroup{ + { + VfRange: "0-0", + ResourceName: "test-resource0", + PolicyName: "test-policy0", + Mtu: 2000, + IsRdma: true, + VdpaType: "vhost_vdpa", + }}, + }}, + []sriovnetworkv1.InterfaceExt{{PciAddress: "0000:d8:00.0"}}, + false)).NotTo(HaveOccurred()) + helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs", "1") + }) + + It("should configure configure swtichdev by switching back to legacy mode and configure smfs", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/sys/bus/pci/devices/0000:d8:00.0"}, + Files: map[string][]byte{"/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs": {}}, + }) + + dputilsLibMock.EXPECT().GetSriovVFcapacity("0000:d8:00.0").Return(1) + dputilsLibMock.EXPECT().GetVFconfigured("0000:d8:00.0").Return(0) + dputilsLibMock.EXPECT().GetDriverName("0000:d8:00.0").Return("mlx5_core", nil).Times(2) + hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.0").Return(nil) + hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.0").Return(nil) + hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.0").Return(nil) + hostMock.EXPECT().AddDisableNMUdevRule("0000:d8:00.0").Return(nil) + hostMock.EXPECT().AddPersistPFNameUdevRule("0000:d8:00.0", "enp216s0f0np0").Return(nil) + hostMock.EXPECT().EnableHwTcOffload("enp216s0f0np0").Return(nil) + hostMock.EXPECT().GetDevlinkDeviceParam("0000:d8:00.0", "flow_steering_mode").Return("test", nil) + dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2"}, nil).Times(4) + pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + netlinkLibMock.EXPECT().LinkByName("enp216s0f0np0").Return(pfLinkMock, nil).Times(2) + netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false) + netlinkLibMock.EXPECT().LinkSetUp(pfLinkMock).Return(nil) + netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return(&netlink.DevlinkDevice{ + Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: sriovnetworkv1.ESwithModeSwitchDev}}}, nil).Times(6) + netlinkLibMock.EXPECT().DevLinkSetEswitchMode(gomock.Any(), "legacy").Return(nil).Times(2) + netlinkLibMock.EXPECT().DevLinkSetEswitchMode(gomock.Any(), "switchdev").Return(nil) + hostMock.EXPECT().SetDevlinkDeviceParam("0000:d8:00.0", "flow_steering_mode", "smfs").Return(nil) + + dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(2) + hostMock.EXPECT().Unbind("0000:d8:00.2").Return(nil).Times(3) + hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(false, "") + hostMock.EXPECT().BindDefaultDriver("0000:d8:00.2").Return(nil) + hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(true, "test") + hostMock.EXPECT().UnbindDriverIfNeeded("0000:d8:00.2", true).Return(nil) + hostMock.EXPECT().BindDefaultDriver("0000:d8:00.2").Return(nil) + hostMock.EXPECT().SetNetdevMTU("0000:d8:00.2", 2000).Return(nil) + hostMock.EXPECT().GetInterfaceIndex("0000:d8:00.2").Return(42, nil).AnyTimes() + vf0LinkMock := netlinkMockPkg.NewMockLink(testCtrl) + vf0Mac, _ := net.ParseMAC("02:42:19:51:2f:af") + vf0LinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Name: "enp216s0f0_0", HardwareAddr: vf0Mac}) + netlinkLibMock.EXPECT().LinkByIndex(42).Return(vf0LinkMock, nil).AnyTimes() + netlinkLibMock.EXPECT().LinkSetVfHardwareAddr(vf0LinkMock, 0, vf0Mac).Return(nil) + hostMock.EXPECT().GetPhysPortName("enp216s0f0np0").Return("p0", nil) + hostMock.EXPECT().GetPhysSwitchID("enp216s0f0np0").Return("7cfe90ff2cc0", nil) + hostMock.EXPECT().AddVfRepresentorUdevRule("0000:d8:00.0", "enp216s0f0np0", "7cfe90ff2cc0", "p0").Return(nil) + hostMock.EXPECT().CreateVDPADevice("0000:d8:00.2", "vhost_vdpa") + hostMock.EXPECT().LoadUdevRules().Return(nil) + + storeManagerMode.EXPECT().SaveLastPfAppliedStatus(gomock.Any()).Return(nil) + + Expect(s.ConfigSriovInterfaces(storeManagerMode, + []sriovnetworkv1.Interface{{ + Name: "enp216s0f0np0", + PciAddress: "0000:d8:00.0", + NumVfs: 1, + LinkType: "ETH", + EswitchMode: "switchdev", + VfGroups: []sriovnetworkv1.VfGroup{ + { + VfRange: "0-0", + ResourceName: "test-resource0", + PolicyName: "test-policy0", + Mtu: 2000, + IsRdma: true, + VdpaType: "vhost_vdpa", + }}, + }}, + []sriovnetworkv1.InterfaceExt{{PciAddress: "0000:d8:00.0"}}, + false)).NotTo(HaveOccurred()) + helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs", "1") + }) + It("should configure switchdev on ice driver", func() { helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ Dirs: []string{"/sys/bus/pci/devices/0000:d8:00.0"}, @@ -412,7 +757,7 @@ var _ = Describe("SRIOV", func() { hostMock.EXPECT().AddPersistPFNameUdevRule("0000:d8:00.0", "enp216s0f0np0").Return(nil) hostMock.EXPECT().EnableHwTcOffload("enp216s0f0np0").Return(nil) hostMock.EXPECT().GetDevlinkDeviceParam("0000:d8:00.0", "flow_steering_mode").Return("", syscall.EINVAL) - dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2"}, nil).Times(1) + dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2"}, nil) pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) netlinkLibMock.EXPECT().LinkByName("enp216s0f0np0").Return(pfLinkMock, nil).Times(2) netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false) @@ -545,6 +890,68 @@ var _ = Describe("SRIOV", func() { }}, false)).NotTo(HaveOccurred()) helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs", "0") }) + + It("should reset devices in parallel", func() { + vars.ParallelNicConfig = true + defer func() { + vars.ParallelNicConfig = false + }() + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/sys/bus/pci/devices/0000:d8:00.0", "/sys/bus/pci/devices/0000:d8:00.1"}, + Files: map[string][]byte{"/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs": {}, + "/sys/bus/pci/devices/0000:d8:00.1/sriov_numvfs": {}}, + }) + + storeManagerMode.EXPECT().LoadPfsStatus("0000:d8:00.0").Return(&sriovnetworkv1.Interface{ + Name: "enp216s0f0np0", + PciAddress: "0000:d8:00.0", + NumVfs: 2, + }, true, nil) + storeManagerMode.EXPECT().RemovePfAppliedStatus("0000:d8:00.0").Return(nil) + netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return( + &netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, + nil) + hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.0").Return(nil) + hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.0").Return(nil) + hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.0").Return(nil) + dputilsLibMock.EXPECT().GetDriverName("0000:d8:00.0").Return("mlx5_core", nil) + hostMock.EXPECT().SetNetdevMTU("0000:d8:00.0", 1500).Return(nil) + + storeManagerMode.EXPECT().LoadPfsStatus("0000:d8:00.1").Return(&sriovnetworkv1.Interface{ + Name: "enp216s0f0np0", + PciAddress: "0000:d8:00.0", + NumVfs: 2, + }, true, nil) + storeManagerMode.EXPECT().RemovePfAppliedStatus("0000:d8:00.1").Return(nil) + netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.1").Return( + &netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, + nil) + hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.1").Return(nil) + hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.1").Return(nil) + hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.1").Return(nil) + dputilsLibMock.EXPECT().GetDriverName("0000:d8:00.1").Return("mlx5_core", nil) + hostMock.EXPECT().SetNetdevMTU("0000:d8:00.1", 1500).Return(nil) + + Expect(s.ConfigSriovInterfaces(storeManagerMode, + []sriovnetworkv1.Interface{}, + []sriovnetworkv1.InterfaceExt{ + { + Name: "enp216s0f0np0", + PciAddress: "0000:d8:00.0", + LinkType: "ETH", + NumVfs: 2, + TotalVfs: 2, + }, + { + Name: "enp216s0f0np1", + PciAddress: "0000:d8:00.1", + LinkType: "ETH", + NumVfs: 2, + TotalVfs: 2, + }}, false)).NotTo(HaveOccurred()) + helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs", "0") + }) + It("reset device - skip external", func() { storeManagerMode.EXPECT().LoadPfsStatus("0000:d8:00.0").Return(&sriovnetworkv1.Interface{ Name: "enp216s0f0np0", @@ -563,6 +970,7 @@ var _ = Describe("SRIOV", func() { TotalVfs: 2, }}, false)).NotTo(HaveOccurred()) }) + It("should configure - skipVFConfiguration is true", func() { helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ Dirs: []string{"/sys/bus/pci/devices/0000:d8:00.0"}, @@ -628,91 +1036,94 @@ var _ = Describe("SRIOV", func() { }) }) -func getTestPCIDevices() []*ghw.PCIDevice { - return []*ghw.PCIDevice{{ - Driver: "mlx5_core", - Address: "0000:d8:00.0", - Vendor: &pcidb.Vendor{ - ID: "15b3", - Name: "Mellanox Technologies", - }, - Product: &pcidb.Product{ - ID: "101d", - Name: "MT2892 Family [ConnectX-6 Dx]", - }, - Revision: "0x00", - Subsystem: &pcidb.Product{ - ID: "0083", - Name: "unknown", - }, - Class: &pcidb.Class{ - ID: "02", - Name: "Network controller", - }, - Subclass: &pcidb.Subclass{ - ID: "00", - Name: "Ethernet controller", - }, - ProgrammingInterface: &pcidb.ProgrammingInterface{ - ID: "00", - Name: "unknonw", - }, - }, - { - Driver: "mlx5_core", - Address: "0000:d8:00.2", - Vendor: &pcidb.Vendor{ - ID: "15b3", - Name: "Mellanox Technologies", - }, - Product: &pcidb.Product{ - ID: "101e", - Name: "ConnectX Family mlx5Gen Virtual Function", - }, - Revision: "0x00", - Subsystem: &pcidb.Product{ - ID: "0083", - Name: "unknown", - }, - Class: &pcidb.Class{ - ID: "02", - Name: "Network controller", - }, - Subclass: &pcidb.Subclass{ - ID: "00", - Name: "Ethernet controller", +func getTestPCIDevices() *pci.Info { + return &pci.Info{ + Devices: []*pci.Device{ + { + Driver: "mlx5_core", + Address: "0000:d8:00.0", + Vendor: &pcidb.Vendor{ + ID: "15b3", + Name: "Mellanox Technologies", + }, + Product: &pcidb.Product{ + ID: "101d", + Name: "MT2892 Family [ConnectX-6 Dx]", + }, + Revision: "0x00", + Subsystem: &pcidb.Product{ + ID: "0083", + Name: "unknown", + }, + Class: &pcidb.Class{ + ID: "02", + Name: "Network controller", + }, + Subclass: &pcidb.Subclass{ + ID: "00", + Name: "Ethernet controller", + }, + ProgrammingInterface: &pcidb.ProgrammingInterface{ + ID: "00", + Name: "unknonw", + }, }, - ProgrammingInterface: &pcidb.ProgrammingInterface{ - ID: "00", - Name: "unknonw", + { + Driver: "mlx5_core", + Address: "0000:d8:00.2", + Vendor: &pcidb.Vendor{ + ID: "15b3", + Name: "Mellanox Technologies", + }, + Product: &pcidb.Product{ + ID: "101e", + Name: "ConnectX Family mlx5Gen Virtual Function", + }, + Revision: "0x00", + Subsystem: &pcidb.Product{ + ID: "0083", + Name: "unknown", + }, + Class: &pcidb.Class{ + ID: "02", + Name: "Network controller", + }, + Subclass: &pcidb.Subclass{ + ID: "00", + Name: "Ethernet controller", + }, + ProgrammingInterface: &pcidb.ProgrammingInterface{ + ID: "00", + Name: "unknonw", + }, }, - }, - { - Driver: "mlx5_core", - Address: "0000:3b:00.0", - Vendor: &pcidb.Vendor{ - ID: "15b3", - Name: "Mellanox Technologies", - }, - Product: &pcidb.Product{ - ID: "aaaa", // not supported - Name: "not supported", - }, - Class: &pcidb.Class{ - ID: "02", - Name: "Network controller", - }, - }, - { - Driver: "test", - Address: "0000:d7:16.5", - Vendor: &pcidb.Vendor{ - ID: "8086", - Name: "Intel Corporation", + { + Driver: "mlx5_core", + Address: "0000:3b:00.0", + Vendor: &pcidb.Vendor{ + ID: "15b3", + Name: "Mellanox Technologies", + }, + Product: &pcidb.Product{ + ID: "aaaa", // not supported + Name: "not supported", + }, + Class: &pcidb.Class{ + ID: "02", + Name: "Network controller", + }, }, - Class: &pcidb.Class{ - ID: "11", // not network device - Name: "Signal processing controller", + { + Driver: "test", + Address: "0000:d7:16.5", + Vendor: &pcidb.Vendor{ + ID: "8086", + Name: "Intel Corporation", + }, + Class: &pcidb.Class{ + ID: "11", // not network device + Name: "Signal processing controller", + }, }, }, } diff --git a/pkg/host/internal/systemd/systemd.go b/pkg/host/internal/systemd/systemd.go new file mode 100644 index 000000000..eb2b771f3 --- /dev/null +++ b/pkg/host/internal/systemd/systemd.go @@ -0,0 +1,336 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package systemd + +import ( + "bytes" + "fmt" + "os" + "strings" + + "gopkg.in/yaml.v3" + "sigs.k8s.io/controller-runtime/pkg/log" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" +) + +type systemd struct{} + +func New() types.SystemdInterface { + return &systemd{} +} + +// ReadConfFile reads the SR-IOV config file from the host +// Unmarshal YAML content into SriovConfig object +func (s *systemd) ReadConfFile() (spec *types.SriovConfig, err error) { + rawConfig, err := os.ReadFile(utils.GetHostExtensionPath(consts.SriovSystemdConfigPath)) + if err != nil { + return nil, err + } + + err = yaml.Unmarshal(rawConfig, &spec) + + return spec, err +} + +// WriteConfFile generates or updates a SriovNetwork configuration file based on the provided state. +// It creates the necessary directory structure if the file doesn't exist, +// reads the existing content to check for changes, and writes new content only when needed. +func (s *systemd) WriteConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (bool, error) { + newFile := false + sriovConfig := &types.SriovConfig{ + Spec: newState.Spec, + UnsupportedNics: vars.DevMode, + PlatformType: vars.PlatformType, + ManageSoftwareBridges: vars.ManageSoftwareBridges, + OVSDBSocketPath: vars.OVSDBSocketPath, + } + + _, err := os.Stat(utils.GetHostExtensionPath(consts.SriovSystemdConfigPath)) + if err != nil { + if os.IsNotExist(err) { + // Create the sriov-operator folder on the host if it doesn't exist + if _, err := os.Stat(utils.GetHostExtensionPath(consts.SriovConfBasePath)); os.IsNotExist(err) { + err = os.Mkdir(utils.GetHostExtensionPath(consts.SriovConfBasePath), 0777) + if err != nil { + log.Log.Error(err, "WriteConfFile(): fail to create sriov-operator folder", + "path", utils.GetHostExtensionPath(consts.SriovConfBasePath)) + return false, err + } + } + + log.Log.V(2).Info("WriteConfFile(): file not existed, create it", + "path", utils.GetHostExtensionPath(consts.SriovSystemdConfigPath)) + _, err = os.Create(utils.GetHostExtensionPath(consts.SriovSystemdConfigPath)) + if err != nil { + log.Log.Error(err, "WriteConfFile(): fail to create file") + return false, err + } + newFile = true + } else { + return false, err + } + } + + oldContent, err := os.ReadFile(utils.GetHostExtensionPath(consts.SriovSystemdConfigPath)) + if err != nil { + log.Log.Error(err, "WriteConfFile(): fail to read file", "path", utils.GetHostExtensionPath(consts.SriovSystemdConfigPath)) + return false, err + } + + oldContentObj := &types.SriovConfig{} + err = yaml.Unmarshal(oldContent, oldContentObj) + if err != nil { + log.Log.Error(err, "WriteConfFile(): fail to unmarshal old file") + return false, err + } + + var newContent []byte + newContent, err = yaml.Marshal(sriovConfig) + if err != nil { + log.Log.Error(err, "WriteConfFile(): fail to marshal sriov config") + return false, err + } + + if bytes.Equal(newContent, oldContent) { + log.Log.V(2).Info("WriteConfFile(): no update") + return false, nil + } + log.Log.V(2).Info("WriteConfFile(): old and new configuration are not equal", + "old", string(oldContent), "new", string(newContent)) + + log.Log.V(2).Info("WriteConfFile(): write content to file", + "content", newContent, "path", utils.GetHostExtensionPath(consts.SriovSystemdConfigPath)) + err = os.WriteFile(utils.GetHostExtensionPath(consts.SriovSystemdConfigPath), newContent, 0644) + if err != nil { + log.Log.Error(err, "WriteConfFile(): fail to write file") + return false, err + } + + // this will be used to mark the first time we create this file. + // this helps to avoid the first reboot after installation + if newFile && len(sriovConfig.Spec.Interfaces) == 0 { + log.Log.V(2).Info("WriteConfFile(): first file creation and no interfaces to configure returning reboot false") + return false, nil + } + + return true, nil +} + +// WriteSriovResult writes SR-IOV results to the host. +// It creates the file if it doesn't exist +func (s *systemd) WriteSriovResult(result *types.SriovResult) error { + _, err := os.Stat(utils.GetHostExtensionPath(consts.SriovSystemdResultPath)) + if err != nil { + if os.IsNotExist(err) { + // Create the sriov-operator folder on the host if it doesn't exist + if _, err := os.Stat(utils.GetHostExtensionPath(consts.SriovConfBasePath)); os.IsNotExist(err) { + err = os.Mkdir(utils.GetHostExtensionPath(consts.SriovConfBasePath), 0777) + if err != nil { + log.Log.Error(err, "WriteConfFile(): fail to create sriov-operator folder", + "path", utils.GetHostExtensionPath(consts.SriovConfBasePath)) + return err + } + } + + log.Log.V(2).Info("WriteSriovResult(): file not existed, create it") + _, err = os.Create(utils.GetHostExtensionPath(consts.SriovSystemdResultPath)) + if err != nil { + log.Log.Error(err, "WriteSriovResult(): failed to create sriov result file", "path", utils.GetHostExtensionPath(consts.SriovSystemdResultPath)) + return err + } + } else { + log.Log.Error(err, "WriteSriovResult(): failed to check sriov result file", "path", utils.GetHostExtensionPath(consts.SriovSystemdResultPath)) + return err + } + } + + out, err := yaml.Marshal(result) + if err != nil { + log.Log.Error(err, "WriteSriovResult(): failed to marshal sriov result") + return err + } + + log.Log.V(2).Info("WriteSriovResult(): write results", + "content", string(out), "path", utils.GetHostExtensionPath(consts.SriovSystemdResultPath)) + err = os.WriteFile(utils.GetHostExtensionPath(consts.SriovSystemdResultPath), out, 0644) + if err != nil { + log.Log.Error(err, "WriteSriovResult(): failed to write sriov result file", "path", utils.GetHostExtensionPath(consts.SriovSystemdResultPath)) + return err + } + + return nil +} + +// ReadSriovResult reads and parses the sriov result file from the host. +// The function first checks if the result file exists. If it doesn't, it returns nil with a success flag of false and no error. +// If the file exists, it reads its contents and attempts to unmarshal the YAML data into the SriovResult struct. +func (s *systemd) ReadSriovResult() (*types.SriovResult, error) { + _, err := os.Stat(utils.GetHostExtensionPath(consts.SriovSystemdResultPath)) + if err != nil { + if os.IsNotExist(err) { + log.Log.V(2).Info("ReadSriovResult(): file does not exist") + return nil, nil + } else { + log.Log.Error(err, "ReadSriovResult(): failed to check sriov result file", "path", utils.GetHostExtensionPath(consts.SriovSystemdResultPath)) + return nil, err + } + } + + rawConfig, err := os.ReadFile(utils.GetHostExtensionPath(consts.SriovSystemdResultPath)) + if err != nil { + log.Log.Error(err, "ReadSriovResult(): failed to read sriov result file", "path", utils.GetHostExtensionPath(consts.SriovSystemdResultPath)) + return nil, err + } + + result := &types.SriovResult{} + err = yaml.Unmarshal(rawConfig, &result) + if err != nil { + log.Log.Error(err, "ReadSriovResult(): failed to unmarshal sriov result file", "path", utils.GetHostExtensionPath(consts.SriovSystemdResultPath)) + return nil, err + } + return result, err +} + +// RemoveSriovResult: Removes the Sriov result file from the host. +func (s *systemd) RemoveSriovResult() error { + err := os.Remove(utils.GetHostExtensionPath(consts.SriovSystemdResultPath)) + if err != nil { + if os.IsNotExist(err) { + log.Log.V(2).Info("RemoveSriovResult(): result file not found") + return nil + } + log.Log.Error(err, "RemoveSriovResult(): failed to remove sriov result file", "path", utils.GetHostExtensionPath(consts.SriovSystemdResultPath)) + return err + } + log.Log.V(2).Info("RemoveSriovResult(): result file removed") + return nil +} + +// WriteSriovSupportedNics() creates or updates a file containing the list of supported SR-IOV NIC IDs +// If the file does not exist, it will create it +// It reads from sriovnetworkv1.NicIDMap to gather the list of NIC identifiers +func (s *systemd) WriteSriovSupportedNics() error { + _, err := os.Stat(utils.GetHostExtensionPath(consts.SriovSystemdSupportedNicPath)) + if err != nil { + if os.IsNotExist(err) { + // Create the sriov-operator folder on the host if it doesn't exist + if _, err := os.Stat(utils.GetHostExtensionPath(consts.SriovConfBasePath)); os.IsNotExist(err) { + err = os.Mkdir(utils.GetHostExtensionPath(consts.SriovConfBasePath), 0777) + if err != nil { + log.Log.Error(err, "WriteConfFile(): fail to create sriov-operator folder", + "path", utils.GetHostExtensionPath(consts.SriovConfBasePath)) + return err + } + } + + log.Log.V(2).Info("WriteSriovSupportedNics(): file does not exist, create it") + _, err = os.Create(utils.GetHostExtensionPath(consts.SriovSystemdSupportedNicPath)) + if err != nil { + log.Log.Error(err, "WriteSriovSupportedNics(): failed to create sriov supporter nics ids file", + "path", utils.GetHostExtensionPath(consts.SriovSystemdSupportedNicPath)) + return err + } + } else { + log.Log.Error(err, "WriteSriovSupportedNics(): failed to check sriov supported nics ids file", "path", utils.GetHostExtensionPath(consts.SriovSystemdSupportedNicPath)) + return err + } + } + + rawNicList := []byte{} + for _, line := range sriovnetworkv1.NicIDMap { + rawNicList = append(rawNicList, []byte(fmt.Sprintf("%s\n", line))...) + } + + err = os.WriteFile(utils.GetHostExtensionPath(consts.SriovSystemdSupportedNicPath), rawNicList, 0644) + if err != nil { + log.Log.Error(err, "WriteSriovSupportedNics(): failed to write sriov supported nics ids file", + "path", utils.GetHostExtensionPath(consts.SriovSystemdSupportedNicPath)) + return err + } + + return nil +} + +// ReadSriovSupportedNics reads the list of SR-IOV supported network interface cards (NICs) from the host. +// It returns a slice of strings where each string represents a line from the file, +// with each line corresponding to an SR-IOV supported NIC. If the file does not exist, it returns nil and an error. +// If there is an error reading the file, it returns the error along with the file path for debugging purposes. +func (s *systemd) ReadSriovSupportedNics() ([]string, error) { + _, err := os.Stat(utils.GetHostExtensionPath(consts.SriovSystemdSupportedNicPath)) + if err != nil { + if os.IsNotExist(err) { + log.Log.V(2).Info("ReadSriovSupportedNics(): file does not exist, return empty result") + return nil, err + } else { + log.Log.Error(err, "ReadSriovSupportedNics(): failed to check sriov supported nics file", "path", utils.GetHostExtensionPath(consts.SriovSystemdSupportedNicPath)) + return nil, err + } + } + + rawConfig, err := os.ReadFile(utils.GetHostExtensionPath(consts.SriovSystemdSupportedNicPath)) + if err != nil { + log.Log.Error(err, "ReadSriovSupportedNics(): failed to read sriov supported nics file", "path", utils.GetHostExtensionPath(consts.SriovSystemdSupportedNicPath)) + return nil, err + } + + lines := strings.Split(string(rawConfig), "\n") + return lines, nil +} + +// CleanSriovFilesFromHost removes SR-IOV related configuration and service files from the host system. +// It deletes several systemd-related files including configuration paths, result paths, supported NICs path, +// and service binary path. If not in an OpenShift environment, it also removes the main SR-IOV +// service and post-networking service files. +func (s *systemd) CleanSriovFilesFromHost(isOpenShift bool) error { + err := os.Remove(utils.GetHostExtensionPath(consts.SriovSystemdConfigPath)) + if err != nil && !os.IsNotExist(err) { + return err + } + + err = os.Remove(utils.GetHostExtensionPath(consts.SriovSystemdResultPath)) + if err != nil && !os.IsNotExist(err) { + return err + } + + err = os.Remove(utils.GetHostExtensionPath(consts.SriovSystemdSupportedNicPath)) + if err != nil && !os.IsNotExist(err) { + return err + } + + err = os.Remove(utils.GetHostExtensionPath(consts.SriovSystemdServiceBinaryPath)) + if err != nil && !os.IsNotExist(err) { + return err + } + + // in openshift we should not remove the systemd service it will be done by the machine config operator + if !isOpenShift { + err = os.Remove(utils.GetHostExtensionPath(consts.SriovServicePath)) + if err != nil && !os.IsNotExist(err) { + return err + } + err = os.Remove(utils.GetHostExtensionPath(consts.SriovPostNetworkServicePath)) + if err != nil && !os.IsNotExist(err) { + return err + } + } + + return nil +} diff --git a/pkg/host/internal/systemd/systemd_suite_test.go b/pkg/host/internal/systemd/systemd_suite_test.go new file mode 100644 index 000000000..f5962a129 --- /dev/null +++ b/pkg/host/internal/systemd/systemd_suite_test.go @@ -0,0 +1,21 @@ +package systemd + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "go.uber.org/zap/zapcore" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +func TestSystemd(t *testing.T) { + log.SetLogger(zap.New( + zap.WriteTo(GinkgoWriter), + zap.Level(zapcore.Level(-2)), + zap.UseDevMode(true))) + RegisterFailHandler(Fail) + RunSpecs(t, "Package Systemd Suite") +} diff --git a/pkg/host/internal/systemd/systemd_test.go b/pkg/host/internal/systemd/systemd_test.go new file mode 100644 index 000000000..6a9ed2e2a --- /dev/null +++ b/pkg/host/internal/systemd/systemd_test.go @@ -0,0 +1,362 @@ +package systemd + +import ( + "os" + "path" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" +) + +var _ = Describe("Systemd", func() { + var ( + tempDir = "/tmp/sriov-test/" + validTestContentJson = []byte(`{ + "spec": { + "interfaces": [ + { + "name": "enp216s0f0np0", + "pciAddress": "0000:d8:00.0", + "numVfs": 1, + "linkType": "IB", + "vfGroups": [ + { + "vfRange": "0-0", + "resourceName": "test-resource0" + } + ] + } + ], + "policyName": "test-policy0" + }, + "mtu": 2000, + "isRdma": true, + "unsupportedNics": false, + "platformType": 0, + "manageSoftwareBridges": false, + "ovsdbSocketPath": "" +}`) + + validTestContentYaml = []byte(`spec: + interfaces: + - name: "enp216s0f0np0" + pciAddress: "0000:d8:00.0" + numVfs: 1 + linkType: "IB" + vfGroups: + - vfRange: "0-0" + resourceName: "test-resource0" + policyName: "test-policy0" + mtu: 2000 + isRdma: true +unsupportedNics: false +platformType: 0 +manageSoftwareBridges: false +ovsdbSocketPath: ""`) + + testNodeStateData = &sriovnetworkv1.SriovNetworkNodeState{ + Spec: sriovnetworkv1.SriovNetworkNodeStateSpec{ + Interfaces: []sriovnetworkv1.Interface{ + { + Name: "enp216s0f0np0", + PciAddress: "0000:d8:00.0", + NumVfs: 1, + LinkType: "IB", + VfGroups: []sriovnetworkv1.VfGroup{ + { + VfRange: "0-0", + ResourceName: "test-resource0", + }, + }, + }, + }, + }, + } + + resultData = &types.SriovResult{LastSyncError: "", SyncStatus: consts.SyncStatusSucceeded} + validResultData = []byte(`syncStatus: Succeeded +lastSyncError: ""`) + + testSriovSupportedNicIDs = []byte(`8086 1583 154c +8086 0d58 154c +8086 10c9 10ca +`) + + s types.SystemdInterface + ) + + BeforeEach(func() { + err := os.MkdirAll(path.Join(tempDir, consts.SriovConfBasePath), 0777) + Expect(err).ToNot(HaveOccurred()) + + err = os.MkdirAll(path.Join(tempDir, consts.SriovServiceBasePath), 0777) + Expect(err).ToNot(HaveOccurred()) + + err = os.MkdirAll(path.Join(tempDir, "/var/lib/sriov/"), 0777) + Expect(err).ToNot(HaveOccurred()) + + vars.InChroot = true + vars.FilesystemRoot = tempDir + sriovnetworkv1.NicIDMap = []string{} + + s = &systemd{} + }) + + AfterEach(func() { + err := os.RemoveAll(tempDir) + Expect(err).ToNot(HaveOccurred()) + }) + + Context("ReadConfFile", func() { + It("should read the content of the file as a json", func() { + err := os.WriteFile(path.Join(tempDir, consts.SriovSystemdConfigPath), validTestContentJson, 0644) + Expect(err).ToNot(HaveOccurred()) + + sr, err := s.ReadConfFile() + Expect(err).ToNot(HaveOccurred()) + Expect(len(sr.Spec.Interfaces)).To(Equal(1)) + }) + + It("should read the content of the file as a yaml", func() { + err := os.WriteFile(path.Join(tempDir, consts.SriovSystemdConfigPath), validTestContentYaml, 0644) + Expect(err).ToNot(HaveOccurred()) + + sr, err := s.ReadConfFile() + Expect(err).ToNot(HaveOccurred()) + Expect(len(sr.Spec.Interfaces)).To(Equal(1)) + }) + + It("should return error if the file doesn't exist ", func() { + _, err := s.ReadConfFile() + Expect(err).To(HaveOccurred()) + }) + + It("should return error if not able to parse the content of the file as a yaml", func() { + err := os.WriteFile(path.Join(tempDir, consts.SriovSystemdConfigPath), []byte("test"), 0644) + Expect(err).ToNot(HaveOccurred()) + + _, err = s.ReadConfFile() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `test` into types.SriovConfig")) + }) + }) + + Context("WriteConfFile", func() { + It("should create the sriov-operator folder if it doesn't exist", func() { + err := os.Remove(path.Join(tempDir, consts.SriovConfBasePath)) + Expect(err).ToNot(HaveOccurred()) + + updated, err := s.WriteConfFile(testNodeStateData) + Expect(err).ToNot(HaveOccurred()) + Expect(updated).To(BeTrue()) + _, err = os.Stat(path.Join(tempDir, consts.SriovSystemdConfigPath)) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should return false if the file was not updated", func() { + updated, err := s.WriteConfFile(testNodeStateData) + Expect(err).ToNot(HaveOccurred()) + Expect(updated).To(BeTrue()) + updated, err = s.WriteConfFile(testNodeStateData) + Expect(err).ToNot(HaveOccurred()) + Expect(updated).To(BeFalse()) + }) + + It("should write the content of the file as a yaml", func() { + updated, err := s.WriteConfFile(testNodeStateData) + Expect(err).ToNot(HaveOccurred()) + Expect(updated).To(BeTrue()) + + content, err := s.ReadConfFile() + Expect(err).ToNot(HaveOccurred()) + Expect(content.PlatformType).To(Equal(consts.Baremetal)) + + _, err = os.Stat(utils.GetHostExtensionPath(consts.SriovSystemdConfigPath)) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should write the content of the file as a yaml and return false if the file didn't exist and no interfaces exist", func() { + testNodeStateData := &sriovnetworkv1.SriovNetworkNodeState{ + Spec: sriovnetworkv1.SriovNetworkNodeStateSpec{}, + } + + updated, err := s.WriteConfFile(testNodeStateData) + Expect(err).ToNot(HaveOccurred()) + Expect(updated).To(BeFalse()) + + content, err := s.ReadConfFile() + Expect(err).ToNot(HaveOccurred()) + Expect(content.PlatformType).To(Equal(consts.Baremetal)) + + _, err = os.Stat(utils.GetHostExtensionPath(consts.SriovSystemdConfigPath)) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should update the existing config file", func() { + emptyTestNodeStateData := &sriovnetworkv1.SriovNetworkNodeState{ + Spec: sriovnetworkv1.SriovNetworkNodeStateSpec{}, + } + + updated, err := s.WriteConfFile(emptyTestNodeStateData) + Expect(err).ToNot(HaveOccurred()) + Expect(updated).To(BeFalse()) + + content, err := s.ReadConfFile() + Expect(err).ToNot(HaveOccurred()) + Expect(len(content.Spec.Interfaces)).To(Equal(0)) + + updated, err = s.WriteConfFile(testNodeStateData) + Expect(err).ToNot(HaveOccurred()) + Expect(updated).To(BeTrue()) + + content, err = s.ReadConfFile() + Expect(err).ToNot(HaveOccurred()) + Expect(len(content.Spec.Interfaces)).To(Equal(1)) + }) + }) + + Context("WriteSriovResult", func() { + It("should write the result file", func() { + err := s.WriteSriovResult(resultData) + Expect(err).ToNot(HaveOccurred()) + + _, err = os.Stat(utils.GetHostExtensionPath(consts.SriovSystemdResultPath)) + Expect(err).ToNot(HaveOccurred()) + + result, err := s.ReadSriovResult() + Expect(err).ToNot(HaveOccurred()) + Expect(result.SyncStatus).To(Equal(resultData.SyncStatus)) + }) + + It("should create the folder if doesn't exist", func() { + err := os.Remove(path.Join(tempDir, consts.SriovConfBasePath)) + Expect(err).ToNot(HaveOccurred()) + + err = s.WriteSriovResult(resultData) + Expect(err).ToNot(HaveOccurred()) + + result, err := s.ReadSriovResult() + Expect(err).ToNot(HaveOccurred()) + Expect(result.SyncStatus).To(Equal(resultData.SyncStatus)) + }) + }) + + Context("ReadSriovResult", func() { + It("should read the content of the file as a yaml", func() { + err := os.WriteFile(path.Join(tempDir, consts.SriovSystemdResultPath), validResultData, 0644) + Expect(err).ToNot(HaveOccurred()) + + sr, err := s.ReadSriovResult() + Expect(err).ToNot(HaveOccurred()) + Expect(sr.SyncStatus).To(Equal(consts.SyncStatusSucceeded)) + }) + + It("should return error if not able to parse the content of the file as a yaml", func() { + err := os.WriteFile(path.Join(tempDir, consts.SriovSystemdResultPath), []byte("test"), 0644) + Expect(err).ToNot(HaveOccurred()) + + _, err = s.ReadSriovResult() + Expect(err).To(HaveOccurred()) + }) + }) + + Context("RemoveSriovResult", func() { + It("should not return an error if the file doesn't exist", func() { + err := s.RemoveSriovResult() + Expect(err).ToNot(HaveOccurred()) + }) + + It("should not return", func() { + err := os.WriteFile(path.Join(tempDir, consts.SriovSystemdResultPath), []byte("test"), 0644) + Expect(err).ToNot(HaveOccurred()) + + err = s.RemoveSriovResult() + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Context("WriteSriovSupportedNics", func() { + It("should write the nic map", func() { + sriovnetworkv1.NicIDMap = []string{"test", "test1"} + err := s.WriteSriovSupportedNics() + Expect(err).ToNot(HaveOccurred()) + + _, err = os.Stat(utils.GetHostExtensionPath(consts.SriovSystemdSupportedNicPath)) + Expect(err).ToNot(HaveOccurred()) + + result, err := s.ReadSriovSupportedNics() + Expect(err).ToNot(HaveOccurred()) + Expect(len(result)).To(Equal(3)) + Expect(result[0]).To(Equal("test")) + }) + + It("should create the folder if doesn't exist", func() { + err := os.Remove(path.Join(tempDir, consts.SriovConfBasePath)) + Expect(err).ToNot(HaveOccurred()) + + err = s.WriteSriovSupportedNics() + Expect(err).ToNot(HaveOccurred()) + + result, err := s.ReadSriovSupportedNics() + Expect(err).ToNot(HaveOccurred()) + Expect(len(result)).To(Equal(1)) + Expect(result[0]).To(Equal("")) + }) + }) + + Context("ReadSriovSupportedNics", func() { + It("should read the content of the file", func() { + err := os.WriteFile(path.Join(tempDir, consts.SriovSystemdSupportedNicPath), testSriovSupportedNicIDs, 0644) + Expect(err).ToNot(HaveOccurred()) + + sr, err := s.ReadSriovSupportedNics() + Expect(err).ToNot(HaveOccurred()) + Expect(len(sr)).To(Equal(4)) + Expect(sr[0]).To(Equal("8086 1583 154c")) + }) + + It("should return error if the files doesn't exist", func() { + _, err := s.ReadSriovSupportedNics() + Expect(err).To(HaveOccurred()) + }) + }) + + Context("CleanSriovFilesFromHost", func() { + It("should remove the files that exist", func() { + err := os.WriteFile(path.Join(tempDir, consts.SriovSystemdConfigPath), []byte("test"), 0644) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(path.Join(tempDir, consts.SriovSystemdResultPath), []byte("test"), 0644) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(path.Join(tempDir, consts.SriovSystemdSupportedNicPath), []byte("test"), 0644) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(path.Join(tempDir, consts.SriovSystemdServiceBinaryPath), []byte("test"), 0644) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(path.Join(tempDir, consts.SriovSystemdSupportedNicPath), []byte("test"), 0644) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(path.Join(tempDir, consts.SriovServicePath), []byte("test"), 0644) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(path.Join(tempDir, consts.SriovPostNetworkServicePath), []byte("test"), 0644) + Expect(err).ToNot(HaveOccurred()) + + err = s.CleanSriovFilesFromHost(false) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should not return error if the files don't exist", func() { + err := s.CleanSriovFilesFromHost(false) + Expect(err).ToNot(HaveOccurred()) + }) + }) +}) diff --git a/pkg/host/internal/udev/udev.go b/pkg/host/internal/udev/udev.go index 3f828bb70..841bc71d7 100644 --- a/pkg/host/internal/udev/udev.go +++ b/pkg/host/internal/udev/udev.go @@ -5,6 +5,7 @@ import ( "os" "path" "path/filepath" + "strconv" "strings" "sigs.k8s.io/controller-runtime/pkg/log" @@ -126,6 +127,18 @@ func (u *udev) LoadUdevRules() error { return nil } +// WaitUdevEventsProcessed calls `udevadm settle“ with provided timeout +// The command watches the udev event queue, and exits if all current events are handled. +func (u *udev) WaitUdevEventsProcessed(timeout int) error { + log.Log.V(2).Info("WaitUdevEventsProcessed()") + _, stderr, err := u.utilsHelper.RunCommand("udevadm", "settle", "-t", strconv.Itoa(timeout)) + if err != nil { + log.Log.Error(err, "WaitUdevEventsProcessed(): failed to wait for udev rules to process", "error", stderr, "timeout", timeout) + return err + } + return nil +} + func (u *udev) addUdevRule(pfPciAddress, ruleName, ruleContent string) error { log.Log.V(2).Info("addUdevRule()", "device", pfPciAddress, "rule", ruleName) rulePath := u.getRuleFolderPath() diff --git a/pkg/host/internal/udev/udev_test.go b/pkg/host/internal/udev/udev_test.go index 4a2e17e7e..5289ad731 100644 --- a/pkg/host/internal/udev/udev_test.go +++ b/pkg/host/internal/udev/udev_test.go @@ -5,7 +5,7 @@ import ( "os" "path/filepath" - "github.com/golang/mock/gomock" + "go.uber.org/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -210,4 +210,14 @@ var _ = Describe("UDEV", func() { Expect(s.LoadUdevRules()).To(MatchError(testError)) }) }) + Context("WaitUdevEventsProcessed", func() { + It("Succeed", func() { + utilsMock.EXPECT().RunCommand("udevadm", "settle", "-t", "10").Return("", "", nil) + Expect(s.WaitUdevEventsProcessed(10)).NotTo(HaveOccurred()) + }) + It("Command Failed", func() { + utilsMock.EXPECT().RunCommand("udevadm", "settle", "-t", "20").Return("", "", testError) + Expect(s.WaitUdevEventsProcessed(20)).To(MatchError(testError)) + }) + }) }) diff --git a/pkg/host/internal/vdpa/vdpa.go b/pkg/host/internal/vdpa/vdpa.go index 4a41c63d1..e21d00cb6 100644 --- a/pkg/host/internal/vdpa/vdpa.go +++ b/pkg/host/internal/vdpa/vdpa.go @@ -94,11 +94,9 @@ func (v *vdpa) DeleteVDPADevice(pciAddr string) error { func (v *vdpa) DiscoverVDPAType(pciAddr string) string { expectedVDPAName := generateVDPADevName(pciAddr) funcLog := log.Log.WithValues("device", pciAddr, "name", expectedVDPAName) - funcLog.V(2).Info("DiscoverVDPAType() discover device type") _, err := v.netlinkLib.VDPAGetDevByName(expectedVDPAName) if err != nil { if errors.Is(err, syscall.ENODEV) { - funcLog.V(2).Info("DiscoverVDPAType(): VDPA device for VF not found") return "" } if errors.Is(err, syscall.ENOENT) { diff --git a/pkg/host/internal/vdpa/vdpa_test.go b/pkg/host/internal/vdpa/vdpa_test.go index d58bf0b97..83b15da0f 100644 --- a/pkg/host/internal/vdpa/vdpa_test.go +++ b/pkg/host/internal/vdpa/vdpa_test.go @@ -4,8 +4,8 @@ import ( "fmt" "syscall" - "github.com/golang/mock/gomock" "github.com/vishvananda/netlink" + "go.uber.org/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" diff --git a/pkg/host/manager.go b/pkg/host/manager.go index 02a77a659..7eb8e23a1 100644 --- a/pkg/host/manager.go +++ b/pkg/host/manager.go @@ -2,6 +2,7 @@ package host import ( "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/bridge" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/cpu" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/infiniband" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/kernel" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/dputils" @@ -12,6 +13,7 @@ import ( "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/network" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/service" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/sriov" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/systemd" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/udev" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/vdpa" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" @@ -30,6 +32,8 @@ type HostManagerInterface interface { types.VdpaInterface types.InfinibandInterface types.BridgeInterface + types.CPUInfoProviderInterface + types.SystemdInterface } type hostManager struct { @@ -42,6 +46,8 @@ type hostManager struct { types.VdpaInterface types.InfinibandInterface types.BridgeInterface + types.CPUInfoProviderInterface + types.SystemdInterface } func NewHostManager(utilsInterface utils.CmdInterface) (HostManagerInterface, error) { @@ -61,6 +67,8 @@ func NewHostManager(utilsInterface utils.CmdInterface) (HostManagerInterface, er } br := bridge.New() sr := sriov.New(utilsInterface, k, n, u, v, ib, netlinkLib, dpUtils, sriovnetLib, ghwLib, br) + cpuInfoProvider := cpu.New(ghwLib) + s := systemd.New() return &hostManager{ utilsInterface, k, @@ -71,5 +79,7 @@ func NewHostManager(utilsInterface utils.CmdInterface) (HostManagerInterface, er v, ib, br, + cpuInfoProvider, + s, }, nil } diff --git a/pkg/host/mock/mock_host.go b/pkg/host/mock/mock_host.go index cb4d1480a..be37c4245 100644 --- a/pkg/host/mock/mock_host.go +++ b/pkg/host/mock/mock_host.go @@ -1,23 +1,30 @@ // Code generated by MockGen. DO NOT EDIT. // Source: manager.go +// +// Generated by this command: +// +// mockgen -destination mock/mock_host.go -source manager.go +// // Package mock_host is a generated GoMock package. package mock_host import ( + "net" reflect "reflect" - gomock "github.com/golang/mock/gomock" v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" store "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/store" types "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" netlink "github.com/vishvananda/netlink" + gomock "go.uber.org/mock/gomock" ) // MockHostManagerInterface is a mock of HostManagerInterface interface. type MockHostManagerInterface struct { ctrl *gomock.Controller recorder *MockHostManagerInterfaceMockRecorder + isgomock struct{} } // MockHostManagerInterfaceMockRecorder is the mock recorder for MockHostManagerInterface. @@ -46,7 +53,7 @@ func (m *MockHostManagerInterface) AddDisableNMUdevRule(pfPciAddress string) err } // AddDisableNMUdevRule indicates an expected call of AddDisableNMUdevRule. -func (mr *MockHostManagerInterfaceMockRecorder) AddDisableNMUdevRule(pfPciAddress interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) AddDisableNMUdevRule(pfPciAddress any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddDisableNMUdevRule", reflect.TypeOf((*MockHostManagerInterface)(nil).AddDisableNMUdevRule), pfPciAddress) } @@ -60,7 +67,7 @@ func (m *MockHostManagerInterface) AddPersistPFNameUdevRule(pfPciAddress, pfName } // AddPersistPFNameUdevRule indicates an expected call of AddPersistPFNameUdevRule. -func (mr *MockHostManagerInterfaceMockRecorder) AddPersistPFNameUdevRule(pfPciAddress, pfName interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) AddPersistPFNameUdevRule(pfPciAddress, pfName any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddPersistPFNameUdevRule", reflect.TypeOf((*MockHostManagerInterface)(nil).AddPersistPFNameUdevRule), pfPciAddress, pfName) } @@ -74,7 +81,7 @@ func (m *MockHostManagerInterface) AddVfRepresentorUdevRule(pfPciAddress, pfName } // AddVfRepresentorUdevRule indicates an expected call of AddVfRepresentorUdevRule. -func (mr *MockHostManagerInterfaceMockRecorder) AddVfRepresentorUdevRule(pfPciAddress, pfName, pfSwitchID, pfSwitchPort interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) AddVfRepresentorUdevRule(pfPciAddress, pfName, pfSwitchID, pfSwitchPort any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddVfRepresentorUdevRule", reflect.TypeOf((*MockHostManagerInterface)(nil).AddVfRepresentorUdevRule), pfPciAddress, pfName, pfSwitchID, pfSwitchPort) } @@ -88,7 +95,7 @@ func (m *MockHostManagerInterface) BindDefaultDriver(pciAddr string) error { } // BindDefaultDriver indicates an expected call of BindDefaultDriver. -func (mr *MockHostManagerInterfaceMockRecorder) BindDefaultDriver(pciAddr interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) BindDefaultDriver(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BindDefaultDriver", reflect.TypeOf((*MockHostManagerInterface)(nil).BindDefaultDriver), pciAddr) } @@ -102,7 +109,7 @@ func (m *MockHostManagerInterface) BindDpdkDriver(pciAddr, driver string) error } // BindDpdkDriver indicates an expected call of BindDpdkDriver. -func (mr *MockHostManagerInterfaceMockRecorder) BindDpdkDriver(pciAddr, driver interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) BindDpdkDriver(pciAddr, driver any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BindDpdkDriver", reflect.TypeOf((*MockHostManagerInterface)(nil).BindDpdkDriver), pciAddr, driver) } @@ -116,11 +123,16 @@ func (m *MockHostManagerInterface) BindDriverByBusAndDevice(bus, device, driver } // BindDriverByBusAndDevice indicates an expected call of BindDriverByBusAndDevice. -func (mr *MockHostManagerInterfaceMockRecorder) BindDriverByBusAndDevice(bus, device, driver interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) BindDriverByBusAndDevice(bus, device, driver any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BindDriverByBusAndDevice", reflect.TypeOf((*MockHostManagerInterface)(nil).BindDriverByBusAndDevice), bus, device, driver) } +func (mr *MockHostManagerInterfaceMockRecorder) GetVfGUID(vfAddr string, pfAddr string, vfID int) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVfGUID", reflect.TypeOf((*MockHostManagerInterface)(nil).GetVfGUID), vfAddr, pfAddr, vfID) +} + // CheckRDMAEnabled mocks base method. func (m *MockHostManagerInterface) CheckRDMAEnabled() (bool, error) { m.ctrl.T.Helper() @@ -136,6 +148,20 @@ func (mr *MockHostManagerInterfaceMockRecorder) CheckRDMAEnabled() *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckRDMAEnabled", reflect.TypeOf((*MockHostManagerInterface)(nil).CheckRDMAEnabled)) } +// CleanSriovFilesFromHost mocks base method. +func (m *MockHostManagerInterface) CleanSriovFilesFromHost(isOpenShift bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CleanSriovFilesFromHost", isOpenShift) + ret0, _ := ret[0].(error) + return ret0 +} + +// CleanSriovFilesFromHost indicates an expected call of CleanSriovFilesFromHost. +func (mr *MockHostManagerInterfaceMockRecorder) CleanSriovFilesFromHost(isOpenShift any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanSriovFilesFromHost", reflect.TypeOf((*MockHostManagerInterface)(nil).CleanSriovFilesFromHost), isOpenShift) +} + // CompareServices mocks base method. func (m *MockHostManagerInterface) CompareServices(serviceA, serviceB *types.Service) (bool, error) { m.ctrl.T.Helper() @@ -146,7 +172,7 @@ func (m *MockHostManagerInterface) CompareServices(serviceA, serviceB *types.Ser } // CompareServices indicates an expected call of CompareServices. -func (mr *MockHostManagerInterfaceMockRecorder) CompareServices(serviceA, serviceB interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) CompareServices(serviceA, serviceB any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompareServices", reflect.TypeOf((*MockHostManagerInterface)(nil).CompareServices), serviceA, serviceB) } @@ -160,7 +186,7 @@ func (m *MockHostManagerInterface) ConfigSriovDeviceVirtual(iface *v1.Interface) } // ConfigSriovDeviceVirtual indicates an expected call of ConfigSriovDeviceVirtual. -func (mr *MockHostManagerInterfaceMockRecorder) ConfigSriovDeviceVirtual(iface interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) ConfigSriovDeviceVirtual(iface any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigSriovDeviceVirtual", reflect.TypeOf((*MockHostManagerInterface)(nil).ConfigSriovDeviceVirtual), iface) } @@ -174,7 +200,7 @@ func (m *MockHostManagerInterface) ConfigSriovInterfaces(storeManager store.Mana } // ConfigSriovInterfaces indicates an expected call of ConfigSriovInterfaces. -func (mr *MockHostManagerInterfaceMockRecorder) ConfigSriovInterfaces(storeManager, interfaces, ifaceStatuses, skipVFConfiguration interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) ConfigSriovInterfaces(storeManager, interfaces, ifaceStatuses, skipVFConfiguration any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigSriovInterfaces", reflect.TypeOf((*MockHostManagerInterface)(nil).ConfigSriovInterfaces), storeManager, interfaces, ifaceStatuses, skipVFConfiguration) } @@ -188,7 +214,7 @@ func (m *MockHostManagerInterface) ConfigureBridges(bridgesSpec, bridgesStatus v } // ConfigureBridges indicates an expected call of ConfigureBridges. -func (mr *MockHostManagerInterfaceMockRecorder) ConfigureBridges(bridgesSpec, bridgesStatus interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) ConfigureBridges(bridgesSpec, bridgesStatus any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigureBridges", reflect.TypeOf((*MockHostManagerInterface)(nil).ConfigureBridges), bridgesSpec, bridgesStatus) } @@ -202,11 +228,20 @@ func (m *MockHostManagerInterface) ConfigureVfGUID(vfAddr, pfAddr string, vfID i } // ConfigureVfGUID indicates an expected call of ConfigureVfGUID. -func (mr *MockHostManagerInterfaceMockRecorder) ConfigureVfGUID(vfAddr, pfAddr, vfID, pfLink interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) ConfigureVfGUID(vfAddr, pfAddr, vfID, pfLink any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigureVfGUID", reflect.TypeOf((*MockHostManagerInterface)(nil).ConfigureVfGUID), vfAddr, pfAddr, vfID, pfLink) } +// GetVfGUID mocks base method. +func (m *MockHostManagerInterface) GetVfGUID(vfAddr, pfAddr string, vfID int) (net.HardwareAddr, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVfGUID", vfAddr, pfAddr, vfID) + ret0, _ := ret[0].(net.HardwareAddr) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + // CreateVDPADevice mocks base method. func (m *MockHostManagerInterface) CreateVDPADevice(pciAddr, vdpaType string) error { m.ctrl.T.Helper() @@ -216,7 +251,7 @@ func (m *MockHostManagerInterface) CreateVDPADevice(pciAddr, vdpaType string) er } // CreateVDPADevice indicates an expected call of CreateVDPADevice. -func (mr *MockHostManagerInterfaceMockRecorder) CreateVDPADevice(pciAddr, vdpaType interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) CreateVDPADevice(pciAddr, vdpaType any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateVDPADevice", reflect.TypeOf((*MockHostManagerInterface)(nil).CreateVDPADevice), pciAddr, vdpaType) } @@ -230,7 +265,7 @@ func (m *MockHostManagerInterface) DeleteVDPADevice(pciAddr string) error { } // DeleteVDPADevice indicates an expected call of DeleteVDPADevice. -func (mr *MockHostManagerInterfaceMockRecorder) DeleteVDPADevice(pciAddr interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) DeleteVDPADevice(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteVDPADevice", reflect.TypeOf((*MockHostManagerInterface)(nil).DeleteVDPADevice), pciAddr) } @@ -244,7 +279,7 @@ func (m *MockHostManagerInterface) DetachInterfaceFromManagedBridge(pciAddr stri } // DetachInterfaceFromManagedBridge indicates an expected call of DetachInterfaceFromManagedBridge. -func (mr *MockHostManagerInterfaceMockRecorder) DetachInterfaceFromManagedBridge(pciAddr interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) DetachInterfaceFromManagedBridge(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DetachInterfaceFromManagedBridge", reflect.TypeOf((*MockHostManagerInterface)(nil).DetachInterfaceFromManagedBridge), pciAddr) } @@ -264,6 +299,21 @@ func (mr *MockHostManagerInterfaceMockRecorder) DiscoverBridges() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiscoverBridges", reflect.TypeOf((*MockHostManagerInterface)(nil).DiscoverBridges)) } +// DiscoverRDMASubsystem mocks base method. +func (m *MockHostManagerInterface) DiscoverRDMASubsystem() (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DiscoverRDMASubsystem") + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DiscoverRDMASubsystem indicates an expected call of DiscoverRDMASubsystem. +func (mr *MockHostManagerInterfaceMockRecorder) DiscoverRDMASubsystem() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiscoverRDMASubsystem", reflect.TypeOf((*MockHostManagerInterface)(nil).DiscoverRDMASubsystem)) +} + // DiscoverSriovDevices mocks base method. func (m *MockHostManagerInterface) DiscoverSriovDevices(storeManager store.ManagerInterface) ([]v1.InterfaceExt, error) { m.ctrl.T.Helper() @@ -274,7 +324,7 @@ func (m *MockHostManagerInterface) DiscoverSriovDevices(storeManager store.Manag } // DiscoverSriovDevices indicates an expected call of DiscoverSriovDevices. -func (mr *MockHostManagerInterfaceMockRecorder) DiscoverSriovDevices(storeManager interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) DiscoverSriovDevices(storeManager any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiscoverSriovDevices", reflect.TypeOf((*MockHostManagerInterface)(nil).DiscoverSriovDevices), storeManager) } @@ -288,7 +338,7 @@ func (m *MockHostManagerInterface) DiscoverVDPAType(pciAddr string) string { } // DiscoverVDPAType indicates an expected call of DiscoverVDPAType. -func (mr *MockHostManagerInterfaceMockRecorder) DiscoverVDPAType(pciAddr interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) DiscoverVDPAType(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiscoverVDPAType", reflect.TypeOf((*MockHostManagerInterface)(nil).DiscoverVDPAType), pciAddr) } @@ -302,7 +352,7 @@ func (m *MockHostManagerInterface) EnableHwTcOffload(ifaceName string) error { } // EnableHwTcOffload indicates an expected call of EnableHwTcOffload. -func (mr *MockHostManagerInterfaceMockRecorder) EnableHwTcOffload(ifaceName interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) EnableHwTcOffload(ifaceName any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnableHwTcOffload", reflect.TypeOf((*MockHostManagerInterface)(nil).EnableHwTcOffload), ifaceName) } @@ -316,11 +366,26 @@ func (m *MockHostManagerInterface) EnableService(service *types.Service) error { } // EnableService indicates an expected call of EnableService. -func (mr *MockHostManagerInterfaceMockRecorder) EnableService(service interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) EnableService(service any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnableService", reflect.TypeOf((*MockHostManagerInterface)(nil).EnableService), service) } +// GetCPUVendor mocks base method. +func (m *MockHostManagerInterface) GetCPUVendor() (types.CPUVendor, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCPUVendor") + ret0, _ := ret[0].(types.CPUVendor) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCPUVendor indicates an expected call of GetCPUVendor. +func (mr *MockHostManagerInterfaceMockRecorder) GetCPUVendor() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCPUVendor", reflect.TypeOf((*MockHostManagerInterface)(nil).GetCPUVendor)) +} + // GetCurrentKernelArgs mocks base method. func (m *MockHostManagerInterface) GetCurrentKernelArgs() (string, error) { m.ctrl.T.Helper() @@ -346,7 +411,7 @@ func (m *MockHostManagerInterface) GetDevlinkDeviceParam(pciAddr, paramName stri } // GetDevlinkDeviceParam indicates an expected call of GetDevlinkDeviceParam. -func (mr *MockHostManagerInterfaceMockRecorder) GetDevlinkDeviceParam(pciAddr, paramName interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) GetDevlinkDeviceParam(pciAddr, paramName any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDevlinkDeviceParam", reflect.TypeOf((*MockHostManagerInterface)(nil).GetDevlinkDeviceParam), pciAddr, paramName) } @@ -361,7 +426,7 @@ func (m *MockHostManagerInterface) GetDriverByBusAndDevice(bus, device string) ( } // GetDriverByBusAndDevice indicates an expected call of GetDriverByBusAndDevice. -func (mr *MockHostManagerInterfaceMockRecorder) GetDriverByBusAndDevice(bus, device interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) GetDriverByBusAndDevice(bus, device any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDriverByBusAndDevice", reflect.TypeOf((*MockHostManagerInterface)(nil).GetDriverByBusAndDevice), bus, device) } @@ -376,7 +441,7 @@ func (m *MockHostManagerInterface) GetInterfaceIndex(pciAddr string) (int, error } // GetInterfaceIndex indicates an expected call of GetInterfaceIndex. -func (mr *MockHostManagerInterfaceMockRecorder) GetInterfaceIndex(pciAddr interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) GetInterfaceIndex(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInterfaceIndex", reflect.TypeOf((*MockHostManagerInterface)(nil).GetInterfaceIndex), pciAddr) } @@ -390,7 +455,7 @@ func (m *MockHostManagerInterface) GetLinkType(name string) string { } // GetLinkType indicates an expected call of GetLinkType. -func (mr *MockHostManagerInterfaceMockRecorder) GetLinkType(name interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) GetLinkType(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLinkType", reflect.TypeOf((*MockHostManagerInterface)(nil).GetLinkType), name) } @@ -404,7 +469,7 @@ func (m *MockHostManagerInterface) GetNetDevLinkAdminState(ifaceName string) str } // GetNetDevLinkAdminState indicates an expected call of GetNetDevLinkAdminState. -func (mr *MockHostManagerInterfaceMockRecorder) GetNetDevLinkAdminState(ifaceName interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) GetNetDevLinkAdminState(ifaceName any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNetDevLinkAdminState", reflect.TypeOf((*MockHostManagerInterface)(nil).GetNetDevLinkAdminState), ifaceName) } @@ -418,7 +483,7 @@ func (m *MockHostManagerInterface) GetNetDevLinkSpeed(name string) string { } // GetNetDevLinkSpeed indicates an expected call of GetNetDevLinkSpeed. -func (mr *MockHostManagerInterfaceMockRecorder) GetNetDevLinkSpeed(name interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) GetNetDevLinkSpeed(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNetDevLinkSpeed", reflect.TypeOf((*MockHostManagerInterface)(nil).GetNetDevLinkSpeed), name) } @@ -432,7 +497,7 @@ func (m *MockHostManagerInterface) GetNetDevMac(name string) string { } // GetNetDevMac indicates an expected call of GetNetDevMac. -func (mr *MockHostManagerInterfaceMockRecorder) GetNetDevMac(name interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) GetNetDevMac(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNetDevMac", reflect.TypeOf((*MockHostManagerInterface)(nil).GetNetDevMac), name) } @@ -446,7 +511,7 @@ func (m *MockHostManagerInterface) GetNetDevNodeGUID(pciAddr string) string { } // GetNetDevNodeGUID indicates an expected call of GetNetDevNodeGUID. -func (mr *MockHostManagerInterfaceMockRecorder) GetNetDevNodeGUID(pciAddr interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) GetNetDevNodeGUID(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNetDevNodeGUID", reflect.TypeOf((*MockHostManagerInterface)(nil).GetNetDevNodeGUID), pciAddr) } @@ -460,7 +525,7 @@ func (m *MockHostManagerInterface) GetNetdevMTU(pciAddr string) int { } // GetNetdevMTU indicates an expected call of GetNetdevMTU. -func (mr *MockHostManagerInterfaceMockRecorder) GetNetdevMTU(pciAddr interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) GetNetdevMTU(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNetdevMTU", reflect.TypeOf((*MockHostManagerInterface)(nil).GetNetdevMTU), pciAddr) } @@ -474,7 +539,7 @@ func (m *MockHostManagerInterface) GetNicSriovMode(pciAddr string) string { } // GetNicSriovMode indicates an expected call of GetNicSriovMode. -func (mr *MockHostManagerInterfaceMockRecorder) GetNicSriovMode(pciAddr interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) GetNicSriovMode(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNicSriovMode", reflect.TypeOf((*MockHostManagerInterface)(nil).GetNicSriovMode), pciAddr) } @@ -489,7 +554,7 @@ func (m *MockHostManagerInterface) GetPciAddressFromInterfaceName(interfaceName } // GetPciAddressFromInterfaceName indicates an expected call of GetPciAddressFromInterfaceName. -func (mr *MockHostManagerInterfaceMockRecorder) GetPciAddressFromInterfaceName(interfaceName interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) GetPciAddressFromInterfaceName(interfaceName any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPciAddressFromInterfaceName", reflect.TypeOf((*MockHostManagerInterface)(nil).GetPciAddressFromInterfaceName), interfaceName) } @@ -504,7 +569,7 @@ func (m *MockHostManagerInterface) GetPhysPortName(name string) (string, error) } // GetPhysPortName indicates an expected call of GetPhysPortName. -func (mr *MockHostManagerInterfaceMockRecorder) GetPhysPortName(name interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) GetPhysPortName(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPhysPortName", reflect.TypeOf((*MockHostManagerInterface)(nil).GetPhysPortName), name) } @@ -519,7 +584,7 @@ func (m *MockHostManagerInterface) GetPhysSwitchID(name string) (string, error) } // GetPhysSwitchID indicates an expected call of GetPhysSwitchID. -func (mr *MockHostManagerInterfaceMockRecorder) GetPhysSwitchID(name interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) GetPhysSwitchID(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPhysSwitchID", reflect.TypeOf((*MockHostManagerInterface)(nil).GetPhysSwitchID), name) } @@ -534,7 +599,7 @@ func (m *MockHostManagerInterface) HasDriver(pciAddr string) (bool, string) { } // HasDriver indicates an expected call of HasDriver. -func (mr *MockHostManagerInterfaceMockRecorder) HasDriver(pciAddr interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) HasDriver(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasDriver", reflect.TypeOf((*MockHostManagerInterface)(nil).HasDriver), pciAddr) } @@ -548,7 +613,7 @@ func (m *MockHostManagerInterface) IsKernelArgsSet(cmdLine, karg string) bool { } // IsKernelArgsSet indicates an expected call of IsKernelArgsSet. -func (mr *MockHostManagerInterfaceMockRecorder) IsKernelArgsSet(cmdLine, karg interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) IsKernelArgsSet(cmdLine, karg any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsKernelArgsSet", reflect.TypeOf((*MockHostManagerInterface)(nil).IsKernelArgsSet), cmdLine, karg) } @@ -577,7 +642,7 @@ func (m *MockHostManagerInterface) IsKernelModuleLoaded(name string) (bool, erro } // IsKernelModuleLoaded indicates an expected call of IsKernelModuleLoaded. -func (mr *MockHostManagerInterfaceMockRecorder) IsKernelModuleLoaded(name interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) IsKernelModuleLoaded(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsKernelModuleLoaded", reflect.TypeOf((*MockHostManagerInterface)(nil).IsKernelModuleLoaded), name) } @@ -592,7 +657,7 @@ func (m *MockHostManagerInterface) IsServiceEnabled(servicePath string) (bool, e } // IsServiceEnabled indicates an expected call of IsServiceEnabled. -func (mr *MockHostManagerInterfaceMockRecorder) IsServiceEnabled(servicePath interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) IsServiceEnabled(servicePath any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsServiceEnabled", reflect.TypeOf((*MockHostManagerInterface)(nil).IsServiceEnabled), servicePath) } @@ -607,7 +672,7 @@ func (m *MockHostManagerInterface) IsServiceExist(servicePath string) (bool, err } // IsServiceExist indicates an expected call of IsServiceExist. -func (mr *MockHostManagerInterfaceMockRecorder) IsServiceExist(servicePath interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) IsServiceExist(servicePath any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsServiceExist", reflect.TypeOf((*MockHostManagerInterface)(nil).IsServiceExist), servicePath) } @@ -621,7 +686,7 @@ func (m *MockHostManagerInterface) IsSwitchdev(name string) bool { } // IsSwitchdev indicates an expected call of IsSwitchdev. -func (mr *MockHostManagerInterfaceMockRecorder) IsSwitchdev(name interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) IsSwitchdev(name any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSwitchdev", reflect.TypeOf((*MockHostManagerInterface)(nil).IsSwitchdev), name) } @@ -629,7 +694,7 @@ func (mr *MockHostManagerInterfaceMockRecorder) IsSwitchdev(name interface{}) *g // LoadKernelModule mocks base method. func (m *MockHostManagerInterface) LoadKernelModule(name string, args ...string) error { m.ctrl.T.Helper() - varargs := []interface{}{name} + varargs := []any{name} for _, a := range args { varargs = append(varargs, a) } @@ -639,9 +704,9 @@ func (m *MockHostManagerInterface) LoadKernelModule(name string, args ...string) } // LoadKernelModule indicates an expected call of LoadKernelModule. -func (mr *MockHostManagerInterfaceMockRecorder) LoadKernelModule(name interface{}, args ...interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) LoadKernelModule(name any, args ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{name}, args...) + varargs := append([]any{name}, args...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadKernelModule", reflect.TypeOf((*MockHostManagerInterface)(nil).LoadKernelModule), varargs...) } @@ -668,7 +733,7 @@ func (m *MockHostManagerInterface) PrepareNMUdevRule(supportedVfIds []string) er } // PrepareNMUdevRule indicates an expected call of PrepareNMUdevRule. -func (mr *MockHostManagerInterfaceMockRecorder) PrepareNMUdevRule(supportedVfIds interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) PrepareNMUdevRule(supportedVfIds any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PrepareNMUdevRule", reflect.TypeOf((*MockHostManagerInterface)(nil).PrepareNMUdevRule), supportedVfIds) } @@ -687,6 +752,21 @@ func (mr *MockHostManagerInterfaceMockRecorder) PrepareVFRepUdevRule() *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PrepareVFRepUdevRule", reflect.TypeOf((*MockHostManagerInterface)(nil).PrepareVFRepUdevRule)) } +// ReadConfFile mocks base method. +func (m *MockHostManagerInterface) ReadConfFile() (*types.SriovConfig, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadConfFile") + ret0, _ := ret[0].(*types.SriovConfig) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReadConfFile indicates an expected call of ReadConfFile. +func (mr *MockHostManagerInterfaceMockRecorder) ReadConfFile() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfFile", reflect.TypeOf((*MockHostManagerInterface)(nil).ReadConfFile)) +} + // ReadService mocks base method. func (m *MockHostManagerInterface) ReadService(servicePath string) (*types.Service, error) { m.ctrl.T.Helper() @@ -697,7 +777,7 @@ func (m *MockHostManagerInterface) ReadService(servicePath string) (*types.Servi } // ReadService indicates an expected call of ReadService. -func (mr *MockHostManagerInterfaceMockRecorder) ReadService(servicePath interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) ReadService(servicePath any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadService", reflect.TypeOf((*MockHostManagerInterface)(nil).ReadService), servicePath) } @@ -712,7 +792,7 @@ func (m *MockHostManagerInterface) ReadServiceInjectionManifestFile(path string) } // ReadServiceInjectionManifestFile indicates an expected call of ReadServiceInjectionManifestFile. -func (mr *MockHostManagerInterfaceMockRecorder) ReadServiceInjectionManifestFile(path interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) ReadServiceInjectionManifestFile(path any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadServiceInjectionManifestFile", reflect.TypeOf((*MockHostManagerInterface)(nil).ReadServiceInjectionManifestFile), path) } @@ -727,11 +807,41 @@ func (m *MockHostManagerInterface) ReadServiceManifestFile(path string) (*types. } // ReadServiceManifestFile indicates an expected call of ReadServiceManifestFile. -func (mr *MockHostManagerInterfaceMockRecorder) ReadServiceManifestFile(path interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) ReadServiceManifestFile(path any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadServiceManifestFile", reflect.TypeOf((*MockHostManagerInterface)(nil).ReadServiceManifestFile), path) } +// ReadSriovResult mocks base method. +func (m *MockHostManagerInterface) ReadSriovResult() (*types.SriovResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadSriovResult") + ret0, _ := ret[0].(*types.SriovResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReadSriovResult indicates an expected call of ReadSriovResult. +func (mr *MockHostManagerInterfaceMockRecorder) ReadSriovResult() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadSriovResult", reflect.TypeOf((*MockHostManagerInterface)(nil).ReadSriovResult)) +} + +// ReadSriovSupportedNics mocks base method. +func (m *MockHostManagerInterface) ReadSriovSupportedNics() ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadSriovSupportedNics") + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReadSriovSupportedNics indicates an expected call of ReadSriovSupportedNics. +func (mr *MockHostManagerInterfaceMockRecorder) ReadSriovSupportedNics() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadSriovSupportedNics", reflect.TypeOf((*MockHostManagerInterface)(nil).ReadSriovSupportedNics)) +} + // RebindVfToDefaultDriver mocks base method. func (m *MockHostManagerInterface) RebindVfToDefaultDriver(pciAddr string) error { m.ctrl.T.Helper() @@ -741,7 +851,7 @@ func (m *MockHostManagerInterface) RebindVfToDefaultDriver(pciAddr string) error } // RebindVfToDefaultDriver indicates an expected call of RebindVfToDefaultDriver. -func (mr *MockHostManagerInterfaceMockRecorder) RebindVfToDefaultDriver(pciAddr interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) RebindVfToDefaultDriver(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RebindVfToDefaultDriver", reflect.TypeOf((*MockHostManagerInterface)(nil).RebindVfToDefaultDriver), pciAddr) } @@ -755,7 +865,7 @@ func (m *MockHostManagerInterface) RemoveDisableNMUdevRule(pfPciAddress string) } // RemoveDisableNMUdevRule indicates an expected call of RemoveDisableNMUdevRule. -func (mr *MockHostManagerInterfaceMockRecorder) RemoveDisableNMUdevRule(pfPciAddress interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) RemoveDisableNMUdevRule(pfPciAddress any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveDisableNMUdevRule", reflect.TypeOf((*MockHostManagerInterface)(nil).RemoveDisableNMUdevRule), pfPciAddress) } @@ -769,11 +879,25 @@ func (m *MockHostManagerInterface) RemovePersistPFNameUdevRule(pfPciAddress stri } // RemovePersistPFNameUdevRule indicates an expected call of RemovePersistPFNameUdevRule. -func (mr *MockHostManagerInterfaceMockRecorder) RemovePersistPFNameUdevRule(pfPciAddress interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) RemovePersistPFNameUdevRule(pfPciAddress any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemovePersistPFNameUdevRule", reflect.TypeOf((*MockHostManagerInterface)(nil).RemovePersistPFNameUdevRule), pfPciAddress) } +// RemoveSriovResult mocks base method. +func (m *MockHostManagerInterface) RemoveSriovResult() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveSriovResult") + ret0, _ := ret[0].(error) + return ret0 +} + +// RemoveSriovResult indicates an expected call of RemoveSriovResult. +func (mr *MockHostManagerInterfaceMockRecorder) RemoveSriovResult() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveSriovResult", reflect.TypeOf((*MockHostManagerInterface)(nil).RemoveSriovResult)) +} + // RemoveVfRepresentorUdevRule mocks base method. func (m *MockHostManagerInterface) RemoveVfRepresentorUdevRule(pfPciAddress string) error { m.ctrl.T.Helper() @@ -783,7 +907,7 @@ func (m *MockHostManagerInterface) RemoveVfRepresentorUdevRule(pfPciAddress stri } // RemoveVfRepresentorUdevRule indicates an expected call of RemoveVfRepresentorUdevRule. -func (mr *MockHostManagerInterfaceMockRecorder) RemoveVfRepresentorUdevRule(pfPciAddress interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) RemoveVfRepresentorUdevRule(pfPciAddress any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveVfRepresentorUdevRule", reflect.TypeOf((*MockHostManagerInterface)(nil).RemoveVfRepresentorUdevRule), pfPciAddress) } @@ -797,7 +921,7 @@ func (m *MockHostManagerInterface) ResetSriovDevice(ifaceStatus v1.InterfaceExt) } // ResetSriovDevice indicates an expected call of ResetSriovDevice. -func (mr *MockHostManagerInterfaceMockRecorder) ResetSriovDevice(ifaceStatus interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) ResetSriovDevice(ifaceStatus any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetSriovDevice", reflect.TypeOf((*MockHostManagerInterface)(nil).ResetSriovDevice), ifaceStatus) } @@ -811,7 +935,7 @@ func (m *MockHostManagerInterface) SetDevlinkDeviceParam(pciAddr, paramName, val } // SetDevlinkDeviceParam indicates an expected call of SetDevlinkDeviceParam. -func (mr *MockHostManagerInterfaceMockRecorder) SetDevlinkDeviceParam(pciAddr, paramName, value interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) SetDevlinkDeviceParam(pciAddr, paramName, value any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDevlinkDeviceParam", reflect.TypeOf((*MockHostManagerInterface)(nil).SetDevlinkDeviceParam), pciAddr, paramName, value) } @@ -825,7 +949,7 @@ func (m *MockHostManagerInterface) SetNetdevMTU(pciAddr string, mtu int) error { } // SetNetdevMTU indicates an expected call of SetNetdevMTU. -func (mr *MockHostManagerInterfaceMockRecorder) SetNetdevMTU(pciAddr, mtu interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) SetNetdevMTU(pciAddr, mtu any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetNetdevMTU", reflect.TypeOf((*MockHostManagerInterface)(nil).SetNetdevMTU), pciAddr, mtu) } @@ -839,11 +963,25 @@ func (m *MockHostManagerInterface) SetNicSriovMode(pciAddr, mode string) error { } // SetNicSriovMode indicates an expected call of SetNicSriovMode. -func (mr *MockHostManagerInterfaceMockRecorder) SetNicSriovMode(pciAddr, mode interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) SetNicSriovMode(pciAddr, mode any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetNicSriovMode", reflect.TypeOf((*MockHostManagerInterface)(nil).SetNicSriovMode), pciAddr, mode) } +// SetRDMASubsystem mocks base method. +func (m *MockHostManagerInterface) SetRDMASubsystem(mode string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetRDMASubsystem", mode) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetRDMASubsystem indicates an expected call of SetRDMASubsystem. +func (mr *MockHostManagerInterfaceMockRecorder) SetRDMASubsystem(mode any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetRDMASubsystem", reflect.TypeOf((*MockHostManagerInterface)(nil).SetRDMASubsystem), mode) +} + // SetSriovNumVfs mocks base method. func (m *MockHostManagerInterface) SetSriovNumVfs(pciAddr string, numVfs int) error { m.ctrl.T.Helper() @@ -853,7 +991,7 @@ func (m *MockHostManagerInterface) SetSriovNumVfs(pciAddr string, numVfs int) er } // SetSriovNumVfs indicates an expected call of SetSriovNumVfs. -func (mr *MockHostManagerInterfaceMockRecorder) SetSriovNumVfs(pciAddr, numVfs interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) SetSriovNumVfs(pciAddr, numVfs any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetSriovNumVfs", reflect.TypeOf((*MockHostManagerInterface)(nil).SetSriovNumVfs), pciAddr, numVfs) } @@ -867,7 +1005,7 @@ func (m *MockHostManagerInterface) SetVfAdminMac(vfAddr string, pfLink, vfLink n } // SetVfAdminMac indicates an expected call of SetVfAdminMac. -func (mr *MockHostManagerInterfaceMockRecorder) SetVfAdminMac(vfAddr, pfLink, vfLink interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) SetVfAdminMac(vfAddr, pfLink, vfLink any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetVfAdminMac", reflect.TypeOf((*MockHostManagerInterface)(nil).SetVfAdminMac), vfAddr, pfLink, vfLink) } @@ -905,7 +1043,7 @@ func (m *MockHostManagerInterface) TryGetInterfaceName(pciAddr string) string { } // TryGetInterfaceName indicates an expected call of TryGetInterfaceName. -func (mr *MockHostManagerInterfaceMockRecorder) TryGetInterfaceName(pciAddr interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) TryGetInterfaceName(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TryGetInterfaceName", reflect.TypeOf((*MockHostManagerInterface)(nil).TryGetInterfaceName), pciAddr) } @@ -919,7 +1057,7 @@ func (m *MockHostManagerInterface) TryToGetVirtualInterfaceName(pciAddr string) } // TryToGetVirtualInterfaceName indicates an expected call of TryToGetVirtualInterfaceName. -func (mr *MockHostManagerInterfaceMockRecorder) TryToGetVirtualInterfaceName(pciAddr interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) TryToGetVirtualInterfaceName(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TryToGetVirtualInterfaceName", reflect.TypeOf((*MockHostManagerInterface)(nil).TryToGetVirtualInterfaceName), pciAddr) } @@ -933,7 +1071,7 @@ func (m *MockHostManagerInterface) Unbind(pciAddr string) error { } // Unbind indicates an expected call of Unbind. -func (mr *MockHostManagerInterfaceMockRecorder) Unbind(pciAddr interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) Unbind(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Unbind", reflect.TypeOf((*MockHostManagerInterface)(nil).Unbind), pciAddr) } @@ -947,7 +1085,7 @@ func (m *MockHostManagerInterface) UnbindDriverByBusAndDevice(bus, device string } // UnbindDriverByBusAndDevice indicates an expected call of UnbindDriverByBusAndDevice. -func (mr *MockHostManagerInterfaceMockRecorder) UnbindDriverByBusAndDevice(bus, device interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) UnbindDriverByBusAndDevice(bus, device any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnbindDriverByBusAndDevice", reflect.TypeOf((*MockHostManagerInterface)(nil).UnbindDriverByBusAndDevice), bus, device) } @@ -961,7 +1099,7 @@ func (m *MockHostManagerInterface) UnbindDriverIfNeeded(pciAddr string, isRdma b } // UnbindDriverIfNeeded indicates an expected call of UnbindDriverIfNeeded. -func (mr *MockHostManagerInterfaceMockRecorder) UnbindDriverIfNeeded(pciAddr, isRdma interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) UnbindDriverIfNeeded(pciAddr, isRdma any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnbindDriverIfNeeded", reflect.TypeOf((*MockHostManagerInterface)(nil).UnbindDriverIfNeeded), pciAddr, isRdma) } @@ -975,7 +1113,7 @@ func (m *MockHostManagerInterface) UpdateSystemService(serviceObj *types.Service } // UpdateSystemService indicates an expected call of UpdateSystemService. -func (mr *MockHostManagerInterfaceMockRecorder) UpdateSystemService(serviceObj interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) UpdateSystemService(serviceObj any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateSystemService", reflect.TypeOf((*MockHostManagerInterface)(nil).UpdateSystemService), serviceObj) } @@ -990,7 +1128,64 @@ func (m *MockHostManagerInterface) VFIsReady(pciAddr string) (netlink.Link, erro } // VFIsReady indicates an expected call of VFIsReady. -func (mr *MockHostManagerInterfaceMockRecorder) VFIsReady(pciAddr interface{}) *gomock.Call { +func (mr *MockHostManagerInterfaceMockRecorder) VFIsReady(pciAddr any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VFIsReady", reflect.TypeOf((*MockHostManagerInterface)(nil).VFIsReady), pciAddr) } + +// WaitUdevEventsProcessed mocks base method. +func (m *MockHostManagerInterface) WaitUdevEventsProcessed(timeout int) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WaitUdevEventsProcessed", timeout) + ret0, _ := ret[0].(error) + return ret0 +} + +// WaitUdevEventsProcessed indicates an expected call of WaitUdevEventsProcessed. +func (mr *MockHostManagerInterfaceMockRecorder) WaitUdevEventsProcessed(timeout any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitUdevEventsProcessed", reflect.TypeOf((*MockHostManagerInterface)(nil).WaitUdevEventsProcessed), timeout) +} + +// WriteConfFile mocks base method. +func (m *MockHostManagerInterface) WriteConfFile(newState *v1.SriovNetworkNodeState) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WriteConfFile", newState) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// WriteConfFile indicates an expected call of WriteConfFile. +func (mr *MockHostManagerInterfaceMockRecorder) WriteConfFile(newState any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteConfFile", reflect.TypeOf((*MockHostManagerInterface)(nil).WriteConfFile), newState) +} + +// WriteSriovResult mocks base method. +func (m *MockHostManagerInterface) WriteSriovResult(result *types.SriovResult) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WriteSriovResult", result) + ret0, _ := ret[0].(error) + return ret0 +} + +// WriteSriovResult indicates an expected call of WriteSriovResult. +func (mr *MockHostManagerInterfaceMockRecorder) WriteSriovResult(result any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteSriovResult", reflect.TypeOf((*MockHostManagerInterface)(nil).WriteSriovResult), result) +} + +// WriteSriovSupportedNics mocks base method. +func (m *MockHostManagerInterface) WriteSriovSupportedNics() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WriteSriovSupportedNics") + ret0, _ := ret[0].(error) + return ret0 +} + +// WriteSriovSupportedNics indicates an expected call of WriteSriovSupportedNics. +func (mr *MockHostManagerInterfaceMockRecorder) WriteSriovSupportedNics() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteSriovSupportedNics", reflect.TypeOf((*MockHostManagerInterface)(nil).WriteSriovSupportedNics)) +} diff --git a/pkg/host/store/mock/mock_store.go b/pkg/host/store/mock/mock_store.go index d405543a2..b9cb3a907 100644 --- a/pkg/host/store/mock/mock_store.go +++ b/pkg/host/store/mock/mock_store.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: store.go +// +// Generated by this command: +// +// mockgen -destination mock/mock_store.go -source store.go +// // Package mock_store is a generated GoMock package. package mock_store @@ -7,14 +12,15 @@ package mock_store import ( reflect "reflect" - gomock "github.com/golang/mock/gomock" v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + gomock "go.uber.org/mock/gomock" ) // MockManagerInterface is a mock of ManagerInterface interface. type MockManagerInterface struct { ctrl *gomock.Controller recorder *MockManagerInterfaceMockRecorder + isgomock struct{} } // MockManagerInterfaceMockRecorder is the mock recorder for MockManagerInterface. @@ -74,7 +80,7 @@ func (m *MockManagerInterface) LoadPfsStatus(pciAddress string) (*v1.Interface, } // LoadPfsStatus indicates an expected call of LoadPfsStatus. -func (mr *MockManagerInterfaceMockRecorder) LoadPfsStatus(pciAddress interface{}) *gomock.Call { +func (mr *MockManagerInterfaceMockRecorder) LoadPfsStatus(pciAddress any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadPfsStatus", reflect.TypeOf((*MockManagerInterface)(nil).LoadPfsStatus), pciAddress) } @@ -88,7 +94,7 @@ func (m *MockManagerInterface) RemovePfAppliedStatus(pciAddress string) error { } // RemovePfAppliedStatus indicates an expected call of RemovePfAppliedStatus. -func (mr *MockManagerInterfaceMockRecorder) RemovePfAppliedStatus(pciAddress interface{}) *gomock.Call { +func (mr *MockManagerInterfaceMockRecorder) RemovePfAppliedStatus(pciAddress any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemovePfAppliedStatus", reflect.TypeOf((*MockManagerInterface)(nil).RemovePfAppliedStatus), pciAddress) } @@ -102,7 +108,7 @@ func (m *MockManagerInterface) SaveLastPfAppliedStatus(PfInfo *v1.Interface) err } // SaveLastPfAppliedStatus indicates an expected call of SaveLastPfAppliedStatus. -func (mr *MockManagerInterfaceMockRecorder) SaveLastPfAppliedStatus(PfInfo interface{}) *gomock.Call { +func (mr *MockManagerInterfaceMockRecorder) SaveLastPfAppliedStatus(PfInfo any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveLastPfAppliedStatus", reflect.TypeOf((*MockManagerInterface)(nil).SaveLastPfAppliedStatus), PfInfo) } @@ -116,7 +122,7 @@ func (m *MockManagerInterface) WriteCheckpointFile(arg0 *v1.SriovNetworkNodeStat } // WriteCheckpointFile indicates an expected call of WriteCheckpointFile. -func (mr *MockManagerInterfaceMockRecorder) WriteCheckpointFile(arg0 interface{}) *gomock.Call { +func (mr *MockManagerInterfaceMockRecorder) WriteCheckpointFile(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteCheckpointFile", reflect.TypeOf((*MockManagerInterface)(nil).WriteCheckpointFile), arg0) } diff --git a/pkg/host/store/store.go b/pkg/host/store/store.go index 67c0b17e3..a66542e1d 100644 --- a/pkg/host/store/store.go +++ b/pkg/host/store/store.go @@ -47,7 +47,7 @@ func createOperatorConfigFolderIfNeeded() error { _, err := os.Stat(SriovConfBasePathUse) if err != nil { if os.IsNotExist(err) { - err = os.MkdirAll(SriovConfBasePathUse, os.ModeDir) + err = os.MkdirAll(SriovConfBasePathUse, 0777) if err != nil { return fmt.Errorf("failed to create the sriov config folder on host in path %s: %v", SriovConfBasePathUse, err) } @@ -60,7 +60,7 @@ func createOperatorConfigFolderIfNeeded() error { _, err = os.Stat(PfAppliedConfigUse) if err != nil { if os.IsNotExist(err) { - err = os.MkdirAll(PfAppliedConfigUse, os.ModeDir) + err = os.MkdirAll(PfAppliedConfigUse, 0777) if err != nil { return fmt.Errorf("failed to create the pci folder on host in path %s: %v", PfAppliedConfigUse, err) } @@ -89,7 +89,7 @@ func (s *manager) ClearPCIAddressFolder() error { return fmt.Errorf("failed to remove the PCI address folder on path %s: %v", PfAppliedConfigUse, err) } - err = os.Mkdir(PfAppliedConfigUse, os.ModeDir) + err = os.Mkdir(PfAppliedConfigUse, 0777) if err != nil { return fmt.Errorf("failed to create the pci folder on host in path %s: %v", PfAppliedConfigUse, err) } @@ -174,7 +174,7 @@ func (s *manager) WriteCheckpointFile(ns *sriovnetworkv1.SriovNetworkNodeState) defer file.Close() log.Log.Info("WriteCheckpointFile(): try to decode the checkpoint file") if err = json.NewDecoder(file).Decode(&sriovnetworkv1.InitialState); err != nil { - log.Log.V(2).Error(err, "WriteCheckpointFile(): fail to decode, writing new file instead") + log.Log.Error(err, "WriteCheckpointFile(): fail to decode, writing new file instead") log.Log.Info("WriteCheckpointFile(): write checkpoint file") if err = file.Truncate(0); err != nil { return err diff --git a/pkg/host/store/store_suite_test.go b/pkg/host/store/store_suite_test.go new file mode 100644 index 000000000..c2e2b8319 --- /dev/null +++ b/pkg/host/store/store_suite_test.go @@ -0,0 +1,21 @@ +package store + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "go.uber.org/zap/zapcore" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +func TestStore(t *testing.T) { + log.SetLogger(zap.New( + zap.WriteTo(GinkgoWriter), + zap.Level(zapcore.Level(-2)), + zap.UseDevMode(true))) + RegisterFailHandler(Fail) + RunSpecs(t, "Package Store Suite") +} diff --git a/pkg/host/store/store_test.go b/pkg/host/store/store_test.go new file mode 100644 index 000000000..b5144d54d --- /dev/null +++ b/pkg/host/store/store_test.go @@ -0,0 +1,316 @@ +package store + +import ( + "os" + "path" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" +) + +var _ = Describe("Store", func() { + var ( + tempDir = "/tmp/sriov-test/" + err error + m ManagerInterface + + testInterface = &sriovnetworkv1.Interface{ + Name: "enp216s0f0np0", + PciAddress: "0000:d8:00.0", + NumVfs: 1, + LinkType: "IB", + VfGroups: []sriovnetworkv1.VfGroup{ + { + VfRange: "0-0", + ResourceName: "test-resource0", + }, + }, + } + + testInterfaceData = []byte(`{ + "name": "enp216s0f0np0", + "pciAddress": "0000:d8:00.0", + "numVfs": 1, + "linkType": "IB", + "vfGroups": [ + { + "vfRange": "0-0", + "resourceName": "test-resource0", + "policyName": "test-policy0" + } + ], + "mtu": 2000 +}`) + + testNodeState = &sriovnetworkv1.SriovNetworkNodeState{ObjectMeta: metav1.ObjectMeta{ + Name: "worker-0", + }, + Status: sriovnetworkv1.SriovNetworkNodeStateStatus{ + Interfaces: sriovnetworkv1.InterfaceExts{ + { + Name: "eno1np0", + PciAddress: "0000:8d:00.0", + }, + }, + }, + } + + testNodeStateData = []byte(`{ + "apiVersion": "sriovnetwork.openshift.io/v1", + "kind": "SriovNetworkNodeState", + "metadata": { + "annotations": { + "sriovnetwork.openshift.io/current-state": "Idle", + "sriovnetwork.openshift.io/desired-state": "Idle" + }, + "creationTimestamp": "2024-10-10T13:40:32Z", + "generation": 1, + "name": "worker-0", + "namespace": "sriov-network-operator", + "ownerReferences": [ + { + "apiVersion": "sriovnetwork.openshift.io/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "SriovOperatorConfig", + "name": "default", + "uid": "4bb72ddd-82be-4a8a-a91b-1cb24f500f77" + } + ], + "resourceVersion": "96479400", + "uid": "0d11adf3-dd52-4906-a15c-87bf6271c946" + }, + "spec": { + "bridges": {}, + "system": {} + }, + "status": { + "bridges": {}, + "interfaces": [ + { + "deviceID": "1015", + "driver": "mlx5_core", + "linkAdminState": "up", + "linkSpeed": "10000 Mb/s", + "linkType": "ETH", + "mac": "0c:42:a1:6c:ac:9c", + "mtu": 1500, + "name": "eno1np0", + "pciAddress": "0000:8d:00.0", + "vendor": "15b3" + } + ], + "syncStatus": "Succeeded", + "system": { + "rdmaMode": "shared" + } + } +}`) + ) + + BeforeEach(func() { + vars.InChroot = true + vars.FilesystemRoot = tempDir + vars.Destdir = tempDir + sriovnetworkv1.NicIDMap = []string{} + + m, err = NewManager() + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + err := os.RemoveAll(tempDir) + Expect(err).ToNot(HaveOccurred()) + }) + + Context("NewManager", func() { + It("should not return error if it's able to create the folder", func() { + _, err := NewManager() + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Context("createOperatorConfigFolderIfNeeded", func() { + It("should have the folder created after getting an instance of the store manager", func() { + _, err = os.Stat(utils.GetHostExtensionPath(consts.SriovConfBasePath)) + Expect(err).ToNot(HaveOccurred()) + + _, err = os.Stat(utils.GetHostExtensionPath(consts.PfAppliedConfig)) + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Context("ClearPCIAddressFolder", func() { + It("should return without error if the folder doesn't exist", func() { + err = os.RemoveAll(utils.GetHostExtensionPath(consts.PfAppliedConfig)) + Expect(err).ToNot(HaveOccurred()) + + err = m.ClearPCIAddressFolder() + Expect(err).ToNot(HaveOccurred()) + }) + + It("should remove the folder and recreate an empty new one", func() { + err := os.WriteFile(path.Join(utils.GetHostExtensionPath(consts.PfAppliedConfig), "test"), []byte("test"), 0644) + Expect(err).ToNot(HaveOccurred()) + + err = m.ClearPCIAddressFolder() + Expect(err).ToNot(HaveOccurred()) + + _, err = os.Stat(path.Join(utils.GetHostExtensionPath(consts.PfAppliedConfig), "test")) + Expect(err).To(HaveOccurred()) + Expect(os.IsNotExist(err)).To(BeTrue()) + }) + }) + + Context("SaveLastPfAppliedStatus", func() { + It("should failed to write the file if path doesn't exist", func() { + err = os.RemoveAll(utils.GetHostExtensionPath(consts.PfAppliedConfig)) + Expect(err).ToNot(HaveOccurred()) + + err = m.SaveLastPfAppliedStatus(testInterface) + Expect(err).To(HaveOccurred()) + }) + + It("should write the file", func() { + err = m.SaveLastPfAppliedStatus(testInterface) + Expect(err).ToNot(HaveOccurred()) + + _, err = os.Stat(path.Join(utils.GetHostExtensionPath(consts.PfAppliedConfig), "0000:d8:00.0")) + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Context("RemovePfAppliedStatus", func() { + It("should remove the file", func() { + err = m.SaveLastPfAppliedStatus(testInterface) + Expect(err).ToNot(HaveOccurred()) + + err = m.RemovePfAppliedStatus("0000:d8:00.0") + Expect(err).ToNot(HaveOccurred()) + + _, err = os.Stat(path.Join(utils.GetHostExtensionPath(consts.PfAppliedConfig), "0000:d8:00.0")) + Expect(err).To(HaveOccurred()) + Expect(os.IsNotExist(err)).To(BeTrue()) + }) + + It("should not return error if the file doesn't exist", func() { + err = m.RemovePfAppliedStatus("0000:d8:00.0") + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Context("LoadPfsStatus", func() { + It("should return error if not able to parse the file", func() { + err = os.WriteFile(path.Join(utils.GetHostExtensionPath(consts.PfAppliedConfig), "0000:d8:00.0"), []byte("test"), 0644) + Expect(err).ToNot(HaveOccurred()) + + _, _, err = m.LoadPfsStatus("0000:d8:00.0") + Expect(err).To(HaveOccurred()) + }) + + It("should not return error if the file doesn't exist and false", func() { + _, exist, err := m.LoadPfsStatus("0000:d8:00.0") + Expect(err).ToNot(HaveOccurred()) + Expect(exist).To(BeFalse()) + }) + + It("should return a valid PFStatus", func() { + err = os.WriteFile(path.Join(utils.GetHostExtensionPath(consts.PfAppliedConfig), "0000:d8:00.0"), testInterfaceData, 0644) + Expect(err).ToNot(HaveOccurred()) + + loaded, exist, err := m.LoadPfsStatus("0000:d8:00.0") + Expect(err).ToNot(HaveOccurred()) + Expect(exist).To(BeTrue()) + Expect(loaded.PciAddress).To(Equal(testInterface.PciAddress)) + Expect(loaded.Name).To(Equal(testInterface.Name)) + }) + }) + + Context("GetCheckPointNodeState", func() { + It("should return not error and return empty struct if file doesn't exist", func() { + ns, err := m.GetCheckPointNodeState() + Expect(err).ToNot(HaveOccurred()) + Expect(ns).To(BeNil()) + }) + + It("should return not error if not able to parse the file", func() { + err = os.WriteFile(path.Join(vars.Destdir, consts.CheckpointFileName), []byte("test"), 0644) + Expect(err).ToNot(HaveOccurred()) + + _, err = m.GetCheckPointNodeState() + Expect(err).To(HaveOccurred()) + }) + + It("should be able to decode the file and also save a global variable", func() { + Expect(sriovnetworkv1.InitialState.Name).To(Equal("")) + err = os.WriteFile(path.Join(vars.Destdir, consts.CheckpointFileName), testNodeStateData, 0644) + Expect(err).ToNot(HaveOccurred()) + + ns, err := m.GetCheckPointNodeState() + Expect(err).ToNot(HaveOccurred()) + Expect(ns).ToNot(BeNil()) + Expect(sriovnetworkv1.InitialState.Name).To(Equal("worker-0")) + Expect(equality.Semantic.DeepEqual(*ns, sriovnetworkv1.InitialState)).To(BeTrue()) + }) + }) + + Context("WriteCheckpointFile", func() { + It("should return error if the path doesn't exist", func() { + vars.Destdir = path.Join(vars.Destdir, "not-exist") + defer func() { + vars.Destdir = vars.FilesystemRoot + }() + + err = m.WriteCheckpointFile(testNodeState) + Expect(err).To(HaveOccurred()) + }) + + It("should write the file and return no error", func() { + err = m.WriteCheckpointFile(testNodeState) + Expect(err).ToNot(HaveOccurred()) + + _, err = os.Stat(path.Join(vars.Destdir, consts.CheckpointFileName)) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should not override the file if it already exist", func() { + _, err = os.Stat(path.Join(vars.Destdir, consts.CheckpointFileName)) + Expect(err).To(HaveOccurred()) + Expect(os.IsNotExist(err)).To(BeTrue()) + + err = m.WriteCheckpointFile(testNodeState) + Expect(err).ToNot(HaveOccurred()) + + _, err = os.Stat(path.Join(vars.Destdir, consts.CheckpointFileName)) + Expect(err).ToNot(HaveOccurred()) + + copyTestNode := testNodeState.DeepCopy() + copyTestNode.Name = "worker-1" + err = m.WriteCheckpointFile(copyTestNode) + Expect(err).ToNot(HaveOccurred()) + + ns, err := m.GetCheckPointNodeState() + Expect(err).ToNot(HaveOccurred()) + Expect(ns.Name).To(Equal("worker-0")) + }) + + It("should override the file if not able to decode it", func() { + err = os.WriteFile(path.Join(vars.Destdir, consts.CheckpointFileName), []byte("test"), 0644) + Expect(err).ToNot(HaveOccurred()) + + err = m.WriteCheckpointFile(testNodeState) + Expect(err).ToNot(HaveOccurred()) + + ns, err := m.GetCheckPointNodeState() + Expect(err).ToNot(HaveOccurred()) + Expect(ns.Name).To(Equal("worker-0")) + }) + }) +}) diff --git a/pkg/host/types/interfaces.go b/pkg/host/types/interfaces.go index 5918dca34..aa4d1f77e 100644 --- a/pkg/host/types/interfaces.go +++ b/pkg/host/types/interfaces.go @@ -1,6 +1,8 @@ package types import ( + "net" + "github.com/vishvananda/netlink" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" @@ -90,6 +92,10 @@ type NetworkInterface interface { GetNetDevLinkAdminState(ifaceName string) string // GetPciAddressFromInterfaceName parses sysfs to get pci address of an interface by name GetPciAddressFromInterfaceName(interfaceName string) (string, error) + // DiscoverRDMASubsystem returns RDMA subsystem mode + DiscoverRDMASubsystem() (string, error) + // SetRDMASubsystem changes RDMA subsystem mode + SetRDMASubsystem(mode string) error } type ServiceInterface interface { @@ -160,6 +166,9 @@ type UdevInterface interface { RemoveVfRepresentorUdevRule(pfPciAddress string) error // LoadUdevRules triggers udev rules for network subsystem LoadUdevRules() error + // WaitUdevEventsProcessed calls `udevadm settle“ with provided timeout + // The command watches the udev event queue, and exits if all current events are handled. + WaitUdevEventsProcessed(timeout int) error } type VdpaInterface interface { @@ -186,4 +195,29 @@ type BridgeInterface interface { type InfinibandInterface interface { // ConfigureVfGUID configures and sets a GUID for an IB VF device ConfigureVfGUID(vfAddr string, pfAddr string, vfID int, pfLink netlink.Link) error + GetVfGUID(vfAddr string, pfAddr string, vfID int) (net.HardwareAddr, error) +} + +type CPUVendor int + +const ( + CPUVendorIntel CPUVendor = iota + CPUVendorAMD + CPUVendorARM +) + +type CPUInfoProviderInterface interface { + // Retrieve the CPU vendor of the current system + GetCPUVendor() (CPUVendor, error) +} + +type SystemdInterface interface { + ReadConfFile() (spec *SriovConfig, err error) + WriteConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (bool, error) + WriteSriovResult(result *SriovResult) error + ReadSriovResult() (*SriovResult, error) + RemoveSriovResult() error + WriteSriovSupportedNics() error + ReadSriovSupportedNics() ([]string, error) + CleanSriovFilesFromHost(isOpenShift bool) error } diff --git a/pkg/host/types/types.go b/pkg/host/types/types.go index 248163670..0039e87dc 100644 --- a/pkg/host/types/types.go +++ b/pkg/host/types/types.go @@ -1,5 +1,10 @@ package types +import ( + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" +) + // Service contains info about systemd service type Service struct { Name string @@ -28,3 +33,18 @@ type ScriptManifestFile struct { Inline string } } + +// SriovConfig: Contains the information we saved on the host for the sriov-config service running on the host +type SriovConfig struct { + Spec sriovnetworkv1.SriovNetworkNodeStateSpec `yaml:"spec"` + UnsupportedNics bool `yaml:"unsupportedNics"` + PlatformType consts.PlatformTypes `yaml:"platformType"` + ManageSoftwareBridges bool `yaml:"manageSoftwareBridges"` + OVSDBSocketPath string `yaml:"ovsdbSocketPath"` +} + +// SriovResult: Contains the result from the sriov-config service trying to apply the requested policies +type SriovResult struct { + SyncStatus string `yaml:"syncStatus"` + LastSyncError string `yaml:"lastSyncError"` +} diff --git a/pkg/leaderelection/leaderelection.go b/pkg/leaderelection/leaderelection.go deleted file mode 100644 index ff7570c3b..000000000 --- a/pkg/leaderelection/leaderelection.go +++ /dev/null @@ -1,50 +0,0 @@ -package leaderelection - -import ( - "time" - - "k8s.io/client-go/tools/leaderelection" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" -) - -const ( - // Defaults follow conventions - // https://github.com/openshift/enhancements/blob/master/CONVENTIONS.md#high-availability - // Impl Calculations: https://github.com/openshift/library-go/commit/7e7d216ed91c3119800219c9194e5e57113d059a - defaultLeaseDuration = 137 * time.Second - defaultRenewDeadline = 107 * time.Second - defaultRetryPeriod = 26 * time.Second -) - -func GetLeaderElectionConfig(c client.Client, enabled bool) (defaultConfig leaderelection.LeaderElectionConfig) { - defaultConfig = leaderelection.LeaderElectionConfig{ - LeaseDuration: defaultLeaseDuration, - RenewDeadline: defaultRenewDeadline, - RetryPeriod: defaultRetryPeriod, - } - - if enabled { - isSingleNode, err := utils.IsSingleNodeCluster(c) - if err != nil { - log.Log.Error(err, "warning, unable to get cluster infrastructure status, using HA cluster values for leader election") - return - } - if isSingleNode { - return leaderElectionSingleNodeConfig(defaultConfig) - } - } - return -} - -// Default leader election for Single Node environments -// Impl Calculations: -// https://github.com/openshift/library-go/commit/2612981f3019479805ac8448b997266fc07a236a#diff-61dd95c7fd45fa18038e825205fbfab8a803f1970068157608b6b1e9e6c27248R127 -func leaderElectionSingleNodeConfig(config leaderelection.LeaderElectionConfig) leaderelection.LeaderElectionConfig { - config.LeaseDuration = 270 * time.Second - config.RenewDeadline = 240 * time.Second - config.RetryPeriod = 60 * time.Second - return config -} diff --git a/pkg/log/log.go b/pkg/log/log.go index 1e76facc5..5ecc16893 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -60,6 +60,10 @@ func SetLogLevel(operatorLevel int) { } } +func GetLogLevel() int { + return zapToOperatorLevel(Options.Level.(zzap.AtomicLevel).Level()) +} + func zapToOperatorLevel(zapLevel zapcore.Level) int { return int(zapLevel) * -1 } diff --git a/pkg/platforms/mock/mock_platforms.go b/pkg/platforms/mock/mock_platforms.go index 4218ad045..015b7f731 100644 --- a/pkg/platforms/mock/mock_platforms.go +++ b/pkg/platforms/mock/mock_platforms.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: platforms.go +// +// Generated by this command: +// +// mockgen -destination mock/mock_platforms.go -source platforms.go +// // Package mock_platforms is a generated GoMock package. package mock_platforms @@ -8,10 +13,10 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" openshift "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openshift" - v10 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + v10 "github.com/openshift/api/machineconfiguration/v1" + gomock "go.uber.org/mock/gomock" v11 "k8s.io/api/core/v1" ) @@ -19,6 +24,7 @@ import ( type MockInterface struct { ctrl *gomock.Controller recorder *MockInterfaceMockRecorder + isgomock struct{} } // MockInterfaceMockRecorder is the mock recorder for MockInterface. @@ -47,7 +53,7 @@ func (m *MockInterface) ChangeMachineConfigPoolPause(arg0 context.Context, arg1 } // ChangeMachineConfigPoolPause indicates an expected call of ChangeMachineConfigPoolPause. -func (mr *MockInterfaceMockRecorder) ChangeMachineConfigPoolPause(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockInterfaceMockRecorder) ChangeMachineConfigPoolPause(arg0, arg1, arg2 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChangeMachineConfigPoolPause", reflect.TypeOf((*MockInterface)(nil).ChangeMachineConfigPoolPause), arg0, arg1, arg2) } @@ -73,7 +79,7 @@ func (m *MockInterface) CreateOpenstackDevicesInfoFromNodeStatus(arg0 *v1.SriovN } // CreateOpenstackDevicesInfoFromNodeStatus indicates an expected call of CreateOpenstackDevicesInfoFromNodeStatus. -func (mr *MockInterfaceMockRecorder) CreateOpenstackDevicesInfoFromNodeStatus(arg0 interface{}) *gomock.Call { +func (mr *MockInterfaceMockRecorder) CreateOpenstackDevicesInfoFromNodeStatus(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOpenstackDevicesInfoFromNodeStatus", reflect.TypeOf((*MockInterface)(nil).CreateOpenstackDevicesInfoFromNodeStatus), arg0) } @@ -117,7 +123,7 @@ func (m *MockInterface) GetNodeMachinePoolName(arg0 context.Context, arg1 *v11.N } // GetNodeMachinePoolName indicates an expected call of GetNodeMachinePoolName. -func (mr *MockInterfaceMockRecorder) GetNodeMachinePoolName(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockInterfaceMockRecorder) GetNodeMachinePoolName(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNodeMachinePoolName", reflect.TypeOf((*MockInterface)(nil).GetNodeMachinePoolName), arg0, arg1) } @@ -160,7 +166,7 @@ func (m *MockInterface) OpenshiftAfterCompleteDrainNode(arg0 context.Context, ar } // OpenshiftAfterCompleteDrainNode indicates an expected call of OpenshiftAfterCompleteDrainNode. -func (mr *MockInterfaceMockRecorder) OpenshiftAfterCompleteDrainNode(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockInterfaceMockRecorder) OpenshiftAfterCompleteDrainNode(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OpenshiftAfterCompleteDrainNode", reflect.TypeOf((*MockInterface)(nil).OpenshiftAfterCompleteDrainNode), arg0, arg1) } @@ -175,7 +181,7 @@ func (m *MockInterface) OpenshiftBeforeDrainNode(arg0 context.Context, arg1 *v11 } // OpenshiftBeforeDrainNode indicates an expected call of OpenshiftBeforeDrainNode. -func (mr *MockInterfaceMockRecorder) OpenshiftBeforeDrainNode(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockInterfaceMockRecorder) OpenshiftBeforeDrainNode(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OpenshiftBeforeDrainNode", reflect.TypeOf((*MockInterface)(nil).OpenshiftBeforeDrainNode), arg0, arg1) } diff --git a/pkg/platforms/openshift/mock/mock_openshift.go b/pkg/platforms/openshift/mock/mock_openshift.go index 4bafa8ee5..4692cedba 100644 --- a/pkg/platforms/openshift/mock/mock_openshift.go +++ b/pkg/platforms/openshift/mock/mock_openshift.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: openshift.go +// +// Generated by this command: +// +// mockgen -destination mock/mock_openshift.go -source openshift.go +// // Package mock_openshift is a generated GoMock package. package mock_openshift @@ -8,9 +13,9 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" openshift "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openshift" - v1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + v1 "github.com/openshift/api/machineconfiguration/v1" + gomock "go.uber.org/mock/gomock" v10 "k8s.io/api/core/v1" ) @@ -18,6 +23,7 @@ import ( type MockOpenshiftContextInterface struct { ctrl *gomock.Controller recorder *MockOpenshiftContextInterfaceMockRecorder + isgomock struct{} } // MockOpenshiftContextInterfaceMockRecorder is the mock recorder for MockOpenshiftContextInterface. @@ -46,7 +52,7 @@ func (m *MockOpenshiftContextInterface) ChangeMachineConfigPoolPause(arg0 contex } // ChangeMachineConfigPoolPause indicates an expected call of ChangeMachineConfigPoolPause. -func (mr *MockOpenshiftContextInterfaceMockRecorder) ChangeMachineConfigPoolPause(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockOpenshiftContextInterfaceMockRecorder) ChangeMachineConfigPoolPause(arg0, arg1, arg2 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChangeMachineConfigPoolPause", reflect.TypeOf((*MockOpenshiftContextInterface)(nil).ChangeMachineConfigPoolPause), arg0, arg1, arg2) } @@ -75,7 +81,7 @@ func (m *MockOpenshiftContextInterface) GetNodeMachinePoolName(arg0 context.Cont } // GetNodeMachinePoolName indicates an expected call of GetNodeMachinePoolName. -func (mr *MockOpenshiftContextInterfaceMockRecorder) GetNodeMachinePoolName(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockOpenshiftContextInterfaceMockRecorder) GetNodeMachinePoolName(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNodeMachinePoolName", reflect.TypeOf((*MockOpenshiftContextInterface)(nil).GetNodeMachinePoolName), arg0, arg1) } @@ -118,7 +124,7 @@ func (m *MockOpenshiftContextInterface) OpenshiftAfterCompleteDrainNode(arg0 con } // OpenshiftAfterCompleteDrainNode indicates an expected call of OpenshiftAfterCompleteDrainNode. -func (mr *MockOpenshiftContextInterfaceMockRecorder) OpenshiftAfterCompleteDrainNode(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockOpenshiftContextInterfaceMockRecorder) OpenshiftAfterCompleteDrainNode(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OpenshiftAfterCompleteDrainNode", reflect.TypeOf((*MockOpenshiftContextInterface)(nil).OpenshiftAfterCompleteDrainNode), arg0, arg1) } @@ -133,7 +139,7 @@ func (m *MockOpenshiftContextInterface) OpenshiftBeforeDrainNode(arg0 context.Co } // OpenshiftBeforeDrainNode indicates an expected call of OpenshiftBeforeDrainNode. -func (mr *MockOpenshiftContextInterfaceMockRecorder) OpenshiftBeforeDrainNode(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockOpenshiftContextInterfaceMockRecorder) OpenshiftBeforeDrainNode(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OpenshiftBeforeDrainNode", reflect.TypeOf((*MockOpenshiftContextInterface)(nil).OpenshiftBeforeDrainNode), arg0, arg1) } diff --git a/pkg/platforms/openshift/openshift.go b/pkg/platforms/openshift/openshift.go index 3f7d3421c..8daceb90b 100644 --- a/pkg/platforms/openshift/openshift.go +++ b/pkg/platforms/openshift/openshift.go @@ -4,14 +4,16 @@ import ( "context" "fmt" "sync" + "time" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + configv1 "github.com/openshift/api/config/v1" + mcv1 "github.com/openshift/api/machineconfiguration/v1" mcoconsts "github.com/openshift/machine-config-operator/pkg/daemon/constants" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" @@ -27,6 +29,8 @@ const ( OpenshiftFlavorHypershift OpenshiftFlavor = "hypershift" // OpenshiftFlavorDefault covers all remaining flavors of openshift not explicitly called out above OpenshiftFlavorDefault OpenshiftFlavor = "default" + // default Infrastructure resource name for Openshift + infraResourceName = "cluster" ) //go:generate ../../../bin/mockgen -destination mock/mock_openshift.go -source openshift.go @@ -74,7 +78,7 @@ func New() (OpenshiftContextInterface, error) { return nil, err } - isHypershift, err := utils.IsExternalControlPlaneCluster(infraClient) + isHypershift, err := isExternalControlPlaneCluster(infraClient) if err != nil { return nil, err } @@ -137,7 +141,8 @@ func (c *openshiftContext) OpenshiftBeforeDrainNode(ctx context.Context, node *c if !mcp.Spec.Paused { // if the machine config pool needs to update then we return false // if they are equal we can pause the pool - if mcp.Spec.Configuration.Name != mcp.Status.Configuration.Name { + if mcp.Spec.Configuration.Name == "" || mcp.Status.Configuration.Name == "" || + mcp.Spec.Configuration.Name != mcp.Status.Configuration.Name { return false, err } else { err = c.ChangeMachineConfigPoolPause(ctx, mcp, true) @@ -228,6 +233,18 @@ func (c *openshiftContext) OpenshiftAfterCompleteDrainNode(ctx context.Context, return false, err } + value, exist := mcp.Annotations[consts.MachineConfigPoolPausedAnnotation] + // if the label doesn't exist we just return true here + // this can be a case where the node was moved to another MCP in the time we start the drain + if !exist { + return true, nil + } + // check if the sriov annotation on mcp is idle + // if the value is idle we just return here + if value == consts.MachineConfigPoolPausedAnnotationIdle { + return true, nil + } + // get all the nodes that belong to this machine config pool to validate this is the last node // request to complete the drain nodesInPool := &corev1.NodeList{} @@ -273,17 +290,25 @@ func (c *openshiftContext) OpenshiftAfterCompleteDrainNode(ctx context.Context, } func (c *openshiftContext) GetNodeMachinePoolName(ctx context.Context, node *corev1.Node) (string, error) { + // if it's not an openshift cluster we return error + if !c.IsOpenshiftCluster() { + return "", fmt.Errorf("not an openshift cluster") + } + + // hyperShift cluster don't have machine config + if c.IsHypershift() { + return "", fmt.Errorf("hypershift doesn't have machineConfig") + } + desiredConfig, ok := node.Annotations[mcoconsts.DesiredMachineConfigAnnotationKey] if !ok { - log.Log.Error(nil, "getNodeMachinePool(): Failed to find the the desiredConfig Annotation") - return "", fmt.Errorf("getNodeMachinePool(): Failed to find the the desiredConfig Annotation") + return "", fmt.Errorf("failed to find the the annotation [%s] on node [%s]", mcoconsts.DesiredMachineConfigAnnotationKey, node.Name) } mc := &mcv1.MachineConfig{} err := c.kubeClient.Get(ctx, client.ObjectKey{Name: desiredConfig}, mc) if err != nil { - log.Log.Error(err, "getNodeMachinePool(): Failed to get the desired Machine Config") - return "", err + return "", fmt.Errorf("failed to get the desired MachineConfig [%s] for node [%s]: %w", desiredConfig, node.Name, err) } for _, owner := range mc.OwnerReferences { if owner.Kind == "MachineConfigPool" { @@ -291,12 +316,12 @@ func (c *openshiftContext) GetNodeMachinePoolName(ctx context.Context, node *cor } } - log.Log.Error(nil, "getNodeMachinePool(): Failed to find the MCP of the node") - return "", fmt.Errorf("getNodeMachinePool(): Failed to find the MCP of the node") + return "", fmt.Errorf("failed to find the MCP of the node") } func (c *openshiftContext) ChangeMachineConfigPoolPause(ctx context.Context, mcp *mcv1.MachineConfigPool, pause bool) error { - log.Log.Info("ChangeMachineConfigPoolPause:()") + logger := ctx.Value("logger").(logr.Logger).WithName("ChangeMachineConfigPoolPause") + logger.Info("change machine config pool state", "pause", pause, "mcp", mcp.Name) patchString := []byte(fmt.Sprintf(`{"spec":{"paused":%t}}`, pause)) patch := client.RawPatch(types.MergePatchType, patchString) @@ -307,3 +332,23 @@ func (c *openshiftContext) ChangeMachineConfigPoolPause(ctx context.Context, mcp return nil } + +// IsExternalControlPlaneCluster detects control plane location of the cluster. +// On OpenShift, the control plane topology is configured in configv1.Infrastucture struct. +// On kubernetes, it is determined by which node the sriov operator is scheduled on. If operator +// pod is schedule on worker node, it is considered as external control plane. +func isExternalControlPlaneCluster(c client.Client) (bool, error) { + ctx, cancelFunc := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelFunc() + + infra := &configv1.Infrastructure{} + err := c.Get(ctx, types.NamespacedName{Name: infraResourceName}, infra) + if err != nil { + return false, fmt.Errorf("openshiftControlPlaneTopologyStatus(): Failed to get Infrastructure (name: %s): %w", infraResourceName, err) + } + + if infra.Status.ControlPlaneTopology == "External" { + return true, nil + } + return false, nil +} diff --git a/pkg/platforms/openshift/openshift_suite_test.go b/pkg/platforms/openshift/openshift_suite_test.go new file mode 100644 index 000000000..453b0d3cd --- /dev/null +++ b/pkg/platforms/openshift/openshift_suite_test.go @@ -0,0 +1,139 @@ +package openshift_test + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + openshiftconfigv1 "github.com/openshift/api/config/v1" + mcfgv1 "github.com/openshift/api/machineconfiguration/v1" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "go.uber.org/zap/zapcore" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + k8sClient client.Client + testEnv *envtest.Environment + cfg *rest.Config +) + +// Define utility constants for object names and testing timeouts/durations and intervals. +const testNamespace = "openshift-sriov-network-operator" + +var _ = BeforeSuite(func() { + var err error + + logf.SetLogger(zap.New( + zap.WriteTo(GinkgoWriter), + zap.UseDevMode(true), + func(o *zap.Options) { + o.TimeEncoder = zapcore.RFC3339NanoTimeEncoder + })) + + // Go to project root directory + err = os.Chdir("../../..") + Expect(err).NotTo(HaveOccurred()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("config", "crd", "bases"), filepath.Join("test", "util", "crds")}, + ErrorIfCRDPathMissing: true, + } + + testEnv.ControlPlane.GetAPIServer().Configure().Set("disable-admission-plugins", "MutatingAdmissionWebhook", "ValidatingAdmissionWebhook") + + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + By("registering schemes") + err = sriovnetworkv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = netattdefv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = mcfgv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = openshiftconfigv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = monitoringv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + vars.Config = cfg + vars.Scheme = scheme.Scheme + vars.Namespace = testNamespace + vars.ResourcePrefix = "sriovoperator.io" + + By("creating K8s client") + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + By("creating default/common k8s objects for tests") + // Create test namespace + ns := &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: testNamespace, + }, + Spec: corev1.NamespaceSpec{}, + Status: corev1.NamespaceStatus{}, + } + Expect(k8sClient.Create(context.Background(), ns)).Should(Succeed()) + + sa := &corev1.ServiceAccount{TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: testNamespace, + }} + Expect(k8sClient.Create(context.Background(), sa)).Should(Succeed()) + + // Create openshift Infrastructure + infra := &openshiftconfigv1.Infrastructure{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + Spec: openshiftconfigv1.InfrastructureSpec{}, + } + Expect(k8sClient.Create(context.Background(), infra)).Should(Succeed()) + + infra.Status = openshiftconfigv1.InfrastructureStatus{ + ControlPlaneTopology: openshiftconfigv1.HighlyAvailableTopologyMode, + InfrastructureTopology: openshiftconfigv1.HighlyAvailableTopologyMode, + } + Expect(k8sClient.Status().Update(context.Background(), infra)).Should(Succeed()) +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + if testEnv != nil { + Eventually(func() error { + return testEnv.Stop() + }, util.APITimeout, time.Second).ShouldNot(HaveOccurred()) + } +}) + +func TestOpenshift(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Openshift Suite") +} diff --git a/pkg/platforms/openshift/openshift_test.go b/pkg/platforms/openshift/openshift_test.go new file mode 100644 index 000000000..83643e068 --- /dev/null +++ b/pkg/platforms/openshift/openshift_test.go @@ -0,0 +1,548 @@ +package openshift_test + +import ( + "context" + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + configv1 "github.com/openshift/api/config/v1" + mcv1 "github.com/openshift/api/machineconfiguration/v1" + mcoconsts "github.com/openshift/machine-config-operator/pkg/daemon/constants" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openshift" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" +) + +var ( + cancel context.CancelFunc + ctx context.Context + err error + op openshift.OpenshiftContextInterface +) + +var _ = Describe("Openshift Package", Ordered, func() { + BeforeAll(func() { + Expect(k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovOperatorConfig{}, client.InNamespace(testNamespace))).ToNot(HaveOccurred()) + soc := &sriovnetworkv1.SriovOperatorConfig{ObjectMeta: metav1.ObjectMeta{ + Name: constants.DefaultConfigName, + Namespace: testNamespace, + }, + Spec: sriovnetworkv1.SriovOperatorConfigSpec{ + LogLevel: 2, + }, + } + err := k8sClient.Create(context.Background(), soc) + Expect(err).ToNot(HaveOccurred()) + + snolog.SetLogLevel(2) + // Check if the environment variable CLUSTER_TYPE is set + if clusterType, ok := os.LookupEnv("CLUSTER_TYPE"); ok && clusterType == constants.ClusterTypeOpenshift { + vars.ClusterType = constants.ClusterTypeOpenshift + } else { + vars.ClusterType = constants.ClusterTypeKubernetes + } + }) + + BeforeEach(func() { + ctx, cancel = context.WithCancel(context.Background()) + ctx = context.WithValue(ctx, "logger", log.FromContext(ctx)) + + op, err = openshift.New() + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + Expect(k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovNetworkNodeState{}, client.InNamespace(testNamespace))).ToNot(HaveOccurred()) + Expect(k8sClient.DeleteAllOf(context.Background(), &corev1.Pod{}, client.InNamespace(testNamespace), &client.DeleteAllOfOptions{DeleteOptions: client.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)}})).ToNot(HaveOccurred()) + Expect(k8sClient.DeleteAllOf(context.Background(), &corev1.Node{}, &client.DeleteAllOfOptions{DeleteOptions: client.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)}})).ToNot(HaveOccurred()) + Expect(k8sClient.DeleteAllOf(context.Background(), &mcv1.MachineConfigPool{}, &client.DeleteAllOfOptions{DeleteOptions: client.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)}})).ToNot(HaveOccurred()) + Expect(k8sClient.DeleteAllOf(context.Background(), &mcv1.MachineConfig{}, &client.DeleteAllOfOptions{DeleteOptions: client.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)}})).ToNot(HaveOccurred()) + + By("Shutdown controller manager") + cancel() + }) + + AfterAll(func() { + Expect(k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovOperatorConfig{}, client.InNamespace(testNamespace))).ToNot(HaveOccurred()) + }) + + Context("On Kubernetes cluster", func() { + BeforeEach(func() { + if vars.ClusterType != constants.ClusterTypeKubernetes { + Skip("Cluster type is not Kubernetes") + } + }) + + It("should return an empty struct", func() { + temp, err := openshift.New() + Expect(err).ToNot(HaveOccurred()) + Expect(temp.IsOpenshiftCluster()).To(BeFalse()) + }) + + It("should return false for IsOpenshiftCluster function", func() { + Expect(op.IsOpenshiftCluster()).To(BeFalse()) + }) + + It("should return empty string for GetFlavor function", func() { + Expect(op.GetFlavor()).To(Equal(openshift.OpenshiftFlavor(""))) + }) + + It("should return false for IsHypershift function", func() { + Expect(op.IsHypershift()).To(BeFalse()) + }) + + It("should always return true for OpenshiftBeforeDrainNode", func() { + n := createNode("worker0") + complete, err := op.OpenshiftBeforeDrainNode(ctx, n) + Expect(err).ToNot(HaveOccurred()) + Expect(complete).To(BeTrue()) + }) + + It("should always return true for OpenshiftBeforeDrainNode even for invalid call like non existing node", func() { + n := createNode("worker0") + n.Name = "test" + complete, err := op.OpenshiftBeforeDrainNode(ctx, n) + Expect(err).ToNot(HaveOccurred()) + Expect(complete).To(BeTrue()) + }) + + It("should always return true for OpenshiftAfterCompleteDrainNode", func() { + n := createNode("worker0") + complete, err := op.OpenshiftAfterCompleteDrainNode(ctx, n) + Expect(err).ToNot(HaveOccurred()) + Expect(complete).To(BeTrue()) + }) + + It("should always return true for OpenshiftAfterCompleteDrainNode even for invalid call like non existing node", func() { + n := createNode("worker0") + n.Name = "test" + complete, err := op.OpenshiftAfterCompleteDrainNode(ctx, n) + Expect(err).ToNot(HaveOccurred()) + Expect(complete).To(BeTrue()) + }) + + It("should always return error for GetNodeMachinePoolName on a non openshift cluster", func() { + n := createNode("worker0") + _, err = op.GetNodeMachinePoolName(ctx, n) + Expect(err).To(HaveOccurred()) + }) + }) + + Context("On Openshift cluster", func() { + BeforeEach(func() { + if vars.ClusterType != constants.ClusterTypeOpenshift { + Skip("Cluster type is not openshift") + } + }) + + Context("OpenshiftBeforeDrainNode", func() { + It("should return true if the cluster is not openshift", func() { + vars.ClusterType = constants.ClusterTypeKubernetes + defer func() { + vars.ClusterType = constants.ClusterTypeOpenshift + }() + + op, err = openshift.New() + Expect(err).ToNot(HaveOccurred()) + + n := createNode("worker-0") + completed, err := op.OpenshiftBeforeDrainNode(ctx, n) + Expect(err).ToNot(HaveOccurred()) + Expect(completed).To(BeTrue()) + }) + + It("should return true if the cluster is hypershift", func() { + infra := &configv1.Infrastructure{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: "cluster"}, infra) + Expect(err).ToNot(HaveOccurred()) + infra.Status.ControlPlaneTopology = "External" + err = k8sClient.Status().Update(ctx, infra) + Expect(err).ToNot(HaveOccurred()) + defer func() { + infra.Status.ControlPlaneTopology = "HighlyAvailable" + err = k8sClient.Status().Update(ctx, infra) + Expect(err).ToNot(HaveOccurred()) + }() + + // recreate to update the topology + op, err = openshift.New() + Expect(err).ToNot(HaveOccurred()) + + n := createNode("worker-0") + completed, err := op.OpenshiftBeforeDrainNode(ctx, n) + Expect(err).ToNot(HaveOccurred()) + Expect(completed).To(BeTrue()) + }) + + It("should return true if the machine config is paused and it was done by the sriov operator", func() { + mcp := createMachineConfigPool("worker") + createMachineConfig("machine-config", mcp) + n := createNodeWithMCName("worker-0", "machine-config") + mcp.Annotations = map[string]string{constants.MachineConfigPoolPausedAnnotation: constants.MachineConfigPoolPausedAnnotationPaused} + mcp.Spec.Paused = true + err = k8sClient.Update(ctx, mcp) + Expect(err).ToNot(HaveOccurred()) + + completed, err := op.OpenshiftBeforeDrainNode(ctx, n) + Expect(err).ToNot(HaveOccurred()) + Expect(completed).To(BeTrue()) + }) + + It("should return false if machine config is paused but the machine config daemon is configuring the nodes", func() { + mcp := createMachineConfigPool("worker") + createMachineConfig("machine-config", mcp) + n := createNodeWithMCName("worker-0", "machine-config") + mcp.Annotations = map[string]string{constants.MachineConfigPoolPausedAnnotation: constants.MachineConfigPoolPausedAnnotationPaused} + mcp.Spec.Configuration.Name = "test" + err = k8sClient.Update(ctx, mcp) + Expect(err).ToNot(HaveOccurred()) + mcp.Status.Configuration.Name = "test1" + err = k8sClient.Status().Update(ctx, mcp) + Expect(err).ToNot(HaveOccurred()) + + completed, err := op.OpenshiftBeforeDrainNode(ctx, n) + Expect(err).ToNot(HaveOccurred()) + Expect(completed).To(BeFalse()) + }) + + It("should return true after patching the machineConfigPool with pause true", func() { + mcp := createMachineConfigPool("worker") + createMachineConfig("machine-config", mcp) + n := createNodeWithMCName("worker-0", "machine-config") + mcp.Annotations = map[string]string{constants.MachineConfigPoolPausedAnnotation: constants.MachineConfigPoolPausedAnnotationPaused} + mcp.Spec.Configuration.Name = "test" + err = k8sClient.Update(ctx, mcp) + Expect(err).ToNot(HaveOccurred()) + mcp.Status.Configuration.Name = "test" + err = k8sClient.Status().Update(ctx, mcp) + Expect(err).ToNot(HaveOccurred()) + + completed, err := op.OpenshiftBeforeDrainNode(ctx, n) + Expect(err).ToNot(HaveOccurred()) + Expect(completed).To(BeTrue()) + + err = k8sClient.Get(ctx, client.ObjectKeyFromObject(mcp), mcp) + Expect(err).ToNot(HaveOccurred()) + Expect(mcp.Spec.Paused).To(BeTrue()) + }) + + It("should add the sriov pause label and update the machine config pool spec with pause true", func() { + mcp := createMachineConfigPool("worker") + createMachineConfig("machine-config", mcp) + n := createNodeWithMCName("worker-0", "machine-config") + mcp.Spec.Configuration.Name = "test" + err = k8sClient.Update(ctx, mcp) + Expect(err).ToNot(HaveOccurred()) + mcp.Status.Configuration.Name = "test" + err = k8sClient.Status().Update(ctx, mcp) + Expect(err).ToNot(HaveOccurred()) + + completed, err := op.OpenshiftBeforeDrainNode(ctx, n) + Expect(err).ToNot(HaveOccurred()) + Expect(completed).To(BeTrue()) + + err = k8sClient.Get(ctx, client.ObjectKeyFromObject(mcp), mcp) + Expect(err).ToNot(HaveOccurred()) + Expect(mcp.Spec.Paused).To(BeTrue()) + value, exist := mcp.Annotations[constants.MachineConfigPoolPausedAnnotation] + Expect(exist).To(BeTrue()) + Expect(value).To(Equal(constants.MachineConfigPoolPausedAnnotationPaused)) + }) + }) + + Context("OpenshiftAfterCompleteDrainNode", func() { + It("should return true if the cluster is not openshift", func() { + vars.ClusterType = constants.ClusterTypeKubernetes + defer func() { + vars.ClusterType = constants.ClusterTypeOpenshift + }() + + op, err = openshift.New() + Expect(err).ToNot(HaveOccurred()) + + n := createNode("worker-0") + completed, err := op.OpenshiftAfterCompleteDrainNode(ctx, n) + Expect(err).ToNot(HaveOccurred()) + Expect(completed).To(BeTrue()) + }) + + It("should return true if the cluster is hypershift", func() { + infra := &configv1.Infrastructure{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: "cluster"}, infra) + Expect(err).ToNot(HaveOccurred()) + infra.Status.ControlPlaneTopology = "External" + err = k8sClient.Status().Update(ctx, infra) + Expect(err).ToNot(HaveOccurred()) + defer func() { + infra.Status.ControlPlaneTopology = "HighlyAvailable" + err = k8sClient.Status().Update(ctx, infra) + Expect(err).ToNot(HaveOccurred()) + }() + + // recreate to update the topology + op, err = openshift.New() + Expect(err).ToNot(HaveOccurred()) + + n := createNode("worker-0") + completed, err := op.OpenshiftAfterCompleteDrainNode(ctx, n) + Expect(err).ToNot(HaveOccurred()) + Expect(completed).To(BeTrue()) + }) + + It("should return true if the machine config pool doesn't have sriov annotation", func() { + mcp := createMachineConfigPool("worker") + createMachineConfig("machine-config", mcp) + n := createNodeWithMCName("worker-0", "machine-config") + + completed, err := op.OpenshiftAfterCompleteDrainNode(ctx, n) + Expect(err).ToNot(HaveOccurred()) + Expect(completed).To(BeTrue()) + }) + + It("should return true if the machine config pool have sriov annotation on idle", func() { + mcp := createMachineConfigPool("worker") + createMachineConfig("machine-config", mcp) + n := createNodeWithMCName("worker-0", "machine-config") + mcp.Annotations = map[string]string{constants.MachineConfigPoolPausedAnnotation: constants.MachineConfigPoolPausedAnnotationIdle} + err = k8sClient.Update(ctx, mcp) + Expect(err).ToNot(HaveOccurred()) + + completed, err := op.OpenshiftAfterCompleteDrainNode(ctx, n) + Expect(err).ToNot(HaveOccurred()) + Expect(completed).To(BeTrue()) + }) + + It("should return true without changing the machine config pool pause to false when other nodes are still draining", func() { + mcp := createMachineConfigPool("worker") + createMachineConfig("machine-config", mcp) + mcp.Annotations = map[string]string{constants.MachineConfigPoolPausedAnnotation: constants.MachineConfigPoolPausedAnnotationPaused} + mcp.Spec.Paused = true + mcp.Spec.Configuration.Name = "test" + mcp.Spec.NodeSelector = &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "test": "", + }, + } + err = k8sClient.Update(ctx, mcp) + Expect(err).ToNot(HaveOccurred()) + mcp.Status.Configuration.Name = "test" + err = k8sClient.Status().Update(ctx, mcp) + Expect(err).ToNot(HaveOccurred()) + + n1 := createNodeWithMCName("worker-0", "machine-config") + n1.Annotations[constants.NodeDrainAnnotation] = constants.Draining + err = k8sClient.Update(ctx, n1) + Expect(err).ToNot(HaveOccurred()) + + n2 := createNodeWithMCName("worker-1", "machine-config") + n2.Annotations[constants.NodeDrainAnnotation] = constants.Draining + err = k8sClient.Update(ctx, n2) + Expect(err).ToNot(HaveOccurred()) + + completed, err := op.OpenshiftAfterCompleteDrainNode(ctx, n1) + Expect(err).ToNot(HaveOccurred()) + Expect(completed).To(BeTrue()) + + err = k8sClient.Get(ctx, client.ObjectKeyFromObject(mcp), mcp) + Expect(err).ToNot(HaveOccurred()) + value, exist := mcp.Annotations[constants.MachineConfigPoolPausedAnnotation] + Expect(exist).To(BeTrue()) + Expect(value).To(Equal(constants.MachineConfigPoolPausedAnnotationPaused)) + Expect(mcp.Spec.Paused).To(BeTrue()) + + n1.Annotations[constants.NodeDrainAnnotation] = constants.DrainIdle + err = k8sClient.Update(ctx, n1) + Expect(err).ToNot(HaveOccurred()) + + completed, err = op.OpenshiftAfterCompleteDrainNode(ctx, n2) + Expect(err).ToNot(HaveOccurred()) + Expect(completed).To(BeTrue()) + + err = k8sClient.Get(ctx, client.ObjectKeyFromObject(mcp), mcp) + Expect(err).ToNot(HaveOccurred()) + value, exist = mcp.Annotations[constants.MachineConfigPoolPausedAnnotation] + Expect(exist).To(BeTrue()) + Expect(value).To(Equal(constants.MachineConfigPoolPausedAnnotationIdle)) + Expect(mcp.Spec.Paused).To(BeFalse()) + }) + }) + + Context("GetNodeMachinePoolName", func() { + It("should return error if the cluster is not openshift", func() { + vars.ClusterType = constants.ClusterTypeKubernetes + defer func() { + vars.ClusterType = constants.ClusterTypeOpenshift + }() + n := createNode("worker-0") + op, err = openshift.New() + Expect(err).ToNot(HaveOccurred()) + + _, err = op.GetNodeMachinePoolName(ctx, n) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("not an openshift cluster")) + }) + + It("should return error if the cluster is hypershift", func() { + infra := &configv1.Infrastructure{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: "cluster"}, infra) + Expect(err).ToNot(HaveOccurred()) + infra.Status.ControlPlaneTopology = "External" + err = k8sClient.Status().Update(ctx, infra) + Expect(err).ToNot(HaveOccurred()) + defer func() { + infra.Status.ControlPlaneTopology = "HighlyAvailable" + err = k8sClient.Status().Update(ctx, infra) + Expect(err).ToNot(HaveOccurred()) + }() + + // recreate to update the topology + op, err = openshift.New() + Expect(err).ToNot(HaveOccurred()) + + n := createNode("worker-0") + _, err = op.GetNodeMachinePoolName(ctx, n) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("hypershift doesn't have machineConfig")) + }) + + It("should return error if the node doesn't have the MCO annotation", func() { + n := createNode("worker-0") + delete(n.Annotations, mcoconsts.DesiredMachineConfigAnnotationKey) + err = k8sClient.Update(ctx, n) + Expect(err).ToNot(HaveOccurred()) + + _, err = op.GetNodeMachinePoolName(ctx, n) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("failed to find the the annotation [machineconfiguration.openshift.io/desiredConfig] on node [worker-0]")) + }) + + It("should return error if the name of the MC in the annotation doesn't exist", func() { + n := createNodeWithMCName("worker-0", "worker") + _, err = op.GetNodeMachinePoolName(ctx, n) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("failed to get the desired MachineConfig [worker] for node [worker-0]: machineconfigs.machineconfiguration.openshift.io \"worker\" not found")) + }) + + It("should return error if machine config pool doesn't exist in owner reference of machine config", func() { + n := createNodeWithMCName("worker-0", "worker") + createMachineConfig("worker", nil) + _, err = op.GetNodeMachinePoolName(ctx, n) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("failed to find the MCP of the node")) + }) + + It("should return the machine config pool name for a specific node base on the machine config owner reference", func() { + mcp := createMachineConfigPool("worker") + createMachineConfig("machine-config", mcp) + n := createNodeWithMCName("worker-0", "machine-config") + mcpName, err := op.GetNodeMachinePoolName(ctx, n) + Expect(err).ToNot(HaveOccurred()) + Expect(mcpName).To(Equal("worker")) + }) + }) + + Context("ChangeMachineConfigPoolPause", func() { + It("should failed to change if mcp doesn't exist", func() { + mcp := createMachineConfigPool("worker") + mcp.Name = "test" + err = op.ChangeMachineConfigPoolPause(ctx, mcp, false) + Expect(err).To(HaveOccurred()) + }) + + It("should switch the pause", func() { + mcp := createMachineConfigPool("worker") + Expect(mcp.Spec.Paused).To(BeFalse()) + err = op.ChangeMachineConfigPoolPause(ctx, mcp, true) + Expect(err).ToNot(HaveOccurred()) + Expect(mcp.Spec.Paused).To(BeTrue()) + }) + }) + }) +}) + +func createNode(nodeName string) *corev1.Node { + node := corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Annotations: map[string]string{ + constants.NodeDrainAnnotation: constants.DrainIdle, + mcoconsts.DesiredMachineConfigAnnotationKey: "worker", + }, + Labels: map[string]string{ + "test": "", + }, + }, + } + + Expect(k8sClient.Create(ctx, &node)).ToNot(HaveOccurred()) + vars.NodeName = nodeName + + return &node +} + +func createNodeWithMCName(nodeName, mcName string) *corev1.Node { + node := corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Annotations: map[string]string{ + constants.NodeDrainAnnotation: constants.DrainIdle, + mcoconsts.DesiredMachineConfigAnnotationKey: mcName, + }, + Labels: map[string]string{ + "test": "", + }, + }, + } + + Expect(k8sClient.Create(ctx, &node)).ToNot(HaveOccurred()) + vars.NodeName = nodeName + + return &node +} + +func createMachineConfigPool(poolName string) *mcv1.MachineConfigPool { + mcp := &mcv1.MachineConfigPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: poolName, + }, + Spec: mcv1.MachineConfigPoolSpec{ + Paused: false, + }, + } + + Expect(k8sClient.Create(ctx, mcp)).ToNot(HaveOccurred()) + return mcp +} + +func createMachineConfig(mcName string, mcp *mcv1.MachineConfigPool) *mcv1.MachineConfig { + mc := &mcv1.MachineConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: mcName, + }, + } + + if mcp != nil { + mc.OwnerReferences = []metav1.OwnerReference{ + { + Name: mcp.Name, + APIVersion: "machineconfiguration.openshift.io/v1", + Kind: "MachineConfigPool", + UID: mcp.UID, + }, + } + } + + Expect(k8sClient.Create(ctx, mc)).ToNot(HaveOccurred()) + return mc +} diff --git a/pkg/platforms/openstack/mock/mock_openstack.go b/pkg/platforms/openstack/mock/mock_openstack.go index 9ef989297..da1ea554c 100644 --- a/pkg/platforms/openstack/mock/mock_openstack.go +++ b/pkg/platforms/openstack/mock/mock_openstack.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: openstack.go +// +// Generated by this command: +// +// mockgen -destination mock/mock_openstack.go -source openstack.go +// // Package mock_openstack is a generated GoMock package. package mock_openstack @@ -7,14 +12,15 @@ package mock_openstack import ( reflect "reflect" - gomock "github.com/golang/mock/gomock" v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + gomock "go.uber.org/mock/gomock" ) // MockOpenstackInterface is a mock of OpenstackInterface interface. type MockOpenstackInterface struct { ctrl *gomock.Controller recorder *MockOpenstackInterfaceMockRecorder + isgomock struct{} } // MockOpenstackInterfaceMockRecorder is the mock recorder for MockOpenstackInterface. @@ -55,7 +61,7 @@ func (m *MockOpenstackInterface) CreateOpenstackDevicesInfoFromNodeStatus(arg0 * } // CreateOpenstackDevicesInfoFromNodeStatus indicates an expected call of CreateOpenstackDevicesInfoFromNodeStatus. -func (mr *MockOpenstackInterfaceMockRecorder) CreateOpenstackDevicesInfoFromNodeStatus(arg0 interface{}) *gomock.Call { +func (mr *MockOpenstackInterfaceMockRecorder) CreateOpenstackDevicesInfoFromNodeStatus(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOpenstackDevicesInfoFromNodeStatus", reflect.TypeOf((*MockOpenstackInterface)(nil).CreateOpenstackDevicesInfoFromNodeStatus), arg0) } diff --git a/pkg/platforms/openstack/openstack.go b/pkg/platforms/openstack/openstack.go index 94a9ae433..608ba6f87 100644 --- a/pkg/platforms/openstack/openstack.go +++ b/pkg/platforms/openstack/openstack.go @@ -5,6 +5,8 @@ import ( "fmt" "io" "os" + "os/exec" + "path/filepath" "strconv" "strings" @@ -21,15 +23,18 @@ import ( ) const ( - ospHostMetaDataDir = "/host/var/config/openstack/2018-08-27" - ospMetaDataDir = "/var/config/openstack/2018-08-27" - ospMetaDataBaseURL = "http://169.254.169.254/openstack/2018-08-27" - ospNetworkDataJSON = "network_data.json" - ospMetaDataJSON = "meta_data.json" - ospHostNetworkDataFile = ospHostMetaDataDir + "/" + ospNetworkDataJSON - ospHostMetaDataFile = ospHostMetaDataDir + "/" + ospMetaDataJSON - ospNetworkDataURL = ospMetaDataBaseURL + "/" + ospNetworkDataJSON - ospMetaDataURL = ospMetaDataBaseURL + "/" + ospMetaDataJSON + varConfigPath = "/var/config" + ospMetaDataBaseDir = "/openstack/2018-08-27" + ospMetaDataDir = varConfigPath + ospMetaDataBaseDir + ospMetaDataBaseURL = "http://169.254.169.254" + ospMetaDataBaseDir + ospNetworkDataJSON = "network_data.json" + ospMetaDataJSON = "meta_data.json" + ospNetworkDataURL = ospMetaDataBaseURL + "/" + ospNetworkDataJSON + ospMetaDataURL = ospMetaDataBaseURL + "/" + ospMetaDataJSON + // Config drive is defined as an iso9660 or vfat (deprecated) drive + // with the "config-2" label. + //https://docs.openstack.org/nova/latest/user/config-drive.html + configDriveLabel = "config-2" ) var ( @@ -109,9 +114,10 @@ func New(hostManager host.HostManagerInterface) OpenstackInterface { } // GetOpenstackData gets the metadata and network_data -func getOpenstackData(useHostPath bool) (metaData *OSPMetaData, networkData *OSPNetworkData, err error) { - metaData, networkData, err = getOpenstackDataFromConfigDrive(useHostPath) +func getOpenstackData(mountConfigDrive bool) (metaData *OSPMetaData, networkData *OSPNetworkData, err error) { + metaData, networkData, err = getOpenstackDataFromConfigDrive(mountConfigDrive) if err != nil { + log.Log.Error(err, "GetOpenStackData(): non-fatal error getting OpenStack data from config drive") metaData, networkData, err = getOpenstackDataFromMetadataService() if err != nil { return metaData, networkData, fmt.Errorf("GetOpenStackData(): error getting OpenStack data: %w", err) @@ -153,46 +159,109 @@ func getOpenstackData(useHostPath bool) (metaData *OSPMetaData, networkData *OSP return metaData, networkData, err } +// getConfigDriveDevice returns the config drive device which was found +func getConfigDriveDevice() (string, error) { + dev := "/dev/disk/by-label/" + configDriveLabel + if _, err := os.Stat(dev); os.IsNotExist(err) { + out, err := exec.Command( + "blkid", "-l", + "-t", "LABEL="+configDriveLabel, + "-o", "device", + ).CombinedOutput() + if err != nil { + return "", fmt.Errorf("unable to run blkid: %v", err) + } + dev = strings.TrimSpace(string(out)) + } + log.Log.Info("found config drive device", "device", dev) + return dev, nil +} + +// mountConfigDriveDevice mounts the config drive and return the path +func mountConfigDriveDevice(device string) (string, error) { + if device == "" { + return "", fmt.Errorf("device is empty") + } + tmpDir, err := os.MkdirTemp("", "sriov-configdrive") + if err != nil { + return "", fmt.Errorf("error creating temp directory: %w", err) + } + cmd := exec.Command("mount", "-o", "ro", "-t", "auto", device, tmpDir) + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("error mounting config drive: %w", err) + } + log.Log.V(2).Info("mounted config drive device", "device", device, "path", tmpDir) + return tmpDir, nil +} + +// ummountConfigDriveDevice ummounts the config drive device +func ummountConfigDriveDevice(path string) error { + if path == "" { + return fmt.Errorf("path is empty") + } + cmd := exec.Command("umount", path) + if err := cmd.Run(); err != nil { + return fmt.Errorf("error umounting config drive: %w", err) + } + log.Log.V(2).Info("umounted config drive", "path", path) + return nil +} + // getOpenstackDataFromConfigDrive reads the meta_data and network_data files -func getOpenstackDataFromConfigDrive(useHostPath bool) (metaData *OSPMetaData, networkData *OSPNetworkData, err error) { +func getOpenstackDataFromConfigDrive(mountConfigDrive bool) (metaData *OSPMetaData, networkData *OSPNetworkData, err error) { metaData = &OSPMetaData{} networkData = &OSPNetworkData{} + var configDrivePath string log.Log.Info("reading OpenStack meta_data from config-drive") var metadataf *os.File ospMetaDataFilePath := ospMetaDataFile - if useHostPath { - ospMetaDataFilePath = ospHostMetaDataFile + if mountConfigDrive { + configDriveDevice, err := getConfigDriveDevice() + if err != nil { + return metaData, networkData, fmt.Errorf("error finding config drive device: %w", err) + } + configDrivePath, err = mountConfigDriveDevice(configDriveDevice) + if err != nil { + return metaData, networkData, fmt.Errorf("error mounting config drive device: %w", err) + } + defer func() { + if e := ummountConfigDriveDevice(configDrivePath); err == nil && e != nil { + err = fmt.Errorf("error umounting config drive device: %w", e) + } + if e := os.Remove(configDrivePath); err == nil && e != nil { + err = fmt.Errorf("error removing temp directory %s: %w", configDrivePath, e) + } + }() + ospMetaDataFilePath = filepath.Join(configDrivePath, ospMetaDataBaseDir, ospMetaDataJSON) + ospNetworkDataFile = filepath.Join(configDrivePath, ospMetaDataBaseDir, ospNetworkDataJSON) } metadataf, err = os.Open(ospMetaDataFilePath) if err != nil { - return metaData, networkData, fmt.Errorf("error opening file %s: %w", ospHostMetaDataFile, err) + return metaData, networkData, fmt.Errorf("error opening file %s: %w", ospMetaDataFilePath, err) } defer func() { if e := metadataf.Close(); err == nil && e != nil { - err = fmt.Errorf("error closing file %s: %w", ospHostMetaDataFile, e) + err = fmt.Errorf("error closing file %s: %w", ospMetaDataFilePath, e) } }() if err = json.NewDecoder(metadataf).Decode(&metaData); err != nil { - return metaData, networkData, fmt.Errorf("error unmarshalling metadata from file %s: %w", ospHostMetaDataFile, err) + return metaData, networkData, fmt.Errorf("error unmarshalling metadata from file %s: %w", ospMetaDataFilePath, err) } log.Log.Info("reading OpenStack network_data from config-drive") var networkDataf *os.File ospNetworkDataFilePath := ospNetworkDataFile - if useHostPath { - ospNetworkDataFilePath = ospHostNetworkDataFile - } networkDataf, err = os.Open(ospNetworkDataFilePath) if err != nil { - return metaData, networkData, fmt.Errorf("error opening file %s: %w", ospHostNetworkDataFile, err) + return metaData, networkData, fmt.Errorf("error opening file %s: %w", ospNetworkDataFilePath, err) } defer func() { if e := networkDataf.Close(); err == nil && e != nil { - err = fmt.Errorf("error closing file %s: %w", ospHostNetworkDataFile, e) + err = fmt.Errorf("error closing file %s: %w", ospNetworkDataFilePath, e) } }() if err = json.NewDecoder(networkDataf).Decode(&networkData); err != nil { - return metaData, networkData, fmt.Errorf("error unmarshalling metadata from file %s: %w", ospHostNetworkDataFile, err) + return metaData, networkData, fmt.Errorf("error unmarshalling metadata from file %s: %w", ospNetworkDataFilePath, err) } return metaData, networkData, err } @@ -293,7 +362,7 @@ func (o *openstackContext) CreateOpenstackDevicesInfo() error { return fmt.Errorf("CreateOpenstackDevicesInfo(): error getting PCI info: %v", err) } - devices := pci.ListDevices() + devices := pci.Devices if len(devices) == 0 { return fmt.Errorf("CreateOpenstackDevicesInfo(): could not retrieve PCI devices") } @@ -352,7 +421,7 @@ func (o *openstackContext) DiscoverSriovDevicesVirtual() ([]sriovnetworkv1.Inter return nil, fmt.Errorf("DiscoverSriovDevicesVirtual(): error getting PCI info: %v", err) } - devices := pci.ListDevices() + devices := pci.Devices if len(devices) == 0 { return nil, fmt.Errorf("DiscoverSriovDevicesVirtual(): could not retrieve PCI devices") } diff --git a/pkg/plugins/fake/fake_plugin.go b/pkg/plugins/fake/fake_plugin.go deleted file mode 100644 index 0aa218f28..000000000 --- a/pkg/plugins/fake/fake_plugin.go +++ /dev/null @@ -1,30 +0,0 @@ -package fake - -import ( - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" -) - -// This plugin is used in Daemon unit tests -type FakePlugin struct { - PluginName string -} - -func (f *FakePlugin) Name() string { - return f.PluginName -} - -func (f *FakePlugin) Spec() string { - return "1.0" -} - -func (f *FakePlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) (bool, bool, error) { - return false, false, nil -} - -func (f *FakePlugin) CheckStatusChanges(new *sriovnetworkv1.SriovNetworkNodeState) (bool, error) { - return false, nil -} - -func (f *FakePlugin) Apply() error { - return nil -} diff --git a/pkg/plugins/generic/generic_plugin.go b/pkg/plugins/generic/generic_plugin.go index 14b1903e5..27b237c6c 100644 --- a/pkg/plugins/generic/generic_plugin.go +++ b/pkg/plugins/generic/generic_plugin.go @@ -1,11 +1,8 @@ package generic import ( - "bytes" "errors" - "os/exec" - "strconv" - "strings" + "fmt" "syscall" "sigs.k8s.io/controller-runtime/pkg/log" @@ -13,6 +10,7 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper" + hostTypes "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" @@ -47,12 +45,13 @@ type DriverState struct { type DriverStateMapType map[uint]*DriverState +type KargStateMapType map[string]bool + type GenericPlugin struct { PluginName string - SpecVersion string DesireState *sriovnetworkv1.SriovNetworkNodeState DriverStateMap DriverStateMapType - DesiredKernelArgs map[string]bool + DesiredKernelArgs KargStateMapType helpers helper.HostHelpersInterface skipVFConfiguration bool skipBridgeConfiguration bool @@ -81,7 +80,7 @@ type genericPluginOptions struct { skipBridgeConfiguration bool } -const scriptsPath = "bindata/scripts/enable-kargs.sh" +const scriptsPath = "bindata/scripts/kargs.sh" // Initialize our plugin and set up initial values func NewGenericPlugin(helpers helper.HostHelpersInterface, options ...Option) (plugin.VendorPlugin, error) { @@ -111,11 +110,26 @@ func NewGenericPlugin(helpers helper.HostHelpersInterface, options ...Option) (p NeedDriverFunc: needDriverCheckVdpaType, DriverLoaded: false, } + + // To maintain backward compatibility we don't remove the intel_iommu, iommu and pcirealloc + // kernel args if they are configured + kargs, err := helpers.GetCurrentKernelArgs() + if err != nil { + return nil, err + } + desiredKernelArgs := KargStateMapType{ + consts.KernelArgPciRealloc: helpers.IsKernelArgsSet(kargs, consts.KernelArgPciRealloc), + consts.KernelArgIntelIommu: helpers.IsKernelArgsSet(kargs, consts.KernelArgIntelIommu), + consts.KernelArgIommuOn: helpers.IsKernelArgsSet(kargs, consts.KernelArgIommuOn), + consts.KernelArgIommuPt: helpers.IsKernelArgsSet(kargs, consts.KernelArgIommuPt), + consts.KernelArgRdmaShared: false, + consts.KernelArgRdmaExclusive: false, + } + return &GenericPlugin{ PluginName: PluginName, - SpecVersion: "1.0", DriverStateMap: driverStateMap, - DesiredKernelArgs: make(map[string]bool), + DesiredKernelArgs: desiredKernelArgs, helpers: helpers, skipVFConfiguration: cfg.skipVFConfiguration, skipBridgeConfiguration: cfg.skipBridgeConfiguration, @@ -127,11 +141,6 @@ func (p *GenericPlugin) Name() string { return p.PluginName } -// Spec returns the version of the spec expected by the plugin -func (p *GenericPlugin) Spec() string { - return p.SpecVersion -} - // OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need drain and/or reboot node func (p *GenericPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { log.Log.Info("generic plugin OnNodeStateChange()") @@ -178,18 +187,13 @@ func (p *GenericPlugin) CheckStatusChanges(current *sriovnetworkv1.SriovNetworkN } } - missingKernelArgs, err := p.getMissingKernelArgs() + shouldUpdate, err := p.shouldUpdateKernelArgs() if err != nil { log.Log.Error(err, "generic-plugin CheckStatusChanges(): failed to verify missing kernel arguments") return false, err } - if len(missingKernelArgs) != 0 { - log.Log.V(0).Info("generic-plugin CheckStatusChanges(): kernel args missing", - "kernelArgs", missingKernelArgs) - } - - return len(missingKernelArgs) != 0, nil + return shouldUpdate, nil } func (p *GenericPlugin) syncDriverState() error { @@ -227,7 +231,7 @@ func (p *GenericPlugin) Apply() error { p.DesireState.Status.Interfaces, p.skipVFConfiguration); err != nil { // Catch the "cannot allocate memory" error and try to use PCI realloc if errors.Is(err, syscall.ENOMEM) { - p.addToDesiredKernelArgs(consts.KernelArgPciRealloc) + p.enableDesiredKernelArgs(consts.KernelArgPciRealloc) } return err } @@ -263,85 +267,84 @@ func needDriverCheckVdpaType(state *sriovnetworkv1.SriovNetworkNodeState, driver return false } -// setKernelArg Tries to add the kernel args via ostree or grubby. -func setKernelArg(karg string) (bool, error) { - log.Log.Info("generic plugin setKernelArg()") - var stdout, stderr bytes.Buffer - cmd := exec.Command("/bin/sh", scriptsPath, karg) - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { +// editKernelArg Tries to add the kernel args via ostree or grubby. +func editKernelArg(helper helper.HostHelpersInterface, mode, karg string) error { + log.Log.Info("generic plugin editKernelArg()", "mode", mode, "karg", karg) + _, _, err := helper.RunCommand("/bin/sh", scriptsPath, mode, karg) + if err != nil { // if grubby is not there log and assume kernel args are set correctly. if utils.IsCommandNotFound(err) { - log.Log.Error(err, "generic plugin setKernelArg(): grubby or ostree command not found. Please ensure that kernel arg are set", + log.Log.Error(err, "generic plugin editKernelArg(): grubby or ostree command not found. Please ensure that kernel arg are correct", "kargs", karg) - return false, nil - } - log.Log.Error(err, "generic plugin setKernelArg(): fail to enable kernel arg", "karg", karg) - return false, err - } - - i, err := strconv.Atoi(strings.TrimSpace(stdout.String())) - if err == nil { - if i > 0 { - log.Log.Info("generic plugin setKernelArg(): need to reboot node for kernel arg", "karg", karg) - return true, nil + return nil } + log.Log.Error(err, "generic plugin editKernelArg(): fail to edit kernel arg", "karg", karg) + return err } - return false, err + return nil } -// addToDesiredKernelArgs Should be called to queue a kernel arg to be added to the node. -func (p *GenericPlugin) addToDesiredKernelArgs(karg string) { - if _, ok := p.DesiredKernelArgs[karg]; !ok { - log.Log.Info("generic plugin addToDesiredKernelArgs(): Adding to desired kernel arg", "karg", karg) - p.DesiredKernelArgs[karg] = false - } +// enableDesiredKernelArgs Should be called to mark a kernel arg as enabled. +func (p *GenericPlugin) enableDesiredKernelArgs(karg string) { + log.Log.Info("generic plugin enableDesiredKernelArgs(): enable kernel arg", "karg", karg) + p.DesiredKernelArgs[karg] = true } -// getMissingKernelArgs gets Kernel arguments that have not been set. -func (p *GenericPlugin) getMissingKernelArgs() ([]string, error) { - missingArgs := make([]string, 0, len(p.DesiredKernelArgs)) - if len(p.DesiredKernelArgs) == 0 { - return nil, nil - } +// disableDesiredKernelArgs Should be called to mark a kernel arg as disabled. +func (p *GenericPlugin) disableDesiredKernelArgs(karg string) { + log.Log.Info("generic plugin disableDesiredKernelArgs(): disable kernel arg", "karg", karg) + p.DesiredKernelArgs[karg] = false +} +// shouldUpdateKernelArgs returns true if the DesiredKernelArgs state is not equal to the running kernel args in the system +func (p *GenericPlugin) shouldUpdateKernelArgs() (bool, error) { kargs, err := p.helpers.GetCurrentKernelArgs() if err != nil { - return nil, err + return false, err } - for desiredKarg := range p.DesiredKernelArgs { - if !p.helpers.IsKernelArgsSet(kargs, desiredKarg) { - missingArgs = append(missingArgs, desiredKarg) + for karg, kargState := range p.DesiredKernelArgs { + if kargState && !p.helpers.IsKernelArgsSet(kargs, karg) { + return true, nil + } + + if !kargState && p.helpers.IsKernelArgsSet(kargs, karg) { + return true, nil } } - return missingArgs, nil + return false, nil } // syncDesiredKernelArgs should be called to set all the kernel arguments. Returns bool if node update is needed. -func (p *GenericPlugin) syncDesiredKernelArgs(kargs []string) (bool, error) { +func (p *GenericPlugin) syncDesiredKernelArgs() (bool, error) { + kargs, err := p.helpers.GetCurrentKernelArgs() + if err != nil { + return false, err + } + needReboot := false + for karg, kargState := range p.DesiredKernelArgs { + if kargState { + err = editKernelArg(p.helpers, "add", karg) + if err != nil { + log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to set kernel arg", "karg", karg) + return false, err + } - for _, karg := range kargs { - if p.DesiredKernelArgs[karg] { - log.Log.V(2).Info("generic-plugin syncDesiredKernelArgs(): previously attempted to set kernel arg", - "karg", karg) - } - // There is a case when we try to set the kernel argument here, the daemon could decide to not reboot because - // the daemon encountered a potentially one-time error. However we always want to make sure that the kernel - // argument is set once the daemon goes through node state sync again. - update, err := setKernelArg(karg) - if err != nil { - log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to set kernel arg", "karg", karg) - return false, err - } - if update { - needReboot = true - log.Log.V(2).Info("generic-plugin syncDesiredKernelArgs(): need reboot for setting kernel arg", "karg", karg) + if !p.helpers.IsKernelArgsSet(kargs, karg) { + needReboot = true + } + } else { + err = editKernelArg(p.helpers, "remove", karg) + if err != nil { + log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to remove kernel arg", "karg", karg) + return false, err + } + + if p.helpers.IsKernelArgsSet(kargs, karg) { + needReboot = true + } } - p.DesiredKernelArgs[karg] = true } return needReboot, nil } @@ -419,32 +422,66 @@ func (p *GenericPlugin) shouldConfigureBridges() bool { func (p *GenericPlugin) addVfioDesiredKernelArg(state *sriovnetworkv1.SriovNetworkNodeState) { driverState := p.DriverStateMap[Vfio] + + kernelArgFnByCPUVendor := map[hostTypes.CPUVendor]func(){ + hostTypes.CPUVendorIntel: func() { + p.enableDesiredKernelArgs(consts.KernelArgIntelIommu) + p.enableDesiredKernelArgs(consts.KernelArgIommuOn) + }, + hostTypes.CPUVendorAMD: func() { + p.enableDesiredKernelArgs(consts.KernelArgIommuOn) + }, + } + if !driverState.DriverLoaded && driverState.NeedDriverFunc(state, driverState) { - p.addToDesiredKernelArgs(consts.KernelArgIntelIommu) - p.addToDesiredKernelArgs(consts.KernelArgIommuPt) + cpuVendor, err := p.helpers.GetCPUVendor() + if err != nil { + log.Log.Error(err, "can't get CPU vendor, falling back to Intel") + cpuVendor = hostTypes.CPUVendorIntel + } + + addKernelArgFn := kernelArgFnByCPUVendor[cpuVendor] + if addKernelArgFn != nil { + addKernelArgFn() + } + } +} + +func (p *GenericPlugin) configRdmaKernelArg(state *sriovnetworkv1.SriovNetworkNodeState) error { + if state.Spec.System.RdmaMode == "" { + p.disableDesiredKernelArgs(consts.KernelArgRdmaExclusive) + p.disableDesiredKernelArgs(consts.KernelArgRdmaShared) + } else if state.Spec.System.RdmaMode == "shared" { + p.enableDesiredKernelArgs(consts.KernelArgRdmaShared) + p.disableDesiredKernelArgs(consts.KernelArgRdmaExclusive) + } else if state.Spec.System.RdmaMode == "exclusive" { + p.enableDesiredKernelArgs(consts.KernelArgRdmaExclusive) + p.disableDesiredKernelArgs(consts.KernelArgRdmaShared) + } else { + err := fmt.Errorf("unexpected rdma mode: %s", state.Spec.System.RdmaMode) + log.Log.Error(err, "generic-plugin configRdmaKernelArg(): failed to configure kernel arguments for rdma") + return err } + + return p.helpers.SetRDMASubsystem(state.Spec.System.RdmaMode) } func (p *GenericPlugin) needRebootNode(state *sriovnetworkv1.SriovNetworkNodeState) (bool, error) { needReboot := false p.addVfioDesiredKernelArg(state) - - missingKernelArgs, err := p.getMissingKernelArgs() + err := p.configRdmaKernelArg(state) if err != nil { - log.Log.Error(err, "generic-plugin needRebootNode(): failed to verify missing kernel arguments") return false, err } - if len(missingKernelArgs) != 0 { - needReboot, err = p.syncDesiredKernelArgs(missingKernelArgs) - if err != nil { - log.Log.Error(err, "generic-plugin needRebootNode(): failed to set the desired kernel arguments") - return false, err - } - if needReboot { - log.Log.V(2).Info("generic-plugin needRebootNode(): need reboot for updating kernel arguments") - } + needReboot, err = p.syncDesiredKernelArgs() + if err != nil { + log.Log.Error(err, "generic-plugin needRebootNode(): failed to set the desired kernel arguments") + return false, err + } + if needReboot { + log.Log.V(2).Info("generic-plugin needRebootNode(): need reboot for updating kernel arguments") } return needReboot, nil diff --git a/pkg/plugins/generic/generic_plugin_test.go b/pkg/plugins/generic/generic_plugin_test.go index 0d6701a64..c3ab09331 100644 --- a/pkg/plugins/generic/generic_plugin_test.go +++ b/pkg/plugins/generic/generic_plugin_test.go @@ -3,13 +3,14 @@ package generic import ( "testing" - "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" mock_helper "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper/mock" + hostTypes "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) @@ -33,6 +34,15 @@ var _ = Describe("Generic plugin", func() { ctrl = gomock.NewController(t) hostHelper = mock_helper.NewMockHostHelpersInterface(ctrl) + hostHelper.EXPECT().SetRDMASubsystem("").Return(nil).AnyTimes() + hostHelper.EXPECT().GetCurrentKernelArgs().Return("", nil).AnyTimes() + hostHelper.EXPECT().IsKernelArgsSet("", consts.KernelArgIntelIommu).Return(false).AnyTimes() + hostHelper.EXPECT().IsKernelArgsSet("", consts.KernelArgIommuPt).Return(false).AnyTimes() + hostHelper.EXPECT().IsKernelArgsSet("", consts.KernelArgPciRealloc).Return(false).AnyTimes() + hostHelper.EXPECT().IsKernelArgsSet("", consts.KernelArgRdmaExclusive).Return(false).AnyTimes() + hostHelper.EXPECT().IsKernelArgsSet("", consts.KernelArgRdmaShared).Return(false).AnyTimes() + + hostHelper.EXPECT().RunCommand(gomock.Any(), gomock.Any()).Return("", "", nil).AnyTimes() genericPlugin, err = NewGenericPlugin(hostHelper) Expect(err).ToNot(HaveOccurred()) @@ -850,8 +860,9 @@ var _ = Describe("Generic plugin", func() { Expect(changed).To(BeTrue()) }) - It("should detect changes on status due to missing kernel args", func() { - networkNodeState := &sriovnetworkv1.SriovNetworkNodeState{ + Context("Kernel Args", func() { + + vfioNetworkNodeState := &sriovnetworkv1.SriovNetworkNodeState{ Spec: sriovnetworkv1.SriovNetworkNodeStateSpec{ Interfaces: sriovnetworkv1.Interfaces{{ PciAddress: "0000:00:00.0", @@ -896,16 +907,71 @@ var _ = Describe("Generic plugin", func() { }, } - // Load required kernel args. - genericPlugin.(*GenericPlugin).addVfioDesiredKernelArg(networkNodeState) + rdmaState := &sriovnetworkv1.SriovNetworkNodeState{ + Spec: sriovnetworkv1.SriovNetworkNodeStateSpec{System: sriovnetworkv1.System{ + RdmaMode: consts.RdmaSubsystemModeShared, + }}, + Status: sriovnetworkv1.SriovNetworkNodeStateStatus{}, + } - hostHelper.EXPECT().GetCurrentKernelArgs().Return("", nil) - hostHelper.EXPECT().IsKernelArgsSet("", consts.KernelArgIntelIommu).Return(false) - hostHelper.EXPECT().IsKernelArgsSet("", consts.KernelArgIommuPt).Return(false) + It("should detect changes on status due to missing kernel args", func() { + hostHelper.EXPECT().GetCPUVendor().Return(hostTypes.CPUVendorIntel, nil) - changed, err := genericPlugin.CheckStatusChanges(networkNodeState) - Expect(err).ToNot(HaveOccurred()) - Expect(changed).To(BeTrue()) + // Load required kernel args. + genericPlugin.(*GenericPlugin).addVfioDesiredKernelArg(vfioNetworkNodeState) + + Expect(genericPlugin.(*GenericPlugin).DesiredKernelArgs[consts.KernelArgIntelIommu]).To(BeTrue()) + Expect(genericPlugin.(*GenericPlugin).DesiredKernelArgs[consts.KernelArgIommuPt]).To(BeTrue()) + + changed, err := genericPlugin.CheckStatusChanges(vfioNetworkNodeState) + Expect(err).ToNot(HaveOccurred()) + Expect(changed).To(BeTrue()) + }) + + It("should set the correct kernel args on AMD CPUs", func() { + hostHelper.EXPECT().GetCPUVendor().Return(hostTypes.CPUVendorAMD, nil) + genericPlugin.(*GenericPlugin).addVfioDesiredKernelArg(vfioNetworkNodeState) + Expect(genericPlugin.(*GenericPlugin).DesiredKernelArgs[consts.KernelArgIommuPt]).To(BeTrue()) + }) + + It("should enable rdma shared mode", func() { + hostHelper.EXPECT().SetRDMASubsystem(consts.RdmaSubsystemModeShared).Return(nil) + err := genericPlugin.(*GenericPlugin).configRdmaKernelArg(rdmaState) + Expect(err).ToNot(HaveOccurred()) + + Expect(genericPlugin.(*GenericPlugin).DesiredKernelArgs[consts.KernelArgRdmaShared]).To(BeTrue()) + Expect(genericPlugin.(*GenericPlugin).DesiredKernelArgs[consts.KernelArgRdmaExclusive]).To(BeFalse()) + + changed, err := genericPlugin.CheckStatusChanges(rdmaState) + Expect(err).ToNot(HaveOccurred()) + Expect(changed).To(BeTrue()) + }) + It("should enable rdma exclusive mode", func() { + hostHelper.EXPECT().SetRDMASubsystem(consts.RdmaSubsystemModeExclusive).Return(nil) + rdmaState.Spec.System.RdmaMode = consts.RdmaSubsystemModeExclusive + err := genericPlugin.(*GenericPlugin).configRdmaKernelArg(rdmaState) + Expect(err).ToNot(HaveOccurred()) + + Expect(genericPlugin.(*GenericPlugin).DesiredKernelArgs[consts.KernelArgRdmaShared]).To(BeFalse()) + Expect(genericPlugin.(*GenericPlugin).DesiredKernelArgs[consts.KernelArgRdmaExclusive]).To(BeTrue()) + + changed, err := genericPlugin.CheckStatusChanges(rdmaState) + Expect(err).ToNot(HaveOccurred()) + Expect(changed).To(BeTrue()) + }) + It("should not configure RDMA kernel args", func() { + //hostHelper.EXPECT().SetRDMASubsystem("").Return(nil) + rdmaState.Spec.System = sriovnetworkv1.System{} + err := genericPlugin.(*GenericPlugin).configRdmaKernelArg(rdmaState) + Expect(err).ToNot(HaveOccurred()) + + Expect(genericPlugin.(*GenericPlugin).DesiredKernelArgs[consts.KernelArgRdmaShared]).To(BeFalse()) + Expect(genericPlugin.(*GenericPlugin).DesiredKernelArgs[consts.KernelArgRdmaExclusive]).To(BeFalse()) + + changed, err := genericPlugin.CheckStatusChanges(rdmaState) + Expect(err).ToNot(HaveOccurred()) + Expect(changed).To(BeFalse()) + }) }) It("should load vfio_pci driver", func() { diff --git a/pkg/plugins/intel/intel_plugin.go b/pkg/plugins/intel/intel_plugin.go index e88a186c9..c25aa5abe 100644 --- a/pkg/plugins/intel/intel_plugin.go +++ b/pkg/plugins/intel/intel_plugin.go @@ -12,15 +12,13 @@ var PluginName = "intel" type IntelPlugin struct { PluginName string - SpecVersion string DesireState *sriovnetworkv1.SriovNetworkNodeState LastState *sriovnetworkv1.SriovNetworkNodeState } func NewIntelPlugin(helpers helper.HostHelpersInterface) (plugin.VendorPlugin, error) { return &IntelPlugin{ - PluginName: PluginName, - SpecVersion: "1.0", + PluginName: PluginName, }, nil } @@ -29,11 +27,6 @@ func (p *IntelPlugin) Name() string { return p.PluginName } -// Spec returns the version of the spec expected by the plugin -func (p *IntelPlugin) Spec() string { - return p.SpecVersion -} - // OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need dain and/or reboot node func (p *IntelPlugin) OnNodeStateChange(*sriovnetworkv1.SriovNetworkNodeState) (bool, bool, error) { log.Log.Info("intel-plugin OnNodeStateChange()") diff --git a/pkg/plugins/k8s/k8s_plugin.go b/pkg/plugins/k8s/k8s_plugin.go index 1a0336049..bd6023dd5 100644 --- a/pkg/plugins/k8s/k8s_plugin.go +++ b/pkg/plugins/k8s/k8s_plugin.go @@ -96,7 +96,6 @@ const ( func NewK8sPlugin(helper helper.HostHelpersInterface) (plugins.VendorPlugin, error) { k8sPluging := &K8sPlugin{ PluginName: PluginName, - SpecVersion: "1.0", hostHelper: helper, updateTarget: &k8sUpdateTarget{}, } @@ -109,11 +108,6 @@ func (p *K8sPlugin) Name() string { return p.PluginName } -// Spec returns the version of the spec expected by the plugin -func (p *K8sPlugin) Spec() string { - return p.SpecVersion -} - // OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need dain and/or reboot node func (p *K8sPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { log.Log.Info("k8s plugin OnNodeStateChange()") diff --git a/pkg/plugins/k8s/k8s_plugin_test.go b/pkg/plugins/k8s/k8s_plugin_test.go index b4344b95f..23631169a 100644 --- a/pkg/plugins/k8s/k8s_plugin_test.go +++ b/pkg/plugins/k8s/k8s_plugin_test.go @@ -5,9 +5,9 @@ import ( "os" "testing" - "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" "go.uber.org/zap/zapcore" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" diff --git a/pkg/plugins/mellanox/mellanox_plugin.go b/pkg/plugins/mellanox/mellanox_plugin.go index 10b0152bb..da7344bfc 100644 --- a/pkg/plugins/mellanox/mellanox_plugin.go +++ b/pkg/plugins/mellanox/mellanox_plugin.go @@ -16,9 +16,8 @@ import ( var PluginName = "mellanox" type MellanoxPlugin struct { - PluginName string - SpecVersion string - helpers helper.HostHelpersInterface + PluginName string + helpers helper.HostHelpersInterface } var pciAddressesToReset []string @@ -31,9 +30,8 @@ func NewMellanoxPlugin(helpers helper.HostHelpersInterface) (plugin.VendorPlugin mellanoxNicsStatus = map[string]map[string]sriovnetworkv1.InterfaceExt{} return &MellanoxPlugin{ - PluginName: PluginName, - SpecVersion: "1.0", - helpers: helpers, + PluginName: PluginName, + helpers: helpers, }, nil } @@ -42,11 +40,6 @@ func (p *MellanoxPlugin) Name() string { return p.PluginName } -// SpecVersion returns the version of the spec expected by the plugin -func (p *MellanoxPlugin) Spec() string { - return p.SpecVersion -} - // OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need dain and/or reboot node func (p *MellanoxPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { log.Log.Info("mellanox plugin OnNodeStateChange()") @@ -212,8 +205,8 @@ func (p *MellanoxPlugin) Apply() error { if err := p.helpers.MlxConfigFW(attributesToChange); err != nil { return err } - if vars.MlxPluginFwReset { - return p.helpers.MlxResetFW(pciAddressesToReset) + if vars.FeatureGate.IsEnabled(consts.MellanoxFirmwareResetFeatureGate) { + return p.helpers.MlxResetFW(pciAddressesToReset, mellanoxNicsStatus) } return nil } diff --git a/pkg/plugins/mellanox/mellanox_plugin_test.go b/pkg/plugins/mellanox/mellanox_plugin_test.go new file mode 100644 index 000000000..0333d3079 --- /dev/null +++ b/pkg/plugins/mellanox/mellanox_plugin_test.go @@ -0,0 +1,434 @@ +package mellanox + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "go.uber.org/mock/gomock" + corev1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + mock_helper "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper/mock" + plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" + mlx "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vendors/mellanox" +) + +var _ = Describe("SRIOV", Ordered, func() { + var ( + m plugin.VendorPlugin + h *mock_helper.MockHostHelpersInterface + err error + testCtrl *gomock.Controller + + sriovNetworkNodeState *sriovnetworkv1.SriovNetworkNodeState + ) + + BeforeAll(func() { + sriovnetworkv1.NicIDMap = []string{"15b3 1013 1014"} + }) + + BeforeEach(func() { + testCtrl = gomock.NewController(GinkgoT()) + h = mock_helper.NewMockHostHelpersInterface(testCtrl) + m, err = NewMellanoxPlugin(h) + Expect(err).ToNot(HaveOccurred()) + + sriovNetworkNodeState = &sriovnetworkv1.SriovNetworkNodeState{ObjectMeta: corev1.ObjectMeta{Name: "worker-0", Namespace: "test"}, + Spec: sriovnetworkv1.SriovNetworkNodeStateSpec{}, + Status: sriovnetworkv1.SriovNetworkNodeStateStatus{}, + } + }) + + AfterEach(func() { + testCtrl.Finish() + }) + + Context("CheckStatusChanges", func() { + It("should always return false and nil", func() { + drift, err := m.CheckStatusChanges(nil) + Expect(err).ToNot(HaveOccurred()) + Expect(drift).To(BeFalse()) + }) + }) + + Context("OnNodeStateChange", func() { + It("should return error if we are trying to configure mlx devices and the system is in lock down mode", func() { + h.EXPECT().IsKernelLockdownMode().Return(true) + sriovNetworkNodeState.Spec.Interfaces = sriovnetworkv1.Interfaces{ + {Name: "eno1", + NumVfs: 10, + PciAddress: "0000:d8:00.0", VfGroups: []sriovnetworkv1.VfGroup{ + {ResourceName: "test", + PolicyName: "test", + VfRange: "0-0"}, + }, + }, + } + sriovNetworkNodeState.Status.Interfaces = sriovnetworkv1.InterfaceExts{ + { + Name: "eno1", + NumVfs: 0, + PciAddress: "0000:d8:00.0", + Vendor: "15b3", + }, + } + + _, _, err := m.OnNodeStateChange(sriovNetworkNodeState) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("mellanox device detected when in lockdown mode")) + }) + + It("should not return error the system is in lock down mode but we don't configure any mlx device", func() { + h.EXPECT().IsKernelLockdownMode().Return(true) + sriovNetworkNodeState.Spec.Interfaces = sriovnetworkv1.Interfaces{} + sriovNetworkNodeState.Status.Interfaces = sriovnetworkv1.InterfaceExts{ + { + Name: "eno1", + NumVfs: 0, + PciAddress: "0000:d8:00.0", + Vendor: "15b3", + }, + } + + needDrain, needReboot, err := m.OnNodeStateChange(sriovNetworkNodeState) + Expect(err).ToNot(HaveOccurred()) + Expect(needDrain).To(BeFalse()) + Expect(needReboot).To(BeFalse()) + }) + + It("should return error if the nic require fw changes but the nic is externally manage", func() { + h.EXPECT().IsKernelLockdownMode().Return(false) + h.EXPECT().GetMlxNicFwData("0000:d8:00.0").Return(&mlx.MlxNic{TotalVfs: 0}, &mlx.MlxNic{TotalVfs: 0}, nil) + sriovNetworkNodeState.Spec.Interfaces = sriovnetworkv1.Interfaces{ + {Name: "eno1", + NumVfs: 10, + ExternallyManaged: true, + PciAddress: "0000:d8:00.0", VfGroups: []sriovnetworkv1.VfGroup{ + {ResourceName: "test", + PolicyName: "test", + VfRange: "eno1#0-9"}, + }, + }, + } + sriovNetworkNodeState.Status.Interfaces = sriovnetworkv1.InterfaceExts{ + { + Name: "eno1", + NumVfs: 0, + PciAddress: "0000:d8:00.0", + Vendor: "15b3", + }, + } + + _, _, err := m.OnNodeStateChange(sriovNetworkNodeState) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("interface 0000:d8:00.0 required a change in the TotalVfs but the policy is externally managed failing: firmware TotalVf 0 requested TotalVf 10")) + }) + + It("should return true on reboot if we need to update the number of vfs in the firmware", func() { + h.EXPECT().IsKernelLockdownMode().Return(false) + h.EXPECT().GetMlxNicFwData("0000:d8:00.0").Return(&mlx.MlxNic{TotalVfs: 0}, &mlx.MlxNic{TotalVfs: 0}, nil) + sriovNetworkNodeState.Spec.Interfaces = sriovnetworkv1.Interfaces{ + {Name: "eno1", + NumVfs: 10, + PciAddress: "0000:d8:00.0", VfGroups: []sriovnetworkv1.VfGroup{ + {ResourceName: "test", + PolicyName: "test", + VfRange: "eno1#0-9"}, + }, + }, + } + sriovNetworkNodeState.Status.Interfaces = sriovnetworkv1.InterfaceExts{ + { + Name: "eno1", + NumVfs: 0, + PciAddress: "0000:d8:00.0", + Vendor: "15b3", + }, + } + + needDrain, needReboot, err := m.OnNodeStateChange(sriovNetworkNodeState) + Expect(err).ToNot(HaveOccurred()) + Expect(needDrain).To(BeTrue()) + Expect(needReboot).To(BeTrue()) + }) + + It("should return true on reboot adding vfs for one PF and removing for the other", func() { + h.EXPECT().IsKernelLockdownMode().Return(false) + h.EXPECT().GetMlxNicFwData("0000:d8:00.0").Return(&mlx.MlxNic{TotalVfs: 0}, &mlx.MlxNic{TotalVfs: 0}, nil) + h.EXPECT().GetMlxNicFwData("0000:d9:00.0").Return(&mlx.MlxNic{TotalVfs: 10}, &mlx.MlxNic{TotalVfs: 10}, nil) + h.EXPECT().LoadPfsStatus("0000:d9:00.0").Return(&sriovnetworkv1.Interface{ExternallyManaged: false}, true, nil).AnyTimes() + sriovNetworkNodeState.Spec.Interfaces = sriovnetworkv1.Interfaces{ + {Name: "eno1", + NumVfs: 10, + PciAddress: "0000:d8:00.0", VfGroups: []sriovnetworkv1.VfGroup{ + {ResourceName: "test", + PolicyName: "test", + VfRange: "eno1#0-9"}, + }, + }, + } + sriovNetworkNodeState.Status.Interfaces = sriovnetworkv1.InterfaceExts{ + { + Name: "eno1", + NumVfs: 0, + PciAddress: "0000:d8:00.0", + Vendor: "15b3", + DeviceID: "1013", + }, + { + Name: "eno2", + NumVfs: 10, + PciAddress: "0000:d9:00.0", + Vendor: "15b3", + DeviceID: "1013", + }, + } + + needDrain, needReboot, err := m.OnNodeStateChange(sriovNetworkNodeState) + Expect(err).ToNot(HaveOccurred()) + Expect(needDrain).To(BeTrue()) + Expect(needReboot).To(BeTrue()) + value, exist := attributesToChange["0000:d8:00.0"] + Expect(exist).To(BeTrue()) + Expect(value.TotalVfs).To(Equal(10)) + Expect(value.EnableSriov).To(BeTrue()) + value, exist = attributesToChange["0000:d9:00.0"] + Expect(exist).To(BeTrue()) + Expect(value.TotalVfs).To(Equal(0)) + Expect(value.EnableSriov).To(BeFalse()) + }) + + It("should return false if we just need to reset the vfs", func() { + h.EXPECT().IsKernelLockdownMode().Return(false) + h.EXPECT().GetMlxNicFwData("0000:d8:00.0").Return(&mlx.MlxNic{TotalVfs: 10}, &mlx.MlxNic{TotalVfs: 10}, nil) + h.EXPECT().LoadPfsStatus("0000:d8:00.0").Return(&sriovnetworkv1.Interface{ExternallyManaged: false}, true, nil).AnyTimes() + sriovNetworkNodeState.Spec.Interfaces = sriovnetworkv1.Interfaces{} + sriovNetworkNodeState.Status.Interfaces = sriovnetworkv1.InterfaceExts{ + { + Name: "eno1", + NumVfs: 10, + PciAddress: "0000:d8:00.0", + Vendor: "15b3", + DeviceID: "1013", + }, + } + + needDrain, needReboot, err := m.OnNodeStateChange(sriovNetworkNodeState) + Expect(err).ToNot(HaveOccurred()) + Expect(needDrain).To(BeFalse()) + Expect(needReboot).To(BeFalse()) + value, exist := attributesToChange["0000:d8:00.0"] + Expect(exist).To(BeTrue()) + Expect(value.TotalVfs).To(Equal(0)) + Expect(value.EnableSriov).To(BeFalse()) + + }) + + It("should return error if we are not able to read the PF status file form host", func() { + h.EXPECT().IsKernelLockdownMode().Return(false) + h.EXPECT().LoadPfsStatus("0000:d8:00.0").Return(nil, false, fmt.Errorf("failed to read file")) + sriovNetworkNodeState.Spec.Interfaces = sriovnetworkv1.Interfaces{} + sriovNetworkNodeState.Status.Interfaces = sriovnetworkv1.InterfaceExts{ + { + Name: "eno1", + NumVfs: 10, + PciAddress: "0000:d8:00.0", + Vendor: "15b3", + }, + } + + needDrain, needReboot, err := m.OnNodeStateChange(sriovNetworkNodeState) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("failed to read file")) + Expect(needDrain).To(BeFalse()) + Expect(needReboot).To(BeFalse()) + }) + + It("should failed if policy is externally manage and we need to change nic type", func() { + h.EXPECT().IsKernelLockdownMode().Return(false) + h.EXPECT().GetMlxNicFwData("0000:d8:00.0").Return(&mlx.MlxNic{TotalVfs: 10, LinkTypeP1: "ETH"}, &mlx.MlxNic{TotalVfs: 10, LinkTypeP1: "ETH"}, nil) + sriovNetworkNodeState.Spec.Interfaces = sriovnetworkv1.Interfaces{ + {Name: "eno1", + NumVfs: 10, + LinkType: "IB", + ExternallyManaged: true, + PciAddress: "0000:d8:00.0", VfGroups: []sriovnetworkv1.VfGroup{ + {ResourceName: "test", + PolicyName: "test", + VfRange: "eno1#0-9"}, + }, + }, + } + sriovNetworkNodeState.Status.Interfaces = sriovnetworkv1.InterfaceExts{ + { + Name: "eno1", + NumVfs: 10, + PciAddress: "0000:d8:00.0", + Vendor: "15b3", + LinkType: "ETH", + }, + } + + needDrain, needReboot, err := m.OnNodeStateChange(sriovNetworkNodeState) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("change required for link type but the policy is externally managed, failing")) + Expect(needDrain).To(BeFalse()) + Expect(needReboot).To(BeFalse()) + }) + + It("should failed if not able to check if the nic is externally managed PF", func() { + h.EXPECT().IsKernelLockdownMode().Return(false) + h.EXPECT().LoadPfsStatus("0000:d8:00.0").Return(&sriovnetworkv1.Interface{ExternallyManaged: true}, true, nil).AnyTimes() + sriovNetworkNodeState.Spec.Interfaces = sriovnetworkv1.Interfaces{} + sriovNetworkNodeState.Status.Interfaces = sriovnetworkv1.InterfaceExts{ + { + Name: "eno1", + NumVfs: 10, + PciAddress: "0000:d8:00.0", + Vendor: "15b3", + LinkType: "ETH", + }, + } + + needDrain, needReboot, err := m.OnNodeStateChange(sriovNetworkNodeState) + Expect(err).ToNot(HaveOccurred()) + Expect(needDrain).To(BeFalse()) + Expect(needReboot).To(BeFalse()) + _, exist := attributesToChange["0000:d8:00.0"] + Expect(exist).To(BeFalse()) + }) + + It("should not reset the firmware if the device was not configured by us", func() { + h.EXPECT().IsKernelLockdownMode().Return(false) + h.EXPECT().LoadPfsStatus("0000:d8:00.0").Return(nil, false, nil) + sriovNetworkNodeState.Spec.Interfaces = sriovnetworkv1.Interfaces{} + sriovNetworkNodeState.Status.Interfaces = sriovnetworkv1.InterfaceExts{ + { + Name: "eno1", + NumVfs: 10, + PciAddress: "0000:d8:00.0", + Vendor: "15b3", + LinkType: "ETH", + }, + } + + needDrain, needReboot, err := m.OnNodeStateChange(sriovNetworkNodeState) + Expect(err).ToNot(HaveOccurred()) + Expect(needDrain).To(BeFalse()) + Expect(needReboot).To(BeFalse()) + _, exist := attributesToChange["0000:d8:00.0"] + Expect(exist).To(BeFalse()) + }) + }) + + Context("Apply", func() { + It("should not call fw configuration for mlx devices in secure boot active environment", func() { + h.EXPECT().IsKernelLockdownMode().Return(true) + err := m.Apply() + Expect(err).ToNot(HaveOccurred()) + }) + It("should return eror if call mlx config fw failed", func() { + h.EXPECT().IsKernelLockdownMode().Return(false) + h.EXPECT().MlxConfigFW(gomock.Any()).Return(fmt.Errorf("failed to configure fw")) + err := m.Apply() + Expect(err).To(HaveOccurred()) + }) + + It("should call mlx config fw without fwreset if feature flag is disabled", func() { + vars.FeatureGate.Init(nil) + h.EXPECT().IsKernelLockdownMode().Return(false) + h.EXPECT().MlxConfigFW(gomock.Any()).Return(nil) + err := m.Apply() + Expect(err).ToNot(HaveOccurred()) + }) + + It("should call mlx config fw with fwreset if feature flag is enabled", func() { + vars.FeatureGate.Init(map[string]bool{consts.MellanoxFirmwareResetFeatureGate: true}) + h.EXPECT().IsKernelLockdownMode().Return(false) + h.EXPECT().MlxConfigFW(gomock.Any()).Return(nil) + h.EXPECT().MlxResetFW(gomock.Any(), gomock.Any()).Return(nil) + err := m.Apply() + Expect(err).ToNot(HaveOccurred()) + }) + + It("should reset VFs on both ports for dual-port NIC when firmware reset is required", func() { + vars.FeatureGate.Init(map[string]bool{consts.MellanoxFirmwareResetFeatureGate: true}) + + // Setup dual-port NIC status + mellanoxNicsStatus = map[string]map[string]sriovnetworkv1.InterfaceExt{ + "0000:d8:00.": { + "0000:d8:00.0": {PciAddress: "0000:d8:00.0", Vendor: "15b3"}, + "0000:d8:00.1": {PciAddress: "0000:d8:00.1", Vendor: "15b3"}, + }, + } + + // Setup dual-port NIC spec + mellanoxNicsSpec = map[string]sriovnetworkv1.Interface{ + "0000:d8:00.0": {PciAddress: "0000:d8:00.0", NumVfs: 10}, + "0000:d8:00.1": {PciAddress: "0000:d8:00.1", NumVfs: 8}, + } + + // Only one port needs firmware reset + pciAddressesToReset = []string{"0000:d8:00.0"} + + h.EXPECT().IsKernelLockdownMode().Return(false) + h.EXPECT().MlxConfigFW(gomock.Any()).Return(nil) + + h.EXPECT().MlxResetFW([]string{"0000:d8:00.0"}, gomock.Any()).Return(nil) + + err := m.Apply() + Expect(err).ToNot(HaveOccurred()) + }) + + It("should reset VFs on single port NIC when firmware reset is required", func() { + vars.FeatureGate.Init(map[string]bool{consts.MellanoxFirmwareResetFeatureGate: true}) + + // Setup single-port NIC status + mellanoxNicsStatus = map[string]map[string]sriovnetworkv1.InterfaceExt{ + "0000:d8:00.": { + "0000:d8:00.0": {PciAddress: "0000:d8:00.0", Vendor: "15b3"}, + }, + } + + // Setup single-port NIC spec + mellanoxNicsSpec = map[string]sriovnetworkv1.Interface{ + "0000:d8:00.0": {PciAddress: "0000:d8:00.0", NumVfs: 10}, + } + + pciAddressesToReset = []string{"0000:d8:00.0"} + + h.EXPECT().IsKernelLockdownMode().Return(false) + h.EXPECT().MlxConfigFW(gomock.Any()).Return(nil) + + h.EXPECT().MlxResetFW([]string{"0000:d8:00.0"}, gomock.Any()).Return(nil) + + err := m.Apply() + Expect(err).ToNot(HaveOccurred()) + }) + + It("should call MlxResetFW with correct parameters for multiple NICs", func() { + vars.FeatureGate.Init(map[string]bool{consts.MellanoxFirmwareResetFeatureGate: true}) + + pciAddressesToReset = []string{"0000:d8:00.0", "0000:d9:00.0"} + + h.EXPECT().IsKernelLockdownMode().Return(false) + h.EXPECT().MlxConfigFW(gomock.Any()).Return(nil) + + h.EXPECT().MlxResetFW([]string{"0000:d8:00.0", "0000:d9:00.0"}, gomock.Any()).Return(nil) + + err := m.Apply() + Expect(err).ToNot(HaveOccurred()) + }) + + BeforeEach(func() { + // Reset global state before each test + pciAddressesToReset = []string{} + mellanoxNicsStatus = map[string]map[string]sriovnetworkv1.InterfaceExt{} + mellanoxNicsSpec = map[string]sriovnetworkv1.Interface{} + }) + }) +}) diff --git a/pkg/plugins/mellanox/suite_test.go b/pkg/plugins/mellanox/suite_test.go new file mode 100644 index 000000000..4a5b3224e --- /dev/null +++ b/pkg/plugins/mellanox/suite_test.go @@ -0,0 +1,24 @@ +package mellanox + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "go.uber.org/zap/zapcore" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" +) + +func TestSriov(t *testing.T) { + log.SetLogger(zap.New( + zap.WriteTo(GinkgoWriter), + zap.Level(zapcore.Level(-2)), + zap.UseDevMode(true))) + snolog.InitLog() + RegisterFailHandler(Fail) + RunSpecs(t, "Package Mellanox Plugin Suite") +} diff --git a/pkg/plugins/mock/mock_plugin.go b/pkg/plugins/mock/mock_plugin.go index a6ae38202..7c291a479 100644 --- a/pkg/plugins/mock/mock_plugin.go +++ b/pkg/plugins/mock/mock_plugin.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: plugin.go +// +// Generated by this command: +// +// mockgen -destination mock/mock_plugin.go -source plugin.go +// // Package mock_plugin is a generated GoMock package. package mock_plugin @@ -7,14 +12,15 @@ package mock_plugin import ( reflect "reflect" - gomock "github.com/golang/mock/gomock" v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + gomock "go.uber.org/mock/gomock" ) // MockVendorPlugin is a mock of VendorPlugin interface. type MockVendorPlugin struct { ctrl *gomock.Controller recorder *MockVendorPluginMockRecorder + isgomock struct{} } // MockVendorPluginMockRecorder is the mock recorder for MockVendorPlugin. @@ -58,7 +64,7 @@ func (m *MockVendorPlugin) CheckStatusChanges(arg0 *v1.SriovNetworkNodeState) (b } // CheckStatusChanges indicates an expected call of CheckStatusChanges. -func (mr *MockVendorPluginMockRecorder) CheckStatusChanges(arg0 interface{}) *gomock.Call { +func (mr *MockVendorPluginMockRecorder) CheckStatusChanges(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckStatusChanges", reflect.TypeOf((*MockVendorPlugin)(nil).CheckStatusChanges), arg0) } @@ -88,21 +94,7 @@ func (m *MockVendorPlugin) OnNodeStateChange(arg0 *v1.SriovNetworkNodeState) (bo } // OnNodeStateChange indicates an expected call of OnNodeStateChange. -func (mr *MockVendorPluginMockRecorder) OnNodeStateChange(arg0 interface{}) *gomock.Call { +func (mr *MockVendorPluginMockRecorder) OnNodeStateChange(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OnNodeStateChange", reflect.TypeOf((*MockVendorPlugin)(nil).OnNodeStateChange), arg0) } - -// Spec mocks base method. -func (m *MockVendorPlugin) Spec() string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Spec") - ret0, _ := ret[0].(string) - return ret0 -} - -// Spec indicates an expected call of Spec. -func (mr *MockVendorPluginMockRecorder) Spec() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Spec", reflect.TypeOf((*MockVendorPlugin)(nil).Spec)) -} diff --git a/pkg/plugins/plugin.go b/pkg/plugins/plugin.go index 830f7dd68..88dd17623 100644 --- a/pkg/plugins/plugin.go +++ b/pkg/plugins/plugin.go @@ -8,8 +8,6 @@ import ( type VendorPlugin interface { // Name returns the name of plugin Name() string - // Spec returns the SpecVersion followed by plugin - Spec() string // OnNodeStateChange is invoked when SriovNetworkNodeState CR is created or updated, return if need dain and/or reboot node OnNodeStateChange(*sriovnetworkv1.SriovNetworkNodeState) (bool, bool, error) // Apply config change diff --git a/pkg/plugins/virtual/suite_test.go b/pkg/plugins/virtual/suite_test.go new file mode 100644 index 000000000..25fa80a85 --- /dev/null +++ b/pkg/plugins/virtual/suite_test.go @@ -0,0 +1,24 @@ +package virtual + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "go.uber.org/zap/zapcore" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" +) + +func TestSriov(t *testing.T) { + log.SetLogger(zap.New( + zap.WriteTo(GinkgoWriter), + zap.Level(zapcore.Level(-2)), + zap.UseDevMode(true))) + snolog.InitLog() + RegisterFailHandler(Fail) + RunSpecs(t, "Package Virtual Plugin Suite") +} diff --git a/pkg/plugins/virtual/virtual_plugin.go b/pkg/plugins/virtual/virtual_plugin.go index 211cbee2c..4dfa5eba0 100644 --- a/pkg/plugins/virtual/virtual_plugin.go +++ b/pkg/plugins/virtual/virtual_plugin.go @@ -1,8 +1,7 @@ package virtual import ( - "reflect" - + "k8s.io/apimachinery/pkg/api/equality" "sigs.k8s.io/controller-runtime/pkg/log" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" @@ -34,7 +33,6 @@ const ( func NewVirtualPlugin(helper helper.HostHelpersInterface) (plugin.VendorPlugin, error) { return &VirtualPlugin{ PluginName: PluginName, - SpecVersion: "1.0", LoadVfioDriver: unloaded, helpers: helper, }, nil @@ -45,11 +43,6 @@ func (p *VirtualPlugin) Name() string { return p.PluginName } -// Spec returns the version of the spec expected by the plugin -func (p *VirtualPlugin) Spec() string { - return p.SpecVersion -} - // OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need dain and/or reboot node func (p *VirtualPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { log.Log.Info("virtual plugin OnNodeStateChange()") @@ -96,7 +89,7 @@ func (p *VirtualPlugin) Apply() error { if p.LastState != nil { log.Log.Info("virtual plugin Apply()", "last-state", p.LastState.Spec) - if reflect.DeepEqual(p.LastState.Spec.Interfaces, p.DesireState.Spec.Interfaces) { + if equality.Semantic.DeepEqual(p.LastState.Spec.Interfaces, p.DesireState.Spec.Interfaces) { log.Log.Info("virtual plugin Apply(): nothing to apply") return nil } @@ -115,13 +108,6 @@ func (p *VirtualPlugin) Apply() error { return nil } -func (p *VirtualPlugin) SetSystemdFlag() { -} - -func (p *VirtualPlugin) IsSystemService() bool { - return false -} - func needVfioDriver(state *sriovnetworkv1.SriovNetworkNodeState) bool { for _, iface := range state.Spec.Interfaces { for i := range iface.VfGroups { @@ -159,10 +145,8 @@ func needUpdateVirtual(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetwor // The NumVfs is always 1 if iface.NumVfs > 0 { for _, vf := range ifaceStatus.VFs { - ingroup := false for _, group := range iface.VfGroups { if sriovnetworkv1.IndexInRange(vf.VfID, group.VfRange) { - ingroup = true if group.DeviceType != consts.DeviceTypeNetDevice { if group.DeviceType != vf.Driver { log.Log.V(2).Info("needUpdateVirtual(): Driver needs update", @@ -179,10 +163,6 @@ func needUpdateVirtual(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetwor break } } - if !ingroup && sriovnetworkv1.StringInArray(vf.Driver, vars.DpdkDrivers) { - // VF which has DPDK driver loaded but not in any group, needs to be reset to default driver. - return true - } } } return false diff --git a/pkg/plugins/virtual/virtual_plugin_test.go b/pkg/plugins/virtual/virtual_plugin_test.go new file mode 100644 index 000000000..0114d95d9 --- /dev/null +++ b/pkg/plugins/virtual/virtual_plugin_test.go @@ -0,0 +1,237 @@ +package virtual + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "go.uber.org/mock/gomock" + corev1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + mock_helper "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper/mock" +) + +var _ = Describe("SRIOV", Ordered, func() { + var ( + v *VirtualPlugin + h *mock_helper.MockHostHelpersInterface + testCtrl *gomock.Controller + + sriovNetworkNodeState *sriovnetworkv1.SriovNetworkNodeState + ) + + BeforeAll(func() { + sriovnetworkv1.NicIDMap = []string{"15b3 1013 1014"} + }) + + BeforeEach(func() { + testCtrl = gomock.NewController(GinkgoT()) + h = mock_helper.NewMockHostHelpersInterface(testCtrl) + t, err := NewVirtualPlugin(h) + Expect(err).ToNot(HaveOccurred()) + + v = t.(*VirtualPlugin) + + sriovNetworkNodeState = &sriovnetworkv1.SriovNetworkNodeState{ObjectMeta: corev1.ObjectMeta{Name: "worker-0", Namespace: "test"}, + Spec: sriovnetworkv1.SriovNetworkNodeStateSpec{}, + Status: sriovnetworkv1.SriovNetworkNodeStateStatus{}, + } + }) + + AfterEach(func() { + testCtrl.Finish() + }) + + Context("CheckStatusChanges", func() { + It("should always return false", func() { + drift, err := v.CheckStatusChanges(sriovNetworkNodeState) + Expect(err).ToNot(HaveOccurred()) + Expect(drift).To(BeFalse()) + }) + }) + + Context("OnNodeStateChange", func() { + It("should mark the vfio driver as loading if needed", func() { + sriovNetworkNodeState.Spec.Interfaces = sriovnetworkv1.Interfaces{ + {Name: "eno1", + NumVfs: 10, + PciAddress: "0000:d8:00.0", VfGroups: []sriovnetworkv1.VfGroup{ + { + ResourceName: "test", + PolicyName: "test", + VfRange: "eno1#0-9", + DeviceType: consts.DeviceTypeVfioPci, + }, + }, + }, + } + Expect(v.LoadVfioDriver).To(Equal(uint(0))) + needDrain, needReboot, err := v.OnNodeStateChange(sriovNetworkNodeState) + Expect(err).ToNot(HaveOccurred()) + Expect(needDrain).To(BeFalse()) + Expect(needReboot).To(BeFalse()) + Expect(v.LoadVfioDriver).To(Equal(uint(1))) + }) + + It("should remain loading even if no vfio exist", func() { + sriovNetworkNodeState.Spec.Interfaces = sriovnetworkv1.Interfaces{ + {Name: "eno1", + NumVfs: 10, + PciAddress: "0000:d8:00.0", VfGroups: []sriovnetworkv1.VfGroup{ + { + ResourceName: "test", + PolicyName: "test", + VfRange: "eno1#0-9", + DeviceType: consts.DeviceTypeVfioPci, + }, + }, + }, + } + Expect(v.LoadVfioDriver).To(Equal(uint(unloaded))) + needDrain, needReboot, err := v.OnNodeStateChange(sriovNetworkNodeState) + Expect(err).ToNot(HaveOccurred()) + Expect(needDrain).To(BeFalse()) + Expect(needReboot).To(BeFalse()) + Expect(v.LoadVfioDriver).To(Equal(uint(loading))) + + sriovNetworkNodeState.Spec.Interfaces = sriovnetworkv1.Interfaces{} + needDrain, needReboot, err = v.OnNodeStateChange(sriovNetworkNodeState) + Expect(err).ToNot(HaveOccurred()) + Expect(needDrain).To(BeFalse()) + Expect(needReboot).To(BeFalse()) + Expect(v.LoadVfioDriver).To(Equal(uint(loading))) + }) + }) + + Context("Apply", func() { + It("should not apply any configure if there is no change in the node state and the kernel modules are loaded", func() { + v.LoadVfioDriver = loaded + sriovNetworkNodeState.Spec.Interfaces = sriovnetworkv1.Interfaces{ + {Name: "eno1", + NumVfs: 10, + PciAddress: "0000:d8:00.0", VfGroups: []sriovnetworkv1.VfGroup{ + { + ResourceName: "test", + PolicyName: "test", + VfRange: "eno1#0-9", + DeviceType: consts.DeviceTypeVfioPci, + }, + }, + }, + } + + v.LastState = sriovNetworkNodeState + v.DesireState = sriovNetworkNodeState + err := v.Apply() + Expect(err).ToNot(HaveOccurred()) + }) + + It("should load the kernel modules if they are not loaded", func() { + h.EXPECT().LoadKernelModule("vfio", "enable_unsafe_noiommu_mode=1").Return(nil) + h.EXPECT().LoadKernelModule("vfio_pci").Return(nil) + v.LoadVfioDriver = loading + sriovNetworkNodeState.Spec.Interfaces = sriovnetworkv1.Interfaces{ + {Name: "eno1", + NumVfs: 10, + PciAddress: "0000:d8:00.0", VfGroups: []sriovnetworkv1.VfGroup{ + { + ResourceName: "test", + PolicyName: "test", + VfRange: "eno1#0-9", + DeviceType: consts.DeviceTypeVfioPci, + }, + }, + }, + } + + v.LastState = sriovNetworkNodeState + v.DesireState = sriovNetworkNodeState + err := v.Apply() + Expect(err).ToNot(HaveOccurred()) + Expect(v.LoadVfioDriver).To(Equal(uint(loaded))) + }) + + It("should return error if not able to load the vfio kernel module", func() { + h.EXPECT().LoadKernelModule("vfio", "enable_unsafe_noiommu_mode=1").Return(fmt.Errorf("failed to load kernel module")) + v.LoadVfioDriver = loading + sriovNetworkNodeState.Spec.Interfaces = sriovnetworkv1.Interfaces{ + {Name: "eno1", + NumVfs: 10, + PciAddress: "0000:d8:00.0", VfGroups: []sriovnetworkv1.VfGroup{ + { + ResourceName: "test", + PolicyName: "test", + VfRange: "eno1#0-9", + DeviceType: consts.DeviceTypeVfioPci, + }, + }, + }, + } + v.DesireState = sriovNetworkNodeState + + err := v.Apply() + Expect(err).To(HaveOccurred()) + }) + + It("should return error if not able to load the vfio_pci kernel module", func() { + h.EXPECT().LoadKernelModule("vfio", "enable_unsafe_noiommu_mode=1").Return(nil) + h.EXPECT().LoadKernelModule("vfio_pci").Return(fmt.Errorf("failed to load kernel module")) + v.LoadVfioDriver = loading + sriovNetworkNodeState.Spec.Interfaces = sriovnetworkv1.Interfaces{ + {Name: "eno1", + NumVfs: 10, + PciAddress: "0000:d8:00.0", VfGroups: []sriovnetworkv1.VfGroup{ + { + ResourceName: "test", + PolicyName: "test", + VfRange: "eno1#0-9", + DeviceType: consts.DeviceTypeVfioPci, + }, + }, + }, + } + v.DesireState = sriovNetworkNodeState + err := v.Apply() + Expect(err).To(HaveOccurred()) + }) + + It("should configure the device", func() { + v.LoadVfioDriver = loaded + h.EXPECT().Chroot(gomock.Any()).Return(func() error { return nil }, nil) + h.EXPECT().ConfigSriovDeviceVirtual(gomock.Any()).Return(nil) + sriovNetworkNodeState.Spec.Interfaces = sriovnetworkv1.Interfaces{ + {Name: "eno1", + NumVfs: 1, + PciAddress: "0000:d8:00.0", VfGroups: []sriovnetworkv1.VfGroup{ + { + ResourceName: "test", + PolicyName: "test", + VfRange: "0-0", + DeviceType: consts.DeviceTypeVfioPci}, + }, + }, + } + sriovNetworkNodeState.Status.Interfaces = sriovnetworkv1.InterfaceExts{ + { + Name: "eno1", + NumVfs: 1, + PciAddress: "0000:d8:00.0", + VFs: []sriovnetworkv1.VirtualFunction{ + { + Name: "eno1", + PciAddress: "0000:d8:00.0", + Driver: "virtio-net", + }, + }, + }, + } + + v.DesireState = sriovNetworkNodeState + err := v.Apply() + Expect(err).ToNot(HaveOccurred()) + }) + }) +}) diff --git a/pkg/render/render.go b/pkg/render/render.go index aa26b018b..0492dd735 100644 --- a/pkg/render/render.go +++ b/pkg/render/render.go @@ -16,7 +16,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/yaml" - mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + mcfgv1 "github.com/openshift/api/machineconfiguration/v1" "github.com/openshift/machine-config-operator/pkg/controller/common" ) diff --git a/pkg/systemd/systemd.go b/pkg/systemd/systemd.go deleted file mode 100644 index 704e72c0f..000000000 --- a/pkg/systemd/systemd.go +++ /dev/null @@ -1,312 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package systemd - -import ( - "bytes" - "fmt" - "os" - "strings" - - "gopkg.in/yaml.v3" - "sigs.k8s.io/controller-runtime/pkg/log" - - sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" -) - -const ( - SriovSystemdConfigPath = consts.SriovConfBasePath + "/sriov-interface-config.yaml" - SriovSystemdResultPath = consts.SriovConfBasePath + "/sriov-interface-result.yaml" - sriovSystemdSupportedNicPath = consts.SriovConfBasePath + "/sriov-supported-nics-ids.yaml" - sriovSystemdServiceBinaryPath = "/var/lib/sriov/sriov-network-config-daemon" - - SriovServicePath = "/etc/systemd/system/sriov-config.service" - SriovPostNetworkServicePath = "/etc/systemd/system/sriov-config-post-network.service" -) - -// TODO: move this to the host interface also - -type SriovConfig struct { - Spec sriovnetworkv1.SriovNetworkNodeStateSpec `yaml:"spec"` - UnsupportedNics bool `yaml:"unsupportedNics"` - PlatformType consts.PlatformTypes `yaml:"platformType"` - ManageSoftwareBridges bool `yaml:"manageSoftwareBridges"` - OVSDBSocketPath string `yaml:"ovsdbSocketPath"` -} - -type SriovResult struct { - SyncStatus string `yaml:"syncStatus"` - LastSyncError string `yaml:"lastSyncError"` -} - -func ReadConfFile() (spec *SriovConfig, err error) { - rawConfig, err := os.ReadFile(utils.GetHostExtensionPath(SriovSystemdConfigPath)) - if err != nil { - return nil, err - } - - err = yaml.Unmarshal(rawConfig, &spec) - - return spec, err -} - -func WriteConfFile(newState *sriovnetworkv1.SriovNetworkNodeState) (bool, error) { - newFile := false - sriovConfig := &SriovConfig{ - newState.Spec, - vars.DevMode, - vars.PlatformType, - vars.ManageSoftwareBridges, - vars.OVSDBSocketPath, - } - - _, err := os.Stat(utils.GetHostExtensionPath(SriovSystemdConfigPath)) - if err != nil { - if os.IsNotExist(err) { - // Create the sriov-operator folder on the host if it doesn't exist - if _, err := os.Stat(utils.GetHostExtensionPath(consts.SriovConfBasePath)); os.IsNotExist(err) { - err = os.Mkdir(utils.GetHostExtensionPath(consts.SriovConfBasePath), os.ModeDir) - if err != nil { - log.Log.Error(err, "WriteConfFile(): fail to create sriov-operator folder", - "path", utils.GetHostExtensionPath(consts.SriovConfBasePath)) - return false, err - } - } - - log.Log.V(2).Info("WriteConfFile(): file not existed, create it", - "path", utils.GetHostExtensionPath(SriovSystemdConfigPath)) - _, err = os.Create(utils.GetHostExtensionPath(SriovSystemdConfigPath)) - if err != nil { - log.Log.Error(err, "WriteConfFile(): fail to create file") - return false, err - } - newFile = true - } else { - return false, err - } - } - - oldContent, err := os.ReadFile(utils.GetHostExtensionPath(SriovSystemdConfigPath)) - if err != nil { - log.Log.Error(err, "WriteConfFile(): fail to read file", "path", utils.GetHostExtensionPath(SriovSystemdConfigPath)) - return false, err - } - - oldContentObj := &SriovConfig{} - err = yaml.Unmarshal(oldContent, oldContentObj) - if err != nil { - log.Log.Error(err, "WriteConfFile(): fail to unmarshal old file") - return false, err - } - - var newContent []byte - newContent, err = yaml.Marshal(sriovConfig) - if err != nil { - log.Log.Error(err, "WriteConfFile(): fail to marshal sriov config") - return false, err - } - - if bytes.Equal(newContent, oldContent) { - log.Log.V(2).Info("WriteConfFile(): no update") - return false, nil - } - log.Log.V(2).Info("WriteConfFile(): old and new configuration are not equal", - "old", string(oldContent), "new", string(newContent)) - - log.Log.V(2).Info("WriteConfFile(): write content to file", - "content", newContent, "path", utils.GetHostExtensionPath(SriovSystemdConfigPath)) - err = os.WriteFile(utils.GetHostExtensionPath(SriovSystemdConfigPath), newContent, 0644) - if err != nil { - log.Log.Error(err, "WriteConfFile(): fail to write file") - return false, err - } - - // this will be used to mark the first time we create this file. - // this helps to avoid the first reboot after installation - if newFile && len(sriovConfig.Spec.Interfaces) == 0 { - log.Log.V(2).Info("WriteConfFile(): first file creation and no interfaces to configure returning reboot false") - return false, nil - } - - return true, nil -} - -func WriteSriovResult(result *SriovResult) error { - _, err := os.Stat(utils.GetHostExtensionPath(SriovSystemdResultPath)) - if err != nil { - if os.IsNotExist(err) { - log.Log.V(2).Info("WriteSriovResult(): file not existed, create it") - _, err = os.Create(utils.GetHostExtensionPath(SriovSystemdResultPath)) - if err != nil { - log.Log.Error(err, "WriteSriovResult(): failed to create sriov result file", "path", utils.GetHostExtensionPath(SriovSystemdResultPath)) - return err - } - } else { - log.Log.Error(err, "WriteSriovResult(): failed to check sriov result file", "path", utils.GetHostExtensionPath(SriovSystemdResultPath)) - return err - } - } - - out, err := yaml.Marshal(result) - if err != nil { - log.Log.Error(err, "WriteSriovResult(): failed to marshal sriov result") - return err - } - - log.Log.V(2).Info("WriteSriovResult(): write results", - "content", string(out), "path", utils.GetHostExtensionPath(SriovSystemdResultPath)) - err = os.WriteFile(utils.GetHostExtensionPath(SriovSystemdResultPath), out, 0644) - if err != nil { - log.Log.Error(err, "WriteSriovResult(): failed to write sriov result file", "path", utils.GetHostExtensionPath(SriovSystemdResultPath)) - return err - } - - return nil -} - -func ReadSriovResult() (*SriovResult, error) { - _, err := os.Stat(utils.GetHostExtensionPath(SriovSystemdResultPath)) - if err != nil { - if os.IsNotExist(err) { - log.Log.V(2).Info("ReadSriovResult(): file does not exist, return empty result") - return &SriovResult{}, nil - } else { - log.Log.Error(err, "ReadSriovResult(): failed to check sriov result file", "path", utils.GetHostExtensionPath(SriovSystemdResultPath)) - return nil, err - } - } - - rawConfig, err := os.ReadFile(utils.GetHostExtensionPath(SriovSystemdResultPath)) - if err != nil { - log.Log.Error(err, "ReadSriovResult(): failed to read sriov result file", "path", utils.GetHostExtensionPath(SriovSystemdResultPath)) - return nil, err - } - - result := &SriovResult{} - err = yaml.Unmarshal(rawConfig, &result) - if err != nil { - log.Log.Error(err, "ReadSriovResult(): failed to unmarshal sriov result file", "path", utils.GetHostExtensionPath(SriovSystemdResultPath)) - return nil, err - } - return result, err -} - -func RemoveSriovResult() error { - err := os.Remove(utils.GetHostExtensionPath(SriovSystemdResultPath)) - if err != nil { - if os.IsNotExist(err) { - log.Log.V(2).Info("RemoveSriovResult(): result file not found") - return nil - } - log.Log.Error(err, "RemoveSriovResult(): failed to remove sriov result file", "path", utils.GetHostExtensionPath(SriovSystemdResultPath)) - return err - } - log.Log.V(2).Info("RemoveSriovResult(): result file removed") - return nil -} - -func WriteSriovSupportedNics() error { - _, err := os.Stat(utils.GetHostExtensionPath(sriovSystemdSupportedNicPath)) - if err != nil { - if os.IsNotExist(err) { - log.Log.V(2).Info("WriteSriovSupportedNics(): file does not exist, create it") - _, err = os.Create(utils.GetHostExtensionPath(sriovSystemdSupportedNicPath)) - if err != nil { - log.Log.Error(err, "WriteSriovSupportedNics(): failed to create sriov supporter nics ids file", - "path", utils.GetHostExtensionPath(sriovSystemdSupportedNicPath)) - return err - } - } else { - log.Log.Error(err, "WriteSriovSupportedNics(): failed to check sriov supported nics ids file", "path", utils.GetHostExtensionPath(sriovSystemdSupportedNicPath)) - return err - } - } - - rawNicList := []byte{} - for _, line := range sriovnetworkv1.NicIDMap { - rawNicList = append(rawNicList, []byte(fmt.Sprintf("%s\n", line))...) - } - - err = os.WriteFile(utils.GetHostExtensionPath(sriovSystemdSupportedNicPath), rawNicList, 0644) - if err != nil { - log.Log.Error(err, "WriteSriovSupportedNics(): failed to write sriov supported nics ids file", - "path", utils.GetHostExtensionPath(sriovSystemdSupportedNicPath)) - return err - } - - return nil -} - -func ReadSriovSupportedNics() ([]string, error) { - _, err := os.Stat(utils.GetHostExtensionPath(sriovSystemdSupportedNicPath)) - if err != nil { - if os.IsNotExist(err) { - log.Log.V(2).Info("ReadSriovSupportedNics(): file does not exist, return empty result") - return nil, err - } else { - log.Log.Error(err, "ReadSriovSupportedNics(): failed to check sriov supported nics file", "path", utils.GetHostExtensionPath(sriovSystemdSupportedNicPath)) - return nil, err - } - } - - rawConfig, err := os.ReadFile(utils.GetHostExtensionPath(sriovSystemdSupportedNicPath)) - if err != nil { - log.Log.Error(err, "ReadSriovSupportedNics(): failed to read sriov supported nics file", "path", utils.GetHostExtensionPath(sriovSystemdSupportedNicPath)) - return nil, err - } - - lines := strings.Split(string(rawConfig), "\n") - return lines, nil -} - -func CleanSriovFilesFromHost(isOpenShift bool) error { - err := os.Remove(utils.GetHostExtensionPath(SriovSystemdConfigPath)) - if err != nil && !os.IsNotExist(err) { - return err - } - - err = os.Remove(utils.GetHostExtensionPath(SriovSystemdResultPath)) - if err != nil && !os.IsNotExist(err) { - return err - } - - err = os.Remove(utils.GetHostExtensionPath(sriovSystemdSupportedNicPath)) - if err != nil && !os.IsNotExist(err) { - return err - } - - err = os.Remove(utils.GetHostExtensionPath(sriovSystemdServiceBinaryPath)) - if err != nil && !os.IsNotExist(err) { - return err - } - - // in openshift we should not remove the systemd service it will be done by the machine config operator - if !isOpenShift { - err = os.Remove(utils.GetHostExtensionPath(SriovServicePath)) - if err != nil && !os.IsNotExist(err) { - return err - } - err = os.Remove(utils.GetHostExtensionPath(SriovPostNetworkServicePath)) - if err != nil && !os.IsNotExist(err) { - return err - } - } - - return nil -} diff --git a/pkg/utils/cluster.go b/pkg/utils/cluster.go index 6f8d72e07..994d66c64 100644 --- a/pkg/utils/cluster.go +++ b/pkg/utils/cluster.go @@ -2,148 +2,117 @@ package utils import ( "context" - "fmt" - "os" - "sigs.k8s.io/controller-runtime/pkg/log" - - configv1 "github.com/openshift/api/config/v1" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "sigs.k8s.io/controller-runtime/pkg/log" ) -const ( - // default Infrastructure resource name for Openshift - infraResourceName = "cluster" - workerRoleName = "worker" - masterRoleName = "master" - workerNodeLabelKey = "node-role.kubernetes.io/worker" - masterNodeLabelKey = "node-role.kubernetes.io/master" - controlPlaneNodeLabelKey = "node-role.kubernetes.io/control-plane" -) +// ObjectHasAnnotationKey checks if a kubernetes object already contains annotation +func ObjectHasAnnotationKey(obj metav1.Object, annoKey string) bool { + _, hasKey := obj.GetAnnotations()[annoKey] + return hasKey +} -func getNodeRole(node corev1.Node) string { - for k := range node.Labels { - if k == workerNodeLabelKey { - return workerRoleName - } else if k == masterNodeLabelKey || k == controlPlaneNodeLabelKey { - return masterRoleName - } +// ObjectHasAnnotation checks if a kubernetes object already contains annotation +func ObjectHasAnnotation(obj metav1.Object, annoKey string, value string) bool { + if anno, ok := obj.GetAnnotations()[annoKey]; ok && (anno == value) { + return true } - return "" + return false } -func IsSingleNodeCluster(c client.Client) (bool, error) { - if os.Getenv("CLUSTER_TYPE") == consts.ClusterTypeOpenshift { - topo, err := openshiftControlPlaneTopologyStatus(c) - if err != nil { - return false, err - } - if topo == configv1.SingleReplicaTopologyMode { - return true, nil - } - return false, nil +// AnnotateObject adds annotation to a kubernetes object +func AnnotateObject(ctx context.Context, obj client.Object, key, value string, c client.Client) error { + original := obj.DeepCopyObject().(client.Object) + if obj.GetAnnotations() == nil { + obj.SetAnnotations(map[string]string{}) } - return k8sSingleNodeClusterStatus(c) -} -// IsExternalControlPlaneCluster detects control plane location of the cluster. -// On OpenShift, the control plane topology is configured in configv1.Infrastucture struct. -// On kubernetes, it is determined by which node the sriov operator is scheduled on. If operator -// pod is schedule on worker node, it is considered as external control plane. -func IsExternalControlPlaneCluster(c client.Client) (bool, error) { - if os.Getenv("CLUSTER_TYPE") == consts.ClusterTypeOpenshift { - topo, err := openshiftControlPlaneTopologyStatus(c) - if err != nil { - return false, err - } - if topo == "External" { - return true, nil + if obj.GetAnnotations()[key] != value { + logger, ok := ctx.Value("logger").(logr.Logger) + if !ok { + log.Log.Info("Annotate object", + "name", obj.GetName(), + "key", key, + "value", value) + } else { + logger.Info("Annotate object", + "name", obj.GetName(), + "key", key, + "value", value) } - } else if os.Getenv("CLUSTER_TYPE") == consts.ClusterTypeKubernetes { - role, err := operatorNodeRole(c) + + obj.GetAnnotations()[key] = value + patch := client.MergeFrom(original) + err := c.Patch(ctx, + obj, patch) if err != nil { - return false, err - } - if role == workerRoleName { - return true, nil + log.Log.Error(err, "annotateObject(): Failed to patch object") + return err } } - return false, nil + return nil } -func k8sSingleNodeClusterStatus(c client.Client) (bool, error) { - nodeList := &corev1.NodeList{} - err := c.List(context.TODO(), nodeList) +// AnnotateNode add annotation to a node +func AnnotateNode(ctx context.Context, nodeName string, key, value string, c client.Client) error { + node := &corev1.Node{} + err := c.Get(ctx, client.ObjectKey{Name: nodeName}, node) if err != nil { - log.Log.Error(err, "k8sSingleNodeClusterStatus(): Failed to list nodes") - return false, err + return err } - if len(nodeList.Items) == 1 { - log.Log.Info("k8sSingleNodeClusterStatus(): one node found in the cluster") - return true, nil - } - return false, nil + return AnnotateObject(ctx, node, key, value, c) } -// operatorNodeRole returns role of the node where operator is scheduled on -func operatorNodeRole(c client.Client) (string, error) { - node := corev1.Node{} - err := c.Get(context.TODO(), types.NamespacedName{Name: os.Getenv("NODE_NAME")}, &node) - if err != nil { - log.Log.Error(err, "k8sIsExternalTopologyMode(): Failed to get node") - return "", err +// labelObject adds label to a kubernetes object +func labelObject(ctx context.Context, obj client.Object, key, value string, c client.Client) error { + newObj := obj.DeepCopyObject().(client.Object) + if newObj.GetLabels() == nil { + newObj.SetLabels(map[string]string{}) } - return getNodeRole(node), nil -} - -func openshiftControlPlaneTopologyStatus(c client.Client) (configv1.TopologyMode, error) { - infra := &configv1.Infrastructure{} - err := c.Get(context.TODO(), types.NamespacedName{Name: infraResourceName}, infra) - if err != nil { - return "", fmt.Errorf("openshiftControlPlaneTopologyStatus(): Failed to get Infrastructure (name: %s): %v", infraResourceName, err) + if newObj.GetLabels()[key] != value { + log.Log.V(2).Info("labelObject(): label object", + "objectName", obj.GetName(), + "objectKind", obj.GetObjectKind(), + "labelKey", key, + "labelValue", value) + newObj.GetLabels()[key] = value + patch := client.MergeFrom(obj) + err := c.Patch(ctx, + newObj, patch) + if err != nil { + log.Log.Error(err, "labelObject(): Failed to patch object") + return err + } } - return infra.Status.ControlPlaneTopology, nil -} -// ObjectHasAnnotationKey checks if a kubernetes object already contains annotation -func ObjectHasAnnotationKey(obj metav1.Object, annoKey string) bool { - _, hasKey := obj.GetAnnotations()[annoKey] - return hasKey -} - -// ObjectHasAnnotation checks if a kubernetes object already contains annotation -func ObjectHasAnnotation(obj metav1.Object, annoKey string, value string) bool { - if anno, ok := obj.GetAnnotations()[annoKey]; ok && (anno == value) { - return true - } - return false + return nil } -// AnnotateObject adds annotation to a kubernetes object -func AnnotateObject(ctx context.Context, obj client.Object, key, value string, c client.Client) error { - log.Log.V(2).Info("AnnotateObject(): Annotate object", - "objectName", obj.GetName(), - "objectKind", obj.GetObjectKind(), - "annotation", value) +// removeLabelObject remove a label from a kubernetes object +func removeLabelObject(ctx context.Context, obj client.Object, key string, c client.Client) error { newObj := obj.DeepCopyObject().(client.Object) - if newObj.GetAnnotations() == nil { - newObj.SetAnnotations(map[string]string{}) + if newObj.GetLabels() == nil { + newObj.SetLabels(map[string]string{}) } - if newObj.GetAnnotations()[key] != value { - newObj.GetAnnotations()[key] = value + _, exist := newObj.GetLabels()[key] + if exist { + log.Log.V(2).Info("removeLabelObject(): remove label from object", + "objectName", obj.GetName(), + "objectKind", obj.GetObjectKind(), + "labelKey", key) + delete(newObj.GetLabels(), key) patch := client.MergeFrom(obj) err := c.Patch(ctx, newObj, patch) if err != nil { - log.Log.Error(err, "annotateObject(): Failed to patch object") + log.Log.Error(err, "removeLabelObject(): Failed to patch object") return err } } @@ -151,13 +120,23 @@ func AnnotateObject(ctx context.Context, obj client.Object, key, value string, c return nil } -// AnnotateNode add annotation to a node -func AnnotateNode(ctx context.Context, nodeName string, key, value string, c client.Client) error { +// LabelNode add label to a node +func LabelNode(ctx context.Context, nodeName string, key, value string, c client.Client) error { node := &corev1.Node{} err := c.Get(context.TODO(), client.ObjectKey{Name: nodeName}, node) if err != nil { return err } - return AnnotateObject(ctx, node, key, value, c) + return labelObject(ctx, node, key, value, c) +} + +func RemoveLabelFromNode(ctx context.Context, nodeName string, key string, c client.Client) error { + node := &corev1.Node{} + err := c.Get(context.TODO(), client.ObjectKey{Name: nodeName}, node) + if err != nil { + return err + } + + return removeLabelObject(ctx, node, key, c) } diff --git a/pkg/utils/mock/mock_command.go b/pkg/utils/mock/mock_command.go deleted file mode 100644 index d408a48b0..000000000 --- a/pkg/utils/mock/mock_command.go +++ /dev/null @@ -1,56 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: command.go - -// Package mock_utils is a generated GoMock package. -package mock_utils - -import ( - bytes "bytes" - reflect "reflect" - - gomock "github.com/golang/mock/gomock" -) - -// MockCommandInterface is a mock of CommandInterface interface. -type MockCommandInterface struct { - ctrl *gomock.Controller - recorder *MockCommandInterfaceMockRecorder -} - -// MockCommandInterfaceMockRecorder is the mock recorder for MockCommandInterface. -type MockCommandInterfaceMockRecorder struct { - mock *MockCommandInterface -} - -// NewMockCommandInterface creates a new mock instance. -func NewMockCommandInterface(ctrl *gomock.Controller) *MockCommandInterface { - mock := &MockCommandInterface{ctrl: ctrl} - mock.recorder = &MockCommandInterfaceMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockCommandInterface) EXPECT() *MockCommandInterfaceMockRecorder { - return m.recorder -} - -// Run mocks base method. -func (m *MockCommandInterface) Run(arg0 string, arg1 ...string) (bytes.Buffer, bytes.Buffer, error) { - m.ctrl.T.Helper() - varargs := []interface{}{arg0} - for _, a := range arg1 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Run", varargs...) - ret0, _ := ret[0].(bytes.Buffer) - ret1, _ := ret[1].(bytes.Buffer) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// Run indicates an expected call of Run. -func (mr *MockCommandInterfaceMockRecorder) Run(arg0 interface{}, arg1 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0}, arg1...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockCommandInterface)(nil).Run), varargs...) -} diff --git a/pkg/utils/mock/mock_store.go b/pkg/utils/mock/mock_store.go deleted file mode 100644 index d6ff6b8fd..000000000 --- a/pkg/utils/mock/mock_store.go +++ /dev/null @@ -1,79 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: store.go - -// Package mock_utils is a generated GoMock package. -package mock_utils - -import ( - reflect "reflect" - - gomock "github.com/golang/mock/gomock" - v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" -) - -// MockStoreManagerInterface is a mock of StoreManagerInterface interface. -type MockStoreManagerInterface struct { - ctrl *gomock.Controller - recorder *MockStoreManagerInterfaceMockRecorder -} - -// MockStoreManagerInterfaceMockRecorder is the mock recorder for MockStoreManagerInterface. -type MockStoreManagerInterfaceMockRecorder struct { - mock *MockStoreManagerInterface -} - -// NewMockStoreManagerInterface creates a new mock instance. -func NewMockStoreManagerInterface(ctrl *gomock.Controller) *MockStoreManagerInterface { - mock := &MockStoreManagerInterface{ctrl: ctrl} - mock.recorder = &MockStoreManagerInterfaceMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockStoreManagerInterface) EXPECT() *MockStoreManagerInterfaceMockRecorder { - return m.recorder -} - -// ClearPCIAddressFolder mocks base method. -func (m *MockStoreManagerInterface) ClearPCIAddressFolder() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ClearPCIAddressFolder") - ret0, _ := ret[0].(error) - return ret0 -} - -// ClearPCIAddressFolder indicates an expected call of ClearPCIAddressFolder. -func (mr *MockStoreManagerInterfaceMockRecorder) ClearPCIAddressFolder() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClearPCIAddressFolder", reflect.TypeOf((*MockStoreManagerInterface)(nil).ClearPCIAddressFolder)) -} - -// LoadPfsStatus mocks base method. -func (m *MockStoreManagerInterface) LoadPfsStatus(pciAddress string) (*v1.Interface, bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "LoadPfsStatus", pciAddress) - ret0, _ := ret[0].(*v1.Interface) - ret1, _ := ret[1].(bool) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// LoadPfsStatus indicates an expected call of LoadPfsStatus. -func (mr *MockStoreManagerInterfaceMockRecorder) LoadPfsStatus(pciAddress interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadPfsStatus", reflect.TypeOf((*MockStoreManagerInterface)(nil).LoadPfsStatus), pciAddress) -} - -// SaveLastPfAppliedStatus mocks base method. -func (m *MockStoreManagerInterface) SaveLastPfAppliedStatus(PfInfo *v1.Interface) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SaveLastPfAppliedStatus", PfInfo) - ret0, _ := ret[0].(error) - return ret0 -} - -// SaveLastPfAppliedStatus indicates an expected call of SaveLastPfAppliedStatus. -func (mr *MockStoreManagerInterfaceMockRecorder) SaveLastPfAppliedStatus(PfInfo interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveLastPfAppliedStatus", reflect.TypeOf((*MockStoreManagerInterface)(nil).SaveLastPfAppliedStatus), PfInfo) -} diff --git a/pkg/utils/mock/mock_utils.go b/pkg/utils/mock/mock_utils.go index 636c7f4a1..2d2d8e1c2 100644 --- a/pkg/utils/mock/mock_utils.go +++ b/pkg/utils/mock/mock_utils.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: utils.go +// +// Generated by this command: +// +// mockgen -destination mock/mock_utils.go -source utils.go +// // Package mock_utils is a generated GoMock package. package mock_utils @@ -7,13 +12,14 @@ package mock_utils import ( reflect "reflect" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" ) // MockCmdInterface is a mock of CmdInterface interface. type MockCmdInterface struct { ctrl *gomock.Controller recorder *MockCmdInterfaceMockRecorder + isgomock struct{} } // MockCmdInterfaceMockRecorder is the mock recorder for MockCmdInterface. @@ -43,7 +49,7 @@ func (m *MockCmdInterface) Chroot(arg0 string) (func() error, error) { } // Chroot indicates an expected call of Chroot. -func (mr *MockCmdInterfaceMockRecorder) Chroot(arg0 interface{}) *gomock.Call { +func (mr *MockCmdInterfaceMockRecorder) Chroot(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Chroot", reflect.TypeOf((*MockCmdInterface)(nil).Chroot), arg0) } @@ -51,7 +57,7 @@ func (mr *MockCmdInterfaceMockRecorder) Chroot(arg0 interface{}) *gomock.Call { // RunCommand mocks base method. func (m *MockCmdInterface) RunCommand(arg0 string, arg1 ...string) (string, string, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0} + varargs := []any{arg0} for _, a := range arg1 { varargs = append(varargs, a) } @@ -63,8 +69,8 @@ func (m *MockCmdInterface) RunCommand(arg0 string, arg1 ...string) (string, stri } // RunCommand indicates an expected call of RunCommand. -func (mr *MockCmdInterfaceMockRecorder) RunCommand(arg0 interface{}, arg1 ...interface{}) *gomock.Call { +func (mr *MockCmdInterfaceMockRecorder) RunCommand(arg0 any, arg1 ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0}, arg1...) + varargs := append([]any{arg0}, arg1...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RunCommand", reflect.TypeOf((*MockCmdInterface)(nil).RunCommand), varargs...) } diff --git a/pkg/utils/shutdown.go b/pkg/utils/shutdown.go index f8f9618d4..cce1399f5 100644 --- a/pkg/utils/shutdown.go +++ b/pkg/utils/shutdown.go @@ -7,51 +7,50 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" conf "sigs.k8s.io/controller-runtime/pkg/client/config" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - snclientset "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) var shutdownLog = ctrl.Log.WithName("shutdown") var failurePolicyIgnore = admv1.Ignore -func Shutdown() { - updateFinalizers() +func Shutdown(c client.Client) { + updateFinalizers(c) updateWebhooks() } -func updateFinalizers() { +func updateFinalizers(c client.Client) { shutdownLog.Info("Clearing finalizers on exit") - c, err := snclientset.NewForConfig(conf.GetConfigOrDie()) - if err != nil { - shutdownLog.Error(err, "Error creating client") - } - sriovNetworkClient := c.SriovnetworkV1() - networkList, err := sriovNetworkClient.SriovNetworks("").List(context.TODO(), metav1.ListOptions{}) + networkList := &sriovnetworkv1.SriovNetworkList{} + err := c.List(context.TODO(), networkList, &client.ListOptions{Namespace: vars.Namespace}) if err != nil { shutdownLog.Error(err, "Failed to list SriovNetworks") - } else { - for _, instance := range networkList.Items { - if instance.ObjectMeta.Finalizers == nil || len(instance.ObjectMeta.Finalizers) == 0 { - continue - } + return + } + + for _, instance := range networkList.Items { + if len(instance.ObjectMeta.Finalizers) == 0 { + continue + } + if err != nil { + shutdownLog.Error(err, "Failed get finalizers map") + } + shutdownLog.Info("Clearing finalizers on SriovNetwork ", "namespace", instance.GetNamespace(), "name", instance.GetName()) + var found bool + instance.ObjectMeta.Finalizers, found = sriovnetworkv1.RemoveString(sriovnetworkv1.NETATTDEFFINALIZERNAME, instance.ObjectMeta.Finalizers) + if found { + err = c.Update(context.TODO(), &instance) if err != nil { - shutdownLog.Error(err, "Failed get finalizers map") - } - shutdownLog.Info("Clearing finalizers on SriovNetwork ", "namespace", instance.GetNamespace(), "name", instance.GetName()) - var found bool - instance.ObjectMeta.Finalizers, found = sriovnetworkv1.RemoveString(sriovnetworkv1.NETATTDEFFINALIZERNAME, instance.ObjectMeta.Finalizers) - if found { - _, err = sriovNetworkClient.SriovNetworks(instance.GetNamespace()).Update(context.TODO(), &instance, metav1.UpdateOptions{}) - if err != nil { - shutdownLog.Error(err, "Failed to remove finalizer") - } + shutdownLog.Error(err, "Failed to remove finalizer") } } } + shutdownLog.Info("Done clearing finalizers on exit") } diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go deleted file mode 100644 index 9ca82ff0d..000000000 --- a/pkg/utils/utils_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package utils_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestUtils(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Utils Suite") -} diff --git a/pkg/vars/vars.go b/pkg/vars/vars.go index fc7108ed8..44106575c 100644 --- a/pkg/vars/vars.go +++ b/pkg/vars/vars.go @@ -8,6 +8,7 @@ import ( "k8s.io/client-go/rest" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/featuregate" ) var ( @@ -54,9 +55,6 @@ var ( // ManageSoftwareBridges global variable which reflects state of manageSoftwareBridges feature ManageSoftwareBridges = false - // MlxPluginFwReset global variable enables mstfwreset before rebooting a node on VF changes - MlxPluginFwReset = false - // FilesystemRoot used by test to mock interactions with filesystem FilesystemRoot = "" @@ -75,6 +73,12 @@ var ( // DisableablePlugins contains which plugins can be disabled in sriov config daemon DisableablePlugins = map[string]struct{}{"mellanox": {}} + + // DisableDrain controls if the daemon will drain the node before configuration + DisableDrain = false + + // FeatureGates interface to interact with feature gates + FeatureGate featuregate.FeatureGate ) func init() { @@ -95,4 +99,6 @@ func init() { } ResourcePrefix = os.Getenv("RESOURCE_PREFIX") + + FeatureGate = featuregate.New() } diff --git a/pkg/vendors/mellanox/mellanox.go b/pkg/vendors/mellanox/mellanox.go index 82410c7f8..0cc60b8b7 100644 --- a/pkg/vendors/mellanox/mellanox.go +++ b/pkg/vendors/mellanox/mellanox.go @@ -6,11 +6,13 @@ import ( "strconv" "strings" + "github.com/pkg/errors" kerrors "k8s.io/apimachinery/pkg/util/errors" "sigs.k8s.io/controller-runtime/pkg/log" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" ) @@ -39,7 +41,7 @@ const ( DeviceBF3 = "a2dc" PreconfiguredLinkType = "Preconfigured" - UknownLinkType = "Uknown" + UnknownLinkType = "Unknown" TotalVfs = "NUM_OF_VFS" EnableSriov = "SRIOV_EN" LinkTypeP1 = "LINK_TYPE_P1" @@ -61,16 +63,18 @@ type MellanoxInterface interface { GetMlxNicFwData(pciAddress string) (current, next *MlxNic, err error) MlxConfigFW(attributesToChange map[string]MlxNic) error - MlxResetFW(pciAddresses []string) error + MlxResetFW(pciAddresses []string, mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt) error } type mellanoxHelper struct { - utils utils.CmdInterface + utils utils.CmdInterface + hostHelper host.HostManagerInterface } -func New(utilsHelper utils.CmdInterface) MellanoxInterface { +func New(utilsHelper utils.CmdInterface, hostHelper host.HostManagerInterface) MellanoxInterface { return &mellanoxHelper{ - utils: utilsHelper, + utils: utilsHelper, + hostHelper: hostHelper, } } @@ -98,27 +102,27 @@ func (m *mellanoxHelper) GetMellanoxBlueFieldMode(PciAddress string) (BlueFieldM internalCPUPageSupplierstatus, exist := mstCurrentData[internalCPUPageSupplier] if !exist { - return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUPageSupplier) + return -1, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUPageSupplier) } internalCPUEswitchManagerStatus, exist := mstCurrentData[internalCPUEswitchManager] if !exist { - return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUEswitchManager) + return -1, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUEswitchManager) } internalCPUIbVportoStatus, exist := mstCurrentData[internalCPUIbVporto] if !exist { - return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUIbVporto) + return -1, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUIbVporto) } internalCPUOffloadEngineStatus, exist := mstCurrentData[internalCPUOffloadEngine] if !exist { - return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUOffloadEngine) + return -1, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUOffloadEngine) } internalCPUModelStatus, exist := mstCurrentData[internalCPUModel] if !exist { - return 0, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUModel) + return -1, fmt.Errorf("failed to find %s in the mstconfig output command", internalCPUModel) } // check for DPU @@ -143,10 +147,25 @@ func (m *mellanoxHelper) GetMellanoxBlueFieldMode(PciAddress string) (BlueFieldM return -1, fmt.Errorf("MellanoxBlueFieldMode(): unknown device status for %s", PciAddress) } -func (m *mellanoxHelper) MlxResetFW(pciAddresses []string) error { +func (m *mellanoxHelper) MlxResetFW(pciAddresses []string, mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt) error { log.Log.Info("mellanox-plugin resetFW()") var errs []error for _, pciAddress := range pciAddresses { + err := m.hostHelper.SetSriovNumVfs(pciAddress, 0) + if err != nil { + log.Log.Error(err, "failed to set SR-IOV number of VFs to 0 before firmware reset", "pciAddress", pciAddress) + return err + } + + if IsDualPort(pciAddress, mellanoxNicsStatus) { + otherPortPCIAddress := getOtherPortPCIAddress(pciAddress) + err := m.hostHelper.SetSriovNumVfs(otherPortPCIAddress, 0) + if err != nil { + log.Log.Error(err, "failed to set SR-IOV number of VFs to 0 before firmware reset", "pciAddress", otherPortPCIAddress) + return err + } + } + cmdArgs := []string{"-d", pciAddress, "--skip_driver", "-l", "3", "-y", "reset"} log.Log.Info("mellanox-plugin: resetFW()", "cmd-args", cmdArgs) // We have to ensure that pciutils is installed into the container image Dockerfile.sriov-network-config-daemon @@ -156,12 +175,22 @@ func (m *mellanoxHelper) MlxResetFW(pciAddresses []string) error { errs = append(errs, err) } } + return kerrors.NewAggregate(errs) } func (m *mellanoxHelper) MlxConfigFW(attributesToChange map[string]MlxNic) error { log.Log.Info("mellanox-plugin configFW()") for pciAddr, fwArgs := range attributesToChange { + bfMode, err := m.GetMellanoxBlueFieldMode(pciAddr) + if err != nil { + // NIC is not a DPU or mstconfig failed. It's safe to continue FW configuration + log.Log.V(2).Info("mellanox-plugin: configFW(): can't get DPU mode for NIC", "pciAddress", pciAddr) + } + if bfMode == BluefieldDpu { + // Host reboot won't re-load NIC firmware in DPU mode. To apply FW changes power cycle is required or mstfwreset could be used. + return errors.Errorf("NIC %s is in DPU mode. Firmware configuration changes are not supported in this mode.", pciAddr) + } cmdArgs := []string{"-d", pciAddr, "-y", "set"} if fwArgs.EnableSriov { cmdArgs = append(cmdArgs, fmt.Sprintf("%s=True", EnableSriov)) @@ -232,22 +261,6 @@ func ParseMstconfigOutput(mstOutput string, attributes []string) (fwCurrent, fwN return } -func HasMellanoxInterfacesInSpec(ifaceStatuses sriovnetworkv1.InterfaceExts, ifaceSpecs sriovnetworkv1.Interfaces) bool { - for _, ifaceStatus := range ifaceStatuses { - if ifaceStatus.Vendor == VendorMellanox { - for _, iface := range ifaceSpecs { - if iface.PciAddress == ifaceStatus.PciAddress { - log.Log.V(2).Info("hasMellanoxInterfacesInSpec(): Mellanox device specified in SriovNetworkNodeState spec", - "name", ifaceStatus.Name, - "address", ifaceStatus.PciAddress) - return true - } - } - } - } - return false -} - func GetPciAddressPrefix(pciAddress string) string { return pciAddress[:len(pciAddress)-1] } @@ -262,10 +275,11 @@ func IsDualPort(pciAddress string, mellanoxNicsStatus map[string]map[string]srio func HandleTotalVfs(fwCurrent, fwNext, attrs *MlxNic, ifaceSpec sriovnetworkv1.Interface, isDualPort bool, mellanoxNicsSpec map[string]sriovnetworkv1.Interface) ( totalVfs int, needReboot, changeWithoutReboot bool) { totalVfs = ifaceSpec.NumVfs - // Check if the other port is changing theGetMlnxNicFwData number of VF + // Check if the other port is changing the number of VF if isDualPort { - otherIfaceSpec := getOtherPortSpec(ifaceSpec.PciAddress, mellanoxNicsSpec) - if otherIfaceSpec != nil { + otherPortPCIAddress := getOtherPortPCIAddress(ifaceSpec.PciAddress) + otherIfaceSpec, ok := mellanoxNicsSpec[otherPortPCIAddress] + if ok { if otherIfaceSpec.NumVfs > totalVfs { totalVfs = otherIfaceSpec.NumVfs } @@ -391,7 +405,7 @@ func getLinkType(linkType string) string { } else if len(linkType) > 0 { log.Log.Error(nil, "mellanox-plugin getLinkType(): link type is not one of [ETH, IB], treating as unknown", "link-type", linkType) - return UknownLinkType + return UnknownLinkType } else { log.Log.Info("mellanox-plugin getLinkType(): LINK_TYPE_P* attribute was not found, treating as preconfigured link type") return PreconfiguredLinkType @@ -405,7 +419,7 @@ func isLinkTypeRequireChange(iface sriovnetworkv1.Interface, ifaceStatus sriovne return false, fmt.Errorf("mellanox-plugin OnNodeStateChange(): Not supported link type: %s,"+ " supported link types: [eth, ETH, ib, and IB]", iface.LinkType) } - if fwLinkType == UknownLinkType { + if fwLinkType == UnknownLinkType { return false, fmt.Errorf("mellanox-plugin OnNodeStateChange(): Unknown link type: %s", fwLinkType) } if fwLinkType == PreconfiguredLinkType { @@ -418,18 +432,16 @@ func isLinkTypeRequireChange(iface sriovnetworkv1.Interface, ifaceStatus sriovne return false, nil } -func getOtherPortSpec(pciAddress string, mellanoxNicsSpec map[string]sriovnetworkv1.Interface) *sriovnetworkv1.Interface { +func getOtherPortPCIAddress(pciAddress string) string { log.Log.Info("mellanox-plugin getOtherPortSpec()", "pciAddress", pciAddress) pciAddrPrefix := GetPciAddressPrefix(pciAddress) pciAddrSuffix := pciAddress[len(pciAddrPrefix):] if pciAddrSuffix == "0" { - iface := mellanoxNicsSpec[pciAddrPrefix+"1"] - return &iface + return pciAddrPrefix + "1" } - iface := mellanoxNicsSpec[pciAddrPrefix+"0"] - return &iface + return pciAddrPrefix + "0" } func getIfaceStatus(pciAddress string, mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt) sriovnetworkv1.InterfaceExt { diff --git a/pkg/vendors/mellanox/mellanox_test.go b/pkg/vendors/mellanox/mellanox_test.go new file mode 100644 index 000000000..e583109a1 --- /dev/null +++ b/pkg/vendors/mellanox/mellanox_test.go @@ -0,0 +1,978 @@ +package mlxutils + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "go.uber.org/mock/gomock" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + mock_host "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/mock" + mock_utils "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils/mock" +) + +var _ = Describe("SRIOV", func() { + var ( + m MellanoxInterface + u *mock_utils.MockCmdInterface + mockHostHelper *mock_host.MockHostManagerInterface + testCtrl *gomock.Controller + + testError = fmt.Errorf("test") + ) + BeforeEach(func() { + testCtrl = gomock.NewController(GinkgoT()) + u = mock_utils.NewMockCmdInterface(testCtrl) + mockHostHelper = mock_host.NewMockHostManagerInterface(testCtrl) + m = New(u, mockHostHelper) + }) + + AfterEach(func() { + testCtrl.Finish() + }) + + Context("MstConfigReadData", func() { + It("it should error if not able to run the command", func() { + u.EXPECT().RunCommand("mstconfig", "-e", "-d", "0000:d8:00.3", "q").Return("", "-E- Failed to open the device", testError) + stdOut, stdErr, err := m.MstConfigReadData("0000:d8:00.3") + Expect(err).To(HaveOccurred()) + Expect(stdErr).To(Equal("-E- Failed to open the device")) + Expect(stdOut).To(BeEmpty()) + }) + + It("should return mstconfig output", func() { + u.EXPECT().RunCommand("mstconfig", "-e", "-d", "0000:d8:00.0", "q").Return( + getMstconfigOutput(0, 0, "True", "True", "True", true, false, false), + "", nil) + stdOut, stdErr, err := m.MstConfigReadData("0000:d8:00.0") + Expect(err).ToNot(HaveOccurred()) + Expect(stdErr).To(BeEmpty()) + Expect(stdOut).ToNot(BeEmpty()) + }) + }) + + Context("MlxResetFW", func() { + var ( + mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt + ) + + BeforeEach(func() { + mellanoxNicsStatus = map[string]map[string]sriovnetworkv1.InterfaceExt{} + }) + + It("should reset VFs and firmware for single-port NIC", func() { + mellanoxNicsStatus["0000:d8:00."] = map[string]sriovnetworkv1.InterfaceExt{ + "0000:d8:00.0": {PciAddress: "0000:d8:00.0", Vendor: "15b3"}, + } + + // Expect VF reset for the primary port + mockHostHelper.EXPECT().SetSriovNumVfs("0000:d8:00.0", 0).Return(nil) + + // Expect firmware reset + u.EXPECT().RunCommand("mstfwreset", "-d", "0000:d8:00.0", "--skip_driver", "-l", "3", "-y", "reset").Return("", "", nil) + + err := m.MlxResetFW([]string{"0000:d8:00.0"}, mellanoxNicsStatus) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should reset VFs on both ports and firmware for dual-port NIC", func() { + mellanoxNicsStatus["0000:d8:00."] = map[string]sriovnetworkv1.InterfaceExt{ + "0000:d8:00.0": {PciAddress: "0000:d8:00.0", Vendor: "15b3"}, + "0000:d8:00.1": {PciAddress: "0000:d8:00.1", Vendor: "15b3"}, + } + + // Expect VF reset for both ports + mockHostHelper.EXPECT().SetSriovNumVfs("0000:d8:00.0", 0).Return(nil) + mockHostHelper.EXPECT().SetSriovNumVfs("0000:d8:00.1", 0).Return(nil) + + // Expect firmware reset only for the specified port + u.EXPECT().RunCommand("mstfwreset", "-d", "0000:d8:00.0", "--skip_driver", "-l", "3", "-y", "reset").Return("", "", nil) + + err := m.MlxResetFW([]string{"0000:d8:00.0"}, mellanoxNicsStatus) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should return error if VF reset fails on primary port", func() { + mellanoxNicsStatus["0000:d8:00."] = map[string]sriovnetworkv1.InterfaceExt{ + "0000:d8:00.0": {PciAddress: "0000:d8:00.0", Vendor: "15b3"}, + } + + // VF reset fails on primary port + mockHostHelper.EXPECT().SetSriovNumVfs("0000:d8:00.0", 0).Return(fmt.Errorf("failed to reset VFs")) + + err := m.MlxResetFW([]string{"0000:d8:00.0"}, mellanoxNicsStatus) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to reset VFs")) + }) + + It("should return error if VF reset fails on secondary port of dual-port NIC", func() { + mellanoxNicsStatus["0000:d8:00."] = map[string]sriovnetworkv1.InterfaceExt{ + "0000:d8:00.0": {PciAddress: "0000:d8:00.0", Vendor: "15b3"}, + "0000:d8:00.1": {PciAddress: "0000:d8:00.1", Vendor: "15b3"}, + } + + // Primary port succeeds, secondary port fails + mockHostHelper.EXPECT().SetSriovNumVfs("0000:d8:00.0", 0).Return(nil) + mockHostHelper.EXPECT().SetSriovNumVfs("0000:d8:00.1", 0).Return(fmt.Errorf("failed to reset VFs on secondary port")) + + err := m.MlxResetFW([]string{"0000:d8:00.0"}, mellanoxNicsStatus) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to reset VFs on secondary port")) + }) + + It("should return error if firmware reset fails", func() { + mellanoxNicsStatus["0000:d8:00."] = map[string]sriovnetworkv1.InterfaceExt{ + "0000:d8:00.0": {PciAddress: "0000:d8:00.0", Vendor: "15b3"}, + } + + // VF reset succeeds but firmware reset fails + mockHostHelper.EXPECT().SetSriovNumVfs("0000:d8:00.0", 0).Return(nil) + u.EXPECT().RunCommand("mstfwreset", "-d", "0000:d8:00.0", "--skip_driver", "-l", "3", "-y", "reset").Return("", "-E- Failed to open the device", testError) + + err := m.MlxResetFW([]string{"0000:d8:00.0"}, mellanoxNicsStatus) + Expect(err).To(HaveOccurred()) + }) + + It("should handle multiple single-port NICs", func() { + mellanoxNicsStatus["0000:d8:00."] = map[string]sriovnetworkv1.InterfaceExt{ + "0000:d8:00.0": {PciAddress: "0000:d8:00.0", Vendor: "15b3"}, + } + mellanoxNicsStatus["0000:d9:00."] = map[string]sriovnetworkv1.InterfaceExt{ + "0000:d9:00.0": {PciAddress: "0000:d9:00.0", Vendor: "15b3"}, + } + + // VF reset for both NICs + mockHostHelper.EXPECT().SetSriovNumVfs("0000:d8:00.0", 0).Return(nil) + mockHostHelper.EXPECT().SetSriovNumVfs("0000:d9:00.0", 0).Return(nil) + + // Firmware reset for both NICs + u.EXPECT().RunCommand("mstfwreset", "-d", "0000:d8:00.0", "--skip_driver", "-l", "3", "-y", "reset").Return("", "", nil) + u.EXPECT().RunCommand("mstfwreset", "-d", "0000:d9:00.0", "--skip_driver", "-l", "3", "-y", "reset").Return("", "", nil) + + err := m.MlxResetFW([]string{"0000:d8:00.0", "0000:d9:00.0"}, mellanoxNicsStatus) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should handle multiple dual-port NICs", func() { + mellanoxNicsStatus["0000:d8:00."] = map[string]sriovnetworkv1.InterfaceExt{ + "0000:d8:00.0": {PciAddress: "0000:d8:00.0", Vendor: "15b3"}, + "0000:d8:00.1": {PciAddress: "0000:d8:00.1", Vendor: "15b3"}, + } + mellanoxNicsStatus["0000:d9:00."] = map[string]sriovnetworkv1.InterfaceExt{ + "0000:d9:00.0": {PciAddress: "0000:d9:00.0", Vendor: "15b3"}, + "0000:d9:00.1": {PciAddress: "0000:d9:00.1", Vendor: "15b3"}, + } + + // VF reset for all ports + mockHostHelper.EXPECT().SetSriovNumVfs("0000:d8:00.0", 0).Return(nil) + mockHostHelper.EXPECT().SetSriovNumVfs("0000:d8:00.1", 0).Return(nil) + mockHostHelper.EXPECT().SetSriovNumVfs("0000:d9:00.0", 0).Return(nil) + mockHostHelper.EXPECT().SetSriovNumVfs("0000:d9:00.1", 0).Return(nil) + + // Firmware reset for specified ports + u.EXPECT().RunCommand("mstfwreset", "-d", "0000:d8:00.0", "--skip_driver", "-l", "3", "-y", "reset").Return("", "", nil) + u.EXPECT().RunCommand("mstfwreset", "-d", "0000:d9:00.0", "--skip_driver", "-l", "3", "-y", "reset").Return("", "", nil) + + err := m.MlxResetFW([]string{"0000:d8:00.0", "0000:d9:00.0"}, mellanoxNicsStatus) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should handle mixed single and dual-port NICs", func() { + mellanoxNicsStatus["0000:d8:00."] = map[string]sriovnetworkv1.InterfaceExt{ + "0000:d8:00.0": {PciAddress: "0000:d8:00.0", Vendor: "15b3"}, + "0000:d8:00.1": {PciAddress: "0000:d8:00.1", Vendor: "15b3"}, + } + mellanoxNicsStatus["0000:d9:00."] = map[string]sriovnetworkv1.InterfaceExt{ + "0000:d9:00.0": {PciAddress: "0000:d9:00.0", Vendor: "15b3"}, + } + + // VF reset for dual-port NIC (both ports) and single-port NIC + mockHostHelper.EXPECT().SetSriovNumVfs("0000:d8:00.0", 0).Return(nil) + mockHostHelper.EXPECT().SetSriovNumVfs("0000:d8:00.1", 0).Return(nil) + mockHostHelper.EXPECT().SetSriovNumVfs("0000:d9:00.0", 0).Return(nil) + + // Firmware reset for specified ports + u.EXPECT().RunCommand("mstfwreset", "-d", "0000:d8:00.0", "--skip_driver", "-l", "3", "-y", "reset").Return("", "", nil) + u.EXPECT().RunCommand("mstfwreset", "-d", "0000:d9:00.0", "--skip_driver", "-l", "3", "-y", "reset").Return("", "", nil) + + err := m.MlxResetFW([]string{"0000:d8:00.0", "0000:d9:00.0"}, mellanoxNicsStatus) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should aggregate errors from multiple firmware reset failures", func() { + mellanoxNicsStatus["0000:d8:00."] = map[string]sriovnetworkv1.InterfaceExt{ + "0000:d8:00.0": {PciAddress: "0000:d8:00.0", Vendor: "15b3"}, + } + mellanoxNicsStatus["0000:d9:00."] = map[string]sriovnetworkv1.InterfaceExt{ + "0000:d9:00.0": {PciAddress: "0000:d9:00.0", Vendor: "15b3"}, + } + + // VF reset succeeds for both + mockHostHelper.EXPECT().SetSriovNumVfs("0000:d8:00.0", 0).Return(nil) + mockHostHelper.EXPECT().SetSriovNumVfs("0000:d9:00.0", 0).Return(nil) + + // Both firmware resets fail + u.EXPECT().RunCommand("mstfwreset", "-d", "0000:d8:00.0", "--skip_driver", "-l", "3", "-y", "reset").Return("", "-E- Failed to open the device", testError) + u.EXPECT().RunCommand("mstfwreset", "-d", "0000:d9:00.0", "--skip_driver", "-l", "3", "-y", "reset").Return("", "-E- Failed to open the device", testError) + + err := m.MlxResetFW([]string{"0000:d8:00.0", "0000:d9:00.0"}, mellanoxNicsStatus) + Expect(err).To(HaveOccurred()) + }) + }) + + Context("GetMellanoxBlueFieldMode", func() { + It("should return error if not able to run mstconfig", func() { + u.EXPECT().RunCommand("mstconfig", "-e", "-d", "0000:d8:00.0", "q").Return("", "-E- Failed to open the device", testError) + mode, err := m.GetMellanoxBlueFieldMode("0000:d8:00.0") + Expect(err).To(HaveOccurred()) + Expect(int(mode)).To(Equal(-1)) + }) + + It("should return -1 if the card is not a BF", func() { + u.EXPECT().RunCommand("mstconfig", "-e", "-d", "0000:d8:00.0", "q").Return( + getMstconfigOutput(0, 0, "True", "True", "True", true, false, false), + "", nil) + mode, err := m.GetMellanoxBlueFieldMode("0000:d8:00.0") + Expect(err).To(HaveOccurred()) + Expect(int(mode)).To(Equal(-1)) + }) + + It("should return that the card is on dpu mode", func() { + u.EXPECT().RunCommand("mstconfig", "-e", "-d", "0000:d8:00.0", "q").Return( + getBFMstconfigOutput(true, false), + "", nil) + mode, err := m.GetMellanoxBlueFieldMode("0000:d8:00.0") + Expect(err).ToNot(HaveOccurred()) + Expect(int(mode)).To(Equal(0)) + }) + + It("should return that the card is on connectX mode", func() { + u.EXPECT().RunCommand("mstconfig", "-e", "-d", "0000:d8:00.0", "q").Return( + getBFMstconfigOutput(false, false), + "", nil) + mode, err := m.GetMellanoxBlueFieldMode("0000:d8:00.0") + Expect(err).ToNot(HaveOccurred()) + Expect(int(mode)).To(Equal(1)) + }) + + It("should return unknow if the combination out the output is not expected", func() { + u.EXPECT().RunCommand("mstconfig", "-e", "-d", "0000:d8:00.0", "q").Return( + getBFMstconfigOutput(false, true), + "", nil) + mode, err := m.GetMellanoxBlueFieldMode("0000:d8:00.0") + Expect(err).To(HaveOccurred()) + Expect(int(mode)).To(Equal(-1)) + }) + }) + + Context("MlxConfigFW", func() { + It("should return error if the card is on DPU mode", func() { + u.EXPECT().RunCommand("mstconfig", "-e", "-d", "0000:d8:00.0", "q").Return( + getBFMstconfigOutput(true, false), + "", nil) + + err := m.MlxConfigFW(map[string]MlxNic{"0000:d8:00.0": {}}) + Expect(err).To(HaveOccurred()) + }) + + It("should not run mstconfig if no configuration is needed", func() { + u.EXPECT().RunCommand("mstconfig", "-e", "-d", "0000:d8:00.0", "q").Return( + getBFMstconfigOutput(false, false), + "", nil) + err := m.MlxConfigFW(map[string]MlxNic{"0000:d8:00.0": {EnableSriov: false, TotalVfs: -1}}) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should enable all the args", func() { + u.EXPECT().RunCommand("mstconfig", "-e", "-d", "0000:d8:00.0", "q").Return( + getBFMstconfigOutput(false, false), + "", nil) + u.EXPECT().RunCommand("mstconfig", "-d", "0000:d8:00.0", "-y", "set", "SRIOV_EN=True", "NUM_OF_VFS=10", "LINK_TYPE_P1=ETH", "LINK_TYPE_P2=ETH").Return( + "", + "", nil) + err := m.MlxConfigFW(map[string]MlxNic{"0000:d8:00.0": {EnableSriov: true, TotalVfs: 10, LinkTypeP1: "ETH", LinkTypeP2: "ETH"}}) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should return error if args is not right", func() { + u.EXPECT().RunCommand("mstconfig", "-e", "-d", "0000:d8:00.0", "q").Return( + getBFMstconfigOutput(false, false), + "", nil) + u.EXPECT().RunCommand("mstconfig", "-d", "0000:d8:00.0", "-y", "set", "SRIOV_EN=True", "NUM_OF_VFS=10", "LINK_TYPE_P1=ETH", "LINK_TYPE_P2=test").Return( + "", + "", testError) + err := m.MlxConfigFW(map[string]MlxNic{"0000:d8:00.0": {EnableSriov: true, TotalVfs: 10, LinkTypeP1: "ETH", LinkTypeP2: "test"}}) + Expect(err).To(HaveOccurred()) + }) + }) + + Context("GetMlxNicFwData", func() { + It("should return error if not able to run mstconfig", func() { + u.EXPECT().RunCommand("mstconfig", "-e", "-d", "0000:d8:00.0", "q").Return( + "", "", testError) + current, next, err := m.GetMlxNicFwData("0000:d8:00.0") + Expect(err).To(HaveOccurred()) + Expect(current).To(BeNil()) + Expect(next).To(BeNil()) + }) + + It("should return the current and next firmware configuration", func() { + u.EXPECT().RunCommand("mstconfig", "-e", "-d", "0000:d8:00.0", "q").Return( + getMstconfigOutput(5, 10, "True", "True", "True", true, false, false), + "", nil) + current, next, err := m.GetMlxNicFwData("0000:d8:00.0") + Expect(err).ToNot(HaveOccurred()) + Expect(current.TotalVfs).To(Equal(5)) + Expect(current.EnableSriov).To(BeTrue()) + Expect(current.LinkTypeP1).To(Equal("ETH")) + Expect(current.LinkTypeP2).To(Equal("ETH")) + Expect(next.TotalVfs).To(Equal(10)) + Expect(next.EnableSriov).To(BeTrue()) + Expect(next.LinkTypeP1).To(Equal("ETH")) + Expect(next.LinkTypeP2).To(Equal("ETH")) + }) + + It("should return the current and next firmware configuration without linkType", func() { + u.EXPECT().RunCommand("mstconfig", "-e", "-d", "0000:d8:00.0", "q").Return( + getMstconfigOutput(5, 10, "True", "True", "True", false, false, false), + "", nil) + current, next, err := m.GetMlxNicFwData("0000:d8:00.0") + Expect(err).ToNot(HaveOccurred()) + Expect(current.TotalVfs).To(Equal(5)) + Expect(current.EnableSriov).To(BeTrue()) + Expect(current.LinkTypeP1).To(Equal("Preconfigured")) + Expect(current.LinkTypeP2).To(Equal("")) + Expect(next.TotalVfs).To(Equal(10)) + Expect(next.EnableSriov).To(BeTrue()) + Expect(next.LinkTypeP1).To(Equal("Preconfigured")) + Expect(next.LinkTypeP2).To(Equal("")) + }) + + It("should return the current and next firmware configuration with IB", func() { + u.EXPECT().RunCommand("mstconfig", "-e", "-d", "0000:d8:00.0", "q").Return( + getMstconfigOutput(5, 10, "True", "True", "True", false, true, false), + "", nil) + current, next, err := m.GetMlxNicFwData("0000:d8:00.0") + Expect(err).ToNot(HaveOccurred()) + Expect(current.TotalVfs).To(Equal(5)) + Expect(current.EnableSriov).To(BeTrue()) + Expect(current.LinkTypeP1).To(Equal("IB")) + Expect(current.LinkTypeP2).To(Equal("IB")) + Expect(next.TotalVfs).To(Equal(10)) + Expect(next.EnableSriov).To(BeTrue()) + Expect(next.LinkTypeP1).To(Equal("ETH")) + Expect(next.LinkTypeP2).To(Equal("ETH")) + }) + + It("should return the current and next firmware configuration with unknow", func() { + u.EXPECT().RunCommand("mstconfig", "-e", "-d", "0000:d8:00.0", "q").Return( + getMstconfigOutput(5, 10, "True", "True", "True", false, false, true), + "", nil) + current, next, err := m.GetMlxNicFwData("0000:d8:00.0") + Expect(err).ToNot(HaveOccurred()) + Expect(current.TotalVfs).To(Equal(5)) + Expect(current.EnableSriov).To(BeTrue()) + Expect(current.LinkTypeP1).To(Equal("Unknown")) + Expect(current.LinkTypeP2).To(Equal("IB")) + Expect(next.TotalVfs).To(Equal(10)) + Expect(next.EnableSriov).To(BeTrue()) + Expect(next.LinkTypeP1).To(Equal("Unknown")) + Expect(next.LinkTypeP2).To(Equal("ETH")) + }) + }) + + Context("IsDualPort", func() { + It("should return true if it's a dual port", func() { + mellanoxNicsStatus := map[string]map[string]sriovnetworkv1.InterfaceExt{"0000:d8:00.": {"0000:d8:00.0": {}, "0000:d8:00.1": {}}} + status := IsDualPort("0000:d8:00.0", mellanoxNicsStatus) + Expect(status).To(BeTrue()) + }) + + It("should return false if it's not a dual port", func() { + mellanoxNicsStatus := map[string]map[string]sriovnetworkv1.InterfaceExt{"0000:d8:00.": {"0000:d8:00.0": {}}} + status := IsDualPort("0000:d8:00.0", mellanoxNicsStatus) + Expect(status).To(BeFalse()) + }) + }) + + Context("HandleTotalVfs", func() { + It("should require to reboot the system for a single port card", func() { + fwCurrent := &MlxNic{TotalVfs: 0} + fwNext := &MlxNic{TotalVfs: 0} + ifaceSpec := sriovnetworkv1.Interface{PciAddress: "0000:d8:00.0", NumVfs: 10} + mellanoxNicsSpec := map[string]sriovnetworkv1.Interface{"0000:d8:00.0": ifaceSpec} + attrs := &MlxNic{} + totalvfs, needReboot, changeWithoutReboot := HandleTotalVfs(fwCurrent, fwNext, attrs, ifaceSpec, false, mellanoxNicsSpec) + Expect(totalvfs).To(Equal(10)) + Expect(needReboot).To(BeTrue()) + Expect(changeWithoutReboot).To(BeFalse()) + Expect(attrs.TotalVfs).To(Equal(10)) + }) + + It("should use the higher number of vf for dual port", func() { + fwCurrent := &MlxNic{TotalVfs: 0} + fwNext := &MlxNic{TotalVfs: 0} + ifaceSpec := sriovnetworkv1.Interface{PciAddress: "0000:d8:00.0", NumVfs: 10} + mellanoxNicsSpec := map[string]sriovnetworkv1.Interface{"0000:d8:00.0": ifaceSpec, "0000:d8:00.1": {NumVfs: 20}} + attrs := &MlxNic{} + totalvfs, needReboot, changeWithoutReboot := HandleTotalVfs(fwCurrent, fwNext, attrs, ifaceSpec, true, mellanoxNicsSpec) + Expect(totalvfs).To(Equal(20)) + Expect(needReboot).To(BeTrue()) + Expect(changeWithoutReboot).To(BeFalse()) + Expect(attrs.TotalVfs).To(Equal(20)) + }) + + It("should return for externally manage if the number is higher that the current configured", func() { + fwCurrent := &MlxNic{TotalVfs: 0} + fwNext := &MlxNic{TotalVfs: 0} + ifaceSpec := sriovnetworkv1.Interface{PciAddress: "0000:d8:00.0", NumVfs: 10, ExternallyManaged: true} + mellanoxNicsSpec := map[string]sriovnetworkv1.Interface{"0000:d8:00.0": ifaceSpec} + attrs := &MlxNic{} + totalvfs, needReboot, changeWithoutReboot := HandleTotalVfs(fwCurrent, fwNext, attrs, ifaceSpec, false, mellanoxNicsSpec) + Expect(totalvfs).To(Equal(10)) + Expect(needReboot).To(BeTrue()) + Expect(changeWithoutReboot).To(BeFalse()) + Expect(attrs.TotalVfs).To(Equal(10)) + }) + + It("should not need to reboot if we remove a policy and re-apply it", func() { + fwCurrent := &MlxNic{TotalVfs: 10} + fwNext := &MlxNic{TotalVfs: 0} + ifaceSpec := sriovnetworkv1.Interface{PciAddress: "0000:d8:00.0", NumVfs: 10} + mellanoxNicsSpec := map[string]sriovnetworkv1.Interface{"0000:d8:00.0": ifaceSpec} + attrs := &MlxNic{} + totalvfs, needReboot, changeWithoutReboot := HandleTotalVfs(fwCurrent, fwNext, attrs, ifaceSpec, false, mellanoxNicsSpec) + Expect(totalvfs).To(Equal(10)) + Expect(needReboot).To(BeFalse()) + Expect(changeWithoutReboot).To(BeTrue()) + Expect(attrs.TotalVfs).To(Equal(10)) + }) + }) + + Context("HandleEnableSriov", func() { + It("should disable sriov and return need to reboot", func() { + fwCurrent := &MlxNic{TotalVfs: 10, EnableSriov: true} + fwNext := &MlxNic{TotalVfs: 0} + attrs := &MlxNic{} + needReboot, changeWithoutReboot := HandleEnableSriov(0, fwCurrent, fwNext, attrs) + Expect(needReboot).To(BeTrue()) + Expect(changeWithoutReboot).To(BeFalse()) + Expect(attrs.EnableSriov).To(BeFalse()) + }) + It("should enable sriov and return need to reboot", func() { + fwCurrent := &MlxNic{TotalVfs: 0, EnableSriov: false} + fwNext := &MlxNic{TotalVfs: 0} + attrs := &MlxNic{} + needReboot, changeWithoutReboot := HandleEnableSriov(10, fwCurrent, fwNext, attrs) + Expect(needReboot).To(BeTrue()) + Expect(changeWithoutReboot).To(BeFalse()) + Expect(attrs.EnableSriov).To(BeTrue()) + }) + It("should enable sriov and return no need to reboot if fw next is enable", func() { + fwCurrent := &MlxNic{TotalVfs: 0, EnableSriov: true} + fwNext := &MlxNic{TotalVfs: 0, EnableSriov: false} + attrs := &MlxNic{} + needReboot, changeWithoutReboot := HandleEnableSriov(10, fwCurrent, fwNext, attrs) + Expect(needReboot).To(BeFalse()) + Expect(changeWithoutReboot).To(BeTrue()) + Expect(attrs.EnableSriov).To(BeTrue()) + }) + It("should not enable sriov and return false to reboot in case the totalvf is 0 and sriov is not enabled in current fw", func() { + fwCurrent := &MlxNic{TotalVfs: 0, EnableSriov: false} + fwNext := &MlxNic{TotalVfs: 0, EnableSriov: false} + attrs := &MlxNic{} + needReboot, changeWithoutReboot := HandleEnableSriov(0, fwCurrent, fwNext, attrs) + Expect(needReboot).To(BeFalse()) + Expect(changeWithoutReboot).To(BeFalse()) + Expect(attrs.EnableSriov).To(BeFalse()) + }) + }) + + Context("HandleLinkType", func() { + It("should return false if there is no need to change link type", func() { + fwData := &MlxNic{LinkTypeP1: "ETH", LinkTypeP2: "ETH"} + ifaceSpec := sriovnetworkv1.Interface{PciAddress: "0000:d8:00.0", NumVfs: 10} + mellanoxNicsSpec := map[string]sriovnetworkv1.Interface{"0000:d8:00.0": ifaceSpec, "0000:d8:00.1": {NumVfs: 20}} + mellanoxNicsExt := map[string]sriovnetworkv1.InterfaceExt{"0000:d8:00.0": {PciAddress: "0000:d8:00.0"}, "0000:d8:00.1": {NumVfs: 20}} + mellanoxNicsStatus := map[string]map[string]sriovnetworkv1.InterfaceExt{"0000:d8:00.": mellanoxNicsExt} + attrs := &MlxNic{} + + needChange, err := HandleLinkType("0000:d8:00.", fwData, attrs, mellanoxNicsSpec, mellanoxNicsStatus) + Expect(err).ToNot(HaveOccurred()) + Expect(needChange).To(BeFalse()) + }) + + It("should return true if there is a need to change link type", func() { + fwData := &MlxNic{LinkTypeP1: "ETH", LinkTypeP2: "ETH"} + ifaceSpec := sriovnetworkv1.Interface{PciAddress: "0000:d8:00.0", NumVfs: 10, LinkType: "IB"} + mellanoxNicsSpec := map[string]sriovnetworkv1.Interface{"0000:d8:00.0": ifaceSpec, "0000:d8:00.1": {NumVfs: 20}} + mellanoxNicsExt := map[string]sriovnetworkv1.InterfaceExt{"0000:d8:00.0": {PciAddress: "0000:d8:00.0", LinkType: "ETH"}, "0000:d8:00.1": {NumVfs: 20, LinkType: "ETH"}} + mellanoxNicsStatus := map[string]map[string]sriovnetworkv1.InterfaceExt{"0000:d8:00.": mellanoxNicsExt} + attrs := &MlxNic{} + + needChange, err := HandleLinkType("0000:d8:00.", fwData, attrs, mellanoxNicsSpec, mellanoxNicsStatus) + Expect(err).ToNot(HaveOccurred()) + Expect(needChange).To(BeTrue()) + Expect(attrs.LinkTypeP1).To(Equal("IB")) + }) + + It("should return false and error if the link type is not supported", func() { + fwData := &MlxNic{LinkTypeP1: "ETH", LinkTypeP2: "ETH"} + ifaceSpec := sriovnetworkv1.Interface{PciAddress: "0000:d8:00.0", NumVfs: 10, LinkType: "TEST"} + mellanoxNicsSpec := map[string]sriovnetworkv1.Interface{"0000:d8:00.0": ifaceSpec, "0000:d8:00.1": {NumVfs: 20}} + mellanoxNicsExt := map[string]sriovnetworkv1.InterfaceExt{"0000:d8:00.0": {PciAddress: "0000:d8:00.0", LinkType: "ETH"}, "0000:d8:00.1": {NumVfs: 20, LinkType: "ETH"}} + mellanoxNicsStatus := map[string]map[string]sriovnetworkv1.InterfaceExt{"0000:d8:00.": mellanoxNicsExt} + attrs := &MlxNic{} + + _, err := HandleLinkType("0000:d8:00.", fwData, attrs, mellanoxNicsSpec, mellanoxNicsStatus) + Expect(err).To(HaveOccurred()) + }) + + It("should return false and error if the link type is Unknown", func() { + fwData := &MlxNic{LinkTypeP1: "Unknown", LinkTypeP2: "ETH"} + ifaceSpec := sriovnetworkv1.Interface{PciAddress: "0000:d8:00.0", NumVfs: 10, LinkType: "IB"} + mellanoxNicsSpec := map[string]sriovnetworkv1.Interface{"0000:d8:00.0": ifaceSpec, "0000:d8:00.1": {NumVfs: 20}} + mellanoxNicsExt := map[string]sriovnetworkv1.InterfaceExt{"0000:d8:00.0": {PciAddress: "0000:d8:00.0", LinkType: "ETH"}, "0000:d8:00.1": {NumVfs: 20, LinkType: "ETH"}} + mellanoxNicsStatus := map[string]map[string]sriovnetworkv1.InterfaceExt{"0000:d8:00.": mellanoxNicsExt} + attrs := &MlxNic{} + + _, err := HandleLinkType("0000:d8:00.", fwData, attrs, mellanoxNicsSpec, mellanoxNicsStatus) + Expect(err).To(HaveOccurred()) + }) + + It("should return false and error if the link type is Preconfigured", func() { + fwData := &MlxNic{LinkTypeP1: "Unknown", LinkTypeP2: "ETH"} + ifaceSpec := sriovnetworkv1.Interface{PciAddress: "0000:d8:00.0", NumVfs: 10, LinkType: "IB"} + mellanoxNicsSpec := map[string]sriovnetworkv1.Interface{"0000:d8:00.0": ifaceSpec, "0000:d8:00.1": {NumVfs: 20}} + mellanoxNicsExt := map[string]sriovnetworkv1.InterfaceExt{"0000:d8:00.0": {PciAddress: "0000:d8:00.0", LinkType: "ETH"}, "0000:d8:00.1": {NumVfs: 20, LinkType: "ETH"}} + mellanoxNicsStatus := map[string]map[string]sriovnetworkv1.InterfaceExt{"0000:d8:00.": mellanoxNicsExt} + attrs := &MlxNic{} + + _, err := HandleLinkType("0000:d8:00.", fwData, attrs, mellanoxNicsSpec, mellanoxNicsStatus) + Expect(err).To(HaveOccurred()) + }) + + It("should return true if there is a need to change link type for Port 2", func() { + fwData := &MlxNic{LinkTypeP1: "ETH", LinkTypeP2: "ETH"} + ifaceSpec1 := sriovnetworkv1.Interface{PciAddress: "0000:d8:00.0", NumVfs: 10, LinkType: "ETH"} + ifaceSpec2 := sriovnetworkv1.Interface{PciAddress: "0000:d8:00.0", NumVfs: 10, LinkType: "IB"} + mellanoxNicsSpec := map[string]sriovnetworkv1.Interface{"0000:d8:00.0": ifaceSpec1, "0000:d8:00.1": ifaceSpec2} + mellanoxNicsExt := map[string]sriovnetworkv1.InterfaceExt{"0000:d8:00.0": {PciAddress: "0000:d8:00.0", LinkType: "ETH"}, "0000:d8:00.1": {NumVfs: 20, LinkType: "ETH"}} + mellanoxNicsStatus := map[string]map[string]sriovnetworkv1.InterfaceExt{"0000:d8:00.": mellanoxNicsExt} + attrs := &MlxNic{} + + needChange, err := HandleLinkType("0000:d8:00.", fwData, attrs, mellanoxNicsSpec, mellanoxNicsStatus) + Expect(err).ToNot(HaveOccurred()) + Expect(needChange).To(BeTrue()) + Expect(attrs.LinkTypeP2).To(Equal("IB")) + }) + + It("should return true if there is a need to change both links", func() { + fwData := &MlxNic{LinkTypeP1: "ETH", LinkTypeP2: "ETH"} + ifaceSpec1 := sriovnetworkv1.Interface{PciAddress: "0000:d8:00.0", NumVfs: 10, LinkType: "IB"} + ifaceSpec2 := sriovnetworkv1.Interface{PciAddress: "0000:d8:00.0", NumVfs: 10, LinkType: "IB"} + mellanoxNicsSpec := map[string]sriovnetworkv1.Interface{"0000:d8:00.0": ifaceSpec1, "0000:d8:00.1": ifaceSpec2} + mellanoxNicsExt := map[string]sriovnetworkv1.InterfaceExt{"0000:d8:00.0": {PciAddress: "0000:d8:00.0", LinkType: "ETH"}, "0000:d8:00.1": {NumVfs: 20, LinkType: "ETH"}} + mellanoxNicsStatus := map[string]map[string]sriovnetworkv1.InterfaceExt{"0000:d8:00.": mellanoxNicsExt} + attrs := &MlxNic{} + + needChange, err := HandleLinkType("0000:d8:00.", fwData, attrs, mellanoxNicsSpec, mellanoxNicsStatus) + Expect(err).ToNot(HaveOccurred()) + Expect(needChange).To(BeTrue()) + Expect(attrs.LinkTypeP1).To(Equal("IB")) + Expect(attrs.LinkTypeP2).To(Equal("IB")) + }) + }) + + Context("getOtherPortPCIAddress", func() { + It("should return port 1 when given port 0", func() { + result := getOtherPortPCIAddress("0000:d8:00.0") + Expect(result).To(Equal("0000:d8:00.1")) + }) + + It("should return port 0 when given port 1", func() { + result := getOtherPortPCIAddress("0000:d8:00.1") + Expect(result).To(Equal("0000:d8:00.0")) + }) + + It("should handle different PCI prefixes", func() { + result := getOtherPortPCIAddress("0000:d9:00.0") + Expect(result).To(Equal("0000:d9:00.1")) + }) + }) +}) + +func getMstconfigOutput(numOfVfsCurrent, numofVfsNextBoot int, sriovEnableDefault, sriovEnableCurrent, sriovEnableNextBoot string, withETHLinkType, withIBLinkType, withUnknowLinkType bool) string { + mstconfigOutput := ` +Device #1: +---------- + +Device type: ConnectX4LX +Name: 0R887V +Description: MCX422A-ACAA ConnectX-4 Lx EN Dual Port SFP28; 25GbE for Dell rack NDC +Device: 19:00.0 + +Configurations: Default Current Next Boot +%s + MEMIC_BAR_SIZE 0 0 0 + MEMIC_SIZE_LIMIT _256KB(1) _256KB(1) _256KB(1) + FLEX_PARSER_PROFILE_ENABLE 0 0 0 + FLEX_IPV4_OVER_VXLAN_PORT 0 0 0 + ROCE_NEXT_PROTOCOL 254 254 254 + PF_NUM_OF_VF_VALID False(0) False(0) False(0) + NON_PREFETCHABLE_PF_BAR False(0) False(0) False(0) + VF_VPD_ENABLE False(0) False(0) False(0) + STRICT_VF_MSIX_NUM False(0) False(0) False(0) + VF_NODNIC_ENABLE False(0) False(0) False(0) + NUM_PF_MSIX_VALID True(1) True(1) True(1) +* NUM_OF_VFS 8 %d %d + NUM_OF_PF 2 2 2 +* SRIOV_EN %s(0) %s(1) %s(1) + PF_LOG_BAR_SIZE 5 5 5 + VF_LOG_BAR_SIZE 0 0 0 + NUM_PF_MSIX 63 63 63 + NUM_VF_MSIX 11 11 11 + INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0) AUTOMATIC(0) AUTOMATIC(0) + PCIE_CREDIT_TOKEN_TIMEOUT 0 0 0 + ACCURATE_TX_SCHEDULER False(0) False(0) False(0) + PARTIAL_RESET_EN False(0) False(0) False(0) + SW_RECOVERY_ON_ERRORS False(0) False(0) False(0) + RESET_WITH_HOST_ON_ERRORS False(0) False(0) False(0) + PCI_BUS0_RESTRICT_SPEED PCI_GEN_1(0) PCI_GEN_1(0) PCI_GEN_1(0) + PCI_BUS0_RESTRICT_ASPM False(0) False(0) False(0) + PCI_BUS0_RESTRICT_WIDTH PCI_X1(0) PCI_X1(0) PCI_X1(0) + PCI_BUS0_RESTRICT False(0) False(0) False(0) + PCI_DOWNSTREAM_PORT_OWNER Array[0..15] Array[0..15] Array[0..15] + CQE_COMPRESSION BALANCED(0) BALANCED(0) BALANCED(0) + IP_OVER_VXLAN_EN False(0) False(0) False(0) + MKEY_BY_NAME False(0) False(0) False(0) + UCTX_EN True(1) True(1) True(1) + PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) + TUNNEL_ECN_COPY_DISABLE False(0) False(0) False(0) + LRO_LOG_TIMEOUT0 6 6 6 + LRO_LOG_TIMEOUT1 7 7 7 + LRO_LOG_TIMEOUT2 8 8 8 + LRO_LOG_TIMEOUT3 13 13 13 + ICM_CACHE_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) + TX_SCHEDULER_BURST 0 0 0 + LOG_DCR_HASH_TABLE_SIZE 14 14 14 + MAX_PACKET_LIFETIME 0 0 0 + DCR_LIFO_SIZE 16384 16384 16384 + ROCE_CC_PRIO_MASK_P1 255 255 255 + ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) + ROCE_CC_PRIO_MASK_P2 255 255 255 + ROCE_CC_CNP_MODERATION_P2 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) + CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1) True(1) True(1) + CLAMP_TGT_RATE_P1 False(0) False(0) False(0) + RPG_TIME_RESET_P1 300 300 300 + RPG_BYTE_RESET_P1 32767 32767 32767 + RPG_THRESHOLD_P1 1 1 1 + RPG_MAX_RATE_P1 0 0 0 + RPG_AI_RATE_P1 5 5 5 + RPG_HAI_RATE_P1 50 50 50 + RPG_GD_P1 11 11 11 + RPG_MIN_DEC_FAC_P1 50 50 50 + RPG_MIN_RATE_P1 1 1 1 + RATE_TO_SET_ON_FIRST_CNP_P1 0 0 0 + DCE_TCP_G_P1 1019 1019 1019 + DCE_TCP_RTT_P1 1 1 1 + RATE_REDUCE_MONITOR_PERIOD_P1 4 4 4 + INITIAL_ALPHA_VALUE_P1 1023 1023 1023 + MIN_TIME_BETWEEN_CNPS_P1 4 4 4 + CNP_802P_PRIO_P1 6 6 6 + CNP_DSCP_P1 48 48 48 + CLAMP_TGT_RATE_AFTER_TIME_INC_P2 True(1) True(1) True(1) + CLAMP_TGT_RATE_P2 False(0) False(0) False(0) + RPG_TIME_RESET_P2 300 300 300 + RPG_BYTE_RESET_P2 32767 32767 32767 + RPG_THRESHOLD_P2 1 1 1 + RPG_MAX_RATE_P2 0 0 0 + RPG_AI_RATE_P2 5 5 5 + RPG_HAI_RATE_P2 50 50 50 + RPG_GD_P2 11 11 11 + RPG_MIN_DEC_FAC_P2 50 50 50 + RPG_MIN_RATE_P2 1 1 1 + RATE_TO_SET_ON_FIRST_CNP_P2 0 0 0 + DCE_TCP_G_P2 1019 1019 1019 + DCE_TCP_RTT_P2 1 1 1 + RATE_REDUCE_MONITOR_PERIOD_P2 4 4 4 + INITIAL_ALPHA_VALUE_P2 1023 1023 1023 + MIN_TIME_BETWEEN_CNPS_P2 4 4 4 + CNP_802P_PRIO_P2 6 6 6 + CNP_DSCP_P2 48 48 48 + LLDP_NB_DCBX_P1 False(0) False(0) False(0) + LLDP_NB_RX_MODE_P1 ALL(2) ALL(2) ALL(2) + LLDP_NB_TX_MODE_P1 ALL(2) ALL(2) ALL(2) + LLDP_NB_DCBX_P2 False(0) False(0) False(0) + LLDP_NB_RX_MODE_P2 ALL(2) ALL(2) ALL(2) + LLDP_NB_TX_MODE_P2 ALL(2) ALL(2) ALL(2) + ROCE_RTT_RESP_DSCP_P1 0 0 0 + ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) + ROCE_RTT_RESP_DSCP_P2 0 0 0 + ROCE_RTT_RESP_DSCP_MODE_P2 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) + DCBX_IEEE_P1 True(1) True(1) True(1) + DCBX_CEE_P1 True(1) True(1) True(1) + DCBX_WILLING_P1 True(1) True(1) True(1) + DCBX_IEEE_P2 True(1) True(1) True(1) + DCBX_CEE_P2 True(1) True(1) True(1) + DCBX_WILLING_P2 True(1) True(1) True(1) + KEEP_ETH_LINK_UP_P1 True(1) True(1) True(1) + KEEP_IB_LINK_UP_P1 False(0) False(0) False(0) + KEEP_LINK_UP_ON_BOOT_P1 False(0) False(0) False(0) + KEEP_LINK_UP_ON_STANDBY_P1 False(0) False(0) False(0) + DO_NOT_CLEAR_PORT_STATS_P1 False(0) False(0) False(0) + AUTO_POWER_SAVE_LINK_DOWN_P1 False(0) False(0) False(0) + KEEP_ETH_LINK_UP_P2 True(1) True(1) True(1) + KEEP_IB_LINK_UP_P2 False(0) False(0) False(0) + KEEP_LINK_UP_ON_BOOT_P2 False(0) False(0) False(0) + KEEP_LINK_UP_ON_STANDBY_P2 False(0) False(0) False(0) + DO_NOT_CLEAR_PORT_STATS_P2 False(0) False(0) False(0) + AUTO_POWER_SAVE_LINK_DOWN_P2 False(0) False(0) False(0) + NUM_OF_VL_P1 _4_VLs(3) _4_VLs(3) _4_VLs(3) + NUM_OF_TC_P1 _8_TCs(0) _8_TCs(0) _8_TCs(0) + NUM_OF_PFC_P1 8 8 8 + VL15_BUFFER_SIZE_P1 0 0 0 + NUM_OF_VL_P2 _4_VLs(3) _4_VLs(3) _4_VLs(3) + NUM_OF_TC_P2 _8_TCs(0) _8_TCs(0) _8_TCs(0) + NUM_OF_PFC_P2 8 8 8 + VL15_BUFFER_SIZE_P2 0 0 0 + DUP_MAC_ACTION_P1 LAST_CFG(0) LAST_CFG(0) LAST_CFG(0) + SRIOV_IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) + IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) + DUP_MAC_ACTION_P2 LAST_CFG(0) LAST_CFG(0) LAST_CFG(0) + SRIOV_IB_ROUTING_MODE_P2 LID(1) LID(1) LID(1) + IB_ROUTING_MODE_P2 LID(1) LID(1) LID(1) + PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) + PHY_FEC_OVERRIDE_P2 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) + WOL_MAGIC_EN False(0) False(0) False(0) + PF_SD_GROUP 0 0 0 +* ROCE_CONTROL ROCE_ENABLE(2) DEVICE_DEFAULT(0) ROCE_ENABLE(2) + PCI_WR_ORDERING per_mkey(0) per_mkey(0) per_mkey(0) + MULTI_PORT_VHCA_EN False(0) False(0) False(0) + PORT_OWNER True(1) True(1) True(1) + ALLOW_RD_COUNTERS True(1) True(1) True(1) + RENEG_ON_CHANGE True(1) True(1) True(1) + TRACER_ENABLE True(1) True(1) True(1) + BOOT_UNDI_NETWORK_WAIT 0 0 0 + UEFI_HII_EN True(1) True(1) True(1) + BOOT_DBG_LOG False(0) False(0) False(0) + UEFI_LOGS DISABLED(0) DISABLED(0) DISABLED(0) + BOOT_VLAN 1 1 1 +* LEGACY_BOOT_PROTOCOL PXE(1) PXE(1) NONE(0) + BOOT_INTERRUPT_DIS False(0) False(0) False(0) + BOOT_LACP_DIS True(1) True(1) True(1) + BOOT_VLAN_EN False(0) False(0) False(0) + BOOT_PKEY 0 0 0 + DYNAMIC_VF_MSIX_TABLE False(0) False(0) False(0) + EXP_ROM_UEFI_x86_ENABLE True(1) True(1) True(1) + EXP_ROM_PXE_ENABLE True(1) True(1) True(1) + ADVANCED_PCI_SETTINGS False(0) False(0) False(0) + SAFE_MODE_THRESHOLD 10 10 10 + SAFE_MODE_ENABLE True(1) True(1) True(1) +The '*' shows parameters with next value different from default/current value.` + + if withETHLinkType { + linkType := ` LINK_TYPE_P1 ETH ETH ETH + LINK_TYPE_P2 ETH ETH ETH` + return fmt.Sprintf(mstconfigOutput, linkType, numOfVfsCurrent, numofVfsNextBoot, sriovEnableDefault, sriovEnableCurrent, sriovEnableNextBoot) + } + + if withIBLinkType { + linkType := ` LINK_TYPE_P1 IB IB ETH + LINK_TYPE_P2 IB IB ETH` + return fmt.Sprintf(mstconfigOutput, linkType, numOfVfsCurrent, numofVfsNextBoot, sriovEnableDefault, sriovEnableCurrent, sriovEnableNextBoot) + } + + if withUnknowLinkType { + linkType := ` LINK_TYPE_P1 TEST TEST TEST + LINK_TYPE_P2 IB IB ETH` + return fmt.Sprintf(mstconfigOutput, linkType, numOfVfsCurrent, numofVfsNextBoot, sriovEnableDefault, sriovEnableCurrent, sriovEnableNextBoot) + } + + return fmt.Sprintf(mstconfigOutput, "", numOfVfsCurrent, numofVfsNextBoot, sriovEnableDefault, sriovEnableCurrent, sriovEnableNextBoot) +} + +func getBFMstconfigOutput(DPUMode, unExpected bool) string { + mstconfigOutput := ` +Device #1: +---------- + +Device type: ConnectX4LX +Name: 0R887V +Description: MCX422A-ACAA ConnectX-4 Lx EN Dual Port SFP28; 25GbE for Dell rack NDC +Device: 19:00.0 + +Configurations: Default Current Next Boot + INTERNAL_CPU_PAGE_SUPPLIER %s %s %s + INTERNAL_CPU_ESWITCH_MANAGER %s %s %s + INTERNAL_CPU_IB_VPORT0 %s %s %s + INTERNAL_CPU_OFFLOAD_ENGINE %s %s %s + INTERNAL_CPU_MODEL %s %s %s + MEMIC_BAR_SIZE 0 0 0 + MEMIC_SIZE_LIMIT _256KB(1) _256KB(1) _256KB(1) + FLEX_PARSER_PROFILE_ENABLE 0 0 0 + FLEX_IPV4_OVER_VXLAN_PORT 0 0 0 + ROCE_NEXT_PROTOCOL 254 254 254 + PF_NUM_OF_VF_VALID False(0) False(0) False(0) + NON_PREFETCHABLE_PF_BAR False(0) False(0) False(0) + VF_VPD_ENABLE False(0) False(0) False(0) + STRICT_VF_MSIX_NUM False(0) False(0) False(0) + VF_NODNIC_ENABLE False(0) False(0) False(0) + NUM_PF_MSIX_VALID True(1) True(1) True(1) +* NUM_OF_VFS 8 0 0 + NUM_OF_PF 2 2 2 +* SRIOV_EN True(0) True(1) True(1) + PF_LOG_BAR_SIZE 5 5 5 + VF_LOG_BAR_SIZE 0 0 0 + NUM_PF_MSIX 63 63 63 + NUM_VF_MSIX 11 11 11 + INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0) AUTOMATIC(0) AUTOMATIC(0) + PCIE_CREDIT_TOKEN_TIMEOUT 0 0 0 + ACCURATE_TX_SCHEDULER False(0) False(0) False(0) + PARTIAL_RESET_EN False(0) False(0) False(0) + SW_RECOVERY_ON_ERRORS False(0) False(0) False(0) + RESET_WITH_HOST_ON_ERRORS False(0) False(0) False(0) + PCI_BUS0_RESTRICT_SPEED PCI_GEN_1(0) PCI_GEN_1(0) PCI_GEN_1(0) + PCI_BUS0_RESTRICT_ASPM False(0) False(0) False(0) + PCI_BUS0_RESTRICT_WIDTH PCI_X1(0) PCI_X1(0) PCI_X1(0) + PCI_BUS0_RESTRICT False(0) False(0) False(0) + PCI_DOWNSTREAM_PORT_OWNER Array[0..15] Array[0..15] Array[0..15] + CQE_COMPRESSION BALANCED(0) BALANCED(0) BALANCED(0) + IP_OVER_VXLAN_EN False(0) False(0) False(0) + MKEY_BY_NAME False(0) False(0) False(0) + UCTX_EN True(1) True(1) True(1) + PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) + TUNNEL_ECN_COPY_DISABLE False(0) False(0) False(0) + LRO_LOG_TIMEOUT0 6 6 6 + LRO_LOG_TIMEOUT1 7 7 7 + LRO_LOG_TIMEOUT2 8 8 8 + LRO_LOG_TIMEOUT3 13 13 13 + ICM_CACHE_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) + TX_SCHEDULER_BURST 0 0 0 + LOG_DCR_HASH_TABLE_SIZE 14 14 14 + MAX_PACKET_LIFETIME 0 0 0 + DCR_LIFO_SIZE 16384 16384 16384 + ROCE_CC_PRIO_MASK_P1 255 255 255 + ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) + ROCE_CC_PRIO_MASK_P2 255 255 255 + ROCE_CC_CNP_MODERATION_P2 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) + CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1) True(1) True(1) + CLAMP_TGT_RATE_P1 False(0) False(0) False(0) + RPG_TIME_RESET_P1 300 300 300 + RPG_BYTE_RESET_P1 32767 32767 32767 + RPG_THRESHOLD_P1 1 1 1 + RPG_MAX_RATE_P1 0 0 0 + RPG_AI_RATE_P1 5 5 5 + RPG_HAI_RATE_P1 50 50 50 + RPG_GD_P1 11 11 11 + RPG_MIN_DEC_FAC_P1 50 50 50 + RPG_MIN_RATE_P1 1 1 1 + RATE_TO_SET_ON_FIRST_CNP_P1 0 0 0 + DCE_TCP_G_P1 1019 1019 1019 + DCE_TCP_RTT_P1 1 1 1 + RATE_REDUCE_MONITOR_PERIOD_P1 4 4 4 + INITIAL_ALPHA_VALUE_P1 1023 1023 1023 + MIN_TIME_BETWEEN_CNPS_P1 4 4 4 + CNP_802P_PRIO_P1 6 6 6 + CNP_DSCP_P1 48 48 48 + CLAMP_TGT_RATE_AFTER_TIME_INC_P2 True(1) True(1) True(1) + CLAMP_TGT_RATE_P2 False(0) False(0) False(0) + RPG_TIME_RESET_P2 300 300 300 + RPG_BYTE_RESET_P2 32767 32767 32767 + RPG_THRESHOLD_P2 1 1 1 + RPG_MAX_RATE_P2 0 0 0 + RPG_AI_RATE_P2 5 5 5 + RPG_HAI_RATE_P2 50 50 50 + RPG_GD_P2 11 11 11 + RPG_MIN_DEC_FAC_P2 50 50 50 + RPG_MIN_RATE_P2 1 1 1 + RATE_TO_SET_ON_FIRST_CNP_P2 0 0 0 + DCE_TCP_G_P2 1019 1019 1019 + DCE_TCP_RTT_P2 1 1 1 + RATE_REDUCE_MONITOR_PERIOD_P2 4 4 4 + INITIAL_ALPHA_VALUE_P2 1023 1023 1023 + MIN_TIME_BETWEEN_CNPS_P2 4 4 4 + CNP_802P_PRIO_P2 6 6 6 + CNP_DSCP_P2 48 48 48 + LLDP_NB_DCBX_P1 False(0) False(0) False(0) + LLDP_NB_RX_MODE_P1 ALL(2) ALL(2) ALL(2) + LLDP_NB_TX_MODE_P1 ALL(2) ALL(2) ALL(2) + LLDP_NB_DCBX_P2 False(0) False(0) False(0) + LLDP_NB_RX_MODE_P2 ALL(2) ALL(2) ALL(2) + LLDP_NB_TX_MODE_P2 ALL(2) ALL(2) ALL(2) + ROCE_RTT_RESP_DSCP_P1 0 0 0 + ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) + ROCE_RTT_RESP_DSCP_P2 0 0 0 + ROCE_RTT_RESP_DSCP_MODE_P2 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) + DCBX_IEEE_P1 True(1) True(1) True(1) + DCBX_CEE_P1 True(1) True(1) True(1) + DCBX_WILLING_P1 True(1) True(1) True(1) + DCBX_IEEE_P2 True(1) True(1) True(1) + DCBX_CEE_P2 True(1) True(1) True(1) + DCBX_WILLING_P2 True(1) True(1) True(1) + KEEP_ETH_LINK_UP_P1 True(1) True(1) True(1) + KEEP_IB_LINK_UP_P1 False(0) False(0) False(0) + KEEP_LINK_UP_ON_BOOT_P1 False(0) False(0) False(0) + KEEP_LINK_UP_ON_STANDBY_P1 False(0) False(0) False(0) + DO_NOT_CLEAR_PORT_STATS_P1 False(0) False(0) False(0) + AUTO_POWER_SAVE_LINK_DOWN_P1 False(0) False(0) False(0) + KEEP_ETH_LINK_UP_P2 True(1) True(1) True(1) + KEEP_IB_LINK_UP_P2 False(0) False(0) False(0) + KEEP_LINK_UP_ON_BOOT_P2 False(0) False(0) False(0) + KEEP_LINK_UP_ON_STANDBY_P2 False(0) False(0) False(0) + DO_NOT_CLEAR_PORT_STATS_P2 False(0) False(0) False(0) + AUTO_POWER_SAVE_LINK_DOWN_P2 False(0) False(0) False(0) + NUM_OF_VL_P1 _4_VLs(3) _4_VLs(3) _4_VLs(3) + NUM_OF_TC_P1 _8_TCs(0) _8_TCs(0) _8_TCs(0) + NUM_OF_PFC_P1 8 8 8 + VL15_BUFFER_SIZE_P1 0 0 0 + NUM_OF_VL_P2 _4_VLs(3) _4_VLs(3) _4_VLs(3) + NUM_OF_TC_P2 _8_TCs(0) _8_TCs(0) _8_TCs(0) + NUM_OF_PFC_P2 8 8 8 + VL15_BUFFER_SIZE_P2 0 0 0 + DUP_MAC_ACTION_P1 LAST_CFG(0) LAST_CFG(0) LAST_CFG(0) + SRIOV_IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) + IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) + DUP_MAC_ACTION_P2 LAST_CFG(0) LAST_CFG(0) LAST_CFG(0) + SRIOV_IB_ROUTING_MODE_P2 LID(1) LID(1) LID(1) + IB_ROUTING_MODE_P2 LID(1) LID(1) LID(1) + PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) + PHY_FEC_OVERRIDE_P2 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) + WOL_MAGIC_EN False(0) False(0) False(0) + PF_SD_GROUP 0 0 0 +* ROCE_CONTROL ROCE_ENABLE(2) DEVICE_DEFAULT(0) ROCE_ENABLE(2) + PCI_WR_ORDERING per_mkey(0) per_mkey(0) per_mkey(0) + MULTI_PORT_VHCA_EN False(0) False(0) False(0) + PORT_OWNER True(1) True(1) True(1) + ALLOW_RD_COUNTERS True(1) True(1) True(1) + RENEG_ON_CHANGE True(1) True(1) True(1) + TRACER_ENABLE True(1) True(1) True(1) + BOOT_UNDI_NETWORK_WAIT 0 0 0 + UEFI_HII_EN True(1) True(1) True(1) + BOOT_DBG_LOG False(0) False(0) False(0) + UEFI_LOGS DISABLED(0) DISABLED(0) DISABLED(0) + BOOT_VLAN 1 1 1 +* LEGACY_BOOT_PROTOCOL PXE(1) PXE(1) NONE(0) + BOOT_INTERRUPT_DIS False(0) False(0) False(0) + BOOT_LACP_DIS True(1) True(1) True(1) + BOOT_VLAN_EN False(0) False(0) False(0) + BOOT_PKEY 0 0 0 + DYNAMIC_VF_MSIX_TABLE False(0) False(0) False(0) + EXP_ROM_UEFI_x86_ENABLE True(1) True(1) True(1) + EXP_ROM_PXE_ENABLE True(1) True(1) True(1) + ADVANCED_PCI_SETTINGS False(0) False(0) False(0) + SAFE_MODE_THRESHOLD 10 10 10 + SAFE_MODE_ENABLE True(1) True(1) True(1) +The '*' shows parameters with next value different from default/current value.` + if DPUMode { + return fmt.Sprintf(mstconfigOutput, ecpf, ecpf, ecpf, ecpf, ecpf, ecpf, ecpf, ecpf, ecpf, enabled, enabled, enabled, embeddedCPU, embeddedCPU, embeddedCPU) + } + + if unExpected { + return fmt.Sprintf(mstconfigOutput, extHostPf, extHostPf, extHostPf, ecpf, ecpf, ecpf, ecpf, ecpf, ecpf, enabled, enabled, enabled, embeddedCPU, embeddedCPU, embeddedCPU) + } + + return fmt.Sprintf(mstconfigOutput, extHostPf, extHostPf, extHostPf, extHostPf, extHostPf, extHostPf, extHostPf, extHostPf, extHostPf, disabled, disabled, disabled, embeddedCPU, embeddedCPU, embeddedCPU) +} diff --git a/pkg/vendors/mellanox/mock/mock_mellanox.go b/pkg/vendors/mellanox/mock/mock_mellanox.go index 1a0ca3120..27af18c35 100644 --- a/pkg/vendors/mellanox/mock/mock_mellanox.go +++ b/pkg/vendors/mellanox/mock/mock_mellanox.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: mellanox.go +// +// Generated by this command: +// +// mockgen -destination mock/mock_mellanox.go -source mellanox.go +// // Package mock_mlxutils is a generated GoMock package. package mock_mlxutils @@ -7,14 +12,16 @@ package mock_mlxutils import ( reflect "reflect" - gomock "github.com/golang/mock/gomock" + v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" mlxutils "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vendors/mellanox" + gomock "go.uber.org/mock/gomock" ) // MockMellanoxInterface is a mock of MellanoxInterface interface. type MockMellanoxInterface struct { ctrl *gomock.Controller recorder *MockMellanoxInterfaceMockRecorder + isgomock struct{} } // MockMellanoxInterfaceMockRecorder is the mock recorder for MockMellanoxInterface. @@ -44,7 +51,7 @@ func (m *MockMellanoxInterface) GetMellanoxBlueFieldMode(arg0 string) (mlxutils. } // GetMellanoxBlueFieldMode indicates an expected call of GetMellanoxBlueFieldMode. -func (mr *MockMellanoxInterfaceMockRecorder) GetMellanoxBlueFieldMode(arg0 interface{}) *gomock.Call { +func (mr *MockMellanoxInterfaceMockRecorder) GetMellanoxBlueFieldMode(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMellanoxBlueFieldMode", reflect.TypeOf((*MockMellanoxInterface)(nil).GetMellanoxBlueFieldMode), arg0) } @@ -60,7 +67,7 @@ func (m *MockMellanoxInterface) GetMlxNicFwData(pciAddress string) (*mlxutils.Ml } // GetMlxNicFwData indicates an expected call of GetMlxNicFwData. -func (mr *MockMellanoxInterfaceMockRecorder) GetMlxNicFwData(pciAddress interface{}) *gomock.Call { +func (mr *MockMellanoxInterfaceMockRecorder) GetMlxNicFwData(pciAddress any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMlxNicFwData", reflect.TypeOf((*MockMellanoxInterface)(nil).GetMlxNicFwData), pciAddress) } @@ -74,23 +81,23 @@ func (m *MockMellanoxInterface) MlxConfigFW(attributesToChange map[string]mlxuti } // MlxConfigFW indicates an expected call of MlxConfigFW. -func (mr *MockMellanoxInterfaceMockRecorder) MlxConfigFW(attributesToChange interface{}) *gomock.Call { +func (mr *MockMellanoxInterfaceMockRecorder) MlxConfigFW(attributesToChange any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MlxConfigFW", reflect.TypeOf((*MockMellanoxInterface)(nil).MlxConfigFW), attributesToChange) } // MlxResetFW mocks base method. -func (m *MockMellanoxInterface) MlxResetFW(pciAddresses []string) error { +func (m *MockMellanoxInterface) MlxResetFW(pciAddresses []string, mellanoxNicsStatus map[string]map[string]v1.InterfaceExt) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MlxResetFW", pciAddresses) + ret := m.ctrl.Call(m, "MlxResetFW", pciAddresses, mellanoxNicsStatus) ret0, _ := ret[0].(error) return ret0 } // MlxResetFW indicates an expected call of MlxResetFW. -func (mr *MockMellanoxInterfaceMockRecorder) MlxResetFW(pciAddresses interface{}) *gomock.Call { +func (mr *MockMellanoxInterfaceMockRecorder) MlxResetFW(pciAddresses, mellanoxNicsStatus any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MlxResetFW", reflect.TypeOf((*MockMellanoxInterface)(nil).MlxResetFW), pciAddresses) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MlxResetFW", reflect.TypeOf((*MockMellanoxInterface)(nil).MlxResetFW), pciAddresses, mellanoxNicsStatus) } // MstConfigReadData mocks base method. @@ -104,7 +111,7 @@ func (m *MockMellanoxInterface) MstConfigReadData(arg0 string) (string, string, } // MstConfigReadData indicates an expected call of MstConfigReadData. -func (mr *MockMellanoxInterfaceMockRecorder) MstConfigReadData(arg0 interface{}) *gomock.Call { +func (mr *MockMellanoxInterfaceMockRecorder) MstConfigReadData(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MstConfigReadData", reflect.TypeOf((*MockMellanoxInterface)(nil).MstConfigReadData), arg0) } diff --git a/pkg/vendors/mellanox/suite_test.go b/pkg/vendors/mellanox/suite_test.go new file mode 100644 index 000000000..d0ced4529 --- /dev/null +++ b/pkg/vendors/mellanox/suite_test.go @@ -0,0 +1,24 @@ +package mlxutils + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "go.uber.org/zap/zapcore" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" +) + +func TestSriov(t *testing.T) { + log.SetLogger(zap.New( + zap.WriteTo(GinkgoWriter), + zap.Level(zapcore.Level(-2)), + zap.UseDevMode(true))) + snolog.InitLog() + RegisterFailHandler(Fail) + RunSpecs(t, "Package Mellanox Vendor Suite") +} diff --git a/pkg/webhook/client.go b/pkg/webhook/client.go index a38630c9f..edff52f3c 100644 --- a/pkg/webhook/client.go +++ b/pkg/webhook/client.go @@ -6,12 +6,13 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" - snclientset "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) -var snclient snclientset.Interface +var client runtimeclient.Client var kubeclient *kubernetes.Clientset func SetupInClusterClient() error { @@ -26,13 +27,22 @@ func SetupInClusterClient() error { config, err = rest.InClusterConfig() } + if err != nil { + log.Log.Error(nil, "fail to create config") + return err + } + + client, err = runtimeclient.New(config, runtimeclient.Options{Scheme: vars.Scheme}) if err != nil { log.Log.Error(nil, "fail to setup client") return err } - snclient = snclientset.NewForConfigOrDie(config) - kubeclient = kubernetes.NewForConfigOrDie(config) + kubeclient, err = kubernetes.NewForConfig(config) + if err != nil { + log.Log.Error(nil, "fail to setup kubernetes client") + return err + } return nil } diff --git a/pkg/webhook/scheme.go b/pkg/webhook/scheme.go index 795d581bc..7061e0034 100644 --- a/pkg/webhook/scheme.go +++ b/pkg/webhook/scheme.go @@ -9,13 +9,16 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) -var scheme = runtime.NewScheme() -var Codecs = serializer.NewCodecFactory(scheme) +var Codecs serializer.CodecFactory func init() { + scheme := runtime.NewScheme() addToScheme(scheme) + Codecs = serializer.NewCodecFactory(scheme) + vars.Scheme = scheme } func addToScheme(scheme *runtime.Scheme) { diff --git a/pkg/webhook/validate.go b/pkg/webhook/validate.go index 739d2fc61..42b504280 100644 --- a/pkg/webhook/validate.go +++ b/pkg/webhook/validate.go @@ -13,6 +13,7 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" @@ -62,8 +63,8 @@ func validateSriovOperatorConfigDisableDrain(cr *sriovnetworkv1.SriovOperatorCon if !cr.Spec.DisableDrain { return nil } - - previousConfig, err := snclient.SriovnetworkV1().SriovOperatorConfigs(cr.Namespace).Get(context.Background(), cr.Name, metav1.GetOptions{}) + previousConfig := &sriovnetworkv1.SriovOperatorConfig{} + err := client.Get(context.Background(), runtimeclient.ObjectKey{Name: cr.Name, Namespace: namespace}, previousConfig) if err != nil { if k8serrors.IsNotFound(err) { return nil @@ -77,7 +78,8 @@ func validateSriovOperatorConfigDisableDrain(cr *sriovnetworkv1.SriovOperatorCon } // DisableDrain has been changed `false -> true`, check if any node is updating - nodeStates, err := snclient.SriovnetworkV1().SriovNetworkNodeStates(namespace).List(context.Background(), metav1.ListOptions{}) + nodeStates := &sriovnetworkv1.SriovNetworkNodeStateList{} + err = client.List(context.Background(), nodeStates, &runtimeclient.ListOptions{Namespace: namespace}) if err != nil { return fmt.Errorf("can't validate SriovOperatorConfig[%s] DisableDrain transition to true: %q", cr.Name, err) } @@ -245,11 +247,13 @@ func dynamicValidateSriovNetworkNodePolicy(cr *sriovnetworkv1.SriovNetworkNodePo if err != nil { return false, err } - nsList, err := snclient.SriovnetworkV1().SriovNetworkNodeStates(namespace).List(context.Background(), metav1.ListOptions{}) + nsList := &sriovnetworkv1.SriovNetworkNodeStateList{} + err = client.List(context.Background(), nsList, &runtimeclient.ListOptions{Namespace: namespace}) if err != nil { return false, err } - npList, err := snclient.SriovnetworkV1().SriovNetworkNodePolicies(namespace).List(context.Background(), metav1.ListOptions{}) + npList := &sriovnetworkv1.SriovNetworkNodePolicyList{} + err = client.List(context.Background(), npList, &runtimeclient.ListOptions{Namespace: namespace}) if err != nil { return false, err } diff --git a/pkg/webhook/validate_networks.go b/pkg/webhook/validate_networks.go new file mode 100644 index 000000000..358c38a97 --- /dev/null +++ b/pkg/webhook/validate_networks.go @@ -0,0 +1,43 @@ +package webhook + +import ( + "fmt" + + v1 "k8s.io/api/admission/v1" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/controllers" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" +) + +func validateSriovNetwork(cr *sriovnetworkv1.SriovNetwork, operation v1.Operation) (bool, []string, error) { + err := validateNetworkNamespace(cr) + if err != nil { + return false, nil, err + } + return true, nil, nil +} + +func validateSriovIBNetwork(cr *sriovnetworkv1.SriovIBNetwork, operation v1.Operation) (bool, []string, error) { + err := validateNetworkNamespace(cr) + if err != nil { + return false, nil, err + } + return true, nil, nil +} + +func validateOVSNetwork(cr *sriovnetworkv1.OVSNetwork, operation v1.Operation) (bool, []string, error) { + err := validateNetworkNamespace(cr) + if err != nil { + return false, nil, err + } + return true, nil, nil +} + +func validateNetworkNamespace(cr controllers.NetworkCRInstance) error { + if cr.GetNamespace() != vars.Namespace && cr.NetworkNamespace() != "" { + return fmt.Errorf(".Spec.NetworkNamespace field can't be specified if the resource is not in the %s namespace", vars.Namespace) + } + + return nil +} diff --git a/pkg/webhook/validate_networks_test.go b/pkg/webhook/validate_networks_test.go new file mode 100644 index 000000000..ee79af594 --- /dev/null +++ b/pkg/webhook/validate_networks_test.go @@ -0,0 +1,95 @@ +package webhook + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/controllers" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" +) + +func TestValidate_NetworkNamespace(t *testing.T) { + defer func(previous string) { vars.Namespace = previous }(vars.Namespace) + vars.Namespace = "operator-namespace" + + testCases := []struct { + name string + network controllers.NetworkCRInstance + shouldFail bool + }{ + { + name: "SriovNetwork in operator namespace with empty NetworkNamespace", + network: &SriovNetwork{ObjectMeta: metav1.ObjectMeta{Namespace: "operator-namespace"}, Spec: SriovNetworkSpec{NetworkNamespace: ""}}, + shouldFail: false, + }, + { + name: "SriovNetwork in operator namespace with custom NetworkNamespace", + network: &SriovNetwork{ObjectMeta: metav1.ObjectMeta{Namespace: "operator-namespace"}, Spec: SriovNetworkSpec{NetworkNamespace: "xxx"}}, + shouldFail: false, + }, + { + name: "SriovNetwork in custom namespace with empty NetworkNamespace", + network: &SriovNetwork{ObjectMeta: metav1.ObjectMeta{Namespace: "xxx"}, Spec: SriovNetworkSpec{NetworkNamespace: ""}}, + shouldFail: false, + }, + { + name: "SriovIBNetwork in operator namespace with empty NetworkNamespace", + network: &SriovIBNetwork{ObjectMeta: metav1.ObjectMeta{Namespace: "operator-namespace"}, Spec: SriovIBNetworkSpec{NetworkNamespace: ""}}, + shouldFail: false, + }, + { + name: "SriovIBNetwork in operator namespace with custom NetworkNamespace", + network: &SriovIBNetwork{ObjectMeta: metav1.ObjectMeta{Namespace: "operator-namespace"}, Spec: SriovIBNetworkSpec{NetworkNamespace: "xxx"}}, + shouldFail: false, + }, + { + name: "SriovIBNetwork in custom namespace with empty NetworkNamespace", + network: &SriovIBNetwork{ObjectMeta: metav1.ObjectMeta{Namespace: "xxx"}, Spec: SriovIBNetworkSpec{NetworkNamespace: ""}}, + shouldFail: false, + }, + { + name: "OVSNetwork in operator namespace with empty NetworkNamespace", + network: &OVSNetwork{ObjectMeta: metav1.ObjectMeta{Namespace: "operator-namespace"}, Spec: OVSNetworkSpec{NetworkNamespace: ""}}, + shouldFail: false, + }, + { + name: "OVSNetwork in operator namespace with custom NetworkNamespace", + network: &OVSNetwork{ObjectMeta: metav1.ObjectMeta{Namespace: "operator-namespace"}, Spec: OVSNetworkSpec{NetworkNamespace: "xxx"}}, + shouldFail: false, + }, + { + name: "OVSNetwork in custom namespace with empty NetworkNamespace", + network: &OVSNetwork{ObjectMeta: metav1.ObjectMeta{Namespace: "xxx"}, Spec: OVSNetworkSpec{NetworkNamespace: ""}}, + shouldFail: false, + }, + { + name: "SriovNetwork in custom namespace with custom NetworkNamespace", + network: &SriovNetwork{ObjectMeta: metav1.ObjectMeta{Namespace: "xxx"}, Spec: SriovNetworkSpec{NetworkNamespace: "yyy"}}, + shouldFail: true, + }, + { + name: "SriovIBNetwork in custom namespace with custom NetworkNamespace", + network: &SriovIBNetwork{ObjectMeta: metav1.ObjectMeta{Namespace: "xxx"}, Spec: SriovIBNetworkSpec{NetworkNamespace: "yyy"}}, + shouldFail: true, + }, + { + name: "OVSNetwork in custom namespace with custom NetworkNamespace", + network: &OVSNetwork{ObjectMeta: metav1.ObjectMeta{Namespace: "xxx"}, Spec: OVSNetworkSpec{NetworkNamespace: "yyy"}}, + shouldFail: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := validateNetworkNamespace(tc.network) + if tc.shouldFail && err == nil { + t.Error("expected error but got none") + } + if !tc.shouldFail && err != nil { + t.Errorf("expected no error but got: %v", err) + } + }) + } +} diff --git a/pkg/webhook/validate_test.go b/pkg/webhook/validate_test.go index 4e1193b43..8a683d2aa 100644 --- a/pkg/webhook/validate_test.go +++ b/pkg/webhook/validate_test.go @@ -6,17 +6,16 @@ import ( "os" "testing" - corev1 "k8s.io/api/core/v1" - . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client/fake" . "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" - - fakesnclientset "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned/fake" ) func TestMain(m *testing.M) { @@ -168,7 +167,7 @@ func TestValidateSriovOperatorConfigWithDefaultOperatorConfig(t *testing.T) { g := NewGomegaWithT(t) config := newDefaultOperatorConfig() - snclient = fakesnclientset.NewSimpleClientset() + client = fake.NewClientBuilder().WithScheme(vars.Scheme).Build() ok, w, err := validateSriovOperatorConfig(config, "DELETE") g.Expect(err).NotTo(HaveOccurred()) @@ -202,10 +201,7 @@ func TestValidateSriovOperatorConfigDisableDrain(t *testing.T) { }, } - snclient = fakesnclientset.NewSimpleClientset( - config, - nodeState, - ) + client = fake.NewClientBuilder().WithScheme(vars.Scheme).WithObjects(config, nodeState).Build() config.Spec.DisableDrain = true ok, _, err := validateSriovOperatorConfig(config, "UPDATE") @@ -214,8 +210,8 @@ func TestValidateSriovOperatorConfigDisableDrain(t *testing.T) { // Simulate node update finished nodeState.Status.SyncStatus = "Succeeded" - snclient.SriovnetworkV1().SriovNetworkNodeStates(namespace). - Update(context.Background(), nodeState, metav1.UpdateOptions{}) + err = client.Update(context.Background(), nodeState) + g.Expect(err).ToNot(HaveOccurred()) ok, _, err = validateSriovOperatorConfig(config, "UPDATE") g.Expect(err).NotTo(HaveOccurred()) @@ -226,7 +222,7 @@ func TestValidateSriovNetworkPoolConfigWithDefault(t *testing.T) { g := NewGomegaWithT(t) config := newDefaultNetworkPoolConfig() - snclient = fakesnclientset.NewSimpleClientset() + client = fake.NewClientBuilder().WithScheme(vars.Scheme).Build() ok, _, err := validateSriovNetworkPoolConfig(config, "DELETE") g.Expect(err).ToNot(HaveOccurred()) @@ -246,7 +242,7 @@ func TestValidateSriovNetworkPoolConfigWithParallelAndHWOffload(t *testing.T) { config := newDefaultNetworkPoolConfig() config.Spec.OvsHardwareOffloadConfig.Name = "test" - snclient = fakesnclientset.NewSimpleClientset() + client = fake.NewClientBuilder().WithScheme(vars.Scheme).Build() ok, _, err := validateSriovNetworkPoolConfig(config, "UPDATE") g.Expect(err).To(HaveOccurred()) diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index d560a66d1..42df55409 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -97,6 +97,49 @@ func ValidateCustomResource(ar v1.AdmissionReview) *v1.AdmissionResponse { Reason: metav1.StatusReason(err.Error()), } } + + case "SriovNetwork": + network := sriovnetworkv1.SriovNetwork{} + + err = json.Unmarshal(raw, &network) + if err != nil { + log.Log.Error(err, "failed to unmarshal object") + return toV1AdmissionResponse(err) + } + + if reviewResponse.Allowed, reviewResponse.Warnings, err = validateSriovNetwork(&network, ar.Request.Operation); err != nil { + reviewResponse.Result = &metav1.Status{ + Reason: metav1.StatusReason(err.Error()), + } + } + case "SriovIBNetwork": + network := sriovnetworkv1.SriovIBNetwork{} + + err = json.Unmarshal(raw, &network) + if err != nil { + log.Log.Error(err, "failed to unmarshal object") + return toV1AdmissionResponse(err) + } + + if reviewResponse.Allowed, reviewResponse.Warnings, err = validateSriovIBNetwork(&network, ar.Request.Operation); err != nil { + reviewResponse.Result = &metav1.Status{ + Reason: metav1.StatusReason(err.Error()), + } + } + case "OVSNetwork": + network := sriovnetworkv1.OVSNetwork{} + + err = json.Unmarshal(raw, &network) + if err != nil { + log.Log.Error(err, "failed to unmarshal object") + return toV1AdmissionResponse(err) + } + + if reviewResponse.Allowed, reviewResponse.Warnings, err = validateOVSNetwork(&network, ar.Request.Operation); err != nil { + reviewResponse.Result = &metav1.Status{ + Reason: metav1.StatusReason(err.Error()), + } + } } return &reviewResponse diff --git a/test/conformance/tests/fixtures.go b/test/conformance/tests/fixtures.go index 33748b6e6..3cd833506 100644 --- a/test/conformance/tests/fixtures.go +++ b/test/conformance/tests/fixtures.go @@ -4,6 +4,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/clean" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/cluster" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/discovery" @@ -34,6 +35,10 @@ var _ = BeforeSuite(func() { } } + if !isFeatureFlagEnabled(consts.ResourceInjectorMatchConditionFeatureGate) { + setFeatureFlag(consts.ResourceInjectorMatchConditionFeatureGate, true) + } + err = namespaces.Create(namespaces.Test, clients) Expect(err).ToNot(HaveOccurred()) err = namespaces.Clean(operatorNamespace, namespaces.Test, clients, discovery.Enabled()) diff --git a/test/conformance/tests/test_exporter_metrics.go b/test/conformance/tests/test_exporter_metrics.go index e81f63067..f7bc82d3f 100644 --- a/test/conformance/tests/test_exporter_metrics.go +++ b/test/conformance/tests/test_exporter_metrics.go @@ -2,9 +2,12 @@ package tests import ( "context" + "encoding/json" "fmt" + "net/url" "strings" + sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/cluster" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/discovery" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/namespaces" @@ -13,21 +16,21 @@ import ( dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" + "github.com/prometheus/common/model" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) -var _ = Describe("[sriov] Metrics Exporter", Ordered, func() { +var _ = Describe("[sriov] Metrics Exporter", Ordered, ContinueOnFailure, func() { + var node string + var nic *sriovv1.InterfaceExt BeforeAll(func() { - if cluster.VirtualCluster() { - Skip("IGB driver does not support VF statistics") - } - err := namespaces.Create(namespaces.Test, clients) Expect(err).ToNot(HaveOccurred()) @@ -48,13 +51,11 @@ var _ = Describe("[sriov] Metrics Exporter", Ordered, func() { Expect(err).ToNot(HaveOccurred()) WaitForSRIOVStable() - }) - It("collects metrics regarding receiving traffic via VF", func() { sriovInfos, err := cluster.DiscoverSriov(clients, operatorNamespace) Expect(err).ToNot(HaveOccurred()) - node, nic, err := sriovInfos.FindOneSriovNodeAndDevice() + node, nic, err = sriovInfos.FindOneSriovNodeAndDevice() Expect(err).ToNot(HaveOccurred()) By("Using device " + nic.Name + " on node " + node) @@ -65,7 +66,18 @@ var _ = Describe("[sriov] Metrics Exporter", Ordered, func() { Expect(err).ToNot(HaveOccurred()) waitForNetAttachDef("test-me-network", namespaces.Test) + WaitForSRIOVStable() + + DeferCleanup(namespaces.Clean, operatorNamespace, namespaces.Test, clients, discovery.Enabled()) + }) + + It("collects metrics regarding receiving traffic via VF", func() { + if cluster.VirtualCluster() { + Skip("IGB driver does not support VF statistics") + } + pod := createTestPod(node, []string{"test-me-network"}) + DeferCleanup(namespaces.CleanPods, namespaces.Test, clients) ips, err := network.GetSriovNicIPs(pod, "net1") Expect(err).ToNot(HaveOccurred()) @@ -88,6 +100,79 @@ var _ = Describe("[sriov] Metrics Exporter", Ordered, func() { Expect(finalRxPackets).Should(BeNumerically(">", initialRxPackets)) }) + Context("When Prometheus operator is available", func() { + BeforeEach(func() { + _, err := clients.ServiceMonitors(operatorNamespace).List(context.Background(), metav1.ListOptions{}) + if k8serrors.IsNotFound(err) { + Skip("Prometheus operator not available in the cluster") + } + }) + + It("PrometheusRule should provide namespaced metrics", func() { + pod := createTestPod(node, []string{"test-me-network"}) + DeferCleanup(namespaces.CleanPods, namespaces.Test, clients) + + namespacedMetricNames := []string{ + "network:sriov_vf_rx_bytes", + "network:sriov_vf_tx_bytes", + "network:sriov_vf_rx_packets", + "network:sriov_vf_tx_packets", + "network:sriov_vf_rx_dropped", + "network:sriov_vf_tx_dropped", + "network:sriov_vf_rx_broadcast", + "network:sriov_vf_rx_multicast", + } + + Eventually(func(g Gomega) { + for _, metricName := range namespacedMetricNames { + values := runPromQLQuery(fmt.Sprintf(`%s{namespace="%s",pod="%s"}`, metricName, pod.Namespace, pod.Name)) + g.Expect(values).ToNot(BeEmpty(), "no value for metric %s", metricName) + } + }, "90s", "1s").Should(Succeed()) + }) + + It("Metrics should have the correct labels", func() { + pod := createTestPod(node, []string{"test-me-network"}) + DeferCleanup(namespaces.CleanPods, namespaces.Test, clients) + + metricsName := []string{ + "sriov_vf_rx_bytes", + "sriov_vf_tx_bytes", + "sriov_vf_rx_packets", + "sriov_vf_tx_packets", + "sriov_vf_rx_dropped", + "sriov_vf_tx_dropped", + "sriov_vf_rx_broadcast", + "sriov_vf_rx_multicast", + } + + Eventually(func(g Gomega) { + for _, metricName := range metricsName { + samples := runPromQLQuery(metricName) + g.Expect(samples).ToNot(BeEmpty(), "no value for metric %s", metricName) + g.Expect(samples[0].Metric).To(And( + HaveKey(model.LabelName("pciAddr")), + HaveKey(model.LabelName("node")), + HaveKey(model.LabelName("pf")), + HaveKey(model.LabelName("vf")), + )) + } + }, "90s", "1s").Should(Succeed()) + + // sriov_kubepoddevice has a different sets of label than statistics metrics + Eventually(func(g Gomega) { + samples := runPromQLQuery(fmt.Sprintf(`sriov_kubepoddevice{namespace="%s",pod="%s"}`, pod.Namespace, pod.Name)) + g.Expect(samples).ToNot(BeEmpty(), "no value for metric sriov_kubepoddevice") + g.Expect(samples[0].Metric).To(And( + HaveKey(model.LabelName("pciAddr")), + HaveKeyWithValue(model.LabelName("node"), model.LabelValue(pod.Spec.NodeName)), + HaveKeyWithValue(model.LabelName("dev_type"), model.LabelValue("openshift.io/metricsResource")), + HaveKeyWithValue(model.LabelName("namespace"), model.LabelValue(pod.Namespace)), + HaveKeyWithValue(model.LabelName("pod"), model.LabelValue(pod.Name)), + )) + }, "60s", "1s").Should(Succeed()) + }) + }) }) func getMetricsForNode(nodeName string) map[string]*dto.MetricFamily { @@ -185,3 +270,33 @@ func areLabelsMatching(labels []*dto.LabelPair, labelsToMatch map[string]string) return true } + +func runPromQLQuery(query string) model.Vector { + prometheusPods, err := clients.Pods("").List(context.Background(), metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/component=prometheus", + }) + ExpectWithOffset(1, err).ToNot(HaveOccurred()) + ExpectWithOffset(1, prometheusPods.Items).ToNot(HaveLen(0), "At least one Prometheus operator pod expected") + + prometheusPod := prometheusPods.Items[0] + + url := fmt.Sprintf("localhost:9090/api/v1/query?%s", (url.Values{"query": []string{query}}).Encode()) + command := []string{"curl", url} + stdout, stderr, err := pod.ExecCommand(clients, &prometheusPod, command...) + ExpectWithOffset(1, err).ToNot(HaveOccurred(), + "promQL query failed: [%s/%s] command: [%v]\nstdout: %s\nstderr: %s", prometheusPod.Namespace, prometheusPod.Name, command, stdout, stderr) + + result := struct { + Status string `json:"status"` + Data struct { + ResultType string `json:"resultType"` + Result model.Vector `json:"result"` + } `json:"data"` + }{} + + json.Unmarshal([]byte(stdout), &result) + ExpectWithOffset(1, err).ToNot(HaveOccurred()) + ExpectWithOffset(1, result.Status).To(Equal("success"), "cURL for [%s] failed: %s", url, stdout) + + return result.Data.Result +} diff --git a/test/conformance/tests/test_networkpool.go b/test/conformance/tests/test_networkpool.go new file mode 100644 index 000000000..ecfd1788b --- /dev/null +++ b/test/conformance/tests/test_networkpool.go @@ -0,0 +1,371 @@ +package tests + +import ( + "fmt" + "strconv" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "golang.org/x/net/context" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/cluster" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/discovery" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/namespaces" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/network" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/pod" +) + +var _ = Describe("[sriov] NetworkPool", Ordered, func() { + var testNode string + var interfaces []*sriovv1.InterfaceExt + var resourceName = "testrdma" + + BeforeAll(func() { + WaitForSRIOVStable() + + err := namespaces.Create(namespaces.Test, clients) + Expect(err).ToNot(HaveOccurred()) + err = namespaces.Clean(operatorNamespace, namespaces.Test, clients, discovery.Enabled()) + Expect(err).ToNot(HaveOccurred()) + + sriovInfos, err := cluster.DiscoverSriov(clients, operatorNamespace) + Expect(err).ToNot(HaveOccurred()) + Expect(len(sriovInfos.Nodes)).ToNot(BeZero()) + + testNode, interfaces, err = sriovInfos.FindSriovDevicesAndNode() + Expect(err).ToNot(HaveOccurred()) + + By(fmt.Sprintf("Testing on node %s, %d devices found", testNode, len(interfaces))) + WaitForSRIOVStable() + }) + + AfterEach(func() { + err := namespaces.Clean(operatorNamespace, namespaces.Test, clients, discovery.Enabled()) + Expect(err).ToNot(HaveOccurred()) + + err = clients.DeleteAllOf(context.Background(), &sriovv1.SriovNetworkPoolConfig{}, client.InNamespace(operatorNamespace)) + Expect(err).ToNot(HaveOccurred()) + WaitForSRIOVStable() + }) + + Context("Configure rdma namespace mode", func() { + It("should switch rdma mode", func() { + By("create a pool with only that node") + networkPool := &sriovv1.SriovNetworkPoolConfig{ + ObjectMeta: metav1.ObjectMeta{Name: testNode, Namespace: operatorNamespace}, + Spec: sriovv1.SriovNetworkPoolConfigSpec{RdmaMode: consts.RdmaSubsystemModeExclusive, + NodeSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/hostname": testNode}}}} + + By("configure rdma mode to exclusive") + err := clients.Create(context.Background(), networkPool) + Expect(err).ToNot(HaveOccurred()) + By("waiting for operator to finish the configuration") + WaitForSRIOVStable() + nodeState := &sriovv1.SriovNetworkNodeState{} + Eventually(func(g Gomega) { + err = clients.Get(context.Background(), client.ObjectKey{Name: testNode, Namespace: operatorNamespace}, nodeState) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(nodeState.Spec.System.RdmaMode).To(Equal(consts.RdmaSubsystemModeExclusive)) + g.Expect(nodeState.Status.System.RdmaMode).To(Equal(consts.RdmaSubsystemModeExclusive)) + }, 20*time.Minute, 5*time.Second).Should(Succeed()) + + By("Checking rdma mode and kernel args") + cmdlineOutput, _, err := runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/proc/cmdline") + errDescription := fmt.Sprintf("kernel args are not right, printing current kernel args %s", cmdlineOutput) + Expect(err).ToNot(HaveOccurred()) + Expect(cmdlineOutput).To(ContainSubstring("ib_core.netns_mode=0"), errDescription) + Expect(cmdlineOutput).ToNot(ContainSubstring("ib_core.netns_mode=1"), errDescription) + + output, _, err := runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/etc/modprobe.d/sriov_network_operator_modules_config.conf") + Expect(err).ToNot(HaveOccurred()) + Expect(output).To(ContainSubstring("mode=0")) + + By("configure rdma mode to shared") + Eventually(func(g Gomega) { + err = clients.Get(context.Background(), client.ObjectKey{Name: testNode, Namespace: operatorNamespace}, networkPool) + g.Expect(err).ToNot(HaveOccurred()) + networkPool.Spec.RdmaMode = consts.RdmaSubsystemModeShared + err = clients.Update(context.Background(), networkPool) + g.Expect(err).ToNot(HaveOccurred()) + }, time.Minute, 5*time.Second).Should(Succeed()) + + By("waiting for operator to finish the configuration") + WaitForSRIOVStable() + Eventually(func(g Gomega) { + err = clients.Get(context.Background(), client.ObjectKey{Name: testNode, Namespace: operatorNamespace}, nodeState) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(nodeState.Spec.System.RdmaMode).To(Equal(consts.RdmaSubsystemModeShared)) + g.Expect(nodeState.Status.System.RdmaMode).To(Equal(consts.RdmaSubsystemModeShared)) + }, 20*time.Minute, 5*time.Second).Should(Succeed()) + + By("Checking rdma mode and kernel args") + cmdlineOutput, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/proc/cmdline") + errDescription = fmt.Sprintf("kernel args are not right, printing current kernel args %s", cmdlineOutput) + Expect(err).ToNot(HaveOccurred()) + Expect(cmdlineOutput).ToNot(ContainSubstring("ib_core.netns_mode=0"), errDescription) + Expect(cmdlineOutput).To(ContainSubstring("ib_core.netns_mode=1"), errDescription) + + output, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/etc/modprobe.d/sriov_network_operator_modules_config.conf") + Expect(err).ToNot(HaveOccurred()) + Expect(output).To(ContainSubstring("mode=1")) + + By("removing rdma mode configuration") + Eventually(func(g Gomega) { + err = clients.Get(context.Background(), client.ObjectKey{Name: testNode, Namespace: operatorNamespace}, networkPool) + g.Expect(err).ToNot(HaveOccurred()) + err = clients.Delete(context.Background(), networkPool) + g.Expect(err).ToNot(HaveOccurred()) + }, 5*time.Minute, 5*time.Second).Should(Succeed()) + + By("waiting for operator to finish the configuration") + WaitForSRIOVStable() + Eventually(func(g Gomega) { + err = clients.Get(context.Background(), client.ObjectKey{Name: testNode, Namespace: operatorNamespace}, nodeState) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(nodeState.Spec.System.RdmaMode).To(Equal("")) + g.Expect(nodeState.Status.System.RdmaMode).To(Equal(consts.RdmaSubsystemModeShared)) + }, 20*time.Minute, 5*time.Second).Should(Succeed()) + + By("Checking rdma mode and kernel args") + cmdlineOutput, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/proc/cmdline") + errDescription = fmt.Sprintf("kernel args are not right, printing current kernel args %s", cmdlineOutput) + Expect(cmdlineOutput).ToNot(ContainSubstring("ib_core.netns_mode=0"), errDescription) + Expect(cmdlineOutput).ToNot(ContainSubstring("ib_core.netns_mode=1"), errDescription) + + output, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "ls /host/etc/modprobe.d") + Expect(err).ToNot(HaveOccurred()) + Expect(output).ToNot(ContainSubstring("sriov_network_operator_modules_config.conf")) + }) + }) + + Context("Check rdma metrics inside a pod in exclusive mode", func() { + var iface *sriovv1.InterfaceExt + + BeforeAll(func() { + sriovInfos, err := cluster.DiscoverSriov(clients, operatorNamespace) + Expect(err).ToNot(HaveOccurred()) + Expect(len(sriovInfos.Nodes)).ToNot(BeZero()) + + for _, node := range sriovInfos.Nodes { + iface, err = sriovInfos.FindOneMellanoxSriovDevice(node) + if err == nil { + testNode = node + break + } + } + + if iface == nil { + Skip("no mellanox card available to test rdma") + } + + By("Creating sriov network to use the rdma device") + sriovNetwork := &sriovv1.SriovNetwork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-rdmanetwork", + Namespace: operatorNamespace, + }, + Spec: sriovv1.SriovNetworkSpec{ + ResourceName: resourceName, + IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181"}`, + NetworkNamespace: namespaces.Test, + MetaPluginsConfig: `{"type": "rdma"}`, + }} + + err = clients.Create(context.Background(), sriovNetwork) + Expect(err).ToNot(HaveOccurred()) + waitForNetAttachDef("test-rdmanetwork", namespaces.Test) + + sriovNetwork = &sriovv1.SriovNetwork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-nordmanetwork", + Namespace: operatorNamespace, + }, + Spec: sriovv1.SriovNetworkSpec{ + ResourceName: resourceName, + IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181"}`, + NetworkNamespace: namespaces.Test, + }} + + err = clients.Create(context.Background(), sriovNetwork) + Expect(err).ToNot(HaveOccurred()) + waitForNetAttachDef("test-nordmanetwork", namespaces.Test) + + networkPool := &sriovv1.SriovNetworkPoolConfig{ + ObjectMeta: metav1.ObjectMeta{Name: testNode, Namespace: operatorNamespace}, + Spec: sriovv1.SriovNetworkPoolConfigSpec{RdmaMode: consts.RdmaSubsystemModeExclusive, + NodeSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/hostname": testNode}}}} + err = clients.Create(context.Background(), networkPool) + Expect(err).ToNot(HaveOccurred()) + + By("waiting for operator to finish the configuration") + WaitForSRIOVStable() + nodeState := &sriovv1.SriovNetworkNodeState{} + Eventually(func(g Gomega) { + err = clients.Get(context.Background(), client.ObjectKey{Name: testNode, Namespace: operatorNamespace}, nodeState) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(nodeState.Spec.System.RdmaMode).To(Equal(consts.RdmaSubsystemModeExclusive)) + g.Expect(nodeState.Status.System.RdmaMode).To(Equal(consts.RdmaSubsystemModeExclusive)) + }, 20*time.Minute, 5*time.Second).Should(Succeed()) + }) + + It("should run pod with RDMA cni and expose nic metrics and another one without rdma info", func() { + By("creating a policy") + _, err := network.CreateSriovPolicy(clients, "test-policy-", operatorNamespace, iface.Name, testNode, 5, resourceName, "netdevice", + func(policy *sriovv1.SriovNetworkNodePolicy) { policy.Spec.IsRdma = true }) + Expect(err).ToNot(HaveOccurred()) + + By("waiting for operator to finish the configuration") + WaitForSRIOVStable() + podDefinition := pod.DefineWithNetworks([]string{"test-rdmanetwork"}) + firstPod, err := clients.Pods(namespaces.Test).Create(context.Background(), podDefinition, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + podDefinition = pod.DefineWithNetworks([]string{"test-nordmanetwork"}) + secondPod, err := clients.Pods(namespaces.Test).Create(context.Background(), podDefinition, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + firstPod = waitForPodRunning(firstPod) + secondPod = waitForPodRunning(secondPod) + + testedNode := &corev1.Node{} + err = clients.Get(context.Background(), client.ObjectKey{Name: testNode}, testedNode) + Expect(err).ToNot(HaveOccurred()) + resNum := testedNode.Status.Allocatable[corev1.ResourceName("openshift.io/"+resourceName)] + allocatable, _ := resNum.AsInt64() + Expect(allocatable).ToNot(Equal(5)) + + By("restart device plugin") + pods, err := clients.Pods(operatorNamespace).List(context.Background(), metav1.ListOptions{ + LabelSelector: "app=sriov-device-plugin", + FieldSelector: "spec.nodeName=" + testNode, + }) + Expect(err).ToNot(HaveOccurred()) + + for _, podObj := range pods.Items { + err = clients.Delete(context.Background(), &podObj) + Expect(err).ToNot(HaveOccurred()) + Eventually(func() bool { + searchPod := &corev1.Pod{} + err = clients.Get(context.Background(), client.ObjectKey{Name: podObj.Name, Namespace: podObj.Namespace}, searchPod) + if err != nil && errors.IsNotFound(err) { + return true + } + return false + }, 2*time.Minute, time.Second).Should(BeTrue()) + } + + By("checking the amount of allocatable devices remains after device plugin reset") + Consistently(func() int64 { + err = clients.Get(context.Background(), client.ObjectKey{Name: testNode}, testedNode) + Expect(err).ToNot(HaveOccurred()) + resNum := testedNode.Status.Allocatable[corev1.ResourceName("openshift.io/"+resourceName)] + newAllocatable, _ := resNum.AsInt64() + return newAllocatable + }, 1*time.Minute, 5*time.Second).Should(Equal(allocatable)) + + By("checking counters inside the pods") + strOut, _, err := pod.ExecCommand(clients, firstPod, "/bin/bash", "-c", "ip link show net1") + Expect(err).ToNot(HaveOccurred()) + Expect(strOut).To(ContainSubstring("net1")) + strOut, _, err = pod.ExecCommand(clients, firstPod, "/bin/bash", "-c", "ls /sys/bus/pci/devices/${PCIDEVICE_OPENSHIFT_IO_TESTRDMA}/infiniband/*/ports/*/hw_counters | wc -l") + strOut = strings.TrimSpace(strOut) + Expect(err).ToNot(HaveOccurred()) + num, err := strconv.Atoi(strOut) + Expect(err).ToNot(HaveOccurred()) + Expect(num).To(BeNumerically(">", 0)) + + strOut, _, err = pod.ExecCommand(clients, secondPod, "/bin/bash", "-c", "ls /sys/bus/pci/devices/${PCIDEVICE_OPENSHIFT_IO_TESTRDMA}/infiniband/ | wc -l") + Expect(err).ToNot(HaveOccurred()) + strOut = strings.TrimSpace(strOut) + num, err = strconv.Atoi(strOut) + Expect(err).ToNot(HaveOccurred()) + Expect(num).To(BeNumerically("==", 0)) + }) + }) + + Context("Check rdma metrics inside a pod in shared mode not exist", func() { + var iface *sriovv1.InterfaceExt + BeforeAll(func() { + sriovInfos, err := cluster.DiscoverSriov(clients, operatorNamespace) + Expect(err).ToNot(HaveOccurred()) + Expect(len(sriovInfos.Nodes)).ToNot(BeZero()) + + for _, node := range sriovInfos.Nodes { + iface, err = sriovInfos.FindOneMellanoxSriovDevice(node) + if err == nil { + testNode = node + break + } + } + + if iface == nil { + Skip("no mellanox card available to test rdma") + } + + By("Creating sriov network to use the rdma device") + sriovNetwork := &sriovv1.SriovNetwork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-rdmanetwork", + Namespace: operatorNamespace, + }, + Spec: sriovv1.SriovNetworkSpec{ + ResourceName: resourceName, + IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181"}`, + NetworkNamespace: namespaces.Test, + }} + + err = clients.Create(context.Background(), sriovNetwork) + Expect(err).ToNot(HaveOccurred()) + waitForNetAttachDef("test-rdmanetwork", namespaces.Test) + + networkPool := &sriovv1.SriovNetworkPoolConfig{ + ObjectMeta: metav1.ObjectMeta{Name: testNode, Namespace: operatorNamespace}, + Spec: sriovv1.SriovNetworkPoolConfigSpec{RdmaMode: consts.RdmaSubsystemModeShared, + NodeSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/hostname": testNode}}}} + + err = clients.Create(context.Background(), networkPool) + Expect(err).ToNot(HaveOccurred()) + By("waiting for operator to finish the configuration") + WaitForSRIOVStable() + nodeState := &sriovv1.SriovNetworkNodeState{} + Eventually(func(g Gomega) { + err = clients.Get(context.Background(), client.ObjectKey{Name: testNode, Namespace: operatorNamespace}, nodeState) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(nodeState.Spec.System.RdmaMode).To(Equal(consts.RdmaSubsystemModeShared)) + g.Expect(nodeState.Status.System.RdmaMode).To(Equal(consts.RdmaSubsystemModeShared)) + }, 20*time.Minute, 5*time.Second).Should(Succeed()) + }) + + It("should run pod without RDMA cni and not expose nic metrics", func() { + By("creating a policy") + _, err := network.CreateSriovPolicy(clients, "test-policy-", operatorNamespace, iface.Name, testNode, 5, resourceName, "netdevice", + func(policy *sriovv1.SriovNetworkNodePolicy) { policy.Spec.IsRdma = true }) + Expect(err).ToNot(HaveOccurred()) + WaitForSRIOVStable() + + podDefinition := pod.DefineWithNetworks([]string{"test-rdmanetwork"}) + firstPod, err := clients.Pods(namespaces.Test).Create(context.Background(), podDefinition, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + firstPod = waitForPodRunning(firstPod) + + strOut, _, err := pod.ExecCommand(clients, firstPod, "/bin/bash", "-c", "ip link show net1") + Expect(err).ToNot(HaveOccurred()) + Expect(strOut).To(ContainSubstring("net1")) + strOut, _, err = pod.ExecCommand(clients, firstPod, "/bin/bash", "-c", "ls /sys/bus/pci/devices/${PCIDEVICE_OPENSHIFT_IO_TESTRDMA}/infiniband/*/ports/* | grep hw_counters | wc -l") + strOut = strings.TrimSpace(strOut) + Expect(err).ToNot(HaveOccurred()) + num, err := strconv.Atoi(strOut) + Expect(err).ToNot(HaveOccurred()) + Expect(num).To(BeNumerically("==", 0)) + }) + }) +}) diff --git a/test/conformance/tests/test_no_policy.go b/test/conformance/tests/test_no_policy.go new file mode 100644 index 000000000..c965cf186 --- /dev/null +++ b/test/conformance/tests/test_no_policy.go @@ -0,0 +1,259 @@ +package tests + +import ( + "context" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" + + netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + + sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/cluster" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/discovery" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/namespaces" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/nodes" +) + +var _ = Describe("[sriov] operator", Ordered, ContinueOnFailure, func() { + Describe("No SriovNetworkNodePolicy", func() { + Context("SR-IOV network config daemon can be set by nodeselector", func() { + // 26186 + It("Should schedule the config daemon on selected nodes", func() { + if discovery.Enabled() { + Skip("Test unsuitable to be run in discovery mode") + } + + By("Checking that a daemon is scheduled on each worker node") + Eventually(func() bool { + return daemonsScheduledOnNodes("node-role.kubernetes.io/worker=") + }, 3*time.Minute, 1*time.Second).Should(Equal(true)) + + By("Labeling one worker node with the label needed for the daemon") + allNodes, err := clients.CoreV1Interface.Nodes().List(context.Background(), metav1.ListOptions{ + LabelSelector: "node-role.kubernetes.io/worker", + }) + Expect(err).ToNot(HaveOccurred()) + + selectedNodes, err := nodes.MatchingOptionalSelector(clients, allNodes.Items) + Expect(err).ToNot(HaveOccurred()) + + Expect(len(selectedNodes)).To(BeNumerically(">", 0), "There must be at least one worker") + patch := []byte(`{"metadata":{"labels":{"sriovenabled":"true"}}}`) + candidate, err := clients.CoreV1Interface.Nodes().Patch(context.Background(), selectedNodes[0].Name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}) + Expect(err).ToNot(HaveOccurred()) + selectedNodes[0] = *candidate + + By("Setting the node selector for each daemon") + cfg := sriovv1.SriovOperatorConfig{} + err = clients.Get(context.TODO(), runtimeclient.ObjectKey{ + Name: "default", + Namespace: operatorNamespace, + }, &cfg) + Expect(err).ToNot(HaveOccurred()) + cfg.Spec.ConfigDaemonNodeSelector = map[string]string{ + "sriovenabled": "true", + } + Eventually(func() error { + return clients.Update(context.TODO(), &cfg) + }, 1*time.Minute, 5*time.Second).ShouldNot(HaveOccurred()) + + By("Checking that a daemon is scheduled only on selected node") + Eventually(func() bool { + return !daemonsScheduledOnNodes("sriovenabled!=true") && + daemonsScheduledOnNodes("sriovenabled=true") + }, 1*time.Minute, 1*time.Second).Should(Equal(true)) + + By("Restoring the node selector for daemons") + err = clients.Get(context.TODO(), runtimeclient.ObjectKey{ + Name: "default", + Namespace: operatorNamespace, + }, &cfg) + Expect(err).ToNot(HaveOccurred()) + cfg.Spec.ConfigDaemonNodeSelector = map[string]string{} + Eventually(func() error { + return clients.Update(context.TODO(), &cfg) + }, 1*time.Minute, 5*time.Second).ShouldNot(HaveOccurred()) + + By("Checking that a daemon is scheduled on each worker node") + Eventually(func() bool { + return daemonsScheduledOnNodes("node-role.kubernetes.io/worker") + }, 1*time.Minute, 1*time.Second).Should(Equal(true)) + }) + }) + + Context("LogLevel affects operator's logs", func() { + It("when set to 0 no lifecycle logs are present", func() { + if discovery.Enabled() { + Skip("Test unsuitable to be run in discovery mode") + } + + initialLogLevelValue := getOperatorConfigLogLevel() + DeferCleanup(func() { + By("Restore LogLevel to its initial value") + setOperatorConfigLogLevel(initialLogLevelValue) + }) + + initialDisableDrain, err := cluster.GetNodeDrainState(clients, operatorNamespace) + Expect(err).ToNot(HaveOccurred()) + + DeferCleanup(func() { + By("Restore DisableDrain to its initial value") + Eventually(func() error { + return cluster.SetDisableNodeDrainState(clients, operatorNamespace, initialDisableDrain) + }, 1*time.Minute, 5*time.Second).ShouldNot(HaveOccurred()) + }) + + By("Set operator LogLevel to 2") + setOperatorConfigLogLevel(2) + + By("Flip DisableDrain to trigger operator activity") + since := time.Now().Add(-10 * time.Second) + Eventually(func() error { + return cluster.SetDisableNodeDrainState(clients, operatorNamespace, !initialDisableDrain) + }, 1*time.Minute, 5*time.Second).ShouldNot(HaveOccurred()) + + By("Assert logs contains verbose output") + Eventually(func(g Gomega) { + logs := getOperatorLogs(since) + g.Expect(logs).To( + ContainElement(And( + ContainSubstring("Reconciling SriovOperatorConfig"), + )), + ) + + // Should contain verbose logging + g.Expect(logs).To( + ContainElement( + ContainSubstring("Start to sync webhook objects"), + ), + ) + }, 1*time.Minute, 5*time.Second).Should(Succeed()) + + By("Reduce operator LogLevel to 0") + setOperatorConfigLogLevel(0) + + By("Flip DisableDrain again to trigger operator activity") + since = time.Now().Add(-10 * time.Second) + Eventually(func() error { + return cluster.SetDisableNodeDrainState(clients, operatorNamespace, initialDisableDrain) + }, 1*time.Minute, 5*time.Second).ShouldNot(HaveOccurred()) + + By("Assert logs contains less operator activity") + Eventually(func(g Gomega) { + logs := getOperatorLogs(since) + + // time only contains sec, but we can have race here that in the same sec there was a sync + afterLogs := []string{} + found := false + for _, log := range logs { + if found { + afterLogs = append(afterLogs, log) + } + if strings.Contains(log, "{\"new-level\": 0, \"current-level\": 2}") { + found = true + } + } + g.Expect(found).To(BeTrue()) + g.Expect(afterLogs).To( + ContainElement(And( + ContainSubstring("Reconciling SriovOperatorConfig"), + )), + ) + + // Should not contain verbose logging + g.Expect(afterLogs).ToNot( + ContainElement( + ContainSubstring("Start to sync webhook objects"), + ), + ) + }, 3*time.Minute, 5*time.Second).Should(Succeed()) + }) + }) + + Context("SriovNetworkMetricsExporter", func() { + BeforeEach(func() { + if discovery.Enabled() { + Skip("Test unsuitable to be run in discovery mode") + } + + initialValue := isFeatureFlagEnabled("metricsExporter") + DeferCleanup(func() { + By("Restoring initial feature flag value") + setFeatureFlag("metricsExporter", initialValue) + }) + + By("Enabling `metricsExporter` feature flag") + setFeatureFlag("metricsExporter", true) + }) + + It("should be deployed if the feature gate is enabled", func() { + By("Checking that a daemon is scheduled on selected node") + Eventually(func() bool { + return isDaemonsetScheduledOnNodes("node-role.kubernetes.io/worker", "app=sriov-network-metrics-exporter") + }).WithTimeout(time.Minute).WithPolling(time.Second).Should(Equal(true)) + }) + + It("should deploy ServiceMonitor if the Prometheus operator is installed", func() { + _, err := clients.ServiceMonitors(operatorNamespace).List(context.Background(), metav1.ListOptions{}) + if k8serrors.IsNotFound(err) { + Skip("Prometheus operator not available in the cluster") + } + + By("Checking ServiceMonitor is deployed if needed") + Eventually(func(g Gomega) { + _, err := clients.ServiceMonitors(operatorNamespace).Get(context.Background(), "sriov-network-metrics-exporter", metav1.GetOptions{}) + g.Expect(err).ToNot(HaveOccurred()) + }).WithTimeout(time.Minute).WithPolling(time.Second).Should(Succeed()) + }) + + It("should remove ServiceMonitor when the feature is turned off", func() { + setFeatureFlag("metricsExporter", false) + Eventually(func(g Gomega) { + _, err := clients.ServiceMonitors(operatorNamespace).Get(context.Background(), "sriov-network-metrics-exporter", metav1.GetOptions{}) + g.Expect(k8serrors.IsNotFound(err)).To(BeTrue()) + }).WithTimeout(time.Minute).WithPolling(time.Second).Should(Succeed()) + }) + }) + + Context("Namespaced network objects", func() { + DescribeTable("can be create in every namespaces", func(object runtimeclient.Object) { + err := clients.Create(context.Background(), object) + Expect(err).ToNot(HaveOccurred()) + + waitForNetAttachDef(object.GetName(), object.GetNamespace()) + + err = clients.Delete(context.Background(), object) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() bool { + netAttDef := &netattdefv1.NetworkAttachmentDefinition{} + err := clients.Get(context.Background(), runtimeclient.ObjectKey{Name: object.GetName(), Namespace: object.GetNamespace()}, netAttDef) + return err != nil && k8serrors.IsNotFound(err) + }, 2*time.Minute, 10*time.Second).Should(BeTrue()) + }, + Entry("SriovNetwork", &sriovv1.SriovNetwork{ObjectMeta: metav1.ObjectMeta{Name: "sriovnet1", Namespace: namespaces.Test}}), + Entry("SriovIBNetwork", &sriovv1.SriovIBNetwork{ObjectMeta: metav1.ObjectMeta{Name: "sriovibnet1", Namespace: namespaces.Test}}), + Entry("OVSNetwork", &sriovv1.OVSNetwork{ObjectMeta: metav1.ObjectMeta{Name: "ovsnet1", Namespace: namespaces.Test}}), + ) + + DescribeTable("can NOT be in application namespace and have .Spec.NetworkNamespace != ''", func(object runtimeclient.Object) { + err := clients.Create(context.Background(), object) + Expect(err).To(HaveOccurred()) + Expect(string(k8serrors.ReasonForError(err))). + To(ContainSubstring(".Spec.NetworkNamespace field can't be specified if the resource is not in the ")) + }, + Entry("SriovNetwork", &sriovv1.SriovNetwork{ObjectMeta: metav1.ObjectMeta{Name: "sriovnet1", Namespace: namespaces.Test}, Spec: sriovv1.SriovNetworkSpec{NetworkNamespace: "default"}}), + Entry("SriovIBNetwork", &sriovv1.SriovIBNetwork{ObjectMeta: metav1.ObjectMeta{Name: "sriovibnet1", Namespace: namespaces.Test}, Spec: sriovv1.SriovIBNetworkSpec{NetworkNamespace: "default"}}), + Entry("OVSNetwork", &sriovv1.OVSNetwork{ObjectMeta: metav1.ObjectMeta{Name: "ovsnet1", Namespace: namespaces.Test}, Spec: sriovv1.OVSNetworkSpec{NetworkNamespace: "default"}}), + ) + }) + }) +}) diff --git a/test/conformance/tests/test_policy_configuration.go b/test/conformance/tests/test_policy_configuration.go index 28a2c95cc..f8ed943a5 100644 --- a/test/conformance/tests/test_policy_configuration.go +++ b/test/conformance/tests/test_policy_configuration.go @@ -55,7 +55,8 @@ var _ = Describe("[sriov] operator", Ordered, func() { By("waiting for the node state to be updated") Eventually(func() sriovv1.Interfaces { - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), vfioNode, metav1.GetOptions{}) + nodeState := &sriovv1.SriovNetworkNodeState{} + err := clients.Get(context.Background(), runtimeclient.ObjectKey{Namespace: operatorNamespace, Name: vfioNode}, nodeState) Expect(err).ToNot(HaveOccurred()) return nodeState.Spec.Interfaces }, 1*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( @@ -90,7 +91,7 @@ var _ = Describe("[sriov] operator", Ordered, func() { }, Spec: sriovv1.SriovNetworkSpec{ ResourceName: resourceName, - IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181","routes":[{"dst":"0.0.0.0/0"}],"gateway":"10.10.10.1"}`, + IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181"}`, NetworkNamespace: namespaces.Test, }} @@ -148,7 +149,8 @@ var _ = Describe("[sriov] operator", Ordered, func() { Expect(err).ToNot(HaveOccurred()) Eventually(func() sriovv1.Interfaces { - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), vfioNode, metav1.GetOptions{}) + nodeState := &sriovv1.SriovNetworkNodeState{} + err := clients.Get(context.Background(), runtimeclient.ObjectKey{Namespace: operatorNamespace, Name: vfioNode}, nodeState) Expect(err).ToNot(HaveOccurred()) return nodeState.Spec.Interfaces }, 3*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( @@ -180,7 +182,8 @@ var _ = Describe("[sriov] operator", Ordered, func() { Expect(err).ToNot(HaveOccurred()) Eventually(func() sriovv1.Interfaces { - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), vfioNode, metav1.GetOptions{}) + nodeState := &sriovv1.SriovNetworkNodeState{} + err := clients.Get(context.Background(), runtimeclient.ObjectKey{Namespace: operatorNamespace, Name: vfioNode}, nodeState) Expect(err).ToNot(HaveOccurred()) return nodeState.Spec.Interfaces }, 3*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( @@ -248,7 +251,8 @@ var _ = Describe("[sriov] operator", Ordered, func() { Expect(err).ToNot(HaveOccurred()) Eventually(func() sriovv1.Interfaces { - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), node, metav1.GetOptions{}) + nodeState := &sriovv1.SriovNetworkNodeState{} + err := clients.Get(context.Background(), runtimeclient.ObjectKey{Namespace: operatorNamespace, Name: node}, nodeState) Expect(err).ToNot(HaveOccurred()) return nodeState.Spec.Interfaces }, 3*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( @@ -279,7 +283,8 @@ var _ = Describe("[sriov] operator", Ordered, func() { By(fmt.Sprintf("verifying that only VF 0 and 1 have mtu set to %d", newMtu)) Eventually(func() sriovv1.InterfaceExts { - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), node, metav1.GetOptions{}) + nodeState := &sriovv1.SriovNetworkNodeState{} + err := clients.Get(context.Background(), runtimeclient.ObjectKey{Namespace: operatorNamespace, Name: node}, nodeState) Expect(err).ToNot(HaveOccurred()) return nodeState.Status.Interfaces }, 3*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( @@ -342,7 +347,8 @@ var _ = Describe("[sriov] operator", Ordered, func() { Expect(err).ToNot(HaveOccurred()) Eventually(func() sriovv1.Interfaces { - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), node, metav1.GetOptions{}) + nodeState := &sriovv1.SriovNetworkNodeState{} + err := clients.Get(context.Background(), runtimeclient.ObjectKey{Namespace: operatorNamespace, Name: node}, nodeState) Expect(err).ToNot(HaveOccurred()) return nodeState.Spec.Interfaces }, 3*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( @@ -483,9 +489,7 @@ var _ = Describe("[sriov] operator", Ordered, func() { "type":"host-local", "subnet":"10.10.10.0/24", "rangeStart":"10.10.10.171", - "rangeEnd":"10.10.10.181", - "routes":[{"dst":"0.0.0.0/0"}], - "gateway":"10.10.10.1" + "rangeEnd":"10.10.10.181" }` err = network.CreateSriovNetwork(clients, unusedSriovDevice, sriovNetworkName, namespaces.Test, operatorNamespace, resourceName, ipam) @@ -528,7 +532,8 @@ var _ = Describe("[sriov] operator", Ordered, func() { if len(sriovDeviceList) == 0 { return nil, false } - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), node, metav1.GetOptions{}) + nodeState := &sriovv1.SriovNetworkNodeState{} + err := clients.Get(context.Background(), runtimeclient.ObjectKey{Namespace: operatorNamespace, Name: node}, nodeState) Expect(err).ToNot(HaveOccurred()) for _, ifc := range nodeState.Spec.Interfaces { @@ -600,10 +605,15 @@ var _ = Describe("[sriov] operator", Ordered, func() { }, Spec: sriovv1.SriovNetworkSpec{ ResourceName: resourceName, - IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181","routes":[{"dst":"0.0.0.0/0"}],"gateway":"10.10.10.1"}`, + IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181"}`, NetworkNamespace: namespaces.Test, }} + // for real BM env we enable link state + if !cluster.VirtualCluster() { + sriovNetwork.Spec.LinkState = "enable" + } + // We need this to be able to run the connectivity checks on Mellanox cards if intf.DeviceID == "1015" { sriovNetwork.Spec.SpoofChk = off diff --git a/test/conformance/tests/test_sriov_operator.go b/test/conformance/tests/test_sriov_operator.go index 3c3cc4e84..0701023d2 100644 --- a/test/conformance/tests/test_sriov_operator.go +++ b/test/conformance/tests/test_sriov_operator.go @@ -3,7 +3,6 @@ package tests import ( "bufio" "context" - "encoding/json" "fmt" "os" "strconv" @@ -18,23 +17,22 @@ import ( admission "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + networkv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/cluster" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/discovery" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/execute" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/k8sreporter" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/namespaces" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/network" - "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/nodes" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/pod" ) @@ -54,12 +52,6 @@ const ( ipamIpv4 = `{"type": "host-local","ranges": [[{"subnet": "1.1.1.0/24"}]],"dataDir": "/run/my-orchestrator/container-ipam-state"}` ) -type patchBody struct { - Op string `json:"op"` - Path string `json:"path"` - Value string `json:"value"` -} - func init() { waitingEnv := os.Getenv("SRIOV_WAITING_TIME") newTime, err := strconv.Atoi(waitingEnv) @@ -68,250 +60,11 @@ func init() { } } -var _ = Describe("[sriov] operator", func() { - Describe("No SriovNetworkNodePolicy", func() { - Context("SR-IOV network config daemon can be set by nodeselector", func() { - // 26186 - It("Should schedule the config daemon on selected nodes", func() { - if discovery.Enabled() { - Skip("Test unsuitable to be run in discovery mode") - } - - testStartingTime := time.Now() - - By("Checking that a daemon is scheduled on each worker node") - Eventually(func() bool { - return daemonsScheduledOnNodes("node-role.kubernetes.io/worker=") - }, 3*time.Minute, 1*time.Second).Should(Equal(true)) - - By("Labeling one worker node with the label needed for the daemon") - allNodes, err := clients.CoreV1Interface.Nodes().List(context.Background(), metav1.ListOptions{ - LabelSelector: "node-role.kubernetes.io/worker", - }) - Expect(err).ToNot(HaveOccurred()) - - selectedNodes, err := nodes.MatchingOptionalSelector(clients, allNodes.Items) - Expect(err).ToNot(HaveOccurred()) - - Expect(len(selectedNodes)).To(BeNumerically(">", 0), "There must be at least one worker") - candidate := selectedNodes[0] - candidate.Labels["sriovenabled"] = "true" - _, err = clients.CoreV1Interface.Nodes().Update(context.Background(), &candidate, metav1.UpdateOptions{}) - Expect(err).ToNot(HaveOccurred()) - - By("Setting the node selector for each daemon") - cfg := sriovv1.SriovOperatorConfig{} - err = clients.Get(context.TODO(), runtimeclient.ObjectKey{ - Name: "default", - Namespace: operatorNamespace, - }, &cfg) - Expect(err).ToNot(HaveOccurred()) - cfg.Spec.ConfigDaemonNodeSelector = map[string]string{ - "sriovenabled": "true", - } - Eventually(func() error { - return clients.Update(context.TODO(), &cfg) - }, 1*time.Minute, 5*time.Second).ShouldNot(HaveOccurred()) - - By("Checking that a daemon is scheduled only on selected node") - Eventually(func() bool { - return !daemonsScheduledOnNodes("sriovenabled!=true") && - daemonsScheduledOnNodes("sriovenabled=true") - }, 1*time.Minute, 1*time.Second).Should(Equal(true)) - - By("Restoring the node selector for daemons") - err = clients.Get(context.TODO(), runtimeclient.ObjectKey{ - Name: "default", - Namespace: operatorNamespace, - }, &cfg) - Expect(err).ToNot(HaveOccurred()) - cfg.Spec.ConfigDaemonNodeSelector = map[string]string{} - Eventually(func() error { - return clients.Update(context.TODO(), &cfg) - }, 1*time.Minute, 5*time.Second).ShouldNot(HaveOccurred()) - - By("Checking that a daemon is scheduled on each worker node") - Eventually(func() bool { - return daemonsScheduledOnNodes("node-role.kubernetes.io/worker") - }, 1*time.Minute, 1*time.Second).Should(Equal(true)) - - By("Checking that a daemon is able to publish events") - Eventually(func() bool { - events, err := clients.Events(operatorNamespace).List( - context.Background(), metav1.ListOptions{TypeMeta: sriovv1.SriovNetworkNodeState{}.TypeMeta}) - Expect(err).ToNot(HaveOccurred()) - - for _, e := range events.Items { - if e.Reason == "ConfigDaemonStart" && - e.CreationTimestamp.Time.After(testStartingTime) { - return true - } - } - return false - }, 30*time.Second, 5*time.Second).Should(BeTrue(), "Config Daemon should record an event when starting") - }) - }) - - Context("LogLevel affects operator's logs", func() { - It("when set to 0 no lifecycle logs are present", func() { - if discovery.Enabled() { - Skip("Test unsuitable to be run in discovery mode") - } - - initialLogLevelValue := getOperatorConfigLogLevel() - DeferCleanup(func() { - By("Restore LogLevel to its initial value") - setOperatorConfigLogLevel(initialLogLevelValue) - }) - - initialDisableDrain, err := cluster.GetNodeDrainState(clients, operatorNamespace) - Expect(err).ToNot(HaveOccurred()) - - DeferCleanup(func() { - By("Restore DisableDrain to its initial value") - Eventually(func() error { - return cluster.SetDisableNodeDrainState(clients, operatorNamespace, initialDisableDrain) - }, 1*time.Minute, 5*time.Second).ShouldNot(HaveOccurred()) - }) - - By("Set operator LogLevel to 2") - setOperatorConfigLogLevel(2) - - By("Flip DisableDrain to trigger operator activity") - since := time.Now().Add(-10 * time.Second) - Eventually(func() error { - return cluster.SetDisableNodeDrainState(clients, operatorNamespace, !initialDisableDrain) - }, 1*time.Minute, 5*time.Second).ShouldNot(HaveOccurred()) - - By("Assert logs contains verbose output") - Eventually(func(g Gomega) { - logs := getOperatorLogs(since) - g.Expect(logs).To( - ContainElement(And( - ContainSubstring("Reconciling SriovOperatorConfig"), - )), - ) - - // Should contain verbose logging - g.Expect(logs).To( - ContainElement( - ContainSubstring("Start to sync webhook objects"), - ), - ) - }, 1*time.Minute, 5*time.Second).Should(Succeed()) - - By("Reduce operator LogLevel to 0") - setOperatorConfigLogLevel(0) - - By("Flip DisableDrain again to trigger operator activity") - since = time.Now().Add(-10 * time.Second) - Eventually(func() error { - return cluster.SetDisableNodeDrainState(clients, operatorNamespace, initialDisableDrain) - }, 1*time.Minute, 5*time.Second).ShouldNot(HaveOccurred()) - - By("Assert logs contains less operator activity") - Eventually(func(g Gomega) { - logs := getOperatorLogs(since) - - // time only contains sec, but we can have race here that in the same sec there was a sync - afterLogs := []string{} - found := false - for _, log := range logs { - if found { - afterLogs = append(afterLogs, log) - } - if strings.Contains(log, "{\"new-level\": 0, \"current-level\": 2}") { - found = true - } - } - g.Expect(found).To(BeTrue()) - g.Expect(afterLogs).To( - ContainElement(And( - ContainSubstring("Reconciling SriovOperatorConfig"), - )), - ) - - // Should not contain verbose logging - g.Expect(afterLogs).ToNot( - ContainElement( - ContainSubstring("Start to sync webhook objects"), - ), - ) - }, 3*time.Minute, 5*time.Second).Should(Succeed()) - }) - }) - - DescribeTable("should gracefully restart quickly", func(webookEnabled bool) { - DeferCleanup(setSriovOperatorSpecFlag(operatorNetworkInjectorFlag, webookEnabled)) - DeferCleanup(setSriovOperatorSpecFlag(operatorWebhookFlag, webookEnabled)) - - // This test case ensure leader election process runs smoothly when the operator's pod is restarted. - oldLease, err := clients.CoordinationV1Interface.Leases(operatorNamespace).Get(context.Background(), consts.LeaderElectionID, metav1.GetOptions{}) - if k8serrors.IsNotFound(err) { - Skip("Leader Election is not enabled on the cluster. Skipping") - } - Expect(err).ToNot(HaveOccurred()) - - oldOperatorPod := getOperatorPod() - - By("Delete the operator's pod") - deleteOperatorPod() - - By("Wait the new operator's pod to start") - Eventually(func(g Gomega) { - newOperatorPod := getOperatorPod() - Expect(newOperatorPod.Name).ToNot(Equal(oldOperatorPod.Name)) - Expect(newOperatorPod.Status.Phase).To(Equal(corev1.PodRunning)) - }, 45*time.Second, 5*time.Second) - - By("Assert the new operator's pod acquire the lease before 30 seconds") - Eventually(func(g Gomega) { - newLease, err := clients.CoordinationV1Interface.Leases(operatorNamespace).Get(context.Background(), consts.LeaderElectionID, metav1.GetOptions{}) - g.Expect(err).ToNot(HaveOccurred()) - - g.Expect(newLease.Spec.HolderIdentity).ToNot(Equal(oldLease.Spec.HolderIdentity)) - }, 30*time.Second, 5*time.Second).Should(Succeed()) - }, - Entry("webhooks enabled", true), - Entry("webhooks disabled", true), - ) - - Context("SriovNetworkMetricsExporter", func() { - BeforeEach(func() { - if discovery.Enabled() { - Skip("Test unsuitable to be run in discovery mode") - } - - initialValue := isFeatureFlagEnabled("metricsExporter") - DeferCleanup(func() { - By("Restoring initial feature flag value") - setFeatureFlag("metricsExporter", initialValue) - }) - - By("Enabling `metricsExporter` feature flag") - setFeatureFlag("metricsExporter", true) - }) - - It("should be deployed if the feature gate is enabled", func() { - By("Checking that a daemon is scheduled on selected node") - Eventually(func() bool { - return isDaemonsetScheduledOnNodes("node-role.kubernetes.io/worker", "app=sriov-network-metrics-exporter") - }, 1*time.Minute, 1*time.Second).Should(Equal(true)) - }) - - It("should deploy ServiceMonitor if the Promethueus operator is installed", func() { - _, err := clients.ServiceMonitors(operatorNamespace).List(context.Background(), metav1.ListOptions{}) - if k8serrors.IsNotFound(err) { - Skip("Prometheus operator not available in the cluster") - } - - By("Checking ServiceMonitor is deployed if needed") - Eventually(func(g Gomega) { - _, err := clients.ServiceMonitors(operatorNamespace).Get(context.Background(), "sriov-network-metrics-exporter", metav1.GetOptions{}) - g.Expect(err).ToNot(HaveOccurred()) - }).Should(Succeed()) - }) - }) +var _ = Describe("[sriov] operator", Ordered, func() { + AfterAll(func() { + err := namespaces.Clean(operatorNamespace, namespaces.Test, clients, discovery.Enabled()) + Expect(err).ToNot(HaveOccurred()) + WaitForSRIOVStable() }) Describe("Generic SriovNetworkNodePolicy", func() { @@ -380,7 +133,7 @@ var _ = Describe("[sriov] operator", func() { }, Spec: sriovv1.SriovNetworkSpec{ ResourceName: resourceName, - IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181","routes":[{"dst":"0.0.0.0/0"}],"gateway":"10.10.10.1"}`, + IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181"}`, NetworkNamespace: namespaces.Test, }} err := clients.Create(context.Background(), sriovNetwork) @@ -433,7 +186,7 @@ var _ = Describe("[sriov] operator", func() { }, Spec: sriovv1.SriovNetworkSpec{ ResourceName: resourceName, - IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181","routes":[{"dst":"0.0.0.0/0"}],"gateway":"10.10.10.1"}`, + IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181"}`, NetworkNamespace: namespaces.Test, }} err := clients.Create(context.Background(), sriovNetwork) @@ -493,9 +246,14 @@ var _ = Describe("[sriov] operator", func() { Expect(err).ToNot(HaveOccurred()) Eventually(func() bool { - stdout, stderr, err := pod.ExecCommand(clients, hostNetPod, "ip", "link", "show") - Expect(err).ToNot(HaveOccurred()) - Expect(stderr).To(Equal("")) + var stdout, stderr string + // Adding a retry because some of the time we get `Dump was interrupted and may be inconsistent.` + // output from the ip link command + Eventually(func(g Gomega) { + stdout, stderr, err = pod.ExecCommand(clients, hostNetPod, "ip", "link", "show") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(stderr).To(Equal("")) + }, time.Minute, 2*time.Second).Should(Succeed()) found := false for _, line := range strings.Split(stdout, "\n") { @@ -509,7 +267,7 @@ var _ = Describe("[sriov] operator", func() { } err = clients.Pods(namespaces.Test).Delete(context.Background(), podObj.Name, metav1.DeleteOptions{ - GracePeriodSeconds: pointer.Int64Ptr(0)}) + GracePeriodSeconds: ptr.To(int64(0))}) Expect(err).ToNot(HaveOccurred()) return found @@ -555,9 +313,7 @@ var _ = Describe("[sriov] operator", func() { IPAM: `{"type":"host-local", "subnet":"10.10.10.0/24", "rangeStart":"10.10.10.171", - "rangeEnd":"10.10.10.181", - "routes":[{"dst":"0.0.0.0/0"}], - "gateway":"10.10.10.1"}`, + "rangeEnd":"10.10.10.181"}`, NetworkNamespace: namespaces.Test, }} @@ -600,9 +356,7 @@ var _ = Describe("[sriov] operator", func() { IPAM: `{"type":"host-local", "subnet":"10.10.10.0/24", "rangeStart":"10.10.10.171", - "rangeEnd":"10.10.10.181", - "routes":[{"dst":"0.0.0.0/0"}], - "gateway":"10.10.10.1"}`, + "rangeEnd":"10.10.10.181"}`, NetworkNamespace: namespaces.Test, }} @@ -649,9 +403,7 @@ var _ = Describe("[sriov] operator", func() { IPAM: `{"type":"host-local", "subnet":"10.10.10.0/24", "rangeStart":"10.10.10.171", - "rangeEnd":"10.10.10.181", - "routes":[{"dst":"0.0.0.0/0"}], - "gateway":"10.10.10.1"}`, + "rangeEnd":"10.10.10.181"}`, NetworkNamespace: namespaces.Test, }} @@ -722,9 +474,7 @@ var _ = Describe("[sriov] operator", func() { IPAM: `{"type":"host-local", "subnet":"10.10.10.0/24", "rangeStart":"10.10.10.171", - "rangeEnd":"10.10.10.181", - "routes":[{"dst":"0.0.0.0/0"}], - "gateway":"10.10.10.1"}`, + "rangeEnd":"10.10.10.181"}`, MaxTxRate: &maxTxRate, MinTxRate: &minTxRate, NetworkNamespace: namespaces.Test, @@ -758,9 +508,7 @@ var _ = Describe("[sriov] operator", func() { IPAM: `{"type":"host-local", "subnet":"10.10.10.0/24", "rangeStart":"10.10.10.171", - "rangeEnd":"10.10.10.181", - "routes":[{"dst":"0.0.0.0/0"}], - "gateway":"10.10.10.1"}`, + "rangeEnd":"10.10.10.181"}`, Vlan: 1, VlanQoS: 2, NetworkNamespace: namespaces.Test, @@ -838,13 +586,14 @@ var _ = Describe("[sriov] operator", func() { Expect(err).ToNot(HaveOccurred()) waitForNetAttachDef(sriovNetworkName, ns1) - body, _ := json.Marshal([]patchBody{{ - Op: "replace", - Path: "/spec/networkNamespace", - Value: ns2, - }}) - clients.SriovnetworkV1Interface.RESTClient().Patch(types.JSONPatchType).Namespace(operatorNamespace).Resource("sriovnetworks").Name(sriovNetworkName).Body(body).Do(context.Background()) + srNetwork := &sriovv1.SriovNetwork{} + err = clients.Get(context.Background(), runtimeclient.ObjectKey{Namespace: operatorNamespace, Name: sriovNetworkName}, srNetwork) + Expect(err).ToNot(HaveOccurred()) + original := srNetwork.DeepCopy() + srNetwork.Spec.NetworkNamespace = ns2 + err = clients.Patch(context.Background(), srNetwork, runtimeclient.MergeFrom(original)) + Expect(err).ToNot(HaveOccurred()) waitForNetAttachDef(sriovNetworkName, ns2) Consistently(func() error { @@ -939,7 +688,7 @@ var _ = Describe("[sriov] operator", func() { waitForNetAttachDef(sriovNetworkName, namespaces.Test) testPod := createTestPod(node, []string{sriovNetworkName}) - stdout, _, err := pod.ExecCommand(clients, testPod, "more", "/proc/sys/net/ipv4/conf/net1/accept_redirects") + stdout, _, err := pod.ExecCommand(clients, testPod, "cat", "/proc/sys/net/ipv4/conf/net1/accept_redirects") Expect(err).ToNot(HaveOccurred()) Expect(strings.TrimSpace(stdout)).To(Equal("1")) @@ -1003,8 +752,7 @@ var _ = Describe("[sriov] operator", func() { }, 2*time.Minute, 10*time.Second).Should(BeTrue(), "Error to detect Required Event") By("Delete first pod and release all VFs") err = clients.Pods(namespaces.Test).Delete(context.Background(), runningPodA.Name, metav1.DeleteOptions{ - GracePeriodSeconds: pointer.Int64Ptr(0), - }) + GracePeriodSeconds: ptr.To(int64(0))}) Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Error to delete pod %s", runningPodA.Name)) By("Checking that second pod is able to use released VF") waitForPodRunning(runningPodB) @@ -1060,9 +808,11 @@ var _ = Describe("[sriov] operator", func() { findSriovDevice := func(vendorID, deviceID string) (string, sriovv1.InterfaceExt) { for _, node := range sriovInfos.Nodes { - for _, nic := range sriovInfos.States[node].Status.Interfaces { + devices, err := sriovInfos.FindSriovDevices(node) + Expect(err).ToNot(HaveOccurred()) + for _, nic := range devices { if vendorID != "" && deviceID != "" && nic.Vendor == vendorID && nic.DeviceID == deviceID { - return node, nic + return node, *nic } } } @@ -1102,7 +852,8 @@ var _ = Describe("[sriov] operator", func() { By("waiting for the node state to be updated") Eventually(func() sriovv1.Interfaces { - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), node, metav1.GetOptions{}) + nodeState := &sriovv1.SriovNetworkNodeState{} + err := clients.Get(context.Background(), runtimeclient.ObjectKey{Namespace: operatorNamespace, Name: node}, nodeState) Expect(err).ToNot(HaveOccurred()) return nodeState.Spec.Interfaces }, 1*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( @@ -1181,6 +932,7 @@ var _ = Describe("[sriov] operator", func() { assertObjectIsNotFound("network-resources-injector-role-binding", &rbacv1.ClusterRoleBinding{}) assertObjectIsNotFound("network-resources-injector-config", &admission.MutatingWebhookConfiguration{}) assertObjectIsNotFound("nri-control-switches", &corev1.ConfigMap{}) + assertObjectIsNotFound("network-resources-injector-allow-traffic-api-server", &networkv1.NetworkPolicy{}) }) It("SR-IOV Operator Config, disable Operator Webhook", func() { @@ -1201,6 +953,7 @@ var _ = Describe("[sriov] operator", func() { assertObjectIsNotFound("operator-webhook", &rbacv1.ClusterRole{}) assertObjectIsNotFound("operator-webhook-role-binding", &rbacv1.ClusterRoleBinding{}) assertObjectIsNotFound("sriov-operator-webhook-config", &admission.MutatingWebhookConfiguration{}) + assertObjectIsNotFound("operator-webhook-allow-traffic-api-server", &networkv1.NetworkPolicy{}) }) It("SR-IOV Operator Config, disable Resource Injector and Operator Webhook", func() { @@ -1300,7 +1053,7 @@ var _ = Describe("[sriov] operator", func() { }, Spec: sriovv1.SriovNetworkSpec{ ResourceName: resourceName, - IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181","routes":[{"dst":"0.0.0.0/0"}],"gateway":"10.10.10.1"}`, + IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181"}`, NetworkNamespace: namespaces.Test, }} @@ -1520,7 +1273,8 @@ var _ = Describe("[sriov] operator", func() { if len(sriovDeviceList) == 0 { return nil, false } - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), node, metav1.GetOptions{}) + nodeState := &sriovv1.SriovNetworkNodeState{} + err := clients.Get(context.Background(), runtimeclient.ObjectKey{Namespace: operatorNamespace, Name: node}, nodeState) Expect(err).ToNot(HaveOccurred()) for _, ifc := range nodeState.Spec.Interfaces { @@ -1592,7 +1346,7 @@ var _ = Describe("[sriov] operator", func() { }, Spec: sriovv1.SriovNetworkSpec{ ResourceName: resourceName, - IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181","routes":[{"dst":"0.0.0.0/0"}],"gateway":"10.10.10.1"}`, + IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181"}`, NetworkNamespace: namespaces.Test, }} @@ -1621,7 +1375,8 @@ var _ = Describe("[sriov] operator", func() { By("waiting for the mtu to be updated in the status") Eventually(func() sriovv1.InterfaceExts { - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), node, metav1.GetOptions{}) + nodeState := &sriovv1.SriovNetworkNodeState{} + err := clients.Get(context.Background(), runtimeclient.ObjectKey{Namespace: operatorNamespace, Name: node}, nodeState) Expect(err).ToNot(HaveOccurred()) return nodeState.Status.Interfaces }, 3*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( @@ -1635,7 +1390,8 @@ var _ = Describe("[sriov] operator", func() { By("waiting for the mtu to be restored") Eventually(func() sriovv1.InterfaceExts { - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), node, metav1.GetOptions{}) + nodeState := &sriovv1.SriovNetworkNodeState{} + err := clients.Get(context.Background(), runtimeclient.ObjectKey{Namespace: operatorNamespace, Name: node}, nodeState) Expect(err).ToNot(HaveOccurred()) return nodeState.Status.Interfaces }, 3*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( @@ -1656,7 +1412,8 @@ var _ = Describe("[sriov] operator", func() { By("waiting for the mtu to be updated in the status") Eventually(func() sriovv1.InterfaceExts { - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), node, metav1.GetOptions{}) + nodeState := &sriovv1.SriovNetworkNodeState{} + err := clients.Get(context.Background(), runtimeclient.ObjectKey{Namespace: operatorNamespace, Name: node}, nodeState) Expect(err).ToNot(HaveOccurred()) return nodeState.Status.Interfaces }, 3*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( @@ -1670,7 +1427,8 @@ var _ = Describe("[sriov] operator", func() { By("expecting the mtu to consistently stay at the new higher level") Consistently(func() sriovv1.InterfaceExts { - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), node, metav1.GetOptions{}) + nodeState := &sriovv1.SriovNetworkNodeState{} + err := clients.Get(context.Background(), runtimeclient.ObjectKey{Namespace: operatorNamespace, Name: node}, nodeState) Expect(err).ToNot(HaveOccurred()) return nodeState.Status.Interfaces }, 3*time.Minute, 15*time.Second).Should(ContainElement(MatchFields( @@ -1689,7 +1447,8 @@ var _ = Describe("[sriov] operator", func() { By("waiting for the mtu to be updated in the status") Eventually(func() sriovv1.InterfaceExts { - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), node, metav1.GetOptions{}) + nodeState := &sriovv1.SriovNetworkNodeState{} + err := clients.Get(context.Background(), runtimeclient.ObjectKey{Namespace: operatorNamespace, Name: node}, nodeState) Expect(err).ToNot(HaveOccurred()) return nodeState.Status.Interfaces }, 3*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( @@ -1703,7 +1462,8 @@ var _ = Describe("[sriov] operator", func() { By("expecting the mtu to consistently stay at the original level") Consistently(func() sriovv1.InterfaceExts { - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), node, metav1.GetOptions{}) + nodeState := &sriovv1.SriovNetworkNodeState{} + err := clients.Get(context.Background(), runtimeclient.ObjectKey{Namespace: operatorNamespace, Name: node}, nodeState) Expect(err).ToNot(HaveOccurred()) return nodeState.Status.Interfaces }, 3*time.Minute, 15*time.Second).Should(ContainElement(MatchFields( @@ -1717,6 +1477,57 @@ var _ = Describe("[sriov] operator", func() { }) }) + Context("Daemon reset with shutdown", Ordered, func() { + var testNode string + var policy *sriovv1.SriovNetworkNodePolicy + resourceName := "resetresource" + + BeforeAll(func() { + isSingleNode, err := cluster.IsSingleNode(clients) + Expect(err).ToNot(HaveOccurred()) + if isSingleNode { + Skip("test not supported for single node") + } + + sriovInfos, err := cluster.DiscoverSriov(clients, operatorNamespace) + Expect(err).ToNot(HaveOccurred()) + Expect(len(sriovInfos.Nodes)).ToNot(BeZero()) + testNode = sriovInfos.Nodes[0] + iface, err := sriovInfos.FindOneSriovDevice(testNode) + Expect(err).ToNot(HaveOccurred()) + + policy, err = network.CreateSriovPolicy(clients, "test-policy-", operatorNamespace, iface.Name, testNode, 5, resourceName, "netdevice") + Expect(err).ToNot(HaveOccurred()) + WaitForSRIOVStable() + }) + + It("should remove interface configuration from host even after reboot", func() { + By("force rebooting the node") + _, errOutput, err := runCommandOnConfigDaemon(testNode, "chroot", "/host", "reboot") + Expect(err).ToNot(HaveOccurred(), errOutput) + By("removing the policy") + err = clients.Delete(context.Background(), policy) + Expect(err).ToNot(HaveOccurred()) + + By("waiting for node to be not ready") + waitForNodeCondition(testNode, corev1.NodeReady, corev1.ConditionUnknown) + + By("waiting for node to be ready") + waitForNodeCondition(testNode, corev1.NodeReady, corev1.ConditionTrue) + + WaitForSRIOVStable() + By("Checking files on the host") + Eventually(func(g Gomega) { + output, errOutput, err := runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "ls /host/etc/sriov-operator/pci/ | wc -l") + g.Expect(err).ToNot(HaveOccurred(), errOutput) + g.Expect(strings.HasPrefix(output, "0")).Should(BeTrue()) + + output, errOutput, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "ls /host/etc/udev/rules.d/ | grep 10-nm-disable | wc -l") + g.Expect(err).ToNot(HaveOccurred(), errOutput) + g.Expect(strings.HasPrefix(output, "0")).Should(BeTrue()) + }, 3*time.Minute, 1*time.Second).Should(Succeed()) + }) + }) }) }) @@ -1769,7 +1580,8 @@ func discoverResourceForMainSriov(nodes *cluster.EnabledNodes) (*sriovv1.Interfa return nil, "", "", false } - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), node, metav1.GetOptions{}) + nodeState := &sriovv1.SriovNetworkNodeState{} + err = clients.Get(context.Background(), runtimeclient.ObjectKey{Namespace: operatorNamespace, Name: node}, nodeState) Expect(err).ToNot(HaveOccurred()) resourceName, ok := findSuitableResourceForMain(mainDevice, nodeState) if ok { @@ -1846,9 +1658,17 @@ func findUnusedSriovDevices(testNode string, sriovDevices []*sriovv1.InterfaceEx if isDefaultRouteInterface(device.Name, routes) { continue } - stdout, _, err = pod.ExecCommand(clients, createdPod, "ip", "link", "show", device.Name) - Expect(err).ToNot(HaveOccurred()) - Expect(len(stdout)).Should(Not(Equal(0)), "Unable to query link state") + stdout, stderr, err := pod.ExecCommand(clients, createdPod, "ip", "link", "show", device.Name) + if err != nil { + fmt.Printf("Can't query link state for device [%s]: %s", device.Name, err.Error()) + continue + } + + if len(stdout) == 0 { + fmt.Printf("Can't query link state for device [%s]: stderr:[%s]", device.Name, stderr) + continue + } + if strings.Contains(stdout, "master ovs-system") { continue // The interface is not active } @@ -2059,12 +1879,15 @@ func WaitForSRIOVStable() { time.Sleep((10 + snoTimeoutMultiplier*20) * time.Second) fmt.Println("Waiting for the sriov state to stable") - Eventually(func() bool { - // ignoring the error. This can eventually be executed against a single node cluster, - // and if a reconfiguration triggers a reboot then the api calls will return an error - res, _ := cluster.SriovStable(operatorNamespace, clients) - return res - }, waitingTime, 1*time.Second).Should(BeTrue()) + Eventually(func(g Gomega) { + res, err := cluster.SriovStable(operatorNamespace, clients) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(res).To(BeTrue()) + }, waitingTime, 1*time.Second).Should(Succeed(), func() string { + return "SR-IOV Operator is not stable" + + k8sreporter.SriovNetworkNodeStatesSummary(clients) + + k8sreporter.Events(clients, operatorNamespace) + }) fmt.Println("Sriov state is stable") Eventually(func() bool { @@ -2115,7 +1938,8 @@ func createVanillaNetworkPolicy(node string, sriovInfos *cluster.EnabledNodes, n }, 1*time.Minute, 5*time.Second).ShouldNot(HaveOccurred()) Eventually(func() sriovv1.Interfaces { - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), node, metav1.GetOptions{}) + nodeState := &sriovv1.SriovNetworkNodeState{} + err := clients.Get(context.Background(), runtimeclient.ObjectKey{Namespace: operatorNamespace, Name: node}, nodeState) Expect(err).ToNot(HaveOccurred()) return nodeState.Spec.Interfaces }, 1*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( @@ -2294,13 +2118,6 @@ func getOperatorLogs(since time.Time) []string { return strings.Split(string(rawLogs), "\n") } -func deleteOperatorPod() { - pod := getOperatorPod() - - err := clients.Pods(operatorNamespace).Delete(context.Background(), pod.Name, metav1.DeleteOptions{}) - ExpectWithOffset(1, err).ToNot(HaveOccurred()) -} - func assertObjectIsNotFound(name string, obj runtimeclient.Object) { Eventually(func() bool { err := clients.Get(context.Background(), runtimeclient.ObjectKey{Name: name, Namespace: operatorNamespace}, obj) @@ -2366,7 +2183,8 @@ func waitForPodRunning(p *corev1.Pod) *corev1.Pod { // assertNodeStateHasVFMatching asserts that the given node state has at least one VF matching the given fields func assertNodeStateHasVFMatching(nodeName string, fields Fields) { EventuallyWithOffset(1, func(g Gomega) sriovv1.InterfaceExts { - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), nodeName, metav1.GetOptions{}) + nodeState := &sriovv1.SriovNetworkNodeState{} + err := clients.Get(context.Background(), runtimeclient.ObjectKey{Namespace: operatorNamespace, Name: nodeName}, nodeState) g.Expect(err).ToNot(HaveOccurred()) g.Expect(nodeState.Status.SyncStatus).To(Equal("Succeeded")) return nodeState.Status.Interfaces @@ -2401,3 +2219,16 @@ func getInterfaceFromNodeStateByPciAddress(node, pciAddress string) *sriovv1.Int Expect(found).To(BeTrue()) return intf } + +func waitForNodeCondition(nodeName string, conditionType corev1.NodeConditionType, conditionStatus corev1.ConditionStatus) { + EventuallyWithOffset(1, func(g Gomega) bool { + node, err := clients.CoreV1Interface.Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + g.Expect(err).ToNot(HaveOccurred()) + for _, con := range node.Status.Conditions { + if con.Type == conditionType && con.Status == conditionStatus { + return true + } + } + return false + }, waitingTime, 1*time.Second).Should(BeTrue()) +} diff --git a/test/scripts/enable-kargs_test.sh b/test/scripts/kargs_test.sh similarity index 52% rename from test/scripts/enable-kargs_test.sh rename to test/scripts/kargs_test.sh index 615f3d2b2..3e191f230 100755 --- a/test/scripts/enable-kargs_test.sh +++ b/test/scripts/kargs_test.sh @@ -2,14 +2,15 @@ SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -SUT_SCRIPT="${SCRIPTPATH}/../../bindata/scripts/enable-kargs.sh" +SUT_SCRIPT="${SCRIPTPATH}/../../bindata/scripts/kargs.sh" test_RpmOstree_Add_All_Arguments() { + echo "ID=\"rhel\"" > ${FAKE_HOST}/etc/os-release echo "a b c=d eee=fff" > ${FAKE_HOST}/proc/cmdline touch ${FAKE_HOST}/run/ostree-booted - output=`$SUT_SCRIPT X=Y W=Z` + output=`$SUT_SCRIPT add X=Y W=Z` assertEquals 0 $? assertEquals "2" $output @@ -19,10 +20,11 @@ test_RpmOstree_Add_All_Arguments() { test_RpmOstree_Add_Only_Missing_Arguments() { + echo "ID=\"rhel\"" > ${FAKE_HOST}/etc/os-release echo "a b c=d eee=fff K=L" > ${FAKE_HOST}/proc/cmdline touch ${FAKE_HOST}/run/ostree-booted - output=`$SUT_SCRIPT K=L X=Y` + output=`$SUT_SCRIPT add K=L X=Y` assertEquals 0 $? assertEquals "1" $output @@ -30,6 +32,31 @@ test_RpmOstree_Add_Only_Missing_Arguments() { assertNotContains "`cat ${FAKE_HOST}/rpm-ostree_calls`" "--append K=L" } +test_RpmOstree_Delete_All_Arguments() { + echo "ID=\"rhel\"" > ${FAKE_HOST}/etc/os-release + echo "a b c=d eee=fff X=Y W=Z" > ${FAKE_HOST}/proc/cmdline + touch ${FAKE_HOST}/run/ostree-booted + + output=`$SUT_SCRIPT remove X=Y W=Z` + assertEquals 0 $? + assertEquals "2" $output + + assertContains "`cat ${FAKE_HOST}/rpm-ostree_calls`" "--delete X=Y" + assertContains "`cat ${FAKE_HOST}/rpm-ostree_calls`" "--delete W=Z" +} + +test_RpmOstree_Delete_Only_Exist_Arguments() { + echo "ID=\"rhel\"" > ${FAKE_HOST}/etc/os-release + echo "a b c=d eee=fff X=Y" > ${FAKE_HOST}/proc/cmdline + touch ${FAKE_HOST}/run/ostree-booted + + output=`$SUT_SCRIPT remove X=Y W=Z` + assertEquals 0 $? + assertEquals "1" $output + + assertContains "`cat ${FAKE_HOST}/rpm-ostree_calls`" "--delete X=Y" + assertContains "`cat ${FAKE_HOST}/rpm-ostree_calls`" "--delete W=Z" +} ###### Mock /host directory ###### export FAKE_HOST="$(mktemp -d)" @@ -40,13 +67,14 @@ setUp() { cp $(which cat) ${FAKE_HOST}/usr/bin/ cp $(which test) ${FAKE_HOST}/usr/bin/ cp $(which sh) ${FAKE_HOST}/usr/bin/ + cp $(which grep) ${FAKE_HOST}/usr/bin/ cp "$SCRIPTPATH/rpm-ostree_mock" ${FAKE_HOST}/usr/bin/rpm-ostree } # Mock chroot calls to the temporary test folder export real_chroot=$(which chroot) chroot() { - $real_chroot $FAKE_HOST ${@:2} + $real_chroot $FAKE_HOST "${@:2}" } export -f chroot diff --git a/test/scripts/rpm-ostree_mock b/test/scripts/rpm-ostree_mock index 16e816cc9..06e6b1905 100755 --- a/test/scripts/rpm-ostree_mock +++ b/test/scripts/rpm-ostree_mock @@ -5,7 +5,13 @@ # Write invocation with arguments to a file to allow making assertion. echo "$*" >> /rpm-ostree_calls -if [ "$*" != *"--append"* ] +if ! echo "$*" | grep -q "\--append" +then + # Caller is trying to read kernel arguments. + cat /proc/cmdline +fi + +if ! echo "$*" | grep -q "\--delete" then # Caller is trying to read kernel arguments. cat /proc/cmdline diff --git a/test/util/client/clients.go b/test/util/client/clients.go index 368b9d41b..1a8cc93b5 100644 --- a/test/util/client/clients.go +++ b/test/util/client/clients.go @@ -4,11 +4,10 @@ import ( "os" netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" - clientconfigv1 "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" - clientmachineconfigv1 "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned/typed/machineconfiguration.openshift.io/v1" + clientmachineconfigv1 "github.com/openshift/client-go/machineconfiguration/clientset/versioned/typed/machineconfiguration/v1" apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" - discovery "k8s.io/client-go/discovery" + "k8s.io/client-go/discovery" clientgoscheme "k8s.io/client-go/kubernetes/scheme" appsv1client "k8s.io/client-go/kubernetes/typed/apps/v1" coordinationv1 "k8s.io/client-go/kubernetes/typed/coordination/v1" @@ -21,7 +20,6 @@ import ( monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/typed/monitoring/v1" sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" - clientsriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned/typed/sriovnetwork/v1" snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" ) @@ -32,12 +30,10 @@ func init() { // ClientSet provides the struct to talk with relevant API type ClientSet struct { corev1client.CoreV1Interface - clientconfigv1.ConfigV1Interface clientmachineconfigv1.MachineconfigurationV1Interface appsv1client.AppsV1Interface discovery.DiscoveryInterface - clientsriovv1.SriovnetworkV1Interface Config *rest.Config runtimeclient.Client coordinationv1.CoordinationV1Interface @@ -67,11 +63,9 @@ func New(kubeconfig string) *ClientSet { clientSet := &ClientSet{} clientSet.CoreV1Interface = corev1client.NewForConfigOrDie(config) - clientSet.ConfigV1Interface = clientconfigv1.NewForConfigOrDie(config) clientSet.MachineconfigurationV1Interface = clientmachineconfigv1.NewForConfigOrDie(config) clientSet.AppsV1Interface = appsv1client.NewForConfigOrDie(config) clientSet.DiscoveryInterface = discovery.NewDiscoveryClientForConfigOrDie(config) - clientSet.SriovnetworkV1Interface = clientsriovv1.NewForConfigOrDie(config) clientSet.CoordinationV1Interface = coordinationv1.NewForConfigOrDie(config) clientSet.MonitoringV1Interface = monitoringv1.NewForConfigOrDie(config) clientSet.Config = config diff --git a/test/util/cluster/cluster.go b/test/util/cluster/cluster.go index 2d34c327a..5dcb05b5b 100644 --- a/test/util/cluster/cluster.go +++ b/test/util/cluster/cluster.go @@ -58,7 +58,8 @@ const NodeAndDeviceNameFilterEnvVar string = "SRIOV_NODE_AND_DEVICE_NAME_FILTER" // DiscoverSriov retrieves Sriov related information of a given cluster. func DiscoverSriov(clients *testclient.ClientSet, operatorNamespace string) (*EnabledNodes, error) { - nodeStates, err := clients.SriovNetworkNodeStates(operatorNamespace).List(context.Background(), metav1.ListOptions{}) + nodeStates := &sriovv1.SriovNetworkNodeStateList{} + err := clients.List(context.Background(), nodeStates, &runtimeclient.ListOptions{Namespace: operatorNamespace}) if err != nil { return nil, fmt.Errorf("failed to retrieve note states %v", err) } @@ -230,9 +231,14 @@ func (n *EnabledNodes) FindOneSriovNodeAndDevice() (string, *sriovv1.InterfaceEx // FindOneVfioSriovDevice retrieves a node with a valid sriov device for vfio func (n *EnabledNodes) FindOneVfioSriovDevice() (string, sriovv1.InterfaceExt) { for _, node := range n.Nodes { - for _, nic := range n.States[node].Status.Interfaces { + devices, err := n.FindSriovDevices(node) + if err != nil { + return "", sriovv1.InterfaceExt{} + } + + for _, nic := range devices { if nic.Vendor == intelVendorID && sriovv1.IsSupportedModel(nic.Vendor, nic.DeviceID) && nic.TotalVfs != 0 { - return node, nic + return node, *nic } } } @@ -241,29 +247,30 @@ func (n *EnabledNodes) FindOneVfioSriovDevice() (string, sriovv1.InterfaceExt) { // FindOneMellanoxSriovDevice retrieves a valid sriov device for the given node. func (n *EnabledNodes) FindOneMellanoxSriovDevice(node string) (*sriovv1.InterfaceExt, error) { - s, ok := n.States[node] - if !ok { - return nil, fmt.Errorf("node %s not found", node) - } - // return error here as mlx interfaces are not supported when secure boot is enabled // TODO: remove this when mlx support secure boot/lockdown mode if n.IsSecureBootEnabled[node] { return nil, fmt.Errorf("secure boot is enabled on the node mellanox cards are not supported") } - for _, itf := range s.Status.Interfaces { - if itf.Vendor == mlxVendorID && sriovv1.IsSupportedModel(itf.Vendor, itf.DeviceID) { - return &itf, nil + devices, err := n.FindSriovDevices(node) + if err != nil { + return nil, fmt.Errorf("unable to find a mellanox sriov devices in node %s: %w", node, err) + } + + for _, nic := range devices { + if nic.Vendor == mlxVendorID && sriovv1.IsSupportedModel(nic.Vendor, nic.DeviceID) { + return nic, nil } } - return nil, fmt.Errorf("unable to find a mellanox sriov devices in node %s", node) + return nil, fmt.Errorf("no Mellanox devices found in node %s. Available devices [%+v]", node, devices) } // SriovStable tells if all the node states are in sync (and the cluster is ready for another round of tests) func SriovStable(operatorNamespace string, clients *testclient.ClientSet) (bool, error) { - nodeStates, err := clients.SriovNetworkNodeStates(operatorNamespace).List(context.Background(), metav1.ListOptions{}) + nodeStates := &sriovv1.SriovNetworkNodeStateList{} + err := clients.List(context.Background(), nodeStates, &runtimeclient.ListOptions{Namespace: operatorNamespace}) switch err { case io.ErrUnexpectedEOF: return false, err @@ -401,13 +408,13 @@ func GetNodeSecureBootState(clients *testclient.ClientSet, nodeName, namespace s return false, err } - stdout, _, err := pod.ExecCommand(clients, runningPod, "cat", "/host/sys/kernel/security/lockdown") + stdout, stderr, err := pod.ExecCommand(clients, runningPod, "cat", "/host/sys/kernel/security/lockdown") - if strings.Contains(stdout, "No such file or directory") { + if strings.Contains(stderr, "No such file or directory") { return false, nil } if err != nil { - return false, err + return false, fmt.Errorf("secureboot check error stdout:[%s] stderr:[%s]: %w", stdout, stderr, err) } return strings.Contains(stdout, "[integrity]") || strings.Contains(stdout, "[confidentiality]"), nil diff --git a/test/util/crds/monitoring.coreos.com_prometheusrules.yaml b/test/util/crds/monitoring.coreos.com_prometheusrules.yaml new file mode 100644 index 000000000..6c16e8396 --- /dev/null +++ b/test/util/crds/monitoring.coreos.com_prometheusrules.yaml @@ -0,0 +1,142 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + operator.prometheus.io/version: 0.75.1 + name: prometheusrules.monitoring.coreos.com +spec: + group: monitoring.coreos.com + names: + categories: + - prometheus-operator + kind: PrometheusRule + listKind: PrometheusRuleList + plural: prometheusrules + shortNames: + - promrule + singular: prometheusrule + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: |- + The `PrometheusRule` custom resource definition (CRD) defines [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) and [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) rules to be evaluated by `Prometheus` or `ThanosRuler` objects. + + + `Prometheus` and `ThanosRuler` objects select `PrometheusRule` objects using label and namespace selectors. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of desired alerting rule definitions for Prometheus. + properties: + groups: + description: Content of Prometheus rule file + items: + description: RuleGroup is a list of sequentially evaluated recording + and alerting rules. + properties: + interval: + description: Interval determines how often rules in the group + are evaluated. + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + limit: + description: |- + Limit the number of alerts an alerting rule and series a recording + rule can produce. + Limit is supported starting with Prometheus >= 2.31 and Thanos Ruler >= 0.24. + type: integer + name: + description: Name of the rule group. + minLength: 1 + type: string + partial_response_strategy: + description: |- + PartialResponseStrategy is only used by ThanosRuler and will + be ignored by Prometheus instances. + More info: https://github.com/thanos-io/thanos/blob/main/docs/components/rule.md#partial-response + pattern: ^(?i)(abort|warn)?$ + type: string + rules: + description: List of alerting and recording rules. + items: + description: |- + Rule describes an alerting or recording rule + See Prometheus documentation: [alerting](https://www.prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) or [recording](https://www.prometheus.io/docs/prometheus/latest/configuration/recording_rules/#recording-rules) rule + properties: + alert: + description: |- + Name of the alert. Must be a valid label value. + Only one of `record` and `alert` must be set. + type: string + annotations: + additionalProperties: + type: string + description: |- + Annotations to add to each alert. + Only valid for alerting rules. + type: object + expr: + anyOf: + - type: integer + - type: string + description: PromQL expression to evaluate. + x-kubernetes-int-or-string: true + for: + description: Alerts are considered firing once they have + been returned for this long. + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + keep_firing_for: + description: KeepFiringFor defines how long an alert will + continue firing after the condition that triggered it + has cleared. + minLength: 1 + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + labels: + additionalProperties: + type: string + description: Labels to add or overwrite. + type: object + record: + description: |- + Name of the time series to output to. Must be a valid metric name. + Only one of `record` and `alert` must be set. + type: string + required: + - expr + type: object + type: array + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/test/util/helpers/helpers.go b/test/util/helpers/helpers.go index 35f1bf22f..ead34d977 100644 --- a/test/util/helpers/helpers.go +++ b/test/util/helpers/helpers.go @@ -32,3 +32,12 @@ func GinkgoAssertFileContentsEquals(path, expectedContent string) { ExpectWithOffset(1, err).NotTo(HaveOccurred()) ExpectWithOffset(1, string(d)).To(Equal(expectedContent)) } + +// GinkgoAssertFileDoesNotExist check that the file exist +// prepends vars.FilesystemRoot to the file path to be compatible with +// GinkgoConfigureFakeFS function +func GinkgoAssertFileDoesNotExist(path string) { + _, err := os.Stat(filepath.Join(vars.FilesystemRoot, path)) + ExpectWithOffset(1, err).To(HaveOccurred()) + ExpectWithOffset(1, os.IsNotExist(err)).To(BeTrue()) +} diff --git a/test/util/k8sreporter/descriptions.go b/test/util/k8sreporter/descriptions.go new file mode 100644 index 000000000..2a71cc0aa --- /dev/null +++ b/test/util/k8sreporter/descriptions.go @@ -0,0 +1,41 @@ +package k8sreporter + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" +) + +func SriovNetworkNodeStatesSummary(reader client.Reader) string { + ret := "SriovNetworkNodeStates:\n" + nodeStates := &sriovv1.SriovNetworkNodeStateList{} + err := reader.List(context.Background(), nodeStates, &client.ListOptions{}) + if err != nil { + return ret + "Summary error: " + err.Error() + } + + for _, state := range nodeStates.Items { + ret += fmt.Sprintf("%s\t%s\t%+v\n", state.Name, state.Status.SyncStatus, state.Annotations) + } + + return ret +} + +func Events(reader client.Reader, namespace string) string { + ret := fmt.Sprintf("Events in [%s]:\n", namespace) + events := &corev1.EventList{} + err := reader.List(context.Background(), events, &client.ListOptions{}) + if err != nil { + return ret + fmt.Sprintf("can't retrieve events for namespace %s: %s", namespace, err.Error()) + } + + for _, item := range events.Items { + ret += fmt.Sprintf("%s: %s\t%s\t%s\n", item.LastTimestamp, item.Reason, item.InvolvedObject.Name, item.Message) + } + + return ret +} diff --git a/test/util/k8sreporter/reporter.go b/test/util/k8sreporter/reporter.go index 5a3405a91..141c51309 100644 --- a/test/util/k8sreporter/reporter.go +++ b/test/util/k8sreporter/reporter.go @@ -6,6 +6,8 @@ import ( "strings" kniK8sReporter "github.com/openshift-kni/k8sreporter" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/runtime" sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" @@ -18,6 +20,17 @@ func New(reportPath string) (*kniK8sReporter.KubernetesReporter, error) { if err != nil { return err } + + err = monitoringv1.AddToScheme(s) + if err != nil { + return err + } + + err = rbacv1.AddToScheme(s) + if err != nil { + return err + } + return nil } @@ -38,6 +51,8 @@ func New(reportPath string) (*kniK8sReporter.KubernetesReporter, error) { return true case multusNamespace != "" && ns == multusNamespace: return true + case ns == "openshift-monitoring": + return true } return false } @@ -47,6 +62,11 @@ func New(reportPath string) (*kniK8sReporter.KubernetesReporter, error) { {Cr: &sriovv1.SriovNetworkNodePolicyList{}}, {Cr: &sriovv1.SriovNetworkList{}}, {Cr: &sriovv1.SriovOperatorConfigList{}}, + {Cr: &sriovv1.SriovNetworkPoolConfigList{}}, + {Cr: &monitoringv1.ServiceMonitorList{}, Namespace: &operatorNamespace}, + {Cr: &monitoringv1.PrometheusRuleList{}, Namespace: &operatorNamespace}, + {Cr: &rbacv1.RoleList{}, Namespace: &operatorNamespace}, + {Cr: &rbacv1.RoleBindingList{}, Namespace: &operatorNamespace}, } err := os.Mkdir(reportPath, 0755) diff --git a/test/util/namespaces/namespaces.go b/test/util/namespaces/namespaces.go index 5ed106398..ac6cfdcc0 100644 --- a/test/util/namespaces/namespaces.go +++ b/test/util/namespaces/namespaces.go @@ -129,6 +129,25 @@ func CleanNetworks(operatorNamespace string, cs *testclient.ClientSet) error { return waitForSriovNetworkDeletion(operatorNamespace, cs, 15*time.Second) } +func CleanPools(operatorNamespace string, cs *testclient.ClientSet) error { + pools := sriovv1.SriovNetworkPoolConfigList{} + err := cs.List(context.Background(), + &pools, + runtimeclient.InNamespace(operatorNamespace)) + if err != nil { + return err + } + for _, p := range pools.Items { + if strings.HasPrefix(p.Name, "test-") { + err := cs.Delete(context.Background(), &p) + if err != nil { + return fmt.Errorf("failed to delete networkPoolConfig %v", err) + } + } + } + return err +} + func waitForSriovNetworkDeletion(operatorNamespace string, cs *testclient.ClientSet, timeout time.Duration) error { return wait.PollImmediate(time.Second, timeout, func() (bool, error) { networks := sriovv1.SriovNetworkList{} @@ -164,7 +183,8 @@ func Clean(operatorNamespace, namespace string, cs *testclient.ClientSet, discov if err != nil { return err } - return nil + + return CleanPools(operatorNamespace, cs) } func AddLabel(cs corev1client.NamespacesGetter, ctx context.Context, namespaceName, key, value string) error { diff --git a/test/util/pod/pod.go b/test/util/pod/pod.go index 541eda1ad..c99ae00b2 100644 --- a/test/util/pod/pod.go +++ b/test/util/pod/pod.go @@ -117,7 +117,6 @@ func ExecCommand(cs *testclient.ClientSet, pod *corev1.Pod, command ...string) ( Command: command, Stdout: true, Stderr: true, - TTY: true, }, scheme.ParameterCodec) exec, err := remotecommand.NewSPDYExecutor(cs.Config, "POST", req.URL()) @@ -125,10 +124,9 @@ func ExecCommand(cs *testclient.ClientSet, pod *corev1.Pod, command ...string) ( return buf.String(), errbuf.String(), err } - err = exec.Stream(remotecommand.StreamOptions{ + err = exec.StreamWithContext(context.Background(), remotecommand.StreamOptions{ Stdout: &buf, Stderr: &errbuf, - Tty: true, }) if err != nil { return buf.String(), errbuf.String(), err diff --git a/test/util/util.go b/test/util/util.go index 5103af78f..dfc2f686b 100644 --- a/test/util/util.go +++ b/test/util/util.go @@ -4,10 +4,6 @@ import ( goctx "context" "encoding/json" "fmt" - "reflect" - - // "strings" - // "testing" "time" "github.com/google/uuid" @@ -16,6 +12,7 @@ import ( // "github.com/operator-framework/operator-sdk/pkg/test/e2eutil" appsv1 "k8s.io/api/apps/v1" // corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -229,7 +226,7 @@ func validateSelector(rc *dptypes.NetDeviceSelectors, ns *sriovnetworkv1.SriovNe } } if len(ns.PfNames) > 0 { - if !reflect.DeepEqual(ns.PfNames, rc.PfNames) { + if !equality.Semantic.DeepEqual(ns.PfNames, rc.PfNames) { return false } }