Skip to content

Commit

Permalink
test: add usbdevice and usbdevice controller test case
Browse files Browse the repository at this point in the history
Signed-off-by: Jack Yu <jack.yu@suse.com>
  • Loading branch information
Yu-Jack committed May 24, 2024
1 parent 226bd97 commit 63ca2ec
Show file tree
Hide file tree
Showing 37 changed files with 3,081 additions and 33 deletions.
8 changes: 8 additions & 0 deletions pkg/codegen/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
controllergen "github.com/rancher/wrangler/pkg/controller-gen"
"github.com/rancher/wrangler/pkg/controller-gen/args"
_ "github.com/rancher/wrangler/pkg/generated/controllers/apiextensions.k8s.io/v1"
kubevirtv1 "kubevirt.io/api/core/v1"

"github.com/harvester/pcidevices/pkg/crd"
)
Expand All @@ -33,6 +34,13 @@ func main() {
GenerateTypes: true,
GenerateClients: true,
},
kubevirtv1.SchemeGroupVersion.Group: {
Types: []interface{}{
kubevirtv1.KubeVirt{},
},
GenerateTypes: false,
GenerateClients: true,
},
},
})
}
28 changes: 19 additions & 9 deletions pkg/config/factory_magement.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,23 +7,33 @@ import (
"kubevirt.io/client-go/kubecli"

ctldevices "github.com/harvester/pcidevices/pkg/generated/controllers/devices.harvesterhci.io"
ctlkubevirt "github.com/harvester/pcidevices/pkg/generated/controllers/kubevirt.io"
)

type FactoryManager struct {
DeviceFactory *ctldevices.Factory
CoreFactory *ctlcore.Factory
NetworkFactory *ctlnetwork.Factory
DeviceFactory *ctldevices.Factory
CoreFactory *ctlcore.Factory
NetworkFactory *ctlnetwork.Factory
KubevirtFactory *ctlkubevirt.Factory

KubevirtClient kubecli.KubevirtClient
Cfg *rest.Config
}

func NewFactoryManager(deviceFactory *ctldevices.Factory, coreFactory *ctlcore.Factory, networkFactory *ctlnetwork.Factory, kubevirtClient kubecli.KubevirtClient, cfg *rest.Config) *FactoryManager {
func NewFactoryManager(
deviceFactory *ctldevices.Factory,
coreFactory *ctlcore.Factory,
networkFactory *ctlnetwork.Factory,
kubevirtFactory *ctlkubevirt.Factory,
kubevirtClient kubecli.KubevirtClient,
cfg *rest.Config,
) *FactoryManager {
return &FactoryManager{
DeviceFactory: deviceFactory,
CoreFactory: coreFactory,
NetworkFactory: networkFactory,
KubevirtClient: kubevirtClient,
Cfg: cfg,
DeviceFactory: deviceFactory,
CoreFactory: coreFactory,
NetworkFactory: networkFactory,
KubevirtFactory: kubevirtFactory,
KubevirtClient: kubevirtClient,
Cfg: cfg,
}
}
2 changes: 1 addition & 1 deletion pkg/controller/nodes/node_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ func (h *handler) reconcileNodeDevices(name string, node *v1beta1.Node) (*v1beta
return nil, fmt.Errorf("error reconciling pcidevices for node %s: %v", h.nodeName, err)
}

usbHandler := usbdevice.NewHandler(h.usbCtl, h.usbClaimCtl, h.virtClient)
usbHandler := usbdevice.NewHandler(h.usbCtl, h.usbClaimCtl)
if err := usbHandler.ReconcileUSBDevices(); err != nil {
return nil, fmt.Errorf("error reconciling usb devices for node %s: %v", h.nodeName, err)
}
Expand Down
6 changes: 6 additions & 0 deletions pkg/controller/setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/harvester/pcidevices/pkg/controller/usbdevice"
"github.com/harvester/pcidevices/pkg/crd"
ctldevices "github.com/harvester/pcidevices/pkg/generated/controllers/devices.harvesterhci.io"
ctlkubevirt "github.com/harvester/pcidevices/pkg/generated/controllers/kubevirt.io"
"github.com/harvester/pcidevices/pkg/webhook"
)

Expand Down Expand Up @@ -75,13 +76,18 @@ func Setup(ctx context.Context, cfg *rest.Config, _ *runtime.Scheme) error {
return fmt.Errorf("error building network controllers: %v", err)
}

kubevirtFactory, err := ctlkubevirt.NewFactoryFromConfigWithOptions(cfg, &ctlkubevirt.FactoryOptions{
SharedControllerFactory: factory,
})

clientConfig := kubecli.DefaultClientConfig(&pflag.FlagSet{})
virtClient, err := kubecli.GetKubevirtClientFromClientConfig(clientConfig)

management := config.NewFactoryManager(
deviceFactory,
coreFactory,
networkFactory,
kubevirtFactory,
virtClient,
cfg,
)
Expand Down
2 changes: 1 addition & 1 deletion pkg/controller/usbdevice/label.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ type commonLabel struct {

var cl *commonLabel

func setupCommonLabels() {
func init() {
cl = &commonLabel{
nodeName: os.Getenv(v1beta1.NodeEnvVarName),
}
Expand Down
6 changes: 2 additions & 4 deletions pkg/controller/usbdevice/register.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,9 @@ const (
func Register(ctx context.Context, management *config.FactoryManager) error {
usbDeviceCtrl := management.DeviceFactory.Devices().V1beta1().USBDevice()
usbDeviceClaimCtrl := management.DeviceFactory.Devices().V1beta1().USBDeviceClaim()
virtClient := management.KubevirtClient
virtClient := management.KubevirtFactory.Kubevirt().V1().KubeVirt()

setupCommonLabels()

handler := NewHandler(usbDeviceCtrl, usbDeviceClaimCtrl, virtClient)
handler := NewHandler(usbDeviceCtrl, usbDeviceClaimCtrl)
usbDeviceClaimController := NewClaimHandler(usbDeviceCtrl.Cache(), usbDeviceClaimCtrl, usbDeviceCtrl, virtClient)

usbDeviceClaimCtrl.OnChange(ctx, "usbClaimClient-device-claim", usbDeviceClaimController.OnUSBDeviceClaimChanged)
Expand Down
18 changes: 9 additions & 9 deletions pkg/controller/usbdevice/usbdevice_claim_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,23 +9,23 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubevirtv1 "kubevirt.io/api/core/v1"
"kubevirt.io/client-go/kubecli"

"github.com/harvester/pcidevices/pkg/apis/devices.harvesterhci.io/v1beta1"
"github.com/harvester/pcidevices/pkg/deviceplugins"
ctldevicerv1beta1 "github.com/harvester/pcidevices/pkg/generated/controllers/devices.harvesterhci.io/v1beta1"
ctlkubevirtv1 "github.com/harvester/pcidevices/pkg/generated/controllers/kubevirt.io/v1"
)

type ClaimHandler struct {
usbClaimClient ctldevicerv1beta1.USBDeviceClaimController
usbClient ctldevicerv1beta1.USBDeviceController
virtClient kubecli.KubevirtClient
usbClaimClient ctldevicerv1beta1.USBDeviceClaimClient
usbClient ctldevicerv1beta1.USBDeviceClient
virtClient ctlkubevirtv1.KubeVirtClient
lock *sync.Mutex
usbDeviceCache ctldevicerv1beta1.USBDeviceCache
devicePlugin map[string]*deviceplugins.USBDevicePlugin
}

func NewClaimHandler(usbDeviceCache ctldevicerv1beta1.USBDeviceCache, usbClaimClient ctldevicerv1beta1.USBDeviceClaimController, usbClient ctldevicerv1beta1.USBDeviceController, virtClient kubecli.KubevirtClient) *ClaimHandler {
func NewClaimHandler(usbDeviceCache ctldevicerv1beta1.USBDeviceCache, usbClaimClient ctldevicerv1beta1.USBDeviceClaimClient, usbClient ctldevicerv1beta1.USBDeviceClient, virtClient ctlkubevirtv1.KubeVirtClient) *ClaimHandler {
return &ClaimHandler{
usbDeviceCache: usbDeviceCache,
usbClaimClient: usbClaimClient,
Expand Down Expand Up @@ -59,7 +59,7 @@ func (h *ClaimHandler) OnUSBDeviceClaimChanged(_ string, usbDeviceClaim *v1beta1
h.lock.Lock()
defer h.lock.Unlock()

virt, err := h.virtClient.KubeVirt(KubeVirtNamespace).Get(KubeVirtResource, &metav1.GetOptions{})
virt, err := h.virtClient.Get(KubeVirtNamespace, KubeVirtResource, metav1.GetOptions{})
if err != nil {
logrus.Errorf("failed to get kubevirt: %v", err)
return usbDeviceClaim, err
Expand Down Expand Up @@ -142,7 +142,7 @@ func (h *ClaimHandler) OnRemove(_ string, claim *v1beta1.USBDeviceClaim) (*v1bet
h.lock.Lock()
defer h.lock.Unlock()

virt, err := h.virtClient.KubeVirt(KubeVirtNamespace).Get(KubeVirtResource, &metav1.GetOptions{})
virt, err := h.virtClient.Get(KubeVirtNamespace, KubeVirtResource, metav1.GetOptions{})
if err != nil {
fmt.Println(err)
return nil, err
Expand All @@ -168,7 +168,7 @@ func (h *ClaimHandler) OnRemove(_ string, claim *v1beta1.USBDeviceClaim) (*v1bet
virtDp.Spec.Configuration.PermittedHostDevices.USB = usbs

if !reflect.DeepEqual(virt.Spec.Configuration.PermittedHostDevices.USB, virtDp.Spec.Configuration.PermittedHostDevices.USB) {
if _, err := h.virtClient.KubeVirt(KubeVirtNamespace).Update(virtDp); err != nil {
if _, err := h.virtClient.Update(virtDp); err != nil {
return claim, nil
}
}
Expand Down Expand Up @@ -222,7 +222,7 @@ func (h *ClaimHandler) updateKubeVirt(virt *kubevirtv1.KubeVirt, usbDevice *v1be
})

if virt.Spec.Configuration.PermittedHostDevices == nil || !reflect.DeepEqual(virt.Spec.Configuration.PermittedHostDevices.USB, virtDp.Spec.Configuration.PermittedHostDevices.USB) {
newVirt, err := h.virtClient.KubeVirt(KubeVirtNamespace).Update(virtDp)
newVirt, err := h.virtClient.Update(virtDp)
if err != nil {
logrus.Errorf("failed to update kubevirt: %v", err)
return virt, err
Expand Down
93 changes: 93 additions & 0 deletions pkg/controller/usbdevice/usbdevice_claim_controller_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package usbdevice

import (
"context"
"testing"

"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubevirtv1 "kubevirt.io/api/core/v1"

"github.com/harvester/pcidevices/pkg/apis/devices.harvesterhci.io/v1beta1"
"github.com/harvester/pcidevices/pkg/generated/clientset/versioned/fake"
"github.com/harvester/pcidevices/pkg/util/fakeclients"
)

var (
mockUsbDevice1 = &v1beta1.USBDevice{
ObjectMeta: metav1.ObjectMeta{
Name: "test-node-0951-1666-001002",
Labels: map[string]string{
"nodename": "test-node",
},
},
Status: v1beta1.USBDeviceStatus{
VendorID: "0951",
ProductID: "1666",
ResourceName: "kubevirt.io/test-node-0951-1666-001002",
DevicePath: "/dev/bus/usb/001/002",
NodeName: "test-node",
Description: "DataTraveler 100 G3/G4/SE9 G2/50 Kyson (Kingston Technology)",
},
}

mockKubeVirt = &kubevirtv1.KubeVirt{
ObjectMeta: metav1.ObjectMeta{
Name: "kubevirt",
Namespace: KubeVirtNamespace,
},
Spec: kubevirtv1.KubeVirtSpec{},
}

mockUsbDeviceClaim1 = &v1beta1.USBDeviceClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "test-node-0951-1666-001002",
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: "devices.harvesterhci.io/v1beta1",
Kind: "USBDevice",
Name: "test-node-0951-1666-001002",
UID: "test-uid",
},
},
},
Status: v1beta1.USBDeviceClaimStatus{
NodeName: "test-node",
},
}
)

func Test_OnUSBDeviceClaimChanged(t *testing.T) {
client := fake.NewSimpleClientset(mockUsbDevice1, mockUsbDeviceClaim1, mockKubeVirt)

handler := NewClaimHandler(
fakeclients.USBDeviceCache(client.DevicesV1beta1().USBDevices),
fakeclients.USBDeviceClaimsClient(client.DevicesV1beta1().USBDeviceClaims),
fakeclients.USBDevicesClient(client.DevicesV1beta1().USBDevices),
fakeclients.KubeVirtClient(client.KubevirtV1().KubeVirts),
)

_, err := handler.OnUSBDeviceClaimChanged("", mockUsbDeviceClaim1)
assert.NoError(t, err)

kubevirt, err := client.KubevirtV1().KubeVirts(mockKubeVirt.Namespace).Get(context.Background(), mockKubeVirt.Name, metav1.GetOptions{})
assert.NoError(t, err)
assert.Equal(t, kubevirtv1.KubeVirtSpec{
Configuration: kubevirtv1.KubeVirtConfiguration{
PermittedHostDevices: &kubevirtv1.PermittedHostDevices{
USB: []kubevirtv1.USBHostDevice{
{
ResourceName: "kubevirt.io/test-node-0951-1666-001002",
ExternalResourceProvider: true,
Selectors: []kubevirtv1.USBSelector{
{
Vendor: "0951",
Product: "1666",
},
},
},
},
},
},
}, kubevirt.Spec)
}
13 changes: 6 additions & 7 deletions pkg/controller/usbdevice/usbdevice_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"kubevirt.io/client-go/kubecli"

"github.com/harvester/pcidevices/pkg/apis/devices.harvesterhci.io/v1beta1"
"github.com/harvester/pcidevices/pkg/deviceplugins"
Expand All @@ -18,11 +17,12 @@ import (
)

type Handler struct {
usbClient ctldevicerv1vbeta1.USBDeviceController
usbClaimClient ctldevicerv1vbeta1.USBDeviceClaimController
virtClient kubecli.KubevirtClient
usbClient ctldevicerv1vbeta1.USBDeviceClient
usbClaimClient ctldevicerv1vbeta1.USBDeviceClaimClient
}

var walkUSBDevices = deviceplugins.WalkUSBDevices

type USBDevice struct {
Name string
Manufacturer string
Expand All @@ -39,11 +39,10 @@ func (dev *USBDevice) GetID() string {
return fmt.Sprintf("%04x:%04x-%02d:%02d", dev.Vendor, dev.Product, dev.Bus, dev.DeviceNumber)
}

func NewHandler(usbClient ctldevicerv1vbeta1.USBDeviceController, usbClaimClient ctldevicerv1vbeta1.USBDeviceClaimController, virtClient kubecli.KubevirtClient) *Handler {
func NewHandler(usbClient ctldevicerv1vbeta1.USBDeviceClient, usbClaimClient ctldevicerv1vbeta1.USBDeviceClaimClient) *Handler {
return &Handler{
usbClient: usbClient,
usbClaimClient: usbClaimClient,
virtClient: virtClient,
}
}

Expand Down Expand Up @@ -79,7 +78,7 @@ func (h *Handler) OnDeviceChange(_ string, _ string, obj runtime.Object) ([]rela
func (h *Handler) ReconcileUSBDevices() error {
nodeName := cl.nodeName

localUSBDevices, err := deviceplugins.WalkUSBDevices()
localUSBDevices, err := walkUSBDevices()
if err != nil {
logrus.Errorf("failed to walk USB devices: %v\n", err)
return err
Expand Down
69 changes: 69 additions & 0 deletions pkg/controller/usbdevice/usbdevice_controller_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package usbdevice

import (
"context"
"testing"

"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/harvester/pcidevices/pkg/deviceplugins"
"github.com/harvester/pcidevices/pkg/generated/clientset/versioned/fake"
"github.com/harvester/pcidevices/pkg/util/fakeclients"
)

var mockWalkUSBDevices = func() (map[int][]*deviceplugins.USBDevice, error) {
return map[int][]*deviceplugins.USBDevice{
0: {
{
Name: "test",
Manufacturer: "test",
Vendor: 2385,
Product: 5734,
DevicePath: "/dev/bus/usb/001/002",
PCIAddress: "0000:02:01.0",
},
},
}, nil
}

var mockCommonLabel = &commonLabel{
nodeName: "test-node",
}

func Test_ReconcileUSBDevices(t *testing.T) {
// detect one usb device, create a USBDevice CR
walkUSBDevices = mockWalkUSBDevices
cl = mockCommonLabel
client := fake.NewSimpleClientset()

usbHandler := NewHandler(
fakeclients.USBDevicesClient(client.DevicesV1beta1().USBDevices),
fakeclients.USBDeviceClaimsClient(client.DevicesV1beta1().USBDeviceClaims),
)

err := usbHandler.ReconcileUSBDevices()
assert.NoError(t, err)

list, err := client.DevicesV1beta1().USBDevices().List(context.Background(), metav1.ListOptions{})
assert.NoError(t, err)

assert.Equal(t, 1, len(list.Items))
assert.Equal(t, "test-node-0951-1666-001002", list.Items[0].Name)
assert.Equal(t, "0951", list.Items[0].Status.VendorID)
assert.Equal(t, "1666", list.Items[0].Status.ProductID)
assert.Equal(t, "kubevirt.io/test-node-0951-1666-001002", list.Items[0].Status.ResourceName)
assert.Equal(t, "/dev/bus/usb/001/002", list.Items[0].Status.DevicePath)
assert.Equal(t, cl.nodeName, list.Items[0].Status.NodeName)
assert.Equal(t, "DataTraveler 100 G3/G4/SE9 G2/50 Kyson (Kingston Technology)", list.Items[0].Status.Description)

// detect no usb device after few minutes, delete existing USBDevice CR
walkUSBDevices = func() (map[int][]*deviceplugins.USBDevice, error) { return map[int][]*deviceplugins.USBDevice{}, nil }

err = usbHandler.ReconcileUSBDevices()
assert.NoError(t, err)

list, err = client.DevicesV1beta1().USBDevices().List(context.Background(), metav1.ListOptions{})
assert.NoError(t, err)
assert.Equal(t, 0, len(list.Items))
}
Loading

0 comments on commit 63ca2ec

Please sign in to comment.