Yutpeng/secondar vnic

pkg/cloudprovider/providers/oci/instances.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"strings"
 
 	"github.com/oracle/oci-go-sdk/v65/core"
 	"k8s.io/apimachinery/pkg/labels"
@@ -29,6 +30,12 @@ import (
 	cloudprovider "k8s.io/cloud-provider"
 )
 
+const (
+	OpenShiftTagNamesapcePrefix = "openshift-"
+	OpenShiftBootVolumeType     = "boot-volume-type"
+	OpenShiftBootVolumeISCSI    = "ISCSI"
+)
+
 var _ cloudprovider.Instances = &CloudProvider{}
 
 // mapNodeNameToInstanceName maps a kube NodeName to a OCI instance display
@@ -97,6 +104,38 @@ func (cp *CloudProvider) extractNodeAddresses(ctx context.Context, instanceID st
 		addresses = append(addresses, api.NodeAddress{Type: api.NodeExternalIP, Address: ip.String()})
 	}
 
+	OpenShiftTagNamesapce := cp.getOpenShiftTagNamespaceByInstance(ctx, instanceID)
+
+	if OpenShiftTagNamesapce != "" {
+		secondaryVnics, err := cp.client.Compute().GetSecondaryVNICsForInstance(ctx, compartmentID, instanceID)
+		if err != nil {
+			return nil, err
+		}
+
+		if secondaryVnics == nil || len(secondaryVnics) == 0 {
+			return addresses, nil
+		}
+		for _, secondaryVnic := range secondaryVnics {
+			if cp.checkOpenShiftISCSIBootVolumeTagByVnic(ctx, secondaryVnic, OpenShiftTagNamesapce) {
+				if (secondaryVnic.IsPrimary == nil || !*secondaryVnic.IsPrimary) && secondaryVnic.PrivateIp != nil && *secondaryVnic.PrivateIp != "" {
+					ip := net.ParseIP(*secondaryVnic.PrivateIp)
+					if ip == nil {
+						return nil, fmt.Errorf("instance has invalid private address: %q", *secondaryVnic.PrivateIp)
+					}
+					addresses = append(addresses, api.NodeAddress{Type: api.NodeInternalIP, Address: ip.String()})
+				}
+
+				if (secondaryVnic.IsPrimary == nil || !*secondaryVnic.IsPrimary) && secondaryVnic.PublicIp != nil && *secondaryVnic.PublicIp != "" {
+					ip := net.ParseIP(*secondaryVnic.PublicIp)
+					if ip == nil {
+						return nil, errors.Errorf("instance has invalid public address: %q", *secondaryVnic.PublicIp)
+					}
+					addresses = append(addresses, api.NodeAddress{Type: api.NodeExternalIP, Address: ip.String()})
+				}
+			}
+		}
+	}
+
 	// Changing this can have wide reaching impact.
 	//
 	// if vnic.HostnameLabel != nil && *vnic.HostnameLabel != "" {
@@ -312,3 +351,35 @@ func (cp *CloudProvider) getCompartmentIDByNodeName(nodeName string) (string, er
 	cp.logger.Debug("CompartmentID annotation is not present")
 	return "", errors.New("compartmentID annotation missing in the node. Would retry")
 }
+
+func (cp *CloudProvider) getOpenShiftTagNamespaceByInstance(ctx context.Context, instanceID string) string {
+	instance, err := cp.client.Compute().GetInstance(ctx, instanceID)
+	if err != nil {
+		return ""
+	}
+
+	if instance.DefinedTags == nil {
+		return ""
+	}
+
+	for namespace := range instance.DefinedTags {
+		if strings.HasPrefix(namespace, OpenShiftTagNamesapcePrefix) {
+			return namespace
+		}
+	}
+	return ""
+}
+
+func (cp *CloudProvider) checkOpenShiftISCSIBootVolumeTagByVnic(ctx context.Context, vnic *core.Vnic, namespace string) bool {
+	if vnic.DefinedTags == nil {
+		return false
+	}
+
+	if tags, namespaceExists := vnic.DefinedTags[namespace]; namespaceExists {
+		// Check if the boot volume type key exists and its value is ISCSI
+		if bootVolume, keyExists := tags[OpenShiftBootVolumeType]; keyExists && bootVolume == OpenShiftBootVolumeISCSI {
+			return true
+		}
+	}
+	return false
+}

pkg/cloudprovider/providers/oci/instances_test.go

@@ -344,6 +344,10 @@ func (MockComputeClient) GetPrimaryVNICForInstance(ctx context.Context, compartm
 	return instanceVnics[instanceID], nil
 }
 
+func (c *MockComputeClient) GetSecondaryVNICsForInstance(ctx context.Context, compartmentID, instanceID string) ([]*core.Vnic, error) {
+	return []*core.Vnic{instanceVnics[instanceID]}, nil
+}
+
 func (MockComputeClient) FindVolumeAttachment(ctx context.Context, compartmentID, volumeID string) (core.VolumeAttachment, error) {
 	return nil, nil
 }

pkg/csi/driver/bv_controller_test.go

@@ -568,6 +568,10 @@ func (c *MockComputeClient) GetPrimaryVNICForInstance(ctx context.Context, compa
 	return nil, nil
 }
 
+func (c *MockComputeClient) GetSecondaryVNICsForInstance(ctx context.Context, compartmentID, instanceID string) ([]*core.Vnic, error) {
+	return nil, nil
+}
+
 func (c *MockComputeClient) FindVolumeAttachment(ctx context.Context, compartmentID, volumeID string) (core.VolumeAttachment, error) {
 	var page *string
 	var requestMetadata common.RequestMetadata

pkg/oci/client/compute.go

@@ -16,6 +16,7 @@ package client
 
 import (
 	"context"
+	"log"
 	"strings"
 
 	"github.com/oracle/oci-go-sdk/v65/core"
@@ -34,6 +35,8 @@ type ComputeInterface interface {
 
 	GetPrimaryVNICForInstance(ctx context.Context, compartmentID, instanceID string) (*core.Vnic, error)
 
+	GetSecondaryVNICsForInstance(ctx context.Context, compartmentID, instanceID string) ([]*core.Vnic, error)
+
 	VolumeAttachmentInterface
 }
 
@@ -182,6 +185,55 @@ func (c *client) GetPrimaryVNICForInstance(ctx context.Context, compartmentID, i
 	return nil, errors.WithStack(errNotFound)
 }
 
+func (c *client) GetSecondaryVNICsForInstance(ctx context.Context, compartmentID, instanceID string) ([]*core.Vnic, error) {
+	logger := c.logger.With("instanceID", instanceID, "compartmentID", compartmentID)
+	secondaryVnics := []*core.Vnic{}
+	var page *string
+	for {
+		resp, err := c.listVNICAttachments(ctx, core.ListVnicAttachmentsRequest{
+			InstanceId:      &instanceID,
+			CompartmentId:   &compartmentID,
+			Page:            page,
+			RequestMetadata: c.requestMetadata,
+		})
+
+		if err != nil {
+			return nil, err
+		}
+
+		for _, attachment := range resp.Items {
+			if attachment.LifecycleState != core.VnicAttachmentLifecycleStateAttached {
+				logger.With("vnicAttachmentID", *attachment.Id).Info("VNIC attachment is not in attached state")
+				log.Println(logger)
+				continue
+			}
+
+			if attachment.VnicId == nil {
+				// Should never happen but lets be extra cautious as field is non-mandatory in OCI API.
+				logger.With("vnicAttachmentID", *attachment.Id).Error("VNIC attachment is attached but has no VNIC ID")
+				log.Println(logger)
+				continue
+			}
+
+			// TODO(apryde): Cache map[instanceID]primaryVNICID.
+			vnic, err := c.GetVNIC(ctx, *attachment.VnicId)
+			if err != nil {
+				return nil, err
+			}
+
+			if !*vnic.IsPrimary {
+				secondaryVnics = append(secondaryVnics, vnic)
+			}
+		}
+
+		if page = resp.OpcNextPage; resp.OpcNextPage == nil {
+			break
+		}
+	}
+
+	return secondaryVnics, nil
+}
+
 func (c *client) GetInstanceByNodeName(ctx context.Context, compartmentID, vcnID, nodeName string) (*core.Instance, error) {
 	// First try lookup by display name.
 	instance, err := c.getInstanceByDisplayName(ctx, compartmentID, nodeName)

pkg/volume/provisioner/block/block_test.go

@@ -246,6 +246,10 @@ func (c *MockComputeClient) GetPrimaryVNICForInstance(ctx context.Context, compa
 	return nil, nil
 }
 
+func (c *MockComputeClient) GetSecondaryVNICsForInstance(ctx context.Context, compartmentID, instanceID string) ([]*core.Vnic, error) {
+	return nil, nil
+}
+
 func (c *MockComputeClient) FindVolumeAttachment(ctx context.Context, compartmentID, volumeID string) (core.VolumeAttachment, error) {
 	return nil, nil
 }

pkg/volume/provisioner/fss/fss_test.go

@@ -244,6 +244,10 @@ func (c *MockComputeClient) GetPrimaryVNICForInstance(ctx context.Context, compa
 	return nil, nil
 }
 
+func (c *MockComputeClient) GetSecondaryVNICsForInstance(ctx context.Context, compartmentID, instanceID string) ([]*core.Vnic, error) {
+	return nil, nil
+}
+
 func (c *MockComputeClient) FindVolumeAttachment(ctx context.Context, compartmentID, volumeID string) (core.VolumeAttachment, error) {
 	return nil, nil
 }