WIP update APIs

Signed-off-by: Eric Weber <eric.weber@suse.com>

k8s/pkg/apis/longhorn/v1beta2/volume.go

@@ -148,6 +148,15 @@ const (
 	ReplicaZoneSoftAntiAffinityDisabled = ReplicaZoneSoftAntiAffinity("disabled")
 )
 
+// +kubebuilder:validation:Enum=ignored;enabled;disabled
+type ReplicaDiskSoftAntiAffinity string
+
+const (
+	ReplicaDiskSoftAntiAffinityDefault  = ReplicaDiskSoftAntiAffinity("ignored")
+	ReplicaDiskSoftAntiAffinityEnabled  = ReplicaDiskSoftAntiAffinity("enabled")
+	ReplicaDiskSoftAntiAffinityDisabled = ReplicaDiskSoftAntiAffinity("disabled")
+)
+
 type BackendStoreDriverType string
 
 const (
@@ -231,12 +240,15 @@ type VolumeSpec struct {
 	RevisionCounterDisabled bool `json:"revisionCounterDisabled"`
 	// +optional
 	UnmapMarkSnapChainRemoved UnmapMarkSnapChainRemoved `json:"unmapMarkSnapChainRemoved"`
-	// Replica soft anti affinity of the volume. Set enabled to allow replicas to be scheduled on the same node
+	// Replica soft anti affinity of the volume. Set enabled to allow replicas to be scheduled on the same node.
 	// +optional
 	ReplicaSoftAntiAffinity ReplicaSoftAntiAffinity `json:"replicaSoftAntiAffinity"`
-	// Replica zone soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same zone
+	// Replica zone soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same zone.
 	// +optional
 	ReplicaZoneSoftAntiAffinity ReplicaZoneSoftAntiAffinity `json:"replicaZoneSoftAntiAffinity"`
+	// Replica disk soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same disk.
+	// +optional
+	ReplicaDiskSoftAntiAffinity ReplicaDiskSoftAntiAffinity `json:"replicaDiskSoftAntiAffinity"`
 	// +optional
 	LastAttachedBy string `json:"lastAttachedBy"`
 	// +optional

scheduler/replica_scheduler.go

@@ -164,6 +164,8 @@ func (rcs *ReplicaScheduler) getDiskCandidates(nodeInfo map[string]*longhorn.Nod
 		zoneSoftAntiAffinity = volume.Spec.ReplicaZoneSoftAntiAffinity == longhorn.ReplicaZoneSoftAntiAffinityEnabled
 	}
 
+	diskSoftAntiAffinity := true // TODO: Get this setting from somewhere.
+
 	getDiskCandidatesFromNodes := func(nodes map[string]*longhorn.Node) (diskCandidates map[string]*Disk, multiError util.MultiError) {
 		multiError = util.NewMultiError()
 		for _, node := range nodes {
@@ -224,7 +226,7 @@ func (rcs *ReplicaScheduler) getDiskCandidates(nodeInfo map[string]*longhorn.Nod
 	}
 
 	switch {
-	case !zoneSoftAntiAffinity && !nodeSoftAntiAffinity:
+	case !zoneSoftAntiAffinity && !nodeSoftAntiAffinity: // No change. We can't use two disks from the same node in this situation.
 		diskCandidates, errors := getDiskCandidatesFromNodes(unusedNodesInNewZones)
 		if len(diskCandidates) > 0 {
 			return diskCandidates, nil
@@ -235,7 +237,7 @@ func (rcs *ReplicaScheduler) getDiskCandidates(nodeInfo map[string]*longhorn.Nod
 			return diskCandidates, nil
 		}
 		multiError.Append(errors)
-	case zoneSoftAntiAffinity && !nodeSoftAntiAffinity:
+	case zoneSoftAntiAffinity && !nodeSoftAntiAffinity: // No change. We can't use two disks from the same node in this situation.
 		diskCandidates, errors := getDiskCandidatesFromNodes(unusedNodesInNewZones)
 		if len(diskCandidates) > 0 {
 			return diskCandidates, nil

types/setting.go

@@ -1103,6 +1103,16 @@ var (
 		ReadOnly:    true,
 		Default:     "1024",
 	}
+
+	SettingDefinitionReplicaDiskSoftAntiAffinity = SettingDefinition{
+		DisplayName: "Replica Disk Level Soft Anti-Affinity",
+		Description: "Allow scheduling on disks with existing healthy replicas of the same volume",
+		Category:    SettingCategoryScheduling,
+		Type:        SettingTypeBool,
+		Required:    true,
+		ReadOnly:    false,
+		Default:     "true",
+	}
 )
 
 type NodeDownPodDeletionPolicy string