From a5b67e5ec57ec38be0d50cdbe77428d924f1b2fb Mon Sep 17 00:00:00 2001 From: Phan Le Date: Tue, 11 Jul 2023 12:37:49 -0700 Subject: [PATCH 1/3] Fix bug: if the snapshot is no longer in engine CR, don't block the removal process Longhorn-6298 Signed-off-by: Phan Le --- controller/snapshot_controller.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/controller/snapshot_controller.go b/controller/snapshot_controller.go index e46bd2cabf..86e1635766 100644 --- a/controller/snapshot_controller.go +++ b/controller/snapshot_controller.go @@ -335,7 +335,8 @@ func (sc *SnapshotController) reconcile(snapshotName string) (err error) { } }() - if _, ok := snapshot.Status.Children["volume-head"]; ok && snapshot.Status.MarkRemoved { + _, snapshotExistInEngineCR := engine.Status.Snapshots[snapshot.Name] + if _, hasVolumeHeadChild := snapshot.Status.Children["volume-head"]; snapshotExistInEngineCR && hasVolumeHeadChild && snapshot.Status.MarkRemoved { // This snapshot is the parent of volume-head, so it cannot be purged immediately. // We do not want to keep the volume stuck in attached state. return sc.handleAttachmentTicketDeletion(snapshot) From 1c5e1e2d4d3013daac21e9c8432f824b2b7d2be2 Mon Sep 17 00:00:00 2001 From: Phan Le Date: Mon, 26 Jun 2023 09:58:58 -0700 Subject: [PATCH 2/3] Fix volume_controller unit tests Longhorn-6005 Signed-off-by: Phan Le --- controller/volume_controller_test.go | 137 ++++++++++++++++----------- 1 file changed, 84 insertions(+), 53 deletions(-) diff --git a/controller/volume_controller_test.go b/controller/volume_controller_test.go index 56fb011c94..2a7ea166fe 100644 --- a/controller/volume_controller_test.go +++ b/controller/volume_controller_test.go @@ -82,9 +82,10 @@ type VolumeTestCase struct { expectEngines map[string]*longhorn.Engine expectReplicas map[string]*longhorn.Replica - replicaNodeSoftAntiAffinity string - volumeAutoSalvage string - replicaReplenishmentWaitInterval string + replicaNodeSoftAntiAffinity string + volumeAutoSalvage string + replicaReplenishmentWaitInterval string + allowVolumeCreationWithDegradedAvailability string } func (s *TestSuite) TestVolumeLifeCycle(c *C) { @@ -102,7 +103,6 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { // default replica and engine objects will be copied by copyCurrentToExpect tc.copyCurrentToExpect() tc.expectVolume.Status.State = longhorn.VolumeStateCreating - tc.expectVolume.Status.Robustness = longhorn.VolumeRobustnessUnknown tc.expectVolume.Status.CurrentImage = tc.volume.Spec.EngineImage tc.volume.Status.Conditions = []longhorn.Condition{} tc.engines = nil @@ -128,13 +128,24 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { } tc.expectVolume.Status.State = longhorn.VolumeStateCreating tc.expectVolume.Status.CurrentImage = tc.volume.Spec.EngineImage - tc.expectVolume.Status.Robustness = longhorn.VolumeRobustnessUnknown tc.expectVolume.Status.Conditions = setVolumeConditionWithoutTimestamp(tc.expectVolume.Status.Conditions, longhorn.VolumeConditionTypeScheduled, longhorn.ConditionStatusFalse, longhorn.VolumeConditionReasonReplicaSchedulingFailure, longhorn.ErrorReplicaScheduleNodeUnavailable) testCases["volume create - replica scheduling failure"] = tc + // detaching after creation + tc = generateVolumeTestCaseTemplate() + tc.volume.Status.State = longhorn.VolumeStateCreating + tc.copyCurrentToExpect() + tc.expectVolume.Status.Conditions = setVolumeConditionWithoutTimestamp(tc.volume.Status.Conditions, + longhorn.VolumeConditionTypeRestore, longhorn.ConditionStatusFalse, "", "") + tc.expectVolume.Status.State = longhorn.VolumeStateDetaching + tc.expectVolume.Status.CurrentImage = tc.volume.Spec.EngineImage + tc.expectVolume.Status.Robustness = longhorn.VolumeRobustnessUnknown + testCases["volume detaching after being created"] = tc + // after creation, volume in detached state tc = generateVolumeTestCaseTemplate() + tc.volume.Status.State = longhorn.VolumeStateDetaching for _, e := range tc.engines { e.Status.CurrentState = longhorn.InstanceStateStopped } @@ -142,6 +153,8 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { r.Status.CurrentState = longhorn.InstanceStateStopped } tc.copyCurrentToExpect() + tc.expectVolume.Status.Conditions = setVolumeConditionWithoutTimestamp(tc.volume.Status.Conditions, + longhorn.VolumeConditionTypeRestore, longhorn.ConditionStatusFalse, "", "") tc.expectVolume.Status.State = longhorn.VolumeStateDetached tc.expectVolume.Status.Robustness = longhorn.VolumeRobustnessUnknown tc.expectVolume.Status.CurrentImage = tc.volume.Spec.EngineImage @@ -153,10 +166,10 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { for _, r := range tc.replicas { r.Status.CurrentState = longhorn.InstanceStateStopped } + tc.volume.Status.State = longhorn.VolumeStateDetached tc.copyCurrentToExpect() tc.expectVolume.Status.State = longhorn.VolumeStateAttaching tc.expectVolume.Status.CurrentImage = tc.volume.Spec.EngineImage - tc.expectVolume.Status.CurrentNodeID = tc.volume.Spec.NodeID // replicas will be started first // engine will be started only after all the replicas are running for _, r := range tc.expectReplicas { @@ -174,12 +187,12 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { r.Status.StorageIP = r.Status.IP r.Status.Port = randomPort() } + tc.volume.Status.State = longhorn.VolumeStateAttaching tc.copyCurrentToExpect() tc.expectVolume.Status.State = longhorn.VolumeStateAttaching tc.expectVolume.Status.CurrentImage = tc.volume.Spec.EngineImage - tc.expectVolume.Status.CurrentNodeID = tc.volume.Spec.NodeID for _, e := range tc.expectEngines { - e.Spec.NodeID = tc.expectVolume.Status.CurrentNodeID + e.Spec.NodeID = tc.volume.Spec.NodeID e.Spec.DesireState = longhorn.InstanceStateRunning } for name, r := range tc.expectReplicas { @@ -193,6 +206,7 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { // volume attached tc = generateVolumeTestCaseTemplate() tc.volume.Spec.NodeID = TestNode1 + tc.volume.Status.State = longhorn.VolumeStateAttaching for _, e := range tc.engines { e.Spec.NodeID = tc.volume.Spec.NodeID @@ -227,17 +241,15 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { testCases["volume attached"] = tc tc = generateVolumeTestCaseTemplate() - tc.volume.Spec.NodeID = "" + tc.volume.Spec.NodeID = TestNode1 tc.volume.Spec.FromBackup = testBackupURL tc.volume.Spec.Standby = false - tc.volume.Spec.DisableFrontend = false - tc.volume.Status.CurrentNodeID = TestNode1 + tc.volume.Spec.DisableFrontend = true tc.volume.Status.State = longhorn.VolumeStateAttaching tc.volume.Status.CurrentImage = TestEngineImage tc.volume.Status.RestoreRequired = true tc.volume.Status.RestoreInitiated = true tc.volume.Status.LastBackup = TestBackupName - tc.volume.Status.FrontendDisabled = true tc.volume.Status.Conditions = setVolumeConditionWithoutTimestamp(tc.volume.Status.Conditions, longhorn.VolumeConditionTypeRestore, longhorn.ConditionStatusFalse, "", "") for _, e := range tc.engines { @@ -256,6 +268,7 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { r.Status.Port = randomPort() } tc.copyCurrentToExpect() + tc.expectVolume.Status.FrontendDisabled = true for _, e := range tc.expectEngines { e.Spec.NodeID = TestNode1 e.Spec.DesireState = longhorn.InstanceStateRunning @@ -274,15 +287,13 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { // Newly restored volume changed from attaching to attached tc = generateVolumeTestCaseTemplate() - tc.volume.Spec.NodeID = "" + tc.volume.Spec.NodeID = TestNode1 tc.volume.Spec.FromBackup = testBackupURL tc.volume.Spec.Standby = false - tc.volume.Spec.DisableFrontend = false - tc.volume.Status.CurrentNodeID = TestNode1 + tc.volume.Spec.DisableFrontend = true tc.volume.Status.State = longhorn.VolumeStateAttaching tc.volume.Status.LastBackup = TestBackupName tc.volume.Status.CurrentImage = TestEngineImage - tc.volume.Status.FrontendDisabled = true tc.volume.Status.RestoreRequired = true tc.volume.Status.RestoreInitiated = true tc.volume.Status.Conditions = setVolumeConditionWithoutTimestamp(tc.volume.Status.Conditions, @@ -316,7 +327,9 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { } } tc.copyCurrentToExpect() + tc.expectVolume.Status.FrontendDisabled = true tc.expectVolume.Status.State = longhorn.VolumeStateAttached + tc.expectVolume.Status.CurrentNodeID = TestNode1 tc.expectVolume.Status.Robustness = longhorn.VolumeRobustnessHealthy tc.expectVolume.Status.Conditions = setVolumeConditionWithoutTimestamp(tc.volume.Status.Conditions, longhorn.VolumeConditionTypeRestore, longhorn.ConditionStatusTrue, longhorn.VolumeConditionReasonRestoreInProgress, "") @@ -324,10 +337,10 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { // Newly restored volume is waiting for restoration completed tc = generateVolumeTestCaseTemplate() - tc.volume.Spec.NodeID = "" + tc.volume.Spec.NodeID = TestNode1 tc.volume.Spec.FromBackup = testBackupURL tc.volume.Spec.Standby = false - tc.volume.Spec.DisableFrontend = false + tc.volume.Spec.DisableFrontend = true tc.volume.Status.CurrentNodeID = TestNode1 tc.volume.Status.OwnerID = TestNode1 tc.volume.Status.State = longhorn.VolumeStateAttached @@ -374,7 +387,7 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { tc.volume.Spec.NodeID = "" tc.volume.Spec.FromBackup = testBackupURL tc.volume.Spec.Standby = false - tc.volume.Spec.DisableFrontend = false + tc.volume.Spec.DisableFrontend = true tc.volume.Status.OwnerID = TestNode1 tc.volume.Status.State = longhorn.VolumeStateAttached tc.volume.Status.Robustness = longhorn.VolumeRobustnessHealthy @@ -420,11 +433,11 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { e.Spec.RequestedBackupRestore = "" } tc.expectVolume.Spec.NodeID = "" - tc.expectVolume.Spec.DisableFrontend = false - tc.expectVolume.Status.CurrentNodeID = "" + tc.expectVolume.Spec.DisableFrontend = true + tc.expectVolume.Status.CurrentNodeID = TestNode1 tc.expectVolume.Status.State = longhorn.VolumeStateDetaching tc.expectVolume.Status.Robustness = longhorn.VolumeRobustnessUnknown - tc.expectVolume.Status.FrontendDisabled = false + tc.expectVolume.Status.FrontendDisabled = true tc.expectVolume.Status.Conditions = setVolumeConditionWithoutTimestamp(tc.expectVolume.Status.Conditions, longhorn.VolumeConditionTypeRestore, longhorn.ConditionStatusFalse, "", "") for _, r := range tc.expectReplicas { @@ -437,13 +450,13 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { tc.volume.Spec.NodeID = "" tc.volume.Spec.FromBackup = testBackupURL tc.volume.Spec.Standby = false - tc.volume.Spec.DisableFrontend = false + tc.volume.Spec.DisableFrontend = true tc.volume.Status.OwnerID = TestNode1 tc.volume.Status.State = longhorn.VolumeStateDetaching tc.volume.Status.Robustness = longhorn.VolumeRobustnessUnknown - tc.volume.Status.CurrentNodeID = "" + tc.volume.Status.CurrentNodeID = TestNode1 tc.volume.Status.CurrentImage = TestEngineImage - tc.volume.Status.FrontendDisabled = false + tc.volume.Status.FrontendDisabled = true tc.volume.Status.RestoreInitiated = true tc.volume.Status.LastBackup = TestBackupName tc.volume.Status.Conditions = setVolumeConditionWithoutTimestamp(tc.volume.Status.Conditions, @@ -476,8 +489,8 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { } tc.copyCurrentToExpect() tc.expectVolume.Spec.NodeID = "" - tc.expectVolume.Spec.DisableFrontend = false - tc.expectVolume.Status.CurrentNodeID = "" + tc.expectVolume.Spec.DisableFrontend = true + tc.expectVolume.Status.CurrentNodeID = TestNode1 tc.expectVolume.Status.State = longhorn.VolumeStateDetaching tc.expectVolume.Status.Robustness = longhorn.VolumeRobustnessUnknown for _, r := range tc.expectReplicas { @@ -486,10 +499,10 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { testCases["newly restored volume is being detaching after restoration completed"] = tc tc = generateVolumeTestCaseTemplate() - tc.volume.Spec.NodeID = "" + tc.volume.Spec.NodeID = TestNode1 tc.volume.Spec.FromBackup = testBackupURL tc.volume.Spec.Standby = false - tc.volume.Spec.DisableFrontend = false + tc.volume.Spec.DisableFrontend = true tc.volume.Spec.EngineImage = TestEngineImage tc.volume.Status.OwnerID = TestNode1 tc.volume.Status.State = longhorn.VolumeStateAttached @@ -546,6 +559,7 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { } } } + tc.allowVolumeCreationWithDegradedAvailability = "false" tc.copyCurrentToExpect() newReplicaMap := map[string]*longhorn.Replica{} for name, r := range tc.replicas { @@ -567,7 +581,7 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { tc.volume.Spec.NodeID = "" tc.volume.Spec.FromBackup = testBackupURL tc.volume.Spec.Standby = true - tc.volume.Spec.DisableFrontend = false + tc.volume.Spec.DisableFrontend = true tc.volume.Status.CurrentNodeID = TestNode1 tc.volume.Status.FrontendDisabled = true tc.volume.Status.OwnerID = TestNode1 @@ -575,6 +589,7 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { tc.volume.Status.Robustness = longhorn.VolumeRobustnessHealthy tc.volume.Status.CurrentImage = TestEngineImage tc.volume.Status.RestoreInitiated = true + tc.volume.Status.RestoreRequired = true tc.volume.Status.IsStandby = true tc.volume.Status.LastBackup = TestBackupName tc.volume.Status.Conditions = setVolumeConditionWithoutTimestamp(tc.volume.Status.Conditions, @@ -614,8 +629,8 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { } tc.copyCurrentToExpect() tc.expectVolume.Spec.NodeID = "" - tc.expectVolume.Spec.DisableFrontend = false - tc.expectVolume.Status.CurrentNodeID = "" + tc.expectVolume.Spec.DisableFrontend = true + tc.expectVolume.Status.CurrentNodeID = TestNode1 tc.expectVolume.Status.State = longhorn.VolumeStateDetaching tc.expectVolume.Status.FrontendDisabled = true tc.expectVolume.Status.Robustness = longhorn.VolumeRobustnessFaulted @@ -635,11 +650,11 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { testCases["newly restored volume becomes faulted after all replica error"] = tc tc = generateVolumeTestCaseTemplate() - tc.volume.Spec.NodeID = "" + tc.volume.Spec.NodeID = TestNode1 tc.volume.Spec.FromBackup = testBackupURL tc.volume.Spec.Standby = true - tc.volume.Spec.DisableFrontend = false - tc.volume.Status.CurrentNodeID = TestNode1 + tc.volume.Spec.DisableFrontend = true + tc.volume.Status.CurrentNodeID = "" tc.volume.Status.State = longhorn.VolumeStateAttaching tc.volume.Status.CurrentImage = TestEngineImage tc.volume.Status.FrontendDisabled = true @@ -668,12 +683,12 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { // New standby volume changed from attaching to attached, and it's not automatically detached tc = generateVolumeTestCaseTemplate() - tc.volume.Spec.NodeID = "" + tc.volume.Spec.NodeID = TestNode1 tc.volume.Spec.FromBackup = testBackupURL tc.volume.Spec.Standby = true - tc.volume.Spec.DisableFrontend = false + tc.volume.Spec.DisableFrontend = true tc.volume.Status.CurrentNodeID = TestNode1 - tc.volume.Status.State = longhorn.VolumeStateAttaching + tc.volume.Status.State = longhorn.VolumeStateAttached tc.volume.Status.CurrentImage = TestEngineImage tc.volume.Status.LastBackup = TestBackupName tc.volume.Status.FrontendDisabled = true @@ -716,7 +731,8 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { // volume detaching - stop engine tc = generateVolumeTestCaseTemplate() tc.volume.Spec.NodeID = "" - tc.volume.Status.CurrentNodeID = "" + tc.volume.Status.CurrentNodeID = TestNode1 + tc.volume.Status.State = longhorn.VolumeStateAttached for _, e := range tc.engines { e.Spec.NodeID = TestNode1 e.Spec.DesireState = longhorn.InstanceStateRunning @@ -744,6 +760,8 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { tc.expectVolume.Status.State = longhorn.VolumeStateDetaching tc.expectVolume.Status.Robustness = longhorn.VolumeRobustnessUnknown tc.expectVolume.Status.CurrentImage = tc.volume.Spec.EngineImage + tc.expectVolume.Status.Conditions = setVolumeConditionWithoutTimestamp(tc.volume.Status.Conditions, + longhorn.VolumeConditionTypeRestore, longhorn.ConditionStatusFalse, "", "") for _, e := range tc.expectEngines { e.Spec.NodeID = "" e.Spec.DesireState = longhorn.InstanceStateStopped @@ -753,7 +771,8 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { // volume detaching - stop replicas tc = generateVolumeTestCaseTemplate() tc.volume.Spec.NodeID = "" - tc.volume.Status.CurrentNodeID = "" + tc.volume.Status.CurrentNodeID = TestNode1 + tc.volume.Status.State = longhorn.VolumeStateAttached for _, e := range tc.engines { e.Spec.NodeID = "" e.Status.CurrentState = longhorn.InstanceStateStopped @@ -774,6 +793,8 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { tc.expectVolume.Status.State = longhorn.VolumeStateDetaching tc.expectVolume.Status.Robustness = longhorn.VolumeRobustnessUnknown tc.expectVolume.Status.CurrentImage = tc.volume.Spec.EngineImage + tc.expectVolume.Status.Conditions = setVolumeConditionWithoutTimestamp(tc.volume.Status.Conditions, + longhorn.VolumeConditionTypeRestore, longhorn.ConditionStatusFalse, "", "") for _, r := range tc.expectReplicas { r.Spec.DesireState = longhorn.InstanceStateStopped } @@ -793,6 +814,7 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { // volume attaching, start replicas, one node down tc = generateVolumeTestCaseTemplate() tc.volume.Spec.NodeID = TestNode1 + tc.volume.Status.State = longhorn.VolumeStateDetached tc.volume.Spec.StaleReplicaTimeout = 1<<24 - 1 tc.nodes[1] = newNode(TestNode2, TestNamespace, false, longhorn.ConditionStatusFalse, string(longhorn.NodeConditionReasonKubernetesNodeGone)) for _, r := range tc.replicas { @@ -801,7 +823,7 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { r.Status.CurrentState = longhorn.InstanceStateStopped } tc.copyCurrentToExpect() - tc.expectVolume.Status.CurrentNodeID = TestNode1 + tc.expectVolume.Status.CurrentNodeID = "" tc.expectVolume.Status.State = longhorn.VolumeStateAttaching tc.expectVolume.Status.CurrentImage = tc.volume.Spec.EngineImage for _, r := range tc.expectReplicas { @@ -818,7 +840,7 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { tc = generateVolumeTestCaseTemplate() tc.volume.Spec.NodeID = TestNode1 tc.volume.Status.OwnerID = TestNode1 - tc.volume.Status.CurrentNodeID = TestNode1 + tc.volume.Status.CurrentNodeID = "" // Since default node 2 is scheduling disabled, // enable node soft anti-affinity for 2 replicas. @@ -828,9 +850,9 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { tc.engines = nil tc.replicas = nil - tc.expectVolume.Status.State = longhorn.VolumeStateAttaching + tc.expectVolume.Status.State = longhorn.VolumeStateCreating tc.expectVolume.Status.CurrentImage = tc.volume.Spec.EngineImage - tc.expectVolume.Status.CurrentNodeID = tc.volume.Spec.NodeID + tc.expectVolume.Status.CurrentNodeID = "" expectEngines := map[string]*longhorn.Engine{} for _, e := range tc.expectEngines { e.Spec.RevisionCounterDisabled = true @@ -849,7 +871,8 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { // Salvage Requested tc = generateVolumeTestCaseTemplate() - tc.volume.Spec.NodeID = TestNode1 + // volume is Faulted and the VA controller already unset spec.NodeID for this volume + tc.volume.Spec.NodeID = "" tc.volume.Spec.RevisionCounterDisabled = true tc.volume.Status.State = longhorn.VolumeStateDetached tc.volume.Status.Robustness = longhorn.VolumeRobustnessFaulted @@ -888,7 +911,7 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { tc.expectVolume.Status.State = longhorn.VolumeStateDetached tc.expectVolume.Status.CurrentImage = tc.volume.Spec.EngineImage - tc.expectVolume.Status.CurrentNodeID = TestNode1 + tc.expectVolume.Status.CurrentNodeID = "" tc.expectVolume.Status.PendingNodeID = "" tc.expectVolume.Status.Robustness = longhorn.VolumeRobustnessUnknown tc.expectVolume.Status.RemountRequestedAt = getTestNow() @@ -898,7 +921,8 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { // volume attaching, start replicas, manager restart tc = generateVolumeTestCaseTemplate() tc.volume.Spec.NodeID = TestNode1 - tc.volume.Status.CurrentNodeID = TestNode1 + tc.volume.Status.CurrentNodeID = "" + tc.volume.Status.State = longhorn.VolumeStateDetached tc.nodes[1] = newNode(TestNode2, TestNamespace, false, longhorn.ConditionStatusFalse, string(longhorn.NodeConditionReasonManagerPodDown)) for _, r := range tc.replicas { r.Status.CurrentState = longhorn.InstanceStateStopped @@ -913,7 +937,6 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { } tc.expectReplicas = expectRs testCases["volume attaching - start replicas - manager down"] = tc - //s.runTestCases(c, testCases) // restoring volume reattaching, stop replicas tc = generateVolumeTestCaseTemplate() @@ -940,6 +963,8 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { tc.expectVolume.Status.State = longhorn.VolumeStateDetaching tc.expectVolume.Status.Robustness = longhorn.VolumeRobustnessUnknown tc.expectVolume.Status.CurrentImage = tc.volume.Spec.EngineImage + tc.expectVolume.Status.Conditions = setVolumeConditionWithoutTimestamp(tc.volume.Status.Conditions, + longhorn.VolumeConditionTypeRestore, longhorn.ConditionStatusFalse, "", "") for _, r := range tc.expectReplicas { r.Spec.DesireState = longhorn.InstanceStateStopped } @@ -1041,7 +1066,7 @@ func (s *TestSuite) TestVolumeLifeCycle(c *C) { tc.copyCurrentToExpect() testCases["replica rebuilding - delay replica replenishment"] = tc - //s.runTestCases(c, testCases) + s.runTestCases(c, testCases) } func newVolume(name string, replicaCount int) *longhorn.Volume { @@ -1071,10 +1096,6 @@ func newVolume(name string, replicaCount int) *longhorn.Volume { Type: string(longhorn.VolumeConditionTypeScheduled), Status: longhorn.ConditionStatusTrue, }, - { - Type: string(longhorn.VolumeConditionTypeRestore), - Status: longhorn.ConditionStatusFalse, - }, }, }, } @@ -1392,6 +1413,16 @@ func (s *TestSuite) runTestCases(c *C, testCases map[string]*VolumeTestCase) { c.Assert(err, IsNil) } + // Set allow-volume-creation-with-degraded-availability setting + if tc.allowVolumeCreationWithDegradedAvailability != "" { + s := initSettingsNameValue( + string(types.SettingNameAllowVolumeCreationWithDegradedAvailability), + tc.allowVolumeCreationWithDegradedAvailability) + setting, err := + lhClient.LonghornV1beta2().Settings(TestNamespace).Create(context.TODO(), s, metav1.CreateOptions{}) + c.Assert(err, IsNil) + sIndexer.Add(setting) + } // Set replica node soft anti-affinity setting if tc.replicaNodeSoftAntiAffinity != "" { s := initSettingsNameValue( From c59efbfb9b2d05fbb086c3ee9ddaa10c0a7908d0 Mon Sep 17 00:00:00 2001 From: Phan Le Date: Thu, 6 Jul 2023 20:41:18 -0700 Subject: [PATCH 3/3] Add unit tests for volumeattachment_controller Longhorn-6005 Signed-off-by: Phan Le --- .../volume_attachment_controller_test.go | 896 ++++++++++-------- 1 file changed, 498 insertions(+), 398 deletions(-) diff --git a/controller/volume_attachment_controller_test.go b/controller/volume_attachment_controller_test.go index b0da3c8ef1..9f8334af64 100644 --- a/controller/volume_attachment_controller_test.go +++ b/controller/volume_attachment_controller_test.go @@ -1,400 +1,500 @@ package controller -// -//import ( -// "context" -// "fmt" -// "github.com/longhorn/longhorn-manager/datastore" -// longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" -// lhfake "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned/fake" -// lhinformerfactory "github.com/longhorn/longhorn-manager/k8s/pkg/client/informers/externalversions" -// "github.com/sirupsen/logrus" -// . "gopkg.in/check.v1" -// apiextensionsfake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" -// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// "k8s.io/client-go/informers" -// "k8s.io/client-go/kubernetes/fake" -// "k8s.io/client-go/kubernetes/scheme" -// "k8s.io/client-go/tools/record" -// utilpointer "k8s.io/utils/pointer" -//) -// -//type volumeAttachmentTestCase struct { -// volAttachment *longhorn.VolumeAttachment -// vol *longhorn.Volume -// -// expectedVolAttachment *longhorn.VolumeAttachment -// expectedVol *longhorn.Volume -//} -// -//func (tc *volumeAttachmentTestCase) copyCurrentToExpect() { -// tc.expectedVolAttachment = tc.volAttachment.DeepCopy() -// tc.expectedVol = tc.vol.DeepCopy() -//} -// -//func (s *TestSuite) TestVolumeAttachmentLifeCycle(c *C) { -// var tc *volumeAttachmentTestCase -// testCases := map[string]*volumeAttachmentTestCase{} -// -// /////////////////////////////////////////////////////////////////// -// tc = generateVolumeAttachmentTestCaseTemplate(TestVolumeName) -// tc.volAttachment.Spec.AttachmentSpecs = map[string]*longhorn.AttachmentSpec{ -// "attachment-01": &longhorn.AttachmentSpec{ -// ID: "attachment-01", -// Type: longhorn.AttacherTypeCSIAttacher, -// NodeID: TestNode1, -// Parameters: map[string]string{}, -// Generation: 0, -// }, -// } -// tc.vol.Status.OwnerID = TestNode1 -// tc.vol.Status.State = longhorn.VolumeStateDetached -// tc.copyCurrentToExpect() -// tc.expectedVolAttachment.Status.AttachmentStatuses = map[string]*longhorn.AttachmentStatus{ -// "attachment-01": &longhorn.AttachmentStatus{ -// ID: "attachment-01", -// Satisfied: false, -// Generation: 0, -// }, -// } -// tc.expectedVol.Spec.NodeID = TestNode1 -// testCases["test case 1: attach: basic"] = tc -// /////////////////////////////////////////////////////////////////// -// -// /////////////////////////////////////////////////////////////////// -// tc = generateVolumeAttachmentTestCaseTemplate(TestVolumeName) -// tc.volAttachment.Spec.Attachments = map[string]*longhorn.Attachment{ -// "attachment-01": &longhorn.Attachment{ -// ID: "attachment-01", -// Type: longhorn.AttacherTypeCSIAttacher, -// NodeID: TestNode1, -// Parameters: map[string]string{}, -// }, -// "attachment-02": &longhorn.Attachment{ -// ID: "attachment-02", -// Type: longhorn.AttacherTypeSnapshotController, -// NodeID: TestNode2, -// Parameters: map[string]string{}, -// }, -// } -// tc.vol.Status.OwnerID = TestNode1 -// tc.vol.Status.State = longhorn.VolumeStateDetached -// tc.copyCurrentToExpect() -// tc.expectedVolAttachment.Status.Attachments = map[string]*longhorn.Attachment{ -// "attachment-01": &longhorn.Attachment{ -// ID: "attachment-01", -// Type: longhorn.AttacherTypeCSIAttacher, -// NodeID: TestNode1, -// Parameters: map[string]string{}, -// Attached: utilpointer.Bool(false), -// }, -// "attachment-02": &longhorn.Attachment{ -// ID: "attachment-02", -// Type: longhorn.AttacherTypeSnapshotController, -// NodeID: TestNode2, -// Parameters: map[string]string{}, -// Attached: utilpointer.Bool(false), -// AttachError: &longhorn.VolumeError{ -// Message: fmt.Sprintf("cannot attach the volume to node %v because volume has already desired to be attached to node %v", TestNode2, TestNode1), -// }, -// }, -// } -// tc.expectedVol.Spec.NodeID = TestNode1 -// testCases["test case 2: attach: multiple attachments"] = tc -// /////////////////////////////////////////////////////////////////// -// -// /////////////////////////////////////////////////////////////////// -// tc = generateVolumeAttachmentTestCaseTemplate(TestVolumeName) -// tc.volAttachment.Spec.Attachments = map[string]*longhorn.Attachment{ -// "attachment-01": &longhorn.Attachment{ -// ID: "attachment-01", -// Type: longhorn.AttacherTypeCSIAttacher, -// NodeID: TestNode1, -// Parameters: map[string]string{}, -// }, -// "attachment-02": &longhorn.Attachment{ -// ID: "attachment-02", -// Type: longhorn.AttacherTypeCSIAttacher, -// NodeID: TestNode2, -// Parameters: map[string]string{}, -// }, -// } -// tc.vol.Status.OwnerID = TestNode1 -// tc.vol.Status.State = longhorn.VolumeStateDetached -// tc.copyCurrentToExpect() -// tc.expectedVolAttachment.Status.Attachments = map[string]*longhorn.Attachment{ -// "attachment-01": &longhorn.Attachment{ -// ID: "attachment-01", -// Type: longhorn.AttacherTypeCSIAttacher, -// NodeID: TestNode1, -// Parameters: map[string]string{}, -// Attached: utilpointer.Bool(false), -// }, -// "attachment-02": &longhorn.Attachment{ -// ID: "attachment-02", -// Type: longhorn.AttacherTypeCSIAttacher, -// NodeID: TestNode2, -// Parameters: map[string]string{}, -// Attached: utilpointer.Bool(false), -// AttachError: &longhorn.VolumeError{ -// Message: fmt.Sprintf("cannot attach the volume to node %v because volume has already desired to be attached to node %v", TestNode2, TestNode1), -// }, -// }, -// } -// tc.expectedVol.Spec.NodeID = TestNode1 -// testCases["test case 3: attach: multiple attachments with same priority level"] = tc -// /////////////////////////////////////////////////////////////////// -// -// /////////////////////////////////////////////////////////////////// -// tc = generateVolumeAttachmentTestCaseTemplate(TestVolumeName) -// tc.volAttachment.Spec.Attachments = map[string]*longhorn.Attachment{ -// "attachment-01": &longhorn.Attachment{ -// ID: "attachment-01", -// Type: longhorn.AttacherTypeCSIAttacher, -// NodeID: TestNode1, -// Parameters: map[string]string{}, -// }, -// } -// tc.vol.Status.OwnerID = TestNode1 -// tc.vol.Spec.NodeID = TestNode1 -// tc.vol.Status.CurrentNodeID = TestNode1 -// tc.vol.Status.State = longhorn.VolumeStateAttached -// tc.copyCurrentToExpect() -// tc.expectedVolAttachment.Status.Attachments = map[string]*longhorn.Attachment{ -// "attachment-01": &longhorn.Attachment{ -// ID: "attachment-01", -// Type: longhorn.AttacherTypeCSIAttacher, -// NodeID: TestNode1, -// Parameters: map[string]string{}, -// Attached: utilpointer.Bool(true), -// }, -// } -// testCases["test case 4: attach: successfully attached case"] = tc -// /////////////////////////////////////////////////////////////////// -// -// /////////////////////////////////////////////////////////////////// -// tc = generateVolumeAttachmentTestCaseTemplate(TestVolumeName) -// tc.volAttachment.Spec.Attachments = map[string]*longhorn.Attachment{ -// "attachment-01": &longhorn.Attachment{ -// ID: "attachment-01", -// Type: longhorn.AttacherTypeCSIAttacher, -// NodeID: TestNode1, -// Parameters: map[string]string{ -// "disableFrontend": "true", -// }, -// }, -// "attachment-02": &longhorn.Attachment{ -// ID: "attachment-02", -// Type: longhorn.AttacherTypeSnapshotController, -// NodeID: TestNode1, -// Parameters: map[string]string{}, -// }, -// } -// tc.vol.Status.OwnerID = TestNode1 -// tc.vol.Spec.NodeID = TestNode1 -// tc.vol.Status.CurrentNodeID = TestNode1 -// tc.vol.Status.State = longhorn.VolumeStateAttached -// tc.copyCurrentToExpect() -// tc.expectedVolAttachment.Status.Attachments = map[string]*longhorn.Attachment{ -// "attachment-01": &longhorn.Attachment{ -// ID: "attachment-01", -// Type: longhorn.AttacherTypeCSIAttacher, -// NodeID: TestNode1, -// Parameters: map[string]string{ -// "disableFrontend": "true", -// }, -// Attached: utilpointer.Bool(false), -// AttachError: &longhorn.VolumeError{ -// Message: fmt.Sprintf("volume %v has already attached to node %v with incompatible parameters", tc.vol.Name, tc.vol.Status.CurrentNodeID), -// }, -// }, -// "attachment-02": &longhorn.Attachment{ -// ID: "attachment-02", -// Type: longhorn.AttacherTypeSnapshotController, -// NodeID: TestNode1, -// Parameters: map[string]string{}, -// Attached: utilpointer.Bool(true), -// }, -// } -// testCases["test case 5: attach: fail to attach because the volume is already attached with incompatible parameters"] = tc -// /////////////////////////////////////////////////////////////////// -// -// /////////////////////////////////////////////////////////////////// -// tc = generateVolumeAttachmentTestCaseTemplate(TestVolumeName) -// tc.volAttachment.Spec.Attachments = map[string]*longhorn.Attachment{} -// tc.volAttachment.Status.Attachments = map[string]*longhorn.Attachment{ -// "attachment-01": &longhorn.Attachment{ -// ID: "attachment-01", -// Type: longhorn.AttacherTypeCSIAttacher, -// NodeID: TestNode1, -// Parameters: map[string]string{}, -// Attached: utilpointer.Bool(true), -// }, -// } -// tc.vol.Status.OwnerID = TestNode1 -// tc.vol.Spec.NodeID = TestNode1 -// tc.vol.Status.CurrentNodeID = TestNode1 -// tc.vol.Status.State = longhorn.VolumeStateAttached -// tc.copyCurrentToExpect() -// tc.expectedVolAttachment.Status.Attachments = map[string]*longhorn.Attachment{} -// tc.expectedVol.Spec.NodeID = "" -// testCases["test case 6: detach: basic"] = tc -// /////////////////////////////////////////////////////////////////// -// -// /////////////////////////////////////////////////////////////////// -// tc = generateVolumeAttachmentTestCaseTemplate(TestVolumeName) -// tc.volAttachment.Spec.Attachments = map[string]*longhorn.Attachment{ -// "attachment-01": &longhorn.Attachment{ -// ID: "attachment-01", -// Type: longhorn.AttacherTypeCSIAttacher, -// NodeID: TestNode1, -// Parameters: map[string]string{}, -// }, -// } -// tc.volAttachment.Status.Attachments = map[string]*longhorn.Attachment{ -// "attachment-01": &longhorn.Attachment{ -// ID: "attachment-01", -// Type: longhorn.AttacherTypeCSIAttacher, -// NodeID: TestNode1, -// Parameters: map[string]string{}, -// Attached: utilpointer.Bool(true), -// }, -// "attachment-02": &longhorn.Attachment{ -// ID: "attachment-02", -// Type: longhorn.AttacherTypeSnapshotController, -// NodeID: TestNode1, -// Parameters: map[string]string{}, -// Attached: utilpointer.Bool(true), -// }, -// } -// tc.vol.Status.OwnerID = TestNode1 -// tc.vol.Spec.NodeID = TestNode1 -// tc.vol.Status.CurrentNodeID = TestNode1 -// tc.vol.Status.State = longhorn.VolumeStateAttached -// tc.copyCurrentToExpect() -// tc.expectedVolAttachment.Status.Attachments = map[string]*longhorn.Attachment{ -// "attachment-01": &longhorn.Attachment{ -// ID: "attachment-01", -// Type: longhorn.AttacherTypeCSIAttacher, -// NodeID: TestNode1, -// Parameters: map[string]string{}, -// Attached: utilpointer.Bool(true), -// }, -// } -// testCases["test case 7: detach: detach while there are still other attachments requesting the same node"] = tc -// /////////////////////////////////////////////////////////////////// -// -// /////////////////////////////////////////////////////////////////// -// tc = generateVolumeAttachmentTestCaseTemplate(TestVolumeName) -// tc.volAttachment.Spec.Attachments = map[string]*longhorn.Attachment{ -// "attachment-01": &longhorn.Attachment{ -// ID: "attachment-01", -// Type: longhorn.AttacherTypeCSIAttacher, -// NodeID: TestNode1, -// Parameters: map[string]string{}, -// }, -// } -// tc.vol.Status.OwnerID = TestNode1 -// tc.vol.Spec.NodeID = TestNode1 -// tc.vol.Spec.DisableFrontend = true -// tc.vol.Status.CurrentNodeID = TestNode1 -// tc.vol.Status.State = longhorn.VolumeStateAttached -// tc.copyCurrentToExpect() -// tc.expectedVolAttachment.Status.Attachments = map[string]*longhorn.Attachment{ -// "attachment-01": &longhorn.Attachment{ -// ID: "attachment-01", -// Type: longhorn.AttacherTypeCSIAttacher, -// NodeID: TestNode1, -// Parameters: map[string]string{}, -// Attached: utilpointer.Bool(false), -// }, -// } -// tc.expectedVol.Spec.NodeID = "" -// tc.expectedVol.Spec.DisableFrontend = false -// testCases["test case 8: detach: the current attachment requesting the same node but with incompatible parameters"] = tc -// /////////////////////////////////////////////////////////////////// -// -// for name, tc := range testCases { -// // uncomment this block to test individual test case -// //if name != "test case 5: attach: fail to attach because the volume is already attached with incompatible parameters" { -// // continue -// //} -// fmt.Printf("testing %v\n", name) -// s.runVolumeAttachmentTestCase(c, tc) -// } -// -//} -// -//func (s *TestSuite) runVolumeAttachmentTestCase(c *C, tc *volumeAttachmentTestCase) { -// kubeClient := fake.NewSimpleClientset() -// lhClient := lhfake.NewSimpleClientset() -// extensionsClient := apiextensionsfake.NewSimpleClientset() -// kubeInformerFactory := informers.NewSharedInformerFactory(kubeClient, 0) -// lhInformerFactory := lhinformerfactory.NewSharedInformerFactory(lhClient, 0) -// ds := datastore.NewDataStore(lhInformerFactory, lhClient, kubeInformerFactory, kubeClient, extensionsClient, TestNamespace) -// logger := logrus.StandardLogger() -// -// volumeIndexer := lhInformerFactory.Longhorn().V1beta2().Volumes().Informer().GetIndexer() -// volumeAttachmentIndexer := lhInformerFactory.Longhorn().V1beta2().VolumeAttachments().Informer().GetIndexer() -// -// vac := NewLonghornVolumeAttachmentController(logger, ds, scheme.Scheme, kubeClient, TestOwnerID1, TestNamespace) -// fakeRecorder := record.NewFakeRecorder(100) -// vac.eventRecorder = fakeRecorder -// for index := range vac.cacheSyncs { -// vac.cacheSyncs[index] = alwaysReady -// } -// -// // Seed the data. -// // Need to put it into both fakeclientset and Indexer because -// // the fake client doesn't work well with informers. -// // See details at https://github.com/kubernetes/kubernetes/issues/95372 -// vol, err := lhClient.LonghornV1beta2().Volumes(TestNamespace).Create(context.TODO(), tc.vol, metav1.CreateOptions{}) -// c.Assert(err, IsNil) -// err = volumeIndexer.Add(vol) -// c.Assert(err, IsNil) -// -// volAttachment, err := lhClient.LonghornV1beta2().VolumeAttachments(TestNamespace).Create(context.TODO(), tc.volAttachment, metav1.CreateOptions{}) -// c.Assert(err, IsNil) -// err = volumeAttachmentIndexer.Add(volAttachment) -// c.Assert(err, IsNil) -// -// //////////////////////////////////// -// // main test func -// vac.syncHandler(getKey(volAttachment, c)) -// /////////////////////////////////// -// -// retVol, err := lhClient.LonghornV1beta2().Volumes(TestNamespace).Get(context.TODO(), tc.vol.Name, metav1.GetOptions{}) -// c.Assert(err, IsNil) -// c.Assert(retVol.Spec, DeepEquals, tc.expectedVol.Spec) -// -// retVolAttachment, err := lhClient.LonghornV1beta2().VolumeAttachments(TestNamespace).Get(context.TODO(), tc.volAttachment.Name, metav1.GetOptions{}) -// c.Assert(err, IsNil) -// c.Assert(retVolAttachment.Status, DeepEquals, tc.expectedVolAttachment.Status) -// -//} -// -//func newVolumeAttachment(name string) *longhorn.VolumeAttachment { -// return &longhorn.VolumeAttachment{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: name, -// Namespace: TestNamespace, -// Finalizers: []string{ -// longhorn.SchemeGroupVersion.Group, -// }, -// Labels: map[string]string{ -// "longhornvolume": name, -// }, -// }, -// Spec: longhorn.VolumeAttachmentSpec{ -// Volume: name, -// }, -// Status: longhorn.VolumeAttachmentStatus{}, -// } -//} -// -//func generateVolumeAttachmentTestCaseTemplate(name string) *volumeAttachmentTestCase { -// return &volumeAttachmentTestCase{ -// volAttachment: newVolumeAttachment(name), -// vol: newVolume(name, 1), -// } -//} +import ( + "context" + "fmt" + + "github.com/sirupsen/logrus" + . "gopkg.in/check.v1" + + apiextensionsfake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" + + "github.com/longhorn/longhorn-manager/datastore" + longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" + lhfake "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned/fake" + lhinformerfactory "github.com/longhorn/longhorn-manager/k8s/pkg/client/informers/externalversions" + "github.com/longhorn/longhorn-manager/types" +) + +type volumeAttachmentTestCase struct { + volAttachment *longhorn.VolumeAttachment + vol *longhorn.Volume + + expectedVolAttachment *longhorn.VolumeAttachment + expectedVol *longhorn.Volume +} + +func (tc *volumeAttachmentTestCase) copyCurrentToExpect() { + tc.expectedVolAttachment = tc.volAttachment.DeepCopy() + tc.expectedVol = tc.vol.DeepCopy() +} + +func (s *TestSuite) TestVolumeAttachmentLifeCycle(c *C) { + var tc *volumeAttachmentTestCase + testCases := map[string]*volumeAttachmentTestCase{} + + /////////////////////////////////////////////////////////////////// + tc = generateVolumeAttachmentTestCaseTemplate(TestVolumeName) + tc.volAttachment.Spec.AttachmentTickets = map[string]*longhorn.AttachmentTicket{ + "attachment-01": &longhorn.AttachmentTicket{ + ID: "attachment-01", + Type: longhorn.AttacherTypeCSIAttacher, + NodeID: TestNode1, + Parameters: map[string]string{}, + Generation: 0, + }, + } + tc.vol.Status.OwnerID = TestNode1 + tc.vol.Status.State = longhorn.VolumeStateDetached + tc.copyCurrentToExpect() + tc.expectedVolAttachment.Status.AttachmentTicketStatuses = map[string]*longhorn.AttachmentTicketStatus{ + "attachment-01": &longhorn.AttachmentTicketStatus{ + ID: "attachment-01", + Satisfied: false, + Conditions: types.SetConditionWithoutTimestamp([]longhorn.Condition{}, + longhorn.AttachmentStatusConditionTypeSatisfied, longhorn.ConditionStatusFalse, "", ""), + Generation: 0, + }, + } + tc.expectedVol.Spec.NodeID = TestNode1 + testCases["test case 1: attach: basic"] = tc + /////////////////////////////////////////////////////////////////// + + /////////////////////////////////////////////////////////////////// + tc = generateVolumeAttachmentTestCaseTemplate(TestVolumeName) + tc.volAttachment.Spec.AttachmentTickets = map[string]*longhorn.AttachmentTicket{ + "attachment-01": &longhorn.AttachmentTicket{ + ID: "attachment-01", + Type: longhorn.AttacherTypeSnapshotController, + NodeID: TestNode1, + Parameters: map[string]string{}, + Generation: 0, + }, + "attachment-02": &longhorn.AttachmentTicket{ + ID: "attachment-02", + Type: longhorn.AttacherTypeCSIAttacher, + NodeID: TestNode2, + Parameters: map[string]string{}, + Generation: 0, + }, + } + tc.vol.Status.OwnerID = TestNode1 + tc.vol.Status.State = longhorn.VolumeStateDetached + tc.copyCurrentToExpect() + tc.expectedVolAttachment.Status.AttachmentTicketStatuses = map[string]*longhorn.AttachmentTicketStatus{ + "attachment-01": &longhorn.AttachmentTicketStatus{ + ID: "attachment-01", + Satisfied: false, + Conditions: types.SetConditionWithoutTimestamp([]longhorn.Condition{}, + longhorn.AttachmentStatusConditionTypeSatisfied, longhorn.ConditionStatusFalse, "", ""), + Generation: 0, + }, + "attachment-02": &longhorn.AttachmentTicketStatus{ + ID: "attachment-02", + Satisfied: false, + Conditions: types.SetConditionWithoutTimestamp([]longhorn.Condition{}, + longhorn.AttachmentStatusConditionTypeSatisfied, longhorn.ConditionStatusFalse, "", ""), + Generation: 0, + }, + } + tc.expectedVol.Spec.NodeID = TestNode2 + testCases["test case 2: attach: multiple attachments"] = tc + /////////////////////////////////////////////////////////////////// + + /////////////////////////////////////////////////////////////////// + tc = generateVolumeAttachmentTestCaseTemplate(TestVolumeName) + tc.volAttachment.Spec.AttachmentTickets = map[string]*longhorn.AttachmentTicket{ + "attachment-01": &longhorn.AttachmentTicket{ + ID: "attachment-01", + Type: longhorn.AttacherTypeCSIAttacher, + NodeID: TestNode1, + Parameters: map[string]string{}, + Generation: 0, + }, + "attachment-02": &longhorn.AttachmentTicket{ + ID: "attachment-02", + Type: longhorn.AttacherTypeCSIAttacher, + NodeID: TestNode2, + Parameters: map[string]string{}, + Generation: 0, + }, + } + tc.vol.Status.OwnerID = TestNode1 + tc.vol.Status.State = longhorn.VolumeStateDetached + tc.copyCurrentToExpect() + tc.expectedVolAttachment.Status.AttachmentTicketStatuses = map[string]*longhorn.AttachmentTicketStatus{ + "attachment-01": &longhorn.AttachmentTicketStatus{ + ID: "attachment-01", + Satisfied: false, + Conditions: types.SetConditionWithoutTimestamp([]longhorn.Condition{}, + longhorn.AttachmentStatusConditionTypeSatisfied, longhorn.ConditionStatusFalse, "", ""), + Generation: 0, + }, + "attachment-02": &longhorn.AttachmentTicketStatus{ + ID: "attachment-02", + Satisfied: false, + Conditions: types.SetConditionWithoutTimestamp([]longhorn.Condition{}, + longhorn.AttachmentStatusConditionTypeSatisfied, longhorn.ConditionStatusFalse, "", ""), + Generation: 0, + }, + } + // AD ticket is selected by priority then name. + // Since tickets has same priority, we pick ticker with shorter name, attachment-01 + tc.expectedVol.Spec.NodeID = TestNode1 + testCases["test case 3: attach: multiple attachments with same priority level"] = tc + /////////////////////////////////////////////////////////////////// + + /////////////////////////////////////////////////////////////////// + tc = generateVolumeAttachmentTestCaseTemplate(TestVolumeName) + tc.volAttachment.Spec.AttachmentTickets = map[string]*longhorn.AttachmentTicket{ + "attachment-01": &longhorn.AttachmentTicket{ + ID: "attachment-01", + Type: longhorn.AttacherTypeCSIAttacher, + NodeID: TestNode1, + Parameters: map[string]string{}, + Generation: 0, + }, + } + tc.vol.Status.OwnerID = TestNode1 + tc.vol.Spec.NodeID = TestNode1 + tc.vol.Status.CurrentNodeID = TestNode1 + tc.vol.Status.State = longhorn.VolumeStateAttached + tc.copyCurrentToExpect() + tc.expectedVolAttachment.Status.AttachmentTicketStatuses = map[string]*longhorn.AttachmentTicketStatus{ + "attachment-01": &longhorn.AttachmentTicketStatus{ + ID: "attachment-01", + Satisfied: true, + Conditions: types.SetConditionWithoutTimestamp([]longhorn.Condition{}, + longhorn.AttachmentStatusConditionTypeSatisfied, longhorn.ConditionStatusTrue, "", ""), + Generation: 0, + }, + } + testCases["test case 4: attach: successfully attached case"] = tc + /////////////////////////////////////////////////////////////////// + + /////////////////////////////////////////////////////////////////// + tc = generateVolumeAttachmentTestCaseTemplate(TestVolumeName) + tc.volAttachment.Spec.AttachmentTickets = map[string]*longhorn.AttachmentTicket{ + "attachment-01": &longhorn.AttachmentTicket{ + ID: "attachment-01", + Type: longhorn.AttacherTypeVolumeRestoreController, + NodeID: TestNode1, + Parameters: map[string]string{ + "disableFrontend": "true", + }, + Generation: 0, + }, + "attachment-02": &longhorn.AttachmentTicket{ + ID: "attachment-02", + Type: longhorn.AttacherTypeCSIAttacher, + NodeID: TestNode1, + Parameters: map[string]string{}, + Generation: 0, + }, + } + tc.vol.Status.OwnerID = TestNode1 + tc.vol.Spec.NodeID = TestNode1 + tc.vol.Status.CurrentNodeID = TestNode1 + tc.vol.Status.State = longhorn.VolumeStateAttached + tc.copyCurrentToExpect() + tc.expectedVolAttachment.Status.AttachmentTicketStatuses = map[string]*longhorn.AttachmentTicketStatus{ + "attachment-01": &longhorn.AttachmentTicketStatus{ + ID: "attachment-01", + Satisfied: false, + Conditions: types.SetConditionWithoutTimestamp([]longhorn.Condition{}, + longhorn.AttachmentStatusConditionTypeSatisfied, longhorn.ConditionStatusFalse, + longhorn.AttachmentStatusConditionReasonAttachedWithIncompatibleParameters, + fmt.Sprintf("volume %v has already attached to node %v with incompatible parameters", tc.vol.Name, tc.vol.Status.CurrentNodeID)), + Generation: 0, + }, + "attachment-02": &longhorn.AttachmentTicketStatus{ + ID: "attachment-02", + Satisfied: true, + Conditions: types.SetConditionWithoutTimestamp([]longhorn.Condition{}, + longhorn.AttachmentStatusConditionTypeSatisfied, longhorn.ConditionStatusTrue, "", ""), + Generation: 0, + }, + } + testCases["test case 5: attach: fail to attach because the volume is already attached with incompatible parameters"] = tc + /////////////////////////////////////////////////////////////////// + + /////////////////////////////////////////////////////////////////// + tc = generateVolumeAttachmentTestCaseTemplate(TestVolumeName) + tc.volAttachment.Spec.AttachmentTickets = map[string]*longhorn.AttachmentTicket{} + tc.volAttachment.Status.AttachmentTicketStatuses = map[string]*longhorn.AttachmentTicketStatus{ + "attachment-01": &longhorn.AttachmentTicketStatus{ + ID: "attachment-01", + Satisfied: true, + Conditions: types.SetConditionWithoutTimestamp([]longhorn.Condition{}, + longhorn.AttachmentStatusConditionTypeSatisfied, longhorn.ConditionStatusTrue, "", ""), + Generation: 0, + }, + } + tc.vol.Status.OwnerID = TestNode1 + tc.vol.Spec.NodeID = TestNode1 + tc.vol.Status.CurrentNodeID = TestNode1 + tc.vol.Status.State = longhorn.VolumeStateAttached + tc.copyCurrentToExpect() + tc.expectedVolAttachment.Status.AttachmentTicketStatuses = map[string]*longhorn.AttachmentTicketStatus{} + tc.expectedVol.Spec.NodeID = "" + testCases["test case 6: detach: basic"] = tc + /////////////////////////////////////////////////////////////////// + + /////////////////////////////////////////////////////////////////// + tc = generateVolumeAttachmentTestCaseTemplate(TestVolumeName) + tc.volAttachment.Spec.AttachmentTickets = map[string]*longhorn.AttachmentTicket{ + "attachment-01": &longhorn.AttachmentTicket{ + ID: "attachment-01", + Type: longhorn.AttacherTypeCSIAttacher, + NodeID: TestNode1, + Parameters: map[string]string{}, + Generation: 0, + }, + } + tc.volAttachment.Status.AttachmentTicketStatuses = map[string]*longhorn.AttachmentTicketStatus{ + "attachment-01": &longhorn.AttachmentTicketStatus{ + ID: "attachment-01", + Satisfied: true, + Conditions: types.SetConditionWithoutTimestamp([]longhorn.Condition{}, + longhorn.AttachmentStatusConditionTypeSatisfied, longhorn.ConditionStatusTrue, "", ""), + Generation: 0, + }, + "attachment-02": &longhorn.AttachmentTicketStatus{ + ID: "attachment-02", + Satisfied: true, + Conditions: types.SetConditionWithoutTimestamp([]longhorn.Condition{}, + longhorn.AttachmentStatusConditionTypeSatisfied, longhorn.ConditionStatusTrue, "", ""), + Generation: 0, + }, + } + tc.vol.Status.OwnerID = TestNode1 + tc.vol.Spec.NodeID = TestNode1 + tc.vol.Status.CurrentNodeID = TestNode1 + tc.vol.Status.State = longhorn.VolumeStateAttached + tc.copyCurrentToExpect() + delete(tc.expectedVolAttachment.Status.AttachmentTicketStatuses, "attachment-02") + testCases["test case 7: detach: detach while there are still other attachments requesting the same node"] = tc + /////////////////////////////////////////////////////////////////// + + /////////////////////////////////////////////////////////////////// + tc = generateVolumeAttachmentTestCaseTemplate(TestVolumeName) + tc.volAttachment.Spec.AttachmentTickets = map[string]*longhorn.AttachmentTicket{ + "attachment-01": &longhorn.AttachmentTicket{ + ID: "attachment-01", + Type: longhorn.AttacherTypeCSIAttacher, + NodeID: TestNode1, + Parameters: map[string]string{}, + Generation: 0, + }, + } + tc.vol.Status.OwnerID = TestNode1 + tc.vol.Spec.NodeID = TestNode1 + tc.vol.Spec.DisableFrontend = true + tc.vol.Status.CurrentNodeID = TestNode1 + tc.vol.Status.State = longhorn.VolumeStateAttached + tc.copyCurrentToExpect() + tc.expectedVolAttachment.Status.AttachmentTicketStatuses = map[string]*longhorn.AttachmentTicketStatus{ + "attachment-01": &longhorn.AttachmentTicketStatus{ + ID: "attachment-01", + Satisfied: true, + Conditions: types.SetConditionWithoutTimestamp([]longhorn.Condition{}, + longhorn.AttachmentStatusConditionTypeSatisfied, longhorn.ConditionStatusTrue, "", ""), + Generation: 0, + }, + } + tc.expectedVol.Spec.NodeID = "" + tc.expectedVol.Spec.DisableFrontend = false + testCases["test case 8: detach: the current attachment requesting the same node but with incompatible parameters"] = tc + /////////////////////////////////////////////////////////////////// + + /////////////////////////////////////////////////////////////////// + tc = generateVolumeAttachmentTestCaseTemplate(TestVolumeName) + tc.volAttachment.Status.AttachmentTicketStatuses = map[string]*longhorn.AttachmentTicketStatus{ + "attachment-01": &longhorn.AttachmentTicketStatus{ + ID: "attachment-01", + Satisfied: true, + Conditions: types.SetConditionWithoutTimestamp([]longhorn.Condition{}, + longhorn.AttachmentStatusConditionTypeSatisfied, longhorn.ConditionStatusTrue, "", ""), + Generation: 0, + }, + } + tc.volAttachment.Spec.AttachmentTickets = map[string]*longhorn.AttachmentTicket{ + "attachment-01": &longhorn.AttachmentTicket{ + ID: "attachment-01", + Type: longhorn.AttacherTypeCSIAttacher, + NodeID: TestNode2, + Parameters: map[string]string{}, + Generation: 1, + }, + } + tc.vol.Status.OwnerID = TestNode1 + tc.vol.Spec.NodeID = TestNode1 + tc.vol.Spec.DisableFrontend = false + tc.vol.Status.CurrentNodeID = TestNode1 + tc.vol.Status.State = longhorn.VolumeStateAttached + tc.copyCurrentToExpect() + tc.expectedVolAttachment.Status.AttachmentTicketStatuses = map[string]*longhorn.AttachmentTicketStatus{ + "attachment-01": &longhorn.AttachmentTicketStatus{ + ID: "attachment-01", + Satisfied: false, + Conditions: types.SetConditionWithoutTimestamp([]longhorn.Condition{}, + longhorn.AttachmentStatusConditionTypeSatisfied, longhorn.ConditionStatusFalse, "", + fmt.Sprintf("the volume is currently attached to different node %v ", TestNode1)), + Generation: 1, + }, + } + tc.expectedVol.Spec.NodeID = "" + tc.expectedVol.Spec.DisableFrontend = false + testCases["test case 9: test ticket's generation: attachment ticket change its node ID"] = tc + /////////////////////////////////////////////////////////////////// + + /////////////////////////////////////////////////////////////////// + tc = generateVolumeAttachmentTestCaseTemplate(TestVolumeName) + tc.volAttachment.Status.AttachmentTicketStatuses = map[string]*longhorn.AttachmentTicketStatus{ + "attachment-01": &longhorn.AttachmentTicketStatus{ + ID: "attachment-01", + Satisfied: true, + Conditions: types.SetConditionWithoutTimestamp([]longhorn.Condition{}, + longhorn.AttachmentStatusConditionTypeSatisfied, longhorn.ConditionStatusTrue, "", ""), + Generation: 0, + }, + } + tc.volAttachment.Spec.AttachmentTickets = map[string]*longhorn.AttachmentTicket{ + "attachment-01": &longhorn.AttachmentTicket{ + ID: "attachment-01", + Type: longhorn.AttacherTypeSnapshotController, + NodeID: TestNode1, + Parameters: map[string]string{}, + Generation: 0, + }, + "attachment-02": &longhorn.AttachmentTicket{ + ID: "attachment-02", + Type: longhorn.AttacherTypeCSIAttacher, + NodeID: TestNode2, + Parameters: map[string]string{}, + Generation: 0, + }, + } + tc.vol.Status.OwnerID = TestNode1 + tc.vol.Spec.NodeID = TestNode1 + tc.vol.Spec.DisableFrontend = false + tc.vol.Status.CurrentNodeID = TestNode1 + tc.vol.Status.State = longhorn.VolumeStateAttached + tc.copyCurrentToExpect() + tc.expectedVolAttachment.Status.AttachmentTicketStatuses = map[string]*longhorn.AttachmentTicketStatus{ + "attachment-01": &longhorn.AttachmentTicketStatus{ + ID: "attachment-01", + Satisfied: true, + Conditions: types.SetConditionWithoutTimestamp([]longhorn.Condition{}, + longhorn.AttachmentStatusConditionTypeSatisfied, longhorn.ConditionStatusTrue, "", ""), + Generation: 0, + }, + "attachment-02": &longhorn.AttachmentTicketStatus{ + ID: "attachment-02", + Satisfied: false, + Conditions: types.SetConditionWithoutTimestamp([]longhorn.Condition{}, + longhorn.AttachmentStatusConditionTypeSatisfied, longhorn.ConditionStatusFalse, "", + fmt.Sprintf("the volume is currently attached to different node %v ", TestNode1)), + Generation: 0, + }, + } + tc.expectedVol.Spec.NodeID = "" + testCases["test case 10: ticket with higher priority interrupts ticket with lower priority"] = tc + /////////////////////////////////////////////////////////////////// + + for name, tc := range testCases { + //uncomment this block to test individual test case + //if name != "test case 10: ticket with higher priority interrupts ticket with lower priority" { + // continue + //} + fmt.Printf("testing %v\n", name) + s.runVolumeAttachmentTestCase(c, tc) + } + +} + +func (s *TestSuite) runVolumeAttachmentTestCase(c *C, tc *volumeAttachmentTestCase) { + kubeClient := fake.NewSimpleClientset() + lhClient := lhfake.NewSimpleClientset() + extensionsClient := apiextensionsfake.NewSimpleClientset() + kubeInformerFactory := informers.NewSharedInformerFactory(kubeClient, 0) + lhInformerFactory := lhinformerfactory.NewSharedInformerFactory(lhClient, 0) + ds := datastore.NewDataStore(lhInformerFactory, lhClient, kubeInformerFactory, kubeClient, extensionsClient, TestNamespace) + logger := logrus.StandardLogger() + + volumeIndexer := lhInformerFactory.Longhorn().V1beta2().Volumes().Informer().GetIndexer() + volumeAttachmentIndexer := lhInformerFactory.Longhorn().V1beta2().VolumeAttachments().Informer().GetIndexer() + + vac := NewLonghornVolumeAttachmentController(logger, ds, scheme.Scheme, kubeClient, TestOwnerID1, TestNamespace) + fakeRecorder := record.NewFakeRecorder(100) + vac.eventRecorder = fakeRecorder + for index := range vac.cacheSyncs { + vac.cacheSyncs[index] = alwaysReady + } + + // Seed the data. + // Need to put it into both fakeclientset and Indexer because + // the fake client doesn't work well with informers. + // See details at https://github.com/kubernetes/kubernetes/issues/95372 + vol, err := lhClient.LonghornV1beta2().Volumes(TestNamespace).Create(context.TODO(), tc.vol, metav1.CreateOptions{}) + c.Assert(err, IsNil) + err = volumeIndexer.Add(vol) + c.Assert(err, IsNil) + + volAttachment, err := lhClient.LonghornV1beta2().VolumeAttachments(TestNamespace).Create(context.TODO(), tc.volAttachment, metav1.CreateOptions{}) + c.Assert(err, IsNil) + err = volumeAttachmentIndexer.Add(volAttachment) + c.Assert(err, IsNil) + + //////////////////////////////////// + // main test func + vac.syncHandler(getKey(volAttachment, c)) + /////////////////////////////////// + + retVol, err := lhClient.LonghornV1beta2().Volumes(TestNamespace).Get(context.TODO(), tc.vol.Name, metav1.GetOptions{}) + c.Assert(err, IsNil) + c.Assert(retVol.Spec, DeepEquals, tc.expectedVol.Spec) + + retVolAttachment, err := lhClient.LonghornV1beta2().VolumeAttachments(TestNamespace).Get(context.TODO(), tc.volAttachment.Name, metav1.GetOptions{}) + c.Assert(err, IsNil) + // mask timestamps + for _, ticketStatus := range retVolAttachment.Status.AttachmentTicketStatuses { + for ctype, condition := range ticketStatus.Conditions { + condition.LastTransitionTime = "" + ticketStatus.Conditions[ctype] = condition + } + } + c.Assert(retVolAttachment.Status, DeepEquals, tc.expectedVolAttachment.Status) + +} + +func newVolumeAttachment(name string) *longhorn.VolumeAttachment { + return &longhorn.VolumeAttachment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: TestNamespace, + Finalizers: []string{ + longhorn.SchemeGroupVersion.Group, + }, + Labels: map[string]string{ + "longhornvolume": name, + }, + }, + Spec: longhorn.VolumeAttachmentSpec{ + Volume: name, + }, + Status: longhorn.VolumeAttachmentStatus{}, + } +} + +func generateVolumeAttachmentTestCaseTemplate(name string) *volumeAttachmentTestCase { + return &volumeAttachmentTestCase{ + volAttachment: newVolumeAttachment(name), + vol: newVolume(name, 1), + } +}