operators: ignore node deletion errors on absence

edgelesssys · May 21, 2024 · 26c44ab · 26c44ab
1 parent 71fe73a
commit 26c44ab
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 1 deletion.
diff --git a/operators/constellation-node-operator/internal/cloud/aws/client/nodeimage.go b/operators/constellation-node-operator/internal/cloud/aws/client/nodeimage.go
@@ -9,6 +9,7 @@ package client
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/autoscaling"
@@ -207,7 +208,7 @@ func (c *Client) DeleteNode(ctx context.Context, providerID string) error {
 			ShouldDecrementDesiredCapacity: toPtr(true),
 		},
 	)
-	if err != nil {
+	if err != nil && !isInstanceNotFoundError(err) {
 		return fmt.Errorf("failed to terminate instance: %w", err)
 	}
 
@@ -217,3 +218,10 @@ func (c *Client) DeleteNode(ctx context.Context, providerID string) error {
 func toPtr[T any](v T) *T {
 	return &v
 }
+
+func isInstanceNotFoundError(err error) bool {
+	if err == nil {
+		return false
+	}
+	return strings.Contains(err.Error(), "Instance Id not found")
+}
diff --git a/operators/constellation-node-operator/internal/cloud/aws/client/nodeimage_test.go b/operators/constellation-node-operator/internal/cloud/aws/client/nodeimage_test.go
@@ -8,6 +8,7 @@ package client
 
 import (
 	"context"
+	"fmt"
 	"testing"
 
 	"github.com/aws/aws-sdk-go-v2/service/autoscaling"
@@ -382,6 +383,10 @@ func TestDeleteNode(t *testing.T) {
 			terminateInstanceErr: assert.AnError,
 			wantErr:              true,
 		},
+		"deleting node succeeds when the instance does not exist": {
+			providerID:           "aws:///us-east-2a/i-00000000000000000",
+			terminateInstanceErr: fmt.Errorf("Instance Id not found - No managed instance found for instance ID: i-00000000000000000"),
+		},
 	}
 	for name, tc := range testCases {
 		t.Run(name, func(t *testing.T) {

diff --git a/operators/constellation-node-operator/internal/cloud/gcp/client/nodeimage.go b/operators/constellation-node-operator/internal/cloud/gcp/client/nodeimage.go
@@ -140,10 +140,15 @@ func (c *Client) DeleteNode(ctx context.Context, providerID string) error {
 		Zone:                 instanceGroupZone,
 		InstanceGroupManagersDeleteInstancesRequestResource: &computepb.InstanceGroupManagersDeleteInstancesRequest{
 			Instances: []string{instanceID},
+			SkipInstancesOnValidationError: toPtr(true),
 		},
 	})
 	if err != nil {
 		return fmt.Errorf("deleting instance %q from instance group manager %q: %w", instanceID, scalingGroupID, err)
 	}
 	return op.Wait(ctx)
 }
+
+func toPtr[T any](v T) *T {
+	return &v
+}