V7.2.0

CHANGELOG.md

@@ -20,6 +20,8 @@
 * Fix: GCP and AWS backends were ignoring the custom image option. This is now enabled.
 * Fix: Disable setting terminal to RAW mode for ssh, fixing parallelism.
 * Fix: GCP arm support.
+* Parallelize docker cluster and client listing for speed.
+* If terminationProtection is on, show in inventory, avoid trying to destroy, and do not expire.
 
 #### 7.1.1
 * GCP just made `DiscardLocalSsd` non-optional when stopping instances. Adjusting accordingly.

expiries/aws/aerolab-expire.go

@@ -66,8 +66,18 @@ func HandleLambdaEvent(event MyEvent) (MyResponse, error) {
 				expiry, err := time.Parse(time.RFC3339, expires)
 				if err != nil {
 					log.Printf("Could not handle expiry for instance %s: %s: %s", aws.StringValue(instance.InstanceId), expires, err)
+					continue
 				}
 				if expiry.Before(now) && expiry.After(time.Date(1985, time.April, 29, 0, 0, 0, 0, time.UTC)) {
+					// test for termination protection
+					attr, err := svc.DescribeInstanceAttribute(&ec2.DescribeInstanceAttributeInput{
+						Attribute:  aws.String(ec2.InstanceAttributeNameDisableApiTermination),
+						InstanceId: instance.InstanceId,
+					})
+					if err == nil && aws.BoolValue(attr.DisableApiTermination.Value) {
+						log.Printf("Not terminating %s, ApiTermination protection is enabled", *instance.InstanceId)
+						continue
+					}
 					deleteList = append(deleteList, aws.StringValue(instance.InstanceId))
 					name := tags["Aerolab4ClusterName"]
 					node := tags["Aerolab4NodeNumber"]

src/backendAws.go

@@ -2035,6 +2035,15 @@ func (d *backendAws) ClusterDestroy(name string, nodes []int) error {
 					}
 				}
 				if inslice.HasInt(nodes, nodeNumber) {
+					// test for termination protection
+					attr, err := d.ec2svc.DescribeInstanceAttribute(&ec2.DescribeInstanceAttributeInput{
+						Attribute:  aws.String(ec2.InstanceAttributeNameDisableApiTermination),
+						InstanceId: instance.InstanceId,
+					})
+					if err == nil && aws.BoolValue(attr.DisableApiTermination.Value) {
+						log.Printf("Not terminating %s, ApiTermination protection is enabled", *instance.InstanceId)
+						continue
+					}
 					instanceIds = append(instanceIds, instance.InstanceId)
 					input := &ec2.TerminateInstancesInput{
 						InstanceIds: []*string{
@@ -2044,13 +2053,14 @@ func (d *backendAws) ClusterDestroy(name string, nodes []int) error {
 					}
 					result, err := d.ec2svc.TerminateInstances(input)
 					if err != nil {
-						return fmt.Errorf("error starting instance %s\n%s\n%s", *instance.InstanceId, result, err)
+						return fmt.Errorf("error terminating instance %s\n%s\n%s", *instance.InstanceId, result, err)
 					}
 				}
 			}
 		}
 	}
 	if len(instanceIds) > 0 {
+		log.Printf("Terminate command sent to %d instances, waiting for AWS to finish terminating", len(instanceIds))
 		d.ec2svc.WaitUntilInstanceTerminated(&ec2.DescribeInstancesInput{
 			DryRun:      aws.Bool(false),
 			InstanceIds: instanceIds,

src/backendDocker.go

@@ -12,6 +12,7 @@ import (
 	"os/exec"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/bestmethod/inslice"
@@ -147,114 +148,137 @@ func (d *backendDocker) Inventory(owner string, inventoryItems []int) (inventory
 			return ij, err
 		}
 		scanner := bufio.NewScanner(strings.NewReader(string(out)))
+		lineWait := new(sync.WaitGroup)
+		var lineError error
+		lineErrorLock := new(sync.Mutex)
+		invLock := new(sync.Mutex)
 		for scanner.Scan() {
 			t := scanner.Text()
-			t = strings.Trim(t, "'\" \t\r\n")
-			tt := strings.Split(t, "\t")
-			if len(tt) < 4 || len(tt) > 6 {
-				continue
-			}
-			if !strings.HasPrefix(tt[1], dockerNameHeader) {
-				continue
-			}
-			nameNo := strings.Split(strings.TrimPrefix(tt[1], dockerNameHeader+""), "_")
-			if len(nameNo) != 2 {
-				continue
-			}
-			outl, err := exec.Command("docker", "container", "inspect", "--format", "{{json .Config.Labels}}", tt[1]).CombinedOutput()
-			if err != nil {
-				return ij, err
-			}
-			allLabels := make(map[string]string)
-			err = json.Unmarshal(outl, &allLabels)
-			if err != nil {
-				return ij, err
-			}
-			out2, err := exec.Command("docker", "container", "inspect", "--format", "{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}", tt[1]).CombinedOutput()
-			if err != nil {
-				return ij, err
-			}
-			ip := strings.Trim(string(out2), "'\" \n\r")
-			arch := "amd64"
-			if d.isArm {
-				arch = "arm64"
-			}
-			var i1, asdVer string
-			var i2 []string
-			i3 := []string{""}
-			if i == 1 {
-				i1 = strings.TrimPrefix(tt[3], "aerolab-")
-				i2 = strings.Split(i1, "_")
-				if len(i2) > 1 {
-					i3 = strings.Split(i2[1], ":")
+			lineWait.Add(1)
+			go func(t string) {
+				defer lineWait.Done()
+				t = strings.Trim(t, "'\" \t\r\n")
+				tt := strings.Split(t, "\t")
+				if len(tt) < 4 || len(tt) > 6 {
+					return
 				}
-				if len(i3) > 1 {
-					asdVer = i3[1]
+				if !strings.HasPrefix(tt[1], dockerNameHeader) {
+					return
 				}
-			} else {
-				i2 = strings.Split(tt[3], ":")
-				if len(i2) > 1 {
-					i3[0] = i2[1]
+				nameNo := strings.Split(strings.TrimPrefix(tt[1], dockerNameHeader+""), "_")
+				if len(nameNo) != 2 {
+					return
 				}
-			}
-			clientType := ""
-			if len(tt) > 4 {
-				clientType = tt[4]
-			}
-			exposePorts := ""
-			intPorts := ""
-			if len(tt) > 5 {
-				ep1 := strings.Split(tt[5], "->")
-				if len(ep1) > 1 {
-					ep2 := strings.Split(ep1[0], ":")
-					if len(ep2) > 1 {
-						exposePorts = ep2[1]
+				outl, err := exec.Command("docker", "container", "inspect", "--format", "{{json .Config.Labels}}", tt[1]).CombinedOutput()
+				if err != nil {
+					lineErrorLock.Lock()
+					lineError = err
+					lineErrorLock.Unlock()
+					return
+				}
+				allLabels := make(map[string]string)
+				err = json.Unmarshal(outl, &allLabels)
+				if err != nil {
+					lineErrorLock.Lock()
+					lineError = err
+					lineErrorLock.Unlock()
+					return
+				}
+				out2, err := exec.Command("docker", "container", "inspect", "--format", "{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}", tt[1]).CombinedOutput()
+				if err != nil {
+					lineErrorLock.Lock()
+					lineError = err
+					lineErrorLock.Unlock()
+					return
+				}
+				ip := strings.Trim(string(out2), "'\" \n\r")
+				arch := "amd64"
+				if d.isArm {
+					arch = "arm64"
+				}
+				var i1, asdVer string
+				var i2 []string
+				i3 := []string{""}
+				if i == 1 {
+					i1 = strings.TrimPrefix(tt[3], "aerolab-")
+					i2 = strings.Split(i1, "_")
+					if len(i2) > 1 {
+						i3 = strings.Split(i2[1], ":")
+					}
+					if len(i3) > 1 {
+						asdVer = i3[1]
+					}
+				} else {
+					i2 = strings.Split(tt[3], ":")
+					if len(i2) > 1 {
+						i3[0] = i2[1]
 					}
-					ep2 = strings.Split(ep1[1], "/")
-					intPorts = ep2[0]
 				}
-			}
-			if i == 1 {
-				features, _ := strconv.Atoi(clientType)
-				ij.Clusters = append(ij.Clusters, inventoryCluster{
-					ClusterName:        nameNo[0],
-					NodeNo:             nameNo[1],
-					PublicIp:           "",
-					PrivateIp:          strings.ReplaceAll(ip, " ", ","),
-					InstanceId:         tt[0],
-					ImageId:            tt[3],
-					State:              strings.ReplaceAll(tt[2], " ", "_"),
-					Arch:               arch,
-					Distribution:       i2[0],
-					OSVersion:          i3[0],
-					AerospikeVersion:   asdVer,
-					DockerExposePorts:  exposePorts,
-					DockerInternalPort: intPorts,
-					Features:           FeatureSystem(features),
-					AGILabel:           allLabels["agiLabel"],
-					dockerLabels:       allLabels,
-					Owner:              allLabels["owner"],
-				})
-			} else {
-				ij.Clients = append(ij.Clients, inventoryClient{
-					ClientName:         nameNo[0],
-					NodeNo:             nameNo[1],
-					PublicIp:           "",
-					PrivateIp:          strings.ReplaceAll(ip, " ", ","),
-					InstanceId:         tt[0],
-					ImageId:            tt[3],
-					State:              strings.ReplaceAll(tt[2], " ", "_"),
-					Arch:               arch,
-					Distribution:       i2[0],
-					OSVersion:          i3[0],
-					AerospikeVersion:   asdVer,
-					ClientType:         clientType,
-					DockerExposePorts:  exposePorts,
-					DockerInternalPort: intPorts,
-					dockerLabels:       allLabels,
-					Owner:              allLabels["owner"],
-				})
-			}
+				clientType := ""
+				if len(tt) > 4 {
+					clientType = tt[4]
+				}
+				exposePorts := ""
+				intPorts := ""
+				if len(tt) > 5 {
+					ep1 := strings.Split(tt[5], "->")
+					if len(ep1) > 1 {
+						ep2 := strings.Split(ep1[0], ":")
+						if len(ep2) > 1 {
+							exposePorts = ep2[1]
+						}
+						ep2 = strings.Split(ep1[1], "/")
+						intPorts = ep2[0]
+					}
+				}
+				invLock.Lock()
+				defer invLock.Unlock()
+				if i == 1 {
+					features, _ := strconv.Atoi(clientType)
+					ij.Clusters = append(ij.Clusters, inventoryCluster{
+						ClusterName:        nameNo[0],
+						NodeNo:             nameNo[1],
+						PublicIp:           "",
+						PrivateIp:          strings.ReplaceAll(ip, " ", ","),
+						InstanceId:         tt[0],
+						ImageId:            tt[3],
+						State:              strings.ReplaceAll(tt[2], " ", "_"),
+						Arch:               arch,
+						Distribution:       i2[0],
+						OSVersion:          i3[0],
+						AerospikeVersion:   asdVer,
+						DockerExposePorts:  exposePorts,
+						DockerInternalPort: intPorts,
+						Features:           FeatureSystem(features),
+						AGILabel:           allLabels["agiLabel"],
+						dockerLabels:       allLabels,
+						Owner:              allLabels["owner"],
+					})
+				} else {
+					ij.Clients = append(ij.Clients, inventoryClient{
+						ClientName:         nameNo[0],
+						NodeNo:             nameNo[1],
+						PublicIp:           "",
+						PrivateIp:          strings.ReplaceAll(ip, " ", ","),
+						InstanceId:         tt[0],
+						ImageId:            tt[3],
+						State:              strings.ReplaceAll(tt[2], " ", "_"),
+						Arch:               arch,
+						Distribution:       i2[0],
+						OSVersion:          i3[0],
+						AerospikeVersion:   asdVer,
+						ClientType:         clientType,
+						DockerExposePorts:  exposePorts,
+						DockerInternalPort: intPorts,
+						dockerLabels:       allLabels,
+						Owner:              allLabels["owner"],
+					})
+				}
+			}(t)
+		}
+		lineWait.Wait()
+		if lineError != nil {
+			return ij, err
 		}
 	}
 	return ij, nil

src/backendGcp.go

@@ -1672,6 +1672,9 @@ func (d *backendGcp) ClusterDestroy(name string, nodes []int) error {
 			ctx: ctx,
 		})
 	}
+	if len(ops) > 0 {
+		log.Printf("Terminate command sent to %d instances, waiting for GCP to finish terminating", len(ops))
+	}
 	for _, o := range ops {
 		if err = o.op.Wait(o.ctx); err != nil {
 			return fmt.Errorf("unable to wait for the operation: %w", err)