diff --git a/src/core/infra/provisioning.go b/src/core/infra/provisioning.go index 8e88fabb..b6cc1de4 100644 --- a/src/core/infra/provisioning.go +++ b/src/core/infra/provisioning.go @@ -957,30 +957,62 @@ func CreateMciDynamic(reqID string, nsId string, req *model.TbMciDynamicReq, dep } //If not, generate default resources dynamically. + // Parallel processing of VM requests + var wg sync.WaitGroup + var mutex sync.Mutex + errChan := make(chan error, len(vmRequest)) // Error channel to collect errors + for _, k := range vmRequest { - vmReq, err := getVmReqFromDynamicReq(reqID, nsId, &k) + wg.Add(1) + + // Launch a goroutine for each VM request + go func(vmReq model.TbVmDynamicReq) { + defer wg.Done() + + req, err := getVmReqFromDynamicReq(reqID, nsId, &vmReq) + if err != nil { + log.Error().Err(err).Msg("Failed to prepare resources for dynamic MCI creation") + errChan <- err + return + } + + // Safely append to the shared mciReq.Vm slice + mutex.Lock() + mciReq.Vm = append(mciReq.Vm, *req) + mutex.Unlock() + }(k) + } + + // Wait for all goroutines to complete + wg.Wait() + close(errChan) // Close the error channel after processing + + // Check for any errors from the goroutines + for err := range errChan { if err != nil { - log.Error().Err(err).Msg("Failed to prefare resources for dynamic MCI creation") - // Rollback created default resources + // Rollback if any error occurs + log.Info().Msg("Rolling back created default resources") time.Sleep(5 * time.Second) - log.Info().Msg("Try rollback created default resources") - rollbackResult, rollbackErr := resource.DelAllSharedResources(nsId) - if rollbackErr != nil { - err = fmt.Errorf("Failed in rollback operation: %w", rollbackErr) + if rollbackResult, rollbackErr := resource.DelAllSharedResources(nsId); rollbackErr != nil { + return emptyMci, fmt.Errorf("failed in rollback operation: %w", rollbackErr) } else { ids := strings.Join(rollbackResult.IdList, ", ") - err = fmt.Errorf("Rollback results [%s]: %w", ids, err) + return emptyMci, fmt.Errorf("rollback results [%s]: %w", ids, err) } - return emptyMci, err } - mciReq.Vm = append(mciReq.Vm, *vmReq) } + // Log the prepared MCI request and update the progress common.PrintJsonPretty(mciReq) - common.UpdateRequestProgress(reqID, common.ProgressInfo{Title: "Prepared all resources for provisioning MCI:" + mciReq.Name, Info: mciReq, Time: time.Now()}) - common.UpdateRequestProgress(reqID, common.ProgressInfo{Title: "Start provisioning", Time: time.Now()}) - - // Run create MCI with the generated MCI request (option != register) + common.UpdateRequestProgress(reqID, common.ProgressInfo{ + Title: "Prepared all resources for provisioning MCI: " + mciReq.Name, + Info: mciReq, Time: time.Now(), + }) + common.UpdateRequestProgress(reqID, common.ProgressInfo{ + Title: "Start instance provisioning", Time: time.Now(), + }) + + // Run create MCI with the generated MCI request option := "create" if deployOption == "hold" { option = "hold" diff --git a/src/core/resource/common.go b/src/core/resource/common.go index 1f14ecb4..adbbff6e 100644 --- a/src/core/resource/common.go +++ b/src/core/resource/common.go @@ -71,11 +71,11 @@ func init() { validate.RegisterStructValidation(TbVNetReqStructLevelValidation, model.TbVNetReq{}) } -// DelAllResources deletes all TB Resource object of given resourceType +// DelAllResources deletes all TB Resource objects of the given resourceType. func DelAllResources(nsId string, resourceType string, subString string, forceFlag string) (model.IdList, error) { - deletedResources := model.IdList{} - deleteStatus := "" + var mutex sync.Mutex // Protect shared slice access + var wg sync.WaitGroup // Synchronize all goroutines err := common.CheckString(nsId) if err != nil { @@ -89,27 +89,59 @@ func DelAllResources(nsId string, resourceType string, subString string, forceFl } if len(resourceIdList) == 0 { - errString := "There is no " + resourceType + " resource in " + nsId + errString := fmt.Sprintf("There is no %s resource in %s", resourceType, nsId) err := fmt.Errorf(errString) log.Error().Err(err).Msg("") return deletedResources, err } + // Channel to capture errors + errChan := make(chan error, len(resourceIdList)) + + // Process each resourceId concurrently for _, v := range resourceIdList { - // if subString is provided, check the resourceId contains the subString. - if subString == "" || strings.Contains(v, subString) { + // Increment WaitGroup counter + wg.Add(1) + + // Launch a goroutine for each resource deletion + go func(resourceId string) { + defer wg.Done() + common.RandomSleep(0, len(resourceIdList)/10) + + // Check if the resourceId matches the subString criteria + if subString != "" && !strings.Contains(resourceId, subString) { + return + } - deleteStatus = "[Done] " + // Attempt to delete the resource + deleteStatus := "[Done] " errString := "" - err := DelResource(nsId, resourceType, v, forceFlag) + err := DelResource(nsId, resourceType, resourceId, forceFlag) if err != nil { deleteStatus = "[Failed] " errString = " (" + err.Error() + ")" + errChan <- err // Send error to the error channel } - deletedResources.IdList = append(deletedResources.IdList, deleteStatus+resourceType+": "+v+errString) + + // Safely append the result to deletedResources.IdList using mutex + mutex.Lock() + deletedResources.IdList = append(deletedResources.IdList, deleteStatus+resourceType+": "+resourceId+errString) + mutex.Unlock() + }(v) // Pass loop variable as an argument to avoid race conditions + } + + // Wait for all goroutines to complete + wg.Wait() + close(errChan) // Close the error channel + + // Collect any errors from the error channel + for err := range errChan { + if err != nil { + log.Info().Err(err).Msg("error deleting resource") } } + return deletedResources, nil } diff --git a/src/testclient/scripts/sequentialFullTest/test-mci-dynamic-all-for-one.sh b/src/testclient/scripts/sequentialFullTest/test-mci-dynamic-all-for-one.sh index c53221b5..2ee0b654 100755 --- a/src/testclient/scripts/sequentialFullTest/test-mci-dynamic-all-for-one.sh +++ b/src/testclient/scripts/sequentialFullTest/test-mci-dynamic-all-for-one.sh @@ -4,29 +4,65 @@ echo "####################################################################" echo "## test-mci-dynamic-all.sh (parameters: -x (create or delete) -y numVM)" echo "####################################################################" - - source ../init.sh # create or delete option=${OPTION01} subGroupSizeInput=${OPTION02:-1} - PRINT="index,mciName,connectionName,specId,image,subGroupSize,startTime,endTime,elapsedTime,option" echo "${PRINT}" >./mciTest-$option.csv - description="Made in CB-TB" -installMonAgent="no" label="DynamicVM" echo +maxIterations=3 + +specIdArray=( +aws+ap-northeast-1+t2.small +aws+ap-northeast-2+t2.small +aws+ap-northeast-3+t2.small +aws+ap-south-1+t2.small +aws+ap-southeast-1+t2.small +aws+ap-southeast-2+t2.small +aws+ca-central-1+t2.small +aws+eu-central-1+t2.small +aws+eu-west-1+t2.small +aws+eu-west-2+t2.small +aws+eu-west-3+t2.small +aws+sa-east-1+t2.small +aws+us-east-2+t2.small +aws+us-west-1+t2.small +aws+us-west-2+t2.small +gcp+asia-east1+g1-small +gcp+asia-east2+g1-small +gcp+asia-northeast1+g1-small +gcp+asia-northeast2+g1-small +gcp+asia-northeast3+g1-small +gcp+asia-south1+g1-small +gcp+asia-southeast1+g1-small +gcp+asia-southeast2+g1-small +gcp+australia-southeast1+g1-small +gcp+europe-central2+g1-small +) + +specArray="[]" +for specId in "${specIdArray[@]}"; do + spec=$(curl -H "${AUTH}" -sX GET http://$TumblebugServer/tumblebug/ns/system/resources/spec/${specId}) + specArray=$(echo "$specArray" | jq --argjson spec "$spec" '. + [$spec]') + sleep 0.1 +done + +echo "Number of specs: $(echo "$specArray" | jq length)" -specList=$(curl -H "${AUTH}" -sX GET http://$TumblebugServer/tumblebug/ns/system/resources/spec) -specArray=$(jq -r '.spec' <<<"$specList") +commonImage="ubuntu22.04" + +MainMciName="allforone" +mciName=$MainMciName i=0 + for row in $(echo "${specArray}" | jq -r '.[] | @base64'); do { _jq() { @@ -36,16 +72,17 @@ for row in $(echo "${specArray}" | jq -r '.[] | @base64'); do specId=$(_jq '.id') rootDiskType=$(_jq '.rootDiskType') rootDiskSize=$(_jq '.rootDiskSize') - image="ubuntu18.04" - subGroupSize=$subGroupSizeInput - mciName=$specId if [ "${option}" == "create" ]; then - echo "[$i] connection: $connectionName / specId: $specId / image: $image / replica: $subGroupSize " + echo "[$i] connection: $connectionName / specId: $specId / image: $commonImage / replica: $subGroupSizeInput " elif [ "${option}" == "delete" ]; then - echo "[$i] mciName: $mciName / replica: $subGroupSize " + echo "[$i] mciName: $mciName " fi ((i++)) + + if [ $i -ge $maxIterations ]; then + break + fi } done @@ -75,140 +112,59 @@ done SECONDS=0 -specList=$(curl -H "${AUTH}" -sX GET http://$TumblebugServer/tumblebug/ns/system/resources/spec) -specArray=$(jq -r '.spec' <<<"$specList") - -MainMciName="allforone" -mciName=$MainMciName -firstSpecId="" -if [ "${option}" == "delete" ]; then - echo "Terminate and Delete [$mciName]" - curl -H "${AUTH}" -sX DELETE http://$TumblebugServer/tumblebug/ns/$NSID/mci/${mciName}?option=terminate | jq '.' -elif [ "${option}" == "create" ]; then +# Initialize an empty array correctly +vmArray=$(jq -n '[]') +i=0 - i=0 - for row in $(echo "${specArray}" | jq -r '.[] | @base64'); do +for row in $(echo "${specArray}" | jq -r '.[] | @base64'); do { - if [ "${i}" == "0" ]; then - connectionName=$(_jq '.connectionName') - specId=$(_jq '.id') - firstSpecId=$specId - rootDiskType=$(_jq '.rootDiskType') - rootDiskSize=$(_jq '.rootDiskSize') - image="ubuntu18.04" - subGroupSize=$subGroupSizeInput - - mciName=$MainMciName - - echo - echo "mciName: $mciName specId: $specId image: $image connectionName: $connectionName rootDiskType: $rootDiskType rootDiskSize: $rootDiskSize subGroupSize: $subGroupSize " - - startTime=$SECONDS - - echo "Creat MCI dynamic [$mciName]" - VAR1=$(curl -H "${AUTH}" -sX POST http://$TumblebugServer/tumblebug/ns/$NSID/mciDynamic -H 'Content-Type: application/json' -d @- <>./mciTest-$option.csv + _jq() { + echo ${row} | base64 --decode | jq -r ${1} + } + specId=$(_jq '.id') + echo "specId: $specId" - echo "[$i] Elapsed time: $elapsedTime s" - ((i++)) + # Properly append to the JSON array + vmArray=$(echo "$vmArray" | jq --arg commonImage "$commonImage" --arg specId "$specId" --arg subGroupSizeInput "$subGroupSizeInput" '. + [{"commonImage": $commonImage, "commonSpec": $specId, "subGroupSize": $subGroupSizeInput}]') + ((i++)) - _jq() { - echo ${row} | base64 --decode | jq -r ${1} - } + # Break the loop when max iterations are reached + if [ $i -ge $maxIterations ]; then break fi - } - done - - i=0 - for row in $(echo "${specArray}" | jq -r '.[] | @base64'); do - { - _jq() { - echo ${row} | base64 --decode | jq -r ${1} - } - - if [ "${firstSpecId}" != "$(_jq '.id')" ]; then - - connectionName=$(_jq '.connectionName') - - specId=$(_jq '.id') - rootDiskType=$(_jq '.rootDiskType') - rootDiskSize=$(_jq '.rootDiskSize') - image="ubuntu18.04" - subGroupSize=$subGroupSizeInput - - echo - echo "mciName: $mciName specId: $specId image: $image connectionName: $connectionName rootDiskType: $rootDiskType rootDiskSize: $rootDiskSize subGroupSize: $subGroupSize " - sleepDuration=$((1 + $RANDOM% 1000)) - echo "sleepDuration: $sleepDuration" - #sleep $sleepDuration - - startTime=$SECONDS - - echo "Creat VM dynamic [$mciName]" - VAR1=$(curl -H "${AUTH}" -sX POST http://$TumblebugServer/tumblebug/ns/$NSID/mci/$mciName/vmDynamic -H 'Content-Type: application/json' -d @- <>./mciTest-$option.csv +if [ "${option}" == "delete" ]; then + echo "Terminate and Delete [$mciName]" + curl -H "${AUTH}" -sX DELETE http://$TumblebugServer/tumblebug/ns/$NSID/mci/${mciName}?option=terminate | jq '.' +elif [ "${option}" == "create" ]; then + echo "Create MCI dynamic [$mciName]" + response=$(curl -H "${AUTH}" -sX POST http://$TumblebugServer/tumblebug/ns/$NSID/mciDynamic -H 'Content-Type: application/json' -d "$requestBody") + echo "${response}" | jq '.' - echo "[$i] Elapsed time: $elapsedTime s" - ((i++)) - fi - } #& - done - #wait + mciResponse=$(curl -H "${AUTH}" -sX GET http://$TumblebugServer/tumblebug/ns/$NSID/mci/${mciName}) + echo -e "${BOLD}MCI Status Summary: ${mciName}${NC}" + echo "$mciResponse" | jq '.status' + echo -e "${BOLD}Table: All VMs in the MCI : ${mciName}${NC} ${BLUE} ${BOLD}" + echo "$mciResponse" | + jq '.vm | sort_by(.connectionName)' | + jq -r '(["CloudRegion", "SpecName", "ID(TB)", "ID(CSP)", "Status", "PublicIP", "PrivateIP", "DateTime(Created)"] | + (., map(length*"-"))), + (.[] | [.connectionName, .cspSpecName, .id, .cspResourceId, .status, .publicIP, .privateIP, .createdTime]) | @tsv' | + column -t + echo -e "${NC}" fi - - echo "Done!" + duration=$SECONDS printElapsed $@ echo ""