diff --git a/api-runtime/common-runtime/VMManager.go b/api-runtime/common-runtime/VMManager.go index b883ca5a..7cd357c6 100644 --- a/api-runtime/common-runtime/VMManager.go +++ b/api-runtime/common-runtime/VMManager.go @@ -546,7 +546,7 @@ func StartVM(connectionName string, rsType string, reqInfo cres.VMReqInfo, IDTra MSG string } - waiter := NewWaiter(5, 240) // (sleep, timeout) + waiter := NewWaiter(15, 600) // (sleep, timeout) var publicIP string for { vmInfo, err := handler.GetVM(info.IId) @@ -1657,7 +1657,7 @@ func DeleteVM(connectionName string, rsType string, nameID string, force string) } // Check Sync Called - waiter := NewWaiter(5, 600) // (sleep, timeout) + waiter := NewWaiter(15, 600) // (sleep, timeout) for { status, err := handler.(cres.VMHandler).GetVMStatus(driverIId) diff --git a/cloud-control-manager/cloud-driver/drivers/aws/profile/CallCount.go b/cloud-control-manager/cloud-driver/drivers/aws/profile/CallCount.go index d2f287d8..e2fc4119 100644 --- a/cloud-control-manager/cloud-driver/drivers/aws/profile/CallCount.go +++ b/cloud-control-manager/cloud-driver/drivers/aws/profile/CallCount.go @@ -11,25 +11,25 @@ import ( var cblog = cblogger.GetLogger("CLOUD-BARISTA") -var apiCallCount int -var mutex sync.Mutex +var AWSAPICallCount int +var AWSMutex sync.Mutex func incrementCallCount() { - mutex.Lock() - defer mutex.Unlock() - apiCallCount++ + AWSMutex.Lock() + defer AWSMutex.Unlock() + AWSAPICallCount++ } func ResetCallCount() { - mutex.Lock() - defer mutex.Unlock() - apiCallCount = 0 + AWSMutex.Lock() + defer AWSMutex.Unlock() + AWSAPICallCount = 0 } func GetCallCount() int { - mutex.Lock() - defer mutex.Unlock() - return apiCallCount + AWSMutex.Lock() + defer AWSMutex.Unlock() + return AWSAPICallCount } // creates a new AWS session that counts the number of API calls made diff --git a/cloud-control-manager/cloud-driver/drivers/azure/profile/CallCount.go b/cloud-control-manager/cloud-driver/drivers/azure/profile/CallCount.go index f7d2e932..168494f2 100644 --- a/cloud-control-manager/cloud-driver/drivers/azure/profile/CallCount.go +++ b/cloud-control-manager/cloud-driver/drivers/azure/profile/CallCount.go @@ -10,25 +10,25 @@ import ( var cblog = cblogger.GetLogger("CLOUD-BARISTA") -var apiCallCount int -var mutex sync.Mutex +var azureAPICallCount int +var azureMutex sync.Mutex func incrementCallCount() { - mutex.Lock() - defer mutex.Unlock() - apiCallCount++ + azureMutex.Lock() + defer azureMutex.Unlock() + azureAPICallCount++ } func ResetCallCount() { - mutex.Lock() - defer mutex.Unlock() - apiCallCount = 0 + azureMutex.Lock() + defer azureMutex.Unlock() + azureAPICallCount = 0 } func GetCallCount() int { - mutex.Lock() - defer mutex.Unlock() - return apiCallCount + azureMutex.Lock() + defer azureMutex.Unlock() + return azureAPICallCount } // NewCountingPolicy creates a new custom policy to count Azure SDK API calls. diff --git a/cloud-control-manager/cloud-driver/drivers/azure/resources/VMHandler.go b/cloud-control-manager/cloud-driver/drivers/azure/resources/VMHandler.go index deffbef5..c615c398 100644 --- a/cloud-control-manager/cloud-driver/drivers/azure/resources/VMHandler.go +++ b/cloud-control-manager/cloud-driver/drivers/azure/resources/VMHandler.go @@ -449,7 +449,7 @@ func (vmHandler *AzureVMHandler) StartVM(vmReqInfo irs.VMReqInfo) (irs.VMInfo, e return irs.VMInfo{}, createErr } curRetryCnt := 0 - maxRetryCnt := 120 + maxRetryCnt := 40 // 15sec * 40 = 10min // 5. Wait Running for { resp, _ := vmHandler.Client.InstanceView(vmHandler.Ctx, vmHandler.Region.Region, vmReqInfo.IId.NameId, nil) @@ -459,7 +459,7 @@ func (vmHandler *AzureVMHandler) StartVM(vmReqInfo irs.VMReqInfo) (irs.VMInfo, e break } curRetryCnt++ - time.Sleep(1 * time.Second) + time.Sleep(15 * time.Second) if curRetryCnt > maxRetryCnt { createErr := errors.New(fmt.Sprintf("Failed to Start VM. exceeded maximum retry count %d and Finished to rollback deleting", maxRetryCnt)) cleanErr := vmHandler.cleanDeleteVm(vmReqInfo.IId) @@ -1405,7 +1405,7 @@ type CleanVMClientSet struct { // VMCleanRelatedResource func (vmHandler *AzureVMHandler) cleanVMRelatedResource(cleanRelatedResource VMCleanRelatedResource) (bool, error) { curRetryCnt := 0 - maxRetryCnt := 120 + maxRetryCnt := 40 // 15sec * 40 = 10min (Total waiting time for deletion of related resources) networkInterfaceName := cleanRelatedResource.CleanTargetResource.NetworkInterfaceName publicIPId := cleanRelatedResource.CleanTargetResource.PublicIPName @@ -1445,7 +1445,7 @@ func (vmHandler *AzureVMHandler) cleanVMRelatedResource(cleanRelatedResource VMC } curRetryCnt++ - time.Sleep(1 * time.Second) + time.Sleep(15 * time.Second) if curRetryCnt > maxRetryCnt { createErr := errors.New(fmt.Sprintf("Failed to clean remained vnic ("+networkInterfaceName+"). exceeded maximum retry count %d", maxRetryCnt)) cblogger.Warn(createErr.Error()) @@ -1477,7 +1477,7 @@ func (vmHandler *AzureVMHandler) cleanVMRelatedResource(cleanRelatedResource VMC } curRetryCnt++ - time.Sleep(1 * time.Second) + time.Sleep(15 * time.Second) if curRetryCnt > maxRetryCnt { createErr := errors.New(fmt.Sprintf("Failed to clean remained public IP ("+publicIPId+"). exceeded maximum retry count %d", maxRetryCnt)) cblogger.Warn(createErr.Error()) @@ -1505,7 +1505,7 @@ func (vmHandler *AzureVMHandler) cleanVMRelatedResource(cleanRelatedResource VMC } curRetryCnt++ - time.Sleep(1 * time.Second) + time.Sleep(15 * time.Second) if curRetryCnt > maxRetryCnt { createErr := errors.New(fmt.Sprintf("Failed to clean remained disk ("+vmDiskId+"). exceeded maximum retry count %d", maxRetryCnt)) cblogger.Warn(createErr.Error()) diff --git a/cloud-control-manager/cloud-driver/drivers/gcp/profile/CallCount.go b/cloud-control-manager/cloud-driver/drivers/gcp/profile/CallCount.go index 4f4b3451..ac0263ca 100644 --- a/cloud-control-manager/cloud-driver/drivers/gcp/profile/CallCount.go +++ b/cloud-control-manager/cloud-driver/drivers/gcp/profile/CallCount.go @@ -7,28 +7,28 @@ import ( "golang.org/x/oauth2" ) -var apiCallCount int -var mutex sync.Mutex +var GCPAPICallCount int +var GCPMutex sync.Mutex // incrementCallCount increments the count of API calls. func incrementCallCount() { - mutex.Lock() - defer mutex.Unlock() - apiCallCount++ + GCPMutex.Lock() + defer GCPMutex.Unlock() + GCPAPICallCount++ } // ResetCallCount resets the API call count. func ResetCallCount() { - mutex.Lock() - defer mutex.Unlock() - apiCallCount = 0 + GCPMutex.Lock() + defer GCPMutex.Unlock() + GCPAPICallCount = 0 } // GetCallCount returns the current count of API calls. func GetCallCount() int { - mutex.Lock() - defer mutex.Unlock() - return apiCallCount + GCPMutex.Lock() + defer GCPMutex.Unlock() + return GCPAPICallCount } // NewCountingClient returns an *http.Client that counts each request. diff --git a/cloud-control-manager/cloud-driver/drivers/gcp/resources/VMHandler.go b/cloud-control-manager/cloud-driver/drivers/gcp/resources/VMHandler.go index 9af6271b..673d1391 100644 --- a/cloud-control-manager/cloud-driver/drivers/gcp/resources/VMHandler.go +++ b/cloud-control-manager/cloud-driver/drivers/gcp/resources/VMHandler.go @@ -501,7 +501,7 @@ func (vmHandler *GCPVMHandler) StartVM(vmReqInfo irs.VMReqInfo) (irs.VMInfo, err } break } - time.Sleep(5 * time.Second) + time.Sleep(15 * time.Second) } /* @@ -1318,7 +1318,7 @@ func (vmHandler *GCPVMHandler) WaitForRun(vmIID irs.IID) (irs.VMStatus, error) { // Suspending 되도록 3초 정도 대기 함. //=================================== curRetryCnt := 0 - maxRetryCnt := 120 + maxRetryCnt := 40 // 15sec * 40 = 600sec = 10min for { curStatus, errStatus := vmHandler.GetVMStatus(vmIID) if errStatus != nil { @@ -1334,7 +1334,7 @@ func (vmHandler *GCPVMHandler) WaitForRun(vmIID irs.IID) (irs.VMStatus, error) { //if curStatus != irs.VMStatus(waitStatus) { curRetryCnt++ cblogger.Debugf("The VM status is not [%s], so waiting for 1 second before querying.", waitStatus) - time.Sleep(time.Second * 1) + time.Sleep(time.Second * 15) if curRetryCnt > maxRetryCnt { cblogger.Errorf("Forcibly stopping after waiting for a long time (%d seconds) as the VM's Status value hasn't changed to [%s].", maxRetryCnt, waitStatus) return irs.VMStatus("Failed"), errors.New("Stopped waiting after waiting for a long time, but the status of the created VM did not change to [" + waitStatus + "].")