From 172715d1c40412b944a6d430c2eaa174330f662e Mon Sep 17 00:00:00 2001 From: Seokho Son Date: Thu, 21 Nov 2024 21:45:09 +0900 Subject: [PATCH 1/2] Update gomod gosum --- go.mod | 2 +- go.sum | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 261791fe..998d2d1b 100644 --- a/go.mod +++ b/go.mod @@ -89,7 +89,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect - golang.org/x/net v0.26.0 // indirect + golang.org/x/net v0.26.0 golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect diff --git a/go.sum b/go.sum index c2232faf..a5af5c3f 100644 --- a/go.sum +++ b/go.sum @@ -13,14 +13,6 @@ github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0 github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cloud-barista/mc-terrarium v0.0.10 h1:WrK/bHyiQZ9ZxRLVoUTY4c3CVH7z+vEzdh7yxo9X4ZM= -github.com/cloud-barista/mc-terrarium v0.0.10/go.mod h1:iQxZNRa04d7mHA0h5dEPfF7ch1SBUS/ZFGUynKsKJ6I= -github.com/cloud-barista/mc-terrarium v0.0.15 h1:1yJjCsNMwYZJyYSku9HQPOdj1PwAf5ybYI+DqB1TKsc= -github.com/cloud-barista/mc-terrarium v0.0.15/go.mod h1:iQxZNRa04d7mHA0h5dEPfF7ch1SBUS/ZFGUynKsKJ6I= -github.com/cloud-barista/mc-terrarium v0.0.16 h1:0TpSnFk5IpStpQc1YJSbMzB8q1jgVZ639Cjh7YAyxh4= -github.com/cloud-barista/mc-terrarium v0.0.16/go.mod h1:iQxZNRa04d7mHA0h5dEPfF7ch1SBUS/ZFGUynKsKJ6I= -github.com/cloud-barista/mc-terrarium v0.0.17 h1:W6kyGc2dIKrOMnw0UAOoI66G/bX9Tj/wG4pY9TN9uqI= -github.com/cloud-barista/mc-terrarium v0.0.17/go.mod h1:iQxZNRa04d7mHA0h5dEPfF7ch1SBUS/ZFGUynKsKJ6I= github.com/cloud-barista/mc-terrarium v0.0.18 h1:uDzFOIOoIb7hSXJ5ovrrx8+/hqjHeyNNMU3EDfv8If0= github.com/cloud-barista/mc-terrarium v0.0.18/go.mod h1:iQxZNRa04d7mHA0h5dEPfF7ch1SBUS/ZFGUynKsKJ6I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= From e4233692eb75c99eff403f923217e50418df83f0 Mon Sep 17 00:00:00 2001 From: Seokho Son Date: Mon, 25 Nov 2024 17:57:40 +0900 Subject: [PATCH 2/2] Enhance get image mechanism --- src/api/rest/server/server.go | 5 ++ src/core/common/label/label.go | 6 +-- src/core/infra/provisioning.go | 59 +++++++++++--------- src/core/resource/common.go | 26 +++++++-- src/core/resource/image.go | 98 +++++++++++++++++++++++++--------- 5 files changed, 137 insertions(+), 57 deletions(-) diff --git a/src/api/rest/server/server.go b/src/api/rest/server/server.go index be9612be..00986427 100644 --- a/src/api/rest/server/server.go +++ b/src/api/rest/server/server.go @@ -101,6 +101,11 @@ func RunServer() { APILogSkipPatterns := [][]string{ {"/tumblebug/api"}, {"/mci", "option=status"}, + {"/k8scluster"}, + {"/resources/vNet"}, + {"/resources/securityGroup"}, + {"/resources/vpn"}, + {"/resources/sshKey"}, } e.Use(middlewares.Zerologger(APILogSkipPatterns)) diff --git a/src/core/common/label/label.go b/src/core/common/label/label.go index a261c48a..a82cca46 100644 --- a/src/core/common/label/label.go +++ b/src/core/common/label/label.go @@ -250,7 +250,7 @@ func GetResourcesByLabelSelector(labelType, labelSelector string) ([]interface{} keyValue = kvutil.FilterKvListBy(keyValue, listKey, 1) // Log the number of filtered label entries - log.Info().Int("numLabelEntries", len(keyValue)).Str("listKey", listKey).Msg("Fetched and filtered list of label entries") + //log.Debug().Int("numLabelEntries", len(keyValue)).Str("listKey", listKey).Msg("Fetched and filtered list of label entries") // Get the appropriate resource type constructor resourceConstructor, exists := model.ResourceTypeRegistry[labelType] @@ -264,7 +264,7 @@ func GetResourcesByLabelSelector(labelType, labelSelector string) ([]interface{} labelKey := kv.Key labelData := kv.Value - log.Info().Str("labelKey", labelKey).Msg("Processing label entry") + log.Debug().Str("labelKey", labelKey).Msg("Processing label entry") //log.Debug().Str("labelKey", labelKey).Str("labelData", string(labelData)).Msg("Fetched label data") var labelInfo model.LabelInfo @@ -301,6 +301,6 @@ func GetResourcesByLabelSelector(labelType, labelSelector string) ([]interface{} } } - log.Info().Int("numMatchedResources", len(matchedResources)).Str("labelType", labelType).Msg("Matched resources found") + //log.Debug().Int("numMatchedResources", len(matchedResources)).Str("labelType", labelType).Msg("Matched resources found") return matchedResources, nil } diff --git a/src/core/infra/provisioning.go b/src/core/infra/provisioning.go index d1153675..c6cba835 100644 --- a/src/core/infra/provisioning.go +++ b/src/core/infra/provisioning.go @@ -946,7 +946,7 @@ func CreateMciDynamic(reqID string, nsId string, req *model.TbMciDynamicReq, dep // Check whether VM names meet requirement. errStr := "" for i, k := range vmRequest { - err = checkCommonResAvailableForVmDynamicReq(&k) + err = checkCommonResAvailableForVmDynamicReq(&k, nsId) if err != nil { log.Error().Err(err).Msgf("[%d] Failed to find common resource for MCI provision", i) errStr += "{[" + strconv.Itoa(i+1) + "] " + err.Error() + "} " @@ -1046,7 +1046,7 @@ func CreateMciVmDynamic(nsId string, mciId string, req *model.TbVmDynamicReq) (* } // checkCommonResAvailableForVmDynamicReq is func to check common resources availability for VmDynamicReq -func checkCommonResAvailableForVmDynamicReq(req *model.TbVmDynamicReq) error { +func checkCommonResAvailableForVmDynamicReq(req *model.TbVmDynamicReq, nsId string) error { vmRequest := req // Check whether VM names meet requirement. @@ -1076,20 +1076,20 @@ func checkCommonResAvailableForVmDynamicReq(req *model.TbVmDynamicReq) error { return err } - osType := strings.ReplaceAll(k.CommonImage, " ", "") - vmReq.ImageId = resource.GetProviderRegionZoneResourceKey(connection.ProviderName, connection.RegionDetail.RegionName, "", osType) - // incase of user provided image id completely (e.g. aws+ap-northeast-2+ubuntu22.04) - if strings.Contains(k.CommonImage, "+") { - vmReq.ImageId = k.CommonImage + // 1) try if there is matched custom image in the namespace + _, err = resource.GetImage(nsId, k.CommonImage) + if err == nil { + return nil } - _, err = resource.GetImage(model.SystemCommonNs, vmReq.ImageId) - if err != nil { - err := fmt.Errorf("Failed to get Image " + k.CommonImage + " from " + vmReq.ConnectionName) - log.Error().Err(err).Msg("") - return err + + // 2) try if there is a matched image with modified ImageId by osType in the CommonImage list + modifiedImageKey := resource.GetProviderRegionZoneResourceKey(connection.ProviderName, connection.RegionDetail.RegionName, "", k.CommonImage) + _, err = resource.GetImage(model.SystemCommonNs, modifiedImageKey) + if err == nil { + return nil } - return nil + return err } // getVmReqForDynamicMci is func to getVmReqFromDynamicReq @@ -1129,17 +1129,15 @@ func getVmReqFromDynamicReq(reqID string, nsId string, req *model.TbVmDynamicReq resourceName := nsId + model.StrSharedResourceName + vmReq.ConnectionName vmReq.SpecId = specInfo.Id - osType := strings.ReplaceAll(k.CommonImage, " ", "") - vmReq.ImageId = resource.GetProviderRegionZoneResourceKey(connection.ProviderName, connection.RegionDetail.RegionName, "", osType) - // incase of user provided image id completely (e.g. aws+ap-northeast-2+ubuntu22.04) - if strings.Contains(k.CommonImage, "+") { - vmReq.ImageId = k.CommonImage - } - _, err = resource.GetImage(model.SystemCommonNs, vmReq.ImageId) + vmReq.ImageId = k.CommonImage + _, err = resource.GetImage(nsId, k.CommonImage) if err != nil { - err := fmt.Errorf("Failed to get the Image " + vmReq.ImageId + " from " + vmReq.ConnectionName) - log.Error().Err(err).Msg("") - return &model.TbVmReq{}, err + // set modifiedImageKey if there is no matched custom image in the namespace + modifiedImageKey := resource.GetProviderRegionZoneResourceKey(connection.ProviderName, connection.RegionDetail.RegionName, "", k.CommonImage) + _, err = resource.GetImage(model.SystemCommonNs, modifiedImageKey) + if err == nil { + vmReq.ImageId = modifiedImageKey + } } /* @@ -1157,7 +1155,11 @@ func getVmReqFromDynamicReq(reqID string, nsId string, req *model.TbVmDynamicReq if err != nil { log.Error().Err(err).Msg("Failed to create etcd session") } - defer session.Close() + defer func() { + if err := session.Close(); err != nil { + log.Error().Err(err).Msg("Failed to close etcd session") + } + }() lock, err := kvstore.NewLock(ctx, session, vNetKey) if err != nil { @@ -1168,8 +1170,12 @@ func getVmReqFromDynamicReq(reqID string, nsId string, req *model.TbVmDynamicReq if err != nil { log.Error().Err(err).Msg("Failed to acquire lock") } - // Unlock the lock when the function exits - defer lock.Unlock(ctx) + // Ensure the lock is released when the function exits + defer func() { + if err := lock.Unlock(ctx); err != nil { + log.Error().Err(err).Msg("Failed to release lock") + } + }() common.UpdateRequestProgress(reqID, common.ProgressInfo{Title: "Setting vNet:" + resourceName, Time: time.Now()}) @@ -1297,6 +1303,7 @@ func CreateVmObject(wg *sync.WaitGroup, nsId string, mciId string, vmInfoData *m // CreateVm is func to create VM (option = "register" for register existing VM) func CreateVm(wg *sync.WaitGroup, nsId string, mciId string, vmInfoData *model.TbVmInfo, option string) error { + log.Info().Msgf("Start to create VM: %s", vmInfoData.Name) //goroutin defer wg.Done() diff --git a/src/core/resource/common.go b/src/core/resource/common.go index 219db356..95b02cc0 100644 --- a/src/core/resource/common.go +++ b/src/core/resource/common.go @@ -989,18 +989,15 @@ func GetResource(nsId string, resourceType string, resourceId string) (interface return nil, err } - fmt.Printf("HTTP Status code: %d \n", resp.StatusCode()) switch { case resp.StatusCode() >= 400 || resp.StatusCode() < 200: err := fmt.Errorf(string(resp.Body())) - fmt.Println("body: ", string(resp.Body())) log.Error().Err(err).Msg("") return nil, err } updatedSpiderMyImage := resp.Result().(*model.SpiderMyImageInfo) res.Status = updatedSpiderMyImage.Status - fmt.Printf("res.Status: %s \n", res.Status) // for debug UpdateResourceObject(nsId, model.StrCustomImage, res) return res, nil @@ -1095,6 +1092,7 @@ func GenSpecMapKey(region, specName string) string { return strings.ToLower(fmt.Sprintf("%s-%s", region, specName)) } +// GenResourceKey generates a Resource key for concatenating providerName, regionName, zoneName, resourceName func GetProviderRegionZoneResourceKey(providerName, regionName, zoneName, resourceName string) string { div := "+" @@ -1110,6 +1108,28 @@ func GetProviderRegionZoneResourceKey(providerName, regionName, zoneName, resour return strings.ToLower(fmt.Sprintf("%s%s%s%s%s%s%s", providerName, div, regionName, div, zoneName, div, resourceName)) } +// ResolveProviderRegionZoneResourceKey resolves the Resource key into providerName, regionName, zoneName, resourceName +func ResolveProviderRegionZoneResourceKey(key string) (providerName string, regionName string, zoneName string, resourceName string, err error) { + + div := "+" + + split := strings.Split(key, div) + + if len(split) == 1 { + return "", "", "", "", fmt.Errorf("ResourceKey dose not contain div(%s)", div) + } + + if len(split) == 2 { + return split[0], "", "", split[1], nil + } + + if len(split) == 3 { + return split[0], split[1], "", split[2], nil + } + + return split[0], split[1], split[2], split[3], nil +} + // CheckResource returns the existence of the TB Resource resource in bool form. func CheckResource(nsId string, resourceType string, resourceId string) (bool, error) { diff --git a/src/core/resource/image.go b/src/core/resource/image.go index d23e355f..6d123ea2 100644 --- a/src/core/resource/image.go +++ b/src/core/resource/image.go @@ -543,35 +543,83 @@ func GetImage(nsId string, imageKey string) (model.TbImageInfo, error) { // make comparison case-insensitive nsId = strings.ToLower(nsId) imageKey = strings.ToLower(imageKey) + imageKey = strings.ReplaceAll(imageKey, " ", "") - // ex: tencent+ap-jakarta+ubuntu22.04 - image := model.TbImageInfo{Namespace: nsId, Id: imageKey} - has, err := model.ORM.Where("LOWER(Namespace) = ? AND LOWER(Id) = ?", nsId, imageKey).Get(&image) + providerName, regionName, _, resourceName, err := ResolveProviderRegionZoneResourceKey(imageKey) if err != nil { - log.Info().Err(err).Msgf("Failed to get image %s by ID", imageKey) - } - if has { - return image, nil - } + // imageKey does not include information for providerName, regionName + image := model.TbImageInfo{Namespace: nsId, Id: imageKey} + + // 1) Check if the image is a custom image + // ex: custom-img-487zeit5 + tempInterface, err := GetResource(nsId, model.StrCustomImage, imageKey) + customImage := model.TbCustomImageInfo{} + if err == nil { + err = common.CopySrcToDest(&tempInterface, &customImage) + if err != nil { + log.Error().Err(err).Msg("TbCustomImageInfo CopySrcToDest error") + return model.TbImageInfo{}, err + } + image.CspImageName = customImage.CspResourceName + image.SystemLabel = model.StrCustomImage + return image, nil + } - // ex: img-487zeit5 - image = model.TbImageInfo{Namespace: nsId, CspImageName: imageKey} - has, err = model.ORM.Where("LOWER(Namespace) = ? AND LOWER(CspImageName) = ?", nsId, imageKey).Get(&image) - if err != nil { - log.Info().Err(err).Msgf("Failed to get image %s by CspImageName", imageKey) - } - if has { - return image, nil - } + // 2) Check if the image is a registered image in the given namespace + // ex: img-487zeit5 + image = model.TbImageInfo{Namespace: nsId, Id: imageKey} + has, err := model.ORM.Where("LOWER(Namespace) = ? AND LOWER(Id) = ?", nsId, imageKey).Get(&image) + if err != nil { + log.Info().Err(err).Msgf("Cannot get image %s by ID from %s", imageKey, nsId) + } + if has { + return image, nil + } + + } else { + // imageKey includes information for providerName, regionName + + // 1) Check if the image is a registered image in the common namespace model.SystemCommonNs by ImageId + // ex: tencent+ap-jakarta+ubuntu22.04 or tencent+ap-jakarta+img-487zeit5 + image := model.TbImageInfo{Namespace: model.SystemCommonNs, Id: imageKey} + has, err := model.ORM.Where("LOWER(Namespace) = ? AND LOWER(Id) = ?", model.SystemCommonNs, imageKey).Get(&image) + if err != nil { + log.Info().Err(err).Msgf("Cannot get image %s by ID from %s", imageKey, model.SystemCommonNs) + } + if has { + return image, nil + } + + // 2) Check if the image is a registered image in the common namespace model.SystemCommonNs by CspImageName + // ex: tencent+ap-jakarta+img-487zeit5 + image = model.TbImageInfo{Namespace: model.SystemCommonNs, CspImageName: resourceName} + has, err = model.ORM.Where("LOWER(Namespace) = ? AND LOWER(CspImageName) = ? AND LOWER(Id) LIKE ? AND LOWER(Id) LIKE ?", + model.SystemCommonNs, + resourceName, + "%"+strings.ToLower(providerName)+"%", + "%"+strings.ToLower(regionName)+"%").Get(&image) + if err != nil { + log.Info().Err(err).Msgf("Cannot get image %s by CspImageName", resourceName) + } + if has { + return image, nil + } + + // 3) Check if the image is a registered image in the common namespace model.SystemCommonNs by GuestOS + // ex: tencent+ap-jakarta+Ubuntu22.04 + image = model.TbImageInfo{Namespace: model.SystemCommonNs, GuestOS: resourceName} + has, err = model.ORM.Where("LOWER(Namespace) = ? AND LOWER(GuestOS) LIKE ? AND LOWER(Id) LIKE ? AND LOWER(Id) LIKE ?", + model.SystemCommonNs, + "%"+strings.ToLower(resourceName)+"%", + "%"+strings.ToLower(providerName)+"%", + "%"+strings.ToLower(regionName)+"%").Get(&image) + if err != nil { + log.Info().Err(err).Msgf("Failed to get image %s by GuestOS type", resourceName) + } + if has { + return image, nil + } - // ex: Ubuntu22.04 - image = model.TbImageInfo{Namespace: nsId, GuestOS: imageKey} - has, err = model.ORM.Where("LOWER(Namespace) = ? AND LOWER(GuestOS) LIKE ?", nsId, imageKey).Get(&image) - if err != nil { - log.Info().Err(err).Msgf("Failed to get image %s by GuestOS type", imageKey) - } - if has { - return image, nil } return model.TbImageInfo{}, fmt.Errorf("The imageKey %s not found by any of ID, CspImageName, GuestOS", imageKey)