Skip to content

Commit c2d90a1

Browse files
powerkimhubpowerkimhub
authored
Merge pull request #1266 from ish-hcc/fix_azure_image
Azure: Improve of getting image list
2 parents 38baf5d + 2c49697 commit c2d90a1

File tree

1 file changed

+157
-60
lines changed

1 file changed

+157
-60
lines changed

‎cloud-control-manager/cloud-driver/drivers/azure/resources/ImageHandler.go

Lines changed: 157 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,11 @@ import (
44
"context"
55
"errors"
66
"fmt"
7-
"sort"
7+
"regexp"
8+
"strconv"
89
"strings"
910
"sync"
11+
"time"
1012

1113
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-03-01/compute"
1214
"github.com/Azure/go-autorest/autorest/to"
@@ -127,6 +129,23 @@ func (imageHandler *AzureImageHandler) CreateImage(imageReqInfo irs.ImageReqInfo
127129
return imageInfo, nil
128130
}
129131

132+
func checkRequest(errMessage string) (repeat bool) {
133+
pattern := `Please try again after '(\d+)' seconds`
134+
re := regexp.MustCompile(pattern)
135+
136+
matches := re.FindStringSubmatch(errMessage)
137+
138+
if len(matches) > 1 {
139+
number := matches[1]
140+
sec, _ := strconv.Atoi(number)
141+
time.Sleep(time.Second * time.Duration(sec))
142+
143+
return true
144+
}
145+
146+
return false
147+
}
148+
130149
func (imageHandler *AzureImageHandler) ListImage() ([]*irs.ImageInfo, error) {
131150
// log HisCall
132151
hiscallInfo := GetCallLogScheme(imageHandler.Region, call.VMIMAGE, Image, "ListImage()")
@@ -143,21 +162,28 @@ func (imageHandler *AzureImageHandler) ListImage() ([]*irs.ImageInfo, error) {
143162

144163
var publisherNames []string
145164
for _, p := range *publishers.Value {
146-
if p.Name == nil ||
147-
strings.Contains(strings.ToLower(*p.Name), "test") {
165+
if p.Name == nil {
148166
continue
149167
}
150168
publisherNames = append(publisherNames, *p.Name)
151169
}
152-
sort.Strings(publisherNames)
153170

154-
var routineMax = 200
171+
var routineMax = 300
155172
var wait sync.WaitGroup
156173
var mutex = &sync.Mutex{}
157174
var lenPublisherNames = len(publisherNames)
158175
var errList []string
159176
var errMutex = &sync.Mutex{}
160177

178+
type imageSearchInfo struct {
179+
PublisherName string
180+
OfferName string
181+
Sku string
182+
Version string
183+
}
184+
var imageSearchInfos []imageSearchInfo
185+
var imageSearchInfosMutex = &sync.Mutex{}
186+
161187
for i := 0; i < lenPublisherNames; {
162188
if lenPublisherNames-i < routineMax {
163189
routineMax = lenPublisherNames - i
@@ -166,15 +192,28 @@ func (imageHandler *AzureImageHandler) ListImage() ([]*irs.ImageInfo, error) {
166192
wait.Add(routineMax)
167193

168194
for j := 0; j < routineMax; j++ {
169-
go func(ctx context.Context, wait *sync.WaitGroup, mutex *sync.Mutex, errList []string, errMutex *sync.Mutex, pName string) {
195+
pName := publisherNames[i]
196+
go func(imageHandler *AzureImageHandler, wait *sync.WaitGroup, mutex *sync.Mutex, errList []string, errMutex *sync.Mutex, pName string) {
170197
defer wait.Done()
171-
offers, err := imageHandler.VMImageClient.ListOffers(ctx, imageHandler.Region.Region, pName)
172-
if err != nil {
173-
errMutex.Lock()
174-
errList = append(errList, err.Error())
175-
errMutex.Unlock()
176198

177-
return
199+
var err error
200+
var offers compute.ListVirtualMachineImageResource
201+
202+
for {
203+
offers, err = imageHandler.VMImageClient.ListOffers(imageHandler.Ctx, imageHandler.Region.Region, pName)
204+
if err != nil {
205+
if checkRequest(err.Error()) {
206+
continue
207+
}
208+
209+
cblogger.Error(err)
210+
errMutex.Lock()
211+
errList = append(errList, err.Error())
212+
errMutex.Unlock()
213+
214+
return
215+
}
216+
break
178217
}
179218

180219
if offers.Value == nil {
@@ -183,38 +222,44 @@ func (imageHandler *AzureImageHandler) ListImage() ([]*irs.ImageInfo, error) {
183222

184223
var offerNames []string
185224
for _, o := range *offers.Value {
186-
if o.Name == nil ||
187-
strings.Contains(strings.ToLower(*o.Name), "test") ||
188-
strings.Contains(strings.ToLower(*o.Name), "preview") ||
189-
strings.Contains(strings.ToLower(*o.Name), "daily") {
225+
if o.Name == nil {
190226
continue
191227
}
192228
offerNames = append(offerNames, *o.Name)
193229
}
194-
sort.Strings(offerNames)
195230

196231
var lenOfferNames = len(offerNames)
197232
var wait2 sync.WaitGroup
198-
var routineMax2 = 150
199233

200234
for i := 0; i < lenOfferNames; {
201-
if lenOfferNames-i < routineMax2 {
202-
routineMax2 = lenOfferNames - i
235+
if lenOfferNames-i < routineMax {
236+
routineMax = lenOfferNames - i
203237
}
204238

205-
wait2.Add(routineMax2)
239+
wait2.Add(routineMax)
206240

207-
for j := 0; j < routineMax2; j++ {
241+
for j := 0; j < routineMax; j++ {
208242
go func(wait2 *sync.WaitGroup, oName string) {
209243
defer wait2.Done()
210244

211-
skus, err := imageHandler.VMImageClient.ListSkus(ctx, imageHandler.Region.Region, pName, oName)
212-
if err != nil {
213-
errMutex.Lock()
214-
errList = append(errList, err.Error())
215-
errMutex.Unlock()
245+
var err error
246+
var skus compute.ListVirtualMachineImageResource
216247

217-
return
248+
for {
249+
skus, err = imageHandler.VMImageClient.ListSkus(imageHandler.Ctx, imageHandler.Region.Region, pName, oName)
250+
if err != nil {
251+
if checkRequest(err.Error()) {
252+
continue
253+
}
254+
255+
cblogger.Error(err)
256+
errMutex.Lock()
257+
errList = append(errList, err.Error())
258+
errMutex.Unlock()
259+
260+
return
261+
}
262+
break
218263
}
219264

220265
if skus.Value == nil {
@@ -228,66 +273,66 @@ func (imageHandler *AzureImageHandler) ListImage() ([]*irs.ImageInfo, error) {
228273
}
229274
skuNames = append(skuNames, *s.Name)
230275
}
231-
sort.Strings(skuNames)
232276

233277
var lenSkuNames = len(skuNames)
234278
var wait3 sync.WaitGroup
235-
var routineMax3 = 100
236279

237280
for i := 0; i < lenSkuNames; {
238-
if lenSkuNames-i < routineMax3 {
239-
routineMax3 = lenSkuNames - i
281+
if lenSkuNames-i < routineMax {
282+
routineMax = lenSkuNames - i
240283
}
241284

242-
wait3.Add(routineMax3)
285+
wait3.Add(routineMax)
243286

244-
for j := 0; j < routineMax3; j++ {
287+
for j := 0; j < routineMax; j++ {
245288
go func(wait3 *sync.WaitGroup, sName string) {
246289
defer wait3.Done()
247290

248-
imageVersionList, err := imageHandler.VMImageClient.List(ctx, imageHandler.Region.Region, pName, oName, sName, "", nil, "")
249-
if err != nil {
250-
errMutex.Lock()
251-
errList = append(errList, err.Error())
252-
errMutex.Unlock()
291+
var err error
292+
var imageVersionList compute.ListVirtualMachineImageResource
253293

254-
return
294+
for {
295+
imageVersionList, err = imageHandler.VMImageClient.List(imageHandler.Ctx, imageHandler.Region.Region, pName, oName, sName, "", nil, "")
296+
if err != nil {
297+
if checkRequest(err.Error()) {
298+
continue
299+
}
300+
301+
errMutex.Lock()
302+
cblogger.Error(err)
303+
errList = append(errList, err.Error())
304+
errMutex.Unlock()
305+
306+
return
307+
}
308+
break
255309
}
256310

257311
if imageVersionList.Value == nil {
258312
return
259313
}
260314

261-
var imageVersions []string
262315
for _, iv := range *imageVersionList.Value {
263316
if iv.ID == nil {
264317
continue
265318
}
266-
imageVersions = append(imageVersions, *iv.ID)
267-
}
268-
sort.Strings(skuNames)
269319

270-
for _, vID := range imageVersions {
271-
imageIdArr := strings.Split(vID, "/")
320+
imageIdArr := strings.Split(*iv.ID, "/")
272321
imageVersion := imageIdArr[len(imageIdArr)-1]
273322

274-
vmImage, err := imageHandler.VMImageClient.Get(ctx, imageHandler.Region.Region, pName, oName, sName, imageVersion)
275-
if err != nil {
276-
errMutex.Lock()
277-
errList = append(errList, err.Error())
278-
errMutex.Unlock()
279-
280-
continue
281-
}
282-
vmImageInfo := imageHandler.setterVMImage(vmImage)
283-
mutex.Lock()
284-
imageList = append(imageList, vmImageInfo)
285-
mutex.Unlock()
323+
imageSearchInfosMutex.Lock()
324+
imageSearchInfos = append(imageSearchInfos, imageSearchInfo{
325+
PublisherName: pName,
326+
OfferName: oName,
327+
Sku: sName,
328+
Version: imageVersion,
329+
})
330+
imageSearchInfosMutex.Unlock()
286331
}
287332
}(&wait3, skuNames[i])
288333

289334
i++
290-
if i == lenPublisherNames {
335+
if i == lenSkuNames {
291336
break
292337
}
293338
}
@@ -305,7 +350,7 @@ func (imageHandler *AzureImageHandler) ListImage() ([]*irs.ImageInfo, error) {
305350

306351
wait2.Wait()
307352
}
308-
}(imageHandler.Ctx, &wait, mutex, errList, errMutex, publisherNames[i])
353+
}(imageHandler, &wait, mutex, errList, errMutex, pName)
309354

310355
i++
311356
if i == lenPublisherNames {
@@ -315,6 +360,58 @@ func (imageHandler *AzureImageHandler) ListImage() ([]*irs.ImageInfo, error) {
315360

316361
wait.Wait()
317362
}
363+
364+
if len(errList) == 0 {
365+
var lenImageSearchInfos = len(imageSearchInfos)
366+
var wait4 sync.WaitGroup
367+
368+
for i := 0; i < lenImageSearchInfos; {
369+
if lenImageSearchInfos-i < routineMax {
370+
routineMax = lenImageSearchInfos - i
371+
}
372+
373+
wait4.Add(routineMax)
374+
375+
for j := 0; j < routineMax; j++ {
376+
go func(imageHandler *AzureImageHandler, wait4 *sync.WaitGroup, isInfo imageSearchInfo) {
377+
defer wait4.Done()
378+
379+
var err error
380+
var vmImage compute.VirtualMachineImage
381+
382+
for {
383+
vmImage, err = imageHandler.VMImageClient.Get(imageHandler.Ctx, imageHandler.Region.Region, isInfo.PublisherName, isInfo.OfferName, isInfo.Sku, isInfo.Version)
384+
if err != nil {
385+
if checkRequest(err.Error()) {
386+
continue
387+
}
388+
389+
cblogger.Error(err)
390+
errMutex.Lock()
391+
errList = append(errList, err.Error())
392+
errMutex.Unlock()
393+
394+
return
395+
}
396+
break
397+
}
398+
399+
vmImageInfo := imageHandler.setterVMImage(vmImage)
400+
mutex.Lock()
401+
imageList = append(imageList, vmImageInfo)
402+
mutex.Unlock()
403+
}(imageHandler, &wait4, imageSearchInfos[i])
404+
405+
i++
406+
if i == lenImageSearchInfos {
407+
break
408+
}
409+
}
410+
411+
wait4.Wait()
412+
}
413+
}
414+
318415
if len(errList) > 0 {
319416
cblogger.Error(strings.Join(errList, "\n"))
320417
return nil, errors.New(strings.Join(errList, "\n"))

0 commit comments

Comments
 (0)