Merge pull request #138 from 863056768/master

feat: support USTC mirror site

cli/init.go

@@ -0,0 +1,6 @@
+package cli
+
+import (
+	_ "github.com/voidint/g/collector/aliyun"
+	_ "github.com/voidint/g/collector/ustc"
+)

collector/aliyun/aliyun_collector.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/PuerkitoBio/goquery"
+	"github.com/voidint/g/collector"
 	"github.com/voidint/g/pkg/errs"
 	httppkg "github.com/voidint/g/pkg/http"
 	"github.com/voidint/g/version"
@@ -16,9 +17,12 @@ import (
 
 const (
 	// DownloadPageURL 阿里云镜像站点网址
-	DownloadPageURL = "https://mirrors.aliyun.com/golang/"
+	DownloadPageDomain = "mirrors.aliyun.com"
+	DownloadPageURL    = "https://" + DownloadPageDomain + "/golang/"
 )
 
+func init() { collector.Register(DownloadPageDomain, NewCollector) }
+
 // Collector 阿里云镜像站点版本采集器
 type Collector struct {
 	url  string
@@ -27,7 +31,7 @@ type Collector struct {
 }
 
 // NewCollector 返回采集器实例
-func NewCollector() (*Collector, error) {
+func NewCollector() (collector.Collector, error) {
 	pURL, err := stdurl.Parse(DownloadPageURL)
 	if err != nil {
 		return nil, err

collector/aliyun/aliyun_collector_test.go

@@ -126,8 +126,8 @@ func TestNewCollector(t *testing.T) {
 			got, err := NewCollector()
 			assert.Equal(t, tt.wantErr, err)
 			if err == nil {
-				assert.NotNil(t, got.pURL)
-				assert.NotNil(t, got.doc)
+				assert.NotNil(t, got.(*Collector).pURL)
+				assert.NotNil(t, got.(*Collector).doc)
 			}
 		})
 	}

collector/aliyun/go_file_item_test.go

@@ -33,7 +33,7 @@ func Test_getGoVersion(t *testing.T) {
 			In: &goFileItem{
 				FileName: "go1.18rc1.darwin-amd64.tar.gz",
 				URL:      "https://mirrors.aliyun.com/golang/go1.18rc1.darwin-amd64.tar.gz",
-				Size:     "107.2 MB",
+				Size:     "137.0 MB",
 			},
 			Expected: "1.18rc1",
 		},

collector/collector.go

@@ -1,13 +1,21 @@
 package collector
 
 import (
+	stdurl "net/url"
 	"strings"
 
-	"github.com/voidint/g/collector/aliyun"
 	"github.com/voidint/g/collector/official"
 	"github.com/voidint/g/version"
 )
 
+var collectors = make(map[string]Builder)
+
+func Register(domain string, b Builder) {
+	collectors[domain] = b
+}
+
+type Builder func() (Collector, error)
+
 // Collector 版本信息采集器
 type Collector interface {
 	// 返回稳定版本列表
@@ -25,16 +33,20 @@ func NewCollector(urls ...string) (c Collector, err error) {
 	if len(urls) == 0 {
 		urls = []string{official.DefaultDownloadPageURL}
 	}
-	for i := range urls {
-		urls[i] = strings.TrimSpace(urls[i])
+	for _, rawUrl := range urls {
+		var url *stdurl.URL
+		url, err = stdurl.Parse(strings.TrimSpace(rawUrl))
+		if err != nil {
+			continue
+		}
 
-		if urls[i] != "" && (strings.HasPrefix(aliyun.DownloadPageURL, urls[i]) || strings.HasPrefix(urls[i], aliyun.DownloadPageURL)) {
-			if c, err = aliyun.NewCollector(); err == nil {
-				return c, nil
+		for domain, c := range collectors {
+			if url.Host == strings.ToLower(domain) {
+				return c()
 			}
 		}
 
-		if c, err = official.NewCollector(urls[i]); err == nil {
+		if c, err = official.NewCollector(rawUrl); err == nil {
 			return c, nil
 		}
 	}

collector/ustc/go_file_item.go

@@ -0,0 +1,147 @@
+package ustc
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"unicode"
+
+	"github.com/voidint/g/pkg/checksum"
+	"github.com/voidint/g/version"
+)
+
+type goFileItem struct {
+	FileName string
+	URL      string
+	Size     string
+}
+
+func (item goFileItem) getGoVersion() string {
+	arr := strings.Split(strings.TrimPrefix(item.FileName, "go"), ".")
+	if len(arr) < 3 || !unicode.IsNumber(rune(arr[0][0])) {
+		return ""
+	}
+	if unicode.IsNumber(rune(arr[2][0])) {
+		return fmt.Sprintf("%s.%s.%s", arr[0], arr[1], arr[2])
+	}
+
+	if unicode.IsNumber(rune(arr[1][0])) {
+		return fmt.Sprintf("%s.%s", arr[0], arr[1])
+	}
+	return arr[0]
+}
+
+func (item goFileItem) isSHA256File() bool {
+	return strings.HasSuffix(item.FileName, ".sha256")
+}
+
+func (item goFileItem) isPackageFile() bool {
+	return strings.HasSuffix(item.FileName, ".tar.gz") ||
+		strings.HasSuffix(item.FileName, ".pkg") ||
+		strings.HasSuffix(item.FileName, ".zip") ||
+		strings.HasSuffix(item.FileName, ".msi")
+}
+
+func (item goFileItem) getKind() version.PackageKind {
+	if strings.HasSuffix(item.FileName, ".src.tar.gz") {
+		return version.SourceKind
+	}
+	if strings.HasSuffix(item.FileName, ".tar.gz") || strings.HasSuffix(item.FileName, ".zip") {
+		return version.ArchiveKind
+	}
+	if strings.HasSuffix(item.FileName, ".pkg") || strings.HasSuffix(item.FileName, ".msi") {
+		return version.InstallerKind
+	}
+	return "Unknown"
+}
+
+var osMapping = map[string]string{
+	"linux":     "Linux",
+	"darwin":    "macOS",
+	"windows":   "Windows",
+	"freebsd":   "FreeBSD",
+	"netbsd":    "netbsd",
+	"openbsd":   "openbsd",
+	"solaris":   "solaris",
+	"plan9":     "plan9",
+	"aix":       "aix",
+	"dragonfly": "dragonfly",
+	"illumos":   "illumos",
+}
+
+func (item goFileItem) getOS() string {
+	for k, v := range osMapping {
+		if strings.Contains(item.FileName, k) {
+			return v
+		}
+	}
+	return ""
+}
+
+var archMapping = map[string]string{
+	"-386.":      "x86",
+	"-amd64.":    "x86-64",
+	"-arm.":      "ARMv6",
+	"-arm64.":    "ARM64",
+	"-armv6l.":   "ARMv6",
+	"-ppc64.":    "ppc64",
+	"-ppc64le.":  "ppc64le",
+	"-mips.":     "mips",
+	"-mipsle.":   "mipsle",
+	"-mips64.":   "mips64",
+	"-mips64le.": "mips64le",
+	"-s390x.":    "s390x",
+	"-riscv64.":  "riscv64",
+	"-loong64.":  "loong64",
+}
+
+func (item goFileItem) getArch() string {
+	for k, v := range archMapping {
+		if strings.Contains(item.FileName, k) {
+			return v
+		}
+	}
+	return ""
+}
+
+func convert2Versions(items []*goFileItem) (vers []*version.Version, err error) {
+	pkgMap := make(map[string][]*version.Package, 20)
+
+	for _, pitem := range items {
+		ver := pitem.getGoVersion()
+		if _, ok := pkgMap[ver]; !ok {
+			pkgMap[ver] = make([]*version.Package, 0, 20)
+		}
+
+		if pitem.isPackageFile() {
+			pkgMap[ver] = append(pkgMap[ver], &version.Package{
+				FileName: pitem.FileName,
+				URL:      pitem.URL,
+				Kind:     pitem.getKind(),
+				OS:       pitem.getOS(),
+				Arch:     pitem.getArch(),
+				Size:     pitem.Size,
+			})
+		} else if pitem.isSHA256File() {
+			// 设置校验和及算法
+			for _, ppkg := range pkgMap[ver] {
+				if !strings.HasPrefix(pitem.FileName, ppkg.FileName) {
+					continue
+				}
+				ppkg.Algorithm = string(checksum.SHA256)
+				ppkg.ChecksumURL = pitem.URL
+			}
+		}
+	}
+
+	vers = make([]*version.Version, 0, len(pkgMap))
+	for vname, pkgs := range pkgMap {
+		v, err := version.New(vname, version.WithPackages(pkgs))
+		if err != nil {
+			return nil, err
+		}
+		vers = append(vers, v)
+	}
+	sort.Sort(version.Collection(vers))
+	return vers, nil
+}