diff --git a/.github/workflows/fetch.yml b/.github/workflows/fetch.yml index 7239f3e3..b8581dd1 100644 --- a/.github/workflows/fetch.yml +++ b/.github/workflows/fetch.yml @@ -354,7 +354,7 @@ jobs: --health-timeout 5s --health-retries 5 env: - Version: 32 33 34 35 36 37 38 39 40 + Version: 32 33 34 35 36 37 38 39 40 41 steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 diff --git a/fetcher/fedora/fedora.go b/fetcher/fedora/fedora.go index e0ace76c..f7ad89aa 100644 --- a/fetcher/fedora/fedora.go +++ b/fetcher/fedora/fedora.go @@ -7,6 +7,8 @@ import ( "errors" "fmt" "net/url" + "path" + "path/filepath" "regexp" "strconv" "strings" @@ -192,7 +194,7 @@ var errNoUpdateInfoField = xerrors.New("No updateinfo field in the repomd") func fetchUpdateInfosFedora(results []util.FetchResult) ([]util.FetchResult, error) { log15.Info("start fetch updateinfo in repomd.xml") - updateInfoReqs, err := extractInfoFromRepoMd(results, "updateinfo", util.MIMETypeXz) + updateInfoReqs, err := extractInfoFromRepoMd(results, "updateinfo") if err != nil { return nil, xerrors.Errorf("Failed to extract updateinfo from xml, err: %w", err) } @@ -274,7 +276,7 @@ func fetchModuleFeedFilesFedora(reqs []util.FetchRequest) ([]util.FetchResult, e func fetchModulesYamlFedora(results []util.FetchResult) (moduleInfosPerVersion, error) { log15.Info("start fetch modules.yaml in repomd.xml") - updateInfoReqs, err := extractInfoFromRepoMd(results, "modules", util.MIMETypeGzip) + updateInfoReqs, err := extractInfoFromRepoMd(results, "modules") if err != nil { return nil, xerrors.Errorf("Failed to extract modules from xml, err: %w", err) } @@ -377,7 +379,7 @@ func fetchCveIDsFromBugzilla(id string) ([]string, error) { return ids, nil } -func extractInfoFromRepoMd(results []util.FetchResult, rt string, mt util.MIMEType) ([]util.FetchRequest, error) { +func extractInfoFromRepoMd(results []util.FetchResult, rt string) ([]util.FetchRequest, error) { var updateInfoReqs []util.FetchRequest for _, r := range results { var repoMd repoMd @@ -394,6 +396,23 @@ func extractInfoFromRepoMd(results []util.FetchResult, rt string, mt util.MIMETy return nil, xerrors.Errorf("Failed to parse URL in XML. err: %w", err) } u.Path = strings.Replace(u.Path, "repodata/repomd.xml", repo.Location.Href, 1) + + mt, err := func() (util.MIMEType, error) { + switch ext := filepath.Ext(path.Base(repo.Location.Href)); ext { + case ".gz": + return util.MIMETypeGzip, nil + case ".xz": + return util.MIMETypeXz, nil + case ".zst": + return util.MIMETypeZst, nil + default: + return util.MIMETypeUnknown, xerrors.Errorf("%q is not supported extension", ext) + } + }() + if err != nil { + return nil, xerrors.Errorf("Failed to get MIME Type. err: %w", err) + } + req := util.FetchRequest{ URL: u.String(), Target: r.Target, diff --git a/fetcher/util/fetcher.go b/fetcher/util/fetcher.go index c5d1d782..3c5e60fc 100644 --- a/fetcher/util/fetcher.go +++ b/fetcher/util/fetcher.go @@ -12,6 +12,7 @@ import ( "time" "github.com/inconshreveable/log15" + "github.com/klauspost/compress/zstd" "github.com/spf13/viper" "github.com/ulikunitz/xz" "golang.org/x/xerrors" @@ -23,8 +24,10 @@ import ( type MIMEType int const ( + // MIMETypeUnknown : + MIMETypeUnknown MIMEType = iota // MIMETypeXML : - MIMETypeXML MIMEType = iota + MIMETypeXML // MIMETypeTxt : MIMETypeTxt // MIMETypeJSON : @@ -39,6 +42,8 @@ const ( MIMETypeXz // MIMETypeGzip : MIMETypeGzip + // MIMETypeZst : + MIMETypeZst ) func (m MIMEType) String() string { @@ -59,6 +64,8 @@ func (m MIMEType) String() string { return "xz" case MIMETypeGzip: return "gz" + case MIMETypeZst: + return "zst" default: return "Unknown" } @@ -195,6 +202,8 @@ func fetchFileWithUA(req FetchRequest) (body []byte, err error) { return nil, err } + resp.Header.Get("Content-Type") + var b bytes.Buffer switch req.MIMEType { case MIMETypeXML, MIMETypeTxt, MIMETypeJSON, MIMETypeYml, MIMETypeHTML: @@ -219,6 +228,14 @@ func fetchFileWithUA(req FetchRequest) (body []byte, err error) { if _, err = b.ReadFrom(r); err != nil { return nil, xerrors.Errorf("Failed to read gzip file. err: %w", err) } + case MIMETypeZst: + r, err := zstd.NewReader(bytes.NewReader(buf.Bytes())) + if err != nil { + return nil, xerrors.Errorf("Failed to open zstd file. err: %w", err) + } + if _, err = b.ReadFrom(r); err != nil { + return nil, xerrors.Errorf("Failed to read zstd file. err: %w", err) + } } return b.Bytes(), nil diff --git a/go.mod b/go.mod index 34a6cd1e..0162aa68 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/inconshreveable/log15 v3.0.0-testing.5+incompatible github.com/k0kubun/pp v3.0.1+incompatible + github.com/klauspost/compress v1.17.11 github.com/knqyf263/go-rpm-version v0.0.0-20220614171824-631e686d1075 github.com/labstack/echo/v4 v4.12.0 github.com/mitchellh/go-homedir v1.1.0 diff --git a/go.sum b/go.sum index 9dfeb7f6..1777866d 100644 --- a/go.sum +++ b/go.sum @@ -62,6 +62,8 @@ github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQ github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v3.0.1+incompatible h1:3tqvf7QgUnZ5tXO6pNAZlrvHgl6DvifjDrd9g2S9Z40= github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/knqyf263/go-rpm-version v0.0.0-20220614171824-631e686d1075 h1:aC6MEAs3PE3lWD7lqrJfDxHd6hcced9R4JTZu85cJwU= github.com/knqyf263/go-rpm-version v0.0.0-20220614171824-631e686d1075/go.mod h1:i4sF0l1fFnY1aiw08QQSwVAFxHEm311Me3WsU/X7nL0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=