diff --git a/pkg/sync/handler.go b/pkg/sync/handler.go index e558f3fd..e0421b4f 100644 --- a/pkg/sync/handler.go +++ b/pkg/sync/handler.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "strconv" + "strings" "time" "github.com/pkg/errors" @@ -39,7 +40,7 @@ func (h *HTTPHandler) GetSizeFromURL(url string) (size int64, err error) { return 0, err } - client := http.Client{} + client := NewDownloadHttpClient() resp, err := client.Do(rr) if err != nil { return 0, err @@ -72,7 +73,7 @@ func (h *HTTPHandler) DownloadFromURL(ctx context.Context, url, filePath string, return 0, err } - client := http.Client{} + client := NewDownloadHttpClient() resp, err := client.Do(rr) if err != nil { return 0, err @@ -176,6 +177,27 @@ func IdleTimeoutCopy(ctx context.Context, cancel context.CancelFunc, src io.Read return copied, err } +func removeReferer(req *http.Request) { + for k := range req.Header { + if strings.ToLower(k) == "referer" { + delete(req.Header, k) + } + } +} + +func NewDownloadHttpClient() http.Client { + return http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + // Remove the "Referer" header to enable downloads of files + // that are delivered via CDN and therefore may be redirected + // several times. This is the same behaviour of curl or wget + // in their default configuration. + removeReferer(req) + return nil + }, + } +} + const ( MockFileSize = 4096 ) diff --git a/pkg/sync/handler_test.go b/pkg/sync/handler_test.go new file mode 100644 index 00000000..bc612058 --- /dev/null +++ b/pkg/sync/handler_test.go @@ -0,0 +1,20 @@ +package sync + +import ( + . "gopkg.in/check.v1" + "net/http" +) + +type TestSuite struct{} + +var _ = Suite(&TestSuite{}) + +func (s *TestSuite) TestRemoveReferer(c *C) { + req, err := http.NewRequest("HEAD", "https://foo.bar", nil) + c.Assert(err, IsNil) + req.Header.Set("Referer", "https://foo.bar") + req.Header.Set("Foo", "foo") + removeReferer(req) + c.Assert(req.Referer(), Equals, "") + c.Assert(req.Header, HasLen, 1) +}