diff --git a/cachereader.go b/cachereader.go index 6f4ab48..317457b 100644 --- a/cachereader.go +++ b/cachereader.go @@ -5,7 +5,7 @@ import ( "errors" "github.com/whosonfirst/go-cache" "github.com/whosonfirst/go-reader" - "github.com/whosonfirst/go-reader/ioutil" + "github.com/whosonfirst/go-ioutil" "io" "net/url" ) diff --git a/go.mod b/go.mod index 9529740..e904837 100644 --- a/go.mod +++ b/go.mod @@ -4,5 +4,7 @@ go 1.16 require ( github.com/whosonfirst/go-cache v0.2.0 - github.com/whosonfirst/go-reader v0.4.1 + github.com/whosonfirst/go-ioutil v0.0.1 + github.com/whosonfirst/go-reader v0.5.0 + ) diff --git a/go.sum b/go.sum index fff0969..9c9c159 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/whosonfirst/go-cache v0.1.0 h1:w6o4pWT/q2q/uclV72EU3D9CCKOgU+w+D+GKpa github.com/whosonfirst/go-cache v0.1.0/go.mod h1:M5Iche+26mVKFph49/pxS5W4ULZ9Dr01mebPqswG8QU= github.com/whosonfirst/go-cache v0.2.0 h1:S1tKyS5aVCGkQyh9jQHi2u0i0tp0W/vW8CfX9BB6wJw= github.com/whosonfirst/go-cache v0.2.0/go.mod h1:M5Iche+26mVKFph49/pxS5W4ULZ9Dr01mebPqswG8QU= +github.com/whosonfirst/go-ioutil v0.0.1 h1:cCrEYen6NDvHfjzV2q4u/VB21u2kTOwDnUGRlMI8Z9o= +github.com/whosonfirst/go-ioutil v0.0.1/go.mod h1:2dS1vWdAIkiHDvDF8fYyjv6k2NISmwaIjJJeEDBEdvg= github.com/whosonfirst/go-reader v0.0.4 h1:DucfFJB9IY35QU8kzdy8O99Nmn0r/gE7IIBdTg8qyMk= github.com/whosonfirst/go-reader v0.0.4/go.mod h1:TNCpgVgg71qA+jhPltXpGum8gTG5TAJYNGTMJmxFZtQ= github.com/whosonfirst/go-reader v0.1.1 h1:0fXoYQR/Yyh3PKk+9Huwoc6dCAt8l1tjg2dEShjqcFI= @@ -18,3 +20,5 @@ github.com/whosonfirst/go-reader v0.2.0 h1:Et8rvkh+86msAo4Db4h88m1t2QDSMNr3rLy6R github.com/whosonfirst/go-reader v0.2.0/go.mod h1:qUhz3OWefOUX/G1nzCEUzJskDjkF+l9oKPOM3K2fAJI= github.com/whosonfirst/go-reader v0.4.1 h1:gN73LR/gdtWtuCikOc84rtuMIbIu6fi7RPRZgssKL9I= github.com/whosonfirst/go-reader v0.4.1/go.mod h1:ffg8ww1158rNNqStFx64EGmDL5m5WZPsoMqR6wohOWE= +github.com/whosonfirst/go-reader v0.5.0 h1:nx+ai0F6JXouw+7Dln34dmYglw+3sQ6sG4JZGOJ/sqA= +github.com/whosonfirst/go-reader v0.5.0/go.mod h1:4ou/wZUss2CDZp27QK5ySDc8p98GVWvUiqqmwEprjgk= diff --git a/vendor/github.com/whosonfirst/go-ioutil/.gitignore b/vendor/github.com/whosonfirst/go-ioutil/.gitignore new file mode 100644 index 0000000..afa44cd --- /dev/null +++ b/vendor/github.com/whosonfirst/go-ioutil/.gitignore @@ -0,0 +1,11 @@ +*~ +pkg +src +!vendor/src +bin +!bin/.gitignore +*.log +*.json +.travis.yml +*.db +testdata/*.txt \ No newline at end of file diff --git a/vendor/github.com/whosonfirst/go-ioutil/LICENSE b/vendor/github.com/whosonfirst/go-ioutil/LICENSE new file mode 100644 index 0000000..7832dde --- /dev/null +++ b/vendor/github.com/whosonfirst/go-ioutil/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2021, Aaron Straup Cope +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the {organization} nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/whosonfirst/go-ioutil/README.md b/vendor/github.com/whosonfirst/go-ioutil/README.md new file mode 100644 index 0000000..5992016 --- /dev/null +++ b/vendor/github.com/whosonfirst/go-ioutil/README.md @@ -0,0 +1,5 @@ +# go-ioutil + +## Important + +Work in progress. Documentation to follow. \ No newline at end of file diff --git a/vendor/github.com/whosonfirst/go-ioutil/go.mod b/vendor/github.com/whosonfirst/go-ioutil/go.mod new file mode 100644 index 0000000..7e37699 --- /dev/null +++ b/vendor/github.com/whosonfirst/go-ioutil/go.mod @@ -0,0 +1,3 @@ +module github.com/whosonfirst/go-ioutil + +go 1.16 diff --git a/vendor/github.com/whosonfirst/go-ioutil/readseekcloser.go b/vendor/github.com/whosonfirst/go-ioutil/readseekcloser.go new file mode 100644 index 0000000..805465a --- /dev/null +++ b/vendor/github.com/whosonfirst/go-ioutil/readseekcloser.go @@ -0,0 +1,116 @@ +package ioutil + +// This is only here until there is an equivalent package/construct in the core Go language +// (20210217/thisisaaronland) + +import ( + "bytes" + "fmt" + "io" + "sync" +) + +type ReadSeekCloser struct { + io.Reader + io.Seeker + io.Closer + reader bool + closer bool + seeker bool + fh interface{} + br *bytes.Reader + mu *sync.RWMutex +} + +func (rsc *ReadSeekCloser) Read(p []byte) (n int, err error) { + + if rsc.seeker { + return rsc.fh.(io.Reader).Read(p) + } + + br, err := rsc.bytesReader() + + if err != nil { + return 0, err + } + + return br.Read(p) +} + +func (rsc *ReadSeekCloser) Close() error { + + if rsc.closer { + return rsc.fh.(io.ReadCloser).Close() + } + + return nil +} + +func (rsc *ReadSeekCloser) Seek(offset int64, whence int) (int64, error) { + + if rsc.seeker { + return rsc.fh.(io.Seeker).Seek(offset, whence) + } + + br, err := rsc.bytesReader() + + if err != nil { + return 0, err + } + + return br.Seek(offset, whence) +} + +func (rsc *ReadSeekCloser) bytesReader() (*bytes.Reader, error) { + + rsc.mu.Lock() + defer rsc.mu.Unlock() + + if rsc.br != nil { + return rsc.br, nil + } + + body, err := io.ReadAll(rsc.fh.(io.Reader)) + + if err != nil { + return nil, err + } + + br := bytes.NewReader(body) + rsc.br = br + + return br, nil +} + +func NewReadSeekCloser(fh interface{}) (io.ReadSeekCloser, error) { + + reader := true + seeker := false + closer := false + + switch fh.(type) { + case io.ReadSeekCloser: + return fh.(io.ReadSeekCloser), nil + case io.Reader: + // pass + case io.ReadCloser: + closer = true + case io.ReadSeeker: + seeker = true + default: + return nil, fmt.Errorf("Invalid or unsupported type") + } + + mu := new(sync.RWMutex) + + rsc := &ReadSeekCloser{ + reader: reader, + seeker: seeker, + closer: closer, + fh: fh, + mu: mu, + } + + return rsc, nil + +} diff --git a/vendor/github.com/whosonfirst/go-reader/file.go b/vendor/github.com/whosonfirst/go-reader/file.go new file mode 100644 index 0000000..dfb224f --- /dev/null +++ b/vendor/github.com/whosonfirst/go-reader/file.go @@ -0,0 +1,75 @@ +package reader + +import ( + "context" + "errors" + "io" + "net/url" + "os" + "path/filepath" +) + +type FileReader struct { + Reader + root string +} + +func init() { + + ctx := context.Background() + + err := RegisterReader(ctx, "fs", NewFileReader) // Deprecated + + if err != nil { + panic(err) + } + + err = RegisterReader(ctx, "file", NewFileReader) + + if err != nil { + panic(err) + } +} + +func NewFileReader(ctx context.Context, uri string) (Reader, error) { + + u, err := url.Parse(uri) + + if err != nil { + return nil, err + } + + root := u.Path + info, err := os.Stat(root) + + if err != nil { + return nil, err + } + + if !info.IsDir() { + return nil, errors.New("root is not a directory") + } + + r := &FileReader{ + root: root, + } + + return r, nil +} + +func (r *FileReader) Read(ctx context.Context, path string) (io.ReadSeekCloser, error) { + + abs_path := r.ReaderURI(ctx, path) + + _, err := os.Stat(abs_path) + + if err != nil { + return nil, err + } + + return os.Open(abs_path) +} + +func (r *FileReader) ReaderURI(ctx context.Context, path string) string { + return filepath.Join(r.root, path) +} diff --git a/vendor/github.com/whosonfirst/go-reader/go.mod b/vendor/github.com/whosonfirst/go-reader/go.mod index a1861a5..ef51c9b 100644 --- a/vendor/github.com/whosonfirst/go-reader/go.mod +++ b/vendor/github.com/whosonfirst/go-reader/go.mod @@ -2,4 +2,7 @@ module github.com/whosonfirst/go-reader go 1.16 -require github.com/aaronland/go-roster v0.0.2 +require ( + github.com/aaronland/go-roster v0.0.2 + github.com/whosonfirst/go-ioutil v0.0.1 +) diff --git a/vendor/github.com/whosonfirst/go-reader/go.sum b/vendor/github.com/whosonfirst/go-reader/go.sum index 37b964e..3002b76 100644 --- a/vendor/github.com/whosonfirst/go-reader/go.sum +++ b/vendor/github.com/whosonfirst/go-reader/go.sum @@ -12,6 +12,8 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/whosonfirst/go-ioutil v0.0.1 h1:cCrEYen6NDvHfjzV2q4u/VB21u2kTOwDnUGRlMI8Z9o= +github.com/whosonfirst/go-ioutil v0.0.1/go.mod h1:2dS1vWdAIkiHDvDF8fYyjv6k2NISmwaIjJJeEDBEdvg= github.com/whosonfirst/go-whosonfirst-cache v0.1.0 h1:ryWTsHj7gAEjwHC/WjsjROpFflsz3SCa7W9Qnk4EU7k= github.com/whosonfirst/go-whosonfirst-cache v0.1.0/go.mod h1:P5Tx34j2M50qX/kTfXY516apwGzJGkFSBYQI7TGArKw= github.com/whosonfirst/go-whosonfirst-cli v0.1.0 h1:Wwj9z0R/ryHmPmpVm5jCbXdG4agJzKMWXDtPVReN/KA= diff --git a/vendor/github.com/whosonfirst/go-reader/multi.go b/vendor/github.com/whosonfirst/go-reader/multi.go new file mode 100644 index 0000000..a72111b --- /dev/null +++ b/vendor/github.com/whosonfirst/go-reader/multi.go @@ -0,0 +1,122 @@ +package reader + +import ( + "context" + "errors" + "fmt" + "io" + _ "log" + "sync" +) + +type MultiReader struct { + Reader + readers []Reader + lookup map[string]int + mu *sync.RWMutex +} + +func NewMultiReaderFromURIs(ctx context.Context, uris ...string) (Reader, error) { + + readers := make([]Reader, 0) + + for _, uri := range uris { + + r, err := NewReader(ctx, uri) + + if err != nil { + return nil, err + } + + readers = append(readers, r) + } + + return NewMultiReader(ctx, readers...) +} + +func NewMultiReader(ctx context.Context, readers ...Reader) (Reader, error) { + + lookup := make(map[string]int) + + mu := new(sync.RWMutex) + + mr := MultiReader{ + readers: readers, + lookup: lookup, + mu: mu, + } + + return &mr, nil +} + +func (mr *MultiReader) Read(ctx context.Context, uri string) (io.ReadSeekCloser, error) { + + missing := errors.New("Unable to read URI") + + mr.mu.RLock() + + idx, ok := mr.lookup[uri] + + mr.mu.RUnlock() + + if ok { + + // log.Printf("READ MULTIREADER LOOKUP INDEX FOR %s AS %d\n", uri, idx) + + if idx == -1 { + return nil, missing + } + + r := mr.readers[idx] + return r.Read(ctx, uri) + } + + var fh io.ReadSeekCloser + idx = -1 + + for i, r := range mr.readers { + + rsp, err := r.Read(ctx, uri) + + if err == nil { + + fh = rsp + idx = i + + break + } + } + + // log.Printf("SET MULTIREADER LOOKUP INDEX FOR %s AS %d\n", uri, idx) + + mr.mu.Lock() + mr.lookup[uri] = idx + mr.mu.Unlock() + + if fh == nil { + return nil, missing + } + + return fh, nil +} + +func (mr *MultiReader) ReaderURI(ctx context.Context, uri string) string { + + mr.mu.RLock() + + idx, ok := mr.lookup[uri] + + mr.mu.RUnlock() + + if ok { + return mr.readers[idx].ReaderURI(ctx, uri) + } + + _, err := mr.Read(ctx, uri) + + if err != nil { + return fmt.Sprintf("x-urn:go-reader:multi#%s", uri) + } + + return mr.ReaderURI(ctx, uri) +} diff --git a/vendor/github.com/whosonfirst/go-reader/null.go b/vendor/github.com/whosonfirst/go-reader/null.go index 4811253..c1e4151 100644 --- a/vendor/github.com/whosonfirst/go-reader/null.go +++ b/vendor/github.com/whosonfirst/go-reader/null.go @@ -3,7 +3,7 @@ package reader import ( "bytes" "context" - "github.com/whosonfirst/go-reader/ioutil" + "github.com/whosonfirst/go-ioutil" "io" ) diff --git a/vendor/modules.txt b/vendor/modules.txt index 533b3fa..d33d2f0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -5,7 +5,9 @@ github.com/patrickmn/go-cache # github.com/whosonfirst/go-cache v0.2.0 ## explicit github.com/whosonfirst/go-cache -# github.com/whosonfirst/go-reader v0.4.1 +# github.com/whosonfirst/go-ioutil v0.0.1 +## explicit +github.com/whosonfirst/go-ioutil +# github.com/whosonfirst/go-reader v0.5.0 ## explicit github.com/whosonfirst/go-reader -github.com/whosonfirst/go-reader/ioutil