From 3aea884f2d06d973d07e8b6c027db2d9b15ad89b Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 8 Nov 2023 14:21:19 -0500 Subject: [PATCH 01/63] starting on 'zrok copy from' (#438) --- cmd/zrok/copyFrom.go | 29 +++++++++++++++++++++++++++++ cmd/zrok/main.go | 6 ++++++ go.mod | 1 + go.sum | 2 ++ 4 files changed, 38 insertions(+) create mode 100644 cmd/zrok/copyFrom.go diff --git a/cmd/zrok/copyFrom.go b/cmd/zrok/copyFrom.go new file mode 100644 index 000000000..a6f9290a3 --- /dev/null +++ b/cmd/zrok/copyFrom.go @@ -0,0 +1,29 @@ +package main + +import ( + "github.com/io-developer/go-davsync/pkg/client/webdav" + "github.com/spf13/cobra" +) + +func init() { + copyCmd.AddCommand(newCopyFromCommand().cmd) +} + +type copyFromCommand struct { + cmd *cobra.Command +} + +func newCopyFromCommand() *copyFromCommand { + cmd := &cobra.Command{ + Use: "from []", + Short: "Copy files from zrok drive to destination", + Args: cobra.RangeArgs(1, 2), + } + command := ©FromCommand{cmd: cmd} + cmd.Run = command.run + return command +} + +func (cmd *copyFromCommand) run(_ *cobra.Command, args []string) { + _ = &webdav.Options{} +} diff --git a/cmd/zrok/main.go b/cmd/zrok/main.go index fd8d52f8b..97024f32b 100644 --- a/cmd/zrok/main.go +++ b/cmd/zrok/main.go @@ -26,6 +26,7 @@ func init() { testCmd.AddCommand(loopCmd) rootCmd.AddCommand(adminCmd) rootCmd.AddCommand(configCmd) + rootCmd.AddCommand(copyCmd) rootCmd.AddCommand(shareCmd) rootCmd.AddCommand(testCmd) transport.AddAddressParser(tcp.AddressParser{}) @@ -79,6 +80,11 @@ var configCmd = &cobra.Command{ Short: "Configure your zrok environment", } +var copyCmd = &cobra.Command{ + Use: "copy", + Short: "Copy files to/from zrok drives", +} + var loopCmd = &cobra.Command{ Use: "loopback", Aliases: []string{"loop"}, diff --git a/go.mod b/go.mod index ab9fdf36b..c6b647924 100644 --- a/go.mod +++ b/go.mod @@ -121,6 +121,7 @@ require ( github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect + github.com/io-developer/go-davsync v1.0.1 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgconn v1.14.0 // indirect github.com/jackc/pgio v1.0.0 // indirect diff --git a/go.sum b/go.sum index c1454c49a..198b6afea 100644 --- a/go.sum +++ b/go.sum @@ -718,6 +718,8 @@ github.com/influxdata/influxdb-client-go/v2 v2.11.0/go.mod h1:YteV91FiQxRdccyJ2c github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/io-developer/go-davsync v1.0.1 h1:TDwpE+O3saK4WlFL1K66W6fFWEQMfccGnDjyq9R9MpU= +github.com/io-developer/go-davsync v1.0.1/go.mod h1:s+hh2DEKAHJQjSZiLqoW2ahLw+lcvzRThHPOr1vfOw4= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= From ac4b5b93bc7c817a66b2811a21837c97b91f06ed Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 14 Nov 2023 14:27:32 -0500 Subject: [PATCH 02/63] sync from scratch --- cmd/zrok/copyFrom.go | 35 +++++++++++++++++++++++++++++++++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/cmd/zrok/copyFrom.go b/cmd/zrok/copyFrom.go index a6f9290a3..b456d7638 100644 --- a/cmd/zrok/copyFrom.go +++ b/cmd/zrok/copyFrom.go @@ -1,8 +1,10 @@ package main import ( - "github.com/io-developer/go-davsync/pkg/client/webdav" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/studio-b12/gowebdav" + "path/filepath" ) func init() { @@ -25,5 +27,34 @@ func newCopyFromCommand() *copyFromCommand { } func (cmd *copyFromCommand) run(_ *cobra.Command, args []string) { - _ = &webdav.Options{} + c := gowebdav.NewClient(args[0], "", "") + if err := c.Connect(); err != nil { + panic(err) + } + if err := cmd.recurseTree(c, ""); err != nil { + panic(err) + } +} + +func (cmd *copyFromCommand) recurseTree(c *gowebdav.Client, path string) error { + files, err := c.ReadDir(path) + if err != nil { + return err + } + for _, f := range files { + sub := filepath.ToSlash(filepath.Join(path, f.Name())) + if f.IsDir() { + logrus.Infof("-> %v", sub) + if err := cmd.recurseTree(c, sub); err != nil { + return err + } + } else { + fi, err := c.Stat(sub) + if err != nil { + return err + } + logrus.Infof("++ %v (%v)", sub, fi.Sys()) + } + } + return nil } diff --git a/go.mod b/go.mod index c6b647924..04eb14edc 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 + github.com/studio-b12/gowebdav v0.9.0 github.com/wneessen/go-mail v0.2.7 github.com/zitadel/oidc/v2 v2.7.0 go.uber.org/zap v1.25.0 @@ -121,7 +122,6 @@ require ( github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect - github.com/io-developer/go-davsync v1.0.1 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgconn v1.14.0 // indirect github.com/jackc/pgio v1.0.0 // indirect diff --git a/go.sum b/go.sum index 198b6afea..3410c062a 100644 --- a/go.sum +++ b/go.sum @@ -718,8 +718,6 @@ github.com/influxdata/influxdb-client-go/v2 v2.11.0/go.mod h1:YteV91FiQxRdccyJ2c github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= -github.com/io-developer/go-davsync v1.0.1 h1:TDwpE+O3saK4WlFL1K66W6fFWEQMfccGnDjyq9R9MpU= -github.com/io-developer/go-davsync v1.0.1/go.mod h1:s+hh2DEKAHJQjSZiLqoW2ahLw+lcvzRThHPOr1vfOw4= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= @@ -1292,6 +1290,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/studio-b12/gowebdav v0.9.0 h1:1j1sc9gQnNxbXXM4M/CebPOX4aXYtr7MojAVcN4dHjU= +github.com/studio-b12/gowebdav v0.9.0/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tailscale/tscert v0.0.0-20230509043813-4e9cb4f2b4ad h1:JEOo9j4RzDPBJFTU9YZ/QPkLtfV8+6PbZFFOSUx5VP4= github.com/tailscale/tscert v0.0.0-20230509043813-4e9cb4f2b4ad/go.mod h1:kNGUQ3VESx3VZwRwA9MSCUegIl6+saPL8Noq82ozCaU= From 56ecc330ed741ebc8ff45b08dcc96d290443cdcd Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 27 Nov 2023 14:19:01 -0500 Subject: [PATCH 03/63] copyfrom update --- cmd/zrok/copyFrom.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/zrok/copyFrom.go b/cmd/zrok/copyFrom.go index b456d7638..ec2d2c446 100644 --- a/cmd/zrok/copyFrom.go +++ b/cmd/zrok/copyFrom.go @@ -49,11 +49,11 @@ func (cmd *copyFromCommand) recurseTree(c *gowebdav.Client, path string) error { return err } } else { - fi, err := c.Stat(sub) - if err != nil { - return err + etag := "" + if v, ok := f.(gowebdav.File); ok { + etag = v.ETag() } - logrus.Infof("++ %v (%v)", sub, fi.Sys()) + logrus.Infof("++ %v (%v)", sub, etag) } } return nil From 0832f28dc73277c84996aca65a982ae7fb132a1b Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 27 Nov 2023 15:44:38 -0500 Subject: [PATCH 04/63] framework start; webdav (#438) --- cmd/zrok/copyFrom.go | 38 +++++++++------------------- util/sync/model.go | 14 +++++++++++ util/sync/webdav.go | 59 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 27 deletions(-) create mode 100644 util/sync/model.go create mode 100644 util/sync/webdav.go diff --git a/cmd/zrok/copyFrom.go b/cmd/zrok/copyFrom.go index ec2d2c446..58d1fce76 100644 --- a/cmd/zrok/copyFrom.go +++ b/cmd/zrok/copyFrom.go @@ -1,10 +1,9 @@ package main import ( + "github.com/openziti/zrok/util/sync" "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/studio-b12/gowebdav" - "path/filepath" ) func init() { @@ -27,34 +26,19 @@ func newCopyFromCommand() *copyFromCommand { } func (cmd *copyFromCommand) run(_ *cobra.Command, args []string) { - c := gowebdav.NewClient(args[0], "", "") - if err := c.Connect(); err != nil { - panic(err) - } - if err := cmd.recurseTree(c, ""); err != nil { + t, err := sync.NewWebDAVTarget(&sync.WebDAVTargetConfig{ + URL: args[0], + Username: "", + Password: "", + }) + if err != nil { panic(err) } -} - -func (cmd *copyFromCommand) recurseTree(c *gowebdav.Client, path string) error { - files, err := c.ReadDir(path) + tree, err := t.Inventory() if err != nil { - return err + panic(err) } - for _, f := range files { - sub := filepath.ToSlash(filepath.Join(path, f.Name())) - if f.IsDir() { - logrus.Infof("-> %v", sub) - if err := cmd.recurseTree(c, sub); err != nil { - return err - } - } else { - etag := "" - if v, ok := f.(gowebdav.File); ok { - etag = v.ETag() - } - logrus.Infof("++ %v (%v)", sub, etag) - } + for _, f := range tree { + logrus.Infof("-> %v [%v, %v, %v]", f.Path, f.Size, f.Modified, f.ETag) } - return nil } diff --git a/util/sync/model.go b/util/sync/model.go new file mode 100644 index 000000000..948b86a8b --- /dev/null +++ b/util/sync/model.go @@ -0,0 +1,14 @@ +package sync + +import "time" + +type Object struct { + Path string + Size int64 + Modified time.Time + ETag string +} + +type Target interface { + Inventory() ([]*Object, error) +} diff --git a/util/sync/webdav.go b/util/sync/webdav.go new file mode 100644 index 000000000..ba9578d51 --- /dev/null +++ b/util/sync/webdav.go @@ -0,0 +1,59 @@ +package sync + +import ( + "github.com/pkg/errors" + "github.com/studio-b12/gowebdav" + "path/filepath" +) + +type WebDAVTargetConfig struct { + URL string + Username string + Password string +} + +type WebDAVTarget struct { + c *gowebdav.Client +} + +func NewWebDAVTarget(cfg *WebDAVTargetConfig) (*WebDAVTarget, error) { + c := gowebdav.NewClient(cfg.URL, cfg.Username, cfg.Password) + if err := c.Connect(); err != nil { + return nil, errors.Wrap(err, "error connecting to webdav target") + } + return &WebDAVTarget{c: c}, nil +} + +func (t *WebDAVTarget) Inventory() ([]*Object, error) { + tree, err := t.recurse("", nil) + if err != nil { + return nil, err + } + return tree, nil +} + +func (t *WebDAVTarget) recurse(path string, tree []*Object) ([]*Object, error) { + files, err := t.c.ReadDir(path) + if err != nil { + return nil, err + } + for _, f := range files { + sub := filepath.ToSlash(filepath.Join(path, f.Name())) + if f.IsDir() { + tree, err = t.recurse(sub, tree) + if err != nil { + return nil, err + } + } else { + if v, ok := f.(gowebdav.File); ok { + tree = append(tree, &Object{ + Path: filepath.ToSlash(filepath.Join(path, f.Name())), + Size: v.Size(), + Modified: v.ModTime(), + ETag: v.ETag(), + }) + } + } + } + return tree, nil +} From 57c5a9977e669df41756823c53895045b28765de Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 27 Nov 2023 16:18:22 -0500 Subject: [PATCH 05/63] filesystem target (#438) --- cmd/zrok/copyFrom.go | 22 ++++++++++++++--- util/sync/filesystem.go | 54 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 util/sync/filesystem.go diff --git a/cmd/zrok/copyFrom.go b/cmd/zrok/copyFrom.go index 58d1fce76..c211a2c38 100644 --- a/cmd/zrok/copyFrom.go +++ b/cmd/zrok/copyFrom.go @@ -26,7 +26,23 @@ func newCopyFromCommand() *copyFromCommand { } func (cmd *copyFromCommand) run(_ *cobra.Command, args []string) { - t, err := sync.NewWebDAVTarget(&sync.WebDAVTargetConfig{ + target := "." + if len(args) == 2 { + target = args[1] + } + + t := sync.NewFilesystemTarget(&sync.FilesystemTargetConfig{ + Root: target, + }) + destTree, err := t.Inventory() + if err != nil { + panic(err) + } + for _, f := range destTree { + logrus.Infof("<- %v [%v, %v, %v]", f.Path, f.Size, f.Modified, f.ETag) + } + + s, err := sync.NewWebDAVTarget(&sync.WebDAVTargetConfig{ URL: args[0], Username: "", Password: "", @@ -34,11 +50,11 @@ func (cmd *copyFromCommand) run(_ *cobra.Command, args []string) { if err != nil { panic(err) } - tree, err := t.Inventory() + srcTree, err := s.Inventory() if err != nil { panic(err) } - for _, f := range tree { + for _, f := range srcTree { logrus.Infof("-> %v [%v, %v, %v]", f.Path, f.Size, f.Modified, f.ETag) } } diff --git a/util/sync/filesystem.go b/util/sync/filesystem.go new file mode 100644 index 000000000..867a59a83 --- /dev/null +++ b/util/sync/filesystem.go @@ -0,0 +1,54 @@ +package sync + +import ( + "context" + "fmt" + "golang.org/x/net/webdav" + "io/fs" + "os" +) + +type FilesystemTargetConfig struct { + Root string +} + +type FilesystemTarget struct { + root fs.FS + tree []*Object +} + +func NewFilesystemTarget(cfg *FilesystemTargetConfig) *FilesystemTarget { + root := os.DirFS(cfg.Root) + return &FilesystemTarget{root: root} +} + +func (t *FilesystemTarget) Inventory() ([]*Object, error) { + t.tree = nil + if err := fs.WalkDir(t.root, ".", t.recurse); err != nil { + return nil, err + } + return t.tree, nil +} + +func (t *FilesystemTarget) recurse(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + fi, err := d.Info() + if err != nil { + return err + } + etag := "" + if v, ok := fi.(webdav.ETager); ok { + etag, err = v.ETag(context.Background()) + if err != nil { + return err + } + } else { + etag = fmt.Sprintf(`"%x%x"`, fi.ModTime().UnixNano(), fi.Size()) + } + t.tree = append(t.tree, &Object{path, fi.Size(), fi.ModTime(), etag}) + } + return nil +} From e6aa1420bef772ec6d474025be81846670955e83 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 27 Nov 2023 16:53:02 -0500 Subject: [PATCH 06/63] files to copy (#438) --- cmd/zrok/copyFrom.go | 20 ++++--------------- util/sync/synchronizer.go | 41 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 16 deletions(-) create mode 100644 util/sync/synchronizer.go diff --git a/cmd/zrok/copyFrom.go b/cmd/zrok/copyFrom.go index c211a2c38..8b4e78b35 100644 --- a/cmd/zrok/copyFrom.go +++ b/cmd/zrok/copyFrom.go @@ -2,7 +2,6 @@ package main import ( "github.com/openziti/zrok/util/sync" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -31,18 +30,10 @@ func (cmd *copyFromCommand) run(_ *cobra.Command, args []string) { target = args[1] } - t := sync.NewFilesystemTarget(&sync.FilesystemTargetConfig{ + dst := sync.NewFilesystemTarget(&sync.FilesystemTargetConfig{ Root: target, }) - destTree, err := t.Inventory() - if err != nil { - panic(err) - } - for _, f := range destTree { - logrus.Infof("<- %v [%v, %v, %v]", f.Path, f.Size, f.Modified, f.ETag) - } - - s, err := sync.NewWebDAVTarget(&sync.WebDAVTargetConfig{ + src, err := sync.NewWebDAVTarget(&sync.WebDAVTargetConfig{ URL: args[0], Username: "", Password: "", @@ -50,11 +41,8 @@ func (cmd *copyFromCommand) run(_ *cobra.Command, args []string) { if err != nil { panic(err) } - srcTree, err := s.Inventory() - if err != nil { + + if err := sync.Synchronize(src, dst); err != nil { panic(err) } - for _, f := range srcTree { - logrus.Infof("-> %v [%v, %v, %v]", f.Path, f.Size, f.Modified, f.ETag) - } } diff --git a/util/sync/synchronizer.go b/util/sync/synchronizer.go new file mode 100644 index 000000000..ec168b4df --- /dev/null +++ b/util/sync/synchronizer.go @@ -0,0 +1,41 @@ +package sync + +import ( + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +func Synchronize(src, dst Target) error { + srcTree, err := src.Inventory() + if err != nil { + return errors.Wrap(err, "error creating source inventory") + } + + dstTree, err := dst.Inventory() + if err != nil { + return errors.Wrap(err, "error creating destination inventory") + } + + dstIndex := make(map[string]*Object) + for _, f := range dstTree { + dstIndex[f.Path] = f + } + + var copyList []*Object + for _, srcF := range srcTree { + if dstF, found := dstIndex[srcF.Path]; found { + if dstF.ETag != srcF.ETag { + copyList = append(copyList, srcF) + } + } else { + copyList = append(copyList, srcF) + } + } + + logrus.Infof("files to copy:") + for _, copy := range copyList { + logrus.Infof("-> %v", copy.Path) + } + + return nil +} From 93277a191bd0f1d7684bb8104538a56a4a4d5110 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 27 Nov 2023 17:31:13 -0500 Subject: [PATCH 07/63] very rudimentary sync (#438) --- util/sync/filesystem.go | 29 ++++++++++++++++++++++++++++- util/sync/model.go | 8 +++++++- util/sync/synchronizer.go | 14 +++++++++++--- util/sync/webdav.go | 10 ++++++++++ 4 files changed, 56 insertions(+), 5 deletions(-) diff --git a/util/sync/filesystem.go b/util/sync/filesystem.go index 867a59a83..49d4be8ef 100644 --- a/util/sync/filesystem.go +++ b/util/sync/filesystem.go @@ -4,8 +4,10 @@ import ( "context" "fmt" "golang.org/x/net/webdav" + "io" "io/fs" "os" + "path/filepath" ) type FilesystemTargetConfig struct { @@ -13,16 +15,20 @@ type FilesystemTargetConfig struct { } type FilesystemTarget struct { + cfg *FilesystemTargetConfig root fs.FS tree []*Object } func NewFilesystemTarget(cfg *FilesystemTargetConfig) *FilesystemTarget { root := os.DirFS(cfg.Root) - return &FilesystemTarget{root: root} + return &FilesystemTarget{cfg: cfg, root: root} } func (t *FilesystemTarget) Inventory() ([]*Object, error) { + if _, err := os.Stat(t.cfg.Root); os.IsNotExist(err) { + return nil, nil + } t.tree = nil if err := fs.WalkDir(t.root, ".", t.recurse); err != nil { return nil, err @@ -52,3 +58,24 @@ func (t *FilesystemTarget) recurse(path string, d fs.DirEntry, err error) error } return nil } + +func (t *FilesystemTarget) ReadStream(path string) (io.ReadCloser, error) { + return os.Open(path) +} + +func (t *FilesystemTarget) WriteStream(path string, stream io.Reader, mode os.FileMode) error { + targetPath := filepath.Join(t.cfg.Root, path) + + if err := os.MkdirAll(filepath.Dir(targetPath), mode); err != nil { + return err + } + f, err := os.Create(targetPath) + if err != nil { + return err + } + _, err = io.Copy(f, stream) + if err != nil { + return err + } + return nil +} diff --git a/util/sync/model.go b/util/sync/model.go index 948b86a8b..8ea6c1ed2 100644 --- a/util/sync/model.go +++ b/util/sync/model.go @@ -1,6 +1,10 @@ package sync -import "time" +import ( + "io" + "os" + "time" +) type Object struct { Path string @@ -11,4 +15,6 @@ type Object struct { type Target interface { Inventory() ([]*Object, error) + ReadStream(path string) (io.ReadCloser, error) + WriteStream(path string, stream io.Reader, mode os.FileMode) error } diff --git a/util/sync/synchronizer.go b/util/sync/synchronizer.go index ec168b4df..23e63af4b 100644 --- a/util/sync/synchronizer.go +++ b/util/sync/synchronizer.go @@ -3,6 +3,7 @@ package sync import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" + "os" ) func Synchronize(src, dst Target) error { @@ -32,9 +33,16 @@ func Synchronize(src, dst Target) error { } } - logrus.Infof("files to copy:") - for _, copy := range copyList { - logrus.Infof("-> %v", copy.Path) + for _, target := range copyList { + logrus.Infof("+> %v", target.Path) + ss, err := src.ReadStream(target.Path) + if err != nil { + return err + } + if err := dst.WriteStream(target.Path, ss, os.ModePerm); err != nil { + return err + } + logrus.Infof("=> %v", target.Path) } return nil diff --git a/util/sync/webdav.go b/util/sync/webdav.go index ba9578d51..6f63f21fc 100644 --- a/util/sync/webdav.go +++ b/util/sync/webdav.go @@ -3,6 +3,8 @@ package sync import ( "github.com/pkg/errors" "github.com/studio-b12/gowebdav" + "io" + "os" "path/filepath" ) @@ -57,3 +59,11 @@ func (t *WebDAVTarget) recurse(path string, tree []*Object) ([]*Object, error) { } return tree, nil } + +func (t *WebDAVTarget) ReadStream(path string) (io.ReadCloser, error) { + return t.c.ReadStream(path) +} + +func (t *WebDAVTarget) WriteStream(path string, stream io.Reader, mode os.FileMode) error { + return t.c.WriteStream(path, stream, mode) +} From c397ae883b4ebc80ac41b92a4cfba65f645ba37b Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Mon, 27 Nov 2023 21:11:19 -0500 Subject: [PATCH 08/63] modification time and UTC (#438) --- util/sync/filesystem.go | 11 ++++++++++- util/sync/model.go | 1 + util/sync/synchronizer.go | 6 ++++-- util/sync/webdav.go | 5 +++++ 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/util/sync/filesystem.go b/util/sync/filesystem.go index 49d4be8ef..6877440c2 100644 --- a/util/sync/filesystem.go +++ b/util/sync/filesystem.go @@ -8,6 +8,7 @@ import ( "io/fs" "os" "path/filepath" + "time" ) type FilesystemTargetConfig struct { @@ -52,7 +53,7 @@ func (t *FilesystemTarget) recurse(path string, d fs.DirEntry, err error) error return err } } else { - etag = fmt.Sprintf(`"%x%x"`, fi.ModTime().UnixNano(), fi.Size()) + etag = fmt.Sprintf(`"%x%x"`, fi.ModTime().UTC().UnixNano(), fi.Size()) } t.tree = append(t.tree, &Object{path, fi.Size(), fi.ModTime(), etag}) } @@ -79,3 +80,11 @@ func (t *FilesystemTarget) WriteStream(path string, stream io.Reader, mode os.Fi } return nil } + +func (t *FilesystemTarget) SetModificationTime(path string, mtime time.Time) error { + targetPath := filepath.Join(t.cfg.Root, path) + if err := os.Chtimes(targetPath, time.Now(), mtime); err != nil { + return err + } + return nil +} diff --git a/util/sync/model.go b/util/sync/model.go index 8ea6c1ed2..8fca5ca07 100644 --- a/util/sync/model.go +++ b/util/sync/model.go @@ -17,4 +17,5 @@ type Target interface { Inventory() ([]*Object, error) ReadStream(path string) (io.ReadCloser, error) WriteStream(path string, stream io.Reader, mode os.FileMode) error + SetModificationTime(path string, mtime time.Time) error } diff --git a/util/sync/synchronizer.go b/util/sync/synchronizer.go index 23e63af4b..36ba78fc4 100644 --- a/util/sync/synchronizer.go +++ b/util/sync/synchronizer.go @@ -25,7 +25,7 @@ func Synchronize(src, dst Target) error { var copyList []*Object for _, srcF := range srcTree { if dstF, found := dstIndex[srcF.Path]; found { - if dstF.ETag != srcF.ETag { + if dstF.Size != srcF.Size || dstF.Modified.UTC() != srcF.Modified.UTC() { copyList = append(copyList, srcF) } } else { @@ -34,7 +34,6 @@ func Synchronize(src, dst Target) error { } for _, target := range copyList { - logrus.Infof("+> %v", target.Path) ss, err := src.ReadStream(target.Path) if err != nil { return err @@ -42,6 +41,9 @@ func Synchronize(src, dst Target) error { if err := dst.WriteStream(target.Path, ss, os.ModePerm); err != nil { return err } + if err := dst.SetModificationTime(target.Path, target.Modified); err != nil { + return err + } logrus.Infof("=> %v", target.Path) } diff --git a/util/sync/webdav.go b/util/sync/webdav.go index 6f63f21fc..840a685be 100644 --- a/util/sync/webdav.go +++ b/util/sync/webdav.go @@ -6,6 +6,7 @@ import ( "io" "os" "path/filepath" + "time" ) type WebDAVTargetConfig struct { @@ -67,3 +68,7 @@ func (t *WebDAVTarget) ReadStream(path string) (io.ReadCloser, error) { func (t *WebDAVTarget) WriteStream(path string, stream io.Reader, mode os.FileMode) error { return t.c.WriteStream(path, stream, mode) } + +func (t *WebDAVTarget) SetModificationTime(path string, mtime time.Time) error { + return nil +} From d1fe17fdf639fcb81d088c20f1bcfb522ba31e38 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 29 Nov 2023 13:39:17 -0500 Subject: [PATCH 09/63] local copy of x/net/webdav (#438) --- endpoints/drive/backend.go | 2 +- endpoints/drive/webdav/file.go | 803 +++++++ endpoints/drive/webdav/file_test.go | 1183 ++++++++++ endpoints/drive/webdav/if.go | 173 ++ endpoints/drive/webdav/if_test.go | 322 +++ endpoints/drive/webdav/internal/xml/README | 11 + .../drive/webdav/internal/xml/atom_test.go | 56 + .../drive/webdav/internal/xml/example_test.go | 151 ++ .../drive/webdav/internal/xml/marshal.go | 1223 ++++++++++ .../drive/webdav/internal/xml/marshal_test.go | 1939 ++++++++++++++++ endpoints/drive/webdav/internal/xml/read.go | 691 ++++++ .../drive/webdav/internal/xml/read_test.go | 744 ++++++ .../drive/webdav/internal/xml/typeinfo.go | 371 +++ endpoints/drive/webdav/internal/xml/xml.go | 1998 +++++++++++++++++ .../drive/webdav/internal/xml/xml_test.go | 752 +++++++ endpoints/drive/webdav/litmus_test_server.go | 94 + endpoints/drive/webdav/lock.go | 445 ++++ endpoints/drive/webdav/lock_test.go | 735 ++++++ endpoints/drive/webdav/prop.go | 469 ++++ endpoints/drive/webdav/prop_test.go | 716 ++++++ endpoints/drive/webdav/webdav.go | 736 ++++++ endpoints/drive/webdav/webdav_test.go | 349 +++ endpoints/drive/webdav/xml.go | 519 +++++ endpoints/drive/webdav/xml_test.go | 905 ++++++++ 24 files changed, 15386 insertions(+), 1 deletion(-) create mode 100644 endpoints/drive/webdav/file.go create mode 100644 endpoints/drive/webdav/file_test.go create mode 100644 endpoints/drive/webdav/if.go create mode 100644 endpoints/drive/webdav/if_test.go create mode 100644 endpoints/drive/webdav/internal/xml/README create mode 100644 endpoints/drive/webdav/internal/xml/atom_test.go create mode 100644 endpoints/drive/webdav/internal/xml/example_test.go create mode 100644 endpoints/drive/webdav/internal/xml/marshal.go create mode 100644 endpoints/drive/webdav/internal/xml/marshal_test.go create mode 100644 endpoints/drive/webdav/internal/xml/read.go create mode 100644 endpoints/drive/webdav/internal/xml/read_test.go create mode 100644 endpoints/drive/webdav/internal/xml/typeinfo.go create mode 100644 endpoints/drive/webdav/internal/xml/xml.go create mode 100644 endpoints/drive/webdav/internal/xml/xml_test.go create mode 100644 endpoints/drive/webdav/litmus_test_server.go create mode 100644 endpoints/drive/webdav/lock.go create mode 100644 endpoints/drive/webdav/lock_test.go create mode 100644 endpoints/drive/webdav/prop.go create mode 100644 endpoints/drive/webdav/prop_test.go create mode 100644 endpoints/drive/webdav/webdav.go create mode 100644 endpoints/drive/webdav/webdav_test.go create mode 100644 endpoints/drive/webdav/xml.go create mode 100644 endpoints/drive/webdav/xml_test.go diff --git a/endpoints/drive/backend.go b/endpoints/drive/backend.go index f9147e8c1..35e15d544 100644 --- a/endpoints/drive/backend.go +++ b/endpoints/drive/backend.go @@ -5,8 +5,8 @@ import ( "github.com/openziti/sdk-golang/ziti" "github.com/openziti/sdk-golang/ziti/edge" "github.com/openziti/zrok/endpoints" + "github.com/openziti/zrok/endpoints/drive/webdav" "github.com/pkg/errors" - "golang.org/x/net/webdav" "net/http" "time" ) diff --git a/endpoints/drive/webdav/file.go b/endpoints/drive/webdav/file.go new file mode 100644 index 000000000..3cd19ffbe --- /dev/null +++ b/endpoints/drive/webdav/file.go @@ -0,0 +1,803 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package webdav + +import ( + "context" + "encoding/xml" + "io" + "net/http" + "os" + "path" + "path/filepath" + "runtime" + "strings" + "sync" + "time" +) + +// slashClean is equivalent to but slightly more efficient than +// path.Clean("/" + name). +func slashClean(name string) string { + if name == "" || name[0] != '/' { + name = "/" + name + } + return path.Clean(name) +} + +// A FileSystem implements access to a collection of named files. The elements +// in a file path are separated by slash ('/', U+002F) characters, regardless +// of host operating system convention. +// +// Each method has the same semantics as the os package's function of the same +// name. +// +// Note that the os.Rename documentation says that "OS-specific restrictions +// might apply". In particular, whether or not renaming a file or directory +// overwriting another existing file or directory is an error is OS-dependent. +type FileSystem interface { + Mkdir(ctx context.Context, name string, perm os.FileMode) error + OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (File, error) + RemoveAll(ctx context.Context, name string) error + Rename(ctx context.Context, oldName, newName string) error + Stat(ctx context.Context, name string) (os.FileInfo, error) +} + +// A File is returned by a FileSystem's OpenFile method and can be served by a +// Handler. +// +// A File may optionally implement the DeadPropsHolder interface, if it can +// load and save dead properties. +type File interface { + http.File + io.Writer +} + +// A Dir implements FileSystem using the native file system restricted to a +// specific directory tree. +// +// While the FileSystem.OpenFile method takes '/'-separated paths, a Dir's +// string value is a filename on the native file system, not a URL, so it is +// separated by filepath.Separator, which isn't necessarily '/'. +// +// An empty Dir is treated as ".". +type Dir string + +func (d Dir) resolve(name string) string { + // This implementation is based on Dir.Open's code in the standard net/http package. + if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 || + strings.Contains(name, "\x00") { + return "" + } + dir := string(d) + if dir == "" { + dir = "." + } + return filepath.Join(dir, filepath.FromSlash(slashClean(name))) +} + +func (d Dir) Mkdir(ctx context.Context, name string, perm os.FileMode) error { + if name = d.resolve(name); name == "" { + return os.ErrNotExist + } + return os.Mkdir(name, perm) +} + +func (d Dir) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (File, error) { + if name = d.resolve(name); name == "" { + return nil, os.ErrNotExist + } + f, err := os.OpenFile(name, flag, perm) + if err != nil { + return nil, err + } + return f, nil +} + +func (d Dir) RemoveAll(ctx context.Context, name string) error { + if name = d.resolve(name); name == "" { + return os.ErrNotExist + } + if name == filepath.Clean(string(d)) { + // Prohibit removing the virtual root directory. + return os.ErrInvalid + } + return os.RemoveAll(name) +} + +func (d Dir) Rename(ctx context.Context, oldName, newName string) error { + if oldName = d.resolve(oldName); oldName == "" { + return os.ErrNotExist + } + if newName = d.resolve(newName); newName == "" { + return os.ErrNotExist + } + if root := filepath.Clean(string(d)); root == oldName || root == newName { + // Prohibit renaming from or to the virtual root directory. + return os.ErrInvalid + } + return os.Rename(oldName, newName) +} + +func (d Dir) Stat(ctx context.Context, name string) (os.FileInfo, error) { + if name = d.resolve(name); name == "" { + return nil, os.ErrNotExist + } + return os.Stat(name) +} + +// NewMemFS returns a new in-memory FileSystem implementation. +func NewMemFS() FileSystem { + return &memFS{ + root: memFSNode{ + children: make(map[string]*memFSNode), + mode: 0660 | os.ModeDir, + modTime: time.Now(), + }, + } +} + +// A memFS implements FileSystem, storing all metadata and actual file data +// in-memory. No limits on filesystem size are used, so it is not recommended +// this be used where the clients are untrusted. +// +// Concurrent access is permitted. The tree structure is protected by a mutex, +// and each node's contents and metadata are protected by a per-node mutex. +// +// TODO: Enforce file permissions. +type memFS struct { + mu sync.Mutex + root memFSNode +} + +// TODO: clean up and rationalize the walk/find code. + +// walk walks the directory tree for the fullname, calling f at each step. If f +// returns an error, the walk will be aborted and return that same error. +// +// dir is the directory at that step, frag is the name fragment, and final is +// whether it is the final step. For example, walking "/foo/bar/x" will result +// in 3 calls to f: +// - "/", "foo", false +// - "/foo/", "bar", false +// - "/foo/bar/", "x", true +// +// The frag argument will be empty only if dir is the root node and the walk +// ends at that root node. +func (fs *memFS) walk(op, fullname string, f func(dir *memFSNode, frag string, final bool) error) error { + original := fullname + fullname = slashClean(fullname) + + // Strip any leading "/"s to make fullname a relative path, as the walk + // starts at fs.root. + if fullname[0] == '/' { + fullname = fullname[1:] + } + dir := &fs.root + + for { + frag, remaining := fullname, "" + i := strings.IndexRune(fullname, '/') + final := i < 0 + if !final { + frag, remaining = fullname[:i], fullname[i+1:] + } + if frag == "" && dir != &fs.root { + panic("webdav: empty path fragment for a clean path") + } + if err := f(dir, frag, final); err != nil { + return &os.PathError{ + Op: op, + Path: original, + Err: err, + } + } + if final { + break + } + child := dir.children[frag] + if child == nil { + return &os.PathError{ + Op: op, + Path: original, + Err: os.ErrNotExist, + } + } + if !child.mode.IsDir() { + return &os.PathError{ + Op: op, + Path: original, + Err: os.ErrInvalid, + } + } + dir, fullname = child, remaining + } + return nil +} + +// find returns the parent of the named node and the relative name fragment +// from the parent to the child. For example, if finding "/foo/bar/baz" then +// parent will be the node for "/foo/bar" and frag will be "baz". +// +// If the fullname names the root node, then parent, frag and err will be zero. +// +// find returns an error if the parent does not already exist or the parent +// isn't a directory, but it will not return an error per se if the child does +// not already exist. The error returned is either nil or an *os.PathError +// whose Op is op. +func (fs *memFS) find(op, fullname string) (parent *memFSNode, frag string, err error) { + err = fs.walk(op, fullname, func(parent0 *memFSNode, frag0 string, final bool) error { + if !final { + return nil + } + if frag0 != "" { + parent, frag = parent0, frag0 + } + return nil + }) + return parent, frag, err +} + +func (fs *memFS) Mkdir(ctx context.Context, name string, perm os.FileMode) error { + fs.mu.Lock() + defer fs.mu.Unlock() + + dir, frag, err := fs.find("mkdir", name) + if err != nil { + return err + } + if dir == nil { + // We can't create the root. + return os.ErrInvalid + } + if _, ok := dir.children[frag]; ok { + return os.ErrExist + } + dir.children[frag] = &memFSNode{ + children: make(map[string]*memFSNode), + mode: perm.Perm() | os.ModeDir, + modTime: time.Now(), + } + return nil +} + +func (fs *memFS) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (File, error) { + fs.mu.Lock() + defer fs.mu.Unlock() + + dir, frag, err := fs.find("open", name) + if err != nil { + return nil, err + } + var n *memFSNode + if dir == nil { + // We're opening the root. + if runtime.GOOS == "zos" { + if flag&os.O_WRONLY != 0 { + return nil, os.ErrPermission + } + } else { + if flag&(os.O_WRONLY|os.O_RDWR) != 0 { + return nil, os.ErrPermission + } + } + n, frag = &fs.root, "/" + + } else { + n = dir.children[frag] + if flag&(os.O_SYNC|os.O_APPEND) != 0 { + // memFile doesn't support these flags yet. + return nil, os.ErrInvalid + } + if flag&os.O_CREATE != 0 { + if flag&os.O_EXCL != 0 && n != nil { + return nil, os.ErrExist + } + if n == nil { + n = &memFSNode{ + mode: perm.Perm(), + } + dir.children[frag] = n + } + } + if n == nil { + return nil, os.ErrNotExist + } + if flag&(os.O_WRONLY|os.O_RDWR) != 0 && flag&os.O_TRUNC != 0 { + n.mu.Lock() + n.data = nil + n.mu.Unlock() + } + } + + children := make([]os.FileInfo, 0, len(n.children)) + for cName, c := range n.children { + children = append(children, c.stat(cName)) + } + return &memFile{ + n: n, + nameSnapshot: frag, + childrenSnapshot: children, + }, nil +} + +func (fs *memFS) RemoveAll(ctx context.Context, name string) error { + fs.mu.Lock() + defer fs.mu.Unlock() + + dir, frag, err := fs.find("remove", name) + if err != nil { + return err + } + if dir == nil { + // We can't remove the root. + return os.ErrInvalid + } + delete(dir.children, frag) + return nil +} + +func (fs *memFS) Rename(ctx context.Context, oldName, newName string) error { + fs.mu.Lock() + defer fs.mu.Unlock() + + oldName = slashClean(oldName) + newName = slashClean(newName) + if oldName == newName { + return nil + } + if strings.HasPrefix(newName, oldName+"/") { + // We can't rename oldName to be a sub-directory of itself. + return os.ErrInvalid + } + + oDir, oFrag, err := fs.find("rename", oldName) + if err != nil { + return err + } + if oDir == nil { + // We can't rename from the root. + return os.ErrInvalid + } + + nDir, nFrag, err := fs.find("rename", newName) + if err != nil { + return err + } + if nDir == nil { + // We can't rename to the root. + return os.ErrInvalid + } + + oNode, ok := oDir.children[oFrag] + if !ok { + return os.ErrNotExist + } + if oNode.children != nil { + if nNode, ok := nDir.children[nFrag]; ok { + if nNode.children == nil { + return errNotADirectory + } + if len(nNode.children) != 0 { + return errDirectoryNotEmpty + } + } + } + delete(oDir.children, oFrag) + nDir.children[nFrag] = oNode + return nil +} + +func (fs *memFS) Stat(ctx context.Context, name string) (os.FileInfo, error) { + fs.mu.Lock() + defer fs.mu.Unlock() + + dir, frag, err := fs.find("stat", name) + if err != nil { + return nil, err + } + if dir == nil { + // We're stat'ting the root. + return fs.root.stat("/"), nil + } + if n, ok := dir.children[frag]; ok { + return n.stat(path.Base(name)), nil + } + return nil, os.ErrNotExist +} + +// A memFSNode represents a single entry in the in-memory filesystem and also +// implements os.FileInfo. +type memFSNode struct { + // children is protected by memFS.mu. + children map[string]*memFSNode + + mu sync.Mutex + data []byte + mode os.FileMode + modTime time.Time + deadProps map[xml.Name]Property +} + +func (n *memFSNode) stat(name string) *memFileInfo { + n.mu.Lock() + defer n.mu.Unlock() + return &memFileInfo{ + name: name, + size: int64(len(n.data)), + mode: n.mode, + modTime: n.modTime, + } +} + +func (n *memFSNode) DeadProps() (map[xml.Name]Property, error) { + n.mu.Lock() + defer n.mu.Unlock() + if len(n.deadProps) == 0 { + return nil, nil + } + ret := make(map[xml.Name]Property, len(n.deadProps)) + for k, v := range n.deadProps { + ret[k] = v + } + return ret, nil +} + +func (n *memFSNode) Patch(patches []Proppatch) ([]Propstat, error) { + n.mu.Lock() + defer n.mu.Unlock() + pstat := Propstat{Status: http.StatusOK} + for _, patch := range patches { + for _, p := range patch.Props { + pstat.Props = append(pstat.Props, Property{XMLName: p.XMLName}) + if patch.Remove { + delete(n.deadProps, p.XMLName) + continue + } + if n.deadProps == nil { + n.deadProps = map[xml.Name]Property{} + } + n.deadProps[p.XMLName] = p + } + } + return []Propstat{pstat}, nil +} + +type memFileInfo struct { + name string + size int64 + mode os.FileMode + modTime time.Time +} + +func (f *memFileInfo) Name() string { return f.name } +func (f *memFileInfo) Size() int64 { return f.size } +func (f *memFileInfo) Mode() os.FileMode { return f.mode } +func (f *memFileInfo) ModTime() time.Time { return f.modTime } +func (f *memFileInfo) IsDir() bool { return f.mode.IsDir() } +func (f *memFileInfo) Sys() interface{} { return nil } + +// A memFile is a File implementation for a memFSNode. It is a per-file (not +// per-node) read/write position, and a snapshot of the memFS' tree structure +// (a node's name and children) for that node. +type memFile struct { + n *memFSNode + nameSnapshot string + childrenSnapshot []os.FileInfo + // pos is protected by n.mu. + pos int +} + +// A *memFile implements the optional DeadPropsHolder interface. +var _ DeadPropsHolder = (*memFile)(nil) + +func (f *memFile) DeadProps() (map[xml.Name]Property, error) { return f.n.DeadProps() } +func (f *memFile) Patch(patches []Proppatch) ([]Propstat, error) { return f.n.Patch(patches) } + +func (f *memFile) Close() error { + return nil +} + +func (f *memFile) Read(p []byte) (int, error) { + f.n.mu.Lock() + defer f.n.mu.Unlock() + if f.n.mode.IsDir() { + return 0, os.ErrInvalid + } + if f.pos >= len(f.n.data) { + return 0, io.EOF + } + n := copy(p, f.n.data[f.pos:]) + f.pos += n + return n, nil +} + +func (f *memFile) Readdir(count int) ([]os.FileInfo, error) { + f.n.mu.Lock() + defer f.n.mu.Unlock() + if !f.n.mode.IsDir() { + return nil, os.ErrInvalid + } + old := f.pos + if old >= len(f.childrenSnapshot) { + // The os.File Readdir docs say that at the end of a directory, + // the error is io.EOF if count > 0 and nil if count <= 0. + if count > 0 { + return nil, io.EOF + } + return nil, nil + } + if count > 0 { + f.pos += count + if f.pos > len(f.childrenSnapshot) { + f.pos = len(f.childrenSnapshot) + } + } else { + f.pos = len(f.childrenSnapshot) + old = 0 + } + return f.childrenSnapshot[old:f.pos], nil +} + +func (f *memFile) Seek(offset int64, whence int) (int64, error) { + f.n.mu.Lock() + defer f.n.mu.Unlock() + npos := f.pos + // TODO: How to handle offsets greater than the size of system int? + switch whence { + case io.SeekStart: + npos = int(offset) + case io.SeekCurrent: + npos += int(offset) + case io.SeekEnd: + npos = len(f.n.data) + int(offset) + default: + npos = -1 + } + if npos < 0 { + return 0, os.ErrInvalid + } + f.pos = npos + return int64(f.pos), nil +} + +func (f *memFile) Stat() (os.FileInfo, error) { + return f.n.stat(f.nameSnapshot), nil +} + +func (f *memFile) Write(p []byte) (int, error) { + lenp := len(p) + f.n.mu.Lock() + defer f.n.mu.Unlock() + + if f.n.mode.IsDir() { + return 0, os.ErrInvalid + } + if f.pos < len(f.n.data) { + n := copy(f.n.data[f.pos:], p) + f.pos += n + p = p[n:] + } else if f.pos > len(f.n.data) { + // Write permits the creation of holes, if we've seek'ed past the + // existing end of file. + if f.pos <= cap(f.n.data) { + oldLen := len(f.n.data) + f.n.data = f.n.data[:f.pos] + hole := f.n.data[oldLen:] + for i := range hole { + hole[i] = 0 + } + } else { + d := make([]byte, f.pos, f.pos+len(p)) + copy(d, f.n.data) + f.n.data = d + } + } + + if len(p) > 0 { + // We should only get here if f.pos == len(f.n.data). + f.n.data = append(f.n.data, p...) + f.pos = len(f.n.data) + } + f.n.modTime = time.Now() + return lenp, nil +} + +// moveFiles moves files and/or directories from src to dst. +// +// See section 9.9.4 for when various HTTP status codes apply. +func moveFiles(ctx context.Context, fs FileSystem, src, dst string, overwrite bool) (status int, err error) { + created := false + if _, err := fs.Stat(ctx, dst); err != nil { + if !os.IsNotExist(err) { + return http.StatusForbidden, err + } + created = true + } else if overwrite { + // Section 9.9.3 says that "If a resource exists at the destination + // and the Overwrite header is "T", then prior to performing the move, + // the server must perform a DELETE with "Depth: infinity" on the + // destination resource. + if err := fs.RemoveAll(ctx, dst); err != nil { + return http.StatusForbidden, err + } + } else { + return http.StatusPreconditionFailed, os.ErrExist + } + if err := fs.Rename(ctx, src, dst); err != nil { + return http.StatusForbidden, err + } + if created { + return http.StatusCreated, nil + } + return http.StatusNoContent, nil +} + +func copyProps(dst, src File) error { + d, ok := dst.(DeadPropsHolder) + if !ok { + return nil + } + s, ok := src.(DeadPropsHolder) + if !ok { + return nil + } + m, err := s.DeadProps() + if err != nil { + return err + } + props := make([]Property, 0, len(m)) + for _, prop := range m { + props = append(props, prop) + } + _, err = d.Patch([]Proppatch{{Props: props}}) + return err +} + +// copyFiles copies files and/or directories from src to dst. +// +// See section 9.8.5 for when various HTTP status codes apply. +func copyFiles(ctx context.Context, fs FileSystem, src, dst string, overwrite bool, depth int, recursion int) (status int, err error) { + if recursion == 1000 { + return http.StatusInternalServerError, errRecursionTooDeep + } + recursion++ + + // TODO: section 9.8.3 says that "Note that an infinite-depth COPY of /A/ + // into /A/B/ could lead to infinite recursion if not handled correctly." + + srcFile, err := fs.OpenFile(ctx, src, os.O_RDONLY, 0) + if err != nil { + if os.IsNotExist(err) { + return http.StatusNotFound, err + } + return http.StatusInternalServerError, err + } + defer srcFile.Close() + srcStat, err := srcFile.Stat() + if err != nil { + if os.IsNotExist(err) { + return http.StatusNotFound, err + } + return http.StatusInternalServerError, err + } + srcPerm := srcStat.Mode() & os.ModePerm + + created := false + if _, err := fs.Stat(ctx, dst); err != nil { + if os.IsNotExist(err) { + created = true + } else { + return http.StatusForbidden, err + } + } else { + if !overwrite { + return http.StatusPreconditionFailed, os.ErrExist + } + if err := fs.RemoveAll(ctx, dst); err != nil && !os.IsNotExist(err) { + return http.StatusForbidden, err + } + } + + if srcStat.IsDir() { + if err := fs.Mkdir(ctx, dst, srcPerm); err != nil { + return http.StatusForbidden, err + } + if depth == infiniteDepth { + children, err := srcFile.Readdir(-1) + if err != nil { + return http.StatusForbidden, err + } + for _, c := range children { + name := c.Name() + s := path.Join(src, name) + d := path.Join(dst, name) + cStatus, cErr := copyFiles(ctx, fs, s, d, overwrite, depth, recursion) + if cErr != nil { + // TODO: MultiStatus. + return cStatus, cErr + } + } + } + + } else { + dstFile, err := fs.OpenFile(ctx, dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, srcPerm) + if err != nil { + if os.IsNotExist(err) { + return http.StatusConflict, err + } + return http.StatusForbidden, err + + } + _, copyErr := io.Copy(dstFile, srcFile) + propsErr := copyProps(dstFile, srcFile) + closeErr := dstFile.Close() + if copyErr != nil { + return http.StatusInternalServerError, copyErr + } + if propsErr != nil { + return http.StatusInternalServerError, propsErr + } + if closeErr != nil { + return http.StatusInternalServerError, closeErr + } + } + + if created { + return http.StatusCreated, nil + } + return http.StatusNoContent, nil +} + +// walkFS traverses filesystem fs starting at name up to depth levels. +// +// Allowed values for depth are 0, 1 or infiniteDepth. For each visited node, +// walkFS calls walkFn. If a visited file system node is a directory and +// walkFn returns filepath.SkipDir, walkFS will skip traversal of this node. +func walkFS(ctx context.Context, fs FileSystem, depth int, name string, info os.FileInfo, walkFn filepath.WalkFunc) error { + // This implementation is based on Walk's code in the standard path/filepath package. + err := walkFn(name, info, nil) + if err != nil { + if info.IsDir() && err == filepath.SkipDir { + return nil + } + return err + } + if !info.IsDir() || depth == 0 { + return nil + } + if depth == 1 { + depth = 0 + } + + // Read directory names. + f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0) + if err != nil { + return walkFn(name, info, err) + } + fileInfos, err := f.Readdir(0) + f.Close() + if err != nil { + return walkFn(name, info, err) + } + + for _, fileInfo := range fileInfos { + filename := path.Join(name, fileInfo.Name()) + fileInfo, err := fs.Stat(ctx, filename) + if err != nil { + if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { + return err + } + } else { + err = walkFS(ctx, fs, depth, filename, fileInfo, walkFn) + if err != nil { + if !fileInfo.IsDir() || err != filepath.SkipDir { + return err + } + } + } + } + return nil +} diff --git a/endpoints/drive/webdav/file_test.go b/endpoints/drive/webdav/file_test.go new file mode 100644 index 000000000..e875c136c --- /dev/null +++ b/endpoints/drive/webdav/file_test.go @@ -0,0 +1,1183 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package webdav + +import ( + "context" + "encoding/xml" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "path/filepath" + "reflect" + "runtime" + "sort" + "strconv" + "strings" + "testing" +) + +func TestSlashClean(t *testing.T) { + testCases := []string{ + "", + ".", + "/", + "/./", + "//", + "//.", + "//a", + "/a", + "/a/b/c", + "/a//b/./../c/d/", + "a", + "a/b/c", + } + for _, tc := range testCases { + got := slashClean(tc) + want := path.Clean("/" + tc) + if got != want { + t.Errorf("tc=%q: got %q, want %q", tc, got, want) + } + } +} + +func TestDirResolve(t *testing.T) { + testCases := []struct { + dir, name, want string + }{ + {"/", "", "/"}, + {"/", "/", "/"}, + {"/", ".", "/"}, + {"/", "./a", "/a"}, + {"/", "..", "/"}, + {"/", "..", "/"}, + {"/", "../", "/"}, + {"/", "../.", "/"}, + {"/", "../a", "/a"}, + {"/", "../..", "/"}, + {"/", "../bar/a", "/bar/a"}, + {"/", "../baz/a", "/baz/a"}, + {"/", "...", "/..."}, + {"/", ".../a", "/.../a"}, + {"/", ".../..", "/"}, + {"/", "a", "/a"}, + {"/", "a/./b", "/a/b"}, + {"/", "a/../../b", "/b"}, + {"/", "a/../b", "/b"}, + {"/", "a/b", "/a/b"}, + {"/", "a/b/c/../../d", "/a/d"}, + {"/", "a/b/c/../../../d", "/d"}, + {"/", "a/b/c/../../../../d", "/d"}, + {"/", "a/b/c/d", "/a/b/c/d"}, + + {"/foo/bar", "", "/foo/bar"}, + {"/foo/bar", "/", "/foo/bar"}, + {"/foo/bar", ".", "/foo/bar"}, + {"/foo/bar", "./a", "/foo/bar/a"}, + {"/foo/bar", "..", "/foo/bar"}, + {"/foo/bar", "../", "/foo/bar"}, + {"/foo/bar", "../.", "/foo/bar"}, + {"/foo/bar", "../a", "/foo/bar/a"}, + {"/foo/bar", "../..", "/foo/bar"}, + {"/foo/bar", "../bar/a", "/foo/bar/bar/a"}, + {"/foo/bar", "../baz/a", "/foo/bar/baz/a"}, + {"/foo/bar", "...", "/foo/bar/..."}, + {"/foo/bar", ".../a", "/foo/bar/.../a"}, + {"/foo/bar", ".../..", "/foo/bar"}, + {"/foo/bar", "a", "/foo/bar/a"}, + {"/foo/bar", "a/./b", "/foo/bar/a/b"}, + {"/foo/bar", "a/../../b", "/foo/bar/b"}, + {"/foo/bar", "a/../b", "/foo/bar/b"}, + {"/foo/bar", "a/b", "/foo/bar/a/b"}, + {"/foo/bar", "a/b/c/../../d", "/foo/bar/a/d"}, + {"/foo/bar", "a/b/c/../../../d", "/foo/bar/d"}, + {"/foo/bar", "a/b/c/../../../../d", "/foo/bar/d"}, + {"/foo/bar", "a/b/c/d", "/foo/bar/a/b/c/d"}, + + {"/foo/bar/", "", "/foo/bar"}, + {"/foo/bar/", "/", "/foo/bar"}, + {"/foo/bar/", ".", "/foo/bar"}, + {"/foo/bar/", "./a", "/foo/bar/a"}, + {"/foo/bar/", "..", "/foo/bar"}, + + {"/foo//bar///", "", "/foo/bar"}, + {"/foo//bar///", "/", "/foo/bar"}, + {"/foo//bar///", ".", "/foo/bar"}, + {"/foo//bar///", "./a", "/foo/bar/a"}, + {"/foo//bar///", "..", "/foo/bar"}, + + {"/x/y/z", "ab/c\x00d/ef", ""}, + + {".", "", "."}, + {".", "/", "."}, + {".", ".", "."}, + {".", "./a", "a"}, + {".", "..", "."}, + {".", "..", "."}, + {".", "../", "."}, + {".", "../.", "."}, + {".", "../a", "a"}, + {".", "../..", "."}, + {".", "../bar/a", "bar/a"}, + {".", "../baz/a", "baz/a"}, + {".", "...", "..."}, + {".", ".../a", ".../a"}, + {".", ".../..", "."}, + {".", "a", "a"}, + {".", "a/./b", "a/b"}, + {".", "a/../../b", "b"}, + {".", "a/../b", "b"}, + {".", "a/b", "a/b"}, + {".", "a/b/c/../../d", "a/d"}, + {".", "a/b/c/../../../d", "d"}, + {".", "a/b/c/../../../../d", "d"}, + {".", "a/b/c/d", "a/b/c/d"}, + + {"", "", "."}, + {"", "/", "."}, + {"", ".", "."}, + {"", "./a", "a"}, + {"", "..", "."}, + } + + for _, tc := range testCases { + d := Dir(filepath.FromSlash(tc.dir)) + if got := filepath.ToSlash(d.resolve(tc.name)); got != tc.want { + t.Errorf("dir=%q, name=%q: got %q, want %q", tc.dir, tc.name, got, tc.want) + } + } +} + +func TestWalk(t *testing.T) { + type walkStep struct { + name, frag string + final bool + } + + testCases := []struct { + dir string + want []walkStep + }{ + {"", []walkStep{ + {"", "", true}, + }}, + {"/", []walkStep{ + {"", "", true}, + }}, + {"/a", []walkStep{ + {"", "a", true}, + }}, + {"/a/", []walkStep{ + {"", "a", true}, + }}, + {"/a/b", []walkStep{ + {"", "a", false}, + {"a", "b", true}, + }}, + {"/a/b/", []walkStep{ + {"", "a", false}, + {"a", "b", true}, + }}, + {"/a/b/c", []walkStep{ + {"", "a", false}, + {"a", "b", false}, + {"b", "c", true}, + }}, + // The following test case is the one mentioned explicitly + // in the method description. + {"/foo/bar/x", []walkStep{ + {"", "foo", false}, + {"foo", "bar", false}, + {"bar", "x", true}, + }}, + } + + ctx := context.Background() + + for _, tc := range testCases { + fs := NewMemFS().(*memFS) + + parts := strings.Split(tc.dir, "/") + for p := 2; p < len(parts); p++ { + d := strings.Join(parts[:p], "/") + if err := fs.Mkdir(ctx, d, 0666); err != nil { + t.Errorf("tc.dir=%q: mkdir: %q: %v", tc.dir, d, err) + } + } + + i, prevFrag := 0, "" + err := fs.walk("test", tc.dir, func(dir *memFSNode, frag string, final bool) error { + got := walkStep{ + name: prevFrag, + frag: frag, + final: final, + } + want := tc.want[i] + + if got != want { + return fmt.Errorf("got %+v, want %+v", got, want) + } + i, prevFrag = i+1, frag + return nil + }) + if err != nil { + t.Errorf("tc.dir=%q: %v", tc.dir, err) + } + } +} + +// find appends to ss the names of the named file and its children. It is +// analogous to the Unix find command. +// +// The returned strings are not guaranteed to be in any particular order. +func find(ctx context.Context, ss []string, fs FileSystem, name string) ([]string, error) { + stat, err := fs.Stat(ctx, name) + if err != nil { + return nil, err + } + ss = append(ss, name) + if stat.IsDir() { + f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0) + if err != nil { + return nil, err + } + defer f.Close() + children, err := f.Readdir(-1) + if err != nil { + return nil, err + } + for _, c := range children { + ss, err = find(ctx, ss, fs, path.Join(name, c.Name())) + if err != nil { + return nil, err + } + } + } + return ss, nil +} + +func testFS(t *testing.T, fs FileSystem) { + errStr := func(err error) string { + switch { + case os.IsExist(err): + return "errExist" + case os.IsNotExist(err): + return "errNotExist" + case err != nil: + return "err" + } + return "ok" + } + + // The non-"find" non-"stat" test cases should change the file system state. The + // indentation of the "find"s and "stat"s helps distinguish such test cases. + testCases := []string{ + " stat / want dir", + " stat /a want errNotExist", + " stat /d want errNotExist", + " stat /d/e want errNotExist", + "create /a A want ok", + " stat /a want 1", + "create /d/e EEE want errNotExist", + "mk-dir /a want errExist", + "mk-dir /d/m want errNotExist", + "mk-dir /d want ok", + " stat /d want dir", + "create /d/e EEE want ok", + " stat /d/e want 3", + " find / /a /d /d/e", + "create /d/f FFFF want ok", + "create /d/g GGGGGGG want ok", + "mk-dir /d/m want ok", + "mk-dir /d/m want errExist", + "create /d/m/p PPPPP want ok", + " stat /d/e want 3", + " stat /d/f want 4", + " stat /d/g want 7", + " stat /d/h want errNotExist", + " stat /d/m want dir", + " stat /d/m/p want 5", + " find / /a /d /d/e /d/f /d/g /d/m /d/m/p", + "rm-all /d want ok", + " stat /a want 1", + " stat /d want errNotExist", + " stat /d/e want errNotExist", + " stat /d/f want errNotExist", + " stat /d/g want errNotExist", + " stat /d/m want errNotExist", + " stat /d/m/p want errNotExist", + " find / /a", + "mk-dir /d/m want errNotExist", + "mk-dir /d want ok", + "create /d/f FFFF want ok", + "rm-all /d/f want ok", + "mk-dir /d/m want ok", + "rm-all /z want ok", + "rm-all / want err", + "create /b BB want ok", + " stat / want dir", + " stat /a want 1", + " stat /b want 2", + " stat /c want errNotExist", + " stat /d want dir", + " stat /d/m want dir", + " find / /a /b /d /d/m", + "move__ o=F /b /c want ok", + " stat /b want errNotExist", + " stat /c want 2", + " stat /d/m want dir", + " stat /d/n want errNotExist", + " find / /a /c /d /d/m", + "move__ o=F /d/m /d/n want ok", + "create /d/n/q QQQQ want ok", + " stat /d/m want errNotExist", + " stat /d/n want dir", + " stat /d/n/q want 4", + "move__ o=F /d /d/n/z want err", + "move__ o=T /c /d/n/q want ok", + " stat /c want errNotExist", + " stat /d/n/q want 2", + " find / /a /d /d/n /d/n/q", + "create /d/n/r RRRRR want ok", + "mk-dir /u want ok", + "mk-dir /u/v want ok", + "move__ o=F /d/n /u want errExist", + "create /t TTTTTT want ok", + "move__ o=F /d/n /t want errExist", + "rm-all /t want ok", + "move__ o=F /d/n /t want ok", + " stat /d want dir", + " stat /d/n want errNotExist", + " stat /d/n/r want errNotExist", + " stat /t want dir", + " stat /t/q want 2", + " stat /t/r want 5", + " find / /a /d /t /t/q /t/r /u /u/v", + "move__ o=F /t / want errExist", + "move__ o=T /t /u/v want ok", + " stat /u/v/r want 5", + "move__ o=F / /z want err", + " find / /a /d /u /u/v /u/v/q /u/v/r", + " stat /a want 1", + " stat /b want errNotExist", + " stat /c want errNotExist", + " stat /u/v/r want 5", + "copy__ o=F d=0 /a /b want ok", + "copy__ o=T d=0 /a /c want ok", + " stat /a want 1", + " stat /b want 1", + " stat /c want 1", + " stat /u/v/r want 5", + "copy__ o=F d=0 /u/v/r /b want errExist", + " stat /b want 1", + "copy__ o=T d=0 /u/v/r /b want ok", + " stat /a want 1", + " stat /b want 5", + " stat /u/v/r want 5", + "rm-all /a want ok", + "rm-all /b want ok", + "mk-dir /u/v/w want ok", + "create /u/v/w/s SSSSSSSS want ok", + " stat /d want dir", + " stat /d/x want errNotExist", + " stat /d/y want errNotExist", + " stat /u/v/r want 5", + " stat /u/v/w/s want 8", + " find / /c /d /u /u/v /u/v/q /u/v/r /u/v/w /u/v/w/s", + "copy__ o=T d=0 /u/v /d/x want ok", + "copy__ o=T d=∞ /u/v /d/y want ok", + "rm-all /u want ok", + " stat /d/x want dir", + " stat /d/x/q want errNotExist", + " stat /d/x/r want errNotExist", + " stat /d/x/w want errNotExist", + " stat /d/x/w/s want errNotExist", + " stat /d/y want dir", + " stat /d/y/q want 2", + " stat /d/y/r want 5", + " stat /d/y/w want dir", + " stat /d/y/w/s want 8", + " stat /u want errNotExist", + " find / /c /d /d/x /d/y /d/y/q /d/y/r /d/y/w /d/y/w/s", + "copy__ o=F d=∞ /d/y /d/x want errExist", + } + + ctx := context.Background() + + for i, tc := range testCases { + tc = strings.TrimSpace(tc) + j := strings.IndexByte(tc, ' ') + if j < 0 { + t.Fatalf("test case #%d %q: invalid command", i, tc) + } + op, arg := tc[:j], tc[j+1:] + + switch op { + default: + t.Fatalf("test case #%d %q: invalid operation %q", i, tc, op) + + case "create": + parts := strings.Split(arg, " ") + if len(parts) != 4 || parts[2] != "want" { + t.Fatalf("test case #%d %q: invalid write", i, tc) + } + f, opErr := fs.OpenFile(ctx, parts[0], os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + if got := errStr(opErr); got != parts[3] { + t.Fatalf("test case #%d %q: OpenFile: got %q (%v), want %q", i, tc, got, opErr, parts[3]) + } + if f != nil { + if _, err := f.Write([]byte(parts[1])); err != nil { + t.Fatalf("test case #%d %q: Write: %v", i, tc, err) + } + if err := f.Close(); err != nil { + t.Fatalf("test case #%d %q: Close: %v", i, tc, err) + } + } + + case "find": + got, err := find(ctx, nil, fs, "/") + if err != nil { + t.Fatalf("test case #%d %q: find: %v", i, tc, err) + } + sort.Strings(got) + want := strings.Split(arg, " ") + if !reflect.DeepEqual(got, want) { + t.Fatalf("test case #%d %q:\ngot %s\nwant %s", i, tc, got, want) + } + + case "copy__", "mk-dir", "move__", "rm-all", "stat": + nParts := 3 + switch op { + case "copy__": + nParts = 6 + case "move__": + nParts = 5 + } + parts := strings.Split(arg, " ") + if len(parts) != nParts { + t.Fatalf("test case #%d %q: invalid %s", i, tc, op) + } + + got, opErr := "", error(nil) + switch op { + case "copy__": + depth := 0 + if parts[1] == "d=∞" { + depth = infiniteDepth + } + _, opErr = copyFiles(ctx, fs, parts[2], parts[3], parts[0] == "o=T", depth, 0) + case "mk-dir": + opErr = fs.Mkdir(ctx, parts[0], 0777) + case "move__": + _, opErr = moveFiles(ctx, fs, parts[1], parts[2], parts[0] == "o=T") + case "rm-all": + opErr = fs.RemoveAll(ctx, parts[0]) + case "stat": + var stat os.FileInfo + fileName := parts[0] + if stat, opErr = fs.Stat(ctx, fileName); opErr == nil { + if stat.IsDir() { + got = "dir" + } else { + got = strconv.Itoa(int(stat.Size())) + } + + if fileName == "/" { + // For a Dir FileSystem, the virtual file system root maps to a + // real file system name like "/tmp/webdav-test012345", which does + // not end with "/". We skip such cases. + } else if statName := stat.Name(); path.Base(fileName) != statName { + t.Fatalf("test case #%d %q: file name %q inconsistent with stat name %q", + i, tc, fileName, statName) + } + } + } + if got == "" { + got = errStr(opErr) + } + + if parts[len(parts)-2] != "want" { + t.Fatalf("test case #%d %q: invalid %s", i, tc, op) + } + if want := parts[len(parts)-1]; got != want { + t.Fatalf("test case #%d %q: got %q (%v), want %q", i, tc, got, opErr, want) + } + } + } +} + +func TestDir(t *testing.T) { + switch runtime.GOOS { + case "nacl": + t.Skip("see golang.org/issue/12004") + case "plan9": + t.Skip("see golang.org/issue/11453") + } + + td, err := ioutil.TempDir("", "webdav-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(td) + testFS(t, Dir(td)) +} + +func TestMemFS(t *testing.T) { + testFS(t, NewMemFS()) +} + +func TestMemFSRoot(t *testing.T) { + ctx := context.Background() + fs := NewMemFS() + for i := 0; i < 5; i++ { + stat, err := fs.Stat(ctx, "/") + if err != nil { + t.Fatalf("i=%d: Stat: %v", i, err) + } + if !stat.IsDir() { + t.Fatalf("i=%d: Stat.IsDir is false, want true", i) + } + + f, err := fs.OpenFile(ctx, "/", os.O_RDONLY, 0) + if err != nil { + t.Fatalf("i=%d: OpenFile: %v", i, err) + } + defer f.Close() + children, err := f.Readdir(-1) + if err != nil { + t.Fatalf("i=%d: Readdir: %v", i, err) + } + if len(children) != i { + t.Fatalf("i=%d: got %d children, want %d", i, len(children), i) + } + + if _, err := f.Write(make([]byte, 1)); err == nil { + t.Fatalf("i=%d: Write: got nil error, want non-nil", i) + } + + if err := fs.Mkdir(ctx, fmt.Sprintf("/dir%d", i), 0777); err != nil { + t.Fatalf("i=%d: Mkdir: %v", i, err) + } + } +} + +func TestMemFileReaddir(t *testing.T) { + ctx := context.Background() + fs := NewMemFS() + if err := fs.Mkdir(ctx, "/foo", 0777); err != nil { + t.Fatalf("Mkdir: %v", err) + } + readdir := func(count int) ([]os.FileInfo, error) { + f, err := fs.OpenFile(ctx, "/foo", os.O_RDONLY, 0) + if err != nil { + t.Fatalf("OpenFile: %v", err) + } + defer f.Close() + return f.Readdir(count) + } + if got, err := readdir(-1); len(got) != 0 || err != nil { + t.Fatalf("readdir(-1): got %d fileInfos with err=%v, want 0, ", len(got), err) + } + if got, err := readdir(+1); len(got) != 0 || err != io.EOF { + t.Fatalf("readdir(+1): got %d fileInfos with err=%v, want 0, EOF", len(got), err) + } +} + +func TestMemFile(t *testing.T) { + testCases := []string{ + "wantData ", + "wantSize 0", + "write abc", + "wantData abc", + "write de", + "wantData abcde", + "wantSize 5", + "write 5*x", + "write 4*y+2*z", + "write 3*st", + "wantData abcdexxxxxyyyyzzststst", + "wantSize 22", + "seek set 4 want 4", + "write EFG", + "wantData abcdEFGxxxyyyyzzststst", + "wantSize 22", + "seek set 2 want 2", + "read cdEF", + "read Gx", + "seek cur 0 want 8", + "seek cur 2 want 10", + "seek cur -1 want 9", + "write J", + "wantData abcdEFGxxJyyyyzzststst", + "wantSize 22", + "seek cur -4 want 6", + "write ghijk", + "wantData abcdEFghijkyyyzzststst", + "wantSize 22", + "read yyyz", + "seek cur 0 want 15", + "write ", + "seek cur 0 want 15", + "read ", + "seek cur 0 want 15", + "seek end -3 want 19", + "write ZZ", + "wantData abcdEFghijkyyyzzstsZZt", + "wantSize 22", + "write 4*A", + "wantData abcdEFghijkyyyzzstsZZAAAA", + "wantSize 25", + "seek end 0 want 25", + "seek end -5 want 20", + "read Z+4*A", + "write 5*B", + "wantData abcdEFghijkyyyzzstsZZAAAABBBBB", + "wantSize 30", + "seek end 10 want 40", + "write C", + "wantData abcdEFghijkyyyzzstsZZAAAABBBBB..........C", + "wantSize 41", + "write D", + "wantData abcdEFghijkyyyzzstsZZAAAABBBBB..........CD", + "wantSize 42", + "seek set 43 want 43", + "write E", + "wantData abcdEFghijkyyyzzstsZZAAAABBBBB..........CD.E", + "wantSize 44", + "seek set 0 want 0", + "write 5*123456789_", + "wantData 123456789_123456789_123456789_123456789_123456789_", + "wantSize 50", + "seek cur 0 want 50", + "seek cur -99 want err", + } + + ctx := context.Background() + + const filename = "/foo" + fs := NewMemFS() + f, err := fs.OpenFile(ctx, filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + t.Fatalf("OpenFile: %v", err) + } + defer f.Close() + + for i, tc := range testCases { + j := strings.IndexByte(tc, ' ') + if j < 0 { + t.Fatalf("test case #%d %q: invalid command", i, tc) + } + op, arg := tc[:j], tc[j+1:] + + // Expand an arg like "3*a+2*b" to "aaabb". + parts := strings.Split(arg, "+") + for j, part := range parts { + if k := strings.IndexByte(part, '*'); k >= 0 { + repeatCount, repeatStr := part[:k], part[k+1:] + n, err := strconv.Atoi(repeatCount) + if err != nil { + t.Fatalf("test case #%d %q: invalid repeat count %q", i, tc, repeatCount) + } + parts[j] = strings.Repeat(repeatStr, n) + } + } + arg = strings.Join(parts, "") + + switch op { + default: + t.Fatalf("test case #%d %q: invalid operation %q", i, tc, op) + + case "read": + buf := make([]byte, len(arg)) + if _, err := io.ReadFull(f, buf); err != nil { + t.Fatalf("test case #%d %q: ReadFull: %v", i, tc, err) + } + if got := string(buf); got != arg { + t.Fatalf("test case #%d %q:\ngot %q\nwant %q", i, tc, got, arg) + } + + case "seek": + parts := strings.Split(arg, " ") + if len(parts) != 4 { + t.Fatalf("test case #%d %q: invalid seek", i, tc) + } + + whence := 0 + switch parts[0] { + default: + t.Fatalf("test case #%d %q: invalid seek whence", i, tc) + case "set": + whence = io.SeekStart + case "cur": + whence = io.SeekCurrent + case "end": + whence = io.SeekEnd + } + offset, err := strconv.Atoi(parts[1]) + if err != nil { + t.Fatalf("test case #%d %q: invalid offset %q", i, tc, parts[1]) + } + + if parts[2] != "want" { + t.Fatalf("test case #%d %q: invalid seek", i, tc) + } + if parts[3] == "err" { + _, err := f.Seek(int64(offset), whence) + if err == nil { + t.Fatalf("test case #%d %q: Seek returned nil error, want non-nil", i, tc) + } + } else { + got, err := f.Seek(int64(offset), whence) + if err != nil { + t.Fatalf("test case #%d %q: Seek: %v", i, tc, err) + } + want, err := strconv.Atoi(parts[3]) + if err != nil { + t.Fatalf("test case #%d %q: invalid want %q", i, tc, parts[3]) + } + if got != int64(want) { + t.Fatalf("test case #%d %q: got %d, want %d", i, tc, got, want) + } + } + + case "write": + n, err := f.Write([]byte(arg)) + if err != nil { + t.Fatalf("test case #%d %q: write: %v", i, tc, err) + } + if n != len(arg) { + t.Fatalf("test case #%d %q: write returned %d bytes, want %d", i, tc, n, len(arg)) + } + + case "wantData": + g, err := fs.OpenFile(ctx, filename, os.O_RDONLY, 0666) + if err != nil { + t.Fatalf("test case #%d %q: OpenFile: %v", i, tc, err) + } + gotBytes, err := ioutil.ReadAll(g) + if err != nil { + t.Fatalf("test case #%d %q: ReadAll: %v", i, tc, err) + } + for i, c := range gotBytes { + if c == '\x00' { + gotBytes[i] = '.' + } + } + got := string(gotBytes) + if got != arg { + t.Fatalf("test case #%d %q:\ngot %q\nwant %q", i, tc, got, arg) + } + if err := g.Close(); err != nil { + t.Fatalf("test case #%d %q: Close: %v", i, tc, err) + } + + case "wantSize": + n, err := strconv.Atoi(arg) + if err != nil { + t.Fatalf("test case #%d %q: invalid size %q", i, tc, arg) + } + fi, err := fs.Stat(ctx, filename) + if err != nil { + t.Fatalf("test case #%d %q: Stat: %v", i, tc, err) + } + if got, want := fi.Size(), int64(n); got != want { + t.Fatalf("test case #%d %q: got %d, want %d", i, tc, got, want) + } + } + } +} + +// TestMemFileWriteAllocs tests that writing N consecutive 1KiB chunks to a +// memFile doesn't allocate a new buffer for each of those N times. Otherwise, +// calling io.Copy(aMemFile, src) is likely to have quadratic complexity. +func TestMemFileWriteAllocs(t *testing.T) { + if runtime.Compiler == "gccgo" { + t.Skip("gccgo allocates here") + } + ctx := context.Background() + fs := NewMemFS() + f, err := fs.OpenFile(ctx, "/xxx", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + t.Fatalf("OpenFile: %v", err) + } + defer f.Close() + + xxx := make([]byte, 1024) + for i := range xxx { + xxx[i] = 'x' + } + + a := testing.AllocsPerRun(100, func() { + f.Write(xxx) + }) + // AllocsPerRun returns an integral value, so we compare the rounded-down + // number to zero. + if a > 0 { + t.Fatalf("%v allocs per run, want 0", a) + } +} + +func BenchmarkMemFileWrite(b *testing.B) { + ctx := context.Background() + fs := NewMemFS() + xxx := make([]byte, 1024) + for i := range xxx { + xxx[i] = 'x' + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + f, err := fs.OpenFile(ctx, "/xxx", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + b.Fatalf("OpenFile: %v", err) + } + for j := 0; j < 100; j++ { + f.Write(xxx) + } + if err := f.Close(); err != nil { + b.Fatalf("Close: %v", err) + } + if err := fs.RemoveAll(ctx, "/xxx"); err != nil { + b.Fatalf("RemoveAll: %v", err) + } + } +} + +func TestCopyMoveProps(t *testing.T) { + ctx := context.Background() + fs := NewMemFS() + create := func(name string) error { + f, err := fs.OpenFile(ctx, name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + return err + } + _, wErr := f.Write([]byte("contents")) + cErr := f.Close() + if wErr != nil { + return wErr + } + return cErr + } + patch := func(name string, patches ...Proppatch) error { + f, err := fs.OpenFile(ctx, name, os.O_RDWR, 0666) + if err != nil { + return err + } + _, pErr := f.(DeadPropsHolder).Patch(patches) + cErr := f.Close() + if pErr != nil { + return pErr + } + return cErr + } + props := func(name string) (map[xml.Name]Property, error) { + f, err := fs.OpenFile(ctx, name, os.O_RDWR, 0666) + if err != nil { + return nil, err + } + m, pErr := f.(DeadPropsHolder).DeadProps() + cErr := f.Close() + if pErr != nil { + return nil, pErr + } + if cErr != nil { + return nil, cErr + } + return m, nil + } + + p0 := Property{ + XMLName: xml.Name{Space: "x:", Local: "boat"}, + InnerXML: []byte("pea-green"), + } + p1 := Property{ + XMLName: xml.Name{Space: "x:", Local: "ring"}, + InnerXML: []byte("1 shilling"), + } + p2 := Property{ + XMLName: xml.Name{Space: "x:", Local: "spoon"}, + InnerXML: []byte("runcible"), + } + p3 := Property{ + XMLName: xml.Name{Space: "x:", Local: "moon"}, + InnerXML: []byte("light"), + } + + if err := create("/src"); err != nil { + t.Fatalf("create /src: %v", err) + } + if err := patch("/src", Proppatch{Props: []Property{p0, p1}}); err != nil { + t.Fatalf("patch /src +p0 +p1: %v", err) + } + if _, err := copyFiles(ctx, fs, "/src", "/tmp", true, infiniteDepth, 0); err != nil { + t.Fatalf("copyFiles /src /tmp: %v", err) + } + if _, err := moveFiles(ctx, fs, "/tmp", "/dst", true); err != nil { + t.Fatalf("moveFiles /tmp /dst: %v", err) + } + if err := patch("/src", Proppatch{Props: []Property{p0}, Remove: true}); err != nil { + t.Fatalf("patch /src -p0: %v", err) + } + if err := patch("/src", Proppatch{Props: []Property{p2}}); err != nil { + t.Fatalf("patch /src +p2: %v", err) + } + if err := patch("/dst", Proppatch{Props: []Property{p1}, Remove: true}); err != nil { + t.Fatalf("patch /dst -p1: %v", err) + } + if err := patch("/dst", Proppatch{Props: []Property{p3}}); err != nil { + t.Fatalf("patch /dst +p3: %v", err) + } + + gotSrc, err := props("/src") + if err != nil { + t.Fatalf("props /src: %v", err) + } + wantSrc := map[xml.Name]Property{ + p1.XMLName: p1, + p2.XMLName: p2, + } + if !reflect.DeepEqual(gotSrc, wantSrc) { + t.Fatalf("props /src:\ngot %v\nwant %v", gotSrc, wantSrc) + } + + gotDst, err := props("/dst") + if err != nil { + t.Fatalf("props /dst: %v", err) + } + wantDst := map[xml.Name]Property{ + p0.XMLName: p0, + p3.XMLName: p3, + } + if !reflect.DeepEqual(gotDst, wantDst) { + t.Fatalf("props /dst:\ngot %v\nwant %v", gotDst, wantDst) + } +} + +func TestWalkFS(t *testing.T) { + testCases := []struct { + desc string + buildfs []string + startAt string + depth int + walkFn filepath.WalkFunc + want []string + }{{ + "just root", + []string{}, + "/", + infiniteDepth, + nil, + []string{ + "/", + }, + }, { + "infinite walk from root", + []string{ + "mkdir /a", + "mkdir /a/b", + "touch /a/b/c", + "mkdir /a/d", + "mkdir /e", + "touch /f", + }, + "/", + infiniteDepth, + nil, + []string{ + "/", + "/a", + "/a/b", + "/a/b/c", + "/a/d", + "/e", + "/f", + }, + }, { + "infinite walk from subdir", + []string{ + "mkdir /a", + "mkdir /a/b", + "touch /a/b/c", + "mkdir /a/d", + "mkdir /e", + "touch /f", + }, + "/a", + infiniteDepth, + nil, + []string{ + "/a", + "/a/b", + "/a/b/c", + "/a/d", + }, + }, { + "depth 1 walk from root", + []string{ + "mkdir /a", + "mkdir /a/b", + "touch /a/b/c", + "mkdir /a/d", + "mkdir /e", + "touch /f", + }, + "/", + 1, + nil, + []string{ + "/", + "/a", + "/e", + "/f", + }, + }, { + "depth 1 walk from subdir", + []string{ + "mkdir /a", + "mkdir /a/b", + "touch /a/b/c", + "mkdir /a/b/g", + "mkdir /a/b/g/h", + "touch /a/b/g/i", + "touch /a/b/g/h/j", + }, + "/a/b", + 1, + nil, + []string{ + "/a/b", + "/a/b/c", + "/a/b/g", + }, + }, { + "depth 0 walk from subdir", + []string{ + "mkdir /a", + "mkdir /a/b", + "touch /a/b/c", + "mkdir /a/b/g", + "mkdir /a/b/g/h", + "touch /a/b/g/i", + "touch /a/b/g/h/j", + }, + "/a/b", + 0, + nil, + []string{ + "/a/b", + }, + }, { + "infinite walk from file", + []string{ + "mkdir /a", + "touch /a/b", + "touch /a/c", + }, + "/a/b", + 0, + nil, + []string{ + "/a/b", + }, + }, { + "infinite walk with skipped subdir", + []string{ + "mkdir /a", + "mkdir /a/b", + "touch /a/b/c", + "mkdir /a/b/g", + "mkdir /a/b/g/h", + "touch /a/b/g/i", + "touch /a/b/g/h/j", + "touch /a/b/z", + }, + "/", + infiniteDepth, + func(path string, info os.FileInfo, err error) error { + if path == "/a/b/g" { + return filepath.SkipDir + } + return nil + }, + []string{ + "/", + "/a", + "/a/b", + "/a/b/c", + "/a/b/z", + }, + }} + ctx := context.Background() + for _, tc := range testCases { + fs, err := buildTestFS(tc.buildfs) + if err != nil { + t.Fatalf("%s: cannot create test filesystem: %v", tc.desc, err) + } + var got []string + traceFn := func(path string, info os.FileInfo, err error) error { + if tc.walkFn != nil { + err = tc.walkFn(path, info, err) + if err != nil { + return err + } + } + got = append(got, path) + return nil + } + fi, err := fs.Stat(ctx, tc.startAt) + if err != nil { + t.Fatalf("%s: cannot stat: %v", tc.desc, err) + } + err = walkFS(ctx, fs, tc.depth, tc.startAt, fi, traceFn) + if err != nil { + t.Errorf("%s:\ngot error %v, want nil", tc.desc, err) + continue + } + sort.Strings(got) + sort.Strings(tc.want) + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("%s:\ngot %q\nwant %q", tc.desc, got, tc.want) + continue + } + } +} + +func buildTestFS(buildfs []string) (FileSystem, error) { + // TODO: Could this be merged with the build logic in TestFS? + + ctx := context.Background() + fs := NewMemFS() + for _, b := range buildfs { + op := strings.Split(b, " ") + switch op[0] { + case "mkdir": + err := fs.Mkdir(ctx, op[1], os.ModeDir|0777) + if err != nil { + return nil, err + } + case "touch": + f, err := fs.OpenFile(ctx, op[1], os.O_RDWR|os.O_CREATE, 0666) + if err != nil { + return nil, err + } + f.Close() + case "write": + f, err := fs.OpenFile(ctx, op[1], os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + return nil, err + } + _, err = f.Write([]byte(op[2])) + f.Close() + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unknown file operation %q", op[0]) + } + } + return fs, nil +} diff --git a/endpoints/drive/webdav/if.go b/endpoints/drive/webdav/if.go new file mode 100644 index 000000000..e646570bb --- /dev/null +++ b/endpoints/drive/webdav/if.go @@ -0,0 +1,173 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package webdav + +// The If header is covered by Section 10.4. +// http://www.webdav.org/specs/rfc4918.html#HEADER_If + +import ( + "strings" +) + +// ifHeader is a disjunction (OR) of ifLists. +type ifHeader struct { + lists []ifList +} + +// ifList is a conjunction (AND) of Conditions, and an optional resource tag. +type ifList struct { + resourceTag string + conditions []Condition +} + +// parseIfHeader parses the "If: foo bar" HTTP header. The httpHeader string +// should omit the "If:" prefix and have any "\r\n"s collapsed to a " ", as is +// returned by req.Header.Get("If") for an http.Request req. +func parseIfHeader(httpHeader string) (h ifHeader, ok bool) { + s := strings.TrimSpace(httpHeader) + switch tokenType, _, _ := lex(s); tokenType { + case '(': + return parseNoTagLists(s) + case angleTokenType: + return parseTaggedLists(s) + default: + return ifHeader{}, false + } +} + +func parseNoTagLists(s string) (h ifHeader, ok bool) { + for { + l, remaining, ok := parseList(s) + if !ok { + return ifHeader{}, false + } + h.lists = append(h.lists, l) + if remaining == "" { + return h, true + } + s = remaining + } +} + +func parseTaggedLists(s string) (h ifHeader, ok bool) { + resourceTag, n := "", 0 + for first := true; ; first = false { + tokenType, tokenStr, remaining := lex(s) + switch tokenType { + case angleTokenType: + if !first && n == 0 { + return ifHeader{}, false + } + resourceTag, n = tokenStr, 0 + s = remaining + case '(': + n++ + l, remaining, ok := parseList(s) + if !ok { + return ifHeader{}, false + } + l.resourceTag = resourceTag + h.lists = append(h.lists, l) + if remaining == "" { + return h, true + } + s = remaining + default: + return ifHeader{}, false + } + } +} + +func parseList(s string) (l ifList, remaining string, ok bool) { + tokenType, _, s := lex(s) + if tokenType != '(' { + return ifList{}, "", false + } + for { + tokenType, _, remaining = lex(s) + if tokenType == ')' { + if len(l.conditions) == 0 { + return ifList{}, "", false + } + return l, remaining, true + } + c, remaining, ok := parseCondition(s) + if !ok { + return ifList{}, "", false + } + l.conditions = append(l.conditions, c) + s = remaining + } +} + +func parseCondition(s string) (c Condition, remaining string, ok bool) { + tokenType, tokenStr, s := lex(s) + if tokenType == notTokenType { + c.Not = true + tokenType, tokenStr, s = lex(s) + } + switch tokenType { + case strTokenType, angleTokenType: + c.Token = tokenStr + case squareTokenType: + c.ETag = tokenStr + default: + return Condition{}, "", false + } + return c, s, true +} + +// Single-rune tokens like '(' or ')' have a token type equal to their rune. +// All other tokens have a negative token type. +const ( + errTokenType = rune(-1) + eofTokenType = rune(-2) + strTokenType = rune(-3) + notTokenType = rune(-4) + angleTokenType = rune(-5) + squareTokenType = rune(-6) +) + +func lex(s string) (tokenType rune, tokenStr string, remaining string) { + // The net/textproto Reader that parses the HTTP header will collapse + // Linear White Space that spans multiple "\r\n" lines to a single " ", + // so we don't need to look for '\r' or '\n'. + for len(s) > 0 && (s[0] == '\t' || s[0] == ' ') { + s = s[1:] + } + if len(s) == 0 { + return eofTokenType, "", "" + } + i := 0 +loop: + for ; i < len(s); i++ { + switch s[i] { + case '\t', ' ', '(', ')', '<', '>', '[', ']': + break loop + } + } + + if i != 0 { + tokenStr, remaining = s[:i], s[i:] + if tokenStr == "Not" { + return notTokenType, "", remaining + } + return strTokenType, tokenStr, remaining + } + + j := 0 + switch s[0] { + case '<': + j, tokenType = strings.IndexByte(s, '>'), angleTokenType + case '[': + j, tokenType = strings.IndexByte(s, ']'), squareTokenType + default: + return rune(s[0]), "", s[1:] + } + if j < 0 { + return errTokenType, "", "" + } + return tokenType, s[1:j], s[j+1:] +} diff --git a/endpoints/drive/webdav/if_test.go b/endpoints/drive/webdav/if_test.go new file mode 100644 index 000000000..aad61a401 --- /dev/null +++ b/endpoints/drive/webdav/if_test.go @@ -0,0 +1,322 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package webdav + +import ( + "reflect" + "strings" + "testing" +) + +func TestParseIfHeader(t *testing.T) { + // The "section x.y.z" test cases come from section x.y.z of the spec at + // http://www.webdav.org/specs/rfc4918.html + testCases := []struct { + desc string + input string + want ifHeader + }{{ + "bad: empty", + ``, + ifHeader{}, + }, { + "bad: no parens", + `foobar`, + ifHeader{}, + }, { + "bad: empty list #1", + `()`, + ifHeader{}, + }, { + "bad: empty list #2", + `(a) (b c) () (d)`, + ifHeader{}, + }, { + "bad: no list after resource #1", + ``, + ifHeader{}, + }, { + "bad: no list after resource #2", + ` (a)`, + ifHeader{}, + }, { + "bad: no list after resource #3", + ` (a) (b) `, + ifHeader{}, + }, { + "bad: no-tag-list followed by tagged-list", + `(a) (b) (c)`, + ifHeader{}, + }, { + "bad: unfinished list", + `(a`, + ifHeader{}, + }, { + "bad: unfinished ETag", + `([b`, + ifHeader{}, + }, { + "bad: unfinished Notted list", + `(Not a`, + ifHeader{}, + }, { + "bad: double Not", + `(Not Not a)`, + ifHeader{}, + }, { + "good: one list with a Token", + `(a)`, + ifHeader{ + lists: []ifList{{ + conditions: []Condition{{ + Token: `a`, + }}, + }}, + }, + }, { + "good: one list with an ETag", + `([a])`, + ifHeader{ + lists: []ifList{{ + conditions: []Condition{{ + ETag: `a`, + }}, + }}, + }, + }, { + "good: one list with three Nots", + `(Not a Not b Not [d])`, + ifHeader{ + lists: []ifList{{ + conditions: []Condition{{ + Not: true, + Token: `a`, + }, { + Not: true, + Token: `b`, + }, { + Not: true, + ETag: `d`, + }}, + }}, + }, + }, { + "good: two lists", + `(a) (b)`, + ifHeader{ + lists: []ifList{{ + conditions: []Condition{{ + Token: `a`, + }}, + }, { + conditions: []Condition{{ + Token: `b`, + }}, + }}, + }, + }, { + "good: two Notted lists", + `(Not a) (Not b)`, + ifHeader{ + lists: []ifList{{ + conditions: []Condition{{ + Not: true, + Token: `a`, + }}, + }, { + conditions: []Condition{{ + Not: true, + Token: `b`, + }}, + }}, + }, + }, { + "section 7.5.1", + ` + ()`, + ifHeader{ + lists: []ifList{{ + resourceTag: `http://www.example.com/users/f/fielding/index.html`, + conditions: []Condition{{ + Token: `urn:uuid:f81d4fae-7dec-11d0-a765-00a0c91e6bf6`, + }}, + }}, + }, + }, { + "section 7.5.2 #1", + `()`, + ifHeader{ + lists: []ifList{{ + conditions: []Condition{{ + Token: `urn:uuid:150852e2-3847-42d5-8cbe-0f4f296f26cf`, + }}, + }}, + }, + }, { + "section 7.5.2 #2", + ` + ()`, + ifHeader{ + lists: []ifList{{ + resourceTag: `http://example.com/locked/`, + conditions: []Condition{{ + Token: `urn:uuid:150852e2-3847-42d5-8cbe-0f4f296f26cf`, + }}, + }}, + }, + }, { + "section 7.5.2 #3", + ` + ()`, + ifHeader{ + lists: []ifList{{ + resourceTag: `http://example.com/locked/member`, + conditions: []Condition{{ + Token: `urn:uuid:150852e2-3847-42d5-8cbe-0f4f296f26cf`, + }}, + }}, + }, + }, { + "section 9.9.6", + `() + ()`, + ifHeader{ + lists: []ifList{{ + conditions: []Condition{{ + Token: `urn:uuid:fe184f2e-6eec-41d0-c765-01adc56e6bb4`, + }}, + }, { + conditions: []Condition{{ + Token: `urn:uuid:e454f3f3-acdc-452a-56c7-00a5c91e4b77`, + }}, + }}, + }, + }, { + "section 9.10.8", + `()`, + ifHeader{ + lists: []ifList{{ + conditions: []Condition{{ + Token: `urn:uuid:e71d4fae-5dec-22d6-fea5-00a0c91e6be4`, + }}, + }}, + }, + }, { + "section 10.4.6", + `( + ["I am an ETag"]) + (["I am another ETag"])`, + ifHeader{ + lists: []ifList{{ + conditions: []Condition{{ + Token: `urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2`, + }, { + ETag: `"I am an ETag"`, + }}, + }, { + conditions: []Condition{{ + ETag: `"I am another ETag"`, + }}, + }}, + }, + }, { + "section 10.4.7", + `(Not + )`, + ifHeader{ + lists: []ifList{{ + conditions: []Condition{{ + Not: true, + Token: `urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2`, + }, { + Token: `urn:uuid:58f202ac-22cf-11d1-b12d-002035b29092`, + }}, + }}, + }, + }, { + "section 10.4.8", + `() + (Not )`, + ifHeader{ + lists: []ifList{{ + conditions: []Condition{{ + Token: `urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2`, + }}, + }, { + conditions: []Condition{{ + Not: true, + Token: `DAV:no-lock`, + }}, + }}, + }, + }, { + "section 10.4.9", + ` + ( + [W/"A weak ETag"]) (["strong ETag"])`, + ifHeader{ + lists: []ifList{{ + resourceTag: `/resource1`, + conditions: []Condition{{ + Token: `urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2`, + }, { + ETag: `W/"A weak ETag"`, + }}, + }, { + resourceTag: `/resource1`, + conditions: []Condition{{ + ETag: `"strong ETag"`, + }}, + }}, + }, + }, { + "section 10.4.10", + ` + ()`, + ifHeader{ + lists: []ifList{{ + resourceTag: `http://www.example.com/specs/`, + conditions: []Condition{{ + Token: `urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2`, + }}, + }}, + }, + }, { + "section 10.4.11 #1", + ` (["4217"])`, + ifHeader{ + lists: []ifList{{ + resourceTag: `/specs/rfc2518.doc`, + conditions: []Condition{{ + ETag: `"4217"`, + }}, + }}, + }, + }, { + "section 10.4.11 #2", + ` (Not ["4217"])`, + ifHeader{ + lists: []ifList{{ + resourceTag: `/specs/rfc2518.doc`, + conditions: []Condition{{ + Not: true, + ETag: `"4217"`, + }}, + }}, + }, + }} + + for _, tc := range testCases { + got, ok := parseIfHeader(strings.Replace(tc.input, "\n", "", -1)) + if gotEmpty := reflect.DeepEqual(got, ifHeader{}); gotEmpty == ok { + t.Errorf("%s: should be different: empty header == %t, ok == %t", tc.desc, gotEmpty, ok) + continue + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("%s:\ngot %v\nwant %v", tc.desc, got, tc.want) + continue + } + } +} diff --git a/endpoints/drive/webdav/internal/xml/README b/endpoints/drive/webdav/internal/xml/README new file mode 100644 index 000000000..89656f489 --- /dev/null +++ b/endpoints/drive/webdav/internal/xml/README @@ -0,0 +1,11 @@ +This is a fork of the encoding/xml package at ca1d6c4, the last commit before +https://go.googlesource.com/go/+/c0d6d33 "encoding/xml: restore Go 1.4 name +space behavior" made late in the lead-up to the Go 1.5 release. + +The list of encoding/xml changes is at +https://go.googlesource.com/go/+log/master/src/encoding/xml + +This fork is temporary, and I (nigeltao) expect to revert it after Go 1.6 is +released. + +See http://golang.org/issue/11841 diff --git a/endpoints/drive/webdav/internal/xml/atom_test.go b/endpoints/drive/webdav/internal/xml/atom_test.go new file mode 100644 index 000000000..a71284312 --- /dev/null +++ b/endpoints/drive/webdav/internal/xml/atom_test.go @@ -0,0 +1,56 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xml + +import "time" + +var atomValue = &Feed{ + XMLName: Name{"http://www.w3.org/2005/Atom", "feed"}, + Title: "Example Feed", + Link: []Link{{Href: "http://example.org/"}}, + Updated: ParseTime("2003-12-13T18:30:02Z"), + Author: Person{Name: "John Doe"}, + Id: "urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6", + + Entry: []Entry{ + { + Title: "Atom-Powered Robots Run Amok", + Link: []Link{{Href: "http://example.org/2003/12/13/atom03"}}, + Id: "urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a", + Updated: ParseTime("2003-12-13T18:30:02Z"), + Summary: NewText("Some text."), + }, + }, +} + +var atomXml = `` + + `` + + `Example Feed` + + `urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6` + + `` + + `John Doe` + + `` + + `Atom-Powered Robots Run Amok` + + `urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a` + + `` + + `2003-12-13T18:30:02Z` + + `` + + `Some text.` + + `` + + `` + +func ParseTime(str string) time.Time { + t, err := time.Parse(time.RFC3339, str) + if err != nil { + panic(err) + } + return t +} + +func NewText(text string) Text { + return Text{ + Body: text, + } +} diff --git a/endpoints/drive/webdav/internal/xml/example_test.go b/endpoints/drive/webdav/internal/xml/example_test.go new file mode 100644 index 000000000..21b48dea5 --- /dev/null +++ b/endpoints/drive/webdav/internal/xml/example_test.go @@ -0,0 +1,151 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xml_test + +import ( + "encoding/xml" + "fmt" + "os" +) + +func ExampleMarshalIndent() { + type Address struct { + City, State string + } + type Person struct { + XMLName xml.Name `xml:"person"` + Id int `xml:"id,attr"` + FirstName string `xml:"name>first"` + LastName string `xml:"name>last"` + Age int `xml:"age"` + Height float32 `xml:"height,omitempty"` + Married bool + Address + Comment string `xml:",comment"` + } + + v := &Person{Id: 13, FirstName: "John", LastName: "Doe", Age: 42} + v.Comment = " Need more details. " + v.Address = Address{"Hanga Roa", "Easter Island"} + + output, err := xml.MarshalIndent(v, " ", " ") + if err != nil { + fmt.Printf("error: %v\n", err) + } + + os.Stdout.Write(output) + // Output: + // + // + // John + // Doe + // + // 42 + // false + // Hanga Roa + // Easter Island + // + // +} + +func ExampleEncoder() { + type Address struct { + City, State string + } + type Person struct { + XMLName xml.Name `xml:"person"` + Id int `xml:"id,attr"` + FirstName string `xml:"name>first"` + LastName string `xml:"name>last"` + Age int `xml:"age"` + Height float32 `xml:"height,omitempty"` + Married bool + Address + Comment string `xml:",comment"` + } + + v := &Person{Id: 13, FirstName: "John", LastName: "Doe", Age: 42} + v.Comment = " Need more details. " + v.Address = Address{"Hanga Roa", "Easter Island"} + + enc := xml.NewEncoder(os.Stdout) + enc.Indent(" ", " ") + if err := enc.Encode(v); err != nil { + fmt.Printf("error: %v\n", err) + } + + // Output: + // + // + // John + // Doe + // + // 42 + // false + // Hanga Roa + // Easter Island + // + // +} + +// This example demonstrates unmarshaling an XML excerpt into a value with +// some preset fields. Note that the Phone field isn't modified and that +// the XML element is ignored. Also, the Groups field is assigned +// considering the element path provided in its tag. +func ExampleUnmarshal() { + type Email struct { + Where string `xml:"where,attr"` + Addr string + } + type Address struct { + City, State string + } + type Result struct { + XMLName xml.Name `xml:"Person"` + Name string `xml:"FullName"` + Phone string + Email []Email + Groups []string `xml:"Group>Value"` + Address + } + v := Result{Name: "none", Phone: "none"} + + data := ` + + Grace R. Emlin + Example Inc. + + gre@example.com + + + gre@work.com + + + Friends + Squash + + Hanga Roa + Easter Island + + ` + err := xml.Unmarshal([]byte(data), &v) + if err != nil { + fmt.Printf("error: %v", err) + return + } + fmt.Printf("XMLName: %#v\n", v.XMLName) + fmt.Printf("Name: %q\n", v.Name) + fmt.Printf("Phone: %q\n", v.Phone) + fmt.Printf("Email: %v\n", v.Email) + fmt.Printf("Groups: %v\n", v.Groups) + fmt.Printf("Address: %v\n", v.Address) + // Output: + // XMLName: xml.Name{Space:"", Local:"Person"} + // Name: "Grace R. Emlin" + // Phone: "none" + // Email: [{home gre@example.com} {work gre@work.com}] + // Groups: [Friends Squash] + // Address: {Hanga Roa Easter Island} +} diff --git a/endpoints/drive/webdav/internal/xml/marshal.go b/endpoints/drive/webdav/internal/xml/marshal.go new file mode 100644 index 000000000..4dd0f417f --- /dev/null +++ b/endpoints/drive/webdav/internal/xml/marshal.go @@ -0,0 +1,1223 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xml + +import ( + "bufio" + "bytes" + "encoding" + "fmt" + "io" + "reflect" + "strconv" + "strings" +) + +const ( + // A generic XML header suitable for use with the output of Marshal. + // This is not automatically added to any output of this package, + // it is provided as a convenience. + Header = `` + "\n" +) + +// Marshal returns the XML encoding of v. +// +// Marshal handles an array or slice by marshalling each of the elements. +// Marshal handles a pointer by marshalling the value it points at or, if the +// pointer is nil, by writing nothing. Marshal handles an interface value by +// marshalling the value it contains or, if the interface value is nil, by +// writing nothing. Marshal handles all other data by writing one or more XML +// elements containing the data. +// +// The name for the XML elements is taken from, in order of preference: +// - the tag on the XMLName field, if the data is a struct +// - the value of the XMLName field of type xml.Name +// - the tag of the struct field used to obtain the data +// - the name of the struct field used to obtain the data +// - the name of the marshalled type +// +// The XML element for a struct contains marshalled elements for each of the +// exported fields of the struct, with these exceptions: +// - the XMLName field, described above, is omitted. +// - a field with tag "-" is omitted. +// - a field with tag "name,attr" becomes an attribute with +// the given name in the XML element. +// - a field with tag ",attr" becomes an attribute with the +// field name in the XML element. +// - a field with tag ",chardata" is written as character data, +// not as an XML element. +// - a field with tag ",innerxml" is written verbatim, not subject +// to the usual marshalling procedure. +// - a field with tag ",comment" is written as an XML comment, not +// subject to the usual marshalling procedure. It must not contain +// the "--" string within it. +// - a field with a tag including the "omitempty" option is omitted +// if the field value is empty. The empty values are false, 0, any +// nil pointer or interface value, and any array, slice, map, or +// string of length zero. +// - an anonymous struct field is handled as if the fields of its +// value were part of the outer struct. +// +// If a field uses a tag "a>b>c", then the element c will be nested inside +// parent elements a and b. Fields that appear next to each other that name +// the same parent will be enclosed in one XML element. +// +// See MarshalIndent for an example. +// +// Marshal will return an error if asked to marshal a channel, function, or map. +func Marshal(v interface{}) ([]byte, error) { + var b bytes.Buffer + if err := NewEncoder(&b).Encode(v); err != nil { + return nil, err + } + return b.Bytes(), nil +} + +// Marshaler is the interface implemented by objects that can marshal +// themselves into valid XML elements. +// +// MarshalXML encodes the receiver as zero or more XML elements. +// By convention, arrays or slices are typically encoded as a sequence +// of elements, one per entry. +// Using start as the element tag is not required, but doing so +// will enable Unmarshal to match the XML elements to the correct +// struct field. +// One common implementation strategy is to construct a separate +// value with a layout corresponding to the desired XML and then +// to encode it using e.EncodeElement. +// Another common strategy is to use repeated calls to e.EncodeToken +// to generate the XML output one token at a time. +// The sequence of encoded tokens must make up zero or more valid +// XML elements. +type Marshaler interface { + MarshalXML(e *Encoder, start StartElement) error +} + +// MarshalerAttr is the interface implemented by objects that can marshal +// themselves into valid XML attributes. +// +// MarshalXMLAttr returns an XML attribute with the encoded value of the receiver. +// Using name as the attribute name is not required, but doing so +// will enable Unmarshal to match the attribute to the correct +// struct field. +// If MarshalXMLAttr returns the zero attribute Attr{}, no attribute +// will be generated in the output. +// MarshalXMLAttr is used only for struct fields with the +// "attr" option in the field tag. +type MarshalerAttr interface { + MarshalXMLAttr(name Name) (Attr, error) +} + +// MarshalIndent works like Marshal, but each XML element begins on a new +// indented line that starts with prefix and is followed by one or more +// copies of indent according to the nesting depth. +func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) { + var b bytes.Buffer + enc := NewEncoder(&b) + enc.Indent(prefix, indent) + if err := enc.Encode(v); err != nil { + return nil, err + } + return b.Bytes(), nil +} + +// An Encoder writes XML data to an output stream. +type Encoder struct { + p printer +} + +// NewEncoder returns a new encoder that writes to w. +func NewEncoder(w io.Writer) *Encoder { + e := &Encoder{printer{Writer: bufio.NewWriter(w)}} + e.p.encoder = e + return e +} + +// Indent sets the encoder to generate XML in which each element +// begins on a new indented line that starts with prefix and is followed by +// one or more copies of indent according to the nesting depth. +func (enc *Encoder) Indent(prefix, indent string) { + enc.p.prefix = prefix + enc.p.indent = indent +} + +// Encode writes the XML encoding of v to the stream. +// +// See the documentation for Marshal for details about the conversion +// of Go values to XML. +// +// Encode calls Flush before returning. +func (enc *Encoder) Encode(v interface{}) error { + err := enc.p.marshalValue(reflect.ValueOf(v), nil, nil) + if err != nil { + return err + } + return enc.p.Flush() +} + +// EncodeElement writes the XML encoding of v to the stream, +// using start as the outermost tag in the encoding. +// +// See the documentation for Marshal for details about the conversion +// of Go values to XML. +// +// EncodeElement calls Flush before returning. +func (enc *Encoder) EncodeElement(v interface{}, start StartElement) error { + err := enc.p.marshalValue(reflect.ValueOf(v), nil, &start) + if err != nil { + return err + } + return enc.p.Flush() +} + +var ( + begComment = []byte("") + endProcInst = []byte("?>") + endDirective = []byte(">") +) + +// EncodeToken writes the given XML token to the stream. +// It returns an error if StartElement and EndElement tokens are not +// properly matched. +// +// EncodeToken does not call Flush, because usually it is part of a +// larger operation such as Encode or EncodeElement (or a custom +// Marshaler's MarshalXML invoked during those), and those will call +// Flush when finished. Callers that create an Encoder and then invoke +// EncodeToken directly, without using Encode or EncodeElement, need to +// call Flush when finished to ensure that the XML is written to the +// underlying writer. +// +// EncodeToken allows writing a ProcInst with Target set to "xml" only +// as the first token in the stream. +// +// When encoding a StartElement holding an XML namespace prefix +// declaration for a prefix that is not already declared, contained +// elements (including the StartElement itself) will use the declared +// prefix when encoding names with matching namespace URIs. +func (enc *Encoder) EncodeToken(t Token) error { + + p := &enc.p + switch t := t.(type) { + case StartElement: + if err := p.writeStart(&t); err != nil { + return err + } + case EndElement: + if err := p.writeEnd(t.Name); err != nil { + return err + } + case CharData: + escapeText(p, t, false) + case Comment: + if bytes.Contains(t, endComment) { + return fmt.Errorf("xml: EncodeToken of Comment containing --> marker") + } + p.WriteString("") + return p.cachedWriteError() + case ProcInst: + // First token to be encoded which is also a ProcInst with target of xml + // is the xml declaration. The only ProcInst where target of xml is allowed. + if t.Target == "xml" && p.Buffered() != 0 { + return fmt.Errorf("xml: EncodeToken of ProcInst xml target only valid for xml declaration, first token encoded") + } + if !isNameString(t.Target) { + return fmt.Errorf("xml: EncodeToken of ProcInst with invalid Target") + } + if bytes.Contains(t.Inst, endProcInst) { + return fmt.Errorf("xml: EncodeToken of ProcInst containing ?> marker") + } + p.WriteString(" 0 { + p.WriteByte(' ') + p.Write(t.Inst) + } + p.WriteString("?>") + case Directive: + if !isValidDirective(t) { + return fmt.Errorf("xml: EncodeToken of Directive containing wrong < or > markers") + } + p.WriteString("") + default: + return fmt.Errorf("xml: EncodeToken of invalid token type") + + } + return p.cachedWriteError() +} + +// isValidDirective reports whether dir is a valid directive text, +// meaning angle brackets are matched, ignoring comments and strings. +func isValidDirective(dir Directive) bool { + var ( + depth int + inquote uint8 + incomment bool + ) + for i, c := range dir { + switch { + case incomment: + if c == '>' { + if n := 1 + i - len(endComment); n >= 0 && bytes.Equal(dir[n:i+1], endComment) { + incomment = false + } + } + // Just ignore anything in comment + case inquote != 0: + if c == inquote { + inquote = 0 + } + // Just ignore anything within quotes + case c == '\'' || c == '"': + inquote = c + case c == '<': + if i+len(begComment) < len(dir) && bytes.Equal(dir[i:i+len(begComment)], begComment) { + incomment = true + } else { + depth++ + } + case c == '>': + if depth == 0 { + return false + } + depth-- + } + } + return depth == 0 && inquote == 0 && !incomment +} + +// Flush flushes any buffered XML to the underlying writer. +// See the EncodeToken documentation for details about when it is necessary. +func (enc *Encoder) Flush() error { + return enc.p.Flush() +} + +type printer struct { + *bufio.Writer + encoder *Encoder + seq int + indent string + prefix string + depth int + indentedIn bool + putNewline bool + defaultNS string + attrNS map[string]string // map prefix -> name space + attrPrefix map[string]string // map name space -> prefix + prefixes []printerPrefix + tags []Name +} + +// printerPrefix holds a namespace undo record. +// When an element is popped, the prefix record +// is set back to the recorded URL. The empty +// prefix records the URL for the default name space. +// +// The start of an element is recorded with an element +// that has mark=true. +type printerPrefix struct { + prefix string + url string + mark bool +} + +func (p *printer) prefixForNS(url string, isAttr bool) string { + // The "http://www.w3.org/XML/1998/namespace" name space is predefined as "xml" + // and must be referred to that way. + // (The "http://www.w3.org/2000/xmlns/" name space is also predefined as "xmlns", + // but users should not be trying to use that one directly - that's our job.) + if url == xmlURL { + return "xml" + } + if !isAttr && url == p.defaultNS { + // We can use the default name space. + return "" + } + return p.attrPrefix[url] +} + +// defineNS pushes any namespace definition found in the given attribute. +// If ignoreNonEmptyDefault is true, an xmlns="nonempty" +// attribute will be ignored. +func (p *printer) defineNS(attr Attr, ignoreNonEmptyDefault bool) error { + var prefix string + if attr.Name.Local == "xmlns" { + if attr.Name.Space != "" && attr.Name.Space != "xml" && attr.Name.Space != xmlURL { + return fmt.Errorf("xml: cannot redefine xmlns attribute prefix") + } + } else if attr.Name.Space == "xmlns" && attr.Name.Local != "" { + prefix = attr.Name.Local + if attr.Value == "" { + // Technically, an empty XML namespace is allowed for an attribute. + // From http://www.w3.org/TR/xml-names11/#scoping-defaulting: + // + // The attribute value in a namespace declaration for a prefix may be + // empty. This has the effect, within the scope of the declaration, of removing + // any association of the prefix with a namespace name. + // + // However our namespace prefixes here are used only as hints. There's + // no need to respect the removal of a namespace prefix, so we ignore it. + return nil + } + } else { + // Ignore: it's not a namespace definition + return nil + } + if prefix == "" { + if attr.Value == p.defaultNS { + // No need for redefinition. + return nil + } + if attr.Value != "" && ignoreNonEmptyDefault { + // We have an xmlns="..." value but + // it can't define a name space in this context, + // probably because the element has an empty + // name space. In this case, we just ignore + // the name space declaration. + return nil + } + } else if _, ok := p.attrPrefix[attr.Value]; ok { + // There's already a prefix for the given name space, + // so use that. This prevents us from + // having two prefixes for the same name space + // so attrNS and attrPrefix can remain bijective. + return nil + } + p.pushPrefix(prefix, attr.Value) + return nil +} + +// createNSPrefix creates a name space prefix attribute +// to use for the given name space, defining a new prefix +// if necessary. +// If isAttr is true, the prefix is to be created for an attribute +// prefix, which means that the default name space cannot +// be used. +func (p *printer) createNSPrefix(url string, isAttr bool) { + if _, ok := p.attrPrefix[url]; ok { + // We already have a prefix for the given URL. + return + } + switch { + case !isAttr && url == p.defaultNS: + // We can use the default name space. + return + case url == "": + // The only way we can encode names in the empty + // name space is by using the default name space, + // so we must use that. + if p.defaultNS != "" { + // The default namespace is non-empty, so we + // need to set it to empty. + p.pushPrefix("", "") + } + return + case url == xmlURL: + return + } + // TODO If the URL is an existing prefix, we could + // use it as is. That would enable the + // marshaling of elements that had been unmarshaled + // and with a name space prefix that was not found. + // although technically it would be incorrect. + + // Pick a name. We try to use the final element of the path + // but fall back to _. + prefix := strings.TrimRight(url, "/") + if i := strings.LastIndex(prefix, "/"); i >= 0 { + prefix = prefix[i+1:] + } + if prefix == "" || !isName([]byte(prefix)) || strings.Contains(prefix, ":") { + prefix = "_" + } + if strings.HasPrefix(prefix, "xml") { + // xmlanything is reserved. + prefix = "_" + prefix + } + if p.attrNS[prefix] != "" { + // Name is taken. Find a better one. + for p.seq++; ; p.seq++ { + if id := prefix + "_" + strconv.Itoa(p.seq); p.attrNS[id] == "" { + prefix = id + break + } + } + } + + p.pushPrefix(prefix, url) +} + +// writeNamespaces writes xmlns attributes for all the +// namespace prefixes that have been defined in +// the current element. +func (p *printer) writeNamespaces() { + for i := len(p.prefixes) - 1; i >= 0; i-- { + prefix := p.prefixes[i] + if prefix.mark { + return + } + p.WriteString(" ") + if prefix.prefix == "" { + // Default name space. + p.WriteString(`xmlns="`) + } else { + p.WriteString("xmlns:") + p.WriteString(prefix.prefix) + p.WriteString(`="`) + } + EscapeText(p, []byte(p.nsForPrefix(prefix.prefix))) + p.WriteString(`"`) + } +} + +// pushPrefix pushes a new prefix on the prefix stack +// without checking to see if it is already defined. +func (p *printer) pushPrefix(prefix, url string) { + p.prefixes = append(p.prefixes, printerPrefix{ + prefix: prefix, + url: p.nsForPrefix(prefix), + }) + p.setAttrPrefix(prefix, url) +} + +// nsForPrefix returns the name space for the given +// prefix. Note that this is not valid for the +// empty attribute prefix, which always has an empty +// name space. +func (p *printer) nsForPrefix(prefix string) string { + if prefix == "" { + return p.defaultNS + } + return p.attrNS[prefix] +} + +// markPrefix marks the start of an element on the prefix +// stack. +func (p *printer) markPrefix() { + p.prefixes = append(p.prefixes, printerPrefix{ + mark: true, + }) +} + +// popPrefix pops all defined prefixes for the current +// element. +func (p *printer) popPrefix() { + for len(p.prefixes) > 0 { + prefix := p.prefixes[len(p.prefixes)-1] + p.prefixes = p.prefixes[:len(p.prefixes)-1] + if prefix.mark { + break + } + p.setAttrPrefix(prefix.prefix, prefix.url) + } +} + +// setAttrPrefix sets an attribute name space prefix. +// If url is empty, the attribute is removed. +// If prefix is empty, the default name space is set. +func (p *printer) setAttrPrefix(prefix, url string) { + if prefix == "" { + p.defaultNS = url + return + } + if url == "" { + delete(p.attrPrefix, p.attrNS[prefix]) + delete(p.attrNS, prefix) + return + } + if p.attrPrefix == nil { + // Need to define a new name space. + p.attrPrefix = make(map[string]string) + p.attrNS = make(map[string]string) + } + // Remove any old prefix value. This is OK because we maintain a + // strict one-to-one mapping between prefix and URL (see + // defineNS) + delete(p.attrPrefix, p.attrNS[prefix]) + p.attrPrefix[url] = prefix + p.attrNS[prefix] = url +} + +var ( + marshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem() + marshalerAttrType = reflect.TypeOf((*MarshalerAttr)(nil)).Elem() + textMarshalerType = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem() +) + +// marshalValue writes one or more XML elements representing val. +// If val was obtained from a struct field, finfo must have its details. +func (p *printer) marshalValue(val reflect.Value, finfo *fieldInfo, startTemplate *StartElement) error { + if startTemplate != nil && startTemplate.Name.Local == "" { + return fmt.Errorf("xml: EncodeElement of StartElement with missing name") + } + + if !val.IsValid() { + return nil + } + if finfo != nil && finfo.flags&fOmitEmpty != 0 && isEmptyValue(val) { + return nil + } + + // Drill into interfaces and pointers. + // This can turn into an infinite loop given a cyclic chain, + // but it matches the Go 1 behavior. + for val.Kind() == reflect.Interface || val.Kind() == reflect.Ptr { + if val.IsNil() { + return nil + } + val = val.Elem() + } + + kind := val.Kind() + typ := val.Type() + + // Check for marshaler. + if val.CanInterface() && typ.Implements(marshalerType) { + return p.marshalInterface(val.Interface().(Marshaler), p.defaultStart(typ, finfo, startTemplate)) + } + if val.CanAddr() { + pv := val.Addr() + if pv.CanInterface() && pv.Type().Implements(marshalerType) { + return p.marshalInterface(pv.Interface().(Marshaler), p.defaultStart(pv.Type(), finfo, startTemplate)) + } + } + + // Check for text marshaler. + if val.CanInterface() && typ.Implements(textMarshalerType) { + return p.marshalTextInterface(val.Interface().(encoding.TextMarshaler), p.defaultStart(typ, finfo, startTemplate)) + } + if val.CanAddr() { + pv := val.Addr() + if pv.CanInterface() && pv.Type().Implements(textMarshalerType) { + return p.marshalTextInterface(pv.Interface().(encoding.TextMarshaler), p.defaultStart(pv.Type(), finfo, startTemplate)) + } + } + + // Slices and arrays iterate over the elements. They do not have an enclosing tag. + if (kind == reflect.Slice || kind == reflect.Array) && typ.Elem().Kind() != reflect.Uint8 { + for i, n := 0, val.Len(); i < n; i++ { + if err := p.marshalValue(val.Index(i), finfo, startTemplate); err != nil { + return err + } + } + return nil + } + + tinfo, err := getTypeInfo(typ) + if err != nil { + return err + } + + // Create start element. + // Precedence for the XML element name is: + // 0. startTemplate + // 1. XMLName field in underlying struct; + // 2. field name/tag in the struct field; and + // 3. type name + var start StartElement + + // explicitNS records whether the element's name space has been + // explicitly set (for example an XMLName field). + explicitNS := false + + if startTemplate != nil { + start.Name = startTemplate.Name + explicitNS = true + start.Attr = append(start.Attr, startTemplate.Attr...) + } else if tinfo.xmlname != nil { + xmlname := tinfo.xmlname + if xmlname.name != "" { + start.Name.Space, start.Name.Local = xmlname.xmlns, xmlname.name + } else if v, ok := xmlname.value(val).Interface().(Name); ok && v.Local != "" { + start.Name = v + } + explicitNS = true + } + if start.Name.Local == "" && finfo != nil { + start.Name.Local = finfo.name + if finfo.xmlns != "" { + start.Name.Space = finfo.xmlns + explicitNS = true + } + } + if start.Name.Local == "" { + name := typ.Name() + if name == "" { + return &UnsupportedTypeError{typ} + } + start.Name.Local = name + } + + // defaultNS records the default name space as set by a xmlns="..." + // attribute. We don't set p.defaultNS because we want to let + // the attribute writing code (in p.defineNS) be solely responsible + // for maintaining that. + defaultNS := p.defaultNS + + // Attributes + for i := range tinfo.fields { + finfo := &tinfo.fields[i] + if finfo.flags&fAttr == 0 { + continue + } + attr, err := p.fieldAttr(finfo, val) + if err != nil { + return err + } + if attr.Name.Local == "" { + continue + } + start.Attr = append(start.Attr, attr) + if attr.Name.Space == "" && attr.Name.Local == "xmlns" { + defaultNS = attr.Value + } + } + if !explicitNS { + // Historic behavior: elements use the default name space + // they are contained in by default. + start.Name.Space = defaultNS + } + // Historic behaviour: an element that's in a namespace sets + // the default namespace for all elements contained within it. + start.setDefaultNamespace() + + if err := p.writeStart(&start); err != nil { + return err + } + + if val.Kind() == reflect.Struct { + err = p.marshalStruct(tinfo, val) + } else { + s, b, err1 := p.marshalSimple(typ, val) + if err1 != nil { + err = err1 + } else if b != nil { + EscapeText(p, b) + } else { + p.EscapeString(s) + } + } + if err != nil { + return err + } + + if err := p.writeEnd(start.Name); err != nil { + return err + } + + return p.cachedWriteError() +} + +// fieldAttr returns the attribute of the given field. +// If the returned attribute has an empty Name.Local, +// it should not be used. +// The given value holds the value containing the field. +func (p *printer) fieldAttr(finfo *fieldInfo, val reflect.Value) (Attr, error) { + fv := finfo.value(val) + name := Name{Space: finfo.xmlns, Local: finfo.name} + if finfo.flags&fOmitEmpty != 0 && isEmptyValue(fv) { + return Attr{}, nil + } + if fv.Kind() == reflect.Interface && fv.IsNil() { + return Attr{}, nil + } + if fv.CanInterface() && fv.Type().Implements(marshalerAttrType) { + attr, err := fv.Interface().(MarshalerAttr).MarshalXMLAttr(name) + return attr, err + } + if fv.CanAddr() { + pv := fv.Addr() + if pv.CanInterface() && pv.Type().Implements(marshalerAttrType) { + attr, err := pv.Interface().(MarshalerAttr).MarshalXMLAttr(name) + return attr, err + } + } + if fv.CanInterface() && fv.Type().Implements(textMarshalerType) { + text, err := fv.Interface().(encoding.TextMarshaler).MarshalText() + if err != nil { + return Attr{}, err + } + return Attr{name, string(text)}, nil + } + if fv.CanAddr() { + pv := fv.Addr() + if pv.CanInterface() && pv.Type().Implements(textMarshalerType) { + text, err := pv.Interface().(encoding.TextMarshaler).MarshalText() + if err != nil { + return Attr{}, err + } + return Attr{name, string(text)}, nil + } + } + // Dereference or skip nil pointer, interface values. + switch fv.Kind() { + case reflect.Ptr, reflect.Interface: + if fv.IsNil() { + return Attr{}, nil + } + fv = fv.Elem() + } + s, b, err := p.marshalSimple(fv.Type(), fv) + if err != nil { + return Attr{}, err + } + if b != nil { + s = string(b) + } + return Attr{name, s}, nil +} + +// defaultStart returns the default start element to use, +// given the reflect type, field info, and start template. +func (p *printer) defaultStart(typ reflect.Type, finfo *fieldInfo, startTemplate *StartElement) StartElement { + var start StartElement + // Precedence for the XML element name is as above, + // except that we do not look inside structs for the first field. + if startTemplate != nil { + start.Name = startTemplate.Name + start.Attr = append(start.Attr, startTemplate.Attr...) + } else if finfo != nil && finfo.name != "" { + start.Name.Local = finfo.name + start.Name.Space = finfo.xmlns + } else if typ.Name() != "" { + start.Name.Local = typ.Name() + } else { + // Must be a pointer to a named type, + // since it has the Marshaler methods. + start.Name.Local = typ.Elem().Name() + } + // Historic behaviour: elements use the name space of + // the element they are contained in by default. + if start.Name.Space == "" { + start.Name.Space = p.defaultNS + } + start.setDefaultNamespace() + return start +} + +// marshalInterface marshals a Marshaler interface value. +func (p *printer) marshalInterface(val Marshaler, start StartElement) error { + // Push a marker onto the tag stack so that MarshalXML + // cannot close the XML tags that it did not open. + p.tags = append(p.tags, Name{}) + n := len(p.tags) + + err := val.MarshalXML(p.encoder, start) + if err != nil { + return err + } + + // Make sure MarshalXML closed all its tags. p.tags[n-1] is the mark. + if len(p.tags) > n { + return fmt.Errorf("xml: %s.MarshalXML wrote invalid XML: <%s> not closed", receiverType(val), p.tags[len(p.tags)-1].Local) + } + p.tags = p.tags[:n-1] + return nil +} + +// marshalTextInterface marshals a TextMarshaler interface value. +func (p *printer) marshalTextInterface(val encoding.TextMarshaler, start StartElement) error { + if err := p.writeStart(&start); err != nil { + return err + } + text, err := val.MarshalText() + if err != nil { + return err + } + EscapeText(p, text) + return p.writeEnd(start.Name) +} + +// writeStart writes the given start element. +func (p *printer) writeStart(start *StartElement) error { + if start.Name.Local == "" { + return fmt.Errorf("xml: start tag with no name") + } + + p.tags = append(p.tags, start.Name) + p.markPrefix() + // Define any name spaces explicitly declared in the attributes. + // We do this as a separate pass so that explicitly declared prefixes + // will take precedence over implicitly declared prefixes + // regardless of the order of the attributes. + ignoreNonEmptyDefault := start.Name.Space == "" + for _, attr := range start.Attr { + if err := p.defineNS(attr, ignoreNonEmptyDefault); err != nil { + return err + } + } + // Define any new name spaces implied by the attributes. + for _, attr := range start.Attr { + name := attr.Name + // From http://www.w3.org/TR/xml-names11/#defaulting + // "Default namespace declarations do not apply directly + // to attribute names; the interpretation of unprefixed + // attributes is determined by the element on which they + // appear." + // This means we don't need to create a new namespace + // when an attribute name space is empty. + if name.Space != "" && !name.isNamespace() { + p.createNSPrefix(name.Space, true) + } + } + p.createNSPrefix(start.Name.Space, false) + + p.writeIndent(1) + p.WriteByte('<') + p.writeName(start.Name, false) + p.writeNamespaces() + for _, attr := range start.Attr { + name := attr.Name + if name.Local == "" || name.isNamespace() { + // Namespaces have already been written by writeNamespaces above. + continue + } + p.WriteByte(' ') + p.writeName(name, true) + p.WriteString(`="`) + p.EscapeString(attr.Value) + p.WriteByte('"') + } + p.WriteByte('>') + return nil +} + +// writeName writes the given name. It assumes +// that p.createNSPrefix(name) has already been called. +func (p *printer) writeName(name Name, isAttr bool) { + if prefix := p.prefixForNS(name.Space, isAttr); prefix != "" { + p.WriteString(prefix) + p.WriteByte(':') + } + p.WriteString(name.Local) +} + +func (p *printer) writeEnd(name Name) error { + if name.Local == "" { + return fmt.Errorf("xml: end tag with no name") + } + if len(p.tags) == 0 || p.tags[len(p.tags)-1].Local == "" { + return fmt.Errorf("xml: end tag without start tag", name.Local) + } + if top := p.tags[len(p.tags)-1]; top != name { + if top.Local != name.Local { + return fmt.Errorf("xml: end tag does not match start tag <%s>", name.Local, top.Local) + } + return fmt.Errorf("xml: end tag in namespace %s does not match start tag <%s> in namespace %s", name.Local, name.Space, top.Local, top.Space) + } + p.tags = p.tags[:len(p.tags)-1] + + p.writeIndent(-1) + p.WriteByte('<') + p.WriteByte('/') + p.writeName(name, false) + p.WriteByte('>') + p.popPrefix() + return nil +} + +func (p *printer) marshalSimple(typ reflect.Type, val reflect.Value) (string, []byte, error) { + switch val.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return strconv.FormatInt(val.Int(), 10), nil, nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return strconv.FormatUint(val.Uint(), 10), nil, nil + case reflect.Float32, reflect.Float64: + return strconv.FormatFloat(val.Float(), 'g', -1, val.Type().Bits()), nil, nil + case reflect.String: + return val.String(), nil, nil + case reflect.Bool: + return strconv.FormatBool(val.Bool()), nil, nil + case reflect.Array: + if typ.Elem().Kind() != reflect.Uint8 { + break + } + // [...]byte + var bytes []byte + if val.CanAddr() { + bytes = val.Slice(0, val.Len()).Bytes() + } else { + bytes = make([]byte, val.Len()) + reflect.Copy(reflect.ValueOf(bytes), val) + } + return "", bytes, nil + case reflect.Slice: + if typ.Elem().Kind() != reflect.Uint8 { + break + } + // []byte + return "", val.Bytes(), nil + } + return "", nil, &UnsupportedTypeError{typ} +} + +var ddBytes = []byte("--") + +func (p *printer) marshalStruct(tinfo *typeInfo, val reflect.Value) error { + s := parentStack{p: p} + for i := range tinfo.fields { + finfo := &tinfo.fields[i] + if finfo.flags&fAttr != 0 { + continue + } + vf := finfo.value(val) + + // Dereference or skip nil pointer, interface values. + switch vf.Kind() { + case reflect.Ptr, reflect.Interface: + if !vf.IsNil() { + vf = vf.Elem() + } + } + + switch finfo.flags & fMode { + case fCharData: + if err := s.setParents(&noField, reflect.Value{}); err != nil { + return err + } + if vf.CanInterface() && vf.Type().Implements(textMarshalerType) { + data, err := vf.Interface().(encoding.TextMarshaler).MarshalText() + if err != nil { + return err + } + Escape(p, data) + continue + } + if vf.CanAddr() { + pv := vf.Addr() + if pv.CanInterface() && pv.Type().Implements(textMarshalerType) { + data, err := pv.Interface().(encoding.TextMarshaler).MarshalText() + if err != nil { + return err + } + Escape(p, data) + continue + } + } + var scratch [64]byte + switch vf.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + Escape(p, strconv.AppendInt(scratch[:0], vf.Int(), 10)) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + Escape(p, strconv.AppendUint(scratch[:0], vf.Uint(), 10)) + case reflect.Float32, reflect.Float64: + Escape(p, strconv.AppendFloat(scratch[:0], vf.Float(), 'g', -1, vf.Type().Bits())) + case reflect.Bool: + Escape(p, strconv.AppendBool(scratch[:0], vf.Bool())) + case reflect.String: + if err := EscapeText(p, []byte(vf.String())); err != nil { + return err + } + case reflect.Slice: + if elem, ok := vf.Interface().([]byte); ok { + if err := EscapeText(p, elem); err != nil { + return err + } + } + } + continue + + case fComment: + if err := s.setParents(&noField, reflect.Value{}); err != nil { + return err + } + k := vf.Kind() + if !(k == reflect.String || k == reflect.Slice && vf.Type().Elem().Kind() == reflect.Uint8) { + return fmt.Errorf("xml: bad type for comment field of %s", val.Type()) + } + if vf.Len() == 0 { + continue + } + p.writeIndent(0) + p.WriteString("" is invalid grammar. Make it "- -->" + p.WriteByte(' ') + } + p.WriteString("-->") + continue + + case fInnerXml: + iface := vf.Interface() + switch raw := iface.(type) { + case []byte: + p.Write(raw) + continue + case string: + p.WriteString(raw) + continue + } + + case fElement, fElement | fAny: + if err := s.setParents(finfo, vf); err != nil { + return err + } + } + if err := p.marshalValue(vf, finfo, nil); err != nil { + return err + } + } + if err := s.setParents(&noField, reflect.Value{}); err != nil { + return err + } + return p.cachedWriteError() +} + +var noField fieldInfo + +// return the bufio Writer's cached write error +func (p *printer) cachedWriteError() error { + _, err := p.Write(nil) + return err +} + +func (p *printer) writeIndent(depthDelta int) { + if len(p.prefix) == 0 && len(p.indent) == 0 { + return + } + if depthDelta < 0 { + p.depth-- + if p.indentedIn { + p.indentedIn = false + return + } + p.indentedIn = false + } + if p.putNewline { + p.WriteByte('\n') + } else { + p.putNewline = true + } + if len(p.prefix) > 0 { + p.WriteString(p.prefix) + } + if len(p.indent) > 0 { + for i := 0; i < p.depth; i++ { + p.WriteString(p.indent) + } + } + if depthDelta > 0 { + p.depth++ + p.indentedIn = true + } +} + +type parentStack struct { + p *printer + xmlns string + parents []string +} + +// setParents sets the stack of current parents to those found in finfo. +// It only writes the start elements if vf holds a non-nil value. +// If finfo is &noField, it pops all elements. +func (s *parentStack) setParents(finfo *fieldInfo, vf reflect.Value) error { + xmlns := s.p.defaultNS + if finfo.xmlns != "" { + xmlns = finfo.xmlns + } + commonParents := 0 + if xmlns == s.xmlns { + for ; commonParents < len(finfo.parents) && commonParents < len(s.parents); commonParents++ { + if finfo.parents[commonParents] != s.parents[commonParents] { + break + } + } + } + // Pop off any parents that aren't in common with the previous field. + for i := len(s.parents) - 1; i >= commonParents; i-- { + if err := s.p.writeEnd(Name{ + Space: s.xmlns, + Local: s.parents[i], + }); err != nil { + return err + } + } + s.parents = finfo.parents + s.xmlns = xmlns + if commonParents >= len(s.parents) { + // No new elements to push. + return nil + } + if (vf.Kind() == reflect.Ptr || vf.Kind() == reflect.Interface) && vf.IsNil() { + // The element is nil, so no need for the start elements. + s.parents = s.parents[:commonParents] + return nil + } + // Push any new parents required. + for _, name := range s.parents[commonParents:] { + start := &StartElement{ + Name: Name{ + Space: s.xmlns, + Local: name, + }, + } + // Set the default name space for parent elements + // to match what we do with other elements. + if s.xmlns != s.p.defaultNS { + start.setDefaultNamespace() + } + if err := s.p.writeStart(start); err != nil { + return err + } + } + return nil +} + +// A MarshalXMLError is returned when Marshal encounters a type +// that cannot be converted into XML. +type UnsupportedTypeError struct { + Type reflect.Type +} + +func (e *UnsupportedTypeError) Error() string { + return "xml: unsupported type: " + e.Type.String() +} + +func isEmptyValue(v reflect.Value) bool { + switch v.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + } + return false +} diff --git a/endpoints/drive/webdav/internal/xml/marshal_test.go b/endpoints/drive/webdav/internal/xml/marshal_test.go new file mode 100644 index 000000000..226cfd013 --- /dev/null +++ b/endpoints/drive/webdav/internal/xml/marshal_test.go @@ -0,0 +1,1939 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xml + +import ( + "bytes" + "errors" + "fmt" + "io" + "reflect" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +type DriveType int + +const ( + HyperDrive DriveType = iota + ImprobabilityDrive +) + +type Passenger struct { + Name []string `xml:"name"` + Weight float32 `xml:"weight"` +} + +type Ship struct { + XMLName struct{} `xml:"spaceship"` + + Name string `xml:"name,attr"` + Pilot string `xml:"pilot,attr"` + Drive DriveType `xml:"drive"` + Age uint `xml:"age"` + Passenger []*Passenger `xml:"passenger"` + secret string +} + +type NamedType string + +type Port struct { + XMLName struct{} `xml:"port"` + Type string `xml:"type,attr,omitempty"` + Comment string `xml:",comment"` + Number string `xml:",chardata"` +} + +type Domain struct { + XMLName struct{} `xml:"domain"` + Country string `xml:",attr,omitempty"` + Name []byte `xml:",chardata"` + Comment []byte `xml:",comment"` +} + +type Book struct { + XMLName struct{} `xml:"book"` + Title string `xml:",chardata"` +} + +type Event struct { + XMLName struct{} `xml:"event"` + Year int `xml:",chardata"` +} + +type Movie struct { + XMLName struct{} `xml:"movie"` + Length uint `xml:",chardata"` +} + +type Pi struct { + XMLName struct{} `xml:"pi"` + Approximation float32 `xml:",chardata"` +} + +type Universe struct { + XMLName struct{} `xml:"universe"` + Visible float64 `xml:",chardata"` +} + +type Particle struct { + XMLName struct{} `xml:"particle"` + HasMass bool `xml:",chardata"` +} + +type Departure struct { + XMLName struct{} `xml:"departure"` + When time.Time `xml:",chardata"` +} + +type SecretAgent struct { + XMLName struct{} `xml:"agent"` + Handle string `xml:"handle,attr"` + Identity string + Obfuscate string `xml:",innerxml"` +} + +type NestedItems struct { + XMLName struct{} `xml:"result"` + Items []string `xml:">item"` + Item1 []string `xml:"Items>item1"` +} + +type NestedOrder struct { + XMLName struct{} `xml:"result"` + Field1 string `xml:"parent>c"` + Field2 string `xml:"parent>b"` + Field3 string `xml:"parent>a"` +} + +type MixedNested struct { + XMLName struct{} `xml:"result"` + A string `xml:"parent1>a"` + B string `xml:"b"` + C string `xml:"parent1>parent2>c"` + D string `xml:"parent1>d"` +} + +type NilTest struct { + A interface{} `xml:"parent1>parent2>a"` + B interface{} `xml:"parent1>b"` + C interface{} `xml:"parent1>parent2>c"` +} + +type Service struct { + XMLName struct{} `xml:"service"` + Domain *Domain `xml:"host>domain"` + Port *Port `xml:"host>port"` + Extra1 interface{} + Extra2 interface{} `xml:"host>extra2"` +} + +var nilStruct *Ship + +type EmbedA struct { + EmbedC + EmbedB EmbedB + FieldA string +} + +type EmbedB struct { + FieldB string + *EmbedC +} + +type EmbedC struct { + FieldA1 string `xml:"FieldA>A1"` + FieldA2 string `xml:"FieldA>A2"` + FieldB string + FieldC string +} + +type NameCasing struct { + XMLName struct{} `xml:"casing"` + Xy string + XY string + XyA string `xml:"Xy,attr"` + XYA string `xml:"XY,attr"` +} + +type NamePrecedence struct { + XMLName Name `xml:"Parent"` + FromTag XMLNameWithoutTag `xml:"InTag"` + FromNameVal XMLNameWithoutTag + FromNameTag XMLNameWithTag + InFieldName string +} + +type XMLNameWithTag struct { + XMLName Name `xml:"InXMLNameTag"` + Value string `xml:",chardata"` +} + +type XMLNameWithNSTag struct { + XMLName Name `xml:"ns InXMLNameWithNSTag"` + Value string `xml:",chardata"` +} + +type XMLNameWithoutTag struct { + XMLName Name + Value string `xml:",chardata"` +} + +type NameInField struct { + Foo Name `xml:"ns foo"` +} + +type AttrTest struct { + Int int `xml:",attr"` + Named int `xml:"int,attr"` + Float float64 `xml:",attr"` + Uint8 uint8 `xml:",attr"` + Bool bool `xml:",attr"` + Str string `xml:",attr"` + Bytes []byte `xml:",attr"` +} + +type OmitAttrTest struct { + Int int `xml:",attr,omitempty"` + Named int `xml:"int,attr,omitempty"` + Float float64 `xml:",attr,omitempty"` + Uint8 uint8 `xml:",attr,omitempty"` + Bool bool `xml:",attr,omitempty"` + Str string `xml:",attr,omitempty"` + Bytes []byte `xml:",attr,omitempty"` +} + +type OmitFieldTest struct { + Int int `xml:",omitempty"` + Named int `xml:"int,omitempty"` + Float float64 `xml:",omitempty"` + Uint8 uint8 `xml:",omitempty"` + Bool bool `xml:",omitempty"` + Str string `xml:",omitempty"` + Bytes []byte `xml:",omitempty"` + Ptr *PresenceTest `xml:",omitempty"` +} + +type AnyTest struct { + XMLName struct{} `xml:"a"` + Nested string `xml:"nested>value"` + AnyField AnyHolder `xml:",any"` +} + +type AnyOmitTest struct { + XMLName struct{} `xml:"a"` + Nested string `xml:"nested>value"` + AnyField *AnyHolder `xml:",any,omitempty"` +} + +type AnySliceTest struct { + XMLName struct{} `xml:"a"` + Nested string `xml:"nested>value"` + AnyField []AnyHolder `xml:",any"` +} + +type AnyHolder struct { + XMLName Name + XML string `xml:",innerxml"` +} + +type RecurseA struct { + A string + B *RecurseB +} + +type RecurseB struct { + A *RecurseA + B string +} + +type PresenceTest struct { + Exists *struct{} +} + +type IgnoreTest struct { + PublicSecret string `xml:"-"` +} + +type MyBytes []byte + +type Data struct { + Bytes []byte + Attr []byte `xml:",attr"` + Custom MyBytes +} + +type Plain struct { + V interface{} +} + +type MyInt int + +type EmbedInt struct { + MyInt +} + +type Strings struct { + X []string `xml:"A>B,omitempty"` +} + +type PointerFieldsTest struct { + XMLName Name `xml:"dummy"` + Name *string `xml:"name,attr"` + Age *uint `xml:"age,attr"` + Empty *string `xml:"empty,attr"` + Contents *string `xml:",chardata"` +} + +type ChardataEmptyTest struct { + XMLName Name `xml:"test"` + Contents *string `xml:",chardata"` +} + +type MyMarshalerTest struct { +} + +var _ Marshaler = (*MyMarshalerTest)(nil) + +func (m *MyMarshalerTest) MarshalXML(e *Encoder, start StartElement) error { + e.EncodeToken(start) + e.EncodeToken(CharData([]byte("hello world"))) + e.EncodeToken(EndElement{start.Name}) + return nil +} + +type MyMarshalerAttrTest struct{} + +var _ MarshalerAttr = (*MyMarshalerAttrTest)(nil) + +func (m *MyMarshalerAttrTest) MarshalXMLAttr(name Name) (Attr, error) { + return Attr{name, "hello world"}, nil +} + +type MyMarshalerValueAttrTest struct{} + +var _ MarshalerAttr = MyMarshalerValueAttrTest{} + +func (m MyMarshalerValueAttrTest) MarshalXMLAttr(name Name) (Attr, error) { + return Attr{name, "hello world"}, nil +} + +type MarshalerStruct struct { + Foo MyMarshalerAttrTest `xml:",attr"` +} + +type MarshalerValueStruct struct { + Foo MyMarshalerValueAttrTest `xml:",attr"` +} + +type InnerStruct struct { + XMLName Name `xml:"testns outer"` +} + +type OuterStruct struct { + InnerStruct + IntAttr int `xml:"int,attr"` +} + +type OuterNamedStruct struct { + InnerStruct + XMLName Name `xml:"outerns test"` + IntAttr int `xml:"int,attr"` +} + +type OuterNamedOrderedStruct struct { + XMLName Name `xml:"outerns test"` + InnerStruct + IntAttr int `xml:"int,attr"` +} + +type OuterOuterStruct struct { + OuterStruct +} + +type NestedAndChardata struct { + AB []string `xml:"A>B"` + Chardata string `xml:",chardata"` +} + +type NestedAndComment struct { + AB []string `xml:"A>B"` + Comment string `xml:",comment"` +} + +type XMLNSFieldStruct struct { + Ns string `xml:"xmlns,attr"` + Body string +} + +type NamedXMLNSFieldStruct struct { + XMLName struct{} `xml:"testns test"` + Ns string `xml:"xmlns,attr"` + Body string +} + +type XMLNSFieldStructWithOmitEmpty struct { + Ns string `xml:"xmlns,attr,omitempty"` + Body string +} + +type NamedXMLNSFieldStructWithEmptyNamespace struct { + XMLName struct{} `xml:"test"` + Ns string `xml:"xmlns,attr"` + Body string +} + +type RecursiveXMLNSFieldStruct struct { + Ns string `xml:"xmlns,attr"` + Body *RecursiveXMLNSFieldStruct `xml:",omitempty"` + Text string `xml:",omitempty"` +} + +func ifaceptr(x interface{}) interface{} { + return &x +} + +var ( + nameAttr = "Sarah" + ageAttr = uint(12) + contentsAttr = "lorem ipsum" +) + +// Unless explicitly stated as such (or *Plain), all of the +// tests below are two-way tests. When introducing new tests, +// please try to make them two-way as well to ensure that +// marshalling and unmarshalling are as symmetrical as feasible. +var marshalTests = []struct { + Value interface{} + ExpectXML string + MarshalOnly bool + UnmarshalOnly bool +}{ + // Test nil marshals to nothing + {Value: nil, ExpectXML: ``, MarshalOnly: true}, + {Value: nilStruct, ExpectXML: ``, MarshalOnly: true}, + + // Test value types + {Value: &Plain{true}, ExpectXML: `true`}, + {Value: &Plain{false}, ExpectXML: `false`}, + {Value: &Plain{int(42)}, ExpectXML: `42`}, + {Value: &Plain{int8(42)}, ExpectXML: `42`}, + {Value: &Plain{int16(42)}, ExpectXML: `42`}, + {Value: &Plain{int32(42)}, ExpectXML: `42`}, + {Value: &Plain{uint(42)}, ExpectXML: `42`}, + {Value: &Plain{uint8(42)}, ExpectXML: `42`}, + {Value: &Plain{uint16(42)}, ExpectXML: `42`}, + {Value: &Plain{uint32(42)}, ExpectXML: `42`}, + {Value: &Plain{float32(1.25)}, ExpectXML: `1.25`}, + {Value: &Plain{float64(1.25)}, ExpectXML: `1.25`}, + {Value: &Plain{uintptr(0xFFDD)}, ExpectXML: `65501`}, + {Value: &Plain{"gopher"}, ExpectXML: `gopher`}, + {Value: &Plain{[]byte("gopher")}, ExpectXML: `gopher`}, + {Value: &Plain{""}, ExpectXML: `</>`}, + {Value: &Plain{[]byte("")}, ExpectXML: `</>`}, + {Value: &Plain{[3]byte{'<', '/', '>'}}, ExpectXML: `</>`}, + {Value: &Plain{NamedType("potato")}, ExpectXML: `potato`}, + {Value: &Plain{[]int{1, 2, 3}}, ExpectXML: `123`}, + {Value: &Plain{[3]int{1, 2, 3}}, ExpectXML: `123`}, + {Value: ifaceptr(true), MarshalOnly: true, ExpectXML: `true`}, + + // Test time. + { + Value: &Plain{time.Unix(1e9, 123456789).UTC()}, + ExpectXML: `2001-09-09T01:46:40.123456789Z`, + }, + + // A pointer to struct{} may be used to test for an element's presence. + { + Value: &PresenceTest{new(struct{})}, + ExpectXML: ``, + }, + { + Value: &PresenceTest{}, + ExpectXML: ``, + }, + + // A pointer to struct{} may be used to test for an element's presence. + { + Value: &PresenceTest{new(struct{})}, + ExpectXML: ``, + }, + { + Value: &PresenceTest{}, + ExpectXML: ``, + }, + + // A []byte field is only nil if the element was not found. + { + Value: &Data{}, + ExpectXML: ``, + UnmarshalOnly: true, + }, + { + Value: &Data{Bytes: []byte{}, Custom: MyBytes{}, Attr: []byte{}}, + ExpectXML: ``, + UnmarshalOnly: true, + }, + + // Check that []byte works, including named []byte types. + { + Value: &Data{Bytes: []byte("ab"), Custom: MyBytes("cd"), Attr: []byte{'v'}}, + ExpectXML: `abcd`, + }, + + // Test innerxml + { + Value: &SecretAgent{ + Handle: "007", + Identity: "James Bond", + Obfuscate: "", + }, + ExpectXML: `James Bond`, + MarshalOnly: true, + }, + { + Value: &SecretAgent{ + Handle: "007", + Identity: "James Bond", + Obfuscate: "James Bond", + }, + ExpectXML: `James Bond`, + UnmarshalOnly: true, + }, + + // Test structs + {Value: &Port{Type: "ssl", Number: "443"}, ExpectXML: `443`}, + {Value: &Port{Number: "443"}, ExpectXML: `443`}, + {Value: &Port{Type: ""}, ExpectXML: ``}, + {Value: &Port{Number: "443", Comment: "https"}, ExpectXML: `443`}, + {Value: &Port{Number: "443", Comment: "add space-"}, ExpectXML: `443`, MarshalOnly: true}, + {Value: &Domain{Name: []byte("google.com&friends")}, ExpectXML: `google.com&friends`}, + {Value: &Domain{Name: []byte("google.com"), Comment: []byte(" &friends ")}, ExpectXML: `google.com`}, + {Value: &Book{Title: "Pride & Prejudice"}, ExpectXML: `Pride & Prejudice`}, + {Value: &Event{Year: -3114}, ExpectXML: `-3114`}, + {Value: &Movie{Length: 13440}, ExpectXML: `13440`}, + {Value: &Pi{Approximation: 3.14159265}, ExpectXML: `3.1415927`}, + {Value: &Universe{Visible: 9.3e13}, ExpectXML: `9.3e+13`}, + {Value: &Particle{HasMass: true}, ExpectXML: `true`}, + {Value: &Departure{When: ParseTime("2013-01-09T00:15:00-09:00")}, ExpectXML: `2013-01-09T00:15:00-09:00`}, + {Value: atomValue, ExpectXML: atomXml}, + { + Value: &Ship{ + Name: "Heart of Gold", + Pilot: "Computer", + Age: 1, + Drive: ImprobabilityDrive, + Passenger: []*Passenger{ + { + Name: []string{"Zaphod", "Beeblebrox"}, + Weight: 7.25, + }, + { + Name: []string{"Trisha", "McMillen"}, + Weight: 5.5, + }, + { + Name: []string{"Ford", "Prefect"}, + Weight: 7, + }, + { + Name: []string{"Arthur", "Dent"}, + Weight: 6.75, + }, + }, + }, + ExpectXML: `` + + `` + strconv.Itoa(int(ImprobabilityDrive)) + `` + + `1` + + `` + + `Zaphod` + + `Beeblebrox` + + `7.25` + + `` + + `` + + `Trisha` + + `McMillen` + + `5.5` + + `` + + `` + + `Ford` + + `Prefect` + + `7` + + `` + + `` + + `Arthur` + + `Dent` + + `6.75` + + `` + + ``, + }, + + // Test a>b + { + Value: &NestedItems{Items: nil, Item1: nil}, + ExpectXML: `` + + `` + + `` + + ``, + }, + { + Value: &NestedItems{Items: []string{}, Item1: []string{}}, + ExpectXML: `` + + `` + + `` + + ``, + MarshalOnly: true, + }, + { + Value: &NestedItems{Items: nil, Item1: []string{"A"}}, + ExpectXML: `` + + `` + + `A` + + `` + + ``, + }, + { + Value: &NestedItems{Items: []string{"A", "B"}, Item1: nil}, + ExpectXML: `` + + `` + + `A` + + `B` + + `` + + ``, + }, + { + Value: &NestedItems{Items: []string{"A", "B"}, Item1: []string{"C"}}, + ExpectXML: `` + + `` + + `A` + + `B` + + `C` + + `` + + ``, + }, + { + Value: &NestedOrder{Field1: "C", Field2: "B", Field3: "A"}, + ExpectXML: `` + + `` + + `C` + + `B` + + `A` + + `` + + ``, + }, + { + Value: &NilTest{A: "A", B: nil, C: "C"}, + ExpectXML: `` + + `` + + `A` + + `C` + + `` + + ``, + MarshalOnly: true, // Uses interface{} + }, + { + Value: &MixedNested{A: "A", B: "B", C: "C", D: "D"}, + ExpectXML: `` + + `A` + + `B` + + `` + + `C` + + `D` + + `` + + ``, + }, + { + Value: &Service{Port: &Port{Number: "80"}}, + ExpectXML: `80`, + }, + { + Value: &Service{}, + ExpectXML: ``, + }, + { + Value: &Service{Port: &Port{Number: "80"}, Extra1: "A", Extra2: "B"}, + ExpectXML: `` + + `80` + + `A` + + `B` + + ``, + MarshalOnly: true, + }, + { + Value: &Service{Port: &Port{Number: "80"}, Extra2: "example"}, + ExpectXML: `` + + `80` + + `example` + + ``, + MarshalOnly: true, + }, + { + Value: &struct { + XMLName struct{} `xml:"space top"` + A string `xml:"x>a"` + B string `xml:"x>b"` + C string `xml:"space x>c"` + C1 string `xml:"space1 x>c"` + D1 string `xml:"space1 x>d"` + E1 string `xml:"x>e"` + }{ + A: "a", + B: "b", + C: "c", + C1: "c1", + D1: "d1", + E1: "e1", + }, + ExpectXML: `` + + `abc` + + `` + + `c1` + + `d1` + + `` + + `` + + `e1` + + `` + + ``, + }, + { + Value: &struct { + XMLName Name + A string `xml:"x>a"` + B string `xml:"x>b"` + C string `xml:"space x>c"` + C1 string `xml:"space1 x>c"` + D1 string `xml:"space1 x>d"` + }{ + XMLName: Name{ + Space: "space0", + Local: "top", + }, + A: "a", + B: "b", + C: "c", + C1: "c1", + D1: "d1", + }, + ExpectXML: `` + + `ab` + + `c` + + `` + + `c1` + + `d1` + + `` + + ``, + }, + { + Value: &struct { + XMLName struct{} `xml:"top"` + B string `xml:"space x>b"` + B1 string `xml:"space1 x>b"` + }{ + B: "b", + B1: "b1", + }, + ExpectXML: `` + + `b` + + `b1` + + ``, + }, + + // Test struct embedding + { + Value: &EmbedA{ + EmbedC: EmbedC{ + FieldA1: "", // Shadowed by A.A + FieldA2: "", // Shadowed by A.A + FieldB: "A.C.B", + FieldC: "A.C.C", + }, + EmbedB: EmbedB{ + FieldB: "A.B.B", + EmbedC: &EmbedC{ + FieldA1: "A.B.C.A1", + FieldA2: "A.B.C.A2", + FieldB: "", // Shadowed by A.B.B + FieldC: "A.B.C.C", + }, + }, + FieldA: "A.A", + }, + ExpectXML: `` + + `A.C.B` + + `A.C.C` + + `` + + `A.B.B` + + `` + + `A.B.C.A1` + + `A.B.C.A2` + + `` + + `A.B.C.C` + + `` + + `A.A` + + ``, + }, + + // Test that name casing matters + { + Value: &NameCasing{Xy: "mixed", XY: "upper", XyA: "mixedA", XYA: "upperA"}, + ExpectXML: `mixedupper`, + }, + + // Test the order in which the XML element name is chosen + { + Value: &NamePrecedence{ + FromTag: XMLNameWithoutTag{Value: "A"}, + FromNameVal: XMLNameWithoutTag{XMLName: Name{Local: "InXMLName"}, Value: "B"}, + FromNameTag: XMLNameWithTag{Value: "C"}, + InFieldName: "D", + }, + ExpectXML: `` + + `A` + + `B` + + `C` + + `D` + + ``, + MarshalOnly: true, + }, + { + Value: &NamePrecedence{ + XMLName: Name{Local: "Parent"}, + FromTag: XMLNameWithoutTag{XMLName: Name{Local: "InTag"}, Value: "A"}, + FromNameVal: XMLNameWithoutTag{XMLName: Name{Local: "FromNameVal"}, Value: "B"}, + FromNameTag: XMLNameWithTag{XMLName: Name{Local: "InXMLNameTag"}, Value: "C"}, + InFieldName: "D", + }, + ExpectXML: `` + + `A` + + `B` + + `C` + + `D` + + ``, + UnmarshalOnly: true, + }, + + // xml.Name works in a plain field as well. + { + Value: &NameInField{Name{Space: "ns", Local: "foo"}}, + ExpectXML: ``, + }, + { + Value: &NameInField{Name{Space: "ns", Local: "foo"}}, + ExpectXML: ``, + UnmarshalOnly: true, + }, + + // Marshaling zero xml.Name uses the tag or field name. + { + Value: &NameInField{}, + ExpectXML: ``, + MarshalOnly: true, + }, + + // Test attributes + { + Value: &AttrTest{ + Int: 8, + Named: 9, + Float: 23.5, + Uint8: 255, + Bool: true, + Str: "str", + Bytes: []byte("byt"), + }, + ExpectXML: ``, + }, + { + Value: &AttrTest{Bytes: []byte{}}, + ExpectXML: ``, + }, + { + Value: &OmitAttrTest{ + Int: 8, + Named: 9, + Float: 23.5, + Uint8: 255, + Bool: true, + Str: "str", + Bytes: []byte("byt"), + }, + ExpectXML: ``, + }, + { + Value: &OmitAttrTest{}, + ExpectXML: ``, + }, + + // pointer fields + { + Value: &PointerFieldsTest{Name: &nameAttr, Age: &ageAttr, Contents: &contentsAttr}, + ExpectXML: `lorem ipsum`, + MarshalOnly: true, + }, + + // empty chardata pointer field + { + Value: &ChardataEmptyTest{}, + ExpectXML: ``, + MarshalOnly: true, + }, + + // omitempty on fields + { + Value: &OmitFieldTest{ + Int: 8, + Named: 9, + Float: 23.5, + Uint8: 255, + Bool: true, + Str: "str", + Bytes: []byte("byt"), + Ptr: &PresenceTest{}, + }, + ExpectXML: `` + + `8` + + `9` + + `23.5` + + `255` + + `true` + + `str` + + `byt` + + `` + + ``, + }, + { + Value: &OmitFieldTest{}, + ExpectXML: ``, + }, + + // Test ",any" + { + ExpectXML: `knownunknown`, + Value: &AnyTest{ + Nested: "known", + AnyField: AnyHolder{ + XMLName: Name{Local: "other"}, + XML: "unknown", + }, + }, + }, + { + Value: &AnyTest{Nested: "known", + AnyField: AnyHolder{ + XML: "", + XMLName: Name{Local: "AnyField"}, + }, + }, + ExpectXML: `known`, + }, + { + ExpectXML: `b`, + Value: &AnyOmitTest{ + Nested: "b", + }, + }, + { + ExpectXML: `bei`, + Value: &AnySliceTest{ + Nested: "b", + AnyField: []AnyHolder{ + { + XMLName: Name{Local: "c"}, + XML: "e", + }, + { + XMLName: Name{Space: "f", Local: "g"}, + XML: "i", + }, + }, + }, + }, + { + ExpectXML: `b`, + Value: &AnySliceTest{ + Nested: "b", + }, + }, + + // Test recursive types. + { + Value: &RecurseA{ + A: "a1", + B: &RecurseB{ + A: &RecurseA{"a2", nil}, + B: "b1", + }, + }, + ExpectXML: `a1a2b1`, + }, + + // Test ignoring fields via "-" tag + { + ExpectXML: ``, + Value: &IgnoreTest{}, + }, + { + ExpectXML: ``, + Value: &IgnoreTest{PublicSecret: "can't tell"}, + MarshalOnly: true, + }, + { + ExpectXML: `ignore me`, + Value: &IgnoreTest{}, + UnmarshalOnly: true, + }, + + // Test escaping. + { + ExpectXML: `dquote: "; squote: '; ampersand: &; less: <; greater: >;`, + Value: &AnyTest{ + Nested: `dquote: "; squote: '; ampersand: &; less: <; greater: >;`, + AnyField: AnyHolder{XMLName: Name{Local: "empty"}}, + }, + }, + { + ExpectXML: `newline: ; cr: ; tab: ;`, + Value: &AnyTest{ + Nested: "newline: \n; cr: \r; tab: \t;", + AnyField: AnyHolder{XMLName: Name{Local: "AnyField"}}, + }, + }, + { + ExpectXML: "1\r2\r\n3\n\r4\n5", + Value: &AnyTest{ + Nested: "1\n2\n3\n\n4\n5", + }, + UnmarshalOnly: true, + }, + { + ExpectXML: `42`, + Value: &EmbedInt{ + MyInt: 42, + }, + }, + // Test omitempty with parent chain; see golang.org/issue/4168. + { + ExpectXML: ``, + Value: &Strings{}, + }, + // Custom marshalers. + { + ExpectXML: `hello world`, + Value: &MyMarshalerTest{}, + }, + { + ExpectXML: ``, + Value: &MarshalerStruct{}, + }, + { + ExpectXML: ``, + Value: &MarshalerValueStruct{}, + }, + { + ExpectXML: ``, + Value: &OuterStruct{IntAttr: 10}, + }, + { + ExpectXML: ``, + Value: &OuterNamedStruct{XMLName: Name{Space: "outerns", Local: "test"}, IntAttr: 10}, + }, + { + ExpectXML: ``, + Value: &OuterNamedOrderedStruct{XMLName: Name{Space: "outerns", Local: "test"}, IntAttr: 10}, + }, + { + ExpectXML: ``, + Value: &OuterOuterStruct{OuterStruct{IntAttr: 10}}, + }, + { + ExpectXML: `test`, + Value: &NestedAndChardata{AB: make([]string, 2), Chardata: "test"}, + }, + { + ExpectXML: ``, + Value: &NestedAndComment{AB: make([]string, 2), Comment: "test"}, + }, + { + ExpectXML: `hello world`, + Value: &XMLNSFieldStruct{Ns: "http://example.com/ns", Body: "hello world"}, + }, + { + ExpectXML: `hello world`, + Value: &NamedXMLNSFieldStruct{Ns: "http://example.com/ns", Body: "hello world"}, + }, + { + ExpectXML: `hello world`, + Value: &NamedXMLNSFieldStruct{Ns: "", Body: "hello world"}, + }, + { + ExpectXML: `hello world`, + Value: &XMLNSFieldStructWithOmitEmpty{Body: "hello world"}, + }, + { + // The xmlns attribute must be ignored because the + // element is in the empty namespace, so it's not possible + // to set the default namespace to something non-empty. + ExpectXML: `hello world`, + Value: &NamedXMLNSFieldStructWithEmptyNamespace{Ns: "foo", Body: "hello world"}, + MarshalOnly: true, + }, + { + ExpectXML: `hello world`, + Value: &RecursiveXMLNSFieldStruct{ + Ns: "foo", + Body: &RecursiveXMLNSFieldStruct{ + Text: "hello world", + }, + }, + }, +} + +func TestMarshal(t *testing.T) { + for idx, test := range marshalTests { + if test.UnmarshalOnly { + continue + } + data, err := Marshal(test.Value) + if err != nil { + t.Errorf("#%d: marshal(%#v): %s", idx, test.Value, err) + continue + } + if got, want := string(data), test.ExpectXML; got != want { + if strings.Contains(want, "\n") { + t.Errorf("#%d: marshal(%#v):\nHAVE:\n%s\nWANT:\n%s", idx, test.Value, got, want) + } else { + t.Errorf("#%d: marshal(%#v):\nhave %#q\nwant %#q", idx, test.Value, got, want) + } + } + } +} + +type AttrParent struct { + X string `xml:"X>Y,attr"` +} + +type BadAttr struct { + Name []string `xml:"name,attr"` +} + +var marshalErrorTests = []struct { + Value interface{} + Err string + Kind reflect.Kind +}{ + { + Value: make(chan bool), + Err: "xml: unsupported type: chan bool", + Kind: reflect.Chan, + }, + { + Value: map[string]string{ + "question": "What do you get when you multiply six by nine?", + "answer": "42", + }, + Err: "xml: unsupported type: map[string]string", + Kind: reflect.Map, + }, + { + Value: map[*Ship]bool{nil: false}, + Err: "xml: unsupported type: map[*xml.Ship]bool", + Kind: reflect.Map, + }, + { + Value: &Domain{Comment: []byte("f--bar")}, + Err: `xml: comments must not contain "--"`, + }, + // Reject parent chain with attr, never worked; see golang.org/issue/5033. + { + Value: &AttrParent{}, + Err: `xml: X>Y chain not valid with attr flag`, + }, + { + Value: BadAttr{[]string{"X", "Y"}}, + Err: `xml: unsupported type: []string`, + }, +} + +var marshalIndentTests = []struct { + Value interface{} + Prefix string + Indent string + ExpectXML string +}{ + { + Value: &SecretAgent{ + Handle: "007", + Identity: "James Bond", + Obfuscate: "", + }, + Prefix: "", + Indent: "\t", + ExpectXML: fmt.Sprintf("\n\tJames Bond\n"), + }, +} + +func TestMarshalErrors(t *testing.T) { + for idx, test := range marshalErrorTests { + data, err := Marshal(test.Value) + if err == nil { + t.Errorf("#%d: marshal(%#v) = [success] %q, want error %v", idx, test.Value, data, test.Err) + continue + } + if err.Error() != test.Err { + t.Errorf("#%d: marshal(%#v) = [error] %v, want %v", idx, test.Value, err, test.Err) + } + if test.Kind != reflect.Invalid { + if kind := err.(*UnsupportedTypeError).Type.Kind(); kind != test.Kind { + t.Errorf("#%d: marshal(%#v) = [error kind] %s, want %s", idx, test.Value, kind, test.Kind) + } + } + } +} + +// Do invertibility testing on the various structures that we test +func TestUnmarshal(t *testing.T) { + for i, test := range marshalTests { + if test.MarshalOnly { + continue + } + if _, ok := test.Value.(*Plain); ok { + continue + } + vt := reflect.TypeOf(test.Value) + dest := reflect.New(vt.Elem()).Interface() + err := Unmarshal([]byte(test.ExpectXML), dest) + + switch fix := dest.(type) { + case *Feed: + fix.Author.InnerXML = "" + for i := range fix.Entry { + fix.Entry[i].Author.InnerXML = "" + } + } + + if err != nil { + t.Errorf("#%d: unexpected error: %#v", i, err) + } else if got, want := dest, test.Value; !reflect.DeepEqual(got, want) { + t.Errorf("#%d: unmarshal(%q):\nhave %#v\nwant %#v", i, test.ExpectXML, got, want) + } + } +} + +func TestMarshalIndent(t *testing.T) { + for i, test := range marshalIndentTests { + data, err := MarshalIndent(test.Value, test.Prefix, test.Indent) + if err != nil { + t.Errorf("#%d: Error: %s", i, err) + continue + } + if got, want := string(data), test.ExpectXML; got != want { + t.Errorf("#%d: MarshalIndent:\nGot:%s\nWant:\n%s", i, got, want) + } + } +} + +type limitedBytesWriter struct { + w io.Writer + remain int // until writes fail +} + +func (lw *limitedBytesWriter) Write(p []byte) (n int, err error) { + if lw.remain <= 0 { + println("error") + return 0, errors.New("write limit hit") + } + if len(p) > lw.remain { + p = p[:lw.remain] + n, _ = lw.w.Write(p) + lw.remain = 0 + return n, errors.New("write limit hit") + } + n, err = lw.w.Write(p) + lw.remain -= n + return n, err +} + +func TestMarshalWriteErrors(t *testing.T) { + var buf bytes.Buffer + const writeCap = 1024 + w := &limitedBytesWriter{&buf, writeCap} + enc := NewEncoder(w) + var err error + var i int + const n = 4000 + for i = 1; i <= n; i++ { + err = enc.Encode(&Passenger{ + Name: []string{"Alice", "Bob"}, + Weight: 5, + }) + if err != nil { + break + } + } + if err == nil { + t.Error("expected an error") + } + if i == n { + t.Errorf("expected to fail before the end") + } + if buf.Len() != writeCap { + t.Errorf("buf.Len() = %d; want %d", buf.Len(), writeCap) + } +} + +func TestMarshalWriteIOErrors(t *testing.T) { + enc := NewEncoder(errWriter{}) + + expectErr := "unwritable" + err := enc.Encode(&Passenger{}) + if err == nil || err.Error() != expectErr { + t.Errorf("EscapeTest = [error] %v, want %v", err, expectErr) + } +} + +func TestMarshalFlush(t *testing.T) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + if err := enc.EncodeToken(CharData("hello world")); err != nil { + t.Fatalf("enc.EncodeToken: %v", err) + } + if buf.Len() > 0 { + t.Fatalf("enc.EncodeToken caused actual write: %q", buf.Bytes()) + } + if err := enc.Flush(); err != nil { + t.Fatalf("enc.Flush: %v", err) + } + if buf.String() != "hello world" { + t.Fatalf("after enc.Flush, buf.String() = %q, want %q", buf.String(), "hello world") + } +} + +var encodeElementTests = []struct { + desc string + value interface{} + start StartElement + expectXML string +}{{ + desc: "simple string", + value: "hello", + start: StartElement{ + Name: Name{Local: "a"}, + }, + expectXML: `hello`, +}, { + desc: "string with added attributes", + value: "hello", + start: StartElement{ + Name: Name{Local: "a"}, + Attr: []Attr{{ + Name: Name{Local: "x"}, + Value: "y", + }, { + Name: Name{Local: "foo"}, + Value: "bar", + }}, + }, + expectXML: `hello`, +}, { + desc: "start element with default name space", + value: struct { + Foo XMLNameWithNSTag + }{ + Foo: XMLNameWithNSTag{ + Value: "hello", + }, + }, + start: StartElement{ + Name: Name{Space: "ns", Local: "a"}, + Attr: []Attr{{ + Name: Name{Local: "xmlns"}, + // "ns" is the name space defined in XMLNameWithNSTag + Value: "ns", + }}, + }, + expectXML: `hello`, +}, { + desc: "start element in name space with different default name space", + value: struct { + Foo XMLNameWithNSTag + }{ + Foo: XMLNameWithNSTag{ + Value: "hello", + }, + }, + start: StartElement{ + Name: Name{Space: "ns2", Local: "a"}, + Attr: []Attr{{ + Name: Name{Local: "xmlns"}, + // "ns" is the name space defined in XMLNameWithNSTag + Value: "ns", + }}, + }, + expectXML: `hello`, +}, { + desc: "XMLMarshaler with start element with default name space", + value: &MyMarshalerTest{}, + start: StartElement{ + Name: Name{Space: "ns2", Local: "a"}, + Attr: []Attr{{ + Name: Name{Local: "xmlns"}, + // "ns" is the name space defined in XMLNameWithNSTag + Value: "ns", + }}, + }, + expectXML: `hello world`, +}} + +func TestEncodeElement(t *testing.T) { + for idx, test := range encodeElementTests { + var buf bytes.Buffer + enc := NewEncoder(&buf) + err := enc.EncodeElement(test.value, test.start) + if err != nil { + t.Fatalf("enc.EncodeElement: %v", err) + } + err = enc.Flush() + if err != nil { + t.Fatalf("enc.Flush: %v", err) + } + if got, want := buf.String(), test.expectXML; got != want { + t.Errorf("#%d(%s): EncodeElement(%#v, %#v):\nhave %#q\nwant %#q", idx, test.desc, test.value, test.start, got, want) + } + } +} + +func BenchmarkMarshal(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + Marshal(atomValue) + } +} + +func BenchmarkUnmarshal(b *testing.B) { + b.ReportAllocs() + xml := []byte(atomXml) + for i := 0; i < b.N; i++ { + Unmarshal(xml, &Feed{}) + } +} + +// golang.org/issue/6556 +func TestStructPointerMarshal(t *testing.T) { + type A struct { + XMLName string `xml:"a"` + B []interface{} + } + type C struct { + XMLName Name + Value string `xml:"value"` + } + + a := new(A) + a.B = append(a.B, &C{ + XMLName: Name{Local: "c"}, + Value: "x", + }) + + b, err := Marshal(a) + if err != nil { + t.Fatal(err) + } + if x := string(b); x != "x" { + t.Fatal(x) + } + var v A + err = Unmarshal(b, &v) + if err != nil { + t.Fatal(err) + } +} + +var encodeTokenTests = []struct { + desc string + toks []Token + want string + err string +}{{ + desc: "start element with name space", + toks: []Token{ + StartElement{Name{"space", "local"}, nil}, + }, + want: ``, +}, { + desc: "start element with no name", + toks: []Token{ + StartElement{Name{"space", ""}, nil}, + }, + err: "xml: start tag with no name", +}, { + desc: "end element with no name", + toks: []Token{ + EndElement{Name{"space", ""}}, + }, + err: "xml: end tag with no name", +}, { + desc: "char data", + toks: []Token{ + CharData("foo"), + }, + want: `foo`, +}, { + desc: "char data with escaped chars", + toks: []Token{ + CharData(" \t\n"), + }, + want: " \n", +}, { + desc: "comment", + toks: []Token{ + Comment("foo"), + }, + want: ``, +}, { + desc: "comment with invalid content", + toks: []Token{ + Comment("foo-->"), + }, + err: "xml: EncodeToken of Comment containing --> marker", +}, { + desc: "proc instruction", + toks: []Token{ + ProcInst{"Target", []byte("Instruction")}, + }, + want: ``, +}, { + desc: "proc instruction with empty target", + toks: []Token{ + ProcInst{"", []byte("Instruction")}, + }, + err: "xml: EncodeToken of ProcInst with invalid Target", +}, { + desc: "proc instruction with bad content", + toks: []Token{ + ProcInst{"", []byte("Instruction?>")}, + }, + err: "xml: EncodeToken of ProcInst with invalid Target", +}, { + desc: "directive", + toks: []Token{ + Directive("foo"), + }, + want: ``, +}, { + desc: "more complex directive", + toks: []Token{ + Directive("DOCTYPE doc [ '> ]"), + }, + want: `'> ]>`, +}, { + desc: "directive instruction with bad name", + toks: []Token{ + Directive("foo>"), + }, + err: "xml: EncodeToken of Directive containing wrong < or > markers", +}, { + desc: "end tag without start tag", + toks: []Token{ + EndElement{Name{"foo", "bar"}}, + }, + err: "xml: end tag without start tag", +}, { + desc: "mismatching end tag local name", + toks: []Token{ + StartElement{Name{"", "foo"}, nil}, + EndElement{Name{"", "bar"}}, + }, + err: "xml: end tag does not match start tag ", + want: ``, +}, { + desc: "mismatching end tag namespace", + toks: []Token{ + StartElement{Name{"space", "foo"}, nil}, + EndElement{Name{"another", "foo"}}, + }, + err: "xml: end tag in namespace another does not match start tag in namespace space", + want: ``, +}, { + desc: "start element with explicit namespace", + toks: []Token{ + StartElement{Name{"space", "local"}, []Attr{ + {Name{"xmlns", "x"}, "space"}, + {Name{"space", "foo"}, "value"}, + }}, + }, + want: ``, +}, { + desc: "start element with explicit namespace and colliding prefix", + toks: []Token{ + StartElement{Name{"space", "local"}, []Attr{ + {Name{"xmlns", "x"}, "space"}, + {Name{"space", "foo"}, "value"}, + {Name{"x", "bar"}, "other"}, + }}, + }, + want: ``, +}, { + desc: "start element using previously defined namespace", + toks: []Token{ + StartElement{Name{"", "local"}, []Attr{ + {Name{"xmlns", "x"}, "space"}, + }}, + StartElement{Name{"space", "foo"}, []Attr{ + {Name{"space", "x"}, "y"}, + }}, + }, + want: ``, +}, { + desc: "nested name space with same prefix", + toks: []Token{ + StartElement{Name{"", "foo"}, []Attr{ + {Name{"xmlns", "x"}, "space1"}, + }}, + StartElement{Name{"", "foo"}, []Attr{ + {Name{"xmlns", "x"}, "space2"}, + }}, + StartElement{Name{"", "foo"}, []Attr{ + {Name{"space1", "a"}, "space1 value"}, + {Name{"space2", "b"}, "space2 value"}, + }}, + EndElement{Name{"", "foo"}}, + EndElement{Name{"", "foo"}}, + StartElement{Name{"", "foo"}, []Attr{ + {Name{"space1", "a"}, "space1 value"}, + {Name{"space2", "b"}, "space2 value"}, + }}, + }, + want: ``, +}, { + desc: "start element defining several prefixes for the same name space", + toks: []Token{ + StartElement{Name{"space", "foo"}, []Attr{ + {Name{"xmlns", "a"}, "space"}, + {Name{"xmlns", "b"}, "space"}, + {Name{"space", "x"}, "value"}, + }}, + }, + want: ``, +}, { + desc: "nested element redefines name space", + toks: []Token{ + StartElement{Name{"", "foo"}, []Attr{ + {Name{"xmlns", "x"}, "space"}, + }}, + StartElement{Name{"space", "foo"}, []Attr{ + {Name{"xmlns", "y"}, "space"}, + {Name{"space", "a"}, "value"}, + }}, + }, + want: ``, +}, { + desc: "nested element creates alias for default name space", + toks: []Token{ + StartElement{Name{"space", "foo"}, []Attr{ + {Name{"", "xmlns"}, "space"}, + }}, + StartElement{Name{"space", "foo"}, []Attr{ + {Name{"xmlns", "y"}, "space"}, + {Name{"space", "a"}, "value"}, + }}, + }, + want: ``, +}, { + desc: "nested element defines default name space with existing prefix", + toks: []Token{ + StartElement{Name{"", "foo"}, []Attr{ + {Name{"xmlns", "x"}, "space"}, + }}, + StartElement{Name{"space", "foo"}, []Attr{ + {Name{"", "xmlns"}, "space"}, + {Name{"space", "a"}, "value"}, + }}, + }, + want: ``, +}, { + desc: "nested element uses empty attribute name space when default ns defined", + toks: []Token{ + StartElement{Name{"space", "foo"}, []Attr{ + {Name{"", "xmlns"}, "space"}, + }}, + StartElement{Name{"space", "foo"}, []Attr{ + {Name{"", "attr"}, "value"}, + }}, + }, + want: ``, +}, { + desc: "redefine xmlns", + toks: []Token{ + StartElement{Name{"", "foo"}, []Attr{ + {Name{"foo", "xmlns"}, "space"}, + }}, + }, + err: `xml: cannot redefine xmlns attribute prefix`, +}, { + desc: "xmlns with explicit name space #1", + toks: []Token{ + StartElement{Name{"space", "foo"}, []Attr{ + {Name{"xml", "xmlns"}, "space"}, + }}, + }, + want: ``, +}, { + desc: "xmlns with explicit name space #2", + toks: []Token{ + StartElement{Name{"space", "foo"}, []Attr{ + {Name{xmlURL, "xmlns"}, "space"}, + }}, + }, + want: ``, +}, { + desc: "empty name space declaration is ignored", + toks: []Token{ + StartElement{Name{"", "foo"}, []Attr{ + {Name{"xmlns", "foo"}, ""}, + }}, + }, + want: ``, +}, { + desc: "attribute with no name is ignored", + toks: []Token{ + StartElement{Name{"", "foo"}, []Attr{ + {Name{"", ""}, "value"}, + }}, + }, + want: ``, +}, { + desc: "namespace URL with non-valid name", + toks: []Token{ + StartElement{Name{"/34", "foo"}, []Attr{ + {Name{"/34", "x"}, "value"}, + }}, + }, + want: `<_:foo xmlns:_="/34" _:x="value">`, +}, { + desc: "nested element resets default namespace to empty", + toks: []Token{ + StartElement{Name{"space", "foo"}, []Attr{ + {Name{"", "xmlns"}, "space"}, + }}, + StartElement{Name{"", "foo"}, []Attr{ + {Name{"", "xmlns"}, ""}, + {Name{"", "x"}, "value"}, + {Name{"space", "x"}, "value"}, + }}, + }, + want: ``, +}, { + desc: "nested element requires empty default name space", + toks: []Token{ + StartElement{Name{"space", "foo"}, []Attr{ + {Name{"", "xmlns"}, "space"}, + }}, + StartElement{Name{"", "foo"}, nil}, + }, + want: ``, +}, { + desc: "attribute uses name space from xmlns", + toks: []Token{ + StartElement{Name{"some/space", "foo"}, []Attr{ + {Name{"", "attr"}, "value"}, + {Name{"some/space", "other"}, "other value"}, + }}, + }, + want: ``, +}, { + desc: "default name space should not be used by attributes", + toks: []Token{ + StartElement{Name{"space", "foo"}, []Attr{ + {Name{"", "xmlns"}, "space"}, + {Name{"xmlns", "bar"}, "space"}, + {Name{"space", "baz"}, "foo"}, + }}, + StartElement{Name{"space", "baz"}, nil}, + EndElement{Name{"space", "baz"}}, + EndElement{Name{"space", "foo"}}, + }, + want: ``, +}, { + desc: "default name space not used by attributes, not explicitly defined", + toks: []Token{ + StartElement{Name{"space", "foo"}, []Attr{ + {Name{"", "xmlns"}, "space"}, + {Name{"space", "baz"}, "foo"}, + }}, + StartElement{Name{"space", "baz"}, nil}, + EndElement{Name{"space", "baz"}}, + EndElement{Name{"space", "foo"}}, + }, + want: ``, +}, { + desc: "impossible xmlns declaration", + toks: []Token{ + StartElement{Name{"", "foo"}, []Attr{ + {Name{"", "xmlns"}, "space"}, + }}, + StartElement{Name{"space", "bar"}, []Attr{ + {Name{"space", "attr"}, "value"}, + }}, + }, + want: ``, +}} + +func TestEncodeToken(t *testing.T) { +loop: + for i, tt := range encodeTokenTests { + var buf bytes.Buffer + enc := NewEncoder(&buf) + var err error + for j, tok := range tt.toks { + err = enc.EncodeToken(tok) + if err != nil && j < len(tt.toks)-1 { + t.Errorf("#%d %s token #%d: %v", i, tt.desc, j, err) + continue loop + } + } + errorf := func(f string, a ...interface{}) { + t.Errorf("#%d %s token #%d:%s", i, tt.desc, len(tt.toks)-1, fmt.Sprintf(f, a...)) + } + switch { + case tt.err != "" && err == nil: + errorf(" expected error; got none") + continue + case tt.err == "" && err != nil: + errorf(" got error: %v", err) + continue + case tt.err != "" && err != nil && tt.err != err.Error(): + errorf(" error mismatch; got %v, want %v", err, tt.err) + continue + } + if err := enc.Flush(); err != nil { + errorf(" %v", err) + continue + } + if got := buf.String(); got != tt.want { + errorf("\ngot %v\nwant %v", got, tt.want) + continue + } + } +} + +func TestProcInstEncodeToken(t *testing.T) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + + if err := enc.EncodeToken(ProcInst{"xml", []byte("Instruction")}); err != nil { + t.Fatalf("enc.EncodeToken: expected to be able to encode xml target ProcInst as first token, %s", err) + } + + if err := enc.EncodeToken(ProcInst{"Target", []byte("Instruction")}); err != nil { + t.Fatalf("enc.EncodeToken: expected to be able to add non-xml target ProcInst") + } + + if err := enc.EncodeToken(ProcInst{"xml", []byte("Instruction")}); err == nil { + t.Fatalf("enc.EncodeToken: expected to not be allowed to encode xml target ProcInst when not first token") + } +} + +func TestDecodeEncode(t *testing.T) { + var in, out bytes.Buffer + in.WriteString(` + + + +`) + dec := NewDecoder(&in) + enc := NewEncoder(&out) + for tok, err := dec.Token(); err == nil; tok, err = dec.Token() { + err = enc.EncodeToken(tok) + if err != nil { + t.Fatalf("enc.EncodeToken: Unable to encode token (%#v), %v", tok, err) + } + } +} + +// Issue 9796. Used to fail with GORACE="halt_on_error=1" -race. +func TestRace9796(t *testing.T) { + type A struct{} + type B struct { + C []A `xml:"X>Y"` + } + var wg sync.WaitGroup + for i := 0; i < 2; i++ { + wg.Add(1) + go func() { + Marshal(B{[]A{{}}}) + wg.Done() + }() + } + wg.Wait() +} + +func TestIsValidDirective(t *testing.T) { + testOK := []string{ + "<>", + "< < > >", + "' '>' >", + " ]>", + " '<' ' doc ANY> ]>", + ">>> a < comment --> [ ] >", + } + testKO := []string{ + "<", + ">", + "", + "< > > < < >", + " -->", + "", + "'", + "", + } + for _, s := range testOK { + if !isValidDirective(Directive(s)) { + t.Errorf("Directive %q is expected to be valid", s) + } + } + for _, s := range testKO { + if isValidDirective(Directive(s)) { + t.Errorf("Directive %q is expected to be invalid", s) + } + } +} + +// Issue 11719. EncodeToken used to silently eat tokens with an invalid type. +func TestSimpleUseOfEncodeToken(t *testing.T) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + if err := enc.EncodeToken(&StartElement{Name: Name{"", "object1"}}); err == nil { + t.Errorf("enc.EncodeToken: pointer type should be rejected") + } + if err := enc.EncodeToken(&EndElement{Name: Name{"", "object1"}}); err == nil { + t.Errorf("enc.EncodeToken: pointer type should be rejected") + } + if err := enc.EncodeToken(StartElement{Name: Name{"", "object2"}}); err != nil { + t.Errorf("enc.EncodeToken: StartElement %s", err) + } + if err := enc.EncodeToken(EndElement{Name: Name{"", "object2"}}); err != nil { + t.Errorf("enc.EncodeToken: EndElement %s", err) + } + if err := enc.EncodeToken(Universe{}); err == nil { + t.Errorf("enc.EncodeToken: invalid type not caught") + } + if err := enc.Flush(); err != nil { + t.Errorf("enc.Flush: %s", err) + } + if buf.Len() == 0 { + t.Errorf("enc.EncodeToken: empty buffer") + } + want := "" + if buf.String() != want { + t.Errorf("enc.EncodeToken: expected %q; got %q", want, buf.String()) + } +} diff --git a/endpoints/drive/webdav/internal/xml/read.go b/endpoints/drive/webdav/internal/xml/read.go new file mode 100644 index 000000000..bfaef6f17 --- /dev/null +++ b/endpoints/drive/webdav/internal/xml/read.go @@ -0,0 +1,691 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xml + +import ( + "bytes" + "encoding" + "errors" + "fmt" + "reflect" + "strconv" + "strings" +) + +// BUG(rsc): Mapping between XML elements and data structures is inherently flawed: +// an XML element is an order-dependent collection of anonymous +// values, while a data structure is an order-independent collection +// of named values. +// See package json for a textual representation more suitable +// to data structures. + +// Unmarshal parses the XML-encoded data and stores the result in +// the value pointed to by v, which must be an arbitrary struct, +// slice, or string. Well-formed data that does not fit into v is +// discarded. +// +// Because Unmarshal uses the reflect package, it can only assign +// to exported (upper case) fields. Unmarshal uses a case-sensitive +// comparison to match XML element names to tag values and struct +// field names. +// +// Unmarshal maps an XML element to a struct using the following rules. +// In the rules, the tag of a field refers to the value associated with the +// key 'xml' in the struct field's tag (see the example above). +// +// - If the struct has a field of type []byte or string with tag +// ",innerxml", Unmarshal accumulates the raw XML nested inside the +// element in that field. The rest of the rules still apply. +// +// - If the struct has a field named XMLName of type xml.Name, +// Unmarshal records the element name in that field. +// +// - If the XMLName field has an associated tag of the form +// "name" or "namespace-URL name", the XML element must have +// the given name (and, optionally, name space) or else Unmarshal +// returns an error. +// +// - If the XML element has an attribute whose name matches a +// struct field name with an associated tag containing ",attr" or +// the explicit name in a struct field tag of the form "name,attr", +// Unmarshal records the attribute value in that field. +// +// - If the XML element contains character data, that data is +// accumulated in the first struct field that has tag ",chardata". +// The struct field may have type []byte or string. +// If there is no such field, the character data is discarded. +// +// - If the XML element contains comments, they are accumulated in +// the first struct field that has tag ",comment". The struct +// field may have type []byte or string. If there is no such +// field, the comments are discarded. +// +// - If the XML element contains a sub-element whose name matches +// the prefix of a tag formatted as "a" or "a>b>c", unmarshal +// will descend into the XML structure looking for elements with the +// given names, and will map the innermost elements to that struct +// field. A tag starting with ">" is equivalent to one starting +// with the field name followed by ">". +// +// - If the XML element contains a sub-element whose name matches +// a struct field's XMLName tag and the struct field has no +// explicit name tag as per the previous rule, unmarshal maps +// the sub-element to that struct field. +// +// - If the XML element contains a sub-element whose name matches a +// field without any mode flags (",attr", ",chardata", etc), Unmarshal +// maps the sub-element to that struct field. +// +// - If the XML element contains a sub-element that hasn't matched any +// of the above rules and the struct has a field with tag ",any", +// unmarshal maps the sub-element to that struct field. +// +// - An anonymous struct field is handled as if the fields of its +// value were part of the outer struct. +// +// - A struct field with tag "-" is never unmarshalled into. +// +// Unmarshal maps an XML element to a string or []byte by saving the +// concatenation of that element's character data in the string or +// []byte. The saved []byte is never nil. +// +// Unmarshal maps an attribute value to a string or []byte by saving +// the value in the string or slice. +// +// Unmarshal maps an XML element to a slice by extending the length of +// the slice and mapping the element to the newly created value. +// +// Unmarshal maps an XML element or attribute value to a bool by +// setting it to the boolean value represented by the string. +// +// Unmarshal maps an XML element or attribute value to an integer or +// floating-point field by setting the field to the result of +// interpreting the string value in decimal. There is no check for +// overflow. +// +// Unmarshal maps an XML element to an xml.Name by recording the +// element name. +// +// Unmarshal maps an XML element to a pointer by setting the pointer +// to a freshly allocated value and then mapping the element to that value. +func Unmarshal(data []byte, v interface{}) error { + return NewDecoder(bytes.NewReader(data)).Decode(v) +} + +// Decode works like xml.Unmarshal, except it reads the decoder +// stream to find the start element. +func (d *Decoder) Decode(v interface{}) error { + return d.DecodeElement(v, nil) +} + +// DecodeElement works like xml.Unmarshal except that it takes +// a pointer to the start XML element to decode into v. +// It is useful when a client reads some raw XML tokens itself +// but also wants to defer to Unmarshal for some elements. +func (d *Decoder) DecodeElement(v interface{}, start *StartElement) error { + val := reflect.ValueOf(v) + if val.Kind() != reflect.Ptr { + return errors.New("non-pointer passed to Unmarshal") + } + return d.unmarshal(val.Elem(), start) +} + +// An UnmarshalError represents an error in the unmarshalling process. +type UnmarshalError string + +func (e UnmarshalError) Error() string { return string(e) } + +// Unmarshaler is the interface implemented by objects that can unmarshal +// an XML element description of themselves. +// +// UnmarshalXML decodes a single XML element +// beginning with the given start element. +// If it returns an error, the outer call to Unmarshal stops and +// returns that error. +// UnmarshalXML must consume exactly one XML element. +// One common implementation strategy is to unmarshal into +// a separate value with a layout matching the expected XML +// using d.DecodeElement, and then to copy the data from +// that value into the receiver. +// Another common strategy is to use d.Token to process the +// XML object one token at a time. +// UnmarshalXML may not use d.RawToken. +type Unmarshaler interface { + UnmarshalXML(d *Decoder, start StartElement) error +} + +// UnmarshalerAttr is the interface implemented by objects that can unmarshal +// an XML attribute description of themselves. +// +// UnmarshalXMLAttr decodes a single XML attribute. +// If it returns an error, the outer call to Unmarshal stops and +// returns that error. +// UnmarshalXMLAttr is used only for struct fields with the +// "attr" option in the field tag. +type UnmarshalerAttr interface { + UnmarshalXMLAttr(attr Attr) error +} + +// receiverType returns the receiver type to use in an expression like "%s.MethodName". +func receiverType(val interface{}) string { + t := reflect.TypeOf(val) + if t.Name() != "" { + return t.String() + } + return "(" + t.String() + ")" +} + +// unmarshalInterface unmarshals a single XML element into val. +// start is the opening tag of the element. +func (p *Decoder) unmarshalInterface(val Unmarshaler, start *StartElement) error { + // Record that decoder must stop at end tag corresponding to start. + p.pushEOF() + + p.unmarshalDepth++ + err := val.UnmarshalXML(p, *start) + p.unmarshalDepth-- + if err != nil { + p.popEOF() + return err + } + + if !p.popEOF() { + return fmt.Errorf("xml: %s.UnmarshalXML did not consume entire <%s> element", receiverType(val), start.Name.Local) + } + + return nil +} + +// unmarshalTextInterface unmarshals a single XML element into val. +// The chardata contained in the element (but not its children) +// is passed to the text unmarshaler. +func (p *Decoder) unmarshalTextInterface(val encoding.TextUnmarshaler, start *StartElement) error { + var buf []byte + depth := 1 + for depth > 0 { + t, err := p.Token() + if err != nil { + return err + } + switch t := t.(type) { + case CharData: + if depth == 1 { + buf = append(buf, t...) + } + case StartElement: + depth++ + case EndElement: + depth-- + } + } + return val.UnmarshalText(buf) +} + +// unmarshalAttr unmarshals a single XML attribute into val. +func (p *Decoder) unmarshalAttr(val reflect.Value, attr Attr) error { + if val.Kind() == reflect.Ptr { + if val.IsNil() { + val.Set(reflect.New(val.Type().Elem())) + } + val = val.Elem() + } + + if val.CanInterface() && val.Type().Implements(unmarshalerAttrType) { + // This is an unmarshaler with a non-pointer receiver, + // so it's likely to be incorrect, but we do what we're told. + return val.Interface().(UnmarshalerAttr).UnmarshalXMLAttr(attr) + } + if val.CanAddr() { + pv := val.Addr() + if pv.CanInterface() && pv.Type().Implements(unmarshalerAttrType) { + return pv.Interface().(UnmarshalerAttr).UnmarshalXMLAttr(attr) + } + } + + // Not an UnmarshalerAttr; try encoding.TextUnmarshaler. + if val.CanInterface() && val.Type().Implements(textUnmarshalerType) { + // This is an unmarshaler with a non-pointer receiver, + // so it's likely to be incorrect, but we do what we're told. + return val.Interface().(encoding.TextUnmarshaler).UnmarshalText([]byte(attr.Value)) + } + if val.CanAddr() { + pv := val.Addr() + if pv.CanInterface() && pv.Type().Implements(textUnmarshalerType) { + return pv.Interface().(encoding.TextUnmarshaler).UnmarshalText([]byte(attr.Value)) + } + } + + copyValue(val, []byte(attr.Value)) + return nil +} + +var ( + unmarshalerType = reflect.TypeOf((*Unmarshaler)(nil)).Elem() + unmarshalerAttrType = reflect.TypeOf((*UnmarshalerAttr)(nil)).Elem() + textUnmarshalerType = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem() +) + +// Unmarshal a single XML element into val. +func (p *Decoder) unmarshal(val reflect.Value, start *StartElement) error { + // Find start element if we need it. + if start == nil { + for { + tok, err := p.Token() + if err != nil { + return err + } + if t, ok := tok.(StartElement); ok { + start = &t + break + } + } + } + + // Load value from interface, but only if the result will be + // usefully addressable. + if val.Kind() == reflect.Interface && !val.IsNil() { + e := val.Elem() + if e.Kind() == reflect.Ptr && !e.IsNil() { + val = e + } + } + + if val.Kind() == reflect.Ptr { + if val.IsNil() { + val.Set(reflect.New(val.Type().Elem())) + } + val = val.Elem() + } + + if val.CanInterface() && val.Type().Implements(unmarshalerType) { + // This is an unmarshaler with a non-pointer receiver, + // so it's likely to be incorrect, but we do what we're told. + return p.unmarshalInterface(val.Interface().(Unmarshaler), start) + } + + if val.CanAddr() { + pv := val.Addr() + if pv.CanInterface() && pv.Type().Implements(unmarshalerType) { + return p.unmarshalInterface(pv.Interface().(Unmarshaler), start) + } + } + + if val.CanInterface() && val.Type().Implements(textUnmarshalerType) { + return p.unmarshalTextInterface(val.Interface().(encoding.TextUnmarshaler), start) + } + + if val.CanAddr() { + pv := val.Addr() + if pv.CanInterface() && pv.Type().Implements(textUnmarshalerType) { + return p.unmarshalTextInterface(pv.Interface().(encoding.TextUnmarshaler), start) + } + } + + var ( + data []byte + saveData reflect.Value + comment []byte + saveComment reflect.Value + saveXML reflect.Value + saveXMLIndex int + saveXMLData []byte + saveAny reflect.Value + sv reflect.Value + tinfo *typeInfo + err error + ) + + switch v := val; v.Kind() { + default: + return errors.New("unknown type " + v.Type().String()) + + case reflect.Interface: + // TODO: For now, simply ignore the field. In the near + // future we may choose to unmarshal the start + // element on it, if not nil. + return p.Skip() + + case reflect.Slice: + typ := v.Type() + if typ.Elem().Kind() == reflect.Uint8 { + // []byte + saveData = v + break + } + + // Slice of element values. + // Grow slice. + n := v.Len() + if n >= v.Cap() { + ncap := 2 * n + if ncap < 4 { + ncap = 4 + } + new := reflect.MakeSlice(typ, n, ncap) + reflect.Copy(new, v) + v.Set(new) + } + v.SetLen(n + 1) + + // Recur to read element into slice. + if err := p.unmarshal(v.Index(n), start); err != nil { + v.SetLen(n) + return err + } + return nil + + case reflect.Bool, reflect.Float32, reflect.Float64, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, reflect.String: + saveData = v + + case reflect.Struct: + typ := v.Type() + if typ == nameType { + v.Set(reflect.ValueOf(start.Name)) + break + } + + sv = v + tinfo, err = getTypeInfo(typ) + if err != nil { + return err + } + + // Validate and assign element name. + if tinfo.xmlname != nil { + finfo := tinfo.xmlname + if finfo.name != "" && finfo.name != start.Name.Local { + return UnmarshalError("expected element type <" + finfo.name + "> but have <" + start.Name.Local + ">") + } + if finfo.xmlns != "" && finfo.xmlns != start.Name.Space { + e := "expected element <" + finfo.name + "> in name space " + finfo.xmlns + " but have " + if start.Name.Space == "" { + e += "no name space" + } else { + e += start.Name.Space + } + return UnmarshalError(e) + } + fv := finfo.value(sv) + if _, ok := fv.Interface().(Name); ok { + fv.Set(reflect.ValueOf(start.Name)) + } + } + + // Assign attributes. + // Also, determine whether we need to save character data or comments. + for i := range tinfo.fields { + finfo := &tinfo.fields[i] + switch finfo.flags & fMode { + case fAttr: + strv := finfo.value(sv) + // Look for attribute. + for _, a := range start.Attr { + if a.Name.Local == finfo.name && (finfo.xmlns == "" || finfo.xmlns == a.Name.Space) { + if err := p.unmarshalAttr(strv, a); err != nil { + return err + } + break + } + } + + case fCharData: + if !saveData.IsValid() { + saveData = finfo.value(sv) + } + + case fComment: + if !saveComment.IsValid() { + saveComment = finfo.value(sv) + } + + case fAny, fAny | fElement: + if !saveAny.IsValid() { + saveAny = finfo.value(sv) + } + + case fInnerXml: + if !saveXML.IsValid() { + saveXML = finfo.value(sv) + if p.saved == nil { + saveXMLIndex = 0 + p.saved = new(bytes.Buffer) + } else { + saveXMLIndex = p.savedOffset() + } + } + } + } + } + + // Find end element. + // Process sub-elements along the way. +Loop: + for { + var savedOffset int + if saveXML.IsValid() { + savedOffset = p.savedOffset() + } + tok, err := p.Token() + if err != nil { + return err + } + switch t := tok.(type) { + case StartElement: + consumed := false + if sv.IsValid() { + consumed, err = p.unmarshalPath(tinfo, sv, nil, &t) + if err != nil { + return err + } + if !consumed && saveAny.IsValid() { + consumed = true + if err := p.unmarshal(saveAny, &t); err != nil { + return err + } + } + } + if !consumed { + if err := p.Skip(); err != nil { + return err + } + } + + case EndElement: + if saveXML.IsValid() { + saveXMLData = p.saved.Bytes()[saveXMLIndex:savedOffset] + if saveXMLIndex == 0 { + p.saved = nil + } + } + break Loop + + case CharData: + if saveData.IsValid() { + data = append(data, t...) + } + + case Comment: + if saveComment.IsValid() { + comment = append(comment, t...) + } + } + } + + if saveData.IsValid() && saveData.CanInterface() && saveData.Type().Implements(textUnmarshalerType) { + if err := saveData.Interface().(encoding.TextUnmarshaler).UnmarshalText(data); err != nil { + return err + } + saveData = reflect.Value{} + } + + if saveData.IsValid() && saveData.CanAddr() { + pv := saveData.Addr() + if pv.CanInterface() && pv.Type().Implements(textUnmarshalerType) { + if err := pv.Interface().(encoding.TextUnmarshaler).UnmarshalText(data); err != nil { + return err + } + saveData = reflect.Value{} + } + } + + if err := copyValue(saveData, data); err != nil { + return err + } + + switch t := saveComment; t.Kind() { + case reflect.String: + t.SetString(string(comment)) + case reflect.Slice: + t.Set(reflect.ValueOf(comment)) + } + + switch t := saveXML; t.Kind() { + case reflect.String: + t.SetString(string(saveXMLData)) + case reflect.Slice: + t.Set(reflect.ValueOf(saveXMLData)) + } + + return nil +} + +func copyValue(dst reflect.Value, src []byte) (err error) { + dst0 := dst + + if dst.Kind() == reflect.Ptr { + if dst.IsNil() { + dst.Set(reflect.New(dst.Type().Elem())) + } + dst = dst.Elem() + } + + // Save accumulated data. + switch dst.Kind() { + case reflect.Invalid: + // Probably a comment. + default: + return errors.New("cannot unmarshal into " + dst0.Type().String()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + itmp, err := strconv.ParseInt(string(src), 10, dst.Type().Bits()) + if err != nil { + return err + } + dst.SetInt(itmp) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + utmp, err := strconv.ParseUint(string(src), 10, dst.Type().Bits()) + if err != nil { + return err + } + dst.SetUint(utmp) + case reflect.Float32, reflect.Float64: + ftmp, err := strconv.ParseFloat(string(src), dst.Type().Bits()) + if err != nil { + return err + } + dst.SetFloat(ftmp) + case reflect.Bool: + value, err := strconv.ParseBool(strings.TrimSpace(string(src))) + if err != nil { + return err + } + dst.SetBool(value) + case reflect.String: + dst.SetString(string(src)) + case reflect.Slice: + if len(src) == 0 { + // non-nil to flag presence + src = []byte{} + } + dst.SetBytes(src) + } + return nil +} + +// unmarshalPath walks down an XML structure looking for wanted +// paths, and calls unmarshal on them. +// The consumed result tells whether XML elements have been consumed +// from the Decoder until start's matching end element, or if it's +// still untouched because start is uninteresting for sv's fields. +func (p *Decoder) unmarshalPath(tinfo *typeInfo, sv reflect.Value, parents []string, start *StartElement) (consumed bool, err error) { + recurse := false +Loop: + for i := range tinfo.fields { + finfo := &tinfo.fields[i] + if finfo.flags&fElement == 0 || len(finfo.parents) < len(parents) || finfo.xmlns != "" && finfo.xmlns != start.Name.Space { + continue + } + for j := range parents { + if parents[j] != finfo.parents[j] { + continue Loop + } + } + if len(finfo.parents) == len(parents) && finfo.name == start.Name.Local { + // It's a perfect match, unmarshal the field. + return true, p.unmarshal(finfo.value(sv), start) + } + if len(finfo.parents) > len(parents) && finfo.parents[len(parents)] == start.Name.Local { + // It's a prefix for the field. Break and recurse + // since it's not ok for one field path to be itself + // the prefix for another field path. + recurse = true + + // We can reuse the same slice as long as we + // don't try to append to it. + parents = finfo.parents[:len(parents)+1] + break + } + } + if !recurse { + // We have no business with this element. + return false, nil + } + // The element is not a perfect match for any field, but one + // or more fields have the path to this element as a parent + // prefix. Recurse and attempt to match these. + for { + var tok Token + tok, err = p.Token() + if err != nil { + return true, err + } + switch t := tok.(type) { + case StartElement: + consumed2, err := p.unmarshalPath(tinfo, sv, parents, &t) + if err != nil { + return true, err + } + if !consumed2 { + if err := p.Skip(); err != nil { + return true, err + } + } + case EndElement: + return true, nil + } + } +} + +// Skip reads tokens until it has consumed the end element +// matching the most recent start element already consumed. +// It recurs if it encounters a start element, so it can be used to +// skip nested structures. +// It returns nil if it finds an end element matching the start +// element; otherwise it returns an error describing the problem. +func (d *Decoder) Skip() error { + for { + tok, err := d.Token() + if err != nil { + return err + } + switch tok.(type) { + case StartElement: + if err := d.Skip(); err != nil { + return err + } + case EndElement: + return nil + } + } +} diff --git a/endpoints/drive/webdav/internal/xml/read_test.go b/endpoints/drive/webdav/internal/xml/read_test.go new file mode 100644 index 000000000..02f1e10c3 --- /dev/null +++ b/endpoints/drive/webdav/internal/xml/read_test.go @@ -0,0 +1,744 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xml + +import ( + "bytes" + "fmt" + "io" + "reflect" + "strings" + "testing" + "time" +) + +// Stripped down Atom feed data structures. + +func TestUnmarshalFeed(t *testing.T) { + var f Feed + if err := Unmarshal([]byte(atomFeedString), &f); err != nil { + t.Fatalf("Unmarshal: %s", err) + } + if !reflect.DeepEqual(f, atomFeed) { + t.Fatalf("have %#v\nwant %#v", f, atomFeed) + } +} + +// hget http://codereview.appspot.com/rss/mine/rsc +const atomFeedString = ` + +Code Review - My issueshttp://codereview.appspot.com/rietveld<>rietveld: an attempt at pubsubhubbub +2009-10-04T01:35:58+00:00email-address-removedurn:md5:134d9179c41f806be79b3a5f7877d19a + An attempt at adding pubsubhubbub support to Rietveld. +http://code.google.com/p/pubsubhubbub +http://code.google.com/p/rietveld/issues/detail?id=155 + +The server side of the protocol is trivial: + 1. add a &lt;link rel=&quot;hub&quot; href=&quot;hub-server&quot;&gt; tag to all + feeds that will be pubsubhubbubbed. + 2. every time one of those feeds changes, tell the hub + with a simple POST request. + +I have tested this by adding debug prints to a local hub +server and checking that the server got the right publish +requests. + +I can&#39;t quite get the server to work, but I think the bug +is not in my code. I think that the server expects to be +able to grab the feed and see the feed&#39;s actual URL in +the link rel=&quot;self&quot;, but the default value for that drops +the :port from the URL, and I cannot for the life of me +figure out how to get the Atom generator deep inside +django not to do that, or even where it is doing that, +or even what code is running to generate the Atom feed. +(I thought I knew but I added some assert False statements +and it kept running!) + +Ignoring that particular problem, I would appreciate +feedback on the right way to get the two values at +the top of feeds.py marked NOTE(rsc). + + +rietveld: correct tab handling +2009-10-03T23:02:17+00:00email-address-removedurn:md5:0a2a4f19bb815101f0ba2904aed7c35a + This fixes the buggy tab rendering that can be seen at +http://codereview.appspot.com/116075/diff/1/2 + +The fundamental problem was that the tab code was +not being told what column the text began in, so it +didn&#39;t know where to put the tab stops. Another problem +was that some of the code assumed that string byte +offsets were the same as column offsets, which is only +true if there are no tabs. + +In the process of fixing this, I cleaned up the arguments +to Fold and ExpandTabs and renamed them Break and +_ExpandTabs so that I could be sure that I found all the +call sites. I also wanted to verify that ExpandTabs was +not being used from outside intra_region_diff.py. + + + ` + +type Feed struct { + XMLName Name `xml:"http://www.w3.org/2005/Atom feed"` + Title string `xml:"title"` + Id string `xml:"id"` + Link []Link `xml:"link"` + Updated time.Time `xml:"updated,attr"` + Author Person `xml:"author"` + Entry []Entry `xml:"entry"` +} + +type Entry struct { + Title string `xml:"title"` + Id string `xml:"id"` + Link []Link `xml:"link"` + Updated time.Time `xml:"updated"` + Author Person `xml:"author"` + Summary Text `xml:"summary"` +} + +type Link struct { + Rel string `xml:"rel,attr,omitempty"` + Href string `xml:"href,attr"` +} + +type Person struct { + Name string `xml:"name"` + URI string `xml:"uri"` + Email string `xml:"email"` + InnerXML string `xml:",innerxml"` +} + +type Text struct { + Type string `xml:"type,attr,omitempty"` + Body string `xml:",chardata"` +} + +var atomFeed = Feed{ + XMLName: Name{"http://www.w3.org/2005/Atom", "feed"}, + Title: "Code Review - My issues", + Link: []Link{ + {Rel: "alternate", Href: "http://codereview.appspot.com/"}, + {Rel: "self", Href: "http://codereview.appspot.com/rss/mine/rsc"}, + }, + Id: "http://codereview.appspot.com/", + Updated: ParseTime("2009-10-04T01:35:58+00:00"), + Author: Person{ + Name: "rietveld<>", + InnerXML: "rietveld<>", + }, + Entry: []Entry{ + { + Title: "rietveld: an attempt at pubsubhubbub\n", + Link: []Link{ + {Rel: "alternate", Href: "http://codereview.appspot.com/126085"}, + }, + Updated: ParseTime("2009-10-04T01:35:58+00:00"), + Author: Person{ + Name: "email-address-removed", + InnerXML: "email-address-removed", + }, + Id: "urn:md5:134d9179c41f806be79b3a5f7877d19a", + Summary: Text{ + Type: "html", + Body: ` + An attempt at adding pubsubhubbub support to Rietveld. +http://code.google.com/p/pubsubhubbub +http://code.google.com/p/rietveld/issues/detail?id=155 + +The server side of the protocol is trivial: + 1. add a <link rel="hub" href="hub-server"> tag to all + feeds that will be pubsubhubbubbed. + 2. every time one of those feeds changes, tell the hub + with a simple POST request. + +I have tested this by adding debug prints to a local hub +server and checking that the server got the right publish +requests. + +I can't quite get the server to work, but I think the bug +is not in my code. I think that the server expects to be +able to grab the feed and see the feed's actual URL in +the link rel="self", but the default value for that drops +the :port from the URL, and I cannot for the life of me +figure out how to get the Atom generator deep inside +django not to do that, or even where it is doing that, +or even what code is running to generate the Atom feed. +(I thought I knew but I added some assert False statements +and it kept running!) + +Ignoring that particular problem, I would appreciate +feedback on the right way to get the two values at +the top of feeds.py marked NOTE(rsc). + + +`, + }, + }, + { + Title: "rietveld: correct tab handling\n", + Link: []Link{ + {Rel: "alternate", Href: "http://codereview.appspot.com/124106"}, + }, + Updated: ParseTime("2009-10-03T23:02:17+00:00"), + Author: Person{ + Name: "email-address-removed", + InnerXML: "email-address-removed", + }, + Id: "urn:md5:0a2a4f19bb815101f0ba2904aed7c35a", + Summary: Text{ + Type: "html", + Body: ` + This fixes the buggy tab rendering that can be seen at +http://codereview.appspot.com/116075/diff/1/2 + +The fundamental problem was that the tab code was +not being told what column the text began in, so it +didn't know where to put the tab stops. Another problem +was that some of the code assumed that string byte +offsets were the same as column offsets, which is only +true if there are no tabs. + +In the process of fixing this, I cleaned up the arguments +to Fold and ExpandTabs and renamed them Break and +_ExpandTabs so that I could be sure that I found all the +call sites. I also wanted to verify that ExpandTabs was +not being used from outside intra_region_diff.py. + + +`, + }, + }, + }, +} + +const pathTestString = ` + + 1 + + + A + + + B + + + C + D + + <_> + E + + + 2 + +` + +type PathTestItem struct { + Value string +} + +type PathTestA struct { + Items []PathTestItem `xml:">Item1"` + Before, After string +} + +type PathTestB struct { + Other []PathTestItem `xml:"Items>Item1"` + Before, After string +} + +type PathTestC struct { + Values1 []string `xml:"Items>Item1>Value"` + Values2 []string `xml:"Items>Item2>Value"` + Before, After string +} + +type PathTestSet struct { + Item1 []PathTestItem +} + +type PathTestD struct { + Other PathTestSet `xml:"Items"` + Before, After string +} + +type PathTestE struct { + Underline string `xml:"Items>_>Value"` + Before, After string +} + +var pathTests = []interface{}{ + &PathTestA{Items: []PathTestItem{{"A"}, {"D"}}, Before: "1", After: "2"}, + &PathTestB{Other: []PathTestItem{{"A"}, {"D"}}, Before: "1", After: "2"}, + &PathTestC{Values1: []string{"A", "C", "D"}, Values2: []string{"B"}, Before: "1", After: "2"}, + &PathTestD{Other: PathTestSet{Item1: []PathTestItem{{"A"}, {"D"}}}, Before: "1", After: "2"}, + &PathTestE{Underline: "E", Before: "1", After: "2"}, +} + +func TestUnmarshalPaths(t *testing.T) { + for _, pt := range pathTests { + v := reflect.New(reflect.TypeOf(pt).Elem()).Interface() + if err := Unmarshal([]byte(pathTestString), v); err != nil { + t.Fatalf("Unmarshal: %s", err) + } + if !reflect.DeepEqual(v, pt) { + t.Fatalf("have %#v\nwant %#v", v, pt) + } + } +} + +type BadPathTestA struct { + First string `xml:"items>item1"` + Other string `xml:"items>item2"` + Second string `xml:"items"` +} + +type BadPathTestB struct { + Other string `xml:"items>item2>value"` + First string `xml:"items>item1"` + Second string `xml:"items>item1>value"` +} + +type BadPathTestC struct { + First string + Second string `xml:"First"` +} + +type BadPathTestD struct { + BadPathEmbeddedA + BadPathEmbeddedB +} + +type BadPathEmbeddedA struct { + First string +} + +type BadPathEmbeddedB struct { + Second string `xml:"First"` +} + +var badPathTests = []struct { + v, e interface{} +}{ + {&BadPathTestA{}, &TagPathError{reflect.TypeOf(BadPathTestA{}), "First", "items>item1", "Second", "items"}}, + {&BadPathTestB{}, &TagPathError{reflect.TypeOf(BadPathTestB{}), "First", "items>item1", "Second", "items>item1>value"}}, + {&BadPathTestC{}, &TagPathError{reflect.TypeOf(BadPathTestC{}), "First", "", "Second", "First"}}, + {&BadPathTestD{}, &TagPathError{reflect.TypeOf(BadPathTestD{}), "First", "", "Second", "First"}}, +} + +func TestUnmarshalBadPaths(t *testing.T) { + for _, tt := range badPathTests { + err := Unmarshal([]byte(pathTestString), tt.v) + if !reflect.DeepEqual(err, tt.e) { + t.Fatalf("Unmarshal with %#v didn't fail properly:\nhave %#v,\nwant %#v", tt.v, err, tt.e) + } + } +} + +const OK = "OK" +const withoutNameTypeData = ` + +` + +type TestThree struct { + XMLName Name `xml:"Test3"` + Attr string `xml:",attr"` +} + +func TestUnmarshalWithoutNameType(t *testing.T) { + var x TestThree + if err := Unmarshal([]byte(withoutNameTypeData), &x); err != nil { + t.Fatalf("Unmarshal: %s", err) + } + if x.Attr != OK { + t.Fatalf("have %v\nwant %v", x.Attr, OK) + } +} + +func TestUnmarshalAttr(t *testing.T) { + type ParamVal struct { + Int int `xml:"int,attr"` + } + + type ParamPtr struct { + Int *int `xml:"int,attr"` + } + + type ParamStringPtr struct { + Int *string `xml:"int,attr"` + } + + x := []byte(``) + + p1 := &ParamPtr{} + if err := Unmarshal(x, p1); err != nil { + t.Fatalf("Unmarshal: %s", err) + } + if p1.Int == nil { + t.Fatalf("Unmarshal failed in to *int field") + } else if *p1.Int != 1 { + t.Fatalf("Unmarshal with %s failed:\nhave %#v,\n want %#v", x, p1.Int, 1) + } + + p2 := &ParamVal{} + if err := Unmarshal(x, p2); err != nil { + t.Fatalf("Unmarshal: %s", err) + } + if p2.Int != 1 { + t.Fatalf("Unmarshal with %s failed:\nhave %#v,\n want %#v", x, p2.Int, 1) + } + + p3 := &ParamStringPtr{} + if err := Unmarshal(x, p3); err != nil { + t.Fatalf("Unmarshal: %s", err) + } + if p3.Int == nil { + t.Fatalf("Unmarshal failed in to *string field") + } else if *p3.Int != "1" { + t.Fatalf("Unmarshal with %s failed:\nhave %#v,\n want %#v", x, p3.Int, 1) + } +} + +type Tables struct { + HTable string `xml:"http://www.w3.org/TR/html4/ table"` + FTable string `xml:"http://www.w3schools.com/furniture table"` +} + +var tables = []struct { + xml string + tab Tables + ns string +}{ + { + xml: `` + + `hello
` + + `world
` + + ``, + tab: Tables{"hello", "world"}, + }, + { + xml: `` + + `world
` + + `hello
` + + ``, + tab: Tables{"hello", "world"}, + }, + { + xml: `` + + `world` + + `hello` + + ``, + tab: Tables{"hello", "world"}, + }, + { + xml: `` + + `bogus
` + + ``, + tab: Tables{}, + }, + { + xml: `` + + `only
` + + ``, + tab: Tables{HTable: "only"}, + ns: "http://www.w3.org/TR/html4/", + }, + { + xml: `` + + `only
` + + ``, + tab: Tables{FTable: "only"}, + ns: "http://www.w3schools.com/furniture", + }, + { + xml: `` + + `only
` + + ``, + tab: Tables{}, + ns: "something else entirely", + }, +} + +func TestUnmarshalNS(t *testing.T) { + for i, tt := range tables { + var dst Tables + var err error + if tt.ns != "" { + d := NewDecoder(strings.NewReader(tt.xml)) + d.DefaultSpace = tt.ns + err = d.Decode(&dst) + } else { + err = Unmarshal([]byte(tt.xml), &dst) + } + if err != nil { + t.Errorf("#%d: Unmarshal: %v", i, err) + continue + } + want := tt.tab + if dst != want { + t.Errorf("#%d: dst=%+v, want %+v", i, dst, want) + } + } +} + +func TestRoundTrip(t *testing.T) { + // From issue 7535 + const s = `` + in := bytes.NewBufferString(s) + for i := 0; i < 10; i++ { + out := &bytes.Buffer{} + d := NewDecoder(in) + e := NewEncoder(out) + + for { + t, err := d.Token() + if err == io.EOF { + break + } + if err != nil { + fmt.Println("failed:", err) + return + } + e.EncodeToken(t) + } + e.Flush() + in = out + } + if got := in.String(); got != s { + t.Errorf("have: %q\nwant: %q\n", got, s) + } +} + +func TestMarshalNS(t *testing.T) { + dst := Tables{"hello", "world"} + data, err := Marshal(&dst) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + want := `hello
world
` + str := string(data) + if str != want { + t.Errorf("have: %q\nwant: %q\n", str, want) + } +} + +type TableAttrs struct { + TAttr TAttr +} + +type TAttr struct { + HTable string `xml:"http://www.w3.org/TR/html4/ table,attr"` + FTable string `xml:"http://www.w3schools.com/furniture table,attr"` + Lang string `xml:"http://www.w3.org/XML/1998/namespace lang,attr,omitempty"` + Other1 string `xml:"http://golang.org/xml/ other,attr,omitempty"` + Other2 string `xml:"http://golang.org/xmlfoo/ other,attr,omitempty"` + Other3 string `xml:"http://golang.org/json/ other,attr,omitempty"` + Other4 string `xml:"http://golang.org/2/json/ other,attr,omitempty"` +} + +var tableAttrs = []struct { + xml string + tab TableAttrs + ns string +}{ + { + xml: ``, + tab: TableAttrs{TAttr{HTable: "hello", FTable: "world"}}, + }, + { + xml: ``, + tab: TableAttrs{TAttr{HTable: "hello", FTable: "world"}}, + }, + { + xml: ``, + tab: TableAttrs{TAttr{HTable: "hello", FTable: "world"}}, + }, + { + // Default space does not apply to attribute names. + xml: ``, + tab: TableAttrs{TAttr{HTable: "hello", FTable: ""}}, + }, + { + // Default space does not apply to attribute names. + xml: ``, + tab: TableAttrs{TAttr{HTable: "", FTable: "world"}}, + }, + { + xml: ``, + tab: TableAttrs{}, + }, + { + // Default space does not apply to attribute names. + xml: ``, + tab: TableAttrs{TAttr{HTable: "hello", FTable: ""}}, + ns: "http://www.w3schools.com/furniture", + }, + { + // Default space does not apply to attribute names. + xml: ``, + tab: TableAttrs{TAttr{HTable: "", FTable: "world"}}, + ns: "http://www.w3.org/TR/html4/", + }, + { + xml: ``, + tab: TableAttrs{}, + ns: "something else entirely", + }, +} + +func TestUnmarshalNSAttr(t *testing.T) { + for i, tt := range tableAttrs { + var dst TableAttrs + var err error + if tt.ns != "" { + d := NewDecoder(strings.NewReader(tt.xml)) + d.DefaultSpace = tt.ns + err = d.Decode(&dst) + } else { + err = Unmarshal([]byte(tt.xml), &dst) + } + if err != nil { + t.Errorf("#%d: Unmarshal: %v", i, err) + continue + } + want := tt.tab + if dst != want { + t.Errorf("#%d: dst=%+v, want %+v", i, dst, want) + } + } +} + +func TestMarshalNSAttr(t *testing.T) { + src := TableAttrs{TAttr{"hello", "world", "en_US", "other1", "other2", "other3", "other4"}} + data, err := Marshal(&src) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + want := `` + str := string(data) + if str != want { + t.Errorf("Marshal:\nhave: %#q\nwant: %#q\n", str, want) + } + + var dst TableAttrs + if err := Unmarshal(data, &dst); err != nil { + t.Errorf("Unmarshal: %v", err) + } + + if dst != src { + t.Errorf("Unmarshal = %q, want %q", dst, src) + } +} + +type MyCharData struct { + body string +} + +func (m *MyCharData) UnmarshalXML(d *Decoder, start StartElement) error { + for { + t, err := d.Token() + if err == io.EOF { // found end of element + break + } + if err != nil { + return err + } + if char, ok := t.(CharData); ok { + m.body += string(char) + } + } + return nil +} + +var _ Unmarshaler = (*MyCharData)(nil) + +func (m *MyCharData) UnmarshalXMLAttr(attr Attr) error { + panic("must not call") +} + +type MyAttr struct { + attr string +} + +func (m *MyAttr) UnmarshalXMLAttr(attr Attr) error { + m.attr = attr.Value + return nil +} + +var _ UnmarshalerAttr = (*MyAttr)(nil) + +type MyStruct struct { + Data *MyCharData + Attr *MyAttr `xml:",attr"` + + Data2 MyCharData + Attr2 MyAttr `xml:",attr"` +} + +func TestUnmarshaler(t *testing.T) { + xml := ` + + hello world + howdy world + + ` + + var m MyStruct + if err := Unmarshal([]byte(xml), &m); err != nil { + t.Fatal(err) + } + + if m.Data == nil || m.Attr == nil || m.Data.body != "hello world" || m.Attr.attr != "attr1" || m.Data2.body != "howdy world" || m.Attr2.attr != "attr2" { + t.Errorf("m=%#+v\n", m) + } +} + +type Pea struct { + Cotelydon string +} + +type Pod struct { + Pea interface{} `xml:"Pea"` +} + +// https://golang.org/issue/6836 +func TestUnmarshalIntoInterface(t *testing.T) { + pod := new(Pod) + pod.Pea = new(Pea) + xml := `Green stuff` + err := Unmarshal([]byte(xml), pod) + if err != nil { + t.Fatalf("failed to unmarshal %q: %v", xml, err) + } + pea, ok := pod.Pea.(*Pea) + if !ok { + t.Fatalf("unmarshalled into wrong type: have %T want *Pea", pod.Pea) + } + have, want := pea.Cotelydon, "Green stuff" + if have != want { + t.Errorf("failed to unmarshal into interface, have %q want %q", have, want) + } +} diff --git a/endpoints/drive/webdav/internal/xml/typeinfo.go b/endpoints/drive/webdav/internal/xml/typeinfo.go new file mode 100644 index 000000000..fdde288bc --- /dev/null +++ b/endpoints/drive/webdav/internal/xml/typeinfo.go @@ -0,0 +1,371 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xml + +import ( + "fmt" + "reflect" + "strings" + "sync" +) + +// typeInfo holds details for the xml representation of a type. +type typeInfo struct { + xmlname *fieldInfo + fields []fieldInfo +} + +// fieldInfo holds details for the xml representation of a single field. +type fieldInfo struct { + idx []int + name string + xmlns string + flags fieldFlags + parents []string +} + +type fieldFlags int + +const ( + fElement fieldFlags = 1 << iota + fAttr + fCharData + fInnerXml + fComment + fAny + + fOmitEmpty + + fMode = fElement | fAttr | fCharData | fInnerXml | fComment | fAny +) + +var tinfoMap = make(map[reflect.Type]*typeInfo) +var tinfoLock sync.RWMutex + +var nameType = reflect.TypeOf(Name{}) + +// getTypeInfo returns the typeInfo structure with details necessary +// for marshalling and unmarshalling typ. +func getTypeInfo(typ reflect.Type) (*typeInfo, error) { + tinfoLock.RLock() + tinfo, ok := tinfoMap[typ] + tinfoLock.RUnlock() + if ok { + return tinfo, nil + } + tinfo = &typeInfo{} + if typ.Kind() == reflect.Struct && typ != nameType { + n := typ.NumField() + for i := 0; i < n; i++ { + f := typ.Field(i) + if f.PkgPath != "" || f.Tag.Get("xml") == "-" { + continue // Private field + } + + // For embedded structs, embed its fields. + if f.Anonymous { + t := f.Type + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t.Kind() == reflect.Struct { + inner, err := getTypeInfo(t) + if err != nil { + return nil, err + } + if tinfo.xmlname == nil { + tinfo.xmlname = inner.xmlname + } + for _, finfo := range inner.fields { + finfo.idx = append([]int{i}, finfo.idx...) + if err := addFieldInfo(typ, tinfo, &finfo); err != nil { + return nil, err + } + } + continue + } + } + + finfo, err := structFieldInfo(typ, &f) + if err != nil { + return nil, err + } + + if f.Name == "XMLName" { + tinfo.xmlname = finfo + continue + } + + // Add the field if it doesn't conflict with other fields. + if err := addFieldInfo(typ, tinfo, finfo); err != nil { + return nil, err + } + } + } + tinfoLock.Lock() + tinfoMap[typ] = tinfo + tinfoLock.Unlock() + return tinfo, nil +} + +// structFieldInfo builds and returns a fieldInfo for f. +func structFieldInfo(typ reflect.Type, f *reflect.StructField) (*fieldInfo, error) { + finfo := &fieldInfo{idx: f.Index} + + // Split the tag from the xml namespace if necessary. + tag := f.Tag.Get("xml") + if i := strings.Index(tag, " "); i >= 0 { + finfo.xmlns, tag = tag[:i], tag[i+1:] + } + + // Parse flags. + tokens := strings.Split(tag, ",") + if len(tokens) == 1 { + finfo.flags = fElement + } else { + tag = tokens[0] + for _, flag := range tokens[1:] { + switch flag { + case "attr": + finfo.flags |= fAttr + case "chardata": + finfo.flags |= fCharData + case "innerxml": + finfo.flags |= fInnerXml + case "comment": + finfo.flags |= fComment + case "any": + finfo.flags |= fAny + case "omitempty": + finfo.flags |= fOmitEmpty + } + } + + // Validate the flags used. + valid := true + switch mode := finfo.flags & fMode; mode { + case 0: + finfo.flags |= fElement + case fAttr, fCharData, fInnerXml, fComment, fAny: + if f.Name == "XMLName" || tag != "" && mode != fAttr { + valid = false + } + default: + // This will also catch multiple modes in a single field. + valid = false + } + if finfo.flags&fMode == fAny { + finfo.flags |= fElement + } + if finfo.flags&fOmitEmpty != 0 && finfo.flags&(fElement|fAttr) == 0 { + valid = false + } + if !valid { + return nil, fmt.Errorf("xml: invalid tag in field %s of type %s: %q", + f.Name, typ, f.Tag.Get("xml")) + } + } + + // Use of xmlns without a name is not allowed. + if finfo.xmlns != "" && tag == "" { + return nil, fmt.Errorf("xml: namespace without name in field %s of type %s: %q", + f.Name, typ, f.Tag.Get("xml")) + } + + if f.Name == "XMLName" { + // The XMLName field records the XML element name. Don't + // process it as usual because its name should default to + // empty rather than to the field name. + finfo.name = tag + return finfo, nil + } + + if tag == "" { + // If the name part of the tag is completely empty, get + // default from XMLName of underlying struct if feasible, + // or field name otherwise. + if xmlname := lookupXMLName(f.Type); xmlname != nil { + finfo.xmlns, finfo.name = xmlname.xmlns, xmlname.name + } else { + finfo.name = f.Name + } + return finfo, nil + } + + if finfo.xmlns == "" && finfo.flags&fAttr == 0 { + // If it's an element no namespace specified, get the default + // from the XMLName of enclosing struct if possible. + if xmlname := lookupXMLName(typ); xmlname != nil { + finfo.xmlns = xmlname.xmlns + } + } + + // Prepare field name and parents. + parents := strings.Split(tag, ">") + if parents[0] == "" { + parents[0] = f.Name + } + if parents[len(parents)-1] == "" { + return nil, fmt.Errorf("xml: trailing '>' in field %s of type %s", f.Name, typ) + } + finfo.name = parents[len(parents)-1] + if len(parents) > 1 { + if (finfo.flags & fElement) == 0 { + return nil, fmt.Errorf("xml: %s chain not valid with %s flag", tag, strings.Join(tokens[1:], ",")) + } + finfo.parents = parents[:len(parents)-1] + } + + // If the field type has an XMLName field, the names must match + // so that the behavior of both marshalling and unmarshalling + // is straightforward and unambiguous. + if finfo.flags&fElement != 0 { + ftyp := f.Type + xmlname := lookupXMLName(ftyp) + if xmlname != nil && xmlname.name != finfo.name { + return nil, fmt.Errorf("xml: name %q in tag of %s.%s conflicts with name %q in %s.XMLName", + finfo.name, typ, f.Name, xmlname.name, ftyp) + } + } + return finfo, nil +} + +// lookupXMLName returns the fieldInfo for typ's XMLName field +// in case it exists and has a valid xml field tag, otherwise +// it returns nil. +func lookupXMLName(typ reflect.Type) (xmlname *fieldInfo) { + for typ.Kind() == reflect.Ptr { + typ = typ.Elem() + } + if typ.Kind() != reflect.Struct { + return nil + } + for i, n := 0, typ.NumField(); i < n; i++ { + f := typ.Field(i) + if f.Name != "XMLName" { + continue + } + finfo, err := structFieldInfo(typ, &f) + if finfo.name != "" && err == nil { + return finfo + } + // Also consider errors as a non-existent field tag + // and let getTypeInfo itself report the error. + break + } + return nil +} + +func min(a, b int) int { + if a <= b { + return a + } + return b +} + +// addFieldInfo adds finfo to tinfo.fields if there are no +// conflicts, or if conflicts arise from previous fields that were +// obtained from deeper embedded structures than finfo. In the latter +// case, the conflicting entries are dropped. +// A conflict occurs when the path (parent + name) to a field is +// itself a prefix of another path, or when two paths match exactly. +// It is okay for field paths to share a common, shorter prefix. +func addFieldInfo(typ reflect.Type, tinfo *typeInfo, newf *fieldInfo) error { + var conflicts []int +Loop: + // First, figure all conflicts. Most working code will have none. + for i := range tinfo.fields { + oldf := &tinfo.fields[i] + if oldf.flags&fMode != newf.flags&fMode { + continue + } + if oldf.xmlns != "" && newf.xmlns != "" && oldf.xmlns != newf.xmlns { + continue + } + minl := min(len(newf.parents), len(oldf.parents)) + for p := 0; p < minl; p++ { + if oldf.parents[p] != newf.parents[p] { + continue Loop + } + } + if len(oldf.parents) > len(newf.parents) { + if oldf.parents[len(newf.parents)] == newf.name { + conflicts = append(conflicts, i) + } + } else if len(oldf.parents) < len(newf.parents) { + if newf.parents[len(oldf.parents)] == oldf.name { + conflicts = append(conflicts, i) + } + } else { + if newf.name == oldf.name { + conflicts = append(conflicts, i) + } + } + } + // Without conflicts, add the new field and return. + if conflicts == nil { + tinfo.fields = append(tinfo.fields, *newf) + return nil + } + + // If any conflict is shallower, ignore the new field. + // This matches the Go field resolution on embedding. + for _, i := range conflicts { + if len(tinfo.fields[i].idx) < len(newf.idx) { + return nil + } + } + + // Otherwise, if any of them is at the same depth level, it's an error. + for _, i := range conflicts { + oldf := &tinfo.fields[i] + if len(oldf.idx) == len(newf.idx) { + f1 := typ.FieldByIndex(oldf.idx) + f2 := typ.FieldByIndex(newf.idx) + return &TagPathError{typ, f1.Name, f1.Tag.Get("xml"), f2.Name, f2.Tag.Get("xml")} + } + } + + // Otherwise, the new field is shallower, and thus takes precedence, + // so drop the conflicting fields from tinfo and append the new one. + for c := len(conflicts) - 1; c >= 0; c-- { + i := conflicts[c] + copy(tinfo.fields[i:], tinfo.fields[i+1:]) + tinfo.fields = tinfo.fields[:len(tinfo.fields)-1] + } + tinfo.fields = append(tinfo.fields, *newf) + return nil +} + +// A TagPathError represents an error in the unmarshalling process +// caused by the use of field tags with conflicting paths. +type TagPathError struct { + Struct reflect.Type + Field1, Tag1 string + Field2, Tag2 string +} + +func (e *TagPathError) Error() string { + return fmt.Sprintf("%s field %q with tag %q conflicts with field %q with tag %q", e.Struct, e.Field1, e.Tag1, e.Field2, e.Tag2) +} + +// value returns v's field value corresponding to finfo. +// It's equivalent to v.FieldByIndex(finfo.idx), but initializes +// and dereferences pointers as necessary. +func (finfo *fieldInfo) value(v reflect.Value) reflect.Value { + for i, x := range finfo.idx { + if i > 0 { + t := v.Type() + if t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct { + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + v = v.Elem() + } + } + v = v.Field(x) + } + return v +} diff --git a/endpoints/drive/webdav/internal/xml/xml.go b/endpoints/drive/webdav/internal/xml/xml.go new file mode 100644 index 000000000..7d88dac7b --- /dev/null +++ b/endpoints/drive/webdav/internal/xml/xml.go @@ -0,0 +1,1998 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package xml implements a simple XML 1.0 parser that +// understands XML name spaces. +package xml + +// References: +// Annotated XML spec: http://www.xml.com/axml/testaxml.htm +// XML name spaces: http://www.w3.org/TR/REC-xml-names/ + +// TODO(rsc): +// Test error handling. + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +// A SyntaxError represents a syntax error in the XML input stream. +type SyntaxError struct { + Msg string + Line int +} + +func (e *SyntaxError) Error() string { + return "XML syntax error on line " + strconv.Itoa(e.Line) + ": " + e.Msg +} + +// A Name represents an XML name (Local) annotated with a name space +// identifier (Space). In tokens returned by Decoder.Token, the Space +// identifier is given as a canonical URL, not the short prefix used in +// the document being parsed. +// +// As a special case, XML namespace declarations will use the literal +// string "xmlns" for the Space field instead of the fully resolved URL. +// See Encoder.EncodeToken for more information on namespace encoding +// behaviour. +type Name struct { + Space, Local string +} + +// isNamespace reports whether the name is a namespace-defining name. +func (name Name) isNamespace() bool { + return name.Local == "xmlns" || name.Space == "xmlns" +} + +// An Attr represents an attribute in an XML element (Name=Value). +type Attr struct { + Name Name + Value string +} + +// A Token is an interface holding one of the token types: +// StartElement, EndElement, CharData, Comment, ProcInst, or Directive. +type Token interface{} + +// A StartElement represents an XML start element. +type StartElement struct { + Name Name + Attr []Attr +} + +func (e StartElement) Copy() StartElement { + attrs := make([]Attr, len(e.Attr)) + copy(attrs, e.Attr) + e.Attr = attrs + return e +} + +// End returns the corresponding XML end element. +func (e StartElement) End() EndElement { + return EndElement{e.Name} +} + +// setDefaultNamespace sets the namespace of the element +// as the default for all elements contained within it. +func (e *StartElement) setDefaultNamespace() { + if e.Name.Space == "" { + // If there's no namespace on the element, don't + // set the default. Strictly speaking this might be wrong, as + // we can't tell if the element had no namespace set + // or was just using the default namespace. + return + } + // Don't add a default name space if there's already one set. + for _, attr := range e.Attr { + if attr.Name.Space == "" && attr.Name.Local == "xmlns" { + return + } + } + e.Attr = append(e.Attr, Attr{ + Name: Name{ + Local: "xmlns", + }, + Value: e.Name.Space, + }) +} + +// An EndElement represents an XML end element. +type EndElement struct { + Name Name +} + +// A CharData represents XML character data (raw text), +// in which XML escape sequences have been replaced by +// the characters they represent. +type CharData []byte + +func makeCopy(b []byte) []byte { + b1 := make([]byte, len(b)) + copy(b1, b) + return b1 +} + +func (c CharData) Copy() CharData { return CharData(makeCopy(c)) } + +// A Comment represents an XML comment of the form . +// The bytes do not include the comment markers. +type Comment []byte + +func (c Comment) Copy() Comment { return Comment(makeCopy(c)) } + +// A ProcInst represents an XML processing instruction of the form +type ProcInst struct { + Target string + Inst []byte +} + +func (p ProcInst) Copy() ProcInst { + p.Inst = makeCopy(p.Inst) + return p +} + +// A Directive represents an XML directive of the form . +// The bytes do not include the markers. +type Directive []byte + +func (d Directive) Copy() Directive { return Directive(makeCopy(d)) } + +// CopyToken returns a copy of a Token. +func CopyToken(t Token) Token { + switch v := t.(type) { + case CharData: + return v.Copy() + case Comment: + return v.Copy() + case Directive: + return v.Copy() + case ProcInst: + return v.Copy() + case StartElement: + return v.Copy() + } + return t +} + +// A Decoder represents an XML parser reading a particular input stream. +// The parser assumes that its input is encoded in UTF-8. +type Decoder struct { + // Strict defaults to true, enforcing the requirements + // of the XML specification. + // If set to false, the parser allows input containing common + // mistakes: + // * If an element is missing an end tag, the parser invents + // end tags as necessary to keep the return values from Token + // properly balanced. + // * In attribute values and character data, unknown or malformed + // character entities (sequences beginning with &) are left alone. + // + // Setting: + // + // d.Strict = false; + // d.AutoClose = HTMLAutoClose; + // d.Entity = HTMLEntity + // + // creates a parser that can handle typical HTML. + // + // Strict mode does not enforce the requirements of the XML name spaces TR. + // In particular it does not reject name space tags using undefined prefixes. + // Such tags are recorded with the unknown prefix as the name space URL. + Strict bool + + // When Strict == false, AutoClose indicates a set of elements to + // consider closed immediately after they are opened, regardless + // of whether an end element is present. + AutoClose []string + + // Entity can be used to map non-standard entity names to string replacements. + // The parser behaves as if these standard mappings are present in the map, + // regardless of the actual map content: + // + // "lt": "<", + // "gt": ">", + // "amp": "&", + // "apos": "'", + // "quot": `"`, + Entity map[string]string + + // CharsetReader, if non-nil, defines a function to generate + // charset-conversion readers, converting from the provided + // non-UTF-8 charset into UTF-8. If CharsetReader is nil or + // returns an error, parsing stops with an error. One of the + // the CharsetReader's result values must be non-nil. + CharsetReader func(charset string, input io.Reader) (io.Reader, error) + + // DefaultSpace sets the default name space used for unadorned tags, + // as if the entire XML stream were wrapped in an element containing + // the attribute xmlns="DefaultSpace". + DefaultSpace string + + r io.ByteReader + buf bytes.Buffer + saved *bytes.Buffer + stk *stack + free *stack + needClose bool + toClose Name + nextToken Token + nextByte int + ns map[string]string + err error + line int + offset int64 + unmarshalDepth int +} + +// NewDecoder creates a new XML parser reading from r. +// If r does not implement io.ByteReader, NewDecoder will +// do its own buffering. +func NewDecoder(r io.Reader) *Decoder { + d := &Decoder{ + ns: make(map[string]string), + nextByte: -1, + line: 1, + Strict: true, + } + d.switchToReader(r) + return d +} + +// Token returns the next XML token in the input stream. +// At the end of the input stream, Token returns nil, io.EOF. +// +// Slices of bytes in the returned token data refer to the +// parser's internal buffer and remain valid only until the next +// call to Token. To acquire a copy of the bytes, call CopyToken +// or the token's Copy method. +// +// Token expands self-closing elements such as
+// into separate start and end elements returned by successive calls. +// +// Token guarantees that the StartElement and EndElement +// tokens it returns are properly nested and matched: +// if Token encounters an unexpected end element, +// it will return an error. +// +// Token implements XML name spaces as described by +// http://www.w3.org/TR/REC-xml-names/. Each of the +// Name structures contained in the Token has the Space +// set to the URL identifying its name space when known. +// If Token encounters an unrecognized name space prefix, +// it uses the prefix as the Space rather than report an error. +func (d *Decoder) Token() (t Token, err error) { + if d.stk != nil && d.stk.kind == stkEOF { + err = io.EOF + return + } + if d.nextToken != nil { + t = d.nextToken + d.nextToken = nil + } else if t, err = d.rawToken(); err != nil { + return + } + + if !d.Strict { + if t1, ok := d.autoClose(t); ok { + d.nextToken = t + t = t1 + } + } + switch t1 := t.(type) { + case StartElement: + // In XML name spaces, the translations listed in the + // attributes apply to the element name and + // to the other attribute names, so process + // the translations first. + for _, a := range t1.Attr { + if a.Name.Space == "xmlns" { + v, ok := d.ns[a.Name.Local] + d.pushNs(a.Name.Local, v, ok) + d.ns[a.Name.Local] = a.Value + } + if a.Name.Space == "" && a.Name.Local == "xmlns" { + // Default space for untagged names + v, ok := d.ns[""] + d.pushNs("", v, ok) + d.ns[""] = a.Value + } + } + + d.translate(&t1.Name, true) + for i := range t1.Attr { + d.translate(&t1.Attr[i].Name, false) + } + d.pushElement(t1.Name) + t = t1 + + case EndElement: + d.translate(&t1.Name, true) + if !d.popElement(&t1) { + return nil, d.err + } + t = t1 + } + return +} + +const xmlURL = "http://www.w3.org/XML/1998/namespace" + +// Apply name space translation to name n. +// The default name space (for Space=="") +// applies only to element names, not to attribute names. +func (d *Decoder) translate(n *Name, isElementName bool) { + switch { + case n.Space == "xmlns": + return + case n.Space == "" && !isElementName: + return + case n.Space == "xml": + n.Space = xmlURL + case n.Space == "" && n.Local == "xmlns": + return + } + if v, ok := d.ns[n.Space]; ok { + n.Space = v + } else if n.Space == "" { + n.Space = d.DefaultSpace + } +} + +func (d *Decoder) switchToReader(r io.Reader) { + // Get efficient byte at a time reader. + // Assume that if reader has its own + // ReadByte, it's efficient enough. + // Otherwise, use bufio. + if rb, ok := r.(io.ByteReader); ok { + d.r = rb + } else { + d.r = bufio.NewReader(r) + } +} + +// Parsing state - stack holds old name space translations +// and the current set of open elements. The translations to pop when +// ending a given tag are *below* it on the stack, which is +// more work but forced on us by XML. +type stack struct { + next *stack + kind int + name Name + ok bool +} + +const ( + stkStart = iota + stkNs + stkEOF +) + +func (d *Decoder) push(kind int) *stack { + s := d.free + if s != nil { + d.free = s.next + } else { + s = new(stack) + } + s.next = d.stk + s.kind = kind + d.stk = s + return s +} + +func (d *Decoder) pop() *stack { + s := d.stk + if s != nil { + d.stk = s.next + s.next = d.free + d.free = s + } + return s +} + +// Record that after the current element is finished +// (that element is already pushed on the stack) +// Token should return EOF until popEOF is called. +func (d *Decoder) pushEOF() { + // Walk down stack to find Start. + // It might not be the top, because there might be stkNs + // entries above it. + start := d.stk + for start.kind != stkStart { + start = start.next + } + // The stkNs entries below a start are associated with that + // element too; skip over them. + for start.next != nil && start.next.kind == stkNs { + start = start.next + } + s := d.free + if s != nil { + d.free = s.next + } else { + s = new(stack) + } + s.kind = stkEOF + s.next = start.next + start.next = s +} + +// Undo a pushEOF. +// The element must have been finished, so the EOF should be at the top of the stack. +func (d *Decoder) popEOF() bool { + if d.stk == nil || d.stk.kind != stkEOF { + return false + } + d.pop() + return true +} + +// Record that we are starting an element with the given name. +func (d *Decoder) pushElement(name Name) { + s := d.push(stkStart) + s.name = name +} + +// Record that we are changing the value of ns[local]. +// The old value is url, ok. +func (d *Decoder) pushNs(local string, url string, ok bool) { + s := d.push(stkNs) + s.name.Local = local + s.name.Space = url + s.ok = ok +} + +// Creates a SyntaxError with the current line number. +func (d *Decoder) syntaxError(msg string) error { + return &SyntaxError{Msg: msg, Line: d.line} +} + +// Record that we are ending an element with the given name. +// The name must match the record at the top of the stack, +// which must be a pushElement record. +// After popping the element, apply any undo records from +// the stack to restore the name translations that existed +// before we saw this element. +func (d *Decoder) popElement(t *EndElement) bool { + s := d.pop() + name := t.Name + switch { + case s == nil || s.kind != stkStart: + d.err = d.syntaxError("unexpected end element ") + return false + case s.name.Local != name.Local: + if !d.Strict { + d.needClose = true + d.toClose = t.Name + t.Name = s.name + return true + } + d.err = d.syntaxError("element <" + s.name.Local + "> closed by ") + return false + case s.name.Space != name.Space: + d.err = d.syntaxError("element <" + s.name.Local + "> in space " + s.name.Space + + "closed by in space " + name.Space) + return false + } + + // Pop stack until a Start or EOF is on the top, undoing the + // translations that were associated with the element we just closed. + for d.stk != nil && d.stk.kind != stkStart && d.stk.kind != stkEOF { + s := d.pop() + if s.ok { + d.ns[s.name.Local] = s.name.Space + } else { + delete(d.ns, s.name.Local) + } + } + + return true +} + +// If the top element on the stack is autoclosing and +// t is not the end tag, invent the end tag. +func (d *Decoder) autoClose(t Token) (Token, bool) { + if d.stk == nil || d.stk.kind != stkStart { + return nil, false + } + name := strings.ToLower(d.stk.name.Local) + for _, s := range d.AutoClose { + if strings.ToLower(s) == name { + // This one should be auto closed if t doesn't close it. + et, ok := t.(EndElement) + if !ok || et.Name.Local != name { + return EndElement{d.stk.name}, true + } + break + } + } + return nil, false +} + +var errRawToken = errors.New("xml: cannot use RawToken from UnmarshalXML method") + +// RawToken is like Token but does not verify that +// start and end elements match and does not translate +// name space prefixes to their corresponding URLs. +func (d *Decoder) RawToken() (Token, error) { + if d.unmarshalDepth > 0 { + return nil, errRawToken + } + return d.rawToken() +} + +func (d *Decoder) rawToken() (Token, error) { + if d.err != nil { + return nil, d.err + } + if d.needClose { + // The last element we read was self-closing and + // we returned just the StartElement half. + // Return the EndElement half now. + d.needClose = false + return EndElement{d.toClose}, nil + } + + b, ok := d.getc() + if !ok { + return nil, d.err + } + + if b != '<' { + // Text section. + d.ungetc(b) + data := d.text(-1, false) + if data == nil { + return nil, d.err + } + return CharData(data), nil + } + + if b, ok = d.mustgetc(); !ok { + return nil, d.err + } + switch b { + case '/': + // ' { + d.err = d.syntaxError("invalid characters between ") + return nil, d.err + } + return EndElement{name}, nil + + case '?': + // ' { + break + } + b0 = b + } + data := d.buf.Bytes() + data = data[0 : len(data)-2] // chop ?> + + if target == "xml" { + content := string(data) + ver := procInst("version", content) + if ver != "" && ver != "1.0" { + d.err = fmt.Errorf("xml: unsupported version %q; only version 1.0 is supported", ver) + return nil, d.err + } + enc := procInst("encoding", content) + if enc != "" && enc != "utf-8" && enc != "UTF-8" { + if d.CharsetReader == nil { + d.err = fmt.Errorf("xml: encoding %q declared but Decoder.CharsetReader is nil", enc) + return nil, d.err + } + newr, err := d.CharsetReader(enc, d.r.(io.Reader)) + if err != nil { + d.err = fmt.Errorf("xml: opening charset %q: %v", enc, err) + return nil, d.err + } + if newr == nil { + panic("CharsetReader returned a nil Reader for charset " + enc) + } + d.switchToReader(newr) + } + } + return ProcInst{target, data}, nil + + case '!': + // ' { + break + } + b0, b1 = b1, b + } + data := d.buf.Bytes() + data = data[0 : len(data)-3] // chop --> + return Comment(data), nil + + case '[': // . + data := d.text(-1, true) + if data == nil { + return nil, d.err + } + return CharData(data), nil + } + + // Probably a directive: , , etc. + // We don't care, but accumulate for caller. Quoted angle + // brackets do not count for nesting. + d.buf.Reset() + d.buf.WriteByte(b) + inquote := uint8(0) + depth := 0 + for { + if b, ok = d.mustgetc(); !ok { + return nil, d.err + } + if inquote == 0 && b == '>' && depth == 0 { + break + } + HandleB: + d.buf.WriteByte(b) + switch { + case b == inquote: + inquote = 0 + + case inquote != 0: + // in quotes, no special action + + case b == '\'' || b == '"': + inquote = b + + case b == '>' && inquote == 0: + depth-- + + case b == '<' && inquote == 0: + // Look for ` + +var testEntity = map[string]string{"何": "What", "is-it": "is it?"} + +var rawTokens = []Token{ + CharData("\n"), + ProcInst{"xml", []byte(`version="1.0" encoding="UTF-8"`)}, + CharData("\n"), + Directive(`DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"`), + CharData("\n"), + StartElement{Name{"", "body"}, []Attr{{Name{"xmlns", "foo"}, "ns1"}, {Name{"", "xmlns"}, "ns2"}, {Name{"xmlns", "tag"}, "ns3"}}}, + CharData("\n "), + StartElement{Name{"", "hello"}, []Attr{{Name{"", "lang"}, "en"}}}, + CharData("World <>'\" 白鵬翔"), + EndElement{Name{"", "hello"}}, + CharData("\n "), + StartElement{Name{"", "query"}, []Attr{}}, + CharData("What is it?"), + EndElement{Name{"", "query"}}, + CharData("\n "), + StartElement{Name{"", "goodbye"}, []Attr{}}, + EndElement{Name{"", "goodbye"}}, + CharData("\n "), + StartElement{Name{"", "outer"}, []Attr{{Name{"foo", "attr"}, "value"}, {Name{"xmlns", "tag"}, "ns4"}}}, + CharData("\n "), + StartElement{Name{"", "inner"}, []Attr{}}, + EndElement{Name{"", "inner"}}, + CharData("\n "), + EndElement{Name{"", "outer"}}, + CharData("\n "), + StartElement{Name{"tag", "name"}, []Attr{}}, + CharData("\n "), + CharData("Some text here."), + CharData("\n "), + EndElement{Name{"tag", "name"}}, + CharData("\n"), + EndElement{Name{"", "body"}}, + Comment(" missing final newline "), +} + +var cookedTokens = []Token{ + CharData("\n"), + ProcInst{"xml", []byte(`version="1.0" encoding="UTF-8"`)}, + CharData("\n"), + Directive(`DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"`), + CharData("\n"), + StartElement{Name{"ns2", "body"}, []Attr{{Name{"xmlns", "foo"}, "ns1"}, {Name{"", "xmlns"}, "ns2"}, {Name{"xmlns", "tag"}, "ns3"}}}, + CharData("\n "), + StartElement{Name{"ns2", "hello"}, []Attr{{Name{"", "lang"}, "en"}}}, + CharData("World <>'\" 白鵬翔"), + EndElement{Name{"ns2", "hello"}}, + CharData("\n "), + StartElement{Name{"ns2", "query"}, []Attr{}}, + CharData("What is it?"), + EndElement{Name{"ns2", "query"}}, + CharData("\n "), + StartElement{Name{"ns2", "goodbye"}, []Attr{}}, + EndElement{Name{"ns2", "goodbye"}}, + CharData("\n "), + StartElement{Name{"ns2", "outer"}, []Attr{{Name{"ns1", "attr"}, "value"}, {Name{"xmlns", "tag"}, "ns4"}}}, + CharData("\n "), + StartElement{Name{"ns2", "inner"}, []Attr{}}, + EndElement{Name{"ns2", "inner"}}, + CharData("\n "), + EndElement{Name{"ns2", "outer"}}, + CharData("\n "), + StartElement{Name{"ns3", "name"}, []Attr{}}, + CharData("\n "), + CharData("Some text here."), + CharData("\n "), + EndElement{Name{"ns3", "name"}}, + CharData("\n"), + EndElement{Name{"ns2", "body"}}, + Comment(" missing final newline "), +} + +const testInputAltEncoding = ` + +VALUE` + +var rawTokensAltEncoding = []Token{ + CharData("\n"), + ProcInst{"xml", []byte(`version="1.0" encoding="x-testing-uppercase"`)}, + CharData("\n"), + StartElement{Name{"", "tag"}, []Attr{}}, + CharData("value"), + EndElement{Name{"", "tag"}}, +} + +var xmlInput = []string{ + // unexpected EOF cases + "<", + "", + "", + "", + // "", // let the Token() caller handle + "", + "", + "", + "", + " c;", + "", + "", + "", + // "", // let the Token() caller handle + "", + "", + "cdata]]>", +} + +func TestRawToken(t *testing.T) { + d := NewDecoder(strings.NewReader(testInput)) + d.Entity = testEntity + testRawToken(t, d, testInput, rawTokens) +} + +const nonStrictInput = ` +non&entity +&unknown;entity +{ +&#zzz; +&なまえ3; +<-gt; +&; +&0a; +` + +var nonStringEntity = map[string]string{"": "oops!", "0a": "oops!"} + +var nonStrictTokens = []Token{ + CharData("\n"), + StartElement{Name{"", "tag"}, []Attr{}}, + CharData("non&entity"), + EndElement{Name{"", "tag"}}, + CharData("\n"), + StartElement{Name{"", "tag"}, []Attr{}}, + CharData("&unknown;entity"), + EndElement{Name{"", "tag"}}, + CharData("\n"), + StartElement{Name{"", "tag"}, []Attr{}}, + CharData("{"), + EndElement{Name{"", "tag"}}, + CharData("\n"), + StartElement{Name{"", "tag"}, []Attr{}}, + CharData("&#zzz;"), + EndElement{Name{"", "tag"}}, + CharData("\n"), + StartElement{Name{"", "tag"}, []Attr{}}, + CharData("&なまえ3;"), + EndElement{Name{"", "tag"}}, + CharData("\n"), + StartElement{Name{"", "tag"}, []Attr{}}, + CharData("<-gt;"), + EndElement{Name{"", "tag"}}, + CharData("\n"), + StartElement{Name{"", "tag"}, []Attr{}}, + CharData("&;"), + EndElement{Name{"", "tag"}}, + CharData("\n"), + StartElement{Name{"", "tag"}, []Attr{}}, + CharData("&0a;"), + EndElement{Name{"", "tag"}}, + CharData("\n"), +} + +func TestNonStrictRawToken(t *testing.T) { + d := NewDecoder(strings.NewReader(nonStrictInput)) + d.Strict = false + testRawToken(t, d, nonStrictInput, nonStrictTokens) +} + +type downCaser struct { + t *testing.T + r io.ByteReader +} + +func (d *downCaser) ReadByte() (c byte, err error) { + c, err = d.r.ReadByte() + if c >= 'A' && c <= 'Z' { + c += 'a' - 'A' + } + return +} + +func (d *downCaser) Read(p []byte) (int, error) { + d.t.Fatalf("unexpected Read call on downCaser reader") + panic("unreachable") +} + +func TestRawTokenAltEncoding(t *testing.T) { + d := NewDecoder(strings.NewReader(testInputAltEncoding)) + d.CharsetReader = func(charset string, input io.Reader) (io.Reader, error) { + if charset != "x-testing-uppercase" { + t.Fatalf("unexpected charset %q", charset) + } + return &downCaser{t, input.(io.ByteReader)}, nil + } + testRawToken(t, d, testInputAltEncoding, rawTokensAltEncoding) +} + +func TestRawTokenAltEncodingNoConverter(t *testing.T) { + d := NewDecoder(strings.NewReader(testInputAltEncoding)) + token, err := d.RawToken() + if token == nil { + t.Fatalf("expected a token on first RawToken call") + } + if err != nil { + t.Fatal(err) + } + token, err = d.RawToken() + if token != nil { + t.Errorf("expected a nil token; got %#v", token) + } + if err == nil { + t.Fatalf("expected an error on second RawToken call") + } + const encoding = "x-testing-uppercase" + if !strings.Contains(err.Error(), encoding) { + t.Errorf("expected error to contain %q; got error: %v", + encoding, err) + } +} + +func testRawToken(t *testing.T, d *Decoder, raw string, rawTokens []Token) { + lastEnd := int64(0) + for i, want := range rawTokens { + start := d.InputOffset() + have, err := d.RawToken() + end := d.InputOffset() + if err != nil { + t.Fatalf("token %d: unexpected error: %s", i, err) + } + if !reflect.DeepEqual(have, want) { + var shave, swant string + if _, ok := have.(CharData); ok { + shave = fmt.Sprintf("CharData(%q)", have) + } else { + shave = fmt.Sprintf("%#v", have) + } + if _, ok := want.(CharData); ok { + swant = fmt.Sprintf("CharData(%q)", want) + } else { + swant = fmt.Sprintf("%#v", want) + } + t.Errorf("token %d = %s, want %s", i, shave, swant) + } + + // Check that InputOffset returned actual token. + switch { + case start < lastEnd: + t.Errorf("token %d: position [%d,%d) for %T is before previous token", i, start, end, have) + case start >= end: + // Special case: EndElement can be synthesized. + if start == end && end == lastEnd { + break + } + t.Errorf("token %d: position [%d,%d) for %T is empty", i, start, end, have) + case end > int64(len(raw)): + t.Errorf("token %d: position [%d,%d) for %T extends beyond input", i, start, end, have) + default: + text := raw[start:end] + if strings.ContainsAny(text, "<>") && (!strings.HasPrefix(text, "<") || !strings.HasSuffix(text, ">")) { + t.Errorf("token %d: misaligned raw token %#q for %T", i, text, have) + } + } + lastEnd = end + } +} + +// Ensure that directives (specifically !DOCTYPE) include the complete +// text of any nested directives, noting that < and > do not change +// nesting depth if they are in single or double quotes. + +var nestedDirectivesInput = ` +]> +">]> +]> +'>]> +]> +'>]> +]> +` + +var nestedDirectivesTokens = []Token{ + CharData("\n"), + Directive(`DOCTYPE []`), + CharData("\n"), + Directive(`DOCTYPE [">]`), + CharData("\n"), + Directive(`DOCTYPE []`), + CharData("\n"), + Directive(`DOCTYPE ['>]`), + CharData("\n"), + Directive(`DOCTYPE []`), + CharData("\n"), + Directive(`DOCTYPE ['>]`), + CharData("\n"), + Directive(`DOCTYPE []`), + CharData("\n"), +} + +func TestNestedDirectives(t *testing.T) { + d := NewDecoder(strings.NewReader(nestedDirectivesInput)) + + for i, want := range nestedDirectivesTokens { + have, err := d.Token() + if err != nil { + t.Fatalf("token %d: unexpected error: %s", i, err) + } + if !reflect.DeepEqual(have, want) { + t.Errorf("token %d = %#v want %#v", i, have, want) + } + } +} + +func TestToken(t *testing.T) { + d := NewDecoder(strings.NewReader(testInput)) + d.Entity = testEntity + + for i, want := range cookedTokens { + have, err := d.Token() + if err != nil { + t.Fatalf("token %d: unexpected error: %s", i, err) + } + if !reflect.DeepEqual(have, want) { + t.Errorf("token %d = %#v want %#v", i, have, want) + } + } +} + +func TestSyntax(t *testing.T) { + for i := range xmlInput { + d := NewDecoder(strings.NewReader(xmlInput[i])) + var err error + for _, err = d.Token(); err == nil; _, err = d.Token() { + } + if _, ok := err.(*SyntaxError); !ok { + t.Fatalf(`xmlInput "%s": expected SyntaxError not received`, xmlInput[i]) + } + } +} + +type allScalars struct { + True1 bool + True2 bool + False1 bool + False2 bool + Int int + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 + Uint int + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + Uintptr uintptr + Float32 float32 + Float64 float64 + String string + PtrString *string +} + +var all = allScalars{ + True1: true, + True2: true, + False1: false, + False2: false, + Int: 1, + Int8: -2, + Int16: 3, + Int32: -4, + Int64: 5, + Uint: 6, + Uint8: 7, + Uint16: 8, + Uint32: 9, + Uint64: 10, + Uintptr: 11, + Float32: 13.0, + Float64: 14.0, + String: "15", + PtrString: &sixteen, +} + +var sixteen = "16" + +const testScalarsInput = ` + true + 1 + false + 0 + 1 + -2 + 3 + -4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12.0 + 13.0 + 14.0 + 15 + 16 +` + +func TestAllScalars(t *testing.T) { + var a allScalars + err := Unmarshal([]byte(testScalarsInput), &a) + + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(a, all) { + t.Errorf("have %+v want %+v", a, all) + } +} + +type item struct { + Field_a string +} + +func TestIssue569(t *testing.T) { + data := `abcd` + var i item + err := Unmarshal([]byte(data), &i) + + if err != nil || i.Field_a != "abcd" { + t.Fatal("Expecting abcd") + } +} + +func TestUnquotedAttrs(t *testing.T) { + data := "" + d := NewDecoder(strings.NewReader(data)) + d.Strict = false + token, err := d.Token() + if _, ok := err.(*SyntaxError); ok { + t.Errorf("Unexpected error: %v", err) + } + if token.(StartElement).Name.Local != "tag" { + t.Errorf("Unexpected tag name: %v", token.(StartElement).Name.Local) + } + attr := token.(StartElement).Attr[0] + if attr.Value != "azAZ09:-_" { + t.Errorf("Unexpected attribute value: %v", attr.Value) + } + if attr.Name.Local != "attr" { + t.Errorf("Unexpected attribute name: %v", attr.Name.Local) + } +} + +func TestValuelessAttrs(t *testing.T) { + tests := [][3]string{ + {"

", "p", "nowrap"}, + {"

", "p", "nowrap"}, + {"", "input", "checked"}, + {"", "input", "checked"}, + } + for _, test := range tests { + d := NewDecoder(strings.NewReader(test[0])) + d.Strict = false + token, err := d.Token() + if _, ok := err.(*SyntaxError); ok { + t.Errorf("Unexpected error: %v", err) + } + if token.(StartElement).Name.Local != test[1] { + t.Errorf("Unexpected tag name: %v", token.(StartElement).Name.Local) + } + attr := token.(StartElement).Attr[0] + if attr.Value != test[2] { + t.Errorf("Unexpected attribute value: %v", attr.Value) + } + if attr.Name.Local != test[2] { + t.Errorf("Unexpected attribute name: %v", attr.Name.Local) + } + } +} + +func TestCopyTokenCharData(t *testing.T) { + data := []byte("same data") + var tok1 Token = CharData(data) + tok2 := CopyToken(tok1) + if !reflect.DeepEqual(tok1, tok2) { + t.Error("CopyToken(CharData) != CharData") + } + data[1] = 'o' + if reflect.DeepEqual(tok1, tok2) { + t.Error("CopyToken(CharData) uses same buffer.") + } +} + +func TestCopyTokenStartElement(t *testing.T) { + elt := StartElement{Name{"", "hello"}, []Attr{{Name{"", "lang"}, "en"}}} + var tok1 Token = elt + tok2 := CopyToken(tok1) + if tok1.(StartElement).Attr[0].Value != "en" { + t.Error("CopyToken overwrote Attr[0]") + } + if !reflect.DeepEqual(tok1, tok2) { + t.Error("CopyToken(StartElement) != StartElement") + } + tok1.(StartElement).Attr[0] = Attr{Name{"", "lang"}, "de"} + if reflect.DeepEqual(tok1, tok2) { + t.Error("CopyToken(CharData) uses same buffer.") + } +} + +func TestSyntaxErrorLineNum(t *testing.T) { + testInput := "

Foo

\n\n

Bar\n" + d := NewDecoder(strings.NewReader(testInput)) + var err error + for _, err = d.Token(); err == nil; _, err = d.Token() { + } + synerr, ok := err.(*SyntaxError) + if !ok { + t.Error("Expected SyntaxError.") + } + if synerr.Line != 3 { + t.Error("SyntaxError didn't have correct line number.") + } +} + +func TestTrailingRawToken(t *testing.T) { + input := ` ` + d := NewDecoder(strings.NewReader(input)) + var err error + for _, err = d.RawToken(); err == nil; _, err = d.RawToken() { + } + if err != io.EOF { + t.Fatalf("d.RawToken() = _, %v, want _, io.EOF", err) + } +} + +func TestTrailingToken(t *testing.T) { + input := ` ` + d := NewDecoder(strings.NewReader(input)) + var err error + for _, err = d.Token(); err == nil; _, err = d.Token() { + } + if err != io.EOF { + t.Fatalf("d.Token() = _, %v, want _, io.EOF", err) + } +} + +func TestEntityInsideCDATA(t *testing.T) { + input := `` + d := NewDecoder(strings.NewReader(input)) + var err error + for _, err = d.Token(); err == nil; _, err = d.Token() { + } + if err != io.EOF { + t.Fatalf("d.Token() = _, %v, want _, io.EOF", err) + } +} + +var characterTests = []struct { + in string + err string +}{ + {"\x12", "illegal character code U+0012"}, + {"\x0b", "illegal character code U+000B"}, + {"\xef\xbf\xbe", "illegal character code U+FFFE"}, + {"\r\n\x07", "illegal character code U+0007"}, + {"what's up", "expected attribute name in element"}, + {"&abc\x01;", "invalid character entity &abc (no semicolon)"}, + {"&\x01;", "invalid character entity & (no semicolon)"}, + {"&\xef\xbf\xbe;", "invalid character entity &\uFFFE;"}, + {"&hello;", "invalid character entity &hello;"}, +} + +func TestDisallowedCharacters(t *testing.T) { + + for i, tt := range characterTests { + d := NewDecoder(strings.NewReader(tt.in)) + var err error + + for err == nil { + _, err = d.Token() + } + synerr, ok := err.(*SyntaxError) + if !ok { + t.Fatalf("input %d d.Token() = _, %v, want _, *SyntaxError", i, err) + } + if synerr.Msg != tt.err { + t.Fatalf("input %d synerr.Msg wrong: want %q, got %q", i, tt.err, synerr.Msg) + } + } +} + +type procInstEncodingTest struct { + expect, got string +} + +var procInstTests = []struct { + input string + expect [2]string +}{ + {`version="1.0" encoding="utf-8"`, [2]string{"1.0", "utf-8"}}, + {`version="1.0" encoding='utf-8'`, [2]string{"1.0", "utf-8"}}, + {`version="1.0" encoding='utf-8' `, [2]string{"1.0", "utf-8"}}, + {`version="1.0" encoding=utf-8`, [2]string{"1.0", ""}}, + {`encoding="FOO" `, [2]string{"", "FOO"}}, +} + +func TestProcInstEncoding(t *testing.T) { + for _, test := range procInstTests { + if got := procInst("version", test.input); got != test.expect[0] { + t.Errorf("procInst(version, %q) = %q; want %q", test.input, got, test.expect[0]) + } + if got := procInst("encoding", test.input); got != test.expect[1] { + t.Errorf("procInst(encoding, %q) = %q; want %q", test.input, got, test.expect[1]) + } + } +} + +// Ensure that directives with comments include the complete +// text of any nested directives. + +var directivesWithCommentsInput = ` +]> +]> + --> --> []> +` + +var directivesWithCommentsTokens = []Token{ + CharData("\n"), + Directive(`DOCTYPE []`), + CharData("\n"), + Directive(`DOCTYPE []`), + CharData("\n"), + Directive(`DOCTYPE []`), + CharData("\n"), +} + +func TestDirectivesWithComments(t *testing.T) { + d := NewDecoder(strings.NewReader(directivesWithCommentsInput)) + + for i, want := range directivesWithCommentsTokens { + have, err := d.Token() + if err != nil { + t.Fatalf("token %d: unexpected error: %s", i, err) + } + if !reflect.DeepEqual(have, want) { + t.Errorf("token %d = %#v want %#v", i, have, want) + } + } +} + +// Writer whose Write method always returns an error. +type errWriter struct{} + +func (errWriter) Write(p []byte) (n int, err error) { return 0, fmt.Errorf("unwritable") } + +func TestEscapeTextIOErrors(t *testing.T) { + expectErr := "unwritable" + err := EscapeText(errWriter{}, []byte{'A'}) + + if err == nil || err.Error() != expectErr { + t.Errorf("have %v, want %v", err, expectErr) + } +} + +func TestEscapeTextInvalidChar(t *testing.T) { + input := []byte("A \x00 terminated string.") + expected := "A \uFFFD terminated string." + + buff := new(bytes.Buffer) + if err := EscapeText(buff, input); err != nil { + t.Fatalf("have %v, want nil", err) + } + text := buff.String() + + if text != expected { + t.Errorf("have %v, want %v", text, expected) + } +} + +func TestIssue5880(t *testing.T) { + type T []byte + data, err := Marshal(T{192, 168, 0, 1}) + if err != nil { + t.Errorf("Marshal error: %v", err) + } + if !utf8.Valid(data) { + t.Errorf("Marshal generated invalid UTF-8: %x", data) + } +} diff --git a/endpoints/drive/webdav/litmus_test_server.go b/endpoints/drive/webdav/litmus_test_server.go new file mode 100644 index 000000000..e33b7b74e --- /dev/null +++ b/endpoints/drive/webdav/litmus_test_server.go @@ -0,0 +1,94 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build ignore +// +build ignore + +/* +This program is a server for the WebDAV 'litmus' compliance test at +http://www.webdav.org/neon/litmus/ +To run the test: + +go run litmus_test_server.go + +and separately, from the downloaded litmus-xxx directory: + +make URL=http://localhost:9999/ check +*/ +package main + +import ( + "flag" + "fmt" + "golang.org/x/net/webdav" + "log" + "net/http" + "net/url" +) + +var port = flag.Int("port", 9999, "server port") + +func main() { + flag.Parse() + log.SetFlags(0) + h := &webdav.Handler{ + FileSystem: webdav.NewMemFS(), + LockSystem: webdav.NewMemLS(), + Logger: func(r *http.Request, err error) { + litmus := r.Header.Get("X-Litmus") + if len(litmus) > 19 { + litmus = litmus[:16] + "..." + } + + switch r.Method { + case "COPY", "MOVE": + dst := "" + if u, err := url.Parse(r.Header.Get("Destination")); err == nil { + dst = u.Path + } + o := r.Header.Get("Overwrite") + log.Printf("%-20s%-10s%-30s%-30so=%-2s%v", litmus, r.Method, r.URL.Path, dst, o, err) + default: + log.Printf("%-20s%-10s%-30s%v", litmus, r.Method, r.URL.Path, err) + } + }, + } + + // The next line would normally be: + // http.Handle("/", h) + // but we wrap that HTTP handler h to cater for a special case. + // + // The propfind_invalid2 litmus test case expects an empty namespace prefix + // declaration to be an error. The FAQ in the webdav litmus test says: + // + // "What does the "propfind_invalid2" test check for?... + // + // If a request was sent with an XML body which included an empty namespace + // prefix declaration (xmlns:ns1=""), then the server must reject that with + // a "400 Bad Request" response, as it is invalid according to the XML + // Namespace specification." + // + // On the other hand, the Go standard library's encoding/xml package + // accepts an empty xmlns namespace, as per the discussion at + // https://github.com/golang/go/issues/8068 + // + // Empty namespaces seem disallowed in the second (2006) edition of the XML + // standard, but allowed in a later edition. The grammar differs between + // http://www.w3.org/TR/2006/REC-xml-names-20060816/#ns-decl and + // http://www.w3.org/TR/REC-xml-names/#dt-prefix + // + // Thus, we assume that the propfind_invalid2 test is obsolete, and + // hard-code the 400 Bad Request response that the test expects. + http.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-Litmus") == "props: 3 (propfind_invalid2)" { + http.Error(w, "400 Bad Request", http.StatusBadRequest) + return + } + h.ServeHTTP(w, r) + })) + + addr := fmt.Sprintf(":%d", *port) + log.Printf("Serving %v", addr) + log.Fatal(http.ListenAndServe(addr, nil)) +} diff --git a/endpoints/drive/webdav/lock.go b/endpoints/drive/webdav/lock.go new file mode 100644 index 000000000..344ac5cea --- /dev/null +++ b/endpoints/drive/webdav/lock.go @@ -0,0 +1,445 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package webdav + +import ( + "container/heap" + "errors" + "strconv" + "strings" + "sync" + "time" +) + +var ( + // ErrConfirmationFailed is returned by a LockSystem's Confirm method. + ErrConfirmationFailed = errors.New("webdav: confirmation failed") + // ErrForbidden is returned by a LockSystem's Unlock method. + ErrForbidden = errors.New("webdav: forbidden") + // ErrLocked is returned by a LockSystem's Create, Refresh and Unlock methods. + ErrLocked = errors.New("webdav: locked") + // ErrNoSuchLock is returned by a LockSystem's Refresh and Unlock methods. + ErrNoSuchLock = errors.New("webdav: no such lock") +) + +// Condition can match a WebDAV resource, based on a token or ETag. +// Exactly one of Token and ETag should be non-empty. +type Condition struct { + Not bool + Token string + ETag string +} + +// LockSystem manages access to a collection of named resources. The elements +// in a lock name are separated by slash ('/', U+002F) characters, regardless +// of host operating system convention. +type LockSystem interface { + // Confirm confirms that the caller can claim all of the locks specified by + // the given conditions, and that holding the union of all of those locks + // gives exclusive access to all of the named resources. Up to two resources + // can be named. Empty names are ignored. + // + // Exactly one of release and err will be non-nil. If release is non-nil, + // all of the requested locks are held until release is called. Calling + // release does not unlock the lock, in the WebDAV UNLOCK sense, but once + // Confirm has confirmed that a lock claim is valid, that lock cannot be + // Confirmed again until it has been released. + // + // If Confirm returns ErrConfirmationFailed then the Handler will continue + // to try any other set of locks presented (a WebDAV HTTP request can + // present more than one set of locks). If it returns any other non-nil + // error, the Handler will write a "500 Internal Server Error" HTTP status. + Confirm(now time.Time, name0, name1 string, conditions ...Condition) (release func(), err error) + + // Create creates a lock with the given depth, duration, owner and root + // (name). The depth will either be negative (meaning infinite) or zero. + // + // If Create returns ErrLocked then the Handler will write a "423 Locked" + // HTTP status. If it returns any other non-nil error, the Handler will + // write a "500 Internal Server Error" HTTP status. + // + // See http://www.webdav.org/specs/rfc4918.html#rfc.section.9.10.6 for + // when to use each error. + // + // The token returned identifies the created lock. It should be an absolute + // URI as defined by RFC 3986, Section 4.3. In particular, it should not + // contain whitespace. + Create(now time.Time, details LockDetails) (token string, err error) + + // Refresh refreshes the lock with the given token. + // + // If Refresh returns ErrLocked then the Handler will write a "423 Locked" + // HTTP Status. If Refresh returns ErrNoSuchLock then the Handler will write + // a "412 Precondition Failed" HTTP Status. If it returns any other non-nil + // error, the Handler will write a "500 Internal Server Error" HTTP status. + // + // See http://www.webdav.org/specs/rfc4918.html#rfc.section.9.10.6 for + // when to use each error. + Refresh(now time.Time, token string, duration time.Duration) (LockDetails, error) + + // Unlock unlocks the lock with the given token. + // + // If Unlock returns ErrForbidden then the Handler will write a "403 + // Forbidden" HTTP Status. If Unlock returns ErrLocked then the Handler + // will write a "423 Locked" HTTP status. If Unlock returns ErrNoSuchLock + // then the Handler will write a "409 Conflict" HTTP Status. If it returns + // any other non-nil error, the Handler will write a "500 Internal Server + // Error" HTTP status. + // + // See http://www.webdav.org/specs/rfc4918.html#rfc.section.9.11.1 for + // when to use each error. + Unlock(now time.Time, token string) error +} + +// LockDetails are a lock's metadata. +type LockDetails struct { + // Root is the root resource name being locked. For a zero-depth lock, the + // root is the only resource being locked. + Root string + // Duration is the lock timeout. A negative duration means infinite. + Duration time.Duration + // OwnerXML is the verbatim XML given in a LOCK HTTP request. + // + // TODO: does the "verbatim" nature play well with XML namespaces? + // Does the OwnerXML field need to have more structure? See + // https://codereview.appspot.com/175140043/#msg2 + OwnerXML string + // ZeroDepth is whether the lock has zero depth. If it does not have zero + // depth, it has infinite depth. + ZeroDepth bool +} + +// NewMemLS returns a new in-memory LockSystem. +func NewMemLS() LockSystem { + return &memLS{ + byName: make(map[string]*memLSNode), + byToken: make(map[string]*memLSNode), + gen: uint64(time.Now().Unix()), + } +} + +type memLS struct { + mu sync.Mutex + byName map[string]*memLSNode + byToken map[string]*memLSNode + gen uint64 + // byExpiry only contains those nodes whose LockDetails have a finite + // Duration and are yet to expire. + byExpiry byExpiry +} + +func (m *memLS) nextToken() string { + m.gen++ + return strconv.FormatUint(m.gen, 10) +} + +func (m *memLS) collectExpiredNodes(now time.Time) { + for len(m.byExpiry) > 0 { + if now.Before(m.byExpiry[0].expiry) { + break + } + m.remove(m.byExpiry[0]) + } +} + +func (m *memLS) Confirm(now time.Time, name0, name1 string, conditions ...Condition) (func(), error) { + m.mu.Lock() + defer m.mu.Unlock() + m.collectExpiredNodes(now) + + var n0, n1 *memLSNode + if name0 != "" { + if n0 = m.lookup(slashClean(name0), conditions...); n0 == nil { + return nil, ErrConfirmationFailed + } + } + if name1 != "" { + if n1 = m.lookup(slashClean(name1), conditions...); n1 == nil { + return nil, ErrConfirmationFailed + } + } + + // Don't hold the same node twice. + if n1 == n0 { + n1 = nil + } + + if n0 != nil { + m.hold(n0) + } + if n1 != nil { + m.hold(n1) + } + return func() { + m.mu.Lock() + defer m.mu.Unlock() + if n1 != nil { + m.unhold(n1) + } + if n0 != nil { + m.unhold(n0) + } + }, nil +} + +// lookup returns the node n that locks the named resource, provided that n +// matches at least one of the given conditions and that lock isn't held by +// another party. Otherwise, it returns nil. +// +// n may be a parent of the named resource, if n is an infinite depth lock. +func (m *memLS) lookup(name string, conditions ...Condition) (n *memLSNode) { + // TODO: support Condition.Not and Condition.ETag. + for _, c := range conditions { + n = m.byToken[c.Token] + if n == nil || n.held { + continue + } + if name == n.details.Root { + return n + } + if n.details.ZeroDepth { + continue + } + if n.details.Root == "/" || strings.HasPrefix(name, n.details.Root+"/") { + return n + } + } + return nil +} + +func (m *memLS) hold(n *memLSNode) { + if n.held { + panic("webdav: memLS inconsistent held state") + } + n.held = true + if n.details.Duration >= 0 && n.byExpiryIndex >= 0 { + heap.Remove(&m.byExpiry, n.byExpiryIndex) + } +} + +func (m *memLS) unhold(n *memLSNode) { + if !n.held { + panic("webdav: memLS inconsistent held state") + } + n.held = false + if n.details.Duration >= 0 { + heap.Push(&m.byExpiry, n) + } +} + +func (m *memLS) Create(now time.Time, details LockDetails) (string, error) { + m.mu.Lock() + defer m.mu.Unlock() + m.collectExpiredNodes(now) + details.Root = slashClean(details.Root) + + if !m.canCreate(details.Root, details.ZeroDepth) { + return "", ErrLocked + } + n := m.create(details.Root) + n.token = m.nextToken() + m.byToken[n.token] = n + n.details = details + if n.details.Duration >= 0 { + n.expiry = now.Add(n.details.Duration) + heap.Push(&m.byExpiry, n) + } + return n.token, nil +} + +func (m *memLS) Refresh(now time.Time, token string, duration time.Duration) (LockDetails, error) { + m.mu.Lock() + defer m.mu.Unlock() + m.collectExpiredNodes(now) + + n := m.byToken[token] + if n == nil { + return LockDetails{}, ErrNoSuchLock + } + if n.held { + return LockDetails{}, ErrLocked + } + if n.byExpiryIndex >= 0 { + heap.Remove(&m.byExpiry, n.byExpiryIndex) + } + n.details.Duration = duration + if n.details.Duration >= 0 { + n.expiry = now.Add(n.details.Duration) + heap.Push(&m.byExpiry, n) + } + return n.details, nil +} + +func (m *memLS) Unlock(now time.Time, token string) error { + m.mu.Lock() + defer m.mu.Unlock() + m.collectExpiredNodes(now) + + n := m.byToken[token] + if n == nil { + return ErrNoSuchLock + } + if n.held { + return ErrLocked + } + m.remove(n) + return nil +} + +func (m *memLS) canCreate(name string, zeroDepth bool) bool { + return walkToRoot(name, func(name0 string, first bool) bool { + n := m.byName[name0] + if n == nil { + return true + } + if first { + if n.token != "" { + // The target node is already locked. + return false + } + if !zeroDepth { + // The requested lock depth is infinite, and the fact that n exists + // (n != nil) means that a descendent of the target node is locked. + return false + } + } else if n.token != "" && !n.details.ZeroDepth { + // An ancestor of the target node is locked with infinite depth. + return false + } + return true + }) +} + +func (m *memLS) create(name string) (ret *memLSNode) { + walkToRoot(name, func(name0 string, first bool) bool { + n := m.byName[name0] + if n == nil { + n = &memLSNode{ + details: LockDetails{ + Root: name0, + }, + byExpiryIndex: -1, + } + m.byName[name0] = n + } + n.refCount++ + if first { + ret = n + } + return true + }) + return ret +} + +func (m *memLS) remove(n *memLSNode) { + delete(m.byToken, n.token) + n.token = "" + walkToRoot(n.details.Root, func(name0 string, first bool) bool { + x := m.byName[name0] + x.refCount-- + if x.refCount == 0 { + delete(m.byName, name0) + } + return true + }) + if n.byExpiryIndex >= 0 { + heap.Remove(&m.byExpiry, n.byExpiryIndex) + } +} + +func walkToRoot(name string, f func(name0 string, first bool) bool) bool { + for first := true; ; first = false { + if !f(name, first) { + return false + } + if name == "/" { + break + } + name = name[:strings.LastIndex(name, "/")] + if name == "" { + name = "/" + } + } + return true +} + +type memLSNode struct { + // details are the lock metadata. Even if this node's name is not explicitly locked, + // details.Root will still equal the node's name. + details LockDetails + // token is the unique identifier for this node's lock. An empty token means that + // this node is not explicitly locked. + token string + // refCount is the number of self-or-descendent nodes that are explicitly locked. + refCount int + // expiry is when this node's lock expires. + expiry time.Time + // byExpiryIndex is the index of this node in memLS.byExpiry. It is -1 + // if this node does not expire, or has expired. + byExpiryIndex int + // held is whether this node's lock is actively held by a Confirm call. + held bool +} + +type byExpiry []*memLSNode + +func (b *byExpiry) Len() int { + return len(*b) +} + +func (b *byExpiry) Less(i, j int) bool { + return (*b)[i].expiry.Before((*b)[j].expiry) +} + +func (b *byExpiry) Swap(i, j int) { + (*b)[i], (*b)[j] = (*b)[j], (*b)[i] + (*b)[i].byExpiryIndex = i + (*b)[j].byExpiryIndex = j +} + +func (b *byExpiry) Push(x interface{}) { + n := x.(*memLSNode) + n.byExpiryIndex = len(*b) + *b = append(*b, n) +} + +func (b *byExpiry) Pop() interface{} { + i := len(*b) - 1 + n := (*b)[i] + (*b)[i] = nil + n.byExpiryIndex = -1 + *b = (*b)[:i] + return n +} + +const infiniteTimeout = -1 + +// parseTimeout parses the Timeout HTTP header, as per section 10.7. If s is +// empty, an infiniteTimeout is returned. +func parseTimeout(s string) (time.Duration, error) { + if s == "" { + return infiniteTimeout, nil + } + if i := strings.IndexByte(s, ','); i >= 0 { + s = s[:i] + } + s = strings.TrimSpace(s) + if s == "Infinite" { + return infiniteTimeout, nil + } + const pre = "Second-" + if !strings.HasPrefix(s, pre) { + return 0, errInvalidTimeout + } + s = s[len(pre):] + if s == "" || s[0] < '0' || '9' < s[0] { + return 0, errInvalidTimeout + } + n, err := strconv.ParseInt(s, 10, 64) + if err != nil || 1<<32-1 < n { + return 0, errInvalidTimeout + } + return time.Duration(n) * time.Second, nil +} diff --git a/endpoints/drive/webdav/lock_test.go b/endpoints/drive/webdav/lock_test.go new file mode 100644 index 000000000..e7fe97061 --- /dev/null +++ b/endpoints/drive/webdav/lock_test.go @@ -0,0 +1,735 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package webdav + +import ( + "fmt" + "math/rand" + "path" + "reflect" + "sort" + "strconv" + "strings" + "testing" + "time" +) + +func TestWalkToRoot(t *testing.T) { + testCases := []struct { + name string + want []string + }{{ + "/a/b/c/d", + []string{ + "/a/b/c/d", + "/a/b/c", + "/a/b", + "/a", + "/", + }, + }, { + "/a", + []string{ + "/a", + "/", + }, + }, { + "/", + []string{ + "/", + }, + }} + + for _, tc := range testCases { + var got []string + if !walkToRoot(tc.name, func(name0 string, first bool) bool { + if first != (len(got) == 0) { + t.Errorf("name=%q: first=%t but len(got)==%d", tc.name, first, len(got)) + return false + } + got = append(got, name0) + return true + }) { + continue + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("name=%q:\ngot %q\nwant %q", tc.name, got, tc.want) + } + } +} + +var lockTestDurations = []time.Duration{ + infiniteTimeout, // infiniteTimeout means to never expire. + 0, // A zero duration means to expire immediately. + 100 * time.Hour, // A very large duration will not expire in these tests. +} + +// lockTestNames are the names of a set of mutually compatible locks. For each +// name fragment: +// - _ means no explicit lock. +// - i means an infinite-depth lock, +// - z means a zero-depth lock, +var lockTestNames = []string{ + "/_/_/_/_/z", + "/_/_/i", + "/_/z", + "/_/z/i", + "/_/z/z", + "/_/z/_/i", + "/_/z/_/z", + "/i", + "/z", + "/z/_/i", + "/z/_/z", +} + +func lockTestZeroDepth(name string) bool { + switch name[len(name)-1] { + case 'i': + return false + case 'z': + return true + } + panic(fmt.Sprintf("lock name %q did not end with 'i' or 'z'", name)) +} + +func TestMemLSCanCreate(t *testing.T) { + now := time.Unix(0, 0) + m := NewMemLS().(*memLS) + + for _, name := range lockTestNames { + _, err := m.Create(now, LockDetails{ + Root: name, + Duration: infiniteTimeout, + ZeroDepth: lockTestZeroDepth(name), + }) + if err != nil { + t.Fatalf("creating lock for %q: %v", name, err) + } + } + + wantCanCreate := func(name string, zeroDepth bool) bool { + for _, n := range lockTestNames { + switch { + case n == name: + // An existing lock has the same name as the proposed lock. + return false + case strings.HasPrefix(n, name): + // An existing lock would be a child of the proposed lock, + // which conflicts if the proposed lock has infinite depth. + if !zeroDepth { + return false + } + case strings.HasPrefix(name, n): + // An existing lock would be an ancestor of the proposed lock, + // which conflicts if the ancestor has infinite depth. + if n[len(n)-1] == 'i' { + return false + } + } + } + return true + } + + var check func(int, string) + check = func(recursion int, name string) { + for _, zeroDepth := range []bool{false, true} { + got := m.canCreate(name, zeroDepth) + want := wantCanCreate(name, zeroDepth) + if got != want { + t.Errorf("canCreate name=%q zeroDepth=%t: got %t, want %t", name, zeroDepth, got, want) + } + } + if recursion == 6 { + return + } + if name != "/" { + name += "/" + } + for _, c := range "_iz" { + check(recursion+1, name+string(c)) + } + } + check(0, "/") +} + +func TestMemLSLookup(t *testing.T) { + now := time.Unix(0, 0) + m := NewMemLS().(*memLS) + + badToken := m.nextToken() + t.Logf("badToken=%q", badToken) + + for _, name := range lockTestNames { + token, err := m.Create(now, LockDetails{ + Root: name, + Duration: infiniteTimeout, + ZeroDepth: lockTestZeroDepth(name), + }) + if err != nil { + t.Fatalf("creating lock for %q: %v", name, err) + } + t.Logf("%-15q -> node=%p token=%q", name, m.byName[name], token) + } + + baseNames := append([]string{"/a", "/b/c"}, lockTestNames...) + for _, baseName := range baseNames { + for _, suffix := range []string{"", "/0", "/1/2/3"} { + name := baseName + suffix + + goodToken := "" + base := m.byName[baseName] + if base != nil && (suffix == "" || !lockTestZeroDepth(baseName)) { + goodToken = base.token + } + + for _, token := range []string{badToken, goodToken} { + if token == "" { + continue + } + + got := m.lookup(name, Condition{Token: token}) + want := base + if token == badToken { + want = nil + } + if got != want { + t.Errorf("name=%-20qtoken=%q (bad=%t): got %p, want %p", + name, token, token == badToken, got, want) + } + } + } + } +} + +func TestMemLSConfirm(t *testing.T) { + now := time.Unix(0, 0) + m := NewMemLS().(*memLS) + alice, err := m.Create(now, LockDetails{ + Root: "/alice", + Duration: infiniteTimeout, + ZeroDepth: false, + }) + if err != nil { + t.Fatalf("Create: %v", err) + } + + tweedle, err := m.Create(now, LockDetails{ + Root: "/tweedle", + Duration: infiniteTimeout, + ZeroDepth: false, + }) + if err != nil { + t.Fatalf("Create: %v", err) + } + if err := m.consistent(); err != nil { + t.Fatalf("Create: inconsistent state: %v", err) + } + + // Test a mismatch between name and condition. + _, err = m.Confirm(now, "/tweedle/dee", "", Condition{Token: alice}) + if err != ErrConfirmationFailed { + t.Fatalf("Confirm (mismatch): got %v, want ErrConfirmationFailed", err) + } + if err := m.consistent(); err != nil { + t.Fatalf("Confirm (mismatch): inconsistent state: %v", err) + } + + // Test two names (that fall under the same lock) in the one Confirm call. + release, err := m.Confirm(now, "/tweedle/dee", "/tweedle/dum", Condition{Token: tweedle}) + if err != nil { + t.Fatalf("Confirm (twins): %v", err) + } + if err := m.consistent(); err != nil { + t.Fatalf("Confirm (twins): inconsistent state: %v", err) + } + release() + if err := m.consistent(); err != nil { + t.Fatalf("release (twins): inconsistent state: %v", err) + } + + // Test the same two names in overlapping Confirm / release calls. + releaseDee, err := m.Confirm(now, "/tweedle/dee", "", Condition{Token: tweedle}) + if err != nil { + t.Fatalf("Confirm (sequence #0): %v", err) + } + if err := m.consistent(); err != nil { + t.Fatalf("Confirm (sequence #0): inconsistent state: %v", err) + } + + _, err = m.Confirm(now, "/tweedle/dum", "", Condition{Token: tweedle}) + if err != ErrConfirmationFailed { + t.Fatalf("Confirm (sequence #1): got %v, want ErrConfirmationFailed", err) + } + if err := m.consistent(); err != nil { + t.Fatalf("Confirm (sequence #1): inconsistent state: %v", err) + } + + releaseDee() + if err := m.consistent(); err != nil { + t.Fatalf("release (sequence #2): inconsistent state: %v", err) + } + + releaseDum, err := m.Confirm(now, "/tweedle/dum", "", Condition{Token: tweedle}) + if err != nil { + t.Fatalf("Confirm (sequence #3): %v", err) + } + if err := m.consistent(); err != nil { + t.Fatalf("Confirm (sequence #3): inconsistent state: %v", err) + } + + // Test that you can't unlock a held lock. + err = m.Unlock(now, tweedle) + if err != ErrLocked { + t.Fatalf("Unlock (sequence #4): got %v, want ErrLocked", err) + } + + releaseDum() + if err := m.consistent(); err != nil { + t.Fatalf("release (sequence #5): inconsistent state: %v", err) + } + + err = m.Unlock(now, tweedle) + if err != nil { + t.Fatalf("Unlock (sequence #6): %v", err) + } + if err := m.consistent(); err != nil { + t.Fatalf("Unlock (sequence #6): inconsistent state: %v", err) + } +} + +func TestMemLSNonCanonicalRoot(t *testing.T) { + now := time.Unix(0, 0) + m := NewMemLS().(*memLS) + token, err := m.Create(now, LockDetails{ + Root: "/foo/./bar//", + Duration: 1 * time.Second, + }) + if err != nil { + t.Fatalf("Create: %v", err) + } + if err := m.consistent(); err != nil { + t.Fatalf("Create: inconsistent state: %v", err) + } + if err := m.Unlock(now, token); err != nil { + t.Fatalf("Unlock: %v", err) + } + if err := m.consistent(); err != nil { + t.Fatalf("Unlock: inconsistent state: %v", err) + } +} + +func TestMemLSExpiry(t *testing.T) { + m := NewMemLS().(*memLS) + testCases := []string{ + "setNow 0", + "create /a.5", + "want /a.5", + "create /c.6", + "want /a.5 /c.6", + "create /a/b.7", + "want /a.5 /a/b.7 /c.6", + "setNow 4", + "want /a.5 /a/b.7 /c.6", + "setNow 5", + "want /a/b.7 /c.6", + "setNow 6", + "want /a/b.7", + "setNow 7", + "want ", + "setNow 8", + "want ", + "create /a.12", + "create /b.13", + "create /c.15", + "create /a/d.16", + "want /a.12 /a/d.16 /b.13 /c.15", + "refresh /a.14", + "want /a.14 /a/d.16 /b.13 /c.15", + "setNow 12", + "want /a.14 /a/d.16 /b.13 /c.15", + "setNow 13", + "want /a.14 /a/d.16 /c.15", + "setNow 14", + "want /a/d.16 /c.15", + "refresh /a/d.20", + "refresh /c.20", + "want /a/d.20 /c.20", + "setNow 20", + "want ", + } + + tokens := map[string]string{} + zTime := time.Unix(0, 0) + now := zTime + for i, tc := range testCases { + j := strings.IndexByte(tc, ' ') + if j < 0 { + t.Fatalf("test case #%d %q: invalid command", i, tc) + } + op, arg := tc[:j], tc[j+1:] + switch op { + default: + t.Fatalf("test case #%d %q: invalid operation %q", i, tc, op) + + case "create", "refresh": + parts := strings.Split(arg, ".") + if len(parts) != 2 { + t.Fatalf("test case #%d %q: invalid create", i, tc) + } + root := parts[0] + d, err := strconv.Atoi(parts[1]) + if err != nil { + t.Fatalf("test case #%d %q: invalid duration", i, tc) + } + dur := time.Unix(0, 0).Add(time.Duration(d) * time.Second).Sub(now) + + switch op { + case "create": + token, err := m.Create(now, LockDetails{ + Root: root, + Duration: dur, + ZeroDepth: true, + }) + if err != nil { + t.Fatalf("test case #%d %q: Create: %v", i, tc, err) + } + tokens[root] = token + + case "refresh": + token := tokens[root] + if token == "" { + t.Fatalf("test case #%d %q: no token for %q", i, tc, root) + } + got, err := m.Refresh(now, token, dur) + if err != nil { + t.Fatalf("test case #%d %q: Refresh: %v", i, tc, err) + } + want := LockDetails{ + Root: root, + Duration: dur, + ZeroDepth: true, + } + if got != want { + t.Fatalf("test case #%d %q:\ngot %v\nwant %v", i, tc, got, want) + } + } + + case "setNow": + d, err := strconv.Atoi(arg) + if err != nil { + t.Fatalf("test case #%d %q: invalid duration", i, tc) + } + now = time.Unix(0, 0).Add(time.Duration(d) * time.Second) + + case "want": + m.mu.Lock() + m.collectExpiredNodes(now) + got := make([]string, 0, len(m.byToken)) + for _, n := range m.byToken { + got = append(got, fmt.Sprintf("%s.%d", + n.details.Root, n.expiry.Sub(zTime)/time.Second)) + } + m.mu.Unlock() + sort.Strings(got) + want := []string{} + if arg != "" { + want = strings.Split(arg, " ") + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("test case #%d %q:\ngot %q\nwant %q", i, tc, got, want) + } + } + + if err := m.consistent(); err != nil { + t.Fatalf("test case #%d %q: inconsistent state: %v", i, tc, err) + } + } +} + +func TestMemLS(t *testing.T) { + now := time.Unix(0, 0) + m := NewMemLS().(*memLS) + rng := rand.New(rand.NewSource(0)) + tokens := map[string]string{} + nConfirm, nCreate, nRefresh, nUnlock := 0, 0, 0, 0 + const N = 2000 + + for i := 0; i < N; i++ { + name := lockTestNames[rng.Intn(len(lockTestNames))] + duration := lockTestDurations[rng.Intn(len(lockTestDurations))] + confirmed, unlocked := false, false + + // If the name was already locked, we randomly confirm/release, refresh + // or unlock it. Otherwise, we create a lock. + token := tokens[name] + if token != "" { + switch rng.Intn(3) { + case 0: + confirmed = true + nConfirm++ + release, err := m.Confirm(now, name, "", Condition{Token: token}) + if err != nil { + t.Fatalf("iteration #%d: Confirm %q: %v", i, name, err) + } + if err := m.consistent(); err != nil { + t.Fatalf("iteration #%d: inconsistent state: %v", i, err) + } + release() + + case 1: + nRefresh++ + if _, err := m.Refresh(now, token, duration); err != nil { + t.Fatalf("iteration #%d: Refresh %q: %v", i, name, err) + } + + case 2: + unlocked = true + nUnlock++ + if err := m.Unlock(now, token); err != nil { + t.Fatalf("iteration #%d: Unlock %q: %v", i, name, err) + } + } + + } else { + nCreate++ + var err error + token, err = m.Create(now, LockDetails{ + Root: name, + Duration: duration, + ZeroDepth: lockTestZeroDepth(name), + }) + if err != nil { + t.Fatalf("iteration #%d: Create %q: %v", i, name, err) + } + } + + if !confirmed { + if duration == 0 || unlocked { + // A zero-duration lock should expire immediately and is + // effectively equivalent to being unlocked. + tokens[name] = "" + } else { + tokens[name] = token + } + } + + if err := m.consistent(); err != nil { + t.Fatalf("iteration #%d: inconsistent state: %v", i, err) + } + } + + if nConfirm < N/10 { + t.Fatalf("too few Confirm calls: got %d, want >= %d", nConfirm, N/10) + } + if nCreate < N/10 { + t.Fatalf("too few Create calls: got %d, want >= %d", nCreate, N/10) + } + if nRefresh < N/10 { + t.Fatalf("too few Refresh calls: got %d, want >= %d", nRefresh, N/10) + } + if nUnlock < N/10 { + t.Fatalf("too few Unlock calls: got %d, want >= %d", nUnlock, N/10) + } +} + +func (m *memLS) consistent() error { + m.mu.Lock() + defer m.mu.Unlock() + + // If m.byName is non-empty, then it must contain an entry for the root "/", + // and its refCount should equal the number of locked nodes. + if len(m.byName) > 0 { + n := m.byName["/"] + if n == nil { + return fmt.Errorf(`non-empty m.byName does not contain the root "/"`) + } + if n.refCount != len(m.byToken) { + return fmt.Errorf("root node refCount=%d, differs from len(m.byToken)=%d", n.refCount, len(m.byToken)) + } + } + + for name, n := range m.byName { + // The map keys should be consistent with the node's copy of the key. + if n.details.Root != name { + return fmt.Errorf("node name %q != byName map key %q", n.details.Root, name) + } + + // A name must be clean, and start with a "/". + if len(name) == 0 || name[0] != '/' { + return fmt.Errorf(`node name %q does not start with "/"`, name) + } + if name != path.Clean(name) { + return fmt.Errorf(`node name %q is not clean`, name) + } + + // A node's refCount should be positive. + if n.refCount <= 0 { + return fmt.Errorf("non-positive refCount for node at name %q", name) + } + + // A node's refCount should be the number of self-or-descendents that + // are locked (i.e. have a non-empty token). + var list []string + for name0, n0 := range m.byName { + // All of lockTestNames' name fragments are one byte long: '_', 'i' or 'z', + // so strings.HasPrefix is equivalent to self-or-descendent name match. + // We don't have to worry about "/foo/bar" being a false positive match + // for "/foo/b". + if strings.HasPrefix(name0, name) && n0.token != "" { + list = append(list, name0) + } + } + if n.refCount != len(list) { + sort.Strings(list) + return fmt.Errorf("node at name %q has refCount %d but locked self-or-descendents are %q (len=%d)", + name, n.refCount, list, len(list)) + } + + // A node n is in m.byToken if it has a non-empty token. + if n.token != "" { + if _, ok := m.byToken[n.token]; !ok { + return fmt.Errorf("node at name %q has token %q but not in m.byToken", name, n.token) + } + } + + // A node n is in m.byExpiry if it has a non-negative byExpiryIndex. + if n.byExpiryIndex >= 0 { + if n.byExpiryIndex >= len(m.byExpiry) { + return fmt.Errorf("node at name %q has byExpiryIndex %d but m.byExpiry has length %d", name, n.byExpiryIndex, len(m.byExpiry)) + } + if n != m.byExpiry[n.byExpiryIndex] { + return fmt.Errorf("node at name %q has byExpiryIndex %d but that indexes a different node", name, n.byExpiryIndex) + } + } + } + + for token, n := range m.byToken { + // The map keys should be consistent with the node's copy of the key. + if n.token != token { + return fmt.Errorf("node token %q != byToken map key %q", n.token, token) + } + + // Every node in m.byToken is in m.byName. + if _, ok := m.byName[n.details.Root]; !ok { + return fmt.Errorf("node at name %q in m.byToken but not in m.byName", n.details.Root) + } + } + + for i, n := range m.byExpiry { + // The slice indices should be consistent with the node's copy of the index. + if n.byExpiryIndex != i { + return fmt.Errorf("node byExpiryIndex %d != byExpiry slice index %d", n.byExpiryIndex, i) + } + + // Every node in m.byExpiry is in m.byName. + if _, ok := m.byName[n.details.Root]; !ok { + return fmt.Errorf("node at name %q in m.byExpiry but not in m.byName", n.details.Root) + } + + // No node in m.byExpiry should be held. + if n.held { + return fmt.Errorf("node at name %q in m.byExpiry is held", n.details.Root) + } + } + return nil +} + +func TestParseTimeout(t *testing.T) { + testCases := []struct { + s string + want time.Duration + wantErr error + }{{ + "", + infiniteTimeout, + nil, + }, { + "Infinite", + infiniteTimeout, + nil, + }, { + "Infinitesimal", + 0, + errInvalidTimeout, + }, { + "infinite", + 0, + errInvalidTimeout, + }, { + "Second-0", + 0 * time.Second, + nil, + }, { + "Second-123", + 123 * time.Second, + nil, + }, { + " Second-456 ", + 456 * time.Second, + nil, + }, { + "Second-4100000000", + 4100000000 * time.Second, + nil, + }, { + "junk", + 0, + errInvalidTimeout, + }, { + "Second-", + 0, + errInvalidTimeout, + }, { + "Second--1", + 0, + errInvalidTimeout, + }, { + "Second--123", + 0, + errInvalidTimeout, + }, { + "Second-+123", + 0, + errInvalidTimeout, + }, { + "Second-0x123", + 0, + errInvalidTimeout, + }, { + "second-123", + 0, + errInvalidTimeout, + }, { + "Second-4294967295", + 4294967295 * time.Second, + nil, + }, { + // Section 10.7 says that "The timeout value for TimeType "Second" + // must not be greater than 2^32-1." + "Second-4294967296", + 0, + errInvalidTimeout, + }, { + // This test case comes from section 9.10.9 of the spec. It says, + // + // "In this request, the client has specified that it desires an + // infinite-length lock, if available, otherwise a timeout of 4.1 + // billion seconds, if available." + // + // The Go WebDAV package always supports infinite length locks, + // and ignores the fallback after the comma. + "Infinite, Second-4100000000", + infiniteTimeout, + nil, + }} + + for _, tc := range testCases { + got, gotErr := parseTimeout(tc.s) + if got != tc.want || gotErr != tc.wantErr { + t.Errorf("parsing %q:\ngot %v, %v\nwant %v, %v", tc.s, got, gotErr, tc.want, tc.wantErr) + } + } +} diff --git a/endpoints/drive/webdav/prop.go b/endpoints/drive/webdav/prop.go new file mode 100644 index 000000000..fca3154bd --- /dev/null +++ b/endpoints/drive/webdav/prop.go @@ -0,0 +1,469 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package webdav + +import ( + "bytes" + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "mime" + "net/http" + "os" + "path/filepath" + "strconv" +) + +// Proppatch describes a property update instruction as defined in RFC 4918. +// See http://www.webdav.org/specs/rfc4918.html#METHOD_PROPPATCH +type Proppatch struct { + // Remove specifies whether this patch removes properties. If it does not + // remove them, it sets them. + Remove bool + // Props contains the properties to be set or removed. + Props []Property +} + +// Propstat describes a XML propstat element as defined in RFC 4918. +// See http://www.webdav.org/specs/rfc4918.html#ELEMENT_propstat +type Propstat struct { + // Props contains the properties for which Status applies. + Props []Property + + // Status defines the HTTP status code of the properties in Prop. + // Allowed values include, but are not limited to the WebDAV status + // code extensions for HTTP/1.1. + // http://www.webdav.org/specs/rfc4918.html#status.code.extensions.to.http11 + Status int + + // XMLError contains the XML representation of the optional error element. + // XML content within this field must not rely on any predefined + // namespace declarations or prefixes. If empty, the XML error element + // is omitted. + XMLError string + + // ResponseDescription contains the contents of the optional + // responsedescription field. If empty, the XML element is omitted. + ResponseDescription string +} + +// makePropstats returns a slice containing those of x and y whose Props slice +// is non-empty. If both are empty, it returns a slice containing an otherwise +// zero Propstat whose HTTP status code is 200 OK. +func makePropstats(x, y Propstat) []Propstat { + pstats := make([]Propstat, 0, 2) + if len(x.Props) != 0 { + pstats = append(pstats, x) + } + if len(y.Props) != 0 { + pstats = append(pstats, y) + } + if len(pstats) == 0 { + pstats = append(pstats, Propstat{ + Status: http.StatusOK, + }) + } + return pstats +} + +// DeadPropsHolder holds the dead properties of a resource. +// +// Dead properties are those properties that are explicitly defined. In +// comparison, live properties, such as DAV:getcontentlength, are implicitly +// defined by the underlying resource, and cannot be explicitly overridden or +// removed. See the Terminology section of +// http://www.webdav.org/specs/rfc4918.html#rfc.section.3 +// +// There is a whitelist of the names of live properties. This package handles +// all live properties, and will only pass non-whitelisted names to the Patch +// method of DeadPropsHolder implementations. +type DeadPropsHolder interface { + // DeadProps returns a copy of the dead properties held. + DeadProps() (map[xml.Name]Property, error) + + // Patch patches the dead properties held. + // + // Patching is atomic; either all or no patches succeed. It returns (nil, + // non-nil) if an internal server error occurred, otherwise the Propstats + // collectively contain one Property for each proposed patch Property. If + // all patches succeed, Patch returns a slice of length one and a Propstat + // element with a 200 OK HTTP status code. If none succeed, for reasons + // other than an internal server error, no Propstat has status 200 OK. + // + // For more details on when various HTTP status codes apply, see + // http://www.webdav.org/specs/rfc4918.html#PROPPATCH-status + Patch([]Proppatch) ([]Propstat, error) +} + +// liveProps contains all supported, protected DAV: properties. +var liveProps = map[xml.Name]struct { + // findFn implements the propfind function of this property. If nil, + // it indicates a hidden property. + findFn func(context.Context, FileSystem, LockSystem, string, os.FileInfo) (string, error) + // dir is true if the property applies to directories. + dir bool +}{ + {Space: "DAV:", Local: "resourcetype"}: { + findFn: findResourceType, + dir: true, + }, + {Space: "DAV:", Local: "displayname"}: { + findFn: findDisplayName, + dir: true, + }, + {Space: "DAV:", Local: "getcontentlength"}: { + findFn: findContentLength, + dir: false, + }, + {Space: "DAV:", Local: "getlastmodified"}: { + findFn: findLastModified, + // http://webdav.org/specs/rfc4918.html#PROPERTY_getlastmodified + // suggests that getlastmodified should only apply to GETable + // resources, and this package does not support GET on directories. + // + // Nonetheless, some WebDAV clients expect child directories to be + // sortable by getlastmodified date, so this value is true, not false. + // See golang.org/issue/15334. + dir: true, + }, + {Space: "DAV:", Local: "creationdate"}: { + findFn: nil, + dir: false, + }, + {Space: "DAV:", Local: "getcontentlanguage"}: { + findFn: nil, + dir: false, + }, + {Space: "DAV:", Local: "getcontenttype"}: { + findFn: findContentType, + dir: false, + }, + {Space: "DAV:", Local: "getetag"}: { + findFn: findETag, + // findETag implements ETag as the concatenated hex values of a file's + // modification time and size. This is not a reliable synchronization + // mechanism for directories, so we do not advertise getetag for DAV + // collections. + dir: false, + }, + + // TODO: The lockdiscovery property requires LockSystem to list the + // active locks on a resource. + {Space: "DAV:", Local: "lockdiscovery"}: {}, + {Space: "DAV:", Local: "supportedlock"}: { + findFn: findSupportedLock, + dir: true, + }, +} + +// TODO(nigeltao) merge props and allprop? + +// props returns the status of the properties named pnames for resource name. +// +// Each Propstat has a unique status and each property name will only be part +// of one Propstat element. +func props(ctx context.Context, fs FileSystem, ls LockSystem, name string, pnames []xml.Name) ([]Propstat, error) { + f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0) + if err != nil { + return nil, err + } + defer f.Close() + fi, err := f.Stat() + if err != nil { + return nil, err + } + isDir := fi.IsDir() + + var deadProps map[xml.Name]Property + if dph, ok := f.(DeadPropsHolder); ok { + deadProps, err = dph.DeadProps() + if err != nil { + return nil, err + } + } + + pstatOK := Propstat{Status: http.StatusOK} + pstatNotFound := Propstat{Status: http.StatusNotFound} + for _, pn := range pnames { + // If this file has dead properties, check if they contain pn. + if dp, ok := deadProps[pn]; ok { + pstatOK.Props = append(pstatOK.Props, dp) + continue + } + // Otherwise, it must either be a live property or we don't know it. + if prop := liveProps[pn]; prop.findFn != nil && (prop.dir || !isDir) { + innerXML, err := prop.findFn(ctx, fs, ls, name, fi) + if err != nil { + return nil, err + } + pstatOK.Props = append(pstatOK.Props, Property{ + XMLName: pn, + InnerXML: []byte(innerXML), + }) + } else { + pstatNotFound.Props = append(pstatNotFound.Props, Property{ + XMLName: pn, + }) + } + } + return makePropstats(pstatOK, pstatNotFound), nil +} + +// propnames returns the property names defined for resource name. +func propnames(ctx context.Context, fs FileSystem, ls LockSystem, name string) ([]xml.Name, error) { + f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0) + if err != nil { + return nil, err + } + defer f.Close() + fi, err := f.Stat() + if err != nil { + return nil, err + } + isDir := fi.IsDir() + + var deadProps map[xml.Name]Property + if dph, ok := f.(DeadPropsHolder); ok { + deadProps, err = dph.DeadProps() + if err != nil { + return nil, err + } + } + + pnames := make([]xml.Name, 0, len(liveProps)+len(deadProps)) + for pn, prop := range liveProps { + if prop.findFn != nil && (prop.dir || !isDir) { + pnames = append(pnames, pn) + } + } + for pn := range deadProps { + pnames = append(pnames, pn) + } + return pnames, nil +} + +// allprop returns the properties defined for resource name and the properties +// named in include. +// +// Note that RFC 4918 defines 'allprop' to return the DAV: properties defined +// within the RFC plus dead properties. Other live properties should only be +// returned if they are named in 'include'. +// +// See http://www.webdav.org/specs/rfc4918.html#METHOD_PROPFIND +func allprop(ctx context.Context, fs FileSystem, ls LockSystem, name string, include []xml.Name) ([]Propstat, error) { + pnames, err := propnames(ctx, fs, ls, name) + if err != nil { + return nil, err + } + // Add names from include if they are not already covered in pnames. + nameset := make(map[xml.Name]bool) + for _, pn := range pnames { + nameset[pn] = true + } + for _, pn := range include { + if !nameset[pn] { + pnames = append(pnames, pn) + } + } + return props(ctx, fs, ls, name, pnames) +} + +// patch patches the properties of resource name. The return values are +// constrained in the same manner as DeadPropsHolder.Patch. +func patch(ctx context.Context, fs FileSystem, ls LockSystem, name string, patches []Proppatch) ([]Propstat, error) { + conflict := false +loop: + for _, patch := range patches { + for _, p := range patch.Props { + if _, ok := liveProps[p.XMLName]; ok { + conflict = true + break loop + } + } + } + if conflict { + pstatForbidden := Propstat{ + Status: http.StatusForbidden, + XMLError: ``, + } + pstatFailedDep := Propstat{ + Status: StatusFailedDependency, + } + for _, patch := range patches { + for _, p := range patch.Props { + if _, ok := liveProps[p.XMLName]; ok { + pstatForbidden.Props = append(pstatForbidden.Props, Property{XMLName: p.XMLName}) + } else { + pstatFailedDep.Props = append(pstatFailedDep.Props, Property{XMLName: p.XMLName}) + } + } + } + return makePropstats(pstatForbidden, pstatFailedDep), nil + } + + f, err := fs.OpenFile(ctx, name, os.O_RDWR, 0) + if err != nil { + return nil, err + } + defer f.Close() + if dph, ok := f.(DeadPropsHolder); ok { + ret, err := dph.Patch(patches) + if err != nil { + return nil, err + } + // http://www.webdav.org/specs/rfc4918.html#ELEMENT_propstat says that + // "The contents of the prop XML element must only list the names of + // properties to which the result in the status element applies." + for _, pstat := range ret { + for i, p := range pstat.Props { + pstat.Props[i] = Property{XMLName: p.XMLName} + } + } + return ret, nil + } + // The file doesn't implement the optional DeadPropsHolder interface, so + // all patches are forbidden. + pstat := Propstat{Status: http.StatusForbidden} + for _, patch := range patches { + for _, p := range patch.Props { + pstat.Props = append(pstat.Props, Property{XMLName: p.XMLName}) + } + } + return []Propstat{pstat}, nil +} + +func escapeXML(s string) string { + for i := 0; i < len(s); i++ { + // As an optimization, if s contains only ASCII letters, digits or a + // few special characters, the escaped value is s itself and we don't + // need to allocate a buffer and convert between string and []byte. + switch c := s[i]; { + case c == ' ' || c == '_' || + ('+' <= c && c <= '9') || // Digits as well as + , - . and / + ('A' <= c && c <= 'Z') || + ('a' <= c && c <= 'z'): + continue + } + // Otherwise, go through the full escaping process. + var buf bytes.Buffer + xml.EscapeText(&buf, []byte(s)) + return buf.String() + } + return s +} + +func findResourceType(ctx context.Context, fs FileSystem, ls LockSystem, name string, fi os.FileInfo) (string, error) { + if fi.IsDir() { + return ``, nil + } + return "", nil +} + +func findDisplayName(ctx context.Context, fs FileSystem, ls LockSystem, name string, fi os.FileInfo) (string, error) { + if slashClean(name) == "/" { + // Hide the real name of a possibly prefixed root directory. + return "", nil + } + return escapeXML(fi.Name()), nil +} + +func findContentLength(ctx context.Context, fs FileSystem, ls LockSystem, name string, fi os.FileInfo) (string, error) { + return strconv.FormatInt(fi.Size(), 10), nil +} + +func findLastModified(ctx context.Context, fs FileSystem, ls LockSystem, name string, fi os.FileInfo) (string, error) { + return fi.ModTime().UTC().Format(http.TimeFormat), nil +} + +// ErrNotImplemented should be returned by optional interfaces if they +// want the original implementation to be used. +var ErrNotImplemented = errors.New("not implemented") + +// ContentTyper is an optional interface for the os.FileInfo +// objects returned by the FileSystem. +// +// If this interface is defined then it will be used to read the +// content type from the object. +// +// If this interface is not defined the file will be opened and the +// content type will be guessed from the initial contents of the file. +type ContentTyper interface { + // ContentType returns the content type for the file. + // + // If this returns error ErrNotImplemented then the error will + // be ignored and the base implementation will be used + // instead. + ContentType(ctx context.Context) (string, error) +} + +func findContentType(ctx context.Context, fs FileSystem, ls LockSystem, name string, fi os.FileInfo) (string, error) { + if do, ok := fi.(ContentTyper); ok { + ctype, err := do.ContentType(ctx) + if err != ErrNotImplemented { + return ctype, err + } + } + f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0) + if err != nil { + return "", err + } + defer f.Close() + // This implementation is based on serveContent's code in the standard net/http package. + ctype := mime.TypeByExtension(filepath.Ext(name)) + if ctype != "" { + return ctype, nil + } + // Read a chunk to decide between utf-8 text and binary. + var buf [512]byte + n, err := io.ReadFull(f, buf[:]) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return "", err + } + ctype = http.DetectContentType(buf[:n]) + // Rewind file. + _, err = f.Seek(0, io.SeekStart) + return ctype, err +} + +// ETager is an optional interface for the os.FileInfo objects +// returned by the FileSystem. +// +// If this interface is defined then it will be used to read the ETag +// for the object. +// +// If this interface is not defined an ETag will be computed using the +// ModTime() and the Size() methods of the os.FileInfo object. +type ETager interface { + // ETag returns an ETag for the file. This should be of the + // form "value" or W/"value" + // + // If this returns error ErrNotImplemented then the error will + // be ignored and the base implementation will be used + // instead. + ETag(ctx context.Context) (string, error) +} + +func findETag(ctx context.Context, fs FileSystem, ls LockSystem, name string, fi os.FileInfo) (string, error) { + if do, ok := fi.(ETager); ok { + etag, err := do.ETag(ctx) + if err != ErrNotImplemented { + return etag, err + } + } + // The Apache http 2.4 web server by default concatenates the + // modification time and size of a file. We replicate the heuristic + // with nanosecond granularity. + return fmt.Sprintf(`"%x%x"`, fi.ModTime().UnixNano(), fi.Size()), nil +} + +func findSupportedLock(ctx context.Context, fs FileSystem, ls LockSystem, name string, fi os.FileInfo) (string, error) { + return `` + + `` + + `` + + `` + + ``, nil +} diff --git a/endpoints/drive/webdav/prop_test.go b/endpoints/drive/webdav/prop_test.go new file mode 100644 index 000000000..f4247e69b --- /dev/null +++ b/endpoints/drive/webdav/prop_test.go @@ -0,0 +1,716 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package webdav + +import ( + "context" + "encoding/xml" + "fmt" + "net/http" + "os" + "reflect" + "regexp" + "sort" + "testing" +) + +func TestMemPS(t *testing.T) { + ctx := context.Background() + // calcProps calculates the getlastmodified and getetag DAV: property + // values in pstats for resource name in file-system fs. + calcProps := func(name string, fs FileSystem, ls LockSystem, pstats []Propstat) error { + fi, err := fs.Stat(ctx, name) + if err != nil { + return err + } + for _, pst := range pstats { + for i, p := range pst.Props { + switch p.XMLName { + case xml.Name{Space: "DAV:", Local: "getlastmodified"}: + p.InnerXML = []byte(fi.ModTime().UTC().Format(http.TimeFormat)) + pst.Props[i] = p + case xml.Name{Space: "DAV:", Local: "getetag"}: + if fi.IsDir() { + continue + } + etag, err := findETag(ctx, fs, ls, name, fi) + if err != nil { + return err + } + p.InnerXML = []byte(etag) + pst.Props[i] = p + } + } + } + return nil + } + + const ( + lockEntry = `` + + `` + + `` + + `` + + `` + statForbiddenError = `` + ) + + type propOp struct { + op string + name string + pnames []xml.Name + patches []Proppatch + wantPnames []xml.Name + wantPropstats []Propstat + } + + testCases := []struct { + desc string + noDeadProps bool + buildfs []string + propOp []propOp + }{{ + desc: "propname", + buildfs: []string{"mkdir /dir", "touch /file"}, + propOp: []propOp{{ + op: "propname", + name: "/dir", + wantPnames: []xml.Name{ + {Space: "DAV:", Local: "resourcetype"}, + {Space: "DAV:", Local: "displayname"}, + {Space: "DAV:", Local: "supportedlock"}, + {Space: "DAV:", Local: "getlastmodified"}, + }, + }, { + op: "propname", + name: "/file", + wantPnames: []xml.Name{ + {Space: "DAV:", Local: "resourcetype"}, + {Space: "DAV:", Local: "displayname"}, + {Space: "DAV:", Local: "getcontentlength"}, + {Space: "DAV:", Local: "getlastmodified"}, + {Space: "DAV:", Local: "getcontenttype"}, + {Space: "DAV:", Local: "getetag"}, + {Space: "DAV:", Local: "supportedlock"}, + }, + }}, + }, { + desc: "allprop dir and file", + buildfs: []string{"mkdir /dir", "write /file foobarbaz"}, + propOp: []propOp{{ + op: "allprop", + name: "/dir", + wantPropstats: []Propstat{{ + Status: http.StatusOK, + Props: []Property{{ + XMLName: xml.Name{Space: "DAV:", Local: "resourcetype"}, + InnerXML: []byte(``), + }, { + XMLName: xml.Name{Space: "DAV:", Local: "displayname"}, + InnerXML: []byte("dir"), + }, { + XMLName: xml.Name{Space: "DAV:", Local: "getlastmodified"}, + InnerXML: nil, // Calculated during test. + }, { + XMLName: xml.Name{Space: "DAV:", Local: "supportedlock"}, + InnerXML: []byte(lockEntry), + }}, + }}, + }, { + op: "allprop", + name: "/file", + wantPropstats: []Propstat{{ + Status: http.StatusOK, + Props: []Property{{ + XMLName: xml.Name{Space: "DAV:", Local: "resourcetype"}, + InnerXML: []byte(""), + }, { + XMLName: xml.Name{Space: "DAV:", Local: "displayname"}, + InnerXML: []byte("file"), + }, { + XMLName: xml.Name{Space: "DAV:", Local: "getcontentlength"}, + InnerXML: []byte("9"), + }, { + XMLName: xml.Name{Space: "DAV:", Local: "getlastmodified"}, + InnerXML: nil, // Calculated during test. + }, { + XMLName: xml.Name{Space: "DAV:", Local: "getcontenttype"}, + InnerXML: []byte("text/plain; charset=utf-8"), + }, { + XMLName: xml.Name{Space: "DAV:", Local: "getetag"}, + InnerXML: nil, // Calculated during test. + }, { + XMLName: xml.Name{Space: "DAV:", Local: "supportedlock"}, + InnerXML: []byte(lockEntry), + }}, + }}, + }, { + op: "allprop", + name: "/file", + pnames: []xml.Name{ + {Space: "DAV:", Local: "resourcetype"}, + {Space: "foo", Local: "bar"}, + }, + wantPropstats: []Propstat{{ + Status: http.StatusOK, + Props: []Property{{ + XMLName: xml.Name{Space: "DAV:", Local: "resourcetype"}, + InnerXML: []byte(""), + }, { + XMLName: xml.Name{Space: "DAV:", Local: "displayname"}, + InnerXML: []byte("file"), + }, { + XMLName: xml.Name{Space: "DAV:", Local: "getcontentlength"}, + InnerXML: []byte("9"), + }, { + XMLName: xml.Name{Space: "DAV:", Local: "getlastmodified"}, + InnerXML: nil, // Calculated during test. + }, { + XMLName: xml.Name{Space: "DAV:", Local: "getcontenttype"}, + InnerXML: []byte("text/plain; charset=utf-8"), + }, { + XMLName: xml.Name{Space: "DAV:", Local: "getetag"}, + InnerXML: nil, // Calculated during test. + }, { + XMLName: xml.Name{Space: "DAV:", Local: "supportedlock"}, + InnerXML: []byte(lockEntry), + }}}, { + Status: http.StatusNotFound, + Props: []Property{{ + XMLName: xml.Name{Space: "foo", Local: "bar"}, + }}}, + }, + }}, + }, { + desc: "propfind DAV:resourcetype", + buildfs: []string{"mkdir /dir", "touch /file"}, + propOp: []propOp{{ + op: "propfind", + name: "/dir", + pnames: []xml.Name{{Space: "DAV:", Local: "resourcetype"}}, + wantPropstats: []Propstat{{ + Status: http.StatusOK, + Props: []Property{{ + XMLName: xml.Name{Space: "DAV:", Local: "resourcetype"}, + InnerXML: []byte(``), + }}, + }}, + }, { + op: "propfind", + name: "/file", + pnames: []xml.Name{{Space: "DAV:", Local: "resourcetype"}}, + wantPropstats: []Propstat{{ + Status: http.StatusOK, + Props: []Property{{ + XMLName: xml.Name{Space: "DAV:", Local: "resourcetype"}, + InnerXML: []byte(""), + }}, + }}, + }}, + }, { + desc: "propfind unsupported DAV properties", + buildfs: []string{"mkdir /dir"}, + propOp: []propOp{{ + op: "propfind", + name: "/dir", + pnames: []xml.Name{{Space: "DAV:", Local: "getcontentlanguage"}}, + wantPropstats: []Propstat{{ + Status: http.StatusNotFound, + Props: []Property{{ + XMLName: xml.Name{Space: "DAV:", Local: "getcontentlanguage"}, + }}, + }}, + }, { + op: "propfind", + name: "/dir", + pnames: []xml.Name{{Space: "DAV:", Local: "creationdate"}}, + wantPropstats: []Propstat{{ + Status: http.StatusNotFound, + Props: []Property{{ + XMLName: xml.Name{Space: "DAV:", Local: "creationdate"}, + }}, + }}, + }}, + }, { + desc: "propfind getetag for files but not for directories", + buildfs: []string{"mkdir /dir", "touch /file"}, + propOp: []propOp{{ + op: "propfind", + name: "/dir", + pnames: []xml.Name{{Space: "DAV:", Local: "getetag"}}, + wantPropstats: []Propstat{{ + Status: http.StatusNotFound, + Props: []Property{{ + XMLName: xml.Name{Space: "DAV:", Local: "getetag"}, + }}, + }}, + }, { + op: "propfind", + name: "/file", + pnames: []xml.Name{{Space: "DAV:", Local: "getetag"}}, + wantPropstats: []Propstat{{ + Status: http.StatusOK, + Props: []Property{{ + XMLName: xml.Name{Space: "DAV:", Local: "getetag"}, + InnerXML: nil, // Calculated during test. + }}, + }}, + }}, + }, { + desc: "proppatch property on no-dead-properties file system", + buildfs: []string{"mkdir /dir"}, + noDeadProps: true, + propOp: []propOp{{ + op: "proppatch", + name: "/dir", + patches: []Proppatch{{ + Props: []Property{{ + XMLName: xml.Name{Space: "foo", Local: "bar"}, + }}, + }}, + wantPropstats: []Propstat{{ + Status: http.StatusForbidden, + Props: []Property{{ + XMLName: xml.Name{Space: "foo", Local: "bar"}, + }}, + }}, + }, { + op: "proppatch", + name: "/dir", + patches: []Proppatch{{ + Props: []Property{{ + XMLName: xml.Name{Space: "DAV:", Local: "getetag"}, + }}, + }}, + wantPropstats: []Propstat{{ + Status: http.StatusForbidden, + XMLError: statForbiddenError, + Props: []Property{{ + XMLName: xml.Name{Space: "DAV:", Local: "getetag"}, + }}, + }}, + }}, + }, { + desc: "proppatch dead property", + buildfs: []string{"mkdir /dir"}, + propOp: []propOp{{ + op: "proppatch", + name: "/dir", + patches: []Proppatch{{ + Props: []Property{{ + XMLName: xml.Name{Space: "foo", Local: "bar"}, + InnerXML: []byte("baz"), + }}, + }}, + wantPropstats: []Propstat{{ + Status: http.StatusOK, + Props: []Property{{ + XMLName: xml.Name{Space: "foo", Local: "bar"}, + }}, + }}, + }, { + op: "propfind", + name: "/dir", + pnames: []xml.Name{{Space: "foo", Local: "bar"}}, + wantPropstats: []Propstat{{ + Status: http.StatusOK, + Props: []Property{{ + XMLName: xml.Name{Space: "foo", Local: "bar"}, + InnerXML: []byte("baz"), + }}, + }}, + }}, + }, { + desc: "proppatch dead property with failed dependency", + buildfs: []string{"mkdir /dir"}, + propOp: []propOp{{ + op: "proppatch", + name: "/dir", + patches: []Proppatch{{ + Props: []Property{{ + XMLName: xml.Name{Space: "foo", Local: "bar"}, + InnerXML: []byte("baz"), + }}, + }, { + Props: []Property{{ + XMLName: xml.Name{Space: "DAV:", Local: "displayname"}, + InnerXML: []byte("xxx"), + }}, + }}, + wantPropstats: []Propstat{{ + Status: http.StatusForbidden, + XMLError: statForbiddenError, + Props: []Property{{ + XMLName: xml.Name{Space: "DAV:", Local: "displayname"}, + }}, + }, { + Status: StatusFailedDependency, + Props: []Property{{ + XMLName: xml.Name{Space: "foo", Local: "bar"}, + }}, + }}, + }, { + op: "propfind", + name: "/dir", + pnames: []xml.Name{{Space: "foo", Local: "bar"}}, + wantPropstats: []Propstat{{ + Status: http.StatusNotFound, + Props: []Property{{ + XMLName: xml.Name{Space: "foo", Local: "bar"}, + }}, + }}, + }}, + }, { + desc: "proppatch remove dead property", + buildfs: []string{"mkdir /dir"}, + propOp: []propOp{{ + op: "proppatch", + name: "/dir", + patches: []Proppatch{{ + Props: []Property{{ + XMLName: xml.Name{Space: "foo", Local: "bar"}, + InnerXML: []byte("baz"), + }, { + XMLName: xml.Name{Space: "spam", Local: "ham"}, + InnerXML: []byte("eggs"), + }}, + }}, + wantPropstats: []Propstat{{ + Status: http.StatusOK, + Props: []Property{{ + XMLName: xml.Name{Space: "foo", Local: "bar"}, + }, { + XMLName: xml.Name{Space: "spam", Local: "ham"}, + }}, + }}, + }, { + op: "propfind", + name: "/dir", + pnames: []xml.Name{ + {Space: "foo", Local: "bar"}, + {Space: "spam", Local: "ham"}, + }, + wantPropstats: []Propstat{{ + Status: http.StatusOK, + Props: []Property{{ + XMLName: xml.Name{Space: "foo", Local: "bar"}, + InnerXML: []byte("baz"), + }, { + XMLName: xml.Name{Space: "spam", Local: "ham"}, + InnerXML: []byte("eggs"), + }}, + }}, + }, { + op: "proppatch", + name: "/dir", + patches: []Proppatch{{ + Remove: true, + Props: []Property{{ + XMLName: xml.Name{Space: "foo", Local: "bar"}, + }}, + }}, + wantPropstats: []Propstat{{ + Status: http.StatusOK, + Props: []Property{{ + XMLName: xml.Name{Space: "foo", Local: "bar"}, + }}, + }}, + }, { + op: "propfind", + name: "/dir", + pnames: []xml.Name{ + {Space: "foo", Local: "bar"}, + {Space: "spam", Local: "ham"}, + }, + wantPropstats: []Propstat{{ + Status: http.StatusNotFound, + Props: []Property{{ + XMLName: xml.Name{Space: "foo", Local: "bar"}, + }}, + }, { + Status: http.StatusOK, + Props: []Property{{ + XMLName: xml.Name{Space: "spam", Local: "ham"}, + InnerXML: []byte("eggs"), + }}, + }}, + }}, + }, { + desc: "propname with dead property", + buildfs: []string{"touch /file"}, + propOp: []propOp{{ + op: "proppatch", + name: "/file", + patches: []Proppatch{{ + Props: []Property{{ + XMLName: xml.Name{Space: "foo", Local: "bar"}, + InnerXML: []byte("baz"), + }}, + }}, + wantPropstats: []Propstat{{ + Status: http.StatusOK, + Props: []Property{{ + XMLName: xml.Name{Space: "foo", Local: "bar"}, + }}, + }}, + }, { + op: "propname", + name: "/file", + wantPnames: []xml.Name{ + {Space: "DAV:", Local: "resourcetype"}, + {Space: "DAV:", Local: "displayname"}, + {Space: "DAV:", Local: "getcontentlength"}, + {Space: "DAV:", Local: "getlastmodified"}, + {Space: "DAV:", Local: "getcontenttype"}, + {Space: "DAV:", Local: "getetag"}, + {Space: "DAV:", Local: "supportedlock"}, + {Space: "foo", Local: "bar"}, + }, + }}, + }, { + desc: "proppatch remove unknown dead property", + buildfs: []string{"mkdir /dir"}, + propOp: []propOp{{ + op: "proppatch", + name: "/dir", + patches: []Proppatch{{ + Remove: true, + Props: []Property{{ + XMLName: xml.Name{Space: "foo", Local: "bar"}, + }}, + }}, + wantPropstats: []Propstat{{ + Status: http.StatusOK, + Props: []Property{{ + XMLName: xml.Name{Space: "foo", Local: "bar"}, + }}, + }}, + }}, + }, { + desc: "bad: propfind unknown property", + buildfs: []string{"mkdir /dir"}, + propOp: []propOp{{ + op: "propfind", + name: "/dir", + pnames: []xml.Name{{Space: "foo:", Local: "bar"}}, + wantPropstats: []Propstat{{ + Status: http.StatusNotFound, + Props: []Property{{ + XMLName: xml.Name{Space: "foo:", Local: "bar"}, + }}, + }}, + }}, + }} + + for _, tc := range testCases { + fs, err := buildTestFS(tc.buildfs) + if err != nil { + t.Fatalf("%s: cannot create test filesystem: %v", tc.desc, err) + } + if tc.noDeadProps { + fs = noDeadPropsFS{fs} + } + ls := NewMemLS() + for _, op := range tc.propOp { + desc := fmt.Sprintf("%s: %s %s", tc.desc, op.op, op.name) + if err = calcProps(op.name, fs, ls, op.wantPropstats); err != nil { + t.Fatalf("%s: calcProps: %v", desc, err) + } + + // Call property system. + var propstats []Propstat + switch op.op { + case "propname": + pnames, err := propnames(ctx, fs, ls, op.name) + if err != nil { + t.Errorf("%s: got error %v, want nil", desc, err) + continue + } + sort.Sort(byXMLName(pnames)) + sort.Sort(byXMLName(op.wantPnames)) + if !reflect.DeepEqual(pnames, op.wantPnames) { + t.Errorf("%s: pnames\ngot %q\nwant %q", desc, pnames, op.wantPnames) + } + continue + case "allprop": + propstats, err = allprop(ctx, fs, ls, op.name, op.pnames) + case "propfind": + propstats, err = props(ctx, fs, ls, op.name, op.pnames) + case "proppatch": + propstats, err = patch(ctx, fs, ls, op.name, op.patches) + default: + t.Fatalf("%s: %s not implemented", desc, op.op) + } + if err != nil { + t.Errorf("%s: got error %v, want nil", desc, err) + continue + } + // Compare return values from allprop, propfind or proppatch. + for _, pst := range propstats { + sort.Sort(byPropname(pst.Props)) + } + for _, pst := range op.wantPropstats { + sort.Sort(byPropname(pst.Props)) + } + sort.Sort(byStatus(propstats)) + sort.Sort(byStatus(op.wantPropstats)) + if !reflect.DeepEqual(propstats, op.wantPropstats) { + t.Errorf("%s: propstat\ngot %q\nwant %q", desc, propstats, op.wantPropstats) + } + } + } +} + +func cmpXMLName(a, b xml.Name) bool { + if a.Space != b.Space { + return a.Space < b.Space + } + return a.Local < b.Local +} + +type byXMLName []xml.Name + +func (b byXMLName) Len() int { return len(b) } +func (b byXMLName) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b byXMLName) Less(i, j int) bool { return cmpXMLName(b[i], b[j]) } + +type byPropname []Property + +func (b byPropname) Len() int { return len(b) } +func (b byPropname) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b byPropname) Less(i, j int) bool { return cmpXMLName(b[i].XMLName, b[j].XMLName) } + +type byStatus []Propstat + +func (b byStatus) Len() int { return len(b) } +func (b byStatus) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b byStatus) Less(i, j int) bool { return b[i].Status < b[j].Status } + +type noDeadPropsFS struct { + FileSystem +} + +func (fs noDeadPropsFS) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (File, error) { + f, err := fs.FileSystem.OpenFile(ctx, name, flag, perm) + if err != nil { + return nil, err + } + return noDeadPropsFile{f}, nil +} + +// noDeadPropsFile wraps a File but strips any optional DeadPropsHolder methods +// provided by the underlying File implementation. +type noDeadPropsFile struct { + f File +} + +func (f noDeadPropsFile) Close() error { return f.f.Close() } +func (f noDeadPropsFile) Read(p []byte) (int, error) { return f.f.Read(p) } +func (f noDeadPropsFile) Readdir(count int) ([]os.FileInfo, error) { return f.f.Readdir(count) } +func (f noDeadPropsFile) Seek(off int64, whence int) (int64, error) { return f.f.Seek(off, whence) } +func (f noDeadPropsFile) Stat() (os.FileInfo, error) { return f.f.Stat() } +func (f noDeadPropsFile) Write(p []byte) (int, error) { return f.f.Write(p) } + +type overrideContentType struct { + os.FileInfo + contentType string + err error +} + +func (o *overrideContentType) ContentType(ctx context.Context) (string, error) { + return o.contentType, o.err +} + +func TestFindContentTypeOverride(t *testing.T) { + fs, err := buildTestFS([]string{"touch /file"}) + if err != nil { + t.Fatalf("cannot create test filesystem: %v", err) + } + ctx := context.Background() + fi, err := fs.Stat(ctx, "/file") + if err != nil { + t.Fatalf("cannot Stat /file: %v", err) + } + + // Check non overridden case + originalContentType, err := findContentType(ctx, fs, nil, "/file", fi) + if err != nil { + t.Fatalf("findContentType /file failed: %v", err) + } + if originalContentType != "text/plain; charset=utf-8" { + t.Fatalf("ContentType wrong want %q got %q", "text/plain; charset=utf-8", originalContentType) + } + + // Now try overriding the ContentType + o := &overrideContentType{fi, "OverriddenContentType", nil} + ContentType, err := findContentType(ctx, fs, nil, "/file", o) + if err != nil { + t.Fatalf("findContentType /file failed: %v", err) + } + if ContentType != o.contentType { + t.Fatalf("ContentType wrong want %q got %q", o.contentType, ContentType) + } + + // Now return ErrNotImplemented and check we get the original content type + o = &overrideContentType{fi, "OverriddenContentType", ErrNotImplemented} + ContentType, err = findContentType(ctx, fs, nil, "/file", o) + if err != nil { + t.Fatalf("findContentType /file failed: %v", err) + } + if ContentType != originalContentType { + t.Fatalf("ContentType wrong want %q got %q", originalContentType, ContentType) + } +} + +type overrideETag struct { + os.FileInfo + eTag string + err error +} + +func (o *overrideETag) ETag(ctx context.Context) (string, error) { + return o.eTag, o.err +} + +func TestFindETagOverride(t *testing.T) { + fs, err := buildTestFS([]string{"touch /file"}) + if err != nil { + t.Fatalf("cannot create test filesystem: %v", err) + } + ctx := context.Background() + fi, err := fs.Stat(ctx, "/file") + if err != nil { + t.Fatalf("cannot Stat /file: %v", err) + } + + // Check non overridden case + originalETag, err := findETag(ctx, fs, nil, "/file", fi) + if err != nil { + t.Fatalf("findETag /file failed: %v", err) + } + matchETag := regexp.MustCompile(`^"-?[0-9a-f]{6,}"$`) + if !matchETag.MatchString(originalETag) { + t.Fatalf("ETag wrong, wanted something matching %v got %q", matchETag, originalETag) + } + + // Now try overriding the ETag + o := &overrideETag{fi, `"OverriddenETag"`, nil} + ETag, err := findETag(ctx, fs, nil, "/file", o) + if err != nil { + t.Fatalf("findETag /file failed: %v", err) + } + if ETag != o.eTag { + t.Fatalf("ETag wrong want %q got %q", o.eTag, ETag) + } + + // Now return ErrNotImplemented and check we get the original Etag + o = &overrideETag{fi, `"OverriddenETag"`, ErrNotImplemented} + ETag, err = findETag(ctx, fs, nil, "/file", o) + if err != nil { + t.Fatalf("findETag /file failed: %v", err) + } + if ETag != originalETag { + t.Fatalf("ETag wrong want %q got %q", originalETag, ETag) + } +} diff --git a/endpoints/drive/webdav/webdav.go b/endpoints/drive/webdav/webdav.go new file mode 100644 index 000000000..add2bcd67 --- /dev/null +++ b/endpoints/drive/webdav/webdav.go @@ -0,0 +1,736 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package webdav provides a WebDAV server implementation. +package webdav // import "golang.org/x/net/webdav" + +import ( + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "time" +) + +type Handler struct { + // Prefix is the URL path prefix to strip from WebDAV resource paths. + Prefix string + // FileSystem is the virtual file system. + FileSystem FileSystem + // LockSystem is the lock management system. + LockSystem LockSystem + // Logger is an optional error logger. If non-nil, it will be called + // for all HTTP requests. + Logger func(*http.Request, error) +} + +func (h *Handler) stripPrefix(p string) (string, int, error) { + if h.Prefix == "" { + return p, http.StatusOK, nil + } + if r := strings.TrimPrefix(p, h.Prefix); len(r) < len(p) { + return r, http.StatusOK, nil + } + return p, http.StatusNotFound, errPrefixMismatch +} + +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + status, err := http.StatusBadRequest, errUnsupportedMethod + if h.FileSystem == nil { + status, err = http.StatusInternalServerError, errNoFileSystem + } else if h.LockSystem == nil { + status, err = http.StatusInternalServerError, errNoLockSystem + } else { + switch r.Method { + case "OPTIONS": + status, err = h.handleOptions(w, r) + case "GET", "HEAD", "POST": + status, err = h.handleGetHeadPost(w, r) + case "DELETE": + status, err = h.handleDelete(w, r) + case "PUT": + status, err = h.handlePut(w, r) + case "MKCOL": + status, err = h.handleMkcol(w, r) + case "COPY", "MOVE": + status, err = h.handleCopyMove(w, r) + case "LOCK": + status, err = h.handleLock(w, r) + case "UNLOCK": + status, err = h.handleUnlock(w, r) + case "PROPFIND": + status, err = h.handlePropfind(w, r) + case "PROPPATCH": + status, err = h.handleProppatch(w, r) + } + } + + if status != 0 { + w.WriteHeader(status) + if status != http.StatusNoContent { + w.Write([]byte(StatusText(status))) + } + } + if h.Logger != nil { + h.Logger(r, err) + } +} + +func (h *Handler) lock(now time.Time, root string) (token string, status int, err error) { + token, err = h.LockSystem.Create(now, LockDetails{ + Root: root, + Duration: infiniteTimeout, + ZeroDepth: true, + }) + if err != nil { + if err == ErrLocked { + return "", StatusLocked, err + } + return "", http.StatusInternalServerError, err + } + return token, 0, nil +} + +func (h *Handler) confirmLocks(r *http.Request, src, dst string) (release func(), status int, err error) { + hdr := r.Header.Get("If") + if hdr == "" { + // An empty If header means that the client hasn't previously created locks. + // Even if this client doesn't care about locks, we still need to check that + // the resources aren't locked by another client, so we create temporary + // locks that would conflict with another client's locks. These temporary + // locks are unlocked at the end of the HTTP request. + now, srcToken, dstToken := time.Now(), "", "" + if src != "" { + srcToken, status, err = h.lock(now, src) + if err != nil { + return nil, status, err + } + } + if dst != "" { + dstToken, status, err = h.lock(now, dst) + if err != nil { + if srcToken != "" { + h.LockSystem.Unlock(now, srcToken) + } + return nil, status, err + } + } + + return func() { + if dstToken != "" { + h.LockSystem.Unlock(now, dstToken) + } + if srcToken != "" { + h.LockSystem.Unlock(now, srcToken) + } + }, 0, nil + } + + ih, ok := parseIfHeader(hdr) + if !ok { + return nil, http.StatusBadRequest, errInvalidIfHeader + } + // ih is a disjunction (OR) of ifLists, so any ifList will do. + for _, l := range ih.lists { + lsrc := l.resourceTag + if lsrc == "" { + lsrc = src + } else { + u, err := url.Parse(lsrc) + if err != nil { + continue + } + if u.Host != r.Host { + continue + } + lsrc, status, err = h.stripPrefix(u.Path) + if err != nil { + return nil, status, err + } + } + release, err = h.LockSystem.Confirm(time.Now(), lsrc, dst, l.conditions...) + if err == ErrConfirmationFailed { + continue + } + if err != nil { + return nil, http.StatusInternalServerError, err + } + return release, 0, nil + } + // Section 10.4.1 says that "If this header is evaluated and all state lists + // fail, then the request must fail with a 412 (Precondition Failed) status." + // We follow the spec even though the cond_put_corrupt_token test case from + // the litmus test warns on seeing a 412 instead of a 423 (Locked). + return nil, http.StatusPreconditionFailed, ErrLocked +} + +func (h *Handler) handleOptions(w http.ResponseWriter, r *http.Request) (status int, err error) { + reqPath, status, err := h.stripPrefix(r.URL.Path) + if err != nil { + return status, err + } + ctx := r.Context() + allow := "OPTIONS, LOCK, PUT, MKCOL" + if fi, err := h.FileSystem.Stat(ctx, reqPath); err == nil { + if fi.IsDir() { + allow = "OPTIONS, LOCK, DELETE, PROPPATCH, COPY, MOVE, UNLOCK, PROPFIND" + } else { + allow = "OPTIONS, LOCK, GET, HEAD, POST, DELETE, PROPPATCH, COPY, MOVE, UNLOCK, PROPFIND, PUT" + } + } + w.Header().Set("Allow", allow) + // http://www.webdav.org/specs/rfc4918.html#dav.compliance.classes + w.Header().Set("DAV", "1, 2") + // http://msdn.microsoft.com/en-au/library/cc250217.aspx + w.Header().Set("MS-Author-Via", "DAV") + return 0, nil +} + +func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (status int, err error) { + reqPath, status, err := h.stripPrefix(r.URL.Path) + if err != nil { + return status, err + } + // TODO: check locks for read-only access?? + ctx := r.Context() + f, err := h.FileSystem.OpenFile(ctx, reqPath, os.O_RDONLY, 0) + if err != nil { + return http.StatusNotFound, err + } + defer f.Close() + fi, err := f.Stat() + if err != nil { + return http.StatusNotFound, err + } + if fi.IsDir() { + return http.StatusMethodNotAllowed, nil + } + etag, err := findETag(ctx, h.FileSystem, h.LockSystem, reqPath, fi) + if err != nil { + return http.StatusInternalServerError, err + } + w.Header().Set("ETag", etag) + // Let ServeContent determine the Content-Type header. + http.ServeContent(w, r, reqPath, fi.ModTime(), f) + return 0, nil +} + +func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status int, err error) { + reqPath, status, err := h.stripPrefix(r.URL.Path) + if err != nil { + return status, err + } + release, status, err := h.confirmLocks(r, reqPath, "") + if err != nil { + return status, err + } + defer release() + + ctx := r.Context() + + // TODO: return MultiStatus where appropriate. + + // "godoc os RemoveAll" says that "If the path does not exist, RemoveAll + // returns nil (no error)." WebDAV semantics are that it should return a + // "404 Not Found". We therefore have to Stat before we RemoveAll. + if _, err := h.FileSystem.Stat(ctx, reqPath); err != nil { + if os.IsNotExist(err) { + return http.StatusNotFound, err + } + return http.StatusMethodNotAllowed, err + } + if err := h.FileSystem.RemoveAll(ctx, reqPath); err != nil { + return http.StatusMethodNotAllowed, err + } + return http.StatusNoContent, nil +} + +func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, err error) { + reqPath, status, err := h.stripPrefix(r.URL.Path) + if err != nil { + return status, err + } + release, status, err := h.confirmLocks(r, reqPath, "") + if err != nil { + return status, err + } + defer release() + // TODO(rost): Support the If-Match, If-None-Match headers? See bradfitz' + // comments in http.checkEtag. + ctx := r.Context() + + f, err := h.FileSystem.OpenFile(ctx, reqPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + return http.StatusNotFound, err + } + _, copyErr := io.Copy(f, r.Body) + fi, statErr := f.Stat() + closeErr := f.Close() + // TODO(rost): Returning 405 Method Not Allowed might not be appropriate. + if copyErr != nil { + return http.StatusMethodNotAllowed, copyErr + } + if statErr != nil { + return http.StatusMethodNotAllowed, statErr + } + if closeErr != nil { + return http.StatusMethodNotAllowed, closeErr + } + etag, err := findETag(ctx, h.FileSystem, h.LockSystem, reqPath, fi) + if err != nil { + return http.StatusInternalServerError, err + } + w.Header().Set("ETag", etag) + return http.StatusCreated, nil +} + +func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status int, err error) { + reqPath, status, err := h.stripPrefix(r.URL.Path) + if err != nil { + return status, err + } + release, status, err := h.confirmLocks(r, reqPath, "") + if err != nil { + return status, err + } + defer release() + + ctx := r.Context() + + if r.ContentLength > 0 { + return http.StatusUnsupportedMediaType, nil + } + if err := h.FileSystem.Mkdir(ctx, reqPath, 0777); err != nil { + if os.IsNotExist(err) { + return http.StatusConflict, err + } + return http.StatusMethodNotAllowed, err + } + return http.StatusCreated, nil +} + +func (h *Handler) handleCopyMove(w http.ResponseWriter, r *http.Request) (status int, err error) { + hdr := r.Header.Get("Destination") + if hdr == "" { + return http.StatusBadRequest, errInvalidDestination + } + u, err := url.Parse(hdr) + if err != nil { + return http.StatusBadRequest, errInvalidDestination + } + if u.Host != "" && u.Host != r.Host { + return http.StatusBadGateway, errInvalidDestination + } + + src, status, err := h.stripPrefix(r.URL.Path) + if err != nil { + return status, err + } + + dst, status, err := h.stripPrefix(u.Path) + if err != nil { + return status, err + } + + if dst == "" { + return http.StatusBadGateway, errInvalidDestination + } + if dst == src { + return http.StatusForbidden, errDestinationEqualsSource + } + + ctx := r.Context() + + if r.Method == "COPY" { + // Section 7.5.1 says that a COPY only needs to lock the destination, + // not both destination and source. Strictly speaking, this is racy, + // even though a COPY doesn't modify the source, if a concurrent + // operation modifies the source. However, the litmus test explicitly + // checks that COPYing a locked-by-another source is OK. + release, status, err := h.confirmLocks(r, "", dst) + if err != nil { + return status, err + } + defer release() + + // Section 9.8.3 says that "The COPY method on a collection without a Depth + // header must act as if a Depth header with value "infinity" was included". + depth := infiniteDepth + if hdr := r.Header.Get("Depth"); hdr != "" { + depth = parseDepth(hdr) + if depth != 0 && depth != infiniteDepth { + // Section 9.8.3 says that "A client may submit a Depth header on a + // COPY on a collection with a value of "0" or "infinity"." + return http.StatusBadRequest, errInvalidDepth + } + } + return copyFiles(ctx, h.FileSystem, src, dst, r.Header.Get("Overwrite") != "F", depth, 0) + } + + release, status, err := h.confirmLocks(r, src, dst) + if err != nil { + return status, err + } + defer release() + + // Section 9.9.2 says that "The MOVE method on a collection must act as if + // a "Depth: infinity" header was used on it. A client must not submit a + // Depth header on a MOVE on a collection with any value but "infinity"." + if hdr := r.Header.Get("Depth"); hdr != "" { + if parseDepth(hdr) != infiniteDepth { + return http.StatusBadRequest, errInvalidDepth + } + } + return moveFiles(ctx, h.FileSystem, src, dst, r.Header.Get("Overwrite") == "T") +} + +func (h *Handler) handleLock(w http.ResponseWriter, r *http.Request) (retStatus int, retErr error) { + duration, err := parseTimeout(r.Header.Get("Timeout")) + if err != nil { + return http.StatusBadRequest, err + } + li, status, err := readLockInfo(r.Body) + if err != nil { + return status, err + } + + ctx := r.Context() + token, ld, now, created := "", LockDetails{}, time.Now(), false + if li == (lockInfo{}) { + // An empty lockInfo means to refresh the lock. + ih, ok := parseIfHeader(r.Header.Get("If")) + if !ok { + return http.StatusBadRequest, errInvalidIfHeader + } + if len(ih.lists) == 1 && len(ih.lists[0].conditions) == 1 { + token = ih.lists[0].conditions[0].Token + } + if token == "" { + return http.StatusBadRequest, errInvalidLockToken + } + ld, err = h.LockSystem.Refresh(now, token, duration) + if err != nil { + if err == ErrNoSuchLock { + return http.StatusPreconditionFailed, err + } + return http.StatusInternalServerError, err + } + + } else { + // Section 9.10.3 says that "If no Depth header is submitted on a LOCK request, + // then the request MUST act as if a "Depth:infinity" had been submitted." + depth := infiniteDepth + if hdr := r.Header.Get("Depth"); hdr != "" { + depth = parseDepth(hdr) + if depth != 0 && depth != infiniteDepth { + // Section 9.10.3 says that "Values other than 0 or infinity must not be + // used with the Depth header on a LOCK method". + return http.StatusBadRequest, errInvalidDepth + } + } + reqPath, status, err := h.stripPrefix(r.URL.Path) + if err != nil { + return status, err + } + ld = LockDetails{ + Root: reqPath, + Duration: duration, + OwnerXML: li.Owner.InnerXML, + ZeroDepth: depth == 0, + } + token, err = h.LockSystem.Create(now, ld) + if err != nil { + if err == ErrLocked { + return StatusLocked, err + } + return http.StatusInternalServerError, err + } + defer func() { + if retErr != nil { + h.LockSystem.Unlock(now, token) + } + }() + + // Create the resource if it didn't previously exist. + if _, err := h.FileSystem.Stat(ctx, reqPath); err != nil { + f, err := h.FileSystem.OpenFile(ctx, reqPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + // TODO: detect missing intermediate dirs and return http.StatusConflict? + return http.StatusInternalServerError, err + } + f.Close() + created = true + } + + // http://www.webdav.org/specs/rfc4918.html#HEADER_Lock-Token says that the + // Lock-Token value is a Coded-URL. We add angle brackets. + w.Header().Set("Lock-Token", "<"+token+">") + } + + w.Header().Set("Content-Type", "application/xml; charset=utf-8") + if created { + // This is "w.WriteHeader(http.StatusCreated)" and not "return + // http.StatusCreated, nil" because we write our own (XML) response to w + // and Handler.ServeHTTP would otherwise write "Created". + w.WriteHeader(http.StatusCreated) + } + writeLockInfo(w, token, ld) + return 0, nil +} + +func (h *Handler) handleUnlock(w http.ResponseWriter, r *http.Request) (status int, err error) { + // http://www.webdav.org/specs/rfc4918.html#HEADER_Lock-Token says that the + // Lock-Token value is a Coded-URL. We strip its angle brackets. + t := r.Header.Get("Lock-Token") + if len(t) < 2 || t[0] != '<' || t[len(t)-1] != '>' { + return http.StatusBadRequest, errInvalidLockToken + } + t = t[1 : len(t)-1] + + switch err = h.LockSystem.Unlock(time.Now(), t); err { + case nil: + return http.StatusNoContent, err + case ErrForbidden: + return http.StatusForbidden, err + case ErrLocked: + return StatusLocked, err + case ErrNoSuchLock: + return http.StatusConflict, err + default: + return http.StatusInternalServerError, err + } +} + +func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status int, err error) { + reqPath, status, err := h.stripPrefix(r.URL.Path) + if err != nil { + return status, err + } + ctx := r.Context() + fi, err := h.FileSystem.Stat(ctx, reqPath) + if err != nil { + if os.IsNotExist(err) { + return http.StatusNotFound, err + } + return http.StatusMethodNotAllowed, err + } + depth := infiniteDepth + if hdr := r.Header.Get("Depth"); hdr != "" { + depth = parseDepth(hdr) + if depth == invalidDepth { + return http.StatusBadRequest, errInvalidDepth + } + } + pf, status, err := readPropfind(r.Body) + if err != nil { + return status, err + } + + mw := multistatusWriter{w: w} + + walkFn := func(reqPath string, info os.FileInfo, err error) error { + if err != nil { + return handlePropfindError(err, info) + } + + var pstats []Propstat + if pf.Propname != nil { + pnames, err := propnames(ctx, h.FileSystem, h.LockSystem, reqPath) + if err != nil { + return handlePropfindError(err, info) + } + pstat := Propstat{Status: http.StatusOK} + for _, xmlname := range pnames { + pstat.Props = append(pstat.Props, Property{XMLName: xmlname}) + } + pstats = append(pstats, pstat) + } else if pf.Allprop != nil { + pstats, err = allprop(ctx, h.FileSystem, h.LockSystem, reqPath, pf.Prop) + } else { + pstats, err = props(ctx, h.FileSystem, h.LockSystem, reqPath, pf.Prop) + } + if err != nil { + return handlePropfindError(err, info) + } + href := path.Join(h.Prefix, reqPath) + if href != "/" && info.IsDir() { + href += "/" + } + return mw.write(makePropstatResponse(href, pstats)) + } + + walkErr := walkFS(ctx, h.FileSystem, depth, reqPath, fi, walkFn) + closeErr := mw.close() + if walkErr != nil { + return http.StatusInternalServerError, walkErr + } + if closeErr != nil { + return http.StatusInternalServerError, closeErr + } + return 0, nil +} + +func (h *Handler) handleProppatch(w http.ResponseWriter, r *http.Request) (status int, err error) { + reqPath, status, err := h.stripPrefix(r.URL.Path) + if err != nil { + return status, err + } + release, status, err := h.confirmLocks(r, reqPath, "") + if err != nil { + return status, err + } + defer release() + + ctx := r.Context() + + if _, err := h.FileSystem.Stat(ctx, reqPath); err != nil { + if os.IsNotExist(err) { + return http.StatusNotFound, err + } + return http.StatusMethodNotAllowed, err + } + patches, status, err := readProppatch(r.Body) + if err != nil { + return status, err + } + pstats, err := patch(ctx, h.FileSystem, h.LockSystem, reqPath, patches) + if err != nil { + return http.StatusInternalServerError, err + } + mw := multistatusWriter{w: w} + writeErr := mw.write(makePropstatResponse(r.URL.Path, pstats)) + closeErr := mw.close() + if writeErr != nil { + return http.StatusInternalServerError, writeErr + } + if closeErr != nil { + return http.StatusInternalServerError, closeErr + } + return 0, nil +} + +func makePropstatResponse(href string, pstats []Propstat) *response { + resp := response{ + Href: []string{(&url.URL{Path: href}).EscapedPath()}, + Propstat: make([]propstat, 0, len(pstats)), + } + for _, p := range pstats { + var xmlErr *xmlError + if p.XMLError != "" { + xmlErr = &xmlError{InnerXML: []byte(p.XMLError)} + } + resp.Propstat = append(resp.Propstat, propstat{ + Status: fmt.Sprintf("HTTP/1.1 %d %s", p.Status, StatusText(p.Status)), + Prop: p.Props, + ResponseDescription: p.ResponseDescription, + Error: xmlErr, + }) + } + return &resp +} + +func handlePropfindError(err error, info os.FileInfo) error { + var skipResp error = nil + if info != nil && info.IsDir() { + skipResp = filepath.SkipDir + } + + if errors.Is(err, os.ErrPermission) { + // If the server cannot recurse into a directory because it is not allowed, + // then there is nothing more to say about it. Just skip sending anything. + return skipResp + } + + if _, ok := err.(*os.PathError); ok { + // If the file is just bad, it couldn't be a proper WebDAV resource. Skip it. + return skipResp + } + + // We need to be careful with other errors: there is no way to abort the xml stream + // part way through while returning a valid PROPFIND response. Returning only half + // the data would be misleading, but so would be returning results tainted by errors. + // The current behaviour by returning an error here leads to the stream being aborted, + // and the parent http server complaining about writing a spurious header. We should + // consider further enhancing this error handling to more gracefully fail, or perhaps + // buffer the entire response until we've walked the tree. + return err +} + +const ( + infiniteDepth = -1 + invalidDepth = -2 +) + +// parseDepth maps the strings "0", "1" and "infinity" to 0, 1 and +// infiniteDepth. Parsing any other string returns invalidDepth. +// +// Different WebDAV methods have further constraints on valid depths: +// - PROPFIND has no further restrictions, as per section 9.1. +// - COPY accepts only "0" or "infinity", as per section 9.8.3. +// - MOVE accepts only "infinity", as per section 9.9.2. +// - LOCK accepts only "0" or "infinity", as per section 9.10.3. +// +// These constraints are enforced by the handleXxx methods. +func parseDepth(s string) int { + switch s { + case "0": + return 0 + case "1": + return 1 + case "infinity": + return infiniteDepth + } + return invalidDepth +} + +// http://www.webdav.org/specs/rfc4918.html#status.code.extensions.to.http11 +const ( + StatusMulti = 207 + StatusUnprocessableEntity = 422 + StatusLocked = 423 + StatusFailedDependency = 424 + StatusInsufficientStorage = 507 +) + +func StatusText(code int) string { + switch code { + case StatusMulti: + return "Multi-Status" + case StatusUnprocessableEntity: + return "Unprocessable Entity" + case StatusLocked: + return "Locked" + case StatusFailedDependency: + return "Failed Dependency" + case StatusInsufficientStorage: + return "Insufficient Storage" + } + return http.StatusText(code) +} + +var ( + errDestinationEqualsSource = errors.New("webdav: destination equals source") + errDirectoryNotEmpty = errors.New("webdav: directory not empty") + errInvalidDepth = errors.New("webdav: invalid depth") + errInvalidDestination = errors.New("webdav: invalid destination") + errInvalidIfHeader = errors.New("webdav: invalid If header") + errInvalidLockInfo = errors.New("webdav: invalid lock info") + errInvalidLockToken = errors.New("webdav: invalid lock token") + errInvalidPropfind = errors.New("webdav: invalid propfind") + errInvalidProppatch = errors.New("webdav: invalid proppatch") + errInvalidResponse = errors.New("webdav: invalid response") + errInvalidTimeout = errors.New("webdav: invalid timeout") + errNoFileSystem = errors.New("webdav: no file system") + errNoLockSystem = errors.New("webdav: no lock system") + errNotADirectory = errors.New("webdav: not a directory") + errPrefixMismatch = errors.New("webdav: prefix mismatch") + errRecursionTooDeep = errors.New("webdav: recursion too deep") + errUnsupportedLockInfo = errors.New("webdav: unsupported lock info") + errUnsupportedMethod = errors.New("webdav: unsupported method") +) diff --git a/endpoints/drive/webdav/webdav_test.go b/endpoints/drive/webdav/webdav_test.go new file mode 100644 index 000000000..2baebe3c9 --- /dev/null +++ b/endpoints/drive/webdav/webdav_test.go @@ -0,0 +1,349 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package webdav + +import ( + "context" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "os" + "reflect" + "regexp" + "sort" + "strings" + "testing" +) + +// TODO: add tests to check XML responses with the expected prefix path +func TestPrefix(t *testing.T) { + const dst, blah = "Destination", "blah blah blah" + + // createLockBody comes from the example in Section 9.10.7. + const createLockBody = ` + + + + + http://example.org/~ejw/contact.html + + + ` + + do := func(method, urlStr string, body string, wantStatusCode int, headers ...string) (http.Header, error) { + var bodyReader io.Reader + if body != "" { + bodyReader = strings.NewReader(body) + } + req, err := http.NewRequest(method, urlStr, bodyReader) + if err != nil { + return nil, err + } + for len(headers) >= 2 { + req.Header.Add(headers[0], headers[1]) + headers = headers[2:] + } + res, err := http.DefaultTransport.RoundTrip(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != wantStatusCode { + return nil, fmt.Errorf("got status code %d, want %d", res.StatusCode, wantStatusCode) + } + return res.Header, nil + } + + prefixes := []string{ + "/", + "/a/", + "/a/b/", + "/a/b/c/", + } + ctx := context.Background() + for _, prefix := range prefixes { + fs := NewMemFS() + h := &Handler{ + FileSystem: fs, + LockSystem: NewMemLS(), + } + mux := http.NewServeMux() + if prefix != "/" { + h.Prefix = prefix + } + mux.Handle(prefix, h) + srv := httptest.NewServer(mux) + defer srv.Close() + + // The script is: + // MKCOL /a + // MKCOL /a/b + // PUT /a/b/c + // COPY /a/b/c /a/b/d + // MKCOL /a/b/e + // MOVE /a/b/d /a/b/e/f + // LOCK /a/b/e/g + // PUT /a/b/e/g + // which should yield the (possibly stripped) filenames /a/b/c, + // /a/b/e/f and /a/b/e/g, plus their parent directories. + + wantA := map[string]int{ + "/": http.StatusCreated, + "/a/": http.StatusMovedPermanently, + "/a/b/": http.StatusNotFound, + "/a/b/c/": http.StatusNotFound, + }[prefix] + if _, err := do("MKCOL", srv.URL+"/a", "", wantA); err != nil { + t.Errorf("prefix=%-9q MKCOL /a: %v", prefix, err) + continue + } + + wantB := map[string]int{ + "/": http.StatusCreated, + "/a/": http.StatusCreated, + "/a/b/": http.StatusMovedPermanently, + "/a/b/c/": http.StatusNotFound, + }[prefix] + if _, err := do("MKCOL", srv.URL+"/a/b", "", wantB); err != nil { + t.Errorf("prefix=%-9q MKCOL /a/b: %v", prefix, err) + continue + } + + wantC := map[string]int{ + "/": http.StatusCreated, + "/a/": http.StatusCreated, + "/a/b/": http.StatusCreated, + "/a/b/c/": http.StatusMovedPermanently, + }[prefix] + if _, err := do("PUT", srv.URL+"/a/b/c", blah, wantC); err != nil { + t.Errorf("prefix=%-9q PUT /a/b/c: %v", prefix, err) + continue + } + + wantD := map[string]int{ + "/": http.StatusCreated, + "/a/": http.StatusCreated, + "/a/b/": http.StatusCreated, + "/a/b/c/": http.StatusMovedPermanently, + }[prefix] + if _, err := do("COPY", srv.URL+"/a/b/c", "", wantD, dst, srv.URL+"/a/b/d"); err != nil { + t.Errorf("prefix=%-9q COPY /a/b/c /a/b/d: %v", prefix, err) + continue + } + + wantE := map[string]int{ + "/": http.StatusCreated, + "/a/": http.StatusCreated, + "/a/b/": http.StatusCreated, + "/a/b/c/": http.StatusNotFound, + }[prefix] + if _, err := do("MKCOL", srv.URL+"/a/b/e", "", wantE); err != nil { + t.Errorf("prefix=%-9q MKCOL /a/b/e: %v", prefix, err) + continue + } + + wantF := map[string]int{ + "/": http.StatusCreated, + "/a/": http.StatusCreated, + "/a/b/": http.StatusCreated, + "/a/b/c/": http.StatusNotFound, + }[prefix] + if _, err := do("MOVE", srv.URL+"/a/b/d", "", wantF, dst, srv.URL+"/a/b/e/f"); err != nil { + t.Errorf("prefix=%-9q MOVE /a/b/d /a/b/e/f: %v", prefix, err) + continue + } + + var lockToken string + wantG := map[string]int{ + "/": http.StatusCreated, + "/a/": http.StatusCreated, + "/a/b/": http.StatusCreated, + "/a/b/c/": http.StatusNotFound, + }[prefix] + if h, err := do("LOCK", srv.URL+"/a/b/e/g", createLockBody, wantG); err != nil { + t.Errorf("prefix=%-9q LOCK /a/b/e/g: %v", prefix, err) + continue + } else { + lockToken = h.Get("Lock-Token") + } + + ifHeader := fmt.Sprintf("<%s/a/b/e/g> (%s)", srv.URL, lockToken) + wantH := map[string]int{ + "/": http.StatusCreated, + "/a/": http.StatusCreated, + "/a/b/": http.StatusCreated, + "/a/b/c/": http.StatusNotFound, + }[prefix] + if _, err := do("PUT", srv.URL+"/a/b/e/g", blah, wantH, "If", ifHeader); err != nil { + t.Errorf("prefix=%-9q PUT /a/b/e/g: %v", prefix, err) + continue + } + + got, err := find(ctx, nil, fs, "/") + if err != nil { + t.Errorf("prefix=%-9q find: %v", prefix, err) + continue + } + sort.Strings(got) + want := map[string][]string{ + "/": {"/", "/a", "/a/b", "/a/b/c", "/a/b/e", "/a/b/e/f", "/a/b/e/g"}, + "/a/": {"/", "/b", "/b/c", "/b/e", "/b/e/f", "/b/e/g"}, + "/a/b/": {"/", "/c", "/e", "/e/f", "/e/g"}, + "/a/b/c/": {"/"}, + }[prefix] + if !reflect.DeepEqual(got, want) { + t.Errorf("prefix=%-9q find:\ngot %v\nwant %v", prefix, got, want) + continue + } + } +} + +func TestEscapeXML(t *testing.T) { + // These test cases aren't exhaustive, and there is more than one way to + // escape e.g. a quot (as """ or """) or an apos. We presume that + // the encoding/xml package tests xml.EscapeText more thoroughly. This test + // here is just a sanity check for this package's escapeXML function, and + // its attempt to provide a fast path (and avoid a bytes.Buffer allocation) + // when escaping filenames is obviously a no-op. + testCases := map[string]string{ + "": "", + " ": " ", + "&": "&", + "*": "*", + "+": "+", + ",": ",", + "-": "-", + ".": ".", + "/": "/", + "0": "0", + "9": "9", + ":": ":", + "<": "<", + ">": ">", + "A": "A", + "_": "_", + "a": "a", + "~": "~", + "\u0201": "\u0201", + "&": "&amp;", + "foo&baz": "foo&<b/ar>baz", + } + + for in, want := range testCases { + if got := escapeXML(in); got != want { + t.Errorf("in=%q: got %q, want %q", in, got, want) + } + } +} + +func TestFilenameEscape(t *testing.T) { + hrefRe := regexp.MustCompile(`([^<]*)`) + displayNameRe := regexp.MustCompile(`([^<]*)`) + do := func(method, urlStr string) (string, string, error) { + req, err := http.NewRequest(method, urlStr, nil) + if err != nil { + return "", "", err + } + res, err := http.DefaultClient.Do(req) + if err != nil { + return "", "", err + } + defer res.Body.Close() + + b, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", "", err + } + hrefMatch := hrefRe.FindStringSubmatch(string(b)) + if len(hrefMatch) != 2 { + return "", "", errors.New("D:href not found") + } + displayNameMatch := displayNameRe.FindStringSubmatch(string(b)) + if len(displayNameMatch) != 2 { + return "", "", errors.New("D:displayname not found") + } + + return hrefMatch[1], displayNameMatch[1], nil + } + + testCases := []struct { + name, wantHref, wantDisplayName string + }{{ + name: `/foo%bar`, + wantHref: `/foo%25bar`, + wantDisplayName: `foo%bar`, + }, { + name: `/こんにちわ世界`, + wantHref: `/%E3%81%93%E3%82%93%E3%81%AB%E3%81%A1%E3%82%8F%E4%B8%96%E7%95%8C`, + wantDisplayName: `こんにちわ世界`, + }, { + name: `/Program Files/`, + wantHref: `/Program%20Files/`, + wantDisplayName: `Program Files`, + }, { + name: `/go+lang`, + wantHref: `/go+lang`, + wantDisplayName: `go+lang`, + }, { + name: `/go&lang`, + wantHref: `/go&lang`, + wantDisplayName: `go&lang`, + }, { + name: `/goexclusive"` + Shared *struct{} `xml:"lockscope>shared"` + Write *struct{} `xml:"locktype>write"` + Owner owner `xml:"owner"` +} + +// http://www.webdav.org/specs/rfc4918.html#ELEMENT_owner +type owner struct { + InnerXML string `xml:",innerxml"` +} + +func readLockInfo(r io.Reader) (li lockInfo, status int, err error) { + c := &countingReader{r: r} + if err = ixml.NewDecoder(c).Decode(&li); err != nil { + if err == io.EOF { + if c.n == 0 { + // An empty body means to refresh the lock. + // http://www.webdav.org/specs/rfc4918.html#refreshing-locks + return lockInfo{}, 0, nil + } + err = errInvalidLockInfo + } + return lockInfo{}, http.StatusBadRequest, err + } + // We only support exclusive (non-shared) write locks. In practice, these are + // the only types of locks that seem to matter. + if li.Exclusive == nil || li.Shared != nil || li.Write == nil { + return lockInfo{}, http.StatusNotImplemented, errUnsupportedLockInfo + } + return li, 0, nil +} + +type countingReader struct { + n int + r io.Reader +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += n + return n, err +} + +func writeLockInfo(w io.Writer, token string, ld LockDetails) (int, error) { + depth := "infinity" + if ld.ZeroDepth { + depth = "0" + } + timeout := ld.Duration / time.Second + return fmt.Fprintf(w, "\n"+ + "\n"+ + " \n"+ + " \n"+ + " %s\n"+ + " %s\n"+ + " Second-%d\n"+ + " %s\n"+ + " %s\n"+ + "", + depth, ld.OwnerXML, timeout, escape(token), escape(ld.Root), + ) +} + +func escape(s string) string { + for i := 0; i < len(s); i++ { + switch s[i] { + case '"', '&', '\'', '<', '>': + b := bytes.NewBuffer(nil) + ixml.EscapeText(b, []byte(s)) + return b.String() + } + } + return s +} + +// next returns the next token, if any, in the XML stream of d. +// RFC 4918 requires to ignore comments, processing instructions +// and directives. +// http://www.webdav.org/specs/rfc4918.html#property_values +// http://www.webdav.org/specs/rfc4918.html#xml-extensibility +func next(d *ixml.Decoder) (ixml.Token, error) { + for { + t, err := d.Token() + if err != nil { + return t, err + } + switch t.(type) { + case ixml.Comment, ixml.Directive, ixml.ProcInst: + continue + default: + return t, nil + } + } +} + +// http://www.webdav.org/specs/rfc4918.html#ELEMENT_prop (for propfind) +type propfindProps []xml.Name + +// UnmarshalXML appends the property names enclosed within start to pn. +// +// It returns an error if start does not contain any properties or if +// properties contain values. Character data between properties is ignored. +func (pn *propfindProps) UnmarshalXML(d *ixml.Decoder, start ixml.StartElement) error { + for { + t, err := next(d) + if err != nil { + return err + } + switch t.(type) { + case ixml.EndElement: + if len(*pn) == 0 { + return fmt.Errorf("%s must not be empty", start.Name.Local) + } + return nil + case ixml.StartElement: + name := t.(ixml.StartElement).Name + t, err = next(d) + if err != nil { + return err + } + if _, ok := t.(ixml.EndElement); !ok { + return fmt.Errorf("unexpected token %T", t) + } + *pn = append(*pn, xml.Name(name)) + } + } +} + +// http://www.webdav.org/specs/rfc4918.html#ELEMENT_propfind +type propfind struct { + XMLName ixml.Name `xml:"DAV: propfind"` + Allprop *struct{} `xml:"DAV: allprop"` + Propname *struct{} `xml:"DAV: propname"` + Prop propfindProps `xml:"DAV: prop"` + Include propfindProps `xml:"DAV: include"` +} + +func readPropfind(r io.Reader) (pf propfind, status int, err error) { + c := countingReader{r: r} + if err = ixml.NewDecoder(&c).Decode(&pf); err != nil { + if err == io.EOF { + if c.n == 0 { + // An empty body means to propfind allprop. + // http://www.webdav.org/specs/rfc4918.html#METHOD_PROPFIND + return propfind{Allprop: new(struct{})}, 0, nil + } + err = errInvalidPropfind + } + return propfind{}, http.StatusBadRequest, err + } + + if pf.Allprop == nil && pf.Include != nil { + return propfind{}, http.StatusBadRequest, errInvalidPropfind + } + if pf.Allprop != nil && (pf.Prop != nil || pf.Propname != nil) { + return propfind{}, http.StatusBadRequest, errInvalidPropfind + } + if pf.Prop != nil && pf.Propname != nil { + return propfind{}, http.StatusBadRequest, errInvalidPropfind + } + if pf.Propname == nil && pf.Allprop == nil && pf.Prop == nil { + return propfind{}, http.StatusBadRequest, errInvalidPropfind + } + return pf, 0, nil +} + +// Property represents a single DAV resource property as defined in RFC 4918. +// See http://www.webdav.org/specs/rfc4918.html#data.model.for.resource.properties +type Property struct { + // XMLName is the fully qualified name that identifies this property. + XMLName xml.Name + + // Lang is an optional xml:lang attribute. + Lang string `xml:"xml:lang,attr,omitempty"` + + // InnerXML contains the XML representation of the property value. + // See http://www.webdav.org/specs/rfc4918.html#property_values + // + // Property values of complex type or mixed-content must have fully + // expanded XML namespaces or be self-contained with according + // XML namespace declarations. They must not rely on any XML + // namespace declarations within the scope of the XML document, + // even including the DAV: namespace. + InnerXML []byte `xml:",innerxml"` +} + +// ixmlProperty is the same as the Property type except it holds an ixml.Name +// instead of an xml.Name. +type ixmlProperty struct { + XMLName ixml.Name + Lang string `xml:"xml:lang,attr,omitempty"` + InnerXML []byte `xml:",innerxml"` +} + +// http://www.webdav.org/specs/rfc4918.html#ELEMENT_error +// See multistatusWriter for the "D:" namespace prefix. +type xmlError struct { + XMLName ixml.Name `xml:"D:error"` + InnerXML []byte `xml:",innerxml"` +} + +// http://www.webdav.org/specs/rfc4918.html#ELEMENT_propstat +// See multistatusWriter for the "D:" namespace prefix. +type propstat struct { + Prop []Property `xml:"D:prop>_ignored_"` + Status string `xml:"D:status"` + Error *xmlError `xml:"D:error"` + ResponseDescription string `xml:"D:responsedescription,omitempty"` +} + +// ixmlPropstat is the same as the propstat type except it holds an ixml.Name +// instead of an xml.Name. +type ixmlPropstat struct { + Prop []ixmlProperty `xml:"D:prop>_ignored_"` + Status string `xml:"D:status"` + Error *xmlError `xml:"D:error"` + ResponseDescription string `xml:"D:responsedescription,omitempty"` +} + +// MarshalXML prepends the "D:" namespace prefix on properties in the DAV: namespace +// before encoding. See multistatusWriter. +func (ps propstat) MarshalXML(e *ixml.Encoder, start ixml.StartElement) error { + // Convert from a propstat to an ixmlPropstat. + ixmlPs := ixmlPropstat{ + Prop: make([]ixmlProperty, len(ps.Prop)), + Status: ps.Status, + Error: ps.Error, + ResponseDescription: ps.ResponseDescription, + } + for k, prop := range ps.Prop { + ixmlPs.Prop[k] = ixmlProperty{ + XMLName: ixml.Name(prop.XMLName), + Lang: prop.Lang, + InnerXML: prop.InnerXML, + } + } + + for k, prop := range ixmlPs.Prop { + if prop.XMLName.Space == "DAV:" { + prop.XMLName = ixml.Name{Space: "", Local: "D:" + prop.XMLName.Local} + ixmlPs.Prop[k] = prop + } + } + // Distinct type to avoid infinite recursion of MarshalXML. + type newpropstat ixmlPropstat + return e.EncodeElement(newpropstat(ixmlPs), start) +} + +// http://www.webdav.org/specs/rfc4918.html#ELEMENT_response +// See multistatusWriter for the "D:" namespace prefix. +type response struct { + XMLName ixml.Name `xml:"D:response"` + Href []string `xml:"D:href"` + Propstat []propstat `xml:"D:propstat"` + Status string `xml:"D:status,omitempty"` + Error *xmlError `xml:"D:error"` + ResponseDescription string `xml:"D:responsedescription,omitempty"` +} + +// MultistatusWriter marshals one or more Responses into a XML +// multistatus response. +// See http://www.webdav.org/specs/rfc4918.html#ELEMENT_multistatus +// TODO(rsto, mpl): As a workaround, the "D:" namespace prefix, defined as +// "DAV:" on this element, is prepended on the nested response, as well as on all +// its nested elements. All property names in the DAV: namespace are prefixed as +// well. This is because some versions of Mini-Redirector (on windows 7) ignore +// elements with a default namespace (no prefixed namespace). A less intrusive fix +// should be possible after golang.org/cl/11074. See https://golang.org/issue/11177 +type multistatusWriter struct { + // ResponseDescription contains the optional responsedescription + // of the multistatus XML element. Only the latest content before + // close will be emitted. Empty response descriptions are not + // written. + responseDescription string + + w http.ResponseWriter + enc *ixml.Encoder +} + +// Write validates and emits a DAV response as part of a multistatus response +// element. +// +// It sets the HTTP status code of its underlying http.ResponseWriter to 207 +// (Multi-Status) and populates the Content-Type header. If r is the +// first, valid response to be written, Write prepends the XML representation +// of r with a multistatus tag. Callers must call close after the last response +// has been written. +func (w *multistatusWriter) write(r *response) error { + switch len(r.Href) { + case 0: + return errInvalidResponse + case 1: + if len(r.Propstat) > 0 != (r.Status == "") { + return errInvalidResponse + } + default: + if len(r.Propstat) > 0 || r.Status == "" { + return errInvalidResponse + } + } + err := w.writeHeader() + if err != nil { + return err + } + return w.enc.Encode(r) +} + +// writeHeader writes a XML multistatus start element on w's underlying +// http.ResponseWriter and returns the result of the write operation. +// After the first write attempt, writeHeader becomes a no-op. +func (w *multistatusWriter) writeHeader() error { + if w.enc != nil { + return nil + } + w.w.Header().Add("Content-Type", "text/xml; charset=utf-8") + w.w.WriteHeader(StatusMulti) + _, err := fmt.Fprintf(w.w, ``) + if err != nil { + return err + } + w.enc = ixml.NewEncoder(w.w) + return w.enc.EncodeToken(ixml.StartElement{ + Name: ixml.Name{ + Space: "DAV:", + Local: "multistatus", + }, + Attr: []ixml.Attr{{ + Name: ixml.Name{Space: "xmlns", Local: "D"}, + Value: "DAV:", + }}, + }) +} + +// Close completes the marshalling of the multistatus response. It returns +// an error if the multistatus response could not be completed. If both the +// return value and field enc of w are nil, then no multistatus response has +// been written. +func (w *multistatusWriter) close() error { + if w.enc == nil { + return nil + } + var end []ixml.Token + if w.responseDescription != "" { + name := ixml.Name{Space: "DAV:", Local: "responsedescription"} + end = append(end, + ixml.StartElement{Name: name}, + ixml.CharData(w.responseDescription), + ixml.EndElement{Name: name}, + ) + } + end = append(end, ixml.EndElement{ + Name: ixml.Name{Space: "DAV:", Local: "multistatus"}, + }) + for _, t := range end { + err := w.enc.EncodeToken(t) + if err != nil { + return err + } + } + return w.enc.Flush() +} + +var xmlLangName = ixml.Name{Space: "http://www.w3.org/XML/1998/namespace", Local: "lang"} + +func xmlLang(s ixml.StartElement, d string) string { + for _, attr := range s.Attr { + if attr.Name == xmlLangName { + return attr.Value + } + } + return d +} + +type xmlValue []byte + +func (v *xmlValue) UnmarshalXML(d *ixml.Decoder, start ixml.StartElement) error { + // The XML value of a property can be arbitrary, mixed-content XML. + // To make sure that the unmarshalled value contains all required + // namespaces, we encode all the property value XML tokens into a + // buffer. This forces the encoder to redeclare any used namespaces. + var b bytes.Buffer + e := ixml.NewEncoder(&b) + for { + t, err := next(d) + if err != nil { + return err + } + if e, ok := t.(ixml.EndElement); ok && e.Name == start.Name { + break + } + if err = e.EncodeToken(t); err != nil { + return err + } + } + err := e.Flush() + if err != nil { + return err + } + *v = b.Bytes() + return nil +} + +// http://www.webdav.org/specs/rfc4918.html#ELEMENT_prop (for proppatch) +type proppatchProps []Property + +// UnmarshalXML appends the property names and values enclosed within start +// to ps. +// +// An xml:lang attribute that is defined either on the DAV:prop or property +// name XML element is propagated to the property's Lang field. +// +// UnmarshalXML returns an error if start does not contain any properties or if +// property values contain syntactically incorrect XML. +func (ps *proppatchProps) UnmarshalXML(d *ixml.Decoder, start ixml.StartElement) error { + lang := xmlLang(start, "") + for { + t, err := next(d) + if err != nil { + return err + } + switch elem := t.(type) { + case ixml.EndElement: + if len(*ps) == 0 { + return fmt.Errorf("%s must not be empty", start.Name.Local) + } + return nil + case ixml.StartElement: + p := Property{ + XMLName: xml.Name(t.(ixml.StartElement).Name), + Lang: xmlLang(t.(ixml.StartElement), lang), + } + err = d.DecodeElement(((*xmlValue)(&p.InnerXML)), &elem) + if err != nil { + return err + } + *ps = append(*ps, p) + } + } +} + +// http://www.webdav.org/specs/rfc4918.html#ELEMENT_set +// http://www.webdav.org/specs/rfc4918.html#ELEMENT_remove +type setRemove struct { + XMLName ixml.Name + Lang string `xml:"xml:lang,attr,omitempty"` + Prop proppatchProps `xml:"DAV: prop"` +} + +// http://www.webdav.org/specs/rfc4918.html#ELEMENT_propertyupdate +type propertyupdate struct { + XMLName ixml.Name `xml:"DAV: propertyupdate"` + Lang string `xml:"xml:lang,attr,omitempty"` + SetRemove []setRemove `xml:",any"` +} + +func readProppatch(r io.Reader) (patches []Proppatch, status int, err error) { + var pu propertyupdate + if err = ixml.NewDecoder(r).Decode(&pu); err != nil { + return nil, http.StatusBadRequest, err + } + for _, op := range pu.SetRemove { + remove := false + switch op.XMLName { + case ixml.Name{Space: "DAV:", Local: "set"}: + // No-op. + case ixml.Name{Space: "DAV:", Local: "remove"}: + for _, p := range op.Prop { + if len(p.InnerXML) > 0 { + return nil, http.StatusBadRequest, errInvalidProppatch + } + } + remove = true + default: + return nil, http.StatusBadRequest, errInvalidProppatch + } + patches = append(patches, Proppatch{Remove: remove, Props: op.Prop}) + } + return patches, 0, nil +} diff --git a/endpoints/drive/webdav/xml_test.go b/endpoints/drive/webdav/xml_test.go new file mode 100644 index 000000000..6812330a4 --- /dev/null +++ b/endpoints/drive/webdav/xml_test.go @@ -0,0 +1,905 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package webdav + +import ( + "bytes" + "encoding/xml" + "fmt" + "io" + "net/http" + "net/http/httptest" + "reflect" + "sort" + "strings" + "testing" + + ixml "github.com/openziti/zrok/endpoints/drive/webdav/internal/xml" +) + +func TestReadLockInfo(t *testing.T) { + // The "section x.y.z" test cases come from section x.y.z of the spec at + // http://www.webdav.org/specs/rfc4918.html + testCases := []struct { + desc string + input string + wantLI lockInfo + wantStatus int + }{{ + "bad: junk", + "xxx", + lockInfo{}, + http.StatusBadRequest, + }, { + "bad: invalid owner XML", + "" + + "\n" + + " \n" + + " \n" + + " \n" + + " no end tag \n" + + " \n" + + "", + lockInfo{}, + http.StatusBadRequest, + }, { + "bad: invalid UTF-8", + "" + + "\n" + + " \n" + + " \n" + + " \n" + + " \xff \n" + + " \n" + + "", + lockInfo{}, + http.StatusBadRequest, + }, { + "bad: unfinished XML #1", + "" + + "\n" + + " \n" + + " \n", + lockInfo{}, + http.StatusBadRequest, + }, { + "bad: unfinished XML #2", + "" + + "\n" + + " \n" + + " \n" + + " \n", + lockInfo{}, + http.StatusBadRequest, + }, { + "good: empty", + "", + lockInfo{}, + 0, + }, { + "good: plain-text owner", + "" + + "\n" + + " \n" + + " \n" + + " gopher\n" + + "", + lockInfo{ + XMLName: ixml.Name{Space: "DAV:", Local: "lockinfo"}, + Exclusive: new(struct{}), + Write: new(struct{}), + Owner: owner{ + InnerXML: "gopher", + }, + }, + 0, + }, { + "section 9.10.7", + "" + + "\n" + + " \n" + + " \n" + + " \n" + + " http://example.org/~ejw/contact.html\n" + + " \n" + + "", + lockInfo{ + XMLName: ixml.Name{Space: "DAV:", Local: "lockinfo"}, + Exclusive: new(struct{}), + Write: new(struct{}), + Owner: owner{ + InnerXML: "\n http://example.org/~ejw/contact.html\n ", + }, + }, + 0, + }} + + for _, tc := range testCases { + li, status, err := readLockInfo(strings.NewReader(tc.input)) + if tc.wantStatus != 0 { + if err == nil { + t.Errorf("%s: got nil error, want non-nil", tc.desc) + continue + } + } else if err != nil { + t.Errorf("%s: %v", tc.desc, err) + continue + } + if !reflect.DeepEqual(li, tc.wantLI) || status != tc.wantStatus { + t.Errorf("%s:\ngot lockInfo=%v, status=%v\nwant lockInfo=%v, status=%v", + tc.desc, li, status, tc.wantLI, tc.wantStatus) + continue + } + } +} + +func TestReadPropfind(t *testing.T) { + testCases := []struct { + desc string + input string + wantPF propfind + wantStatus int + }{{ + desc: "propfind: propname", + input: "" + + "\n" + + " \n" + + "", + wantPF: propfind{ + XMLName: ixml.Name{Space: "DAV:", Local: "propfind"}, + Propname: new(struct{}), + }, + }, { + desc: "propfind: empty body means allprop", + input: "", + wantPF: propfind{ + Allprop: new(struct{}), + }, + }, { + desc: "propfind: allprop", + input: "" + + "\n" + + " \n" + + "", + wantPF: propfind{ + XMLName: ixml.Name{Space: "DAV:", Local: "propfind"}, + Allprop: new(struct{}), + }, + }, { + desc: "propfind: allprop followed by include", + input: "" + + "\n" + + " \n" + + " \n" + + "", + wantPF: propfind{ + XMLName: ixml.Name{Space: "DAV:", Local: "propfind"}, + Allprop: new(struct{}), + Include: propfindProps{xml.Name{Space: "DAV:", Local: "displayname"}}, + }, + }, { + desc: "propfind: include followed by allprop", + input: "" + + "\n" + + " \n" + + " \n" + + "", + wantPF: propfind{ + XMLName: ixml.Name{Space: "DAV:", Local: "propfind"}, + Allprop: new(struct{}), + Include: propfindProps{xml.Name{Space: "DAV:", Local: "displayname"}}, + }, + }, { + desc: "propfind: propfind", + input: "" + + "\n" + + " \n" + + "", + wantPF: propfind{ + XMLName: ixml.Name{Space: "DAV:", Local: "propfind"}, + Prop: propfindProps{xml.Name{Space: "DAV:", Local: "displayname"}}, + }, + }, { + desc: "propfind: prop with ignored comments", + input: "" + + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + "", + wantPF: propfind{ + XMLName: ixml.Name{Space: "DAV:", Local: "propfind"}, + Prop: propfindProps{xml.Name{Space: "DAV:", Local: "displayname"}}, + }, + }, { + desc: "propfind: propfind with ignored whitespace", + input: "" + + "\n" + + " \n" + + "", + wantPF: propfind{ + XMLName: ixml.Name{Space: "DAV:", Local: "propfind"}, + Prop: propfindProps{xml.Name{Space: "DAV:", Local: "displayname"}}, + }, + }, { + desc: "propfind: propfind with ignored mixed-content", + input: "" + + "\n" + + " foobar\n" + + "", + wantPF: propfind{ + XMLName: ixml.Name{Space: "DAV:", Local: "propfind"}, + Prop: propfindProps{xml.Name{Space: "DAV:", Local: "displayname"}}, + }, + }, { + desc: "propfind: propname with ignored element (section A.4)", + input: "" + + "\n" + + " \n" + + " *boss*\n" + + "", + wantPF: propfind{ + XMLName: ixml.Name{Space: "DAV:", Local: "propfind"}, + Propname: new(struct{}), + }, + }, { + desc: "propfind: bad: junk", + input: "xxx", + wantStatus: http.StatusBadRequest, + }, { + desc: "propfind: bad: propname and allprop (section A.3)", + input: "" + + "\n" + + " " + + " " + + "", + wantStatus: http.StatusBadRequest, + }, { + desc: "propfind: bad: propname and prop", + input: "" + + "\n" + + " \n" + + " \n" + + "", + wantStatus: http.StatusBadRequest, + }, { + desc: "propfind: bad: allprop and prop", + input: "" + + "\n" + + " \n" + + " \n" + + "", + wantStatus: http.StatusBadRequest, + }, { + desc: "propfind: bad: empty propfind with ignored element (section A.4)", + input: "" + + "\n" + + " \n" + + "", + wantStatus: http.StatusBadRequest, + }, { + desc: "propfind: bad: empty prop", + input: "" + + "\n" + + " \n" + + "", + wantStatus: http.StatusBadRequest, + }, { + desc: "propfind: bad: prop with just chardata", + input: "" + + "\n" + + " foo\n" + + "", + wantStatus: http.StatusBadRequest, + }, { + desc: "bad: interrupted prop", + input: "" + + "\n" + + " \n", + wantStatus: http.StatusBadRequest, + }, { + desc: "bad: malformed end element prop", + input: "" + + "\n" + + " \n", + wantStatus: http.StatusBadRequest, + }, { + desc: "propfind: bad: property with chardata value", + input: "" + + "\n" + + " bar\n" + + "", + wantStatus: http.StatusBadRequest, + }, { + desc: "propfind: bad: property with whitespace value", + input: "" + + "\n" + + " \n" + + "", + wantStatus: http.StatusBadRequest, + }, { + desc: "propfind: bad: include without allprop", + input: "" + + "\n" + + " \n" + + "", + wantStatus: http.StatusBadRequest, + }} + + for _, tc := range testCases { + pf, status, err := readPropfind(strings.NewReader(tc.input)) + if tc.wantStatus != 0 { + if err == nil { + t.Errorf("%s: got nil error, want non-nil", tc.desc) + continue + } + } else if err != nil { + t.Errorf("%s: %v", tc.desc, err) + continue + } + if !reflect.DeepEqual(pf, tc.wantPF) || status != tc.wantStatus { + t.Errorf("%s:\ngot propfind=%v, status=%v\nwant propfind=%v, status=%v", + tc.desc, pf, status, tc.wantPF, tc.wantStatus) + continue + } + } +} + +func TestMultistatusWriter(t *testing.T) { + ///The "section x.y.z" test cases come from section x.y.z of the spec at + // http://www.webdav.org/specs/rfc4918.html + testCases := []struct { + desc string + responses []response + respdesc string + writeHeader bool + wantXML string + wantCode int + wantErr error + }{{ + desc: "section 9.2.2 (failed dependency)", + responses: []response{{ + Href: []string{"http://example.com/foo"}, + Propstat: []propstat{{ + Prop: []Property{{ + XMLName: xml.Name{ + Space: "http://ns.example.com/", + Local: "Authors", + }, + }}, + Status: "HTTP/1.1 424 Failed Dependency", + }, { + Prop: []Property{{ + XMLName: xml.Name{ + Space: "http://ns.example.com/", + Local: "Copyright-Owner", + }, + }}, + Status: "HTTP/1.1 409 Conflict", + }}, + ResponseDescription: "Copyright Owner cannot be deleted or altered.", + }}, + wantXML: `` + + `` + + `` + + ` ` + + ` http://example.com/foo` + + ` ` + + ` ` + + ` ` + + ` ` + + ` HTTP/1.1 424 Failed Dependency` + + ` ` + + ` ` + + ` ` + + ` ` + + ` ` + + ` HTTP/1.1 409 Conflict` + + ` ` + + ` Copyright Owner cannot be deleted or altered.` + + `` + + ``, + wantCode: StatusMulti, + }, { + desc: "section 9.6.2 (lock-token-submitted)", + responses: []response{{ + Href: []string{"http://example.com/foo"}, + Status: "HTTP/1.1 423 Locked", + Error: &xmlError{ + InnerXML: []byte(``), + }, + }}, + wantXML: `` + + `` + + `` + + ` ` + + ` http://example.com/foo` + + ` HTTP/1.1 423 Locked` + + ` ` + + ` ` + + ``, + wantCode: StatusMulti, + }, { + desc: "section 9.1.3", + responses: []response{{ + Href: []string{"http://example.com/foo"}, + Propstat: []propstat{{ + Prop: []Property{{ + XMLName: xml.Name{Space: "http://ns.example.com/boxschema/", Local: "bigbox"}, + InnerXML: []byte(`` + + `` + + `Box type A` + + ``), + }, { + XMLName: xml.Name{Space: "http://ns.example.com/boxschema/", Local: "author"}, + InnerXML: []byte(`` + + `` + + `J.J. Johnson` + + ``), + }}, + Status: "HTTP/1.1 200 OK", + }, { + Prop: []Property{{ + XMLName: xml.Name{Space: "http://ns.example.com/boxschema/", Local: "DingALing"}, + }, { + XMLName: xml.Name{Space: "http://ns.example.com/boxschema/", Local: "Random"}, + }}, + Status: "HTTP/1.1 403 Forbidden", + ResponseDescription: "The user does not have access to the DingALing property.", + }}, + }}, + respdesc: "There has been an access violation error.", + wantXML: `` + + `` + + `` + + ` ` + + ` http://example.com/foo` + + ` ` + + ` ` + + ` Box type A` + + ` J.J. Johnson` + + ` ` + + ` HTTP/1.1 200 OK` + + ` ` + + ` ` + + ` ` + + ` ` + + ` ` + + ` ` + + ` HTTP/1.1 403 Forbidden` + + ` The user does not have access to the DingALing property.` + + ` ` + + ` ` + + ` There has been an access violation error.` + + ``, + wantCode: StatusMulti, + }, { + desc: "no response written", + // default of http.responseWriter + wantCode: http.StatusOK, + }, { + desc: "no response written (with description)", + respdesc: "too bad", + // default of http.responseWriter + wantCode: http.StatusOK, + }, { + desc: "empty multistatus with header", + writeHeader: true, + wantXML: ``, + wantCode: StatusMulti, + }, { + desc: "bad: no href", + responses: []response{{ + Propstat: []propstat{{ + Prop: []Property{{ + XMLName: xml.Name{ + Space: "http://example.com/", + Local: "foo", + }, + }}, + Status: "HTTP/1.1 200 OK", + }}, + }}, + wantErr: errInvalidResponse, + // default of http.responseWriter + wantCode: http.StatusOK, + }, { + desc: "bad: multiple hrefs and no status", + responses: []response{{ + Href: []string{"http://example.com/foo", "http://example.com/bar"}, + }}, + wantErr: errInvalidResponse, + // default of http.responseWriter + wantCode: http.StatusOK, + }, { + desc: "bad: one href and no propstat", + responses: []response{{ + Href: []string{"http://example.com/foo"}, + }}, + wantErr: errInvalidResponse, + // default of http.responseWriter + wantCode: http.StatusOK, + }, { + desc: "bad: status with one href and propstat", + responses: []response{{ + Href: []string{"http://example.com/foo"}, + Propstat: []propstat{{ + Prop: []Property{{ + XMLName: xml.Name{ + Space: "http://example.com/", + Local: "foo", + }, + }}, + Status: "HTTP/1.1 200 OK", + }}, + Status: "HTTP/1.1 200 OK", + }}, + wantErr: errInvalidResponse, + // default of http.responseWriter + wantCode: http.StatusOK, + }, { + desc: "bad: multiple hrefs and propstat", + responses: []response{{ + Href: []string{ + "http://example.com/foo", + "http://example.com/bar", + }, + Propstat: []propstat{{ + Prop: []Property{{ + XMLName: xml.Name{ + Space: "http://example.com/", + Local: "foo", + }, + }}, + Status: "HTTP/1.1 200 OK", + }}, + }}, + wantErr: errInvalidResponse, + // default of http.responseWriter + wantCode: http.StatusOK, + }} + + n := xmlNormalizer{omitWhitespace: true} +loop: + for _, tc := range testCases { + rec := httptest.NewRecorder() + w := multistatusWriter{w: rec, responseDescription: tc.respdesc} + if tc.writeHeader { + if err := w.writeHeader(); err != nil { + t.Errorf("%s: got writeHeader error %v, want nil", tc.desc, err) + continue + } + } + for _, r := range tc.responses { + if err := w.write(&r); err != nil { + if err != tc.wantErr { + t.Errorf("%s: got write error %v, want %v", + tc.desc, err, tc.wantErr) + } + continue loop + } + } + if err := w.close(); err != tc.wantErr { + t.Errorf("%s: got close error %v, want %v", + tc.desc, err, tc.wantErr) + continue + } + if rec.Code != tc.wantCode { + t.Errorf("%s: got HTTP status code %d, want %d\n", + tc.desc, rec.Code, tc.wantCode) + continue + } + gotXML := rec.Body.String() + eq, err := n.equalXML(strings.NewReader(gotXML), strings.NewReader(tc.wantXML)) + if err != nil { + t.Errorf("%s: equalXML: %v", tc.desc, err) + continue + } + if !eq { + t.Errorf("%s: XML body\ngot %s\nwant %s", tc.desc, gotXML, tc.wantXML) + } + } +} + +func TestReadProppatch(t *testing.T) { + ppStr := func(pps []Proppatch) string { + var outer []string + for _, pp := range pps { + var inner []string + for _, p := range pp.Props { + inner = append(inner, fmt.Sprintf("{XMLName: %q, Lang: %q, InnerXML: %q}", + p.XMLName, p.Lang, p.InnerXML)) + } + outer = append(outer, fmt.Sprintf("{Remove: %t, Props: [%s]}", + pp.Remove, strings.Join(inner, ", "))) + } + return "[" + strings.Join(outer, ", ") + "]" + } + + testCases := []struct { + desc string + input string + wantPP []Proppatch + wantStatus int + }{{ + desc: "proppatch: section 9.2 (with simple property value)", + input: `` + + `` + + `` + + ` ` + + ` somevalue` + + ` ` + + ` ` + + ` ` + + ` ` + + ``, + wantPP: []Proppatch{{ + Props: []Property{{ + xml.Name{Space: "http://ns.example.com/z/", Local: "Authors"}, + "", + []byte(`somevalue`), + }}, + }, { + Remove: true, + Props: []Property{{ + xml.Name{Space: "http://ns.example.com/z/", Local: "Copyright-Owner"}, + "", + nil, + }}, + }}, + }, { + desc: "proppatch: lang attribute on prop", + input: `` + + `` + + `` + + ` ` + + ` ` + + ` ` + + ` ` + + ` ` + + ``, + wantPP: []Proppatch{{ + Props: []Property{{ + xml.Name{Space: "http://example.com/ns", Local: "foo"}, + "en", + nil, + }}, + }}, + }, { + desc: "bad: remove with value", + input: `` + + `` + + `` + + ` ` + + ` ` + + ` ` + + ` Jim Whitehead` + + ` ` + + ` ` + + ` ` + + ``, + wantStatus: http.StatusBadRequest, + }, { + desc: "bad: empty propertyupdate", + input: `` + + `` + + ``, + wantStatus: http.StatusBadRequest, + }, { + desc: "bad: empty prop", + input: `` + + `` + + `` + + ` ` + + ` ` + + ` ` + + ``, + wantStatus: http.StatusBadRequest, + }} + + for _, tc := range testCases { + pp, status, err := readProppatch(strings.NewReader(tc.input)) + if tc.wantStatus != 0 { + if err == nil { + t.Errorf("%s: got nil error, want non-nil", tc.desc) + continue + } + } else if err != nil { + t.Errorf("%s: %v", tc.desc, err) + continue + } + if status != tc.wantStatus { + t.Errorf("%s: got status %d, want %d", tc.desc, status, tc.wantStatus) + continue + } + if !reflect.DeepEqual(pp, tc.wantPP) || status != tc.wantStatus { + t.Errorf("%s: proppatch\ngot %v\nwant %v", tc.desc, ppStr(pp), ppStr(tc.wantPP)) + } + } +} + +func TestUnmarshalXMLValue(t *testing.T) { + testCases := []struct { + desc string + input string + wantVal string + }{{ + desc: "simple char data", + input: "foo", + wantVal: "foo", + }, { + desc: "empty element", + input: "", + wantVal: "", + }, { + desc: "preserve namespace", + input: ``, + wantVal: ``, + }, { + desc: "preserve root element namespace", + input: ``, + wantVal: ``, + }, { + desc: "preserve whitespace", + input: " \t ", + wantVal: " \t ", + }, { + desc: "preserve mixed content", + input: ` a `, + wantVal: ` a `, + }, { + desc: "section 9.2", + input: `` + + `` + + ` Jim Whitehead` + + ` Roy Fielding` + + ``, + wantVal: `` + + ` Jim Whitehead` + + ` Roy Fielding`, + }, { + desc: "section 4.3.1 (mixed content)", + input: `` + + `` + + ` Jane Doe` + + ` ` + + ` mailto:jane.doe@example.com` + + ` http://www.example.com` + + ` ` + + ` Jane has been working way too long on the` + + ` long-awaited revision of ]]>.` + + ` ` + + ``, + wantVal: `` + + ` Jane Doe` + + ` ` + + ` mailto:jane.doe@example.com` + + ` http://www.example.com` + + ` ` + + ` Jane has been working way too long on the` + + ` long-awaited revision of <RFC2518>.` + + ` `, + }} + + var n xmlNormalizer + for _, tc := range testCases { + d := ixml.NewDecoder(strings.NewReader(tc.input)) + var v xmlValue + if err := d.Decode(&v); err != nil { + t.Errorf("%s: got error %v, want nil", tc.desc, err) + continue + } + eq, err := n.equalXML(bytes.NewReader(v), strings.NewReader(tc.wantVal)) + if err != nil { + t.Errorf("%s: equalXML: %v", tc.desc, err) + continue + } + if !eq { + t.Errorf("%s:\ngot %s\nwant %s", tc.desc, string(v), tc.wantVal) + } + } +} + +// xmlNormalizer normalizes XML. +type xmlNormalizer struct { + // omitWhitespace instructs to ignore whitespace between element tags. + omitWhitespace bool + // omitComments instructs to ignore XML comments. + omitComments bool +} + +// normalize writes the normalized XML content of r to w. It applies the +// following rules +// +// - Rename namespace prefixes according to an internal heuristic. +// - Remove unnecessary namespace declarations. +// - Sort attributes in XML start elements in lexical order of their +// fully qualified name. +// - Remove XML directives and processing instructions. +// - Remove CDATA between XML tags that only contains whitespace, if +// instructed to do so. +// - Remove comments, if instructed to do so. +func (n *xmlNormalizer) normalize(w io.Writer, r io.Reader) error { + d := ixml.NewDecoder(r) + e := ixml.NewEncoder(w) + for { + t, err := d.Token() + if err != nil { + if t == nil && err == io.EOF { + break + } + return err + } + switch val := t.(type) { + case ixml.Directive, ixml.ProcInst: + continue + case ixml.Comment: + if n.omitComments { + continue + } + case ixml.CharData: + if n.omitWhitespace && len(bytes.TrimSpace(val)) == 0 { + continue + } + case ixml.StartElement: + start, _ := ixml.CopyToken(val).(ixml.StartElement) + attr := start.Attr[:0] + for _, a := range start.Attr { + if a.Name.Space == "xmlns" || a.Name.Local == "xmlns" { + continue + } + attr = append(attr, a) + } + sort.Sort(byName(attr)) + start.Attr = attr + t = start + } + err = e.EncodeToken(t) + if err != nil { + return err + } + } + return e.Flush() +} + +// equalXML tests for equality of the normalized XML contents of a and b. +func (n *xmlNormalizer) equalXML(a, b io.Reader) (bool, error) { + var buf bytes.Buffer + if err := n.normalize(&buf, a); err != nil { + return false, err + } + normA := buf.String() + buf.Reset() + if err := n.normalize(&buf, b); err != nil { + return false, err + } + normB := buf.String() + return normA == normB, nil +} + +type byName []ixml.Attr + +func (a byName) Len() int { return len(a) } +func (a byName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byName) Less(i, j int) bool { + if a[i].Name.Space != a[j].Name.Space { + return a[i].Name.Space < a[j].Name.Space + } + return a[i].Name.Local < a[j].Name.Local +} From 6f3e2171965cc3af24ec109c4446f135c03ff478 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 29 Nov 2023 13:52:34 -0500 Subject: [PATCH 10/63] oops (#438) --- endpoints/drive/webdav/xml.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/endpoints/drive/webdav/xml.go b/endpoints/drive/webdav/xml.go index fbd43cf22..a92248e09 100644 --- a/endpoints/drive/webdav/xml.go +++ b/endpoints/drive/webdav/xml.go @@ -32,7 +32,7 @@ import ( // In the long term, this package should use the standard library's version // only, and the internal fork deleted, once // https://github.com/golang/go/issues/13400 is resolved. - ixml "golang.org/x/net/webdav/internal/xml" + ixml "github.com/openziti/zrok/endpoints/drive/webdav/internal/xml" ) // http://www.webdav.org/specs/rfc4918.html#ELEMENT_lockinfo From 9d80e1cf66fcd8d0cd9f5ad8963c181a83300db3 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Thu, 30 Nov 2023 12:37:17 -0500 Subject: [PATCH 11/63] more use our fork (#438) --- endpoints/drive/webdav/litmus_test_server.go | 2 +- util/sync/filesystem.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/endpoints/drive/webdav/litmus_test_server.go b/endpoints/drive/webdav/litmus_test_server.go index e33b7b74e..b87063d1a 100644 --- a/endpoints/drive/webdav/litmus_test_server.go +++ b/endpoints/drive/webdav/litmus_test_server.go @@ -21,7 +21,7 @@ package main import ( "flag" "fmt" - "golang.org/x/net/webdav" + "github.com/openziti/zrok/endpoints/drive/webdav" "log" "net/http" "net/url" diff --git a/util/sync/filesystem.go b/util/sync/filesystem.go index 6877440c2..759236315 100644 --- a/util/sync/filesystem.go +++ b/util/sync/filesystem.go @@ -3,7 +3,7 @@ package sync import ( "context" "fmt" - "golang.org/x/net/webdav" + "github.com/openziti/zrok/endpoints/drive/webdav" "io" "io/fs" "os" From 837d2289f259c798c9c0a09b7a66ac9b69c3e610 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 1 Dec 2023 11:44:20 -0500 Subject: [PATCH 12/63] first round of modifications to the core webdav stack to support custom dead properties (#438) --- endpoints/drive/webdav/file.go | 64 +++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/endpoints/drive/webdav/file.go b/endpoints/drive/webdav/file.go index 3cd19ffbe..db8f4ecef 100644 --- a/endpoints/drive/webdav/file.go +++ b/endpoints/drive/webdav/file.go @@ -6,13 +6,18 @@ package webdav import ( "context" + "crypto/md5" "encoding/xml" + "fmt" + "github.com/sirupsen/logrus" "io" + "io/fs" "net/http" "os" "path" "path/filepath" "runtime" + "strconv" "strings" "sync" "time" @@ -55,6 +60,63 @@ type File interface { io.Writer } +type webdavFile struct { + File + name string +} + +func (f *webdavFile) DeadProps() (map[xml.Name]Property, error) { + logrus.Infof("DeadProps(%v)", f.name) + var ( + xmlName xml.Name + property Property + properties = make(map[xml.Name]Property) + checksum, err = f.md5() + ) + if err == nil { + xmlName.Space = "http://owncloud.org/ns" + xmlName.Local = "checksums" + property.XMLName = xmlName + property.InnerXML = append(property.InnerXML, ""...) + property.InnerXML = append(property.InnerXML, checksum...) + property.InnerXML = append(property.InnerXML, ""...) + properties[xmlName] = property + } + + var stat fs.FileInfo + stat, err = f.Stat() + if err == nil { + xmlName.Space = "DAV:" + xmlName.Local = "lastmodified" + property.XMLName = xmlName + property.InnerXML = strconv.AppendInt(nil, stat.ModTime().Unix(), 10) + properties[xmlName] = property + } + + return properties, nil +} + +func (f *webdavFile) Patch(proppatches []Proppatch) ([]Propstat, error) { + var stat Propstat + stat.Status = http.StatusOK + return []Propstat{stat}, nil +} + +func (f *webdavFile) md5() (string, error) { + file, err := os.Open(f.name) + if err != nil { + return "", err + } + defer file.Close() + hash := md5.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + xhash := fmt.Sprintf("%x", hash.Sum(nil)) + logrus.Infof("hashed %v = %v", f.name, xhash) + return xhash, nil +} + // A Dir implements FileSystem using the native file system restricted to a // specific directory tree. // @@ -93,7 +155,7 @@ func (d Dir) OpenFile(ctx context.Context, name string, flag int, perm os.FileMo if err != nil { return nil, err } - return f, nil + return &webdavFile{f, name}, nil } func (d Dir) RemoveAll(ctx context.Context, name string) error { From f40fd83b82de01b17ffec9c8958effee0505173c Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 1 Dec 2023 11:44:50 -0500 Subject: [PATCH 13/63] initial import of gowebdav fork to support custom properties (#438) --- util/sync/webdav.go | 8 +- util/sync/webdavClient/LICENSE | 27 + util/sync/webdavClient/Makefile | 42 + util/sync/webdavClient/README.md | 962 ++++++++++++++++++++ util/sync/webdavClient/auth.go | 409 +++++++++ util/sync/webdavClient/auth_test.go | 62 ++ util/sync/webdavClient/basicAuth.go | 42 + util/sync/webdavClient/basicAuth_test.go | 51 ++ util/sync/webdavClient/client.go | 438 +++++++++ util/sync/webdavClient/client_test.go | 574 ++++++++++++ util/sync/webdavClient/digestAuth.go | 164 ++++ util/sync/webdavClient/digestAuth_test.go | 35 + util/sync/webdavClient/doc.go | 3 + util/sync/webdavClient/errors.go | 57 ++ util/sync/webdavClient/file.go | 77 ++ util/sync/webdavClient/netrc.go | 54 ++ util/sync/webdavClient/passportAuth.go | 181 ++++ util/sync/webdavClient/passportAuth_test.go | 66 ++ util/sync/webdavClient/requests.go | 181 ++++ util/sync/webdavClient/utils.go | 113 +++ util/sync/webdavClient/utils_test.go | 67 ++ 21 files changed, 3609 insertions(+), 4 deletions(-) create mode 100644 util/sync/webdavClient/LICENSE create mode 100644 util/sync/webdavClient/Makefile create mode 100644 util/sync/webdavClient/README.md create mode 100644 util/sync/webdavClient/auth.go create mode 100644 util/sync/webdavClient/auth_test.go create mode 100644 util/sync/webdavClient/basicAuth.go create mode 100644 util/sync/webdavClient/basicAuth_test.go create mode 100644 util/sync/webdavClient/client.go create mode 100644 util/sync/webdavClient/client_test.go create mode 100644 util/sync/webdavClient/digestAuth.go create mode 100644 util/sync/webdavClient/digestAuth_test.go create mode 100644 util/sync/webdavClient/doc.go create mode 100644 util/sync/webdavClient/errors.go create mode 100644 util/sync/webdavClient/file.go create mode 100644 util/sync/webdavClient/netrc.go create mode 100644 util/sync/webdavClient/passportAuth.go create mode 100644 util/sync/webdavClient/passportAuth_test.go create mode 100644 util/sync/webdavClient/requests.go create mode 100644 util/sync/webdavClient/utils.go create mode 100644 util/sync/webdavClient/utils_test.go diff --git a/util/sync/webdav.go b/util/sync/webdav.go index 840a685be..7a2b6ca3a 100644 --- a/util/sync/webdav.go +++ b/util/sync/webdav.go @@ -1,8 +1,8 @@ package sync import ( + "github.com/openziti/zrok/util/sync/webdavClient" "github.com/pkg/errors" - "github.com/studio-b12/gowebdav" "io" "os" "path/filepath" @@ -16,11 +16,11 @@ type WebDAVTargetConfig struct { } type WebDAVTarget struct { - c *gowebdav.Client + c *webdavClient.Client } func NewWebDAVTarget(cfg *WebDAVTargetConfig) (*WebDAVTarget, error) { - c := gowebdav.NewClient(cfg.URL, cfg.Username, cfg.Password) + c := webdavClient.NewClient(cfg.URL, cfg.Username, cfg.Password) if err := c.Connect(); err != nil { return nil, errors.Wrap(err, "error connecting to webdav target") } @@ -48,7 +48,7 @@ func (t *WebDAVTarget) recurse(path string, tree []*Object) ([]*Object, error) { return nil, err } } else { - if v, ok := f.(gowebdav.File); ok { + if v, ok := f.(webdavClient.File); ok { tree = append(tree, &Object{ Path: filepath.ToSlash(filepath.Join(path, f.Name())), Size: v.Size(), diff --git a/util/sync/webdavClient/LICENSE b/util/sync/webdavClient/LICENSE new file mode 100644 index 000000000..a7cd4420f --- /dev/null +++ b/util/sync/webdavClient/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2014, Studio B12 GmbH +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/util/sync/webdavClient/Makefile b/util/sync/webdavClient/Makefile new file mode 100644 index 000000000..48dbb4e72 --- /dev/null +++ b/util/sync/webdavClient/Makefile @@ -0,0 +1,42 @@ +BIN := gowebdav +SRC := $(wildcard *.go) cmd/gowebdav/main.go + +all: test cmd + +cmd: ${BIN} + +${BIN}: ${SRC} + go build -o $@ ./cmd/gowebdav + +test: + go test -modfile=go_test.mod -v -short -cover ./... + +api: .go/bin/godoc2md + @sed '/^## API$$/,$$d' -i README.md + @echo '## API' >> README.md + @$< github.com/studio-b12/gowebdav | sed '/^$$/N;/^\n$$/D' |\ + sed '2d' |\ + sed 's/\/src\/github.com\/studio-b12\/gowebdav\//https:\/\/github.com\/studio-b12\/gowebdav\/blob\/master\//g' |\ + sed 's/\/src\/target\//https:\/\/github.com\/studio-b12\/gowebdav\/blob\/master\//g' |\ + sed 's/^#/##/g' >> README.md + +check: .go/bin/gocyclo + gofmt -w -s $(SRC) + @echo + .go/bin/gocyclo -over 15 . + @echo + go vet -modfile=go_test.mod ./... + + +.go/bin/godoc2md: + @mkdir -p $(@D) + @GOPATH="$(CURDIR)/.go" go install github.com/davecheney/godoc2md@latest + +.go/bin/gocyclo: + @mkdir -p $(@D) + @GOPATH="$(CURDIR)/.go" go install github.com/fzipp/gocyclo/cmd/gocyclo@latest + +clean: + @rm -f ${BIN} + +.PHONY: all cmd clean test api check diff --git a/util/sync/webdavClient/README.md b/util/sync/webdavClient/README.md new file mode 100644 index 000000000..6c9a795a0 --- /dev/null +++ b/util/sync/webdavClient/README.md @@ -0,0 +1,962 @@ +# GoWebDAV + +[![Unit Tests Status](https://github.com/studio-b12/gowebdav/actions/workflows/tests.yml/badge.svg)](https://github.com/studio-b12/gowebdav/actions/workflows/tests.yml) +[![Build Artifacts Status](https://github.com/studio-b12/gowebdav/actions/workflows/artifacts.yml/badge.svg)](https://github.com/studio-b12/gowebdav/actions/workflows/artifacts.yml) +[![GoDoc](https://godoc.org/github.com/studio-b12/gowebdav?status.svg)](https://godoc.org/github.com/studio-b12/gowebdav) +[![Go Report Card](https://goreportcard.com/badge/github.com/studio-b12/gowebdav)](https://goreportcard.com/report/github.com/studio-b12/gowebdav) + +A pure Golang WebDAV client library that comes with +a [reference implementation](https://github.com/studio-b12/gowebdav/tree/master/cmd/gowebdav). + +## Features at a glance + +Our `gowebdav` library allows to perform following actions on the remote WebDAV server: + +* [create path](#create-path-on-a-webdav-server) +* [get files list](#get-files-list) +* [download file](#download-file-to-byte-array) +* [upload file](#upload-file-from-byte-array) +* [get information about specified file/folder](#get-information-about-specified-filefolder) +* [move file to another location](#move-file-to-another-location) +* [copy file to another location](#copy-file-to-another-location) +* [delete file](#delete-file) + +It also provides an [authentication API](#type-authenticator) that makes it easy to encapsulate and control complex +authentication challenges. +The default implementation negotiates the algorithm based on the user's preferences and the methods offered by the +remote server. + +Out-of-box authentication support for: + +* [BasicAuth](https://en.wikipedia.org/wiki/Basic_access_authentication) +* [DigestAuth](https://en.wikipedia.org/wiki/Digest_access_authentication) +* [MS-PASS](https://github.com/studio-b12/gowebdav/pull/70#issuecomment-1421713726) +* [WIP Kerberos](https://github.com/studio-b12/gowebdav/pull/71#issuecomment-1416465334) +* [WIP Bearer Token](https://github.com/studio-b12/gowebdav/issues/61) + +## Usage + +First of all you should create `Client` instance using `NewClient()` function: + +```go +root := "https://webdav.mydomain.me" +user := "user" +password := "password" + +c := gowebdav.NewClient(root, user, password) +c.Connect() +// kick of your work! +``` + +After you can use this `Client` to perform actions, described below. + +**NOTICE:** We will not check for errors in the examples, to focus you on the `gowebdav` library's code, but you should +do it in your code! + +### Create path on a WebDAV server + +```go +err := c.Mkdir("folder", 0644) +``` + +In case you want to create several folders you can use `c.MkdirAll()`: + +```go +err := c.MkdirAll("folder/subfolder/subfolder2", 0644) +``` + +### Get files list + +```go +files, _ := c.ReadDir("folder/subfolder") +for _, file := range files { + //notice that [file] has os.FileInfo type + fmt.Println(file.Name()) +} +``` + +### Download file to byte array + +```go +webdavFilePath := "folder/subfolder/file.txt" +localFilePath := "/tmp/webdav/file.txt" + +bytes, _ := c.Read(webdavFilePath) +os.WriteFile(localFilePath, bytes, 0644) +``` + +### Download file via reader + +Also you can use `c.ReadStream()` method: + +```go +webdavFilePath := "folder/subfolder/file.txt" +localFilePath := "/tmp/webdav/file.txt" + +reader, _ := c.ReadStream(webdavFilePath) + +file, _ := os.Create(localFilePath) +defer file.Close() + +io.Copy(file, reader) +``` + +### Upload file from byte array + +```go +webdavFilePath := "folder/subfolder/file.txt" +localFilePath := "/tmp/webdav/file.txt" + +bytes, _ := os.ReadFile(localFilePath) + +c.Write(webdavFilePath, bytes, 0644) +``` + +### Upload file via writer + +```go +webdavFilePath := "folder/subfolder/file.txt" +localFilePath := "/tmp/webdav/file.txt" + +file, _ := os.Open(localFilePath) +defer file.Close() + +c.WriteStream(webdavFilePath, file, 0644) +``` + +### Get information about specified file/folder + +```go +webdavFilePath := "folder/subfolder/file.txt" + +info := c.Stat(webdavFilePath) +//notice that [info] has os.FileInfo type +fmt.Println(info) +``` + +### Move file to another location + +```go +oldPath := "folder/subfolder/file.txt" +newPath := "folder/subfolder/moved.txt" +isOverwrite := true + +c.Rename(oldPath, newPath, isOverwrite) +``` + +### Copy file to another location + +```go +oldPath := "folder/subfolder/file.txt" +newPath := "folder/subfolder/file-copy.txt" +isOverwrite := true + +c.Copy(oldPath, newPath, isOverwrite) +``` + +### Delete file + +```go +webdavFilePath := "folder/subfolder/file.txt" + +c.Remove(webdavFilePath) +``` + +## Links + +More details about WebDAV server you can read from following resources: + +* [RFC 4918 - HTTP Extensions for Web Distributed Authoring and Versioning (WebDAV)](https://tools.ietf.org/html/rfc4918) +* [RFC 5689 - Extended MKCOL for Web Distributed Authoring and Versioning (WebDAV)](https://tools.ietf.org/html/rfc5689) +* [RFC 2616 - HTTP/1.1 Status Code Definitions](http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html "HTTP/1.1 Status Code Definitions") +* [WebDav: Next Generation Collaborative Web Authoring By Lisa Dusseaul](https://books.google.de/books?isbn=0130652083 "WebDav: Next Generation Collaborative Web Authoring By Lisa Dusseault") + +**NOTICE**: RFC 2518 is obsoleted by RFC 4918 in June 2007 + +## Contributing + +All contributing are welcome. If you have any suggestions or find some bug - please create an Issue to let us make this +project better. We appreciate your help! + +## License + +This library is distributed under the BSD 3-Clause license found in +the [LICENSE](https://github.com/studio-b12/gowebdav/blob/master/LICENSE) file. + +## API + +`import "github.com/studio-b12/gowebdav"` + +* [Overview](#pkg-overview) +* [Index](#pkg-index) +* [Examples](#pkg-examples) +* [Subdirectories](#pkg-subdirectories) + +### Overview + +Package gowebdav is a WebDAV client library with a command line tool +included. + +### Index + +* [Constants](#pkg-constants) +* [Variables](#pkg-variables) +* [func FixSlash(s string) string](#FixSlash) +* [func FixSlashes(s string) string](#FixSlashes) +* [func IsErrCode(err error, code int) bool](#IsErrCode) +* [func IsErrNotFound(err error) bool](#IsErrNotFound) +* [func Join(path0 string, path1 string) string](#Join) +* [func NewPathError(op string, path string, statusCode int) error](#NewPathError) +* [func NewPathErrorErr(op string, path string, err error) error](#NewPathErrorErr) +* [func PathEscape(path string) string](#PathEscape) +* [func ReadConfig(uri, netrc string) (string, string)](#ReadConfig) +* [func String(r io.Reader) string](#String) +* [type AuthFactory](#AuthFactory) +* [type Authenticator](#Authenticator) + * [func NewDigestAuth(login, secret string, rs *http.Response) (Authenticator, error)](#NewDigestAuth) + * [func NewPassportAuth(c *http.Client, user, pw, partnerURL string, header *http.Header) (Authenticator, error)](#NewPassportAuth) +* [type Authorizer](#Authorizer) + * [func NewAutoAuth(login string, secret string) Authorizer](#NewAutoAuth) + * [func NewEmptyAuth() Authorizer](#NewEmptyAuth) + * [func NewPreemptiveAuth(auth Authenticator) Authorizer](#NewPreemptiveAuth) +* [type BasicAuth](#BasicAuth) + * [func (b *BasicAuth) Authorize(c *http.Client, rq *http.Request, path string) error](#BasicAuth.Authorize) + * [func (b *BasicAuth) Clone() Authenticator](#BasicAuth.Clone) + * [func (b *BasicAuth) Close() error](#BasicAuth.Close) + * [func (b *BasicAuth) String() string](#BasicAuth.String) + * [func (b *BasicAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error)](#BasicAuth.Verify) +* [type Client](#Client) + * [func NewAuthClient(uri string, auth Authorizer) *Client](#NewAuthClient) + * [func NewClient(uri, user, pw string) *Client](#NewClient) + * [func (c *Client) Connect() error](#Client.Connect) + * [func (c *Client) Copy(oldpath, newpath string, overwrite bool) error](#Client.Copy) + * [func (c *Client) Mkdir(path string, _ os.FileMode) (err error)](#Client.Mkdir) + * [func (c *Client) MkdirAll(path string, _ os.FileMode) (err error)](#Client.MkdirAll) + * [func (c *Client) Read(path string) ([]byte, error)](#Client.Read) + * [func (c *Client) ReadDir(path string) ([]os.FileInfo, error)](#Client.ReadDir) + * [func (c *Client) ReadStream(path string) (io.ReadCloser, error)](#Client.ReadStream) + * [func (c *Client) ReadStreamRange(path string, offset, length int64) (io.ReadCloser, error)](#Client.ReadStreamRange) + * [func (c *Client) Remove(path string) error](#Client.Remove) + * [func (c *Client) RemoveAll(path string) error](#Client.RemoveAll) + * [func (c *Client) Rename(oldpath, newpath string, overwrite bool) error](#Client.Rename) + * [func (c *Client) SetHeader(key, value string)](#Client.SetHeader) + * [func (c *Client) SetInterceptor(interceptor func(method string, rq *http.Request))](#Client.SetInterceptor) + * [func (c *Client) SetJar(jar http.CookieJar)](#Client.SetJar) + * [func (c *Client) SetTimeout(timeout time.Duration)](#Client.SetTimeout) + * [func (c *Client) SetTransport(transport http.RoundTripper)](#Client.SetTransport) + * [func (c *Client) Stat(path string) (os.FileInfo, error)](#Client.Stat) + * [func (c *Client) Write(path string, data []byte, _ os.FileMode) (err error)](#Client.Write) + * [func (c *Client) WriteStream(path string, stream io.Reader, _ os.FileMode) (err error)](#Client.WriteStream) +* [type DigestAuth](#DigestAuth) + * [func (d *DigestAuth) Authorize(c *http.Client, rq *http.Request, path string) error](#DigestAuth.Authorize) + * [func (d *DigestAuth) Clone() Authenticator](#DigestAuth.Clone) + * [func (d *DigestAuth) Close() error](#DigestAuth.Close) + * [func (d *DigestAuth) String() string](#DigestAuth.String) + * [func (d *DigestAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error)](#DigestAuth.Verify) +* [type File](#File) + * [func (f File) ContentType() string](#File.ContentType) + * [func (f File) ETag() string](#File.ETag) + * [func (f File) IsDir() bool](#File.IsDir) + * [func (f File) ModTime() time.Time](#File.ModTime) + * [func (f File) Mode() os.FileMode](#File.Mode) + * [func (f File) Name() string](#File.Name) + * [func (f File) Path() string](#File.Path) + * [func (f File) Size() int64](#File.Size) + * [func (f File) String() string](#File.String) + * [func (f File) Sys() interface{}](#File.Sys) +* [type PassportAuth](#PassportAuth) + * [func (p *PassportAuth) Authorize(c *http.Client, rq *http.Request, path string) error](#PassportAuth.Authorize) + * [func (p *PassportAuth) Clone() Authenticator](#PassportAuth.Clone) + * [func (p *PassportAuth) Close() error](#PassportAuth.Close) + * [func (p *PassportAuth) String() string](#PassportAuth.String) + * [func (p *PassportAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error)](#PassportAuth.Verify) +* [type StatusError](#StatusError) + * [func (se StatusError) Error() string](#StatusError.Error) + +##### Examples + +* [PathEscape](#example_PathEscape) + +##### Package files + +[auth.go](https://github.com/studio-b12/gowebdav/blob/master/auth.go) [basicAuth.go](https://github.com/studio-b12/gowebdav/blob/master/basicAuth.go) [client.go](https://github.com/studio-b12/gowebdav/blob/master/client.go) [digestAuth.go](https://github.com/studio-b12/gowebdav/blob/master/digestAuth.go) [doc.go](https://github.com/studio-b12/gowebdav/blob/master/doc.go) [errors.go](https://github.com/studio-b12/gowebdav/blob/master/errors.go) [file.go](https://github.com/studio-b12/gowebdav/blob/master/file.go) [netrc.go](https://github.com/studio-b12/gowebdav/blob/master/netrc.go) [passportAuth.go](https://github.com/studio-b12/gowebdav/blob/master/passportAuth.go) [requests.go](https://github.com/studio-b12/gowebdav/blob/master/requests.go) [utils.go](https://github.com/studio-b12/gowebdav/blob/master/utils.go) + +### Constants + +``` go +const XInhibitRedirect = "X-Gowebdav-Inhibit-Redirect" +``` + +### Variables + +``` go +var ErrAuthChanged = errors.New("authentication failed, change algorithm") +``` + +ErrAuthChanged must be returned from the Verify method as an error +to trigger a re-authentication / negotiation with a new authenticator. + +``` go +var ErrTooManyRedirects = errors.New("stopped after 10 redirects") +``` + +ErrTooManyRedirects will be used as return error if a request exceeds 10 redirects. + +### func [FixSlash](https://github.com/studio-b12/gowebdav/blob/master/utils.go?s=354:384#L23) + +``` go +func FixSlash(s string) string +``` + +FixSlash appends a trailing / to our string + +### func [FixSlashes](https://github.com/studio-b12/gowebdav/blob/master/utils.go?s=506:538#L31) + +``` go +func FixSlashes(s string) string +``` + +FixSlashes appends and prepends a / if they are missing + +### func [IsErrCode](https://github.com/studio-b12/gowebdav/blob/master/errors.go?s=740:780#L29) + +``` go +func IsErrCode(err error, code int) bool +``` + +IsErrCode returns true if the given error +is an os.PathError wrapping a StatusError +with the given status code. + +### func [IsErrNotFound](https://github.com/studio-b12/gowebdav/blob/master/errors.go?s=972:1006#L39) + +``` go +func IsErrNotFound(err error) bool +``` + +IsErrNotFound is shorthand for IsErrCode +for status 404. + +### func [Join](https://github.com/studio-b12/gowebdav/blob/master/utils.go?s=639:683#L40) + +``` go +func Join(path0 string, path1 string) string +``` + +Join joins two paths + +### func [NewPathError](https://github.com/studio-b12/gowebdav/blob/master/errors.go?s=1040:1103#L43) + +``` go +func NewPathError(op string, path string, statusCode int) error +``` + +### func [NewPathErrorErr](https://github.com/studio-b12/gowebdav/blob/master/errors.go?s=1194:1255#L51) + +``` go +func NewPathErrorErr(op string, path string, err error) error +``` + +### func [PathEscape](https://github.com/studio-b12/gowebdav/blob/master/utils.go?s=153:188#L14) + +``` go +func PathEscape(path string) string +``` + +PathEscape escapes all segments of a given path + +### func [ReadConfig](https://github.com/studio-b12/gowebdav/blob/master/netrc.go?s=428:479#L27) + +``` go +func ReadConfig(uri, netrc string) (string, string) +``` + +ReadConfig reads login and password configuration from ~/.netrc +machine foo.com login username password 123456 + +### func [String](https://github.com/studio-b12/gowebdav/blob/master/utils.go?s=813:844#L45) + +``` go +func String(r io.Reader) string +``` + +String pulls a string out of our io.Reader + +### type [AuthFactory](https://github.com/studio-b12/gowebdav/blob/master/auth.go?s=150:251#L13) + +``` go +type AuthFactory func(c *http.Client, rs *http.Response, path string) (auth Authenticator, err error) +``` + +AuthFactory prototype function to create a new Authenticator + +### type [Authenticator](https://github.com/studio-b12/gowebdav/blob/master/auth.go?s=2155:2695#L56) + +``` go +type Authenticator interface { + // Authorizes a request. Usually by adding some authorization headers. + Authorize(c *http.Client, rq *http.Request, path string) error + // Verifies the response if the authorization was successful. + // May trigger some round trips to pass the authentication. + // May also trigger a new Authenticator negotiation by returning `ErrAuthChenged` + Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) + // Creates a copy of the underlying Authenticator. + Clone() Authenticator + io.Closer +} +``` + +A Authenticator implements a specific way to authorize requests. +Each request is bound to a separate Authenticator instance. + +The authentication flow itself is broken down into `Authorize` +and `Verify` steps. The former method runs before, and the latter +runs after the `Request` is submitted. +This makes it easy to encapsulate and control complex +authentication challenges. + +Some authentication flows causing authentication round trips, +which can be archived by returning the `redo` of the Verify +method. `True` restarts the authentication process for the +current action: A new `Request` is spawned, which must be +authorized, sent, and re-verified again, until the action +is successfully submitted. +The preferred way is to handle the authentication ping-pong +within `Verify`, and then `redo` with fresh credentials. + +The result of the `Verify` method can also trigger an +`Authenticator` change by returning the `ErrAuthChanged` +as an error. Depending on the `Authorizer` this may trigger +an `Authenticator` negotiation. + +Set the `XInhibitRedirect` header to '1' in the `Authorize` +method to get control over request redirection. +Attention! You must handle the incoming request yourself. + +To store a shared session state the `Clone` method **must** +return a new instance, initialized with the shared state. + +#### func [NewDigestAuth](https://github.com/studio-b12/gowebdav/blob/master/digestAuth.go?s=324:406#L21) + +``` go +func NewDigestAuth(login, secret string, rs *http.Response) (Authenticator, error) +``` + +NewDigestAuth creates a new instance of our Digest Authenticator + +#### func [NewPassportAuth](https://github.com/studio-b12/gowebdav/blob/master/passportAuth.go?s=386:495#L21) + +``` go +func NewPassportAuth(c *http.Client, user, pw, partnerURL string, header *http.Header) (Authenticator, error) +``` + +constructor for PassportAuth creates a new PassportAuth object and +automatically authenticates against the given partnerURL + +### type [Authorizer](https://github.com/studio-b12/gowebdav/blob/master/auth.go?s=349:764#L17) + +``` go +type Authorizer interface { + // Creates a new Authenticator Shim per request. + // It may track request related states and perform payload buffering + // for authentication round trips. + // The underlying Authenticator will perform the real authentication. + NewAuthenticator(body io.Reader) (Authenticator, io.Reader) + // Registers a new Authenticator factory to a key. + AddAuthenticator(key string, fn AuthFactory) +} +``` + +Authorizer our Authenticator factory which creates an +`Authenticator` per action/request. + +#### func [NewAutoAuth](https://github.com/studio-b12/gowebdav/blob/master/auth.go?s=3789:3845#L109) + +``` go +func NewAutoAuth(login string, secret string) Authorizer +``` + +NewAutoAuth creates an auto Authenticator factory. +It negotiates the default authentication method +based on the order of the registered Authenticators +and the remotely offered authentication methods. +First In, First Out. + +#### func [NewEmptyAuth](https://github.com/studio-b12/gowebdav/blob/master/auth.go?s=4694:4724#L132) + +``` go +func NewEmptyAuth() Authorizer +``` + +NewEmptyAuth creates an empty Authenticator factory +The order of adding the Authenticator matters. +First In, First Out. +It offers the `NewAutoAuth` features. + +#### func [NewPreemptiveAuth](https://github.com/studio-b12/gowebdav/blob/master/auth.go?s=5300:5353#L148) + +``` go +func NewPreemptiveAuth(auth Authenticator) Authorizer +``` + +NewPreemptiveAuth creates a preemptive Authenticator +The preemptive authorizer uses the provided Authenticator +for every request regardless of any `Www-Authenticate` header. + +It may only have one authentication method, +so calling `AddAuthenticator` **will panic**! + +Look out!! This offers the skinniest and slickest implementation +without any synchronisation!! +Still applicable with `BasicAuth` within go routines. + +### type [BasicAuth](https://github.com/studio-b12/gowebdav/blob/master/basicAuth.go?s=94:145#L9) + +``` go +type BasicAuth struct { + // contains filtered or unexported fields +} + +``` + +BasicAuth structure holds our credentials + +#### func (\*BasicAuth) [Authorize](https://github.com/studio-b12/gowebdav/blob/master/basicAuth.go?s=180:262#L15) + +``` go +func (b *BasicAuth) Authorize(c *http.Client, rq *http.Request, path string) error +``` + +Authorize the current request + +#### func (\*BasicAuth) [Clone](https://github.com/studio-b12/gowebdav/blob/master/basicAuth.go?s=666:707#L34) + +``` go +func (b *BasicAuth) Clone() Authenticator +``` + +Clone creates a Copy of itself + +#### func (\*BasicAuth) [Close](https://github.com/studio-b12/gowebdav/blob/master/basicAuth.go?s=581:614#L29) + +``` go +func (b *BasicAuth) Close() error +``` + +Close cleans up all resources + +#### func (\*BasicAuth) [String](https://github.com/studio-b12/gowebdav/blob/master/basicAuth.go?s=778:813#L40) + +``` go +func (b *BasicAuth) String() string +``` + +String toString + +#### func (\*BasicAuth) [Verify](https://github.com/studio-b12/gowebdav/blob/master/basicAuth.go?s=352:449#L21) + +``` go +func (b *BasicAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) +``` + +Verify verifies if the authentication + +### type [Client](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=220:388#L19) + +``` go +type Client struct { + // contains filtered or unexported fields +} + +``` + +Client defines our structure + +#### func [NewAuthClient](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=608:663#L33) + +``` go +func NewAuthClient(uri string, auth Authorizer) *Client +``` + +NewAuthClient creates a new client instance with a custom Authorizer + +#### func [NewClient](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=436:480#L28) + +``` go +func NewClient(uri, user, pw string) *Client +``` + +NewClient creates a new instance of client + +#### func (\*Client) [Connect](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=1829:1861#L74) + +``` go +func (c *Client) Connect() error +``` + +Connect connects to our dav server + +#### func (\*Client) [Copy](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=6815:6883#L310) + +``` go +func (c *Client) Copy(oldpath, newpath string, overwrite bool) error +``` + +Copy copies a file from A to B + +#### func (\*Client) [Mkdir](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=5790:5852#L259) + +``` go +func (c *Client) Mkdir(path string, _ os.FileMode) (err error) +``` + +Mkdir makes a directory + +#### func (\*Client) [MkdirAll](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=6065:6130#L273) + +``` go +func (c *Client) MkdirAll(path string, _ os.FileMode) (err error) +``` + +MkdirAll like mkdir -p, but for webdav + +#### func (\*Client) [Read](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=6989:7039#L315) + +``` go +func (c *Client) Read(path string) ([]byte, error) +``` + +Read reads the contents of a remote file + +#### func (\*Client) [ReadDir](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=2855:2915#L117) + +``` go +func (c *Client) ReadDir(path string) ([]os.FileInfo, error) +``` + +ReadDir reads the contents of a remote directory + +#### func (\*Client) [ReadStream](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=7350:7413#L333) + +``` go +func (c *Client) ReadStream(path string) (io.ReadCloser, error) +``` + +ReadStream reads the stream for a given path + +#### func (\*Client) [ReadStreamRange](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=8162:8252#L355) + +``` go +func (c *Client) ReadStreamRange(path string, offset, length int64) (io.ReadCloser, error) +``` + +ReadStreamRange reads the stream representing a subset of bytes for a given path, +utilizing HTTP Range Requests if the server supports it. +The range is expressed as offset from the start of the file and length, for example +offset=10, length=10 will return bytes 10 through 19. + +If the server does not support partial content requests and returns full content instead, +this function will emulate the behavior by skipping `offset` bytes and limiting the result +to `length`. + +#### func (\*Client) [Remove](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=5296:5338#L236) + +``` go +func (c *Client) Remove(path string) error +``` + +Remove removes a remote file + +#### func (\*Client) [RemoveAll](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=5404:5449#L241) + +``` go +func (c *Client) RemoveAll(path string) error +``` + +RemoveAll removes remote files + +#### func (\*Client) [Rename](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=6649:6719#L305) + +``` go +func (c *Client) Rename(oldpath, newpath string, overwrite bool) error +``` + +Rename moves a file from A to B + +#### func (\*Client) [SetHeader](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=1092:1137#L49) + +``` go +func (c *Client) SetHeader(key, value string) +``` + +SetHeader lets us set arbitrary headers for a given client + +#### func (\*Client) [SetInterceptor](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=1244:1326#L54) + +``` go +func (c *Client) SetInterceptor(interceptor func(method string, rq *http.Request)) +``` + +SetInterceptor lets us set an arbitrary interceptor for a given client + +#### func (\*Client) [SetJar](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=1727:1770#L69) + +``` go +func (c *Client) SetJar(jar http.CookieJar) +``` + +SetJar exposes the ability to set a cookie jar to the client. + +#### func (\*Client) [SetTimeout](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=1428:1478#L59) + +``` go +func (c *Client) SetTimeout(timeout time.Duration) +``` + +SetTimeout exposes the ability to set a time limit for requests + +#### func (\*Client) [SetTransport](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=1571:1629#L64) + +``` go +func (c *Client) SetTransport(transport http.RoundTripper) +``` + +SetTransport exposes the ability to define custom transports + +#### func (\*Client) [Stat](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=4241:4296#L184) + +``` go +func (c *Client) Stat(path string) (os.FileInfo, error) +``` + +Stat returns the file stats for a specified path + +#### func (\*Client) [Write](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=9272:9347#L389) + +``` go +func (c *Client) Write(path string, data []byte, _ os.FileMode) (err error) +``` + +Write writes data to a given path + +#### func (\*Client) [WriteStream](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=9771:9857#L419) + +``` go +func (c *Client) WriteStream(path string, stream io.Reader, _ os.FileMode) (err error) +``` + +WriteStream writes a stream + +### type [DigestAuth](https://github.com/studio-b12/gowebdav/blob/master/digestAuth.go?s=157:254#L14) + +``` go +type DigestAuth struct { + // contains filtered or unexported fields +} + +``` + +DigestAuth structure holds our credentials + +#### func (\*DigestAuth) [Authorize](https://github.com/studio-b12/gowebdav/blob/master/digestAuth.go?s=525:608#L26) + +``` go +func (d *DigestAuth) Authorize(c *http.Client, rq *http.Request, path string) error +``` + +Authorize the current request + +#### func (\*DigestAuth) [Clone](https://github.com/studio-b12/gowebdav/blob/master/digestAuth.go?s=1228:1270#L49) + +``` go +func (d *DigestAuth) Clone() Authenticator +``` + +Clone creates a copy of itself + +#### func (\*DigestAuth) [Close](https://github.com/studio-b12/gowebdav/blob/master/digestAuth.go?s=1142:1176#L44) + +``` go +func (d *DigestAuth) Close() error +``` + +Close cleans up all resources + +#### func (\*DigestAuth) [String](https://github.com/studio-b12/gowebdav/blob/master/digestAuth.go?s=1466:1502#L58) + +``` go +func (d *DigestAuth) String() string +``` + +String toString + +#### func (\*DigestAuth) [Verify](https://github.com/studio-b12/gowebdav/blob/master/digestAuth.go?s=912:1010#L36) + +``` go +func (d *DigestAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) +``` + +Verify checks for authentication issues and may trigger a re-authentication + +### type [File](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=93:253#L10) + +``` go +type File struct { + // contains filtered or unexported fields +} + +``` + +File is our structure for a given file + +#### func (File) [ContentType](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=476:510#L31) + +``` go +func (f File) ContentType() string +``` + +ContentType returns the content type of a file + +#### func (File) [ETag](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=929:956#L56) + +``` go +func (f File) ETag() string +``` + +ETag returns the ETag of a file + +#### func (File) [IsDir](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=1035:1061#L61) + +``` go +func (f File) IsDir() bool +``` + +IsDir let us see if a given file is a directory or not + +#### func (File) [ModTime](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=836:869#L51) + +``` go +func (f File) ModTime() time.Time +``` + +ModTime returns the modified time of a file + +#### func (File) [Mode](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=665:697#L41) + +``` go +func (f File) Mode() os.FileMode +``` + +Mode will return the mode of a given file + +#### func (File) [Name](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=378:405#L26) + +``` go +func (f File) Name() string +``` + +Name returns the name of a file + +#### func (File) [Path](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=295:322#L21) + +``` go +func (f File) Path() string +``` + +Path returns the full path of a file + +#### func (File) [Size](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=573:599#L36) + +``` go +func (f File) Size() int64 +``` + +Size returns the size of a file + +#### func (File) [String](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=1183:1212#L71) + +``` go +func (f File) String() string +``` + +String lets us see file information + +#### func (File) [Sys](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=1095:1126#L66) + +``` go +func (f File) Sys() interface{} +``` + +Sys ???? + +### type [PassportAuth](https://github.com/studio-b12/gowebdav/blob/master/passportAuth.go?s=125:254#L12) + +``` go +type PassportAuth struct { + // contains filtered or unexported fields +} + +``` + +PassportAuth structure holds our credentials + +#### func (\*PassportAuth) [Authorize](https://github.com/studio-b12/gowebdav/blob/master/passportAuth.go?s=690:775#L32) + +``` go +func (p *PassportAuth) Authorize(c *http.Client, rq *http.Request, path string) error +``` + +Authorize the current request + +#### func (\*PassportAuth) [Clone](https://github.com/studio-b12/gowebdav/blob/master/passportAuth.go?s=1701:1745#L69) + +``` go +func (p *PassportAuth) Clone() Authenticator +``` + +Clone creates a Copy of itself + +#### func (\*PassportAuth) [Close](https://github.com/studio-b12/gowebdav/blob/master/passportAuth.go?s=1613:1649#L64) + +``` go +func (p *PassportAuth) Close() error +``` + +Close cleans up all resources + +#### func (\*PassportAuth) [String](https://github.com/studio-b12/gowebdav/blob/master/passportAuth.go?s=2048:2086#L83) + +``` go +func (p *PassportAuth) String() string +``` + +String toString + +#### func (\*PassportAuth) [Verify](https://github.com/studio-b12/gowebdav/blob/master/passportAuth.go?s=1075:1175#L46) + +``` go +func (p *PassportAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) +``` + +Verify verifies if the authentication is good + +### type [StatusError](https://github.com/studio-b12/gowebdav/blob/master/errors.go?s=499:538#L18) + +``` go +type StatusError struct { + Status int +} + +``` + +StatusError implements error and wraps +an erroneous status code. + +#### func (StatusError) [Error](https://github.com/studio-b12/gowebdav/blob/master/errors.go?s=540:576#L22) + +``` go +func (se StatusError) Error() string +``` + +- - - +Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) diff --git a/util/sync/webdavClient/auth.go b/util/sync/webdavClient/auth.go new file mode 100644 index 000000000..32d761685 --- /dev/null +++ b/util/sync/webdavClient/auth.go @@ -0,0 +1,409 @@ +package webdavClient + +import ( + "bytes" + "errors" + "io" + "net/http" + "strings" + "sync" +) + +// AuthFactory prototype function to create a new Authenticator +type AuthFactory func(c *http.Client, rs *http.Response, path string) (auth Authenticator, err error) + +// Authorizer our Authenticator factory which creates an +// `Authenticator` per action/request. +type Authorizer interface { + // Creates a new Authenticator Shim per request. + // It may track request related states and perform payload buffering + // for authentication round trips. + // The underlying Authenticator will perform the real authentication. + NewAuthenticator(body io.Reader) (Authenticator, io.Reader) + // Registers a new Authenticator factory to a key. + AddAuthenticator(key string, fn AuthFactory) +} + +// A Authenticator implements a specific way to authorize requests. +// Each request is bound to a separate Authenticator instance. +// +// The authentication flow itself is broken down into `Authorize` +// and `Verify` steps. The former method runs before, and the latter +// runs after the `Request` is submitted. +// This makes it easy to encapsulate and control complex +// authentication challenges. +// +// Some authentication flows causing authentication round trips, +// which can be archived by returning the `redo` of the Verify +// method. `True` restarts the authentication process for the +// current action: A new `Request` is spawned, which must be +// authorized, sent, and re-verified again, until the action +// is successfully submitted. +// The preferred way is to handle the authentication ping-pong +// within `Verify`, and then `redo` with fresh credentials. +// +// The result of the `Verify` method can also trigger an +// `Authenticator` change by returning the `ErrAuthChanged` +// as an error. Depending on the `Authorizer` this may trigger +// an `Authenticator` negotiation. +// +// Set the `XInhibitRedirect` header to '1' in the `Authorize` +// method to get control over request redirection. +// Attention! You must handle the incoming request yourself. +// +// To store a shared session state the `Clone` method **must** +// return a new instance, initialized with the shared state. +type Authenticator interface { + // Authorizes a request. Usually by adding some authorization headers. + Authorize(c *http.Client, rq *http.Request, path string) error + // Verifies the response if the authorization was successful. + // May trigger some round trips to pass the authentication. + // May also trigger a new Authenticator negotiation by returning `ErrAuthChenged` + Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) + // Creates a copy of the underlying Authenticator. + Clone() Authenticator + io.Closer +} + +type authfactory struct { + key string + create AuthFactory +} + +// authorizer structure holds our Authenticator create functions +type authorizer struct { + factories []authfactory + defAuthMux sync.Mutex + defAuth Authenticator +} + +// preemptiveAuthorizer structure holds the preemptive Authenticator +type preemptiveAuthorizer struct { + auth Authenticator +} + +// authShim structure that wraps the real Authenticator +type authShim struct { + factory AuthFactory + body io.Reader + auth Authenticator +} + +// negoAuth structure holds the authenticators that are going to be negotiated +type negoAuth struct { + auths []Authenticator + setDefaultAuthenticator func(auth Authenticator) +} + +// nullAuth initializes the whole authentication flow +type nullAuth struct{} + +// noAuth structure to perform no authentication at all +type noAuth struct{} + +// NewAutoAuth creates an auto Authenticator factory. +// It negotiates the default authentication method +// based on the order of the registered Authenticators +// and the remotely offered authentication methods. +// First In, First Out. +func NewAutoAuth(login string, secret string) Authorizer { + fmap := make([]authfactory, 0) + az := &authorizer{factories: fmap, defAuthMux: sync.Mutex{}, defAuth: &nullAuth{}} + + az.AddAuthenticator("basic", func(c *http.Client, rs *http.Response, path string) (auth Authenticator, err error) { + return &BasicAuth{user: login, pw: secret}, nil + }) + + az.AddAuthenticator("digest", func(c *http.Client, rs *http.Response, path string) (auth Authenticator, err error) { + return NewDigestAuth(login, secret, rs) + }) + + az.AddAuthenticator("passport1.4", func(c *http.Client, rs *http.Response, path string) (auth Authenticator, err error) { + return NewPassportAuth(c, login, secret, rs.Request.URL.String(), &rs.Header) + }) + + return az +} + +// NewEmptyAuth creates an empty Authenticator factory +// The order of adding the Authenticator matters. +// First In, First Out. +// It offers the `NewAutoAuth` features. +func NewEmptyAuth() Authorizer { + fmap := make([]authfactory, 0) + az := &authorizer{factories: fmap, defAuthMux: sync.Mutex{}, defAuth: &nullAuth{}} + return az +} + +// NewPreemptiveAuth creates a preemptive Authenticator +// The preemptive authorizer uses the provided Authenticator +// for every request regardless of any `Www-Authenticate` header. +// +// It may only have one authentication method, +// so calling `AddAuthenticator` **will panic**! +// +// Look out!! This offers the skinniest and slickest implementation +// without any synchronisation!! +// Still applicable with `BasicAuth` within go routines. +func NewPreemptiveAuth(auth Authenticator) Authorizer { + return &preemptiveAuthorizer{auth: auth} +} + +// NewAuthenticator creates an Authenticator (Shim) per request +func (a *authorizer) NewAuthenticator(body io.Reader) (Authenticator, io.Reader) { + var retryBuf = body + if body != nil { + // If the authorization fails, we will need to restart reading + // from the passed body stream. + // When body is seekable, use seek to reset the streams + // cursor to the start. + // Otherwise, copy the stream into a buffer while uploading + // and use the buffers content on retry. + if _, ok := retryBuf.(io.Seeker); ok { + body = io.NopCloser(body) + } else { + buff := &bytes.Buffer{} + retryBuf = buff + body = io.TeeReader(body, buff) + } + } + a.defAuthMux.Lock() + defAuth := a.defAuth.Clone() + a.defAuthMux.Unlock() + + return &authShim{factory: a.factory, body: retryBuf, auth: defAuth}, body +} + +// AddAuthenticator appends the AuthFactory to our factories. +// It converts the key to lower case and preserves the order. +func (a *authorizer) AddAuthenticator(key string, fn AuthFactory) { + key = strings.ToLower(key) + for _, f := range a.factories { + if f.key == key { + panic("Authenticator exists: " + key) + } + } + a.factories = append(a.factories, authfactory{key, fn}) +} + +// factory picks all valid Authenticators based on Www-Authenticate headers +func (a *authorizer) factory(c *http.Client, rs *http.Response, path string) (auth Authenticator, err error) { + headers := rs.Header.Values("Www-Authenticate") + if len(headers) > 0 { + auths := make([]Authenticator, 0) + for _, f := range a.factories { + for _, header := range headers { + headerLower := strings.ToLower(header) + if strings.Contains(headerLower, f.key) { + rs.Header.Set("Www-Authenticate", header) + if auth, err = f.create(c, rs, path); err == nil { + auths = append(auths, auth) + break + } + } + } + } + + switch len(auths) { + case 0: + return nil, NewPathError("NoAuthenticator", path, rs.StatusCode) + case 1: + auth = auths[0] + default: + auth = &negoAuth{auths: auths, setDefaultAuthenticator: a.setDefaultAuthenticator} + } + } else { + auth = &noAuth{} + } + + a.setDefaultAuthenticator(auth) + + return auth, nil +} + +// setDefaultAuthenticator sets the default Authenticator +func (a *authorizer) setDefaultAuthenticator(auth Authenticator) { + a.defAuthMux.Lock() + a.defAuth.Close() + a.defAuth = auth + a.defAuthMux.Unlock() +} + +// Authorize the current request +func (s *authShim) Authorize(c *http.Client, rq *http.Request, path string) error { + if err := s.auth.Authorize(c, rq, path); err != nil { + return err + } + body := s.body + rq.GetBody = func() (io.ReadCloser, error) { + if body != nil { + if sk, ok := body.(io.Seeker); ok { + if _, err := sk.Seek(0, io.SeekStart); err != nil { + return nil, err + } + } + return io.NopCloser(body), nil + } + return nil, nil + } + return nil +} + +// Verify checks for authentication issues and may trigger a re-authentication. +// Catches AlgoChangedErr to update the current Authenticator +func (s *authShim) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) { + redo, err = s.auth.Verify(c, rs, path) + if err != nil && errors.Is(err, ErrAuthChanged) { + if auth, aerr := s.factory(c, rs, path); aerr == nil { + s.auth.Close() + s.auth = auth + return true, nil + } else { + return false, aerr + } + } + return +} + +// Close closes all resources +func (s *authShim) Close() error { + s.auth.Close() + s.auth, s.factory = nil, nil + if s.body != nil { + if closer, ok := s.body.(io.Closer); ok { + return closer.Close() + } + } + return nil +} + +// It's not intend to Clone the shim +// therefore it returns a noAuth instance +func (s *authShim) Clone() Authenticator { + return &noAuth{} +} + +// String toString +func (s *authShim) String() string { + return "AuthShim" +} + +// Authorize authorizes the current request with the top most Authorizer +func (n *negoAuth) Authorize(c *http.Client, rq *http.Request, path string) error { + if len(n.auths) == 0 { + return NewPathError("NoAuthenticator", path, 400) + } + return n.auths[0].Authorize(c, rq, path) +} + +// Verify verifies the authentication and selects the next one based on the result +func (n *negoAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) { + if len(n.auths) == 0 { + return false, NewPathError("NoAuthenticator", path, 400) + } + redo, err = n.auths[0].Verify(c, rs, path) + if err != nil { + if len(n.auths) > 1 { + n.auths[0].Close() + n.auths = n.auths[1:] + return true, nil + } + } else if redo { + return + } else { + auth := n.auths[0] + n.auths = n.auths[1:] + n.setDefaultAuthenticator(auth) + return + } + + return false, NewPathError("NoAuthenticator", path, rs.StatusCode) +} + +// Close will close the underlying authenticators. +func (n *negoAuth) Close() error { + for _, a := range n.auths { + a.Close() + } + n.setDefaultAuthenticator = nil + return nil +} + +// Clone clones the underlying authenticators. +func (n *negoAuth) Clone() Authenticator { + auths := make([]Authenticator, len(n.auths)) + for i, e := range n.auths { + auths[i] = e.Clone() + } + return &negoAuth{auths: auths, setDefaultAuthenticator: n.setDefaultAuthenticator} +} + +func (n *negoAuth) String() string { + return "NegoAuth" +} + +// Authorize the current request +func (n *noAuth) Authorize(c *http.Client, rq *http.Request, path string) error { + return nil +} + +// Verify checks for authentication issues and may trigger a re-authentication +func (n *noAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) { + if "" != rs.Header.Get("Www-Authenticate") { + err = ErrAuthChanged + } + return +} + +// Close closes all resources +func (n *noAuth) Close() error { + return nil +} + +// Clone creates a copy of itself +func (n *noAuth) Clone() Authenticator { + // no copy due to read only access + return n +} + +// String toString +func (n *noAuth) String() string { + return "NoAuth" +} + +// Authorize the current request +func (n *nullAuth) Authorize(c *http.Client, rq *http.Request, path string) error { + rq.Header.Set(XInhibitRedirect, "1") + return nil +} + +// Verify checks for authentication issues and may trigger a re-authentication +func (n *nullAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) { + return true, ErrAuthChanged +} + +// Close closes all resources +func (n *nullAuth) Close() error { + return nil +} + +// Clone creates a copy of itself +func (n *nullAuth) Clone() Authenticator { + // no copy due to read only access + return n +} + +// String toString +func (n *nullAuth) String() string { + return "NullAuth" +} + +// NewAuthenticator creates an Authenticator (Shim) per request +func (b *preemptiveAuthorizer) NewAuthenticator(body io.Reader) (Authenticator, io.Reader) { + return b.auth.Clone(), body +} + +// AddAuthenticator Will PANIC because it may only have a single authentication method +func (b *preemptiveAuthorizer) AddAuthenticator(key string, fn AuthFactory) { + panic("You're funny! A preemptive authorizer may only have a single authentication method") +} diff --git a/util/sync/webdavClient/auth_test.go b/util/sync/webdavClient/auth_test.go new file mode 100644 index 000000000..d12c0f0d2 --- /dev/null +++ b/util/sync/webdavClient/auth_test.go @@ -0,0 +1,62 @@ +package webdavClient + +import ( + "bytes" + "net/http" + "strings" + "testing" +) + +func TestEmptyAuth(t *testing.T) { + auth := NewEmptyAuth() + srv, _, _ := newAuthSrv(t, basicAuth) + defer srv.Close() + cli := NewAuthClient(srv.URL, auth) + if err := cli.Connect(); err == nil { + t.Fatalf("got nil want error") + } +} + +func TestRedirectAuthWIP(t *testing.T) { + hasPassedAuthServer := false + authHandler := func(h http.Handler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if user, passwd, ok := r.BasicAuth(); ok { + if user == "user" && passwd == "password" { + hasPassedAuthServer = true + w.WriteHeader(200) + return + } + } + w.Header().Set("Www-Authenticate", `Basic realm="x"`) + w.WriteHeader(401) + } + } + + psrv, _, _ := newAuthSrv(t, authHandler) + defer psrv.Close() + + dataHandler := func(h http.Handler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + hasAuth := strings.Contains(r.Header.Get("Authorization"), "Basic dXNlcjpwYXNzd29yZA==") + + if hasPassedAuthServer && hasAuth { + h.ServeHTTP(w, r) + return + } + w.Header().Set("Www-Authenticate", `Basic realm="x"`) + http.Redirect(w, r, psrv.URL+"/", 302) + } + } + + srv, _, _ := newAuthSrv(t, dataHandler) + defer srv.Close() + cli := NewClient(srv.URL, "user", "password") + data, err := cli.Read("/hello.txt") + if err != nil { + t.Logf("WIP got error=%v; want nil", err) + } + if bytes.Compare(data, []byte("hello gowebdav\n")) != 0 { + t.Logf("WIP got data=%v; want=hello gowebdav", data) + } +} diff --git a/util/sync/webdavClient/basicAuth.go b/util/sync/webdavClient/basicAuth.go new file mode 100644 index 000000000..10e5b8f4c --- /dev/null +++ b/util/sync/webdavClient/basicAuth.go @@ -0,0 +1,42 @@ +package webdavClient + +import ( + "fmt" + "net/http" +) + +// BasicAuth structure holds our credentials +type BasicAuth struct { + user string + pw string +} + +// Authorize the current request +func (b *BasicAuth) Authorize(c *http.Client, rq *http.Request, path string) error { + rq.SetBasicAuth(b.user, b.pw) + return nil +} + +// Verify verifies if the authentication +func (b *BasicAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) { + if rs.StatusCode == 401 { + err = NewPathError("Authorize", path, rs.StatusCode) + } + return +} + +// Close cleans up all resources +func (b *BasicAuth) Close() error { + return nil +} + +// Clone creates a Copy of itself +func (b *BasicAuth) Clone() Authenticator { + // no copy due to read only access + return b +} + +// String toString +func (b *BasicAuth) String() string { + return fmt.Sprintf("BasicAuth login: %s", b.user) +} diff --git a/util/sync/webdavClient/basicAuth_test.go b/util/sync/webdavClient/basicAuth_test.go new file mode 100644 index 000000000..3a62713af --- /dev/null +++ b/util/sync/webdavClient/basicAuth_test.go @@ -0,0 +1,51 @@ +package webdavClient + +import ( + "net/http" + "testing" +) + +func TestNewBasicAuth(t *testing.T) { + a := &BasicAuth{user: "user", pw: "password"} + + ex := "BasicAuth login: user" + if a.String() != ex { + t.Error("expected: " + ex + " got: " + a.String()) + } + + if a.Clone() != a { + t.Error("expected the same instance") + } + + if a.Close() != nil { + t.Error("expected close without errors") + } +} + +func TestBasicAuthAuthorize(t *testing.T) { + a := &BasicAuth{user: "user", pw: "password"} + rq, _ := http.NewRequest("GET", "http://localhost/", nil) + a.Authorize(nil, rq, "/") + if rq.Header.Get("Authorization") != "Basic dXNlcjpwYXNzd29yZA==" { + t.Error("got wrong Authorization header: " + rq.Header.Get("Authorization")) + } +} + +func TestPreemtiveBasicAuth(t *testing.T) { + a := &BasicAuth{user: "user", pw: "password"} + auth := NewPreemptiveAuth(a) + n, b := auth.NewAuthenticator(nil) + if b != nil { + t.Error("expected body to be nil") + } + if n != a { + t.Error("expected the same instance") + } + + srv, _, _ := newAuthSrv(t, basicAuth) + defer srv.Close() + cli := NewAuthClient(srv.URL, auth) + if err := cli.Connect(); err != nil { + t.Fatalf("got error: %v, want nil", err) + } +} diff --git a/util/sync/webdavClient/client.go b/util/sync/webdavClient/client.go new file mode 100644 index 000000000..fa806c5a0 --- /dev/null +++ b/util/sync/webdavClient/client.go @@ -0,0 +1,438 @@ +package webdavClient + +import ( + "bytes" + "encoding/xml" + "fmt" + "io" + "net/http" + "net/url" + "os" + pathpkg "path" + "strings" + "time" +) + +const XInhibitRedirect = "X-Gowebdav-Inhibit-Redirect" + +// Client defines our structure +type Client struct { + root string + headers http.Header + interceptor func(method string, rq *http.Request) + c *http.Client + auth Authorizer +} + +// NewClient creates a new instance of client +func NewClient(uri, user, pw string) *Client { + return NewAuthClient(uri, NewAutoAuth(user, pw)) +} + +// NewAuthClient creates a new client instance with a custom Authorizer +func NewAuthClient(uri string, auth Authorizer) *Client { + c := &http.Client{ + CheckRedirect: func(rq *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return ErrTooManyRedirects + } + if via[0].Header.Get(XInhibitRedirect) != "" { + return http.ErrUseLastResponse + } + return nil + }, + } + return &Client{root: FixSlash(uri), headers: make(http.Header), interceptor: nil, c: c, auth: auth} +} + +// SetHeader lets us set arbitrary headers for a given client +func (c *Client) SetHeader(key, value string) { + c.headers.Add(key, value) +} + +// SetInterceptor lets us set an arbitrary interceptor for a given client +func (c *Client) SetInterceptor(interceptor func(method string, rq *http.Request)) { + c.interceptor = interceptor +} + +// SetTimeout exposes the ability to set a time limit for requests +func (c *Client) SetTimeout(timeout time.Duration) { + c.c.Timeout = timeout +} + +// SetTransport exposes the ability to define custom transports +func (c *Client) SetTransport(transport http.RoundTripper) { + c.c.Transport = transport +} + +// SetJar exposes the ability to set a cookie jar to the client. +func (c *Client) SetJar(jar http.CookieJar) { + c.c.Jar = jar +} + +// Connect connects to our dav server +func (c *Client) Connect() error { + rs, err := c.options("/") + if err != nil { + return err + } + + err = rs.Body.Close() + if err != nil { + return err + } + + if rs.StatusCode != 200 { + return NewPathError("Connect", c.root, rs.StatusCode) + } + + return nil +} + +type props struct { + Status string `xml:"DAV: status"` + Name string `xml:"DAV: prop>displayname,omitempty"` + Type xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"` + Size string `xml:"DAV: prop>getcontentlength,omitempty"` + ContentType string `xml:"DAV: prop>getcontenttype,omitempty"` + ETag string `xml:"DAV: prop>getetag,omitempty"` + Modified string `xml:"DAV: prop>getlastmodified,omitempty"` +} + +type response struct { + Href string `xml:"DAV: href"` + Props []props `xml:"DAV: propstat"` +} + +func getProps(r *response, status string) *props { + for _, prop := range r.Props { + if strings.Contains(prop.Status, status) { + return &prop + } + } + return nil +} + +// ReadDir reads the contents of a remote directory +func (c *Client) ReadDir(path string) ([]os.FileInfo, error) { + path = FixSlashes(path) + files := make([]os.FileInfo, 0) + skipSelf := true + parse := func(resp interface{}) error { + r := resp.(*response) + + if skipSelf { + skipSelf = false + if p := getProps(r, "200"); p != nil && p.Type.Local == "collection" { + r.Props = nil + return nil + } + return NewPathError("ReadDir", path, 405) + } + + if p := getProps(r, "200"); p != nil { + f := new(File) + if ps, err := url.PathUnescape(r.Href); err == nil { + f.name = pathpkg.Base(ps) + } else { + f.name = p.Name + } + f.path = path + f.name + f.modified = parseModified(&p.Modified) + f.etag = p.ETag + f.contentType = p.ContentType + + if p.Type.Local == "collection" { + f.path += "/" + f.size = 0 + f.isdir = true + } else { + f.size = parseInt64(&p.Size) + f.isdir = false + } + + files = append(files, *f) + } + + r.Props = nil + return nil + } + + err := c.propfind(path, false, + ` + + + + + + + + + `, + &response{}, + parse) + + if err != nil { + if _, ok := err.(*os.PathError); !ok { + err = NewPathErrorErr("ReadDir", path, err) + } + } + return files, err +} + +// Stat returns the file stats for a specified path +func (c *Client) Stat(path string) (os.FileInfo, error) { + var f *File + parse := func(resp interface{}) error { + r := resp.(*response) + if p := getProps(r, "200"); p != nil && f == nil { + f = new(File) + f.name = p.Name + f.path = path + f.etag = p.ETag + f.contentType = p.ContentType + + if p.Type.Local == "collection" { + if !strings.HasSuffix(f.path, "/") { + f.path += "/" + } + f.size = 0 + f.modified = parseModified(&p.Modified) + f.isdir = true + } else { + f.size = parseInt64(&p.Size) + f.modified = parseModified(&p.Modified) + f.isdir = false + } + } + + r.Props = nil + return nil + } + + err := c.propfind(path, true, + ` + + + + + + + + + `, + &response{}, + parse) + + if err != nil { + if _, ok := err.(*os.PathError); !ok { + err = NewPathErrorErr("ReadDir", path, err) + } + } + return f, err +} + +// Remove removes a remote file +func (c *Client) Remove(path string) error { + return c.RemoveAll(path) +} + +// RemoveAll removes remote files +func (c *Client) RemoveAll(path string) error { + rs, err := c.req("DELETE", path, nil, nil) + if err != nil { + return NewPathError("Remove", path, 400) + } + err = rs.Body.Close() + if err != nil { + return err + } + + if rs.StatusCode == 200 || rs.StatusCode == 204 || rs.StatusCode == 404 { + return nil + } + + return NewPathError("Remove", path, rs.StatusCode) +} + +// Mkdir makes a directory +func (c *Client) Mkdir(path string, _ os.FileMode) (err error) { + path = FixSlashes(path) + status, err := c.mkcol(path) + if err != nil { + return + } + if status == 201 { + return nil + } + + return NewPathError("Mkdir", path, status) +} + +// MkdirAll like mkdir -p, but for webdav +func (c *Client) MkdirAll(path string, _ os.FileMode) (err error) { + path = FixSlashes(path) + status, err := c.mkcol(path) + if err != nil { + return + } + if status == 201 { + return nil + } + if status == 409 { + paths := strings.Split(path, "/") + sub := "/" + for _, e := range paths { + if e == "" { + continue + } + sub += e + "/" + status, err = c.mkcol(sub) + if err != nil { + return + } + if status != 201 { + return NewPathError("MkdirAll", sub, status) + } + } + return nil + } + + return NewPathError("MkdirAll", path, status) +} + +// Rename moves a file from A to B +func (c *Client) Rename(oldpath, newpath string, overwrite bool) error { + return c.copymove("MOVE", oldpath, newpath, overwrite) +} + +// Copy copies a file from A to B +func (c *Client) Copy(oldpath, newpath string, overwrite bool) error { + return c.copymove("COPY", oldpath, newpath, overwrite) +} + +// Read reads the contents of a remote file +func (c *Client) Read(path string) ([]byte, error) { + var stream io.ReadCloser + var err error + + if stream, err = c.ReadStream(path); err != nil { + return nil, err + } + defer stream.Close() + + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(stream) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// ReadStream reads the stream for a given path +func (c *Client) ReadStream(path string) (io.ReadCloser, error) { + rs, err := c.req("GET", path, nil, nil) + if err != nil { + return nil, NewPathErrorErr("ReadStream", path, err) + } + + if rs.StatusCode == 200 { + return rs.Body, nil + } + + rs.Body.Close() + return nil, NewPathError("ReadStream", path, rs.StatusCode) +} + +// ReadStreamRange reads the stream representing a subset of bytes for a given path, +// utilizing HTTP Range Requests if the server supports it. +// The range is expressed as offset from the start of the file and length, for example +// offset=10, length=10 will return bytes 10 through 19. +// +// If the server does not support partial content requests and returns full content instead, +// this function will emulate the behavior by skipping `offset` bytes and limiting the result +// to `length`. +func (c *Client) ReadStreamRange(path string, offset, length int64) (io.ReadCloser, error) { + rs, err := c.req("GET", path, nil, func(r *http.Request) { + if length > 0 { + r.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)) + } else { + r.Header.Add("Range", fmt.Sprintf("bytes=%d-", offset)) + } + }) + if err != nil { + return nil, NewPathErrorErr("ReadStreamRange", path, err) + } + + if rs.StatusCode == http.StatusPartialContent { + // server supported partial content, return as-is. + return rs.Body, nil + } + + // server returned success, but did not support partial content, so we have the whole + // stream in rs.Body + if rs.StatusCode == 200 { + // discard first 'offset' bytes. + if _, err := io.Copy(io.Discard, io.LimitReader(rs.Body, offset)); err != nil { + return nil, NewPathErrorErr("ReadStreamRange", path, err) + } + + // return a io.ReadCloser that is limited to `length` bytes. + return &limitedReadCloser{rc: rs.Body, remaining: int(length)}, nil + } + + rs.Body.Close() + return nil, NewPathError("ReadStream", path, rs.StatusCode) +} + +// Write writes data to a given path +func (c *Client) Write(path string, data []byte, _ os.FileMode) (err error) { + s, err := c.put(path, bytes.NewReader(data)) + if err != nil { + return + } + + switch s { + + case 200, 201, 204: + return nil + + case 404, 409: + err = c.createParentCollection(path) + if err != nil { + return + } + + s, err = c.put(path, bytes.NewReader(data)) + if err != nil { + return + } + if s == 200 || s == 201 || s == 204 { + return + } + } + + return NewPathError("Write", path, s) +} + +// WriteStream writes a stream +func (c *Client) WriteStream(path string, stream io.Reader, _ os.FileMode) (err error) { + + err = c.createParentCollection(path) + if err != nil { + return err + } + + s, err := c.put(path, stream) + if err != nil { + return err + } + + switch s { + case 200, 201, 204: + return nil + + default: + return NewPathError("WriteStream", path, s) + } +} diff --git a/util/sync/webdavClient/client_test.go b/util/sync/webdavClient/client_test.go new file mode 100644 index 000000000..8d60477a8 --- /dev/null +++ b/util/sync/webdavClient/client_test.go @@ -0,0 +1,574 @@ +package webdavClient + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + "time" + + "golang.org/x/net/webdav" +) + +func noAuthHndl(h http.Handler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + h.ServeHTTP(w, r) + } +} + +func basicAuth(h http.Handler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if user, passwd, ok := r.BasicAuth(); ok { + if user == "user" && passwd == "password" { + h.ServeHTTP(w, r) + return + } + + http.Error(w, "not authorized", 403) + } else { + w.Header().Set("WWW-Authenticate", `Basic realm="x"`) + w.WriteHeader(401) + } + } +} + +func multipleAuth(h http.Handler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + notAuthed := false + if r.Header.Get("Authorization") == "" { + notAuthed = true + } else if user, passwd, ok := r.BasicAuth(); ok { + if user == "user" && passwd == "password" { + h.ServeHTTP(w, r) + return + } + notAuthed = true + } else if strings.HasPrefix(r.Header.Get("Authorization"), "Digest ") { + pairs := strings.TrimPrefix(r.Header.Get("Authorization"), "Digest ") + digestParts := make(map[string]string) + for _, pair := range strings.Split(pairs, ",") { + kv := strings.SplitN(strings.TrimSpace(pair), "=", 2) + key, value := kv[0], kv[1] + value = strings.Trim(value, `"`) + digestParts[key] = value + } + if digestParts["qop"] == "" { + digestParts["qop"] = "auth" + } + + ha1 := getMD5(fmt.Sprint(digestParts["username"], ":", digestParts["realm"], ":", "digestPW")) + ha2 := getMD5(fmt.Sprint(r.Method, ":", digestParts["uri"])) + expected := getMD5(fmt.Sprint(ha1, + ":", digestParts["nonce"], + ":", digestParts["nc"], + ":", digestParts["cnonce"], + ":", digestParts["qop"], + ":", ha2)) + + if expected == digestParts["response"] { + h.ServeHTTP(w, r) + return + } + notAuthed = true + } + + if notAuthed { + w.Header().Add("WWW-Authenticate", `Digest realm="testrealm@host.com", qop="auth,auth-int",nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093",opaque="5ccc069c403ebaf9f0171e9517f40e41"`) + w.Header().Add("WWW-Authenticate", `Basic realm="x"`) + w.WriteHeader(401) + } + } +} + +func fillFs(t *testing.T, fs webdav.FileSystem) context.Context { + ctx := context.Background() + f, err := fs.OpenFile(ctx, "hello.txt", os.O_CREATE, 0644) + if err != nil { + t.Errorf("fail to crate file: %v", err) + } + f.Write([]byte("hello gowebdav\n")) + f.Close() + err = fs.Mkdir(ctx, "/test", 0755) + if err != nil { + t.Errorf("fail to crate directory: %v", err) + } + f, err = fs.OpenFile(ctx, "/test/test.txt", os.O_CREATE, 0644) + if err != nil { + t.Errorf("fail to crate file: %v", err) + } + f.Write([]byte("test test gowebdav\n")) + f.Close() + return ctx +} + +func newServer(t *testing.T) (*Client, *httptest.Server, webdav.FileSystem, context.Context) { + return newAuthServer(t, basicAuth) +} + +func newAuthServer(t *testing.T, auth func(h http.Handler) http.HandlerFunc) (*Client, *httptest.Server, webdav.FileSystem, context.Context) { + srv, fs, ctx := newAuthSrv(t, auth) + cli := NewClient(srv.URL, "user", "password") + return cli, srv, fs, ctx +} + +func newAuthSrv(t *testing.T, auth func(h http.Handler) http.HandlerFunc) (*httptest.Server, webdav.FileSystem, context.Context) { + mux := http.NewServeMux() + fs := webdav.NewMemFS() + ctx := fillFs(t, fs) + mux.HandleFunc("/", auth(&webdav.Handler{ + FileSystem: fs, + LockSystem: webdav.NewMemLS(), + })) + srv := httptest.NewServer(mux) + return srv, fs, ctx +} + +func TestConnect(t *testing.T) { + cli, srv, _, _ := newServer(t) + defer srv.Close() + if err := cli.Connect(); err != nil { + t.Fatalf("got error: %v, want nil", err) + } + + cli = NewClient(srv.URL, "no", "no") + if err := cli.Connect(); err == nil { + t.Fatalf("got nil, want error: %v", err) + } +} + +func TestConnectMultipleAuth(t *testing.T) { + cli, srv, _, _ := newAuthServer(t, multipleAuth) + defer srv.Close() + if err := cli.Connect(); err != nil { + t.Fatalf("got error: %v, want nil", err) + } + + cli = NewClient(srv.URL, "digestUser", "digestPW") + if err := cli.Connect(); err != nil { + t.Fatalf("got nil, want error: %v", err) + } + + cli = NewClient(srv.URL, "no", "no") + if err := cli.Connect(); err == nil { + t.Fatalf("got nil, want error: %v", err) + } +} + +func TestConnectMultiAuthII(t *testing.T) { + cli, srv, _, _ := newAuthServer(t, func(h http.Handler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if user, passwd, ok := r.BasicAuth(); ok { + if user == "user" && passwd == "password" { + h.ServeHTTP(w, r) + return + } + + http.Error(w, "not authorized", 403) + } else { + w.Header().Add("WWW-Authenticate", `FooAuth`) + w.Header().Add("WWW-Authenticate", `BazAuth`) + w.Header().Add("WWW-Authenticate", `BarAuth`) + w.Header().Add("WWW-Authenticate", `Basic realm="x"`) + w.WriteHeader(401) + } + } + }) + defer srv.Close() + if err := cli.Connect(); err != nil { + t.Fatalf("got error: %v, want nil", err) + } + + cli = NewClient(srv.URL, "no", "no") + if err := cli.Connect(); err == nil { + t.Fatalf("got nil, want error: %v", err) + } +} + +func TestReadDirConcurrent(t *testing.T) { + cli, srv, _, _ := newServer(t) + defer srv.Close() + + var wg sync.WaitGroup + errs := make(chan error, 2) + for i := 0; i < 2; i++ { + wg.Add(1) + + go func() { + defer wg.Done() + f, err := cli.ReadDir("/") + if err != nil { + errs <- errors.New(fmt.Sprintf("got error: %v, want file listing: %v", err, f)) + } + if len(f) != 2 { + errs <- errors.New(fmt.Sprintf("f: %v err: %v", f, err)) + } + if f[0].Name() != "hello.txt" && f[1].Name() != "hello.txt" { + errs <- errors.New(fmt.Sprintf("got: %v, want file: %s", f, "hello.txt")) + } + if f[0].Name() != "test" && f[1].Name() != "test" { + errs <- errors.New(fmt.Sprintf("got: %v, want directory: %s", f, "test")) + } + }() + } + + wg.Wait() + close(errs) + + for err := range errs { + if err != nil { + t.Fatal(err) + } + } +} + +func TestRead(t *testing.T) { + cli, srv, _, _ := newServer(t) + defer srv.Close() + + data, err := cli.Read("/hello.txt") + if err != nil || bytes.Compare(data, []byte("hello gowebdav\n")) != 0 { + t.Fatalf("got: %v, want data: %s", err, []byte("hello gowebdav\n")) + } + + data, err = cli.Read("/404.txt") + if err == nil { + t.Fatalf("got: %v, want error: %v", data, err) + } + if !IsErrNotFound(err) { + t.Fatalf("got: %v, want 404 error", err) + } +} + +func TestReadNoAuth(t *testing.T) { + cli, srv, _, _ := newAuthServer(t, noAuthHndl) + defer srv.Close() + + data, err := cli.Read("/hello.txt") + if err != nil || bytes.Compare(data, []byte("hello gowebdav\n")) != 0 { + t.Fatalf("got: %v, want data: %s", err, []byte("hello gowebdav\n")) + } + + data, err = cli.Read("/404.txt") + if err == nil { + t.Fatalf("got: %v, want error: %v", data, err) + } + if !IsErrNotFound(err) { + t.Fatalf("got: %v, want 404 error", err) + } +} + +func TestReadStream(t *testing.T) { + cli, srv, _, _ := newServer(t) + defer srv.Close() + + stream, err := cli.ReadStream("/hello.txt") + if err != nil { + t.Fatalf("got: %v, want data: %v", err, stream) + } + buf := new(bytes.Buffer) + buf.ReadFrom(stream) + if buf.String() != "hello gowebdav\n" { + t.Fatalf("got: %v, want stream: hello gowebdav", buf.String()) + } + + stream, err = cli.ReadStream("/404/hello.txt") + if err == nil { + t.Fatalf("got: %v, want error: %v", stream, err) + } +} + +func TestReadStreamRange(t *testing.T) { + cli, srv, _, _ := newServer(t) + defer srv.Close() + + stream, err := cli.ReadStreamRange("/hello.txt", 4, 4) + if err != nil { + t.Fatalf("got: %v, want data: %v", err, stream) + } + buf := new(bytes.Buffer) + buf.ReadFrom(stream) + if buf.String() != "o go" { + t.Fatalf("got: %v, want stream: o go", buf.String()) + } + + stream, err = cli.ReadStream("/404/hello.txt") + if err == nil { + t.Fatalf("got: %v, want error: %v", stream, err) + } +} + +func TestReadStreamRangeUnkownLength(t *testing.T) { + cli, srv, _, _ := newServer(t) + defer srv.Close() + + stream, err := cli.ReadStreamRange("/hello.txt", 6, 0) + if err != nil { + t.Fatalf("got: %v, want data: %v", err, stream) + } + buf := new(bytes.Buffer) + buf.ReadFrom(stream) + if buf.String() != "gowebdav\n" { + t.Fatalf("got: %v, want stream: gowebdav\n", buf.String()) + } + + stream, err = cli.ReadStream("/404/hello.txt") + if err == nil { + t.Fatalf("got: %v, want error: %v", stream, err) + } +} + +func TestStat(t *testing.T) { + cli, srv, _, _ := newServer(t) + defer srv.Close() + + info, err := cli.Stat("/hello.txt") + if err != nil { + t.Fatalf("got: %v, want os.Info: %v", err, info) + } + if info.Name() != "hello.txt" { + t.Fatalf("got: %v, want file hello.txt", info) + } + + info, err = cli.Stat("/404.txt") + if err == nil { + t.Fatalf("got: %v, want error: %v", info, err) + } + if !IsErrNotFound(err) { + t.Fatalf("got: %v, want 404 error", err) + } +} + +func TestMkdir(t *testing.T) { + cli, srv, fs, ctx := newServer(t) + defer srv.Close() + + info, err := cli.Stat("/newdir") + if err == nil { + t.Fatalf("got: %v, want error: %v", info, err) + } + + if err := cli.Mkdir("/newdir", 0755); err != nil { + t.Fatalf("got: %v, want mkdir /newdir", err) + } + + if err := cli.Mkdir("/newdir", 0755); err != nil { + t.Fatalf("got: %v, want mkdir /newdir", err) + } + + info, err = fs.Stat(ctx, "/newdir") + if err != nil { + t.Fatalf("got: %v, want dir info: %v", err, info) + } + + if err := cli.Mkdir("/404/newdir", 0755); err == nil { + t.Fatalf("expected Mkdir error due to missing parent directory") + } +} + +func TestMkdirAll(t *testing.T) { + cli, srv, fs, ctx := newServer(t) + defer srv.Close() + + if err := cli.MkdirAll("/dir/dir/dir", 0755); err != nil { + t.Fatalf("got: %v, want mkdirAll /dir/dir/dir", err) + } + + info, err := fs.Stat(ctx, "/dir/dir/dir") + if err != nil { + t.Fatalf("got: %v, want dir info: %v", err, info) + } +} + +func TestCopy(t *testing.T) { + cli, srv, fs, ctx := newServer(t) + defer srv.Close() + + info, err := fs.Stat(ctx, "/copy.txt") + if err == nil { + t.Fatalf("got: %v, want error: %v", info, err) + } + + if err := cli.Copy("/hello.txt", "/copy.txt", false); err != nil { + t.Fatalf("got: %v, want copy /hello.txt to /copy.txt", err) + } + + info, err = fs.Stat(ctx, "/copy.txt") + if err != nil { + t.Fatalf("got: %v, want file info: %v", err, info) + } + if info.Size() != 15 { + t.Fatalf("got: %v, want file size: %d bytes", info.Size(), 15) + } + + info, err = fs.Stat(ctx, "/hello.txt") + if err != nil { + t.Fatalf("got: %v, want file info: %v", err, info) + } + if info.Size() != 15 { + t.Fatalf("got: %v, want file size: %d bytes", info.Size(), 15) + } + + if err := cli.Copy("/hello.txt", "/copy.txt", false); err == nil { + t.Fatalf("expected copy error due to overwrite false") + } + + if err := cli.Copy("/hello.txt", "/copy.txt", true); err != nil { + t.Fatalf("got: %v, want overwrite /copy.txt with /hello.txt", err) + } +} + +func TestRename(t *testing.T) { + cli, srv, fs, ctx := newServer(t) + defer srv.Close() + + info, err := fs.Stat(ctx, "/copy.txt") + if err == nil { + t.Fatalf("got: %v, want error: %v", info, err) + } + + if err := cli.Rename("/hello.txt", "/copy.txt", false); err != nil { + t.Fatalf("got: %v, want mv /hello.txt to /copy.txt", err) + } + + info, err = fs.Stat(ctx, "/copy.txt") + if err != nil { + t.Fatalf("got: %v, want file info: %v", err, info) + } + if info.Size() != 15 { + t.Fatalf("got: %v, want file size: %d bytes", info.Size(), 15) + } + + if info, err = fs.Stat(ctx, "/hello.txt"); err == nil { + t.Fatalf("got: %v, want error: %v", info, err) + } + + if err := cli.Rename("/test/test.txt", "/copy.txt", true); err != nil { + t.Fatalf("got: %v, want overwrite /copy.txt with /hello.txt", err) + } + info, err = fs.Stat(ctx, "/copy.txt") + if err != nil { + t.Fatalf("got: %v, want file info: %v", err, info) + } + if info.Size() != 19 { + t.Fatalf("got: %v, want file size: %d bytes", info.Size(), 19) + } +} + +func TestRemove(t *testing.T) { + cli, srv, fs, ctx := newServer(t) + defer srv.Close() + + if err := cli.Remove("/hello.txt"); err != nil { + t.Fatalf("got: %v, want nil", err) + } + + if info, err := fs.Stat(ctx, "/hello.txt"); err == nil { + t.Fatalf("got: %v, want error: %v", info, err) + } + + if err := cli.Remove("/404.txt"); err != nil { + t.Fatalf("got: %v, want nil", err) + } +} + +func TestRemoveAll(t *testing.T) { + cli, srv, fs, ctx := newServer(t) + defer srv.Close() + + if err := cli.RemoveAll("/test/test.txt"); err != nil { + t.Fatalf("got: %v, want nil", err) + } + + if info, err := fs.Stat(ctx, "/test/test.txt"); err == nil { + t.Fatalf("got: %v, want error: %v", info, err) + } + + if err := cli.RemoveAll("/404.txt"); err != nil { + t.Fatalf("got: %v, want nil", err) + } + + if err := cli.RemoveAll("/404/404/404.txt"); err != nil { + t.Fatalf("got: %v, want nil", err) + } +} + +func TestWrite(t *testing.T) { + cli, srv, fs, ctx := newServer(t) + defer srv.Close() + + if err := cli.Write("/newfile.txt", []byte("foo bar\n"), 0660); err != nil { + t.Fatalf("got: %v, want nil", err) + } + + info, err := fs.Stat(ctx, "/newfile.txt") + if err != nil { + t.Fatalf("got: %v, want file info: %v", err, info) + } + if info.Size() != 8 { + t.Fatalf("got: %v, want file size: %d bytes", info.Size(), 8) + } + + if err := cli.Write("/404/newfile.txt", []byte("foo bar\n"), 0660); err != nil { + t.Fatalf("got: %v, want nil", err) + } +} + +func TestWriteStream(t *testing.T) { + cli, srv, fs, ctx := newServer(t) + defer srv.Close() + + if err := cli.WriteStream("/newfile.txt", strings.NewReader("foo bar\n"), 0660); err != nil { + t.Fatalf("got: %v, want nil", err) + } + + info, err := fs.Stat(ctx, "/newfile.txt") + if err != nil { + t.Fatalf("got: %v, want file info: %v", err, info) + } + if info.Size() != 8 { + t.Fatalf("got: %v, want file size: %d bytes", info.Size(), 8) + } + + if err := cli.WriteStream("/404/works.txt", strings.NewReader("foo bar\n"), 0660); err != nil { + t.Fatalf("got: %v, want nil", err) + } + + if info, err := fs.Stat(ctx, "/404/works.txt"); err != nil { + t.Fatalf("got: %v, want file info: %v", err, info) + } +} + +func TestWriteStreamFromPipe(t *testing.T) { + cli, srv, fs, ctx := newServer(t) + defer srv.Close() + + r, w := io.Pipe() + + go func() { + defer w.Close() + fmt.Fprint(w, "foo") + time.Sleep(1 * time.Second) + fmt.Fprint(w, " ") + time.Sleep(1 * time.Second) + fmt.Fprint(w, "bar\n") + }() + + if err := cli.WriteStream("/newfile.txt", r, 0660); err != nil { + t.Fatalf("got: %v, want nil", err) + } + + info, err := fs.Stat(ctx, "/newfile.txt") + if err != nil { + t.Fatalf("got: %v, want file info: %v", err, info) + } + if info.Size() != 8 { + t.Fatalf("got: %v, want file size: %d bytes", info.Size(), 8) + } +} diff --git a/util/sync/webdavClient/digestAuth.go b/util/sync/webdavClient/digestAuth.go new file mode 100644 index 000000000..5ac632051 --- /dev/null +++ b/util/sync/webdavClient/digestAuth.go @@ -0,0 +1,164 @@ +package webdavClient + +import ( + "crypto/md5" + "crypto/rand" + "encoding/hex" + "fmt" + "io" + "net/http" + "strings" +) + +// DigestAuth structure holds our credentials +type DigestAuth struct { + user string + pw string + digestParts map[string]string +} + +// NewDigestAuth creates a new instance of our Digest Authenticator +func NewDigestAuth(login, secret string, rs *http.Response) (Authenticator, error) { + return &DigestAuth{user: login, pw: secret, digestParts: digestParts(rs)}, nil +} + +// Authorize the current request +func (d *DigestAuth) Authorize(c *http.Client, rq *http.Request, path string) error { + d.digestParts["uri"] = path + d.digestParts["method"] = rq.Method + d.digestParts["username"] = d.user + d.digestParts["password"] = d.pw + rq.Header.Set("Authorization", getDigestAuthorization(d.digestParts)) + return nil +} + +// Verify checks for authentication issues and may trigger a re-authentication +func (d *DigestAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) { + if rs.StatusCode == 401 { + err = NewPathError("Authorize", path, rs.StatusCode) + } + return +} + +// Close cleans up all resources +func (d *DigestAuth) Close() error { + return nil +} + +// Clone creates a copy of itself +func (d *DigestAuth) Clone() Authenticator { + parts := make(map[string]string, len(d.digestParts)) + for k, v := range d.digestParts { + parts[k] = v + } + return &DigestAuth{user: d.user, pw: d.pw, digestParts: parts} +} + +// String toString +func (d *DigestAuth) String() string { + return fmt.Sprintf("DigestAuth login: %s", d.user) +} + +func digestParts(resp *http.Response) map[string]string { + result := map[string]string{} + if len(resp.Header["Www-Authenticate"]) > 0 { + wantedHeaders := []string{"nonce", "realm", "qop", "opaque", "algorithm", "entityBody"} + responseHeaders := strings.Split(resp.Header["Www-Authenticate"][0], ",") + for _, r := range responseHeaders { + for _, w := range wantedHeaders { + if strings.Contains(r, w) { + result[w] = strings.Trim( + strings.SplitN(r, `=`, 2)[1], + `"`, + ) + } + } + } + } + return result +} + +func getMD5(text string) string { + hasher := md5.New() + hasher.Write([]byte(text)) + return hex.EncodeToString(hasher.Sum(nil)) +} + +func getCnonce() string { + b := make([]byte, 8) + io.ReadFull(rand.Reader, b) + return fmt.Sprintf("%x", b)[:16] +} + +func getDigestAuthorization(digestParts map[string]string) string { + d := digestParts + // These are the correct ha1 and ha2 for qop=auth. We should probably check for other types of qop. + + var ( + ha1 string + ha2 string + nonceCount = 00000001 + cnonce = getCnonce() + response string + ) + + // 'ha1' value depends on value of "algorithm" field + switch d["algorithm"] { + case "MD5", "": + ha1 = getMD5(d["username"] + ":" + d["realm"] + ":" + d["password"]) + case "MD5-sess": + ha1 = getMD5( + fmt.Sprintf("%s:%v:%s", + getMD5(d["username"]+":"+d["realm"]+":"+d["password"]), + nonceCount, + cnonce, + ), + ) + } + + // 'ha2' value depends on value of "qop" field + switch d["qop"] { + case "auth", "": + ha2 = getMD5(d["method"] + ":" + d["uri"]) + case "auth-int": + if d["entityBody"] != "" { + ha2 = getMD5(d["method"] + ":" + d["uri"] + ":" + getMD5(d["entityBody"])) + } + } + + // 'response' value depends on value of "qop" field + switch d["qop"] { + case "": + response = getMD5( + fmt.Sprintf("%s:%s:%s", + ha1, + d["nonce"], + ha2, + ), + ) + case "auth", "auth-int": + response = getMD5( + fmt.Sprintf("%s:%s:%v:%s:%s:%s", + ha1, + d["nonce"], + nonceCount, + cnonce, + d["qop"], + ha2, + ), + ) + } + + authorization := fmt.Sprintf(`Digest username="%s", realm="%s", nonce="%s", uri="%s", nc=%v, cnonce="%s", response="%s"`, + d["username"], d["realm"], d["nonce"], d["uri"], nonceCount, cnonce, response) + + if d["qop"] != "" { + authorization += fmt.Sprintf(`, qop=%s`, d["qop"]) + } + + if d["opaque"] != "" { + authorization += fmt.Sprintf(`, opaque="%s"`, d["opaque"]) + } + + return authorization +} diff --git a/util/sync/webdavClient/digestAuth_test.go b/util/sync/webdavClient/digestAuth_test.go new file mode 100644 index 000000000..2adae4a68 --- /dev/null +++ b/util/sync/webdavClient/digestAuth_test.go @@ -0,0 +1,35 @@ +package webdavClient + +import ( + "net/http" + "strings" + "testing" +) + +func TestNewDigestAuth(t *testing.T) { + a := &DigestAuth{user: "user", pw: "password", digestParts: make(map[string]string, 0)} + + ex := "DigestAuth login: user" + if a.String() != ex { + t.Error("expected: " + ex + " got: " + a.String()) + } + + if a.Clone() == a { + t.Error("expected a different instance") + } + + if a.Close() != nil { + t.Error("expected close without errors") + } +} + +func TestDigestAuthAuthorize(t *testing.T) { + a := &DigestAuth{user: "user", pw: "password", digestParts: make(map[string]string, 0)} + rq, _ := http.NewRequest("GET", "http://localhost/", nil) + a.Authorize(nil, rq, "/") + // TODO this is a very lazy test it cuts of cnonce + ex := `Digest username="user", realm="", nonce="", uri="/", nc=1, cnonce="` + if strings.Index(rq.Header.Get("Authorization"), ex) != 0 { + t.Error("got wrong Authorization header: " + rq.Header.Get("Authorization")) + } +} diff --git a/util/sync/webdavClient/doc.go b/util/sync/webdavClient/doc.go new file mode 100644 index 000000000..db5c24558 --- /dev/null +++ b/util/sync/webdavClient/doc.go @@ -0,0 +1,3 @@ +// Package gowebdav is a WebDAV client library with a command line tool +// included. +package webdavClient diff --git a/util/sync/webdavClient/errors.go b/util/sync/webdavClient/errors.go new file mode 100644 index 000000000..13488f812 --- /dev/null +++ b/util/sync/webdavClient/errors.go @@ -0,0 +1,57 @@ +package webdavClient + +import ( + "errors" + "fmt" + "os" +) + +// ErrAuthChanged must be returned from the Verify method as an error +// to trigger a re-authentication / negotiation with a new authenticator. +var ErrAuthChanged = errors.New("authentication failed, change algorithm") + +// ErrTooManyRedirects will be used as return error if a request exceeds 10 redirects. +var ErrTooManyRedirects = errors.New("stopped after 10 redirects") + +// StatusError implements error and wraps +// an erroneous status code. +type StatusError struct { + Status int +} + +func (se StatusError) Error() string { + return fmt.Sprintf("%d", se.Status) +} + +// IsErrCode returns true if the given error +// is an os.PathError wrapping a StatusError +// with the given status code. +func IsErrCode(err error, code int) bool { + if pe, ok := err.(*os.PathError); ok { + se, ok := pe.Err.(StatusError) + return ok && se.Status == code + } + return false +} + +// IsErrNotFound is shorthand for IsErrCode +// for status 404. +func IsErrNotFound(err error) bool { + return IsErrCode(err, 404) +} + +func NewPathError(op string, path string, statusCode int) error { + return &os.PathError{ + Op: op, + Path: path, + Err: StatusError{statusCode}, + } +} + +func NewPathErrorErr(op string, path string, err error) error { + return &os.PathError{ + Op: op, + Path: path, + Err: err, + } +} diff --git a/util/sync/webdavClient/file.go b/util/sync/webdavClient/file.go new file mode 100644 index 000000000..0925be671 --- /dev/null +++ b/util/sync/webdavClient/file.go @@ -0,0 +1,77 @@ +package webdavClient + +import ( + "fmt" + "os" + "time" +) + +// File is our structure for a given file +type File struct { + path string + name string + contentType string + size int64 + modified time.Time + etag string + isdir bool +} + +// Path returns the full path of a file +func (f File) Path() string { + return f.path +} + +// Name returns the name of a file +func (f File) Name() string { + return f.name +} + +// ContentType returns the content type of a file +func (f File) ContentType() string { + return f.contentType +} + +// Size returns the size of a file +func (f File) Size() int64 { + return f.size +} + +// Mode will return the mode of a given file +func (f File) Mode() os.FileMode { + // TODO check webdav perms + if f.isdir { + return 0775 | os.ModeDir + } + + return 0664 +} + +// ModTime returns the modified time of a file +func (f File) ModTime() time.Time { + return f.modified +} + +// ETag returns the ETag of a file +func (f File) ETag() string { + return f.etag +} + +// IsDir let us see if a given file is a directory or not +func (f File) IsDir() bool { + return f.isdir +} + +// Sys ???? +func (f File) Sys() interface{} { + return nil +} + +// String lets us see file information +func (f File) String() string { + if f.isdir { + return fmt.Sprintf("Dir : '%s' - '%s'", f.path, f.name) + } + + return fmt.Sprintf("File: '%s' SIZE: %d MODIFIED: %s ETAG: %s CTYPE: %s", f.path, f.size, f.modified.String(), f.etag, f.contentType) +} diff --git a/util/sync/webdavClient/netrc.go b/util/sync/webdavClient/netrc.go new file mode 100644 index 000000000..1bf0eaab3 --- /dev/null +++ b/util/sync/webdavClient/netrc.go @@ -0,0 +1,54 @@ +package webdavClient + +import ( + "bufio" + "fmt" + "net/url" + "os" + "regexp" + "strings" +) + +func parseLine(s string) (login, pass string) { + fields := strings.Fields(s) + for i, f := range fields { + if f == "login" { + login = fields[i+1] + } + if f == "password" { + pass = fields[i+1] + } + } + return login, pass +} + +// ReadConfig reads login and password configuration from ~/.netrc +// machine foo.com login username password 123456 +func ReadConfig(uri, netrc string) (string, string) { + u, err := url.Parse(uri) + if err != nil { + return "", "" + } + + file, err := os.Open(netrc) + if err != nil { + return "", "" + } + defer file.Close() + + re := fmt.Sprintf(`^.*machine %s.*$`, u.Host) + scanner := bufio.NewScanner(file) + for scanner.Scan() { + s := scanner.Text() + + matched, err := regexp.MatchString(re, s) + if err != nil { + return "", "" + } + if matched { + return parseLine(s) + } + } + + return "", "" +} diff --git a/util/sync/webdavClient/passportAuth.go b/util/sync/webdavClient/passportAuth.go new file mode 100644 index 000000000..35633849e --- /dev/null +++ b/util/sync/webdavClient/passportAuth.go @@ -0,0 +1,181 @@ +package webdavClient + +import ( + "fmt" + "io" + "net/http" + "net/url" + "strings" +) + +// PassportAuth structure holds our credentials +type PassportAuth struct { + user string + pw string + cookies []http.Cookie + inhibitRedirect bool +} + +// constructor for PassportAuth creates a new PassportAuth object and +// automatically authenticates against the given partnerURL +func NewPassportAuth(c *http.Client, user, pw, partnerURL string, header *http.Header) (Authenticator, error) { + p := &PassportAuth{ + user: user, + pw: pw, + inhibitRedirect: true, + } + err := p.genCookies(c, partnerURL, header) + return p, err +} + +// Authorize the current request +func (p *PassportAuth) Authorize(c *http.Client, rq *http.Request, path string) error { + // prevent redirects to detect subsequent authentication requests + if p.inhibitRedirect { + rq.Header.Set(XInhibitRedirect, "1") + } else { + p.inhibitRedirect = true + } + for _, cookie := range p.cookies { + rq.AddCookie(&cookie) + } + return nil +} + +// Verify verifies if the authentication is good +func (p *PassportAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) { + switch rs.StatusCode { + case 301, 302, 307, 308: + redo = true + if rs.Header.Get("Www-Authenticate") != "" { + // re-authentication required as we are redirected to the login page + err = p.genCookies(c, rs.Request.URL.String(), &rs.Header) + } else { + // just a redirect, follow it + p.inhibitRedirect = false + } + case 401: + err = NewPathError("Authorize", path, rs.StatusCode) + } + return +} + +// Close cleans up all resources +func (p *PassportAuth) Close() error { + return nil +} + +// Clone creates a Copy of itself +func (p *PassportAuth) Clone() Authenticator { + // create a copy to allow independent cookie updates + clonedCookies := make([]http.Cookie, len(p.cookies)) + copy(clonedCookies, p.cookies) + + return &PassportAuth{ + user: p.user, + pw: p.pw, + cookies: clonedCookies, + inhibitRedirect: true, + } +} + +// String toString +func (p *PassportAuth) String() string { + return fmt.Sprintf("PassportAuth login: %s", p.user) +} + +func (p *PassportAuth) genCookies(c *http.Client, partnerUrl string, header *http.Header) error { + // For more details refer to: + // https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-pass/2c80637d-438c-4d4b-adc5-903170a779f3 + // Skipping step 1 and 2 as we already have the partner server challenge + + baseAuthenticationServer := header.Get("Location") + baseAuthenticationServerURL, err := url.Parse(baseAuthenticationServer) + if err != nil { + return err + } + + // Skipping step 3 and 4 as we already know that we need and have the user's credentials + // Step 5 (Sign-in request) + authenticationServerUrl := url.URL{ + Scheme: baseAuthenticationServerURL.Scheme, + Host: baseAuthenticationServerURL.Host, + Path: "/login2.srf", + } + + partnerServerChallenge := strings.Split(header.Get("Www-Authenticate"), " ")[1] + + req := http.Request{ + Method: "GET", + URL: &authenticationServerUrl, + Header: http.Header{ + "Authorization": []string{"Passport1.4 sign-in=" + url.QueryEscape(p.user) + ",pwd=" + url.QueryEscape(p.pw) + ",OrgVerb=GET,OrgUrl=" + partnerUrl + "," + partnerServerChallenge}, + }, + } + + rs, err := c.Do(&req) + if err != nil { + return err + } + io.Copy(io.Discard, rs.Body) + rs.Body.Close() + if rs.StatusCode != 200 { + return NewPathError("Authorize", "/", rs.StatusCode) + } + + // Step 6 (Token Response from Authentication Server) + tokenResponseHeader := rs.Header.Get("Authentication-Info") + if tokenResponseHeader == "" { + return NewPathError("Authorize", "/", 401) + } + tokenResponseHeaderList := strings.Split(tokenResponseHeader, ",") + token := "" + for _, tokenResponseHeader := range tokenResponseHeaderList { + if strings.HasPrefix(tokenResponseHeader, "from-PP='") { + token = tokenResponseHeader + break + } + } + if token == "" { + return NewPathError("Authorize", "/", 401) + } + + // Step 7 (First Authentication Request to Partner Server) + origUrl, err := url.Parse(partnerUrl) + if err != nil { + return err + } + req = http.Request{ + Method: "GET", + URL: origUrl, + Header: http.Header{ + "Authorization": []string{"Passport1.4 " + token}, + }, + } + + rs, err = c.Do(&req) + if err != nil { + return err + } + io.Copy(io.Discard, rs.Body) + rs.Body.Close() + if rs.StatusCode != 200 && rs.StatusCode != 302 { + return NewPathError("Authorize", "/", rs.StatusCode) + } + + // Step 8 (Set Token Message from Partner Server) + cookies := rs.Header.Values("Set-Cookie") + p.cookies = make([]http.Cookie, len(cookies)) + for i, cookie := range cookies { + cookieParts := strings.Split(cookie, ";") + cookieName := strings.Split(cookieParts[0], "=")[0] + cookieValue := strings.Split(cookieParts[0], "=")[1] + + p.cookies[i] = http.Cookie{ + Name: cookieName, + Value: cookieValue, + } + } + + return nil +} diff --git a/util/sync/webdavClient/passportAuth_test.go b/util/sync/webdavClient/passportAuth_test.go new file mode 100644 index 000000000..27b8b6f0a --- /dev/null +++ b/util/sync/webdavClient/passportAuth_test.go @@ -0,0 +1,66 @@ +package webdavClient + +import ( + "bytes" + "net/http" + "net/url" + "regexp" + "testing" +) + +// testing the creation is enough as it handles the authorization during init +func TestNewPassportAuth(t *testing.T) { + user := "user" + pass := "password" + p1 := "some,comma,separated,values" + token := "from-PP='token'" + + authHandler := func(h http.Handler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + reg, err := regexp.Compile("Passport1\\.4 sign-in=" + url.QueryEscape(user) + ",pwd=" + url.QueryEscape(pass) + ",OrgVerb=GET,OrgUrl=.*," + p1) + if err != nil { + t.Error(err) + } + if reg.MatchString(r.Header.Get("Authorization")) { + w.Header().Set("Authentication-Info", token) + w.WriteHeader(200) + return + } + } + } + authsrv, _, _ := newAuthSrv(t, authHandler) + defer authsrv.Close() + + dataHandler := func(h http.Handler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + reg, err := regexp.Compile("Passport1\\.4 " + token) + if err != nil { + t.Error(err) + } + if reg.MatchString(r.Header.Get("Authorization")) { + w.Header().Set("Set-Cookie", "Pass=port") + h.ServeHTTP(w, r) + return + } + for _, c := range r.Cookies() { + if c.Name == "Pass" && c.Value == "port" { + h.ServeHTTP(w, r) + return + } + } + w.Header().Set("Www-Authenticate", "Passport1.4 "+p1) + http.Redirect(w, r, authsrv.URL+"/", 302) + } + } + srv, _, _ := newAuthSrv(t, dataHandler) + defer srv.Close() + + cli := NewClient(srv.URL, user, pass) + data, err := cli.Read("/hello.txt") + if err != nil { + t.Errorf("got error=%v; want nil", err) + } + if !bytes.Equal(data, []byte("hello gowebdav\n")) { + t.Logf("got data=%v; want=hello gowebdav", data) + } +} diff --git a/util/sync/webdavClient/requests.go b/util/sync/webdavClient/requests.go new file mode 100644 index 000000000..e9c77b37a --- /dev/null +++ b/util/sync/webdavClient/requests.go @@ -0,0 +1,181 @@ +package webdavClient + +import ( + "io" + "log" + "net/http" + "path" + "strings" +) + +func (c *Client) req(method, path string, body io.Reader, intercept func(*http.Request)) (rs *http.Response, err error) { + var redo bool + var r *http.Request + var uri = PathEscape(Join(c.root, path)) + auth, body := c.auth.NewAuthenticator(body) + defer auth.Close() + + for { // TODO auth.continue() strategy(true|n times|until)? + if r, err = http.NewRequest(method, uri, body); err != nil { + return + } + + for k, vals := range c.headers { + for _, v := range vals { + r.Header.Add(k, v) + } + } + + if err = auth.Authorize(c.c, r, path); err != nil { + return + } + + if intercept != nil { + intercept(r) + } + + if c.interceptor != nil { + c.interceptor(method, r) + } + + if rs, err = c.c.Do(r); err != nil { + return + } + + if redo, err = auth.Verify(c.c, rs, path); err != nil { + rs.Body.Close() + return nil, err + } + if redo { + rs.Body.Close() + if body, err = r.GetBody(); err != nil { + return nil, err + } + continue + } + break + } + + return rs, err +} + +func (c *Client) mkcol(path string) (status int, err error) { + rs, err := c.req("MKCOL", path, nil, nil) + if err != nil { + return + } + defer rs.Body.Close() + + status = rs.StatusCode + if status == 405 { + status = 201 + } + + return +} + +func (c *Client) options(path string) (*http.Response, error) { + return c.req("OPTIONS", path, nil, func(rq *http.Request) { + rq.Header.Add("Depth", "0") + }) +} + +func (c *Client) propfind(path string, self bool, body string, resp interface{}, parse func(resp interface{}) error) error { + rs, err := c.req("PROPFIND", path, strings.NewReader(body), func(rq *http.Request) { + if self { + rq.Header.Add("Depth", "0") + } else { + rq.Header.Add("Depth", "1") + } + rq.Header.Add("Content-Type", "application/xml;charset=UTF-8") + rq.Header.Add("Accept", "application/xml,text/xml") + rq.Header.Add("Accept-Charset", "utf-8") + // TODO add support for 'gzip,deflate;q=0.8,q=0.7' + rq.Header.Add("Accept-Encoding", "") + }) + if err != nil { + return err + } + defer rs.Body.Close() + + if rs.StatusCode != 207 { + return NewPathError("PROPFIND", path, rs.StatusCode) + } + + return parseXML(rs.Body, resp, parse) +} + +func (c *Client) doCopyMove( + method string, + oldpath string, + newpath string, + overwrite bool, +) ( + status int, + r io.ReadCloser, + err error, +) { + rs, err := c.req(method, oldpath, nil, func(rq *http.Request) { + rq.Header.Add("Destination", PathEscape(Join(c.root, newpath))) + if overwrite { + rq.Header.Add("Overwrite", "T") + } else { + rq.Header.Add("Overwrite", "F") + } + }) + if err != nil { + return + } + status = rs.StatusCode + r = rs.Body + return +} + +func (c *Client) copymove(method string, oldpath string, newpath string, overwrite bool) (err error) { + s, data, err := c.doCopyMove(method, oldpath, newpath, overwrite) + if err != nil { + return + } + if data != nil { + defer data.Close() + } + + switch s { + case 201, 204: + return nil + + case 207: + // TODO handle multistat errors, worst case ... + log.Printf("TODO handle %s - %s multistatus result %s\n", method, oldpath, String(data)) + + case 409: + err := c.createParentCollection(newpath) + if err != nil { + return err + } + + return c.copymove(method, oldpath, newpath, overwrite) + } + + return NewPathError(method, oldpath, s) +} + +func (c *Client) put(path string, stream io.Reader) (status int, err error) { + rs, err := c.req("PUT", path, stream, nil) + if err != nil { + return + } + defer rs.Body.Close() + + status = rs.StatusCode + return +} + +func (c *Client) createParentCollection(itemPath string) (err error) { + parentPath := path.Dir(itemPath) + if parentPath == "." || parentPath == "/" { + return nil + } + + return c.MkdirAll(parentPath, 0755) +} diff --git a/util/sync/webdavClient/utils.go b/util/sync/webdavClient/utils.go new file mode 100644 index 000000000..e1d4c678a --- /dev/null +++ b/util/sync/webdavClient/utils.go @@ -0,0 +1,113 @@ +package webdavClient + +import ( + "bytes" + "encoding/xml" + "io" + "net/url" + "strconv" + "strings" + "time" +) + +// PathEscape escapes all segments of a given path +func PathEscape(path string) string { + s := strings.Split(path, "/") + for i, e := range s { + s[i] = url.PathEscape(e) + } + return strings.Join(s, "/") +} + +// FixSlash appends a trailing / to our string +func FixSlash(s string) string { + if !strings.HasSuffix(s, "/") { + s += "/" + } + return s +} + +// FixSlashes appends and prepends a / if they are missing +func FixSlashes(s string) string { + if !strings.HasPrefix(s, "/") { + s = "/" + s + } + + return FixSlash(s) +} + +// Join joins two paths +func Join(path0 string, path1 string) string { + return strings.TrimSuffix(path0, "/") + "/" + strings.TrimPrefix(path1, "/") +} + +// String pulls a string out of our io.Reader +func String(r io.Reader) string { + buf := new(bytes.Buffer) + // TODO - make String return an error as well + _, _ = buf.ReadFrom(r) + return buf.String() +} + +func parseUint(s *string) uint { + if n, e := strconv.ParseUint(*s, 10, 32); e == nil { + return uint(n) + } + return 0 +} + +func parseInt64(s *string) int64 { + if n, e := strconv.ParseInt(*s, 10, 64); e == nil { + return n + } + return 0 +} + +func parseModified(s *string) time.Time { + if t, e := time.Parse(time.RFC1123, *s); e == nil { + return t + } + return time.Unix(0, 0) +} + +func parseXML(data io.Reader, resp interface{}, parse func(resp interface{}) error) error { + decoder := xml.NewDecoder(data) + for t, _ := decoder.Token(); t != nil; t, _ = decoder.Token() { + switch se := t.(type) { + case xml.StartElement: + if se.Name.Local == "response" { + if e := decoder.DecodeElement(resp, &se); e == nil { + if err := parse(resp); err != nil { + return err + } + } + } + } + } + return nil +} + +// limitedReadCloser wraps a io.ReadCloser and limits the number of bytes that can be read from it. +type limitedReadCloser struct { + rc io.ReadCloser + remaining int +} + +func (l *limitedReadCloser) Read(buf []byte) (int, error) { + if l.remaining <= 0 { + return 0, io.EOF + } + + if len(buf) > l.remaining { + buf = buf[0:l.remaining] + } + + n, err := l.rc.Read(buf) + l.remaining -= n + + return n, err +} + +func (l *limitedReadCloser) Close() error { + return l.rc.Close() +} diff --git a/util/sync/webdavClient/utils_test.go b/util/sync/webdavClient/utils_test.go new file mode 100644 index 000000000..74cb25060 --- /dev/null +++ b/util/sync/webdavClient/utils_test.go @@ -0,0 +1,67 @@ +package webdavClient + +import ( + "fmt" + "net/url" + "testing" +) + +func TestJoin(t *testing.T) { + eq(t, "/", "", "") + eq(t, "/", "/", "/") + eq(t, "/foo", "", "/foo") + eq(t, "foo/foo", "foo/", "/foo") + eq(t, "foo/foo", "foo/", "foo") +} + +func eq(t *testing.T, expected string, s0 string, s1 string) { + s := Join(s0, s1) + if s != expected { + t.Error("For", "'"+s0+"','"+s1+"'", "expeted", "'"+expected+"'", "got", "'"+s+"'") + } +} + +func ExamplePathEscape() { + fmt.Println(PathEscape("")) + fmt.Println(PathEscape("/")) + fmt.Println(PathEscape("/web")) + fmt.Println(PathEscape("/web/")) + fmt.Println(PathEscape("/w e b/d a v/s%u&c#k:s/")) + + // Output: + // + // / + // /web + // /web/ + // /w%20e%20b/d%20a%20v/s%25u&c%23k:s/ +} + +func TestEscapeURL(t *testing.T) { + ex := "https://foo.com/w%20e%20b/d%20a%20v/s%25u&c%23k:s/" + u, _ := url.Parse("https://foo.com" + PathEscape("/w e b/d a v/s%u&c#k:s/")) + if ex != u.String() { + t.Error("expected: " + ex + " got: " + u.String()) + } +} + +func TestFixSlashes(t *testing.T) { + expected := "/" + + if got := FixSlashes(""); got != expected { + t.Errorf("expected: %q, got: %q", expected, got) + } + + expected = "/path/" + + if got := FixSlashes("path"); got != expected { + t.Errorf("expected: %q, got: %q", expected, got) + } + + if got := FixSlashes("/path"); got != expected { + t.Errorf("expected: %q, got: %q", expected, got) + } + + if got := FixSlashes("path/"); got != expected { + t.Errorf("expected: %q, got: %q", expected, got) + } +} From d955a77af7f7bedd56af219f21353a931e1bffcc Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 1 Dec 2023 12:36:06 -0500 Subject: [PATCH 14/63] start of 'copy to'; more plumbing to support passing modification times through (#438) --- cmd/zrok/copyTo.go | 43 ++++++++++++++++++++++++++++++++ endpoints/drive/webdav/file.go | 24 +++++++----------- endpoints/drive/webdav/webdav.go | 5 ++++ util/sync/filesystem.go | 2 +- util/sync/webdav.go | 2 ++ util/sync/webdavClient/client.go | 3 +-- 6 files changed, 61 insertions(+), 18 deletions(-) create mode 100644 cmd/zrok/copyTo.go diff --git a/cmd/zrok/copyTo.go b/cmd/zrok/copyTo.go new file mode 100644 index 000000000..f451fb978 --- /dev/null +++ b/cmd/zrok/copyTo.go @@ -0,0 +1,43 @@ +package main + +import ( + "github.com/openziti/zrok/util/sync" + "github.com/spf13/cobra" +) + +func init() { + copyCmd.AddCommand(newCopyToCommand().cmd) +} + +type copyToCommand struct { + cmd *cobra.Command +} + +func newCopyToCommand() *copyToCommand { + cmd := &cobra.Command{ + Use: "to ", + Short: "Copy files to a zrok drive from source", + Args: cobra.ExactArgs(2), + } + command := ©ToCommand{cmd: cmd} + cmd.Run = command.run + return command +} + +func (cmd *copyToCommand) run(_ *cobra.Command, args []string) { + dst, err := sync.NewWebDAVTarget(&sync.WebDAVTargetConfig{ + URL: args[0], + Username: "", + Password: "", + }) + if err != nil { + panic(err) + } + src := sync.NewFilesystemTarget(&sync.FilesystemTargetConfig{ + Root: args[1], + }) + + if err := sync.Synchronize(src, dst); err != nil { + panic(err) + } +} diff --git a/endpoints/drive/webdav/file.go b/endpoints/drive/webdav/file.go index db8f4ecef..437d9db88 100644 --- a/endpoints/drive/webdav/file.go +++ b/endpoints/drive/webdav/file.go @@ -6,10 +6,9 @@ package webdav import ( "context" - "crypto/md5" + "crypto/sha512" "encoding/xml" "fmt" - "github.com/sirupsen/logrus" "io" "io/fs" "net/http" @@ -66,27 +65,24 @@ type webdavFile struct { } func (f *webdavFile) DeadProps() (map[xml.Name]Property, error) { - logrus.Infof("DeadProps(%v)", f.name) var ( xmlName xml.Name property Property properties = make(map[xml.Name]Property) - checksum, err = f.md5() + checksum, err = f.checksum() ) if err == nil { - xmlName.Space = "http://owncloud.org/ns" - xmlName.Local = "checksums" + xmlName.Space = "zrok:" + xmlName.Local = "checksum" property.XMLName = xmlName - property.InnerXML = append(property.InnerXML, ""...) - property.InnerXML = append(property.InnerXML, checksum...) - property.InnerXML = append(property.InnerXML, ""...) + property.InnerXML = []byte(checksum) properties[xmlName] = property } var stat fs.FileInfo stat, err = f.Stat() if err == nil { - xmlName.Space = "DAV:" + xmlName.Space = "zrok:" xmlName.Local = "lastmodified" property.XMLName = xmlName property.InnerXML = strconv.AppendInt(nil, stat.ModTime().Unix(), 10) @@ -102,19 +98,17 @@ func (f *webdavFile) Patch(proppatches []Proppatch) ([]Propstat, error) { return []Propstat{stat}, nil } -func (f *webdavFile) md5() (string, error) { +func (f *webdavFile) checksum() (string, error) { file, err := os.Open(f.name) if err != nil { return "", err } defer file.Close() - hash := md5.New() + hash := sha512.New() if _, err := io.Copy(hash, file); err != nil { return "", err } - xhash := fmt.Sprintf("%x", hash.Sum(nil)) - logrus.Infof("hashed %v = %v", f.name, xhash) - return xhash, nil + return fmt.Sprintf("%x", hash.Sum(nil)), nil } // A Dir implements FileSystem using the native file system restricted to a diff --git a/endpoints/drive/webdav/webdav.go b/endpoints/drive/webdav/webdav.go index add2bcd67..d6e35ef8f 100644 --- a/endpoints/drive/webdav/webdav.go +++ b/endpoints/drive/webdav/webdav.go @@ -8,6 +8,7 @@ package webdav // import "golang.org/x/net/webdav" import ( "errors" "fmt" + "github.com/sirupsen/logrus" "io" "net/http" "net/url" @@ -287,6 +288,10 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, return http.StatusInternalServerError, err } w.Header().Set("ETag", etag) + ts := r.Header.Get("zrok-timestamp") + if ts != "" { + logrus.Infof("zrok-timestamp = %v", ts) + } return http.StatusCreated, nil } diff --git a/util/sync/filesystem.go b/util/sync/filesystem.go index 759236315..3a4522b6b 100644 --- a/util/sync/filesystem.go +++ b/util/sync/filesystem.go @@ -61,7 +61,7 @@ func (t *FilesystemTarget) recurse(path string, d fs.DirEntry, err error) error } func (t *FilesystemTarget) ReadStream(path string) (io.ReadCloser, error) { - return os.Open(path) + return os.Open(filepath.Join(t.cfg.Root, path)) } func (t *FilesystemTarget) WriteStream(path string, stream io.Reader, mode os.FileMode) error { diff --git a/util/sync/webdav.go b/util/sync/webdav.go index 7a2b6ca3a..408796edf 100644 --- a/util/sync/webdav.go +++ b/util/sync/webdav.go @@ -1,6 +1,7 @@ package sync import ( + "fmt" "github.com/openziti/zrok/util/sync/webdavClient" "github.com/pkg/errors" "io" @@ -66,6 +67,7 @@ func (t *WebDAVTarget) ReadStream(path string) (io.ReadCloser, error) { } func (t *WebDAVTarget) WriteStream(path string, stream io.Reader, mode os.FileMode) error { + t.c.SetHeader("zrok-timestamp", fmt.Sprintf("%d", time.Now().UnixNano())) return t.c.WriteStream(path, stream, mode) } diff --git a/util/sync/webdavClient/client.go b/util/sync/webdavClient/client.go index fa806c5a0..0797e5f13 100644 --- a/util/sync/webdavClient/client.go +++ b/util/sync/webdavClient/client.go @@ -47,7 +47,7 @@ func NewAuthClient(uri string, auth Authorizer) *Client { // SetHeader lets us set arbitrary headers for a given client func (c *Client) SetHeader(key, value string) { - c.headers.Add(key, value) + c.headers.Set(key, value) } // SetInterceptor lets us set an arbitrary interceptor for a given client @@ -417,7 +417,6 @@ func (c *Client) Write(path string, data []byte, _ os.FileMode) (err error) { // WriteStream writes a stream func (c *Client) WriteStream(path string, stream io.Reader, _ os.FileMode) (err error) { - err = c.createParentCollection(path) if err != nil { return err From b5210a61c6230d5ab6e4b8368a7e00f724b2991f Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 1 Dec 2023 13:26:54 -0500 Subject: [PATCH 15/63] support updating the lastmodtime property (#438) --- endpoints/drive/webdav/file.go | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/endpoints/drive/webdav/file.go b/endpoints/drive/webdav/file.go index 437d9db88..bae07af1c 100644 --- a/endpoints/drive/webdav/file.go +++ b/endpoints/drive/webdav/file.go @@ -92,9 +92,22 @@ func (f *webdavFile) DeadProps() (map[xml.Name]Property, error) { return properties, nil } -func (f *webdavFile) Patch(proppatches []Proppatch) ([]Propstat, error) { +func (f *webdavFile) Patch(patches []Proppatch) ([]Propstat, error) { var stat Propstat stat.Status = http.StatusOK + for _, patch := range patches { + for _, prop := range patch.Props { + if prop.XMLName.Space == "zrok:" && prop.XMLName.Local == "lastmodified" { + modtimeUnix, err := strconv.ParseInt(string(prop.InnerXML), 10, 64) + if err != nil { + return nil, err + } + if err := f.updateModtime(f.name, time.Unix(modtimeUnix, 0)); err != nil { + return nil, err + } + } + } + } return []Propstat{stat}, nil } @@ -111,6 +124,13 @@ func (f *webdavFile) checksum() (string, error) { return fmt.Sprintf("%x", hash.Sum(nil)), nil } +func (f *webdavFile) updateModtime(path string, modtime time.Time) error { + if err := os.Chtimes(f.name, time.Now(), modtime); err != nil { + return err + } + return nil +} + // A Dir implements FileSystem using the native file system restricted to a // specific directory tree. // From ba3d1b503284bec01d0bd0b0c8b2161b3202c386 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 1 Dec 2023 13:53:36 -0500 Subject: [PATCH 16/63] hacky proppatch support for lastmodified property (#438) --- util/sync/webdav.go | 10 ++++++++++ util/sync/webdavClient/client.go | 12 ++++++------ util/sync/webdavClient/requests.go | 20 ++++++++++++++++++++ 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/util/sync/webdav.go b/util/sync/webdav.go index 408796edf..77a1633a1 100644 --- a/util/sync/webdav.go +++ b/util/sync/webdav.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/openziti/zrok/util/sync/webdavClient" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "io" "os" "path/filepath" @@ -72,5 +73,14 @@ func (t *WebDAVTarget) WriteStream(path string, stream io.Reader, mode os.FileMo } func (t *WebDAVTarget) SetModificationTime(path string, mtime time.Time) error { + modtimeUnix := mtime.Unix() + body := "" + + "" + + fmt.Sprintf("%d", modtimeUnix) + + "" + logrus.Infof("sending '%v'", body) + if err := t.c.Proppatch(path, body, nil, nil); err != nil { + return err + } return nil } diff --git a/util/sync/webdavClient/client.go b/util/sync/webdavClient/client.go index 0797e5f13..fd4d4d51e 100644 --- a/util/sync/webdavClient/client.go +++ b/util/sync/webdavClient/client.go @@ -99,12 +99,12 @@ type props struct { Modified string `xml:"DAV: prop>getlastmodified,omitempty"` } -type response struct { +type Response struct { Href string `xml:"DAV: href"` Props []props `xml:"DAV: propstat"` } -func getProps(r *response, status string) *props { +func getProps(r *Response, status string) *props { for _, prop := range r.Props { if strings.Contains(prop.Status, status) { return &prop @@ -119,7 +119,7 @@ func (c *Client) ReadDir(path string) ([]os.FileInfo, error) { files := make([]os.FileInfo, 0) skipSelf := true parse := func(resp interface{}) error { - r := resp.(*response) + r := resp.(*Response) if skipSelf { skipSelf = false @@ -169,7 +169,7 @@ func (c *Client) ReadDir(path string) ([]os.FileInfo, error) { `, - &response{}, + &Response{}, parse) if err != nil { @@ -184,7 +184,7 @@ func (c *Client) ReadDir(path string) ([]os.FileInfo, error) { func (c *Client) Stat(path string) (os.FileInfo, error) { var f *File parse := func(resp interface{}) error { - r := resp.(*response) + r := resp.(*Response) if p := getProps(r, "200"); p != nil && f == nil { f = new(File) f.name = p.Name @@ -221,7 +221,7 @@ func (c *Client) Stat(path string) (os.FileInfo, error) { `, - &response{}, + &Response{}, parse) if err != nil { diff --git a/util/sync/webdavClient/requests.go b/util/sync/webdavClient/requests.go index e9c77b37a..40b1c90ab 100644 --- a/util/sync/webdavClient/requests.go +++ b/util/sync/webdavClient/requests.go @@ -105,6 +105,26 @@ func (c *Client) propfind(path string, self bool, body string, resp interface{}, return parseXML(rs.Body, resp, parse) } +func (c *Client) Proppatch(path string, body string, resp interface{}, parse func(resp interface{}) error) error { + rs, err := c.req("PROPPATCH", path, strings.NewReader(body), func(rq *http.Request) { + rq.Header.Add("Content-Type", "application/xml;charset=UTF-8") + rq.Header.Add("Accept", "application/xml,text/xml") + rq.Header.Add("Accept-Charset", "utf-8") + // TODO add support for 'gzip,deflate;q=0.8,q=0.7' + rq.Header.Add("Accept-Encoding", "") + }) + if err != nil { + return err + } + defer rs.Body.Close() + + if rs.StatusCode != 207 { + return NewPathError("PROPPATCH", path, rs.StatusCode) + } + + return parseXML(rs.Body, resp, parse) +} + func (c *Client) doCopyMove( method string, oldpath string, From 31dc90afdac170f26824e1b9e1c393fcb78e6bf4 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 1 Dec 2023 13:56:22 -0500 Subject: [PATCH 17/63] remove timestamp header experiment (#438) --- endpoints/drive/webdav/webdav.go | 5 ----- util/sync/webdav.go | 3 --- 2 files changed, 8 deletions(-) diff --git a/endpoints/drive/webdav/webdav.go b/endpoints/drive/webdav/webdav.go index d6e35ef8f..add2bcd67 100644 --- a/endpoints/drive/webdav/webdav.go +++ b/endpoints/drive/webdav/webdav.go @@ -8,7 +8,6 @@ package webdav // import "golang.org/x/net/webdav" import ( "errors" "fmt" - "github.com/sirupsen/logrus" "io" "net/http" "net/url" @@ -288,10 +287,6 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, return http.StatusInternalServerError, err } w.Header().Set("ETag", etag) - ts := r.Header.Get("zrok-timestamp") - if ts != "" { - logrus.Infof("zrok-timestamp = %v", ts) - } return http.StatusCreated, nil } diff --git a/util/sync/webdav.go b/util/sync/webdav.go index 77a1633a1..c173da65a 100644 --- a/util/sync/webdav.go +++ b/util/sync/webdav.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/openziti/zrok/util/sync/webdavClient" "github.com/pkg/errors" - "github.com/sirupsen/logrus" "io" "os" "path/filepath" @@ -68,7 +67,6 @@ func (t *WebDAVTarget) ReadStream(path string) (io.ReadCloser, error) { } func (t *WebDAVTarget) WriteStream(path string, stream io.Reader, mode os.FileMode) error { - t.c.SetHeader("zrok-timestamp", fmt.Sprintf("%d", time.Now().UnixNano())) return t.c.WriteStream(path, stream, mode) } @@ -78,7 +76,6 @@ func (t *WebDAVTarget) SetModificationTime(path string, mtime time.Time) error { "" + fmt.Sprintf("%d", modtimeUnix) + "" - logrus.Infof("sending '%v'", body) if err := t.c.Proppatch(path, body, nil, nil); err != nil { return err } From fa9f77bd1d5fda8c2e8c2d69e7bd36f5e11515e8 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 6 Dec 2023 17:13:45 -0500 Subject: [PATCH 18/63] checksum plumbing (#438) --- endpoints/drive/webdav/file.go | 5 ++++- util/sync/webdavClient/client.go | 8 ++++++-- util/sync/webdavClient/file.go | 5 +++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/endpoints/drive/webdav/file.go b/endpoints/drive/webdav/file.go index bae07af1c..ab1ee410c 100644 --- a/endpoints/drive/webdav/file.go +++ b/endpoints/drive/webdav/file.go @@ -9,6 +9,7 @@ import ( "crypto/sha512" "encoding/xml" "fmt" + "github.com/sirupsen/logrus" "io" "io/fs" "net/http" @@ -121,7 +122,9 @@ func (f *webdavFile) checksum() (string, error) { if _, err := io.Copy(hash, file); err != nil { return "", err } - return fmt.Sprintf("%x", hash.Sum(nil)), nil + sha512 := fmt.Sprintf("%x", hash.Sum(nil)) + logrus.Infof("%v = %v", f.name, sha512) + return sha512, nil } func (f *webdavFile) updateModtime(path string, modtime time.Time) error { diff --git a/util/sync/webdavClient/client.go b/util/sync/webdavClient/client.go index fd4d4d51e..cb21a5240 100644 --- a/util/sync/webdavClient/client.go +++ b/util/sync/webdavClient/client.go @@ -97,6 +97,7 @@ type props struct { ContentType string `xml:"DAV: prop>getcontenttype,omitempty"` ETag string `xml:"DAV: prop>getetag,omitempty"` Modified string `xml:"DAV: prop>getlastmodified,omitempty"` + Checksum string `xml:"zrok: prop>checksum,omitempty"` } type Response struct { @@ -159,7 +160,7 @@ func (c *Client) ReadDir(path string) ([]os.FileInfo, error) { } err := c.propfind(path, false, - ` + ` @@ -167,6 +168,8 @@ func (c *Client) ReadDir(path string) ([]os.FileInfo, error) { + + `, &Response{}, @@ -211,7 +214,7 @@ func (c *Client) Stat(path string) (os.FileInfo, error) { } err := c.propfind(path, true, - ` + ` @@ -219,6 +222,7 @@ func (c *Client) Stat(path string) (os.FileInfo, error) { + `, &Response{}, diff --git a/util/sync/webdavClient/file.go b/util/sync/webdavClient/file.go index 0925be671..ce1033ef1 100644 --- a/util/sync/webdavClient/file.go +++ b/util/sync/webdavClient/file.go @@ -15,6 +15,7 @@ type File struct { modified time.Time etag string isdir bool + checksum string } // Path returns the full path of a file @@ -62,6 +63,10 @@ func (f File) IsDir() bool { return f.isdir } +func (f File) Checksum() string { + return f.checksum +} + // Sys ???? func (f File) Sys() interface{} { return nil From a975b69541756e8ea205cae81ac80df74001a01f Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Thu, 14 Dec 2023 12:58:54 -0500 Subject: [PATCH 19/63] remove 'checksum' support until next phase (#438) --- endpoints/drive/webdav/file.go | 35 ++++------------------------------ 1 file changed, 4 insertions(+), 31 deletions(-) diff --git a/endpoints/drive/webdav/file.go b/endpoints/drive/webdav/file.go index ab1ee410c..32ae6d1c4 100644 --- a/endpoints/drive/webdav/file.go +++ b/endpoints/drive/webdav/file.go @@ -6,10 +6,7 @@ package webdav import ( "context" - "crypto/sha512" "encoding/xml" - "fmt" - "github.com/sirupsen/logrus" "io" "io/fs" "net/http" @@ -67,21 +64,12 @@ type webdavFile struct { func (f *webdavFile) DeadProps() (map[xml.Name]Property, error) { var ( - xmlName xml.Name - property Property - properties = make(map[xml.Name]Property) - checksum, err = f.checksum() + xmlName xml.Name + property Property + properties = make(map[xml.Name]Property) ) - if err == nil { - xmlName.Space = "zrok:" - xmlName.Local = "checksum" - property.XMLName = xmlName - property.InnerXML = []byte(checksum) - properties[xmlName] = property - } - var stat fs.FileInfo - stat, err = f.Stat() + stat, err := f.Stat() if err == nil { xmlName.Space = "zrok:" xmlName.Local = "lastmodified" @@ -112,21 +100,6 @@ func (f *webdavFile) Patch(patches []Proppatch) ([]Propstat, error) { return []Propstat{stat}, nil } -func (f *webdavFile) checksum() (string, error) { - file, err := os.Open(f.name) - if err != nil { - return "", err - } - defer file.Close() - hash := sha512.New() - if _, err := io.Copy(hash, file); err != nil { - return "", err - } - sha512 := fmt.Sprintf("%x", hash.Sum(nil)) - logrus.Infof("%v = %v", f.name, sha512) - return sha512, nil -} - func (f *webdavFile) updateModtime(path string, modtime time.Time) error { if err := os.Chtimes(f.name, time.Now(), modtime); err != nil { return err From cbb69373f44f004c5b1c05d536a76eae1b08c893 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Thu, 14 Dec 2023 14:25:24 -0500 Subject: [PATCH 20/63] updated 'zrok copy' (#438) --- cmd/zrok/copy.go | 118 +++++++++++++++++++++++++++++++ cmd/zrok/copyFrom.go | 48 ------------- cmd/zrok/copyTo.go | 43 ----------- cmd/zrok/main.go | 6 -- util/sync/webdav.go | 10 ++- util/sync/webdavClient/client.go | 33 +++++++++ 6 files changed, 159 insertions(+), 99 deletions(-) create mode 100644 cmd/zrok/copy.go delete mode 100644 cmd/zrok/copyFrom.go delete mode 100644 cmd/zrok/copyTo.go diff --git a/cmd/zrok/copy.go b/cmd/zrok/copy.go new file mode 100644 index 000000000..020574d7b --- /dev/null +++ b/cmd/zrok/copy.go @@ -0,0 +1,118 @@ +package main + +import ( + "fmt" + "github.com/openziti/zrok/environment" + "github.com/openziti/zrok/environment/env_core" + "github.com/openziti/zrok/sdk/golang/sdk" + "github.com/openziti/zrok/tui" + "github.com/openziti/zrok/util/sync" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "net/url" +) + +func init() { + rootCmd.AddCommand(newCopyCommand().cmd) +} + +type copyCommand struct { + cmd *cobra.Command +} + +func newCopyCommand() *copyCommand { + cmd := &cobra.Command{ + Use: "copy []", + Short: "Copy zrok drive contents from to ('file://' and 'zrok://' supported)", + Args: cobra.RangeArgs(1, 2), + } + command := ©Command{cmd: cmd} + cmd.Run = command.run + return command +} + +func (cmd *copyCommand) run(_ *cobra.Command, args []string) { + sourceUrl, err := url.Parse(args[0]) + if err != nil { + tui.Error(fmt.Sprintf("invalid source URL '%v'", args[0]), err) + } + if sourceUrl.Scheme != "zrok" && sourceUrl.Scheme != "file" { + tui.Error("source URL must be 'file://' or 'zrok://", nil) + } + + targetStr := "file://." + if len(args) == 2 { + targetStr = args[1] + } + targetUrl, err := url.Parse(targetStr) + if err != nil { + tui.Error(fmt.Sprintf("invalid target URL '%v'", targetStr), err) + } + if targetUrl.Scheme != "zrok" && targetUrl.Scheme != "file" { + tui.Error("target URL must be 'file://' or 'zrok://", nil) + } + + if sourceUrl.Scheme != "zrok" && targetUrl.Scheme != "zrok" { + tui.Error("either or must be a 'zrok://' URL", nil) + } + if targetUrl.Scheme != "file" && sourceUrl.Scheme != "file" { + tui.Error("either or must be a 'file://' URL", nil) + } + + root, err := environment.LoadRoot() + if err != nil { + tui.Error("error loading root", err) + } + + var access *sdk.Access + if sourceUrl.Scheme == "zrok" { + access, err = sdk.CreateAccess(root, &sdk.AccessRequest{ShareToken: sourceUrl.Host}) + if err != nil { + tui.Error("error creating access", err) + } + } + if targetUrl.Scheme == "zrok" { + access, err = sdk.CreateAccess(root, &sdk.AccessRequest{ShareToken: targetUrl.Host}) + if err != nil { + tui.Error("error creating access", err) + } + } + defer func() { + err := sdk.DeleteAccess(root, access) + if err != nil { + tui.Error("error deleting access", err) + } + }() + + source, err := cmd.createTarget(sourceUrl, root) + if err != nil { + tui.Error("error creating target", err) + } + target, err := cmd.createTarget(targetUrl, root) + if err != nil { + tui.Error("error creating target", err) + } + + if err := sync.Synchronize(source, target); err != nil { + tui.Error("error copying", err) + } + + fmt.Println("copy complete!") +} + +func (cmd *copyCommand) createTarget(t *url.URL, root env_core.Root) (sync.Target, error) { + switch t.Scheme { + case "zrok": + target, err := sync.NewWebDAVTarget(&sync.WebDAVTargetConfig{URL: t, Username: "", Password: "", Root: root}) + if err != nil { + return nil, err + } + return target, nil + + case "file": + return sync.NewFilesystemTarget(&sync.FilesystemTargetConfig{Root: t.Path}), nil + + default: + return nil, errors.Errorf("invalid scheme") + } +} diff --git a/cmd/zrok/copyFrom.go b/cmd/zrok/copyFrom.go deleted file mode 100644 index 8b4e78b35..000000000 --- a/cmd/zrok/copyFrom.go +++ /dev/null @@ -1,48 +0,0 @@ -package main - -import ( - "github.com/openziti/zrok/util/sync" - "github.com/spf13/cobra" -) - -func init() { - copyCmd.AddCommand(newCopyFromCommand().cmd) -} - -type copyFromCommand struct { - cmd *cobra.Command -} - -func newCopyFromCommand() *copyFromCommand { - cmd := &cobra.Command{ - Use: "from []", - Short: "Copy files from zrok drive to destination", - Args: cobra.RangeArgs(1, 2), - } - command := ©FromCommand{cmd: cmd} - cmd.Run = command.run - return command -} - -func (cmd *copyFromCommand) run(_ *cobra.Command, args []string) { - target := "." - if len(args) == 2 { - target = args[1] - } - - dst := sync.NewFilesystemTarget(&sync.FilesystemTargetConfig{ - Root: target, - }) - src, err := sync.NewWebDAVTarget(&sync.WebDAVTargetConfig{ - URL: args[0], - Username: "", - Password: "", - }) - if err != nil { - panic(err) - } - - if err := sync.Synchronize(src, dst); err != nil { - panic(err) - } -} diff --git a/cmd/zrok/copyTo.go b/cmd/zrok/copyTo.go deleted file mode 100644 index f451fb978..000000000 --- a/cmd/zrok/copyTo.go +++ /dev/null @@ -1,43 +0,0 @@ -package main - -import ( - "github.com/openziti/zrok/util/sync" - "github.com/spf13/cobra" -) - -func init() { - copyCmd.AddCommand(newCopyToCommand().cmd) -} - -type copyToCommand struct { - cmd *cobra.Command -} - -func newCopyToCommand() *copyToCommand { - cmd := &cobra.Command{ - Use: "to ", - Short: "Copy files to a zrok drive from source", - Args: cobra.ExactArgs(2), - } - command := ©ToCommand{cmd: cmd} - cmd.Run = command.run - return command -} - -func (cmd *copyToCommand) run(_ *cobra.Command, args []string) { - dst, err := sync.NewWebDAVTarget(&sync.WebDAVTargetConfig{ - URL: args[0], - Username: "", - Password: "", - }) - if err != nil { - panic(err) - } - src := sync.NewFilesystemTarget(&sync.FilesystemTargetConfig{ - Root: args[1], - }) - - if err := sync.Synchronize(src, dst); err != nil { - panic(err) - } -} diff --git a/cmd/zrok/main.go b/cmd/zrok/main.go index 97024f32b..fd8d52f8b 100644 --- a/cmd/zrok/main.go +++ b/cmd/zrok/main.go @@ -26,7 +26,6 @@ func init() { testCmd.AddCommand(loopCmd) rootCmd.AddCommand(adminCmd) rootCmd.AddCommand(configCmd) - rootCmd.AddCommand(copyCmd) rootCmd.AddCommand(shareCmd) rootCmd.AddCommand(testCmd) transport.AddAddressParser(tcp.AddressParser{}) @@ -80,11 +79,6 @@ var configCmd = &cobra.Command{ Short: "Configure your zrok environment", } -var copyCmd = &cobra.Command{ - Use: "copy", - Short: "Copy files to/from zrok drives", -} - var loopCmd = &cobra.Command{ Use: "loopback", Aliases: []string{"loop"}, diff --git a/util/sync/webdav.go b/util/sync/webdav.go index c173da65a..efc458794 100644 --- a/util/sync/webdav.go +++ b/util/sync/webdav.go @@ -2,18 +2,21 @@ package sync import ( "fmt" + "github.com/openziti/zrok/environment/env_core" "github.com/openziti/zrok/util/sync/webdavClient" "github.com/pkg/errors" "io" + "net/url" "os" "path/filepath" "time" ) type WebDAVTargetConfig struct { - URL string + URL *url.URL Username string Password string + Root env_core.Root } type WebDAVTarget struct { @@ -21,7 +24,10 @@ type WebDAVTarget struct { } func NewWebDAVTarget(cfg *WebDAVTargetConfig) (*WebDAVTarget, error) { - c := webdavClient.NewClient(cfg.URL, cfg.Username, cfg.Password) + c, err := webdavClient.NewZrokClient(cfg.URL, cfg.Root, webdavClient.NewAutoAuth(cfg.Username, cfg.Password)) + if err != nil { + return nil, err + } if err := c.Connect(); err != nil { return nil, errors.Wrap(err, "error connecting to webdav target") } diff --git a/util/sync/webdavClient/client.go b/util/sync/webdavClient/client.go index cb21a5240..7d45b30e2 100644 --- a/util/sync/webdavClient/client.go +++ b/util/sync/webdavClient/client.go @@ -2,9 +2,13 @@ package webdavClient import ( "bytes" + "context" "encoding/xml" "fmt" + "github.com/openziti/zrok/environment/env_core" + "github.com/openziti/zrok/sdk/golang/sdk" "io" + "net" "net/http" "net/url" "os" @@ -29,6 +33,35 @@ func NewClient(uri, user, pw string) *Client { return NewAuthClient(uri, NewAutoAuth(user, pw)) } +func NewZrokClient(zrokUrl *url.URL, root env_core.Root, auth Authorizer) (*Client, error) { + conn, err := sdk.NewDialer(zrokUrl.Host, root) + if err != nil { + return nil, err + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.DialContext = func(_ context.Context, _, _ string) (net.Conn, error) { + return conn, nil + } + c := &http.Client{ + Transport: transport, + CheckRedirect: func(rq *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return ErrTooManyRedirects + } + if via[0].Header.Get(XInhibitRedirect) != "" { + return http.ErrUseLastResponse + } + return nil + }, + } + httpUrl, err := url.Parse(zrokUrl.String()) + if err != nil { + return nil, err + } + httpUrl.Scheme = "http" + return &Client{root: FixSlash(httpUrl.String()), headers: make(http.Header), interceptor: nil, c: c, auth: auth}, nil +} + // NewAuthClient creates a new client instance with a custom Authorizer func NewAuthClient(uri string, auth Authorizer) *Client { c := &http.Client{ From 103b60c2c2ed5bded4a5772a905cfd9bf286019d Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 15 Dec 2023 12:10:35 -0500 Subject: [PATCH 21/63] for when the drive source is a file instead of a directory (#438) --- util/sync/filesystem.go | 15 ++++++++++++++- util/sync/model.go | 1 + util/sync/synchronizer.go | 10 +++++----- util/sync/webdav.go | 26 +++++++++++++++++++++++--- 4 files changed, 43 insertions(+), 9 deletions(-) diff --git a/util/sync/filesystem.go b/util/sync/filesystem.go index 3a4522b6b..3ed94ef47 100644 --- a/util/sync/filesystem.go +++ b/util/sync/filesystem.go @@ -27,9 +27,18 @@ func NewFilesystemTarget(cfg *FilesystemTargetConfig) *FilesystemTarget { } func (t *FilesystemTarget) Inventory() ([]*Object, error) { - if _, err := os.Stat(t.cfg.Root); os.IsNotExist(err) { + fi, err := os.Stat(t.cfg.Root) + if os.IsNotExist(err) { return nil, nil } + if err != nil { + return nil, err + } + + if !fi.IsDir() { + return []*Object{{Path: t.cfg.Root, Size: fi.Size(), Modified: fi.ModTime()}}, nil + } + t.tree = nil if err := fs.WalkDir(t.root, ".", t.recurse); err != nil { return nil, err @@ -37,6 +46,10 @@ func (t *FilesystemTarget) Inventory() ([]*Object, error) { return t.tree, nil } +func (t *FilesystemTarget) IsDir() bool { + return true +} + func (t *FilesystemTarget) recurse(path string, d fs.DirEntry, err error) error { if err != nil { return err diff --git a/util/sync/model.go b/util/sync/model.go index 8fca5ca07..b1a6b0435 100644 --- a/util/sync/model.go +++ b/util/sync/model.go @@ -15,6 +15,7 @@ type Object struct { type Target interface { Inventory() ([]*Object, error) + IsDir() bool ReadStream(path string) (io.ReadCloser, error) WriteStream(path string, stream io.Reader, mode os.FileMode) error SetModificationTime(path string, mtime time.Time) error diff --git a/util/sync/synchronizer.go b/util/sync/synchronizer.go index 36ba78fc4..082325fae 100644 --- a/util/sync/synchronizer.go +++ b/util/sync/synchronizer.go @@ -33,18 +33,18 @@ func Synchronize(src, dst Target) error { } } - for _, target := range copyList { - ss, err := src.ReadStream(target.Path) + for _, copyPath := range copyList { + ss, err := src.ReadStream(copyPath.Path) if err != nil { return err } - if err := dst.WriteStream(target.Path, ss, os.ModePerm); err != nil { + if err := dst.WriteStream(copyPath.Path, ss, os.ModePerm); err != nil { return err } - if err := dst.SetModificationTime(target.Path, target.Modified); err != nil { + if err := dst.SetModificationTime(copyPath.Path, copyPath.Modified); err != nil { return err } - logrus.Infof("=> %v", target.Path) + logrus.Infof("=> %v", copyPath.Path) } return nil diff --git a/util/sync/webdav.go b/util/sync/webdav.go index efc458794..61e07e47c 100644 --- a/util/sync/webdav.go +++ b/util/sync/webdav.go @@ -20,7 +20,9 @@ type WebDAVTargetConfig struct { } type WebDAVTarget struct { - c *webdavClient.Client + cfg *WebDAVTargetConfig + c *webdavClient.Client + isDir bool } func NewWebDAVTarget(cfg *WebDAVTargetConfig) (*WebDAVTarget, error) { @@ -31,10 +33,21 @@ func NewWebDAVTarget(cfg *WebDAVTargetConfig) (*WebDAVTarget, error) { if err := c.Connect(); err != nil { return nil, errors.Wrap(err, "error connecting to webdav target") } - return &WebDAVTarget{c: c}, nil + return &WebDAVTarget{cfg: cfg, c: c}, nil } func (t *WebDAVTarget) Inventory() ([]*Object, error) { + fi, err := t.c.Stat("") + if !fi.IsDir() { + t.isDir = false + return []*Object{{ + Path: fi.Name(), + Size: fi.Size(), + Modified: fi.ModTime(), + }}, nil + } + + t.isDir = true tree, err := t.recurse("", nil) if err != nil { return nil, err @@ -42,6 +55,10 @@ func (t *WebDAVTarget) Inventory() ([]*Object, error) { return tree, nil } +func (t *WebDAVTarget) IsDir() bool { + return t.isDir +} + func (t *WebDAVTarget) recurse(path string, tree []*Object) ([]*Object, error) { files, err := t.c.ReadDir(path) if err != nil { @@ -69,7 +86,10 @@ func (t *WebDAVTarget) recurse(path string, tree []*Object) ([]*Object, error) { } func (t *WebDAVTarget) ReadStream(path string) (io.ReadCloser, error) { - return t.c.ReadStream(path) + if t.isDir { + return t.c.ReadStream(path) + } + return t.c.ReadStream("") } func (t *WebDAVTarget) WriteStream(path string, stream io.Reader, mode os.FileMode) error { From 7e1a42e8a1ba2c507a5600a125e76329b0a691de Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 3 Jan 2024 14:32:45 -0500 Subject: [PATCH 22/63] - gowebdav --- go.mod | 1 - go.sum | 2 -- 2 files changed, 3 deletions(-) diff --git a/go.mod b/go.mod index 0e1c0123e..17b3997ce 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,6 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.8.4 - github.com/studio-b12/gowebdav v0.9.0 github.com/wneessen/go-mail v0.2.7 github.com/zitadel/oidc/v2 v2.7.0 go.uber.org/zap v1.25.0 diff --git a/go.sum b/go.sum index 78df916a4..ab0cb3edb 100644 --- a/go.sum +++ b/go.sum @@ -943,8 +943,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/studio-b12/gowebdav v0.9.0 h1:1j1sc9gQnNxbXXM4M/CebPOX4aXYtr7MojAVcN4dHjU= -github.com/studio-b12/gowebdav v0.9.0/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tailscale/tscert v0.0.0-20230806124524-28a91b69a046 h1:8rUlviSVOEe7TMk7W0gIPrW8MqEzYfZHpsNWSf8s2vg= github.com/tailscale/tscert v0.0.0-20230806124524-28a91b69a046/go.mod h1:kNGUQ3VESx3VZwRwA9MSCUegIl6+saPL8Noq82ozCaU= From 0c39cd875283a13f02378109e123bbb2061db1ae Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 3 Jan 2024 16:58:32 -0500 Subject: [PATCH 23/63] snapshot; starting 'driveClient' --- util/sync/driveClient/client.go | 15 +++++++++++++++ util/sync/webdav.go | 3 ++- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 util/sync/driveClient/client.go diff --git a/util/sync/driveClient/client.go b/util/sync/driveClient/client.go new file mode 100644 index 000000000..9a8d0f776 --- /dev/null +++ b/util/sync/driveClient/client.go @@ -0,0 +1,15 @@ +package driveClient + +import "net/http" + +type Client struct { + client *http.Client +} + +func NewHttpClient(uri string) *Client { + return &Client{&http.Client{}} +} + +func (c *Client) Connect() error { + return nil +} diff --git a/util/sync/webdav.go b/util/sync/webdav.go index 61e07e47c..c349d38bc 100644 --- a/util/sync/webdav.go +++ b/util/sync/webdav.go @@ -38,7 +38,8 @@ func NewWebDAVTarget(cfg *WebDAVTargetConfig) (*WebDAVTarget, error) { func (t *WebDAVTarget) Inventory() ([]*Object, error) { fi, err := t.c.Stat("") - if !fi.IsDir() { + + if fi != nil && !fi.IsDir() { t.isDir = false return []*Object{{ Path: fi.Name(), From ef0ac1e140a7bf0475de133c11961898f02f459d Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 3 Jan 2024 17:05:18 -0500 Subject: [PATCH 24/63] snapshot --- util/sync/driveClient/client.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/util/sync/driveClient/client.go b/util/sync/driveClient/client.go index 9a8d0f776..9de246548 100644 --- a/util/sync/driveClient/client.go +++ b/util/sync/driveClient/client.go @@ -6,10 +6,27 @@ type Client struct { client *http.Client } -func NewHttpClient(uri string) *Client { +func NewHttpClient() *Client { return &Client{&http.Client{}} } func (c *Client) Connect() error { return nil } + +func (c *Client) options(uri string) (*http.Response, error) { + return c.request("OPTIONS", uri) +} + +func (c *Client) request(method, uri string) (resp *http.Response, err error) { + req, err := http.NewRequest(method, uri, nil) + if err != nil { + return nil, err + } + + if resp, err = c.client.Do(req); err != nil { + return resp, err + } + + return resp, err +} From f46e3db4de7f0cf3ae724ceb10f73e1a435226f2 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 5 Jan 2024 14:41:54 -0500 Subject: [PATCH 25/63] initial 'driveClient' based on 'github.com/emersion/go-webdav' (#511) --- cmd/zrok/davtest.go | 42 ++ util/sync/driveClient/client.go | 262 +++++++++++- util/sync/driveClient/internal/client.go | 256 ++++++++++++ util/sync/driveClient/internal/elements.go | 452 +++++++++++++++++++++ util/sync/driveClient/internal/internal.go | 108 +++++ util/sync/driveClient/internal/xml.go | 175 ++++++++ util/sync/driveClient/model.go | 119 ++++++ 7 files changed, 1401 insertions(+), 13 deletions(-) create mode 100644 cmd/zrok/davtest.go create mode 100644 util/sync/driveClient/internal/client.go create mode 100644 util/sync/driveClient/internal/elements.go create mode 100644 util/sync/driveClient/internal/internal.go create mode 100644 util/sync/driveClient/internal/xml.go create mode 100644 util/sync/driveClient/model.go diff --git a/cmd/zrok/davtest.go b/cmd/zrok/davtest.go new file mode 100644 index 000000000..78e919ba2 --- /dev/null +++ b/cmd/zrok/davtest.go @@ -0,0 +1,42 @@ +package main + +import ( + "context" + "github.com/openziti/zrok/util/sync/driveClient" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "net/http" +) + +func init() { + rootCmd.AddCommand(newDavtestCommand().cmd) +} + +type davtestCommand struct { + cmd *cobra.Command +} + +func newDavtestCommand() *davtestCommand { + cmd := &cobra.Command{ + Use: "davtest", + Short: "WebDAV testing wrapper", + Args: cobra.ExactArgs(1), + } + command := &davtestCommand{cmd: cmd} + cmd.Run = command.run + return command +} + +func (cmd *davtestCommand) run(_ *cobra.Command, args []string) { + client, err := driveClient.NewClient(http.DefaultClient, args[0]) + if err != nil { + panic(err) + } + fis, err := client.Readdir(context.Background(), "/", true) + if err != nil { + panic(err) + } + for _, fi := range fis { + logrus.Infof("=> %s", fi.Path) + } +} diff --git a/util/sync/driveClient/client.go b/util/sync/driveClient/client.go index 9de246548..77cd22158 100644 --- a/util/sync/driveClient/client.go +++ b/util/sync/driveClient/client.go @@ -1,32 +1,268 @@ package driveClient -import "net/http" +import ( + "context" + "fmt" + "github.com/openziti/zrok/util/sync/driveClient/internal" + "io" + "net/http" + "time" +) +// HTTPClient performs HTTP requests. It's implemented by *http.Client. +type HTTPClient interface { + Do(req *http.Request) (*http.Response, error) +} + +type basicAuthHTTPClient struct { + c HTTPClient + username, password string +} + +func (c *basicAuthHTTPClient) Do(req *http.Request) (*http.Response, error) { + req.SetBasicAuth(c.username, c.password) + return c.c.Do(req) +} + +// HTTPClientWithBasicAuth returns an HTTP client that adds basic +// authentication to all outgoing requests. If c is nil, http.DefaultClient is +// used. +func HTTPClientWithBasicAuth(c HTTPClient, username, password string) HTTPClient { + if c == nil { + c = http.DefaultClient + } + return &basicAuthHTTPClient{c, username, password} +} + +// Client provides access to a remote WebDAV filesystem. type Client struct { - client *http.Client + ic *internal.Client } -func NewHttpClient() *Client { - return &Client{&http.Client{}} +func NewClient(c HTTPClient, endpoint string) (*Client, error) { + ic, err := internal.NewClient(c, endpoint) + if err != nil { + return nil, err + } + return &Client{ic}, nil } -func (c *Client) Connect() error { - return nil +func (c *Client) FindCurrentUserPrincipal(ctx context.Context) (string, error) { + propfind := internal.NewPropNamePropFind(internal.CurrentUserPrincipalName) + + // TODO: consider retrying on the root URI "/" if this fails, as suggested + // by the RFC? + resp, err := c.ic.PropFindFlat(ctx, "", propfind) + if err != nil { + return "", err + } + + var prop internal.CurrentUserPrincipal + if err := resp.DecodeProp(&prop); err != nil { + return "", err + } + if prop.Unauthenticated != nil { + return "", fmt.Errorf("webdav: unauthenticated") + } + + return prop.Href.Path, nil +} + +var fileInfoPropFind = internal.NewPropNamePropFind( + internal.ResourceTypeName, + internal.GetContentLengthName, + internal.GetLastModifiedName, + internal.GetContentTypeName, + internal.GetETagName, +) + +func fileInfoFromResponse(resp *internal.Response) (*FileInfo, error) { + path, err := resp.Path() + if err != nil { + return nil, err + } + + fi := &FileInfo{Path: path} + + var resType internal.ResourceType + if err := resp.DecodeProp(&resType); err != nil { + return nil, err + } + + if resType.Is(internal.CollectionName) { + fi.IsDir = true + } else { + var getLen internal.GetContentLength + if err := resp.DecodeProp(&getLen); err != nil { + return nil, err + } + + var getType internal.GetContentType + if err := resp.DecodeProp(&getType); err != nil && !internal.IsNotFound(err) { + return nil, err + } + + var getETag internal.GetETag + if err := resp.DecodeProp(&getETag); err != nil && !internal.IsNotFound(err) { + return nil, err + } + + fi.Size = getLen.Length + fi.MIMEType = getType.Type + fi.ETag = string(getETag.ETag) + } + + var getMod internal.GetLastModified + if err := resp.DecodeProp(&getMod); err != nil && !internal.IsNotFound(err) { + return nil, err + } + fi.ModTime = time.Time(getMod.LastModified) + + return fi, nil } -func (c *Client) options(uri string) (*http.Response, error) { - return c.request("OPTIONS", uri) +func (c *Client) Stat(ctx context.Context, name string) (*FileInfo, error) { + resp, err := c.ic.PropFindFlat(ctx, name, fileInfoPropFind) + if err != nil { + return nil, err + } + return fileInfoFromResponse(resp) } -func (c *Client) request(method, uri string) (resp *http.Response, err error) { - req, err := http.NewRequest(method, uri, nil) +func (c *Client) Open(ctx context.Context, name string) (io.ReadCloser, error) { + req, err := c.ic.NewRequest(http.MethodGet, name, nil) + if err != nil { + return nil, err + } + + resp, err := c.ic.Do(req.WithContext(ctx)) if err != nil { return nil, err } - if resp, err = c.client.Do(req); err != nil { - return resp, err + return resp.Body, nil +} + +func (c *Client) Readdir(ctx context.Context, name string, recursive bool) ([]FileInfo, error) { + depth := internal.DepthOne + if recursive { + depth = internal.DepthInfinity + } + + ms, err := c.ic.PropFind(ctx, name, depth, fileInfoPropFind) + if err != nil { + return nil, err } - return resp, err + l := make([]FileInfo, 0, len(ms.Responses)) + for _, resp := range ms.Responses { + fi, err := fileInfoFromResponse(&resp) + if err != nil { + return l, err + } + l = append(l, *fi) + } + + return l, nil +} + +type fileWriter struct { + pw *io.PipeWriter + done <-chan error +} + +func (fw *fileWriter) Write(b []byte) (int, error) { + return fw.pw.Write(b) +} + +func (fw *fileWriter) Close() error { + if err := fw.pw.Close(); err != nil { + return err + } + return <-fw.done +} + +func (c *Client) Create(ctx context.Context, name string) (io.WriteCloser, error) { + pr, pw := io.Pipe() + + req, err := c.ic.NewRequest(http.MethodPut, name, pr) + if err != nil { + pw.Close() + return nil, err + } + + done := make(chan error, 1) + go func() { + resp, err := c.ic.Do(req.WithContext(ctx)) + if err != nil { + done <- err + return + } + resp.Body.Close() + done <- nil + }() + + return &fileWriter{pw, done}, nil +} + +func (c *Client) RemoveAll(ctx context.Context, name string) error { + req, err := c.ic.NewRequest(http.MethodDelete, name, nil) + if err != nil { + return err + } + + resp, err := c.ic.Do(req.WithContext(ctx)) + if err != nil { + return err + } + resp.Body.Close() + return nil +} + +func (c *Client) Mkdir(ctx context.Context, name string) error { + req, err := c.ic.NewRequest("MKCOL", name, nil) + if err != nil { + return err + } + + resp, err := c.ic.Do(req.WithContext(ctx)) + if err != nil { + return err + } + resp.Body.Close() + return nil +} + +func (c *Client) CopyAll(ctx context.Context, name, dest string, overwrite bool) error { + req, err := c.ic.NewRequest("COPY", name, nil) + if err != nil { + return err + } + + req.Header.Set("Destination", c.ic.ResolveHref(dest).String()) + req.Header.Set("Overwrite", internal.FormatOverwrite(overwrite)) + + resp, err := c.ic.Do(req.WithContext(ctx)) + if err != nil { + return err + } + resp.Body.Close() + return nil +} + +func (c *Client) MoveAll(ctx context.Context, name, dest string, overwrite bool) error { + req, err := c.ic.NewRequest("MOVE", name, nil) + if err != nil { + return err + } + + req.Header.Set("Destination", c.ic.ResolveHref(dest).String()) + req.Header.Set("Overwrite", internal.FormatOverwrite(overwrite)) + + resp, err := c.ic.Do(req.WithContext(ctx)) + if err != nil { + return err + } + resp.Body.Close() + return nil } diff --git a/util/sync/driveClient/internal/client.go b/util/sync/driveClient/internal/client.go new file mode 100644 index 000000000..718f436a4 --- /dev/null +++ b/util/sync/driveClient/internal/client.go @@ -0,0 +1,256 @@ +package internal + +import ( + "bytes" + "context" + "encoding/xml" + "fmt" + "io" + "mime" + "net" + "net/http" + "net/url" + "path" + "strings" + "unicode" +) + +// DiscoverContextURL performs a DNS-based CardDAV/CalDAV service discovery as +// described in RFC 6352 section 11. It returns the URL to the CardDAV server. +func DiscoverContextURL(ctx context.Context, service, domain string) (string, error) { + var resolver net.Resolver + + // Only lookup TLS records, plaintext connections are insecure + _, addrs, err := resolver.LookupSRV(ctx, service+"s", "tcp", domain) + if dnsErr, ok := err.(*net.DNSError); ok { + if dnsErr.IsTemporary { + return "", err + } + } else if err != nil { + return "", err + } + + if len(addrs) == 0 { + return "", fmt.Errorf("webdav: domain doesn't have an SRV record") + } + addr := addrs[0] + + target := strings.TrimSuffix(addr.Target, ".") + if target == "" { + return "", fmt.Errorf("webdav: empty target in SRV record") + } + + // TODO: perform a TXT lookup, check for a "path" key in the response + u := url.URL{Scheme: "https"} + if addr.Port == 443 { + u.Host = target + } else { + u.Host = fmt.Sprintf("%v:%v", target, addr.Port) + } + u.Path = "/.well-known/" + service + return u.String(), nil +} + +// HTTPClient performs HTTP requests. It's implemented by *http.Client. +type HTTPClient interface { + Do(req *http.Request) (*http.Response, error) +} + +type Client struct { + http HTTPClient + endpoint *url.URL +} + +func NewClient(c HTTPClient, endpoint string) (*Client, error) { + if c == nil { + c = http.DefaultClient + } + + u, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + if u.Path == "" { + // This is important to avoid issues with path.Join + u.Path = "/" + } + return &Client{http: c, endpoint: u}, nil +} + +func (c *Client) ResolveHref(p string) *url.URL { + if !strings.HasPrefix(p, "/") { + p = path.Join(c.endpoint.Path, p) + } + return &url.URL{ + Scheme: c.endpoint.Scheme, + User: c.endpoint.User, + Host: c.endpoint.Host, + Path: p, + } +} + +func (c *Client) NewRequest(method string, path string, body io.Reader) (*http.Request, error) { + return http.NewRequest(method, c.ResolveHref(path).String(), body) +} + +func (c *Client) NewXMLRequest(method string, path string, v interface{}) (*http.Request, error) { + var buf bytes.Buffer + buf.WriteString(xml.Header) + if err := xml.NewEncoder(&buf).Encode(v); err != nil { + return nil, err + } + + req, err := c.NewRequest(method, path, &buf) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", "text/xml; charset=\"utf-8\"") + + return req, nil +} + +func (c *Client) Do(req *http.Request) (*http.Response, error) { + resp, err := c.http.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode/100 != 2 { + defer resp.Body.Close() + + contentType := resp.Header.Get("Content-Type") + if contentType == "" { + contentType = "text/plain" + } + + var wrappedErr error + t, _, _ := mime.ParseMediaType(contentType) + if t == "application/xml" || t == "text/xml" { + var davErr Error + if err := xml.NewDecoder(resp.Body).Decode(&davErr); err != nil { + wrappedErr = err + } else { + wrappedErr = &davErr + } + } else if strings.HasPrefix(t, "text/") { + lr := io.LimitedReader{R: resp.Body, N: 1024} + var buf bytes.Buffer + io.Copy(&buf, &lr) + resp.Body.Close() + if s := strings.TrimSpace(buf.String()); s != "" { + if lr.N == 0 { + s += " […]" + } + wrappedErr = fmt.Errorf("%v", s) + } + } + return nil, &HTTPError{Code: resp.StatusCode, Err: wrappedErr} + } + return resp, nil +} + +func (c *Client) DoMultiStatus(req *http.Request) (*MultiStatus, error) { + resp, err := c.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusMultiStatus { + return nil, fmt.Errorf("HTTP multi-status request failed: %v", resp.Status) + } + + // TODO: the response can be quite large, support streaming Response elements + var ms MultiStatus + if err := xml.NewDecoder(resp.Body).Decode(&ms); err != nil { + return nil, err + } + + return &ms, nil +} + +func (c *Client) PropFind(ctx context.Context, path string, depth Depth, propfind *PropFind) (*MultiStatus, error) { + req, err := c.NewXMLRequest("PROPFIND", path, propfind) + if err != nil { + return nil, err + } + + req.Header.Add("Depth", depth.String()) + + return c.DoMultiStatus(req.WithContext(ctx)) +} + +// PropfindFlat performs a PROPFIND request with a zero depth. +func (c *Client) PropFindFlat(ctx context.Context, path string, propfind *PropFind) (*Response, error) { + ms, err := c.PropFind(ctx, path, DepthZero, propfind) + if err != nil { + return nil, err + } + + // If the client followed a redirect, the Href might be different from the request path + if len(ms.Responses) != 1 { + return nil, fmt.Errorf("PROPFIND with Depth: 0 returned %d responses", len(ms.Responses)) + } + return &ms.Responses[0], nil +} + +func parseCommaSeparatedSet(values []string, upper bool) map[string]bool { + m := make(map[string]bool) + for _, v := range values { + fields := strings.FieldsFunc(v, func(r rune) bool { + return unicode.IsSpace(r) || r == ',' + }) + for _, f := range fields { + if upper { + f = strings.ToUpper(f) + } else { + f = strings.ToLower(f) + } + m[f] = true + } + } + return m +} + +func (c *Client) Options(ctx context.Context, path string) (classes map[string]bool, methods map[string]bool, err error) { + req, err := c.NewRequest(http.MethodOptions, path, nil) + if err != nil { + return nil, nil, err + } + + resp, err := c.Do(req.WithContext(ctx)) + if err != nil { + return nil, nil, err + } + resp.Body.Close() + + classes = parseCommaSeparatedSet(resp.Header["Dav"], false) + if !classes["1"] { + return nil, nil, fmt.Errorf("webdav: server doesn't support DAV class 1") + } + + methods = parseCommaSeparatedSet(resp.Header["Allow"], true) + return classes, methods, nil +} + +// SyncCollection perform a `sync-collection` REPORT operation on a resource +func (c *Client) SyncCollection(ctx context.Context, path, syncToken string, level Depth, limit *Limit, prop *Prop) (*MultiStatus, error) { + q := SyncCollectionQuery{ + SyncToken: syncToken, + SyncLevel: level.String(), + Limit: limit, + Prop: prop, + } + + req, err := c.NewXMLRequest("REPORT", path, &q) + if err != nil { + return nil, err + } + + ms, err := c.DoMultiStatus(req.WithContext(ctx)) + if err != nil { + return nil, err + } + + return ms, nil +} diff --git a/util/sync/driveClient/internal/elements.go b/util/sync/driveClient/internal/elements.go new file mode 100644 index 000000000..db7d9603c --- /dev/null +++ b/util/sync/driveClient/internal/elements.go @@ -0,0 +1,452 @@ +package internal + +import ( + "encoding/xml" + "errors" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +const Namespace = "DAV:" + +var ( + ResourceTypeName = xml.Name{Namespace, "resourcetype"} + DisplayNameName = xml.Name{Namespace, "displayname"} + GetContentLengthName = xml.Name{Namespace, "getcontentlength"} + GetContentTypeName = xml.Name{Namespace, "getcontenttype"} + GetLastModifiedName = xml.Name{Namespace, "getlastmodified"} + GetETagName = xml.Name{Namespace, "getetag"} + + CurrentUserPrincipalName = xml.Name{Namespace, "current-user-principal"} +) + +type Status struct { + Code int + Text string +} + +func (s *Status) MarshalText() ([]byte, error) { + text := s.Text + if text == "" { + text = http.StatusText(s.Code) + } + return []byte(fmt.Sprintf("HTTP/1.1 %v %v", s.Code, text)), nil +} + +func (s *Status) UnmarshalText(b []byte) error { + if len(b) == 0 { + return nil + } + + parts := strings.SplitN(string(b), " ", 3) + if len(parts) != 3 { + return fmt.Errorf("webdav: invalid HTTP status %q: expected 3 fields", s) + } + code, err := strconv.Atoi(parts[1]) + if err != nil { + return fmt.Errorf("webdav: invalid HTTP status %q: failed to parse code: %v", s, err) + } + + s.Code = code + s.Text = parts[2] + return nil +} + +func (s *Status) Err() error { + if s == nil { + return nil + } + + // TODO: handle 2xx, 3xx + if s.Code != http.StatusOK { + return &HTTPError{Code: s.Code} + } + return nil +} + +type Href url.URL + +func (h *Href) String() string { + u := (*url.URL)(h) + return u.String() +} + +func (h *Href) MarshalText() ([]byte, error) { + return []byte(h.String()), nil +} + +func (h *Href) UnmarshalText(b []byte) error { + u, err := url.Parse(string(b)) + if err != nil { + return err + } + *h = Href(*u) + return nil +} + +// https://tools.ietf.org/html/rfc4918#section-14.16 +type MultiStatus struct { + XMLName xml.Name `xml:"DAV: multistatus"` + Responses []Response `xml:"response"` + ResponseDescription string `xml:"responsedescription,omitempty"` + SyncToken string `xml:"sync-token,omitempty"` +} + +func NewMultiStatus(resps ...Response) *MultiStatus { + return &MultiStatus{Responses: resps} +} + +// https://tools.ietf.org/html/rfc4918#section-14.24 +type Response struct { + XMLName xml.Name `xml:"DAV: response"` + Hrefs []Href `xml:"href"` + PropStats []PropStat `xml:"propstat,omitempty"` + ResponseDescription string `xml:"responsedescription,omitempty"` + Status *Status `xml:"status,omitempty"` + Error *Error `xml:"error,omitempty"` + Location *Location `xml:"location,omitempty"` +} + +func NewOKResponse(path string) *Response { + href := Href{Path: path} + return &Response{ + Hrefs: []Href{href}, + Status: &Status{Code: http.StatusOK}, + } +} + +func NewErrorResponse(path string, err error) *Response { + code := http.StatusInternalServerError + var httpErr *HTTPError + if errors.As(err, &httpErr) { + code = httpErr.Code + } + + var errElt *Error + errors.As(err, &errElt) + + href := Href{Path: path} + return &Response{ + Hrefs: []Href{href}, + Status: &Status{Code: code}, + ResponseDescription: err.Error(), + Error: errElt, + } +} + +func (resp *Response) Err() error { + if resp.Status == nil || resp.Status.Code/100 == 2 { + return nil + } + + var err error + if resp.Error != nil { + err = resp.Error + } + if resp.ResponseDescription != "" { + if err != nil { + err = fmt.Errorf("%v (%w)", resp.ResponseDescription, err) + } else { + err = fmt.Errorf("%v", resp.ResponseDescription) + } + } + + return &HTTPError{ + Code: resp.Status.Code, + Err: err, + } +} + +func (resp *Response) Path() (string, error) { + err := resp.Err() + var path string + if len(resp.Hrefs) == 1 { + path = resp.Hrefs[0].Path + } else if err == nil { + err = fmt.Errorf("webdav: malformed response: expected exactly one href element, got %v", len(resp.Hrefs)) + } + return path, err +} + +func (resp *Response) DecodeProp(values ...interface{}) error { + for _, v := range values { + // TODO wrap errors with more context (XML name) + name, err := valueXMLName(v) + if err != nil { + return err + } + if err := resp.Err(); err != nil { + return newPropError(name, err) + } + for _, propstat := range resp.PropStats { + raw := propstat.Prop.Get(name) + if raw == nil { + continue + } + if err := propstat.Status.Err(); err != nil { + return newPropError(name, err) + } + if err := raw.Decode(v); err != nil { + return newPropError(name, err) + } + return nil + } + return newPropError(name, &HTTPError{ + Code: http.StatusNotFound, + Err: fmt.Errorf("missing property"), + }) + } + + return nil +} + +func newPropError(name xml.Name, err error) error { + return fmt.Errorf("property <%v %v>: %w", name.Space, name.Local, err) +} + +func (resp *Response) EncodeProp(code int, v interface{}) error { + raw, err := EncodeRawXMLElement(v) + if err != nil { + return err + } + + for i := range resp.PropStats { + propstat := &resp.PropStats[i] + if propstat.Status.Code == code { + propstat.Prop.Raw = append(propstat.Prop.Raw, *raw) + return nil + } + } + + resp.PropStats = append(resp.PropStats, PropStat{ + Status: Status{Code: code}, + Prop: Prop{Raw: []RawXMLValue{*raw}}, + }) + return nil +} + +// https://tools.ietf.org/html/rfc4918#section-14.9 +type Location struct { + XMLName xml.Name `xml:"DAV: location"` + Href Href `xml:"href"` +} + +// https://tools.ietf.org/html/rfc4918#section-14.22 +type PropStat struct { + XMLName xml.Name `xml:"DAV: propstat"` + Prop Prop `xml:"prop"` + Status Status `xml:"status"` + ResponseDescription string `xml:"responsedescription,omitempty"` + Error *Error `xml:"error,omitempty"` +} + +// https://tools.ietf.org/html/rfc4918#section-14.18 +type Prop struct { + XMLName xml.Name `xml:"DAV: prop"` + Raw []RawXMLValue `xml:",any"` +} + +func EncodeProp(values ...interface{}) (*Prop, error) { + l := make([]RawXMLValue, len(values)) + for i, v := range values { + raw, err := EncodeRawXMLElement(v) + if err != nil { + return nil, err + } + l[i] = *raw + } + return &Prop{Raw: l}, nil +} + +func (p *Prop) Get(name xml.Name) *RawXMLValue { + for i := range p.Raw { + raw := &p.Raw[i] + if n, ok := raw.XMLName(); ok && name == n { + return raw + } + } + return nil +} + +func (p *Prop) Decode(v interface{}) error { + name, err := valueXMLName(v) + if err != nil { + return err + } + + raw := p.Get(name) + if raw == nil { + return HTTPErrorf(http.StatusNotFound, "missing property %s", name) + } + + return raw.Decode(v) +} + +// https://tools.ietf.org/html/rfc4918#section-14.20 +type PropFind struct { + XMLName xml.Name `xml:"DAV: propfind"` + Prop *Prop `xml:"prop,omitempty"` + AllProp *struct{} `xml:"allprop,omitempty"` + Include *Include `xml:"include,omitempty"` + PropName *struct{} `xml:"propname,omitempty"` +} + +func xmlNamesToRaw(names []xml.Name) []RawXMLValue { + l := make([]RawXMLValue, len(names)) + for i, name := range names { + l[i] = *NewRawXMLElement(name, nil, nil) + } + return l +} + +func NewPropNamePropFind(names ...xml.Name) *PropFind { + return &PropFind{Prop: &Prop{Raw: xmlNamesToRaw(names)}} +} + +// https://tools.ietf.org/html/rfc4918#section-14.8 +type Include struct { + XMLName xml.Name `xml:"DAV: include"` + Raw []RawXMLValue `xml:",any"` +} + +// https://tools.ietf.org/html/rfc4918#section-15.9 +type ResourceType struct { + XMLName xml.Name `xml:"DAV: resourcetype"` + Raw []RawXMLValue `xml:",any"` +} + +func NewResourceType(names ...xml.Name) *ResourceType { + return &ResourceType{Raw: xmlNamesToRaw(names)} +} + +func (t *ResourceType) Is(name xml.Name) bool { + for _, raw := range t.Raw { + if n, ok := raw.XMLName(); ok && name == n { + return true + } + } + return false +} + +var CollectionName = xml.Name{Namespace, "collection"} + +// https://tools.ietf.org/html/rfc4918#section-15.4 +type GetContentLength struct { + XMLName xml.Name `xml:"DAV: getcontentlength"` + Length int64 `xml:",chardata"` +} + +// https://tools.ietf.org/html/rfc4918#section-15.5 +type GetContentType struct { + XMLName xml.Name `xml:"DAV: getcontenttype"` + Type string `xml:",chardata"` +} + +type Time time.Time + +func (t *Time) UnmarshalText(b []byte) error { + tt, err := http.ParseTime(string(b)) + if err != nil { + return err + } + *t = Time(tt) + return nil +} + +func (t *Time) MarshalText() ([]byte, error) { + s := time.Time(*t).UTC().Format(http.TimeFormat) + return []byte(s), nil +} + +// https://tools.ietf.org/html/rfc4918#section-15.7 +type GetLastModified struct { + XMLName xml.Name `xml:"DAV: getlastmodified"` + LastModified Time `xml:",chardata"` +} + +// https://tools.ietf.org/html/rfc4918#section-15.6 +type GetETag struct { + XMLName xml.Name `xml:"DAV: getetag"` + ETag ETag `xml:",chardata"` +} + +type ETag string + +func (etag *ETag) UnmarshalText(b []byte) error { + s, err := strconv.Unquote(string(b)) + if err != nil { + return fmt.Errorf("webdav: failed to unquote ETag: %v", err) + } + *etag = ETag(s) + return nil +} + +func (etag ETag) MarshalText() ([]byte, error) { + return []byte(etag.String()), nil +} + +func (etag ETag) String() string { + return fmt.Sprintf("%q", string(etag)) +} + +// https://tools.ietf.org/html/rfc4918#section-14.5 +type Error struct { + XMLName xml.Name `xml:"DAV: error"` + Raw []RawXMLValue `xml:",any"` +} + +func (err *Error) Error() string { + b, _ := xml.Marshal(err) + return string(b) +} + +// https://tools.ietf.org/html/rfc4918#section-15.2 +type DisplayName struct { + XMLName xml.Name `xml:"DAV: displayname"` + Name string `xml:",chardata"` +} + +// https://tools.ietf.org/html/rfc5397#section-3 +type CurrentUserPrincipal struct { + XMLName xml.Name `xml:"DAV: current-user-principal"` + Href Href `xml:"href,omitempty"` + Unauthenticated *struct{} `xml:"unauthenticated,omitempty"` +} + +// https://tools.ietf.org/html/rfc4918#section-14.19 +type PropertyUpdate struct { + XMLName xml.Name `xml:"DAV: propertyupdate"` + Remove []Remove `xml:"remove"` + Set []Set `xml:"set"` +} + +// https://tools.ietf.org/html/rfc4918#section-14.23 +type Remove struct { + XMLName xml.Name `xml:"DAV: remove"` + Prop Prop `xml:"prop"` +} + +// https://tools.ietf.org/html/rfc4918#section-14.26 +type Set struct { + XMLName xml.Name `xml:"DAV: set"` + Prop Prop `xml:"prop"` +} + +// https://tools.ietf.org/html/rfc6578#section-6.1 +type SyncCollectionQuery struct { + XMLName xml.Name `xml:"DAV: sync-collection"` + SyncToken string `xml:"sync-token"` + Limit *Limit `xml:"limit,omitempty"` + SyncLevel string `xml:"sync-level"` + Prop *Prop `xml:"prop"` +} + +// https://tools.ietf.org/html/rfc5323#section-5.17 +type Limit struct { + XMLName xml.Name `xml:"DAV: limit"` + NResults uint `xml:"nresults"` +} diff --git a/util/sync/driveClient/internal/internal.go b/util/sync/driveClient/internal/internal.go new file mode 100644 index 000000000..1f3e0e525 --- /dev/null +++ b/util/sync/driveClient/internal/internal.go @@ -0,0 +1,108 @@ +package internal // Package internal provides low-level helpers for WebDAV clients and servers. +import ( + "errors" + "fmt" + "net/http" +) + +// Depth indicates whether a request applies to the resource's members. It's +// defined in RFC 4918 section 10.2. +type Depth int + +const ( + // DepthZero indicates that the request applies only to the resource. + DepthZero Depth = 0 + // DepthOne indicates that the request applies to the resource and its + // internal members only. + DepthOne Depth = 1 + // DepthInfinity indicates that the request applies to the resource and all + // of its members. + DepthInfinity Depth = -1 +) + +// ParseDepth parses a Depth header. +func ParseDepth(s string) (Depth, error) { + switch s { + case "0": + return DepthZero, nil + case "1": + return DepthOne, nil + case "infinity": + return DepthInfinity, nil + } + return 0, fmt.Errorf("webdav: invalid Depth value") +} + +// String formats the depth. +func (d Depth) String() string { + switch d { + case DepthZero: + return "0" + case DepthOne: + return "1" + case DepthInfinity: + return "infinity" + } + panic("webdav: invalid Depth value") +} + +// ParseOverwrite parses an Overwrite header. +func ParseOverwrite(s string) (bool, error) { + switch s { + case "T": + return true, nil + case "F": + return false, nil + } + return false, fmt.Errorf("webdav: invalid Overwrite value") +} + +// FormatOverwrite formats an Overwrite header. +func FormatOverwrite(overwrite bool) string { + if overwrite { + return "T" + } else { + return "F" + } +} + +type HTTPError struct { + Code int + Err error +} + +func HTTPErrorFromError(err error) *HTTPError { + if err == nil { + return nil + } + if httpErr, ok := err.(*HTTPError); ok { + return httpErr + } else { + return &HTTPError{http.StatusInternalServerError, err} + } +} + +func IsNotFound(err error) bool { + var httpErr *HTTPError + if errors.As(err, &httpErr) { + return httpErr.Code == http.StatusNotFound + } + return false +} + +func HTTPErrorf(code int, format string, a ...interface{}) *HTTPError { + return &HTTPError{code, fmt.Errorf(format, a...)} +} + +func (err *HTTPError) Error() string { + s := fmt.Sprintf("%v %v", err.Code, http.StatusText(err.Code)) + if err.Err != nil { + return fmt.Sprintf("%v: %v", s, err.Err) + } else { + return s + } +} + +func (err *HTTPError) Unwrap() error { + return err.Err +} diff --git a/util/sync/driveClient/internal/xml.go b/util/sync/driveClient/internal/xml.go new file mode 100644 index 000000000..2f4c61ca8 --- /dev/null +++ b/util/sync/driveClient/internal/xml.go @@ -0,0 +1,175 @@ +package internal + +import ( + "encoding/xml" + "fmt" + "io" + "reflect" + "strings" +) + +// RawXMLValue is a raw XML value. It implements xml.Unmarshaler and +// xml.Marshaler and can be used to delay XML decoding or precompute an XML +// encoding. +type RawXMLValue struct { + tok xml.Token // guaranteed not to be xml.EndElement + children []RawXMLValue + + // Unfortunately encoding/xml doesn't offer TokenWriter, so we need to + // cache outgoing data. + out interface{} +} + +// NewRawXMLElement creates a new RawXMLValue for an element. +func NewRawXMLElement(name xml.Name, attr []xml.Attr, children []RawXMLValue) *RawXMLValue { + return &RawXMLValue{tok: xml.StartElement{name, attr}, children: children} +} + +// EncodeRawXMLElement encodes a value into a new RawXMLValue. The XML value +// can only be used for marshalling. +func EncodeRawXMLElement(v interface{}) (*RawXMLValue, error) { + return &RawXMLValue{out: v}, nil +} + +// UnmarshalXML implements xml.Unmarshaler. +func (val *RawXMLValue) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + val.tok = start + val.children = nil + val.out = nil + + for { + tok, err := d.Token() + if err != nil { + return err + } + switch tok := tok.(type) { + case xml.StartElement: + child := RawXMLValue{} + if err := child.UnmarshalXML(d, tok); err != nil { + return err + } + val.children = append(val.children, child) + case xml.EndElement: + return nil + default: + val.children = append(val.children, RawXMLValue{tok: xml.CopyToken(tok)}) + } + } +} + +// MarshalXML implements xml.Marshaler. +func (val *RawXMLValue) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if val.out != nil { + return e.Encode(val.out) + } + + switch tok := val.tok.(type) { + case xml.StartElement: + if err := e.EncodeToken(tok); err != nil { + return err + } + for _, child := range val.children { + // TODO: find a sensible value for the start argument? + if err := child.MarshalXML(e, xml.StartElement{}); err != nil { + return err + } + } + return e.EncodeToken(tok.End()) + case xml.EndElement: + panic("unexpected end element") + default: + return e.EncodeToken(tok) + } +} + +var _ xml.Marshaler = (*RawXMLValue)(nil) +var _ xml.Unmarshaler = (*RawXMLValue)(nil) + +func (val *RawXMLValue) Decode(v interface{}) error { + return xml.NewTokenDecoder(val.TokenReader()).Decode(&v) +} + +func (val *RawXMLValue) XMLName() (name xml.Name, ok bool) { + if start, ok := val.tok.(xml.StartElement); ok { + return start.Name, true + } + return xml.Name{}, false +} + +// TokenReader returns a stream of tokens for the XML value. +func (val *RawXMLValue) TokenReader() xml.TokenReader { + if val.out != nil { + panic("webdav: called RawXMLValue.TokenReader on a marshal-only XML value") + } + return &rawXMLValueReader{val: val} +} + +type rawXMLValueReader struct { + val *RawXMLValue + start, end bool + child int + childReader xml.TokenReader +} + +func (tr *rawXMLValueReader) Token() (xml.Token, error) { + if tr.end { + return nil, io.EOF + } + + start, ok := tr.val.tok.(xml.StartElement) + if !ok { + tr.end = true + return tr.val.tok, nil + } + + if !tr.start { + tr.start = true + return start, nil + } + + for tr.child < len(tr.val.children) { + if tr.childReader == nil { + tr.childReader = tr.val.children[tr.child].TokenReader() + } + + tok, err := tr.childReader.Token() + if err == io.EOF { + tr.childReader = nil + tr.child++ + } else { + return tok, err + } + } + + tr.end = true + return start.End(), nil +} + +var _ xml.TokenReader = (*rawXMLValueReader)(nil) + +func valueXMLName(v interface{}) (xml.Name, error) { + t := reflect.TypeOf(v) + for t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t.Kind() != reflect.Struct { + return xml.Name{}, fmt.Errorf("webdav: %T is not a struct", v) + } + nameField, ok := t.FieldByName("XMLName") + if !ok { + return xml.Name{}, fmt.Errorf("webdav: %T is missing an XMLName struct field", v) + } + if nameField.Type != reflect.TypeOf(xml.Name{}) { + return xml.Name{}, fmt.Errorf("webdav: %T.XMLName isn't an xml.Name", v) + } + tag := nameField.Tag.Get("xml") + if tag == "" { + return xml.Name{}, fmt.Errorf(`webdav: %T.XMLName is missing an "xml" tag`, v) + } + name := strings.Split(tag, ",")[0] + nameParts := strings.Split(name, " ") + if len(nameParts) != 2 { + return xml.Name{}, fmt.Errorf("webdav: expected a namespace and local name in %T.XMLName's xml tag", v) + } + return xml.Name{nameParts[0], nameParts[1]}, nil +} diff --git a/util/sync/driveClient/model.go b/util/sync/driveClient/model.go new file mode 100644 index 000000000..100994d30 --- /dev/null +++ b/util/sync/driveClient/model.go @@ -0,0 +1,119 @@ +package driveClient + +import ( + "errors" + "fmt" + "net/http" + "time" +) + +// Depth indicates whether a request applies to the resource's members. It's +// defined in RFC 4918 section 10.2. +type Depth int + +const ( + // DepthZero indicates that the request applies only to the resource. + DepthZero Depth = 0 + // DepthOne indicates that the request applies to the resource and its + // internal members only. + DepthOne Depth = 1 + // DepthInfinity indicates that the request applies to the resource and all + // of its members. + DepthInfinity Depth = -1 +) + +// ParseDepth parses a Depth header. +func ParseDepth(s string) (Depth, error) { + switch s { + case "0": + return DepthZero, nil + case "1": + return DepthOne, nil + case "infinity": + return DepthInfinity, nil + } + return 0, fmt.Errorf("webdav: invalid Depth value") +} + +// String formats the depth. +func (d Depth) String() string { + switch d { + case DepthZero: + return "0" + case DepthOne: + return "1" + case DepthInfinity: + return "infinity" + } + panic("webdav: invalid Depth value") +} + +// ParseOverwrite parses an Overwrite header. +func ParseOverwrite(s string) (bool, error) { + switch s { + case "T": + return true, nil + case "F": + return false, nil + } + return false, fmt.Errorf("webdav: invalid Overwrite value") +} + +// FormatOverwrite formats an Overwrite header. +func FormatOverwrite(overwrite bool) string { + if overwrite { + return "T" + } else { + return "F" + } +} + +type HTTPError struct { + Code int + Err error +} + +func HTTPErrorFromError(err error) *HTTPError { + if err == nil { + return nil + } + if httpErr, ok := err.(*HTTPError); ok { + return httpErr + } else { + return &HTTPError{http.StatusInternalServerError, err} + } +} + +func IsNotFound(err error) bool { + var httpErr *HTTPError + if errors.As(err, &httpErr) { + return httpErr.Code == http.StatusNotFound + } + return false +} + +func HTTPErrorf(code int, format string, a ...interface{}) *HTTPError { + return &HTTPError{code, fmt.Errorf(format, a...)} +} + +func (err *HTTPError) Error() string { + s := fmt.Sprintf("%v %v", err.Code, http.StatusText(err.Code)) + if err.Err != nil { + return fmt.Sprintf("%v: %v", s, err.Err) + } else { + return s + } +} + +func (err *HTTPError) Unwrap() error { + return err.Err +} + +type FileInfo struct { + Path string + Size int64 + ModTime time.Time + IsDir bool + MIMEType string + ETag string +} From 03cf7cc8b4c2c3132f5654b5134a61eb46a606f4 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 9 Jan 2024 11:52:22 -0500 Subject: [PATCH 26/63] support for updating timestamps in 'driveClient' (#511) --- cmd/zrok/davtest.go | 10 +++----- util/sync/driveClient/client.go | 13 ++++++++++ util/sync/driveClient/internal/client.go | 30 ++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/cmd/zrok/davtest.go b/cmd/zrok/davtest.go index 78e919ba2..9a85923d1 100644 --- a/cmd/zrok/davtest.go +++ b/cmd/zrok/davtest.go @@ -3,9 +3,9 @@ package main import ( "context" "github.com/openziti/zrok/util/sync/driveClient" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" "net/http" + "time" ) func init() { @@ -20,7 +20,7 @@ func newDavtestCommand() *davtestCommand { cmd := &cobra.Command{ Use: "davtest", Short: "WebDAV testing wrapper", - Args: cobra.ExactArgs(1), + Args: cobra.ExactArgs(2), } command := &davtestCommand{cmd: cmd} cmd.Run = command.run @@ -32,11 +32,7 @@ func (cmd *davtestCommand) run(_ *cobra.Command, args []string) { if err != nil { panic(err) } - fis, err := client.Readdir(context.Background(), "/", true) - if err != nil { + if err := client.Touch(context.Background(), args[1], time.Now().Add(-(24 * time.Hour))); err != nil { panic(err) } - for _, fi := range fis { - logrus.Infof("=> %s", fi.Path) - } } diff --git a/util/sync/driveClient/client.go b/util/sync/driveClient/client.go index 77cd22158..8dbdad586 100644 --- a/util/sync/driveClient/client.go +++ b/util/sync/driveClient/client.go @@ -205,6 +205,19 @@ func (c *Client) Create(ctx context.Context, name string) (io.WriteCloser, error return &fileWriter{pw, done}, nil } +func (c *Client) Touch(ctx context.Context, path string, mtime time.Time) error { + status, err := c.ic.Touch(ctx, path, mtime) + if err != nil { + return err + } + for _, resp := range status.Responses { + if resp.Err() != nil { + return resp.Err() + } + } + return nil +} + func (c *Client) RemoveAll(ctx context.Context, name string) error { req, err := c.ic.NewRequest(http.MethodDelete, name, nil) if err != nil { diff --git a/util/sync/driveClient/internal/client.go b/util/sync/driveClient/internal/client.go index 718f436a4..78a2b3aa0 100644 --- a/util/sync/driveClient/internal/client.go +++ b/util/sync/driveClient/internal/client.go @@ -12,6 +12,7 @@ import ( "net/url" "path" "strings" + "time" "unicode" ) @@ -180,6 +181,35 @@ func (c *Client) PropFind(ctx context.Context, path string, depth Depth, propfin return c.DoMultiStatus(req.WithContext(ctx)) } +func (c *Client) Touch(ctx context.Context, path string, mtime time.Time) (*MultiStatus, error) { + tstr := fmt.Sprintf("%d", mtime.Unix()) + var v []RawXMLValue + for _, c := range tstr { + v = append(v, RawXMLValue{tok: xml.CharData{byte(c)}}) + } + pup := &PropertyUpdate{ + Set: []Set{ + { + Prop: Prop{ + Raw: []RawXMLValue{ + *NewRawXMLElement(xml.Name{Space: "zrok:", Local: "lastmodified"}, nil, v), + }, + }, + }, + }, + } + status, err := c.PropUpdate(ctx, path, pup) + return status, err +} + +func (c *Client) PropUpdate(ctx context.Context, path string, propupd *PropertyUpdate) (*MultiStatus, error) { + req, err := c.NewXMLRequest("PROPPATCH", path, propupd) + if err != nil { + return nil, err + } + return c.DoMultiStatus(req.WithContext(ctx)) +} + // PropfindFlat performs a PROPFIND request with a zero depth. func (c *Client) PropFindFlat(ctx context.Context, path string, propfind *PropFind) (*Response, error) { ms, err := c.PropFind(ctx, path, DepthZero, propfind) From 562e4226b386840d3c5978b84484248b0f2a097f Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 9 Jan 2024 12:57:42 -0500 Subject: [PATCH 27/63] integrate 'driveClient' into the synchronizer (#511) --- cmd/zrok/copy.go | 39 +- cmd/zrok/davtest.go | 18 +- util/sync/model.go | 1 - util/sync/webdav.go | 97 +- util/sync/webdavClient/LICENSE | 27 - util/sync/webdavClient/Makefile | 42 - util/sync/webdavClient/README.md | 962 -------------------- util/sync/webdavClient/auth.go | 409 --------- util/sync/webdavClient/auth_test.go | 62 -- util/sync/webdavClient/basicAuth.go | 42 - util/sync/webdavClient/basicAuth_test.go | 51 -- util/sync/webdavClient/client.go | 474 ---------- util/sync/webdavClient/client_test.go | 574 ------------ util/sync/webdavClient/digestAuth.go | 164 ---- util/sync/webdavClient/digestAuth_test.go | 35 - util/sync/webdavClient/doc.go | 3 - util/sync/webdavClient/errors.go | 57 -- util/sync/webdavClient/file.go | 82 -- util/sync/webdavClient/netrc.go | 54 -- util/sync/webdavClient/passportAuth.go | 181 ---- util/sync/webdavClient/passportAuth_test.go | 66 -- util/sync/webdavClient/requests.go | 201 ---- util/sync/webdavClient/utils.go | 113 --- util/sync/webdavClient/utils_test.go | 67 -- 24 files changed, 57 insertions(+), 3764 deletions(-) delete mode 100644 util/sync/webdavClient/LICENSE delete mode 100644 util/sync/webdavClient/Makefile delete mode 100644 util/sync/webdavClient/README.md delete mode 100644 util/sync/webdavClient/auth.go delete mode 100644 util/sync/webdavClient/auth_test.go delete mode 100644 util/sync/webdavClient/basicAuth.go delete mode 100644 util/sync/webdavClient/basicAuth_test.go delete mode 100644 util/sync/webdavClient/client.go delete mode 100644 util/sync/webdavClient/client_test.go delete mode 100644 util/sync/webdavClient/digestAuth.go delete mode 100644 util/sync/webdavClient/digestAuth_test.go delete mode 100644 util/sync/webdavClient/doc.go delete mode 100644 util/sync/webdavClient/errors.go delete mode 100644 util/sync/webdavClient/file.go delete mode 100644 util/sync/webdavClient/netrc.go delete mode 100644 util/sync/webdavClient/passportAuth.go delete mode 100644 util/sync/webdavClient/passportAuth_test.go delete mode 100644 util/sync/webdavClient/requests.go delete mode 100644 util/sync/webdavClient/utils.go delete mode 100644 util/sync/webdavClient/utils_test.go diff --git a/cmd/zrok/copy.go b/cmd/zrok/copy.go index 020574d7b..58a1e4f1d 100644 --- a/cmd/zrok/copy.go +++ b/cmd/zrok/copy.go @@ -7,7 +7,6 @@ import ( "github.com/openziti/zrok/sdk/golang/sdk" "github.com/openziti/zrok/tui" "github.com/openziti/zrok/util/sync" - "github.com/pkg/errors" "github.com/spf13/cobra" "net/url" ) @@ -36,9 +35,6 @@ func (cmd *copyCommand) run(_ *cobra.Command, args []string) { if err != nil { tui.Error(fmt.Sprintf("invalid source URL '%v'", args[0]), err) } - if sourceUrl.Scheme != "zrok" && sourceUrl.Scheme != "file" { - tui.Error("source URL must be 'file://' or 'zrok://", nil) - } targetStr := "file://." if len(args) == 2 { @@ -48,16 +44,6 @@ func (cmd *copyCommand) run(_ *cobra.Command, args []string) { if err != nil { tui.Error(fmt.Sprintf("invalid target URL '%v'", targetStr), err) } - if targetUrl.Scheme != "zrok" && targetUrl.Scheme != "file" { - tui.Error("target URL must be 'file://' or 'zrok://", nil) - } - - if sourceUrl.Scheme != "zrok" && targetUrl.Scheme != "zrok" { - tui.Error("either or must be a 'zrok://' URL", nil) - } - if targetUrl.Scheme != "file" && sourceUrl.Scheme != "file" { - tui.Error("either or must be a 'file://' URL", nil) - } root, err := environment.LoadRoot() if err != nil { @@ -77,12 +63,14 @@ func (cmd *copyCommand) run(_ *cobra.Command, args []string) { tui.Error("error creating access", err) } } - defer func() { - err := sdk.DeleteAccess(root, access) - if err != nil { - tui.Error("error deleting access", err) - } - }() + if access != nil { + defer func() { + err := sdk.DeleteAccess(root, access) + if err != nil { + tui.Error("error deleting access", err) + } + }() + } source, err := cmd.createTarget(sourceUrl, root) if err != nil { @@ -102,17 +90,14 @@ func (cmd *copyCommand) run(_ *cobra.Command, args []string) { func (cmd *copyCommand) createTarget(t *url.URL, root env_core.Root) (sync.Target, error) { switch t.Scheme { - case "zrok": + case "file": + return sync.NewFilesystemTarget(&sync.FilesystemTargetConfig{Root: t.Path}), nil + + default: target, err := sync.NewWebDAVTarget(&sync.WebDAVTargetConfig{URL: t, Username: "", Password: "", Root: root}) if err != nil { return nil, err } return target, nil - - case "file": - return sync.NewFilesystemTarget(&sync.FilesystemTargetConfig{Root: t.Path}), nil - - default: - return nil, errors.Errorf("invalid scheme") } } diff --git a/cmd/zrok/davtest.go b/cmd/zrok/davtest.go index 9a85923d1..9d03168fb 100644 --- a/cmd/zrok/davtest.go +++ b/cmd/zrok/davtest.go @@ -4,8 +4,9 @@ import ( "context" "github.com/openziti/zrok/util/sync/driveClient" "github.com/spf13/cobra" + "io" "net/http" - "time" + "os" ) func init() { @@ -20,7 +21,7 @@ func newDavtestCommand() *davtestCommand { cmd := &cobra.Command{ Use: "davtest", Short: "WebDAV testing wrapper", - Args: cobra.ExactArgs(2), + Args: cobra.ExactArgs(3), } command := &davtestCommand{cmd: cmd} cmd.Run = command.run @@ -32,7 +33,18 @@ func (cmd *davtestCommand) run(_ *cobra.Command, args []string) { if err != nil { panic(err) } - if err := client.Touch(context.Background(), args[1], time.Now().Add(-(24 * time.Hour))); err != nil { + ws, err := client.Open(context.Background(), args[1]) + if err != nil { + panic(err) + } + fs, err := os.Create(args[2]) + if err != nil { + panic(err) + } + _, err = io.Copy(fs, ws) + if err != nil { panic(err) } + ws.Close() + fs.Close() } diff --git a/util/sync/model.go b/util/sync/model.go index b1a6b0435..8fca5ca07 100644 --- a/util/sync/model.go +++ b/util/sync/model.go @@ -15,7 +15,6 @@ type Object struct { type Target interface { Inventory() ([]*Object, error) - IsDir() bool ReadStream(path string) (io.ReadCloser, error) WriteStream(path string, stream io.Reader, mode os.FileMode) error SetModificationTime(path string, mtime time.Time) error diff --git a/util/sync/webdav.go b/util/sync/webdav.go index c349d38bc..b9a0c871b 100644 --- a/util/sync/webdav.go +++ b/util/sync/webdav.go @@ -1,14 +1,13 @@ package sync import ( - "fmt" + "context" "github.com/openziti/zrok/environment/env_core" - "github.com/openziti/zrok/util/sync/webdavClient" - "github.com/pkg/errors" + "github.com/openziti/zrok/util/sync/driveClient" "io" + "net/http" "net/url" "os" - "path/filepath" "time" ) @@ -21,90 +20,54 @@ type WebDAVTargetConfig struct { type WebDAVTarget struct { cfg *WebDAVTargetConfig - c *webdavClient.Client + dc *driveClient.Client isDir bool } func NewWebDAVTarget(cfg *WebDAVTargetConfig) (*WebDAVTarget, error) { - c, err := webdavClient.NewZrokClient(cfg.URL, cfg.Root, webdavClient.NewAutoAuth(cfg.Username, cfg.Password)) + dc, err := driveClient.NewClient(http.DefaultClient, cfg.URL.String()) if err != nil { return nil, err } - if err := c.Connect(); err != nil { - return nil, errors.Wrap(err, "error connecting to webdav target") - } - return &WebDAVTarget{cfg: cfg, c: c}, nil + return &WebDAVTarget{cfg: cfg, dc: dc}, nil } func (t *WebDAVTarget) Inventory() ([]*Object, error) { - fi, err := t.c.Stat("") - - if fi != nil && !fi.IsDir() { - t.isDir = false - return []*Object{{ - Path: fi.Name(), - Size: fi.Size(), - Modified: fi.ModTime(), - }}, nil - } - - t.isDir = true - tree, err := t.recurse("", nil) - if err != nil { - return nil, err - } - return tree, nil -} - -func (t *WebDAVTarget) IsDir() bool { - return t.isDir -} - -func (t *WebDAVTarget) recurse(path string, tree []*Object) ([]*Object, error) { - files, err := t.c.ReadDir(path) + fis, err := t.dc.Readdir(context.Background(), "", true) if err != nil { return nil, err } - for _, f := range files { - sub := filepath.ToSlash(filepath.Join(path, f.Name())) - if f.IsDir() { - tree, err = t.recurse(sub, tree) - if err != nil { - return nil, err - } - } else { - if v, ok := f.(webdavClient.File); ok { - tree = append(tree, &Object{ - Path: filepath.ToSlash(filepath.Join(path, f.Name())), - Size: v.Size(), - Modified: v.ModTime(), - ETag: v.ETag(), - }) - } + var objects []*Object + for _, fi := range fis { + if !fi.IsDir { + objects = append(objects, &Object{ + Path: fi.Path, + Size: fi.Size, + Modified: fi.ModTime, + ETag: fi.ETag, + }) } } - return tree, nil + return objects, nil } func (t *WebDAVTarget) ReadStream(path string) (io.ReadCloser, error) { - if t.isDir { - return t.c.ReadStream(path) - } - return t.c.ReadStream("") -} - -func (t *WebDAVTarget) WriteStream(path string, stream io.Reader, mode os.FileMode) error { - return t.c.WriteStream(path, stream, mode) + return t.dc.Open(context.Background(), path) } -func (t *WebDAVTarget) SetModificationTime(path string, mtime time.Time) error { - modtimeUnix := mtime.Unix() - body := "" + - "" + - fmt.Sprintf("%d", modtimeUnix) + - "" - if err := t.c.Proppatch(path, body, nil, nil); err != nil { +func (t *WebDAVTarget) WriteStream(path string, rs io.Reader, _ os.FileMode) error { + ws, err := t.dc.Create(context.Background(), path) + if err != nil { + return err + } + defer ws.Close() + _, err = io.Copy(ws, rs) + if err != nil { return err } return nil } + +func (t *WebDAVTarget) SetModificationTime(path string, mtime time.Time) error { + return t.dc.Touch(context.Background(), path, mtime) +} diff --git a/util/sync/webdavClient/LICENSE b/util/sync/webdavClient/LICENSE deleted file mode 100644 index a7cd4420f..000000000 --- a/util/sync/webdavClient/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2014, Studio B12 GmbH -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/util/sync/webdavClient/Makefile b/util/sync/webdavClient/Makefile deleted file mode 100644 index 48dbb4e72..000000000 --- a/util/sync/webdavClient/Makefile +++ /dev/null @@ -1,42 +0,0 @@ -BIN := gowebdav -SRC := $(wildcard *.go) cmd/gowebdav/main.go - -all: test cmd - -cmd: ${BIN} - -${BIN}: ${SRC} - go build -o $@ ./cmd/gowebdav - -test: - go test -modfile=go_test.mod -v -short -cover ./... - -api: .go/bin/godoc2md - @sed '/^## API$$/,$$d' -i README.md - @echo '## API' >> README.md - @$< github.com/studio-b12/gowebdav | sed '/^$$/N;/^\n$$/D' |\ - sed '2d' |\ - sed 's/\/src\/github.com\/studio-b12\/gowebdav\//https:\/\/github.com\/studio-b12\/gowebdav\/blob\/master\//g' |\ - sed 's/\/src\/target\//https:\/\/github.com\/studio-b12\/gowebdav\/blob\/master\//g' |\ - sed 's/^#/##/g' >> README.md - -check: .go/bin/gocyclo - gofmt -w -s $(SRC) - @echo - .go/bin/gocyclo -over 15 . - @echo - go vet -modfile=go_test.mod ./... - - -.go/bin/godoc2md: - @mkdir -p $(@D) - @GOPATH="$(CURDIR)/.go" go install github.com/davecheney/godoc2md@latest - -.go/bin/gocyclo: - @mkdir -p $(@D) - @GOPATH="$(CURDIR)/.go" go install github.com/fzipp/gocyclo/cmd/gocyclo@latest - -clean: - @rm -f ${BIN} - -.PHONY: all cmd clean test api check diff --git a/util/sync/webdavClient/README.md b/util/sync/webdavClient/README.md deleted file mode 100644 index 6c9a795a0..000000000 --- a/util/sync/webdavClient/README.md +++ /dev/null @@ -1,962 +0,0 @@ -# GoWebDAV - -[![Unit Tests Status](https://github.com/studio-b12/gowebdav/actions/workflows/tests.yml/badge.svg)](https://github.com/studio-b12/gowebdav/actions/workflows/tests.yml) -[![Build Artifacts Status](https://github.com/studio-b12/gowebdav/actions/workflows/artifacts.yml/badge.svg)](https://github.com/studio-b12/gowebdav/actions/workflows/artifacts.yml) -[![GoDoc](https://godoc.org/github.com/studio-b12/gowebdav?status.svg)](https://godoc.org/github.com/studio-b12/gowebdav) -[![Go Report Card](https://goreportcard.com/badge/github.com/studio-b12/gowebdav)](https://goreportcard.com/report/github.com/studio-b12/gowebdav) - -A pure Golang WebDAV client library that comes with -a [reference implementation](https://github.com/studio-b12/gowebdav/tree/master/cmd/gowebdav). - -## Features at a glance - -Our `gowebdav` library allows to perform following actions on the remote WebDAV server: - -* [create path](#create-path-on-a-webdav-server) -* [get files list](#get-files-list) -* [download file](#download-file-to-byte-array) -* [upload file](#upload-file-from-byte-array) -* [get information about specified file/folder](#get-information-about-specified-filefolder) -* [move file to another location](#move-file-to-another-location) -* [copy file to another location](#copy-file-to-another-location) -* [delete file](#delete-file) - -It also provides an [authentication API](#type-authenticator) that makes it easy to encapsulate and control complex -authentication challenges. -The default implementation negotiates the algorithm based on the user's preferences and the methods offered by the -remote server. - -Out-of-box authentication support for: - -* [BasicAuth](https://en.wikipedia.org/wiki/Basic_access_authentication) -* [DigestAuth](https://en.wikipedia.org/wiki/Digest_access_authentication) -* [MS-PASS](https://github.com/studio-b12/gowebdav/pull/70#issuecomment-1421713726) -* [WIP Kerberos](https://github.com/studio-b12/gowebdav/pull/71#issuecomment-1416465334) -* [WIP Bearer Token](https://github.com/studio-b12/gowebdav/issues/61) - -## Usage - -First of all you should create `Client` instance using `NewClient()` function: - -```go -root := "https://webdav.mydomain.me" -user := "user" -password := "password" - -c := gowebdav.NewClient(root, user, password) -c.Connect() -// kick of your work! -``` - -After you can use this `Client` to perform actions, described below. - -**NOTICE:** We will not check for errors in the examples, to focus you on the `gowebdav` library's code, but you should -do it in your code! - -### Create path on a WebDAV server - -```go -err := c.Mkdir("folder", 0644) -``` - -In case you want to create several folders you can use `c.MkdirAll()`: - -```go -err := c.MkdirAll("folder/subfolder/subfolder2", 0644) -``` - -### Get files list - -```go -files, _ := c.ReadDir("folder/subfolder") -for _, file := range files { - //notice that [file] has os.FileInfo type - fmt.Println(file.Name()) -} -``` - -### Download file to byte array - -```go -webdavFilePath := "folder/subfolder/file.txt" -localFilePath := "/tmp/webdav/file.txt" - -bytes, _ := c.Read(webdavFilePath) -os.WriteFile(localFilePath, bytes, 0644) -``` - -### Download file via reader - -Also you can use `c.ReadStream()` method: - -```go -webdavFilePath := "folder/subfolder/file.txt" -localFilePath := "/tmp/webdav/file.txt" - -reader, _ := c.ReadStream(webdavFilePath) - -file, _ := os.Create(localFilePath) -defer file.Close() - -io.Copy(file, reader) -``` - -### Upload file from byte array - -```go -webdavFilePath := "folder/subfolder/file.txt" -localFilePath := "/tmp/webdav/file.txt" - -bytes, _ := os.ReadFile(localFilePath) - -c.Write(webdavFilePath, bytes, 0644) -``` - -### Upload file via writer - -```go -webdavFilePath := "folder/subfolder/file.txt" -localFilePath := "/tmp/webdav/file.txt" - -file, _ := os.Open(localFilePath) -defer file.Close() - -c.WriteStream(webdavFilePath, file, 0644) -``` - -### Get information about specified file/folder - -```go -webdavFilePath := "folder/subfolder/file.txt" - -info := c.Stat(webdavFilePath) -//notice that [info] has os.FileInfo type -fmt.Println(info) -``` - -### Move file to another location - -```go -oldPath := "folder/subfolder/file.txt" -newPath := "folder/subfolder/moved.txt" -isOverwrite := true - -c.Rename(oldPath, newPath, isOverwrite) -``` - -### Copy file to another location - -```go -oldPath := "folder/subfolder/file.txt" -newPath := "folder/subfolder/file-copy.txt" -isOverwrite := true - -c.Copy(oldPath, newPath, isOverwrite) -``` - -### Delete file - -```go -webdavFilePath := "folder/subfolder/file.txt" - -c.Remove(webdavFilePath) -``` - -## Links - -More details about WebDAV server you can read from following resources: - -* [RFC 4918 - HTTP Extensions for Web Distributed Authoring and Versioning (WebDAV)](https://tools.ietf.org/html/rfc4918) -* [RFC 5689 - Extended MKCOL for Web Distributed Authoring and Versioning (WebDAV)](https://tools.ietf.org/html/rfc5689) -* [RFC 2616 - HTTP/1.1 Status Code Definitions](http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html "HTTP/1.1 Status Code Definitions") -* [WebDav: Next Generation Collaborative Web Authoring By Lisa Dusseaul](https://books.google.de/books?isbn=0130652083 "WebDav: Next Generation Collaborative Web Authoring By Lisa Dusseault") - -**NOTICE**: RFC 2518 is obsoleted by RFC 4918 in June 2007 - -## Contributing - -All contributing are welcome. If you have any suggestions or find some bug - please create an Issue to let us make this -project better. We appreciate your help! - -## License - -This library is distributed under the BSD 3-Clause license found in -the [LICENSE](https://github.com/studio-b12/gowebdav/blob/master/LICENSE) file. - -## API - -`import "github.com/studio-b12/gowebdav"` - -* [Overview](#pkg-overview) -* [Index](#pkg-index) -* [Examples](#pkg-examples) -* [Subdirectories](#pkg-subdirectories) - -### Overview - -Package gowebdav is a WebDAV client library with a command line tool -included. - -### Index - -* [Constants](#pkg-constants) -* [Variables](#pkg-variables) -* [func FixSlash(s string) string](#FixSlash) -* [func FixSlashes(s string) string](#FixSlashes) -* [func IsErrCode(err error, code int) bool](#IsErrCode) -* [func IsErrNotFound(err error) bool](#IsErrNotFound) -* [func Join(path0 string, path1 string) string](#Join) -* [func NewPathError(op string, path string, statusCode int) error](#NewPathError) -* [func NewPathErrorErr(op string, path string, err error) error](#NewPathErrorErr) -* [func PathEscape(path string) string](#PathEscape) -* [func ReadConfig(uri, netrc string) (string, string)](#ReadConfig) -* [func String(r io.Reader) string](#String) -* [type AuthFactory](#AuthFactory) -* [type Authenticator](#Authenticator) - * [func NewDigestAuth(login, secret string, rs *http.Response) (Authenticator, error)](#NewDigestAuth) - * [func NewPassportAuth(c *http.Client, user, pw, partnerURL string, header *http.Header) (Authenticator, error)](#NewPassportAuth) -* [type Authorizer](#Authorizer) - * [func NewAutoAuth(login string, secret string) Authorizer](#NewAutoAuth) - * [func NewEmptyAuth() Authorizer](#NewEmptyAuth) - * [func NewPreemptiveAuth(auth Authenticator) Authorizer](#NewPreemptiveAuth) -* [type BasicAuth](#BasicAuth) - * [func (b *BasicAuth) Authorize(c *http.Client, rq *http.Request, path string) error](#BasicAuth.Authorize) - * [func (b *BasicAuth) Clone() Authenticator](#BasicAuth.Clone) - * [func (b *BasicAuth) Close() error](#BasicAuth.Close) - * [func (b *BasicAuth) String() string](#BasicAuth.String) - * [func (b *BasicAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error)](#BasicAuth.Verify) -* [type Client](#Client) - * [func NewAuthClient(uri string, auth Authorizer) *Client](#NewAuthClient) - * [func NewClient(uri, user, pw string) *Client](#NewClient) - * [func (c *Client) Connect() error](#Client.Connect) - * [func (c *Client) Copy(oldpath, newpath string, overwrite bool) error](#Client.Copy) - * [func (c *Client) Mkdir(path string, _ os.FileMode) (err error)](#Client.Mkdir) - * [func (c *Client) MkdirAll(path string, _ os.FileMode) (err error)](#Client.MkdirAll) - * [func (c *Client) Read(path string) ([]byte, error)](#Client.Read) - * [func (c *Client) ReadDir(path string) ([]os.FileInfo, error)](#Client.ReadDir) - * [func (c *Client) ReadStream(path string) (io.ReadCloser, error)](#Client.ReadStream) - * [func (c *Client) ReadStreamRange(path string, offset, length int64) (io.ReadCloser, error)](#Client.ReadStreamRange) - * [func (c *Client) Remove(path string) error](#Client.Remove) - * [func (c *Client) RemoveAll(path string) error](#Client.RemoveAll) - * [func (c *Client) Rename(oldpath, newpath string, overwrite bool) error](#Client.Rename) - * [func (c *Client) SetHeader(key, value string)](#Client.SetHeader) - * [func (c *Client) SetInterceptor(interceptor func(method string, rq *http.Request))](#Client.SetInterceptor) - * [func (c *Client) SetJar(jar http.CookieJar)](#Client.SetJar) - * [func (c *Client) SetTimeout(timeout time.Duration)](#Client.SetTimeout) - * [func (c *Client) SetTransport(transport http.RoundTripper)](#Client.SetTransport) - * [func (c *Client) Stat(path string) (os.FileInfo, error)](#Client.Stat) - * [func (c *Client) Write(path string, data []byte, _ os.FileMode) (err error)](#Client.Write) - * [func (c *Client) WriteStream(path string, stream io.Reader, _ os.FileMode) (err error)](#Client.WriteStream) -* [type DigestAuth](#DigestAuth) - * [func (d *DigestAuth) Authorize(c *http.Client, rq *http.Request, path string) error](#DigestAuth.Authorize) - * [func (d *DigestAuth) Clone() Authenticator](#DigestAuth.Clone) - * [func (d *DigestAuth) Close() error](#DigestAuth.Close) - * [func (d *DigestAuth) String() string](#DigestAuth.String) - * [func (d *DigestAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error)](#DigestAuth.Verify) -* [type File](#File) - * [func (f File) ContentType() string](#File.ContentType) - * [func (f File) ETag() string](#File.ETag) - * [func (f File) IsDir() bool](#File.IsDir) - * [func (f File) ModTime() time.Time](#File.ModTime) - * [func (f File) Mode() os.FileMode](#File.Mode) - * [func (f File) Name() string](#File.Name) - * [func (f File) Path() string](#File.Path) - * [func (f File) Size() int64](#File.Size) - * [func (f File) String() string](#File.String) - * [func (f File) Sys() interface{}](#File.Sys) -* [type PassportAuth](#PassportAuth) - * [func (p *PassportAuth) Authorize(c *http.Client, rq *http.Request, path string) error](#PassportAuth.Authorize) - * [func (p *PassportAuth) Clone() Authenticator](#PassportAuth.Clone) - * [func (p *PassportAuth) Close() error](#PassportAuth.Close) - * [func (p *PassportAuth) String() string](#PassportAuth.String) - * [func (p *PassportAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error)](#PassportAuth.Verify) -* [type StatusError](#StatusError) - * [func (se StatusError) Error() string](#StatusError.Error) - -##### Examples - -* [PathEscape](#example_PathEscape) - -##### Package files - -[auth.go](https://github.com/studio-b12/gowebdav/blob/master/auth.go) [basicAuth.go](https://github.com/studio-b12/gowebdav/blob/master/basicAuth.go) [client.go](https://github.com/studio-b12/gowebdav/blob/master/client.go) [digestAuth.go](https://github.com/studio-b12/gowebdav/blob/master/digestAuth.go) [doc.go](https://github.com/studio-b12/gowebdav/blob/master/doc.go) [errors.go](https://github.com/studio-b12/gowebdav/blob/master/errors.go) [file.go](https://github.com/studio-b12/gowebdav/blob/master/file.go) [netrc.go](https://github.com/studio-b12/gowebdav/blob/master/netrc.go) [passportAuth.go](https://github.com/studio-b12/gowebdav/blob/master/passportAuth.go) [requests.go](https://github.com/studio-b12/gowebdav/blob/master/requests.go) [utils.go](https://github.com/studio-b12/gowebdav/blob/master/utils.go) - -### Constants - -``` go -const XInhibitRedirect = "X-Gowebdav-Inhibit-Redirect" -``` - -### Variables - -``` go -var ErrAuthChanged = errors.New("authentication failed, change algorithm") -``` - -ErrAuthChanged must be returned from the Verify method as an error -to trigger a re-authentication / negotiation with a new authenticator. - -``` go -var ErrTooManyRedirects = errors.New("stopped after 10 redirects") -``` - -ErrTooManyRedirects will be used as return error if a request exceeds 10 redirects. - -### func [FixSlash](https://github.com/studio-b12/gowebdav/blob/master/utils.go?s=354:384#L23) - -``` go -func FixSlash(s string) string -``` - -FixSlash appends a trailing / to our string - -### func [FixSlashes](https://github.com/studio-b12/gowebdav/blob/master/utils.go?s=506:538#L31) - -``` go -func FixSlashes(s string) string -``` - -FixSlashes appends and prepends a / if they are missing - -### func [IsErrCode](https://github.com/studio-b12/gowebdav/blob/master/errors.go?s=740:780#L29) - -``` go -func IsErrCode(err error, code int) bool -``` - -IsErrCode returns true if the given error -is an os.PathError wrapping a StatusError -with the given status code. - -### func [IsErrNotFound](https://github.com/studio-b12/gowebdav/blob/master/errors.go?s=972:1006#L39) - -``` go -func IsErrNotFound(err error) bool -``` - -IsErrNotFound is shorthand for IsErrCode -for status 404. - -### func [Join](https://github.com/studio-b12/gowebdav/blob/master/utils.go?s=639:683#L40) - -``` go -func Join(path0 string, path1 string) string -``` - -Join joins two paths - -### func [NewPathError](https://github.com/studio-b12/gowebdav/blob/master/errors.go?s=1040:1103#L43) - -``` go -func NewPathError(op string, path string, statusCode int) error -``` - -### func [NewPathErrorErr](https://github.com/studio-b12/gowebdav/blob/master/errors.go?s=1194:1255#L51) - -``` go -func NewPathErrorErr(op string, path string, err error) error -``` - -### func [PathEscape](https://github.com/studio-b12/gowebdav/blob/master/utils.go?s=153:188#L14) - -``` go -func PathEscape(path string) string -``` - -PathEscape escapes all segments of a given path - -### func [ReadConfig](https://github.com/studio-b12/gowebdav/blob/master/netrc.go?s=428:479#L27) - -``` go -func ReadConfig(uri, netrc string) (string, string) -``` - -ReadConfig reads login and password configuration from ~/.netrc -machine foo.com login username password 123456 - -### func [String](https://github.com/studio-b12/gowebdav/blob/master/utils.go?s=813:844#L45) - -``` go -func String(r io.Reader) string -``` - -String pulls a string out of our io.Reader - -### type [AuthFactory](https://github.com/studio-b12/gowebdav/blob/master/auth.go?s=150:251#L13) - -``` go -type AuthFactory func(c *http.Client, rs *http.Response, path string) (auth Authenticator, err error) -``` - -AuthFactory prototype function to create a new Authenticator - -### type [Authenticator](https://github.com/studio-b12/gowebdav/blob/master/auth.go?s=2155:2695#L56) - -``` go -type Authenticator interface { - // Authorizes a request. Usually by adding some authorization headers. - Authorize(c *http.Client, rq *http.Request, path string) error - // Verifies the response if the authorization was successful. - // May trigger some round trips to pass the authentication. - // May also trigger a new Authenticator negotiation by returning `ErrAuthChenged` - Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) - // Creates a copy of the underlying Authenticator. - Clone() Authenticator - io.Closer -} -``` - -A Authenticator implements a specific way to authorize requests. -Each request is bound to a separate Authenticator instance. - -The authentication flow itself is broken down into `Authorize` -and `Verify` steps. The former method runs before, and the latter -runs after the `Request` is submitted. -This makes it easy to encapsulate and control complex -authentication challenges. - -Some authentication flows causing authentication round trips, -which can be archived by returning the `redo` of the Verify -method. `True` restarts the authentication process for the -current action: A new `Request` is spawned, which must be -authorized, sent, and re-verified again, until the action -is successfully submitted. -The preferred way is to handle the authentication ping-pong -within `Verify`, and then `redo` with fresh credentials. - -The result of the `Verify` method can also trigger an -`Authenticator` change by returning the `ErrAuthChanged` -as an error. Depending on the `Authorizer` this may trigger -an `Authenticator` negotiation. - -Set the `XInhibitRedirect` header to '1' in the `Authorize` -method to get control over request redirection. -Attention! You must handle the incoming request yourself. - -To store a shared session state the `Clone` method **must** -return a new instance, initialized with the shared state. - -#### func [NewDigestAuth](https://github.com/studio-b12/gowebdav/blob/master/digestAuth.go?s=324:406#L21) - -``` go -func NewDigestAuth(login, secret string, rs *http.Response) (Authenticator, error) -``` - -NewDigestAuth creates a new instance of our Digest Authenticator - -#### func [NewPassportAuth](https://github.com/studio-b12/gowebdav/blob/master/passportAuth.go?s=386:495#L21) - -``` go -func NewPassportAuth(c *http.Client, user, pw, partnerURL string, header *http.Header) (Authenticator, error) -``` - -constructor for PassportAuth creates a new PassportAuth object and -automatically authenticates against the given partnerURL - -### type [Authorizer](https://github.com/studio-b12/gowebdav/blob/master/auth.go?s=349:764#L17) - -``` go -type Authorizer interface { - // Creates a new Authenticator Shim per request. - // It may track request related states and perform payload buffering - // for authentication round trips. - // The underlying Authenticator will perform the real authentication. - NewAuthenticator(body io.Reader) (Authenticator, io.Reader) - // Registers a new Authenticator factory to a key. - AddAuthenticator(key string, fn AuthFactory) -} -``` - -Authorizer our Authenticator factory which creates an -`Authenticator` per action/request. - -#### func [NewAutoAuth](https://github.com/studio-b12/gowebdav/blob/master/auth.go?s=3789:3845#L109) - -``` go -func NewAutoAuth(login string, secret string) Authorizer -``` - -NewAutoAuth creates an auto Authenticator factory. -It negotiates the default authentication method -based on the order of the registered Authenticators -and the remotely offered authentication methods. -First In, First Out. - -#### func [NewEmptyAuth](https://github.com/studio-b12/gowebdav/blob/master/auth.go?s=4694:4724#L132) - -``` go -func NewEmptyAuth() Authorizer -``` - -NewEmptyAuth creates an empty Authenticator factory -The order of adding the Authenticator matters. -First In, First Out. -It offers the `NewAutoAuth` features. - -#### func [NewPreemptiveAuth](https://github.com/studio-b12/gowebdav/blob/master/auth.go?s=5300:5353#L148) - -``` go -func NewPreemptiveAuth(auth Authenticator) Authorizer -``` - -NewPreemptiveAuth creates a preemptive Authenticator -The preemptive authorizer uses the provided Authenticator -for every request regardless of any `Www-Authenticate` header. - -It may only have one authentication method, -so calling `AddAuthenticator` **will panic**! - -Look out!! This offers the skinniest and slickest implementation -without any synchronisation!! -Still applicable with `BasicAuth` within go routines. - -### type [BasicAuth](https://github.com/studio-b12/gowebdav/blob/master/basicAuth.go?s=94:145#L9) - -``` go -type BasicAuth struct { - // contains filtered or unexported fields -} - -``` - -BasicAuth structure holds our credentials - -#### func (\*BasicAuth) [Authorize](https://github.com/studio-b12/gowebdav/blob/master/basicAuth.go?s=180:262#L15) - -``` go -func (b *BasicAuth) Authorize(c *http.Client, rq *http.Request, path string) error -``` - -Authorize the current request - -#### func (\*BasicAuth) [Clone](https://github.com/studio-b12/gowebdav/blob/master/basicAuth.go?s=666:707#L34) - -``` go -func (b *BasicAuth) Clone() Authenticator -``` - -Clone creates a Copy of itself - -#### func (\*BasicAuth) [Close](https://github.com/studio-b12/gowebdav/blob/master/basicAuth.go?s=581:614#L29) - -``` go -func (b *BasicAuth) Close() error -``` - -Close cleans up all resources - -#### func (\*BasicAuth) [String](https://github.com/studio-b12/gowebdav/blob/master/basicAuth.go?s=778:813#L40) - -``` go -func (b *BasicAuth) String() string -``` - -String toString - -#### func (\*BasicAuth) [Verify](https://github.com/studio-b12/gowebdav/blob/master/basicAuth.go?s=352:449#L21) - -``` go -func (b *BasicAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) -``` - -Verify verifies if the authentication - -### type [Client](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=220:388#L19) - -``` go -type Client struct { - // contains filtered or unexported fields -} - -``` - -Client defines our structure - -#### func [NewAuthClient](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=608:663#L33) - -``` go -func NewAuthClient(uri string, auth Authorizer) *Client -``` - -NewAuthClient creates a new client instance with a custom Authorizer - -#### func [NewClient](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=436:480#L28) - -``` go -func NewClient(uri, user, pw string) *Client -``` - -NewClient creates a new instance of client - -#### func (\*Client) [Connect](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=1829:1861#L74) - -``` go -func (c *Client) Connect() error -``` - -Connect connects to our dav server - -#### func (\*Client) [Copy](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=6815:6883#L310) - -``` go -func (c *Client) Copy(oldpath, newpath string, overwrite bool) error -``` - -Copy copies a file from A to B - -#### func (\*Client) [Mkdir](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=5790:5852#L259) - -``` go -func (c *Client) Mkdir(path string, _ os.FileMode) (err error) -``` - -Mkdir makes a directory - -#### func (\*Client) [MkdirAll](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=6065:6130#L273) - -``` go -func (c *Client) MkdirAll(path string, _ os.FileMode) (err error) -``` - -MkdirAll like mkdir -p, but for webdav - -#### func (\*Client) [Read](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=6989:7039#L315) - -``` go -func (c *Client) Read(path string) ([]byte, error) -``` - -Read reads the contents of a remote file - -#### func (\*Client) [ReadDir](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=2855:2915#L117) - -``` go -func (c *Client) ReadDir(path string) ([]os.FileInfo, error) -``` - -ReadDir reads the contents of a remote directory - -#### func (\*Client) [ReadStream](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=7350:7413#L333) - -``` go -func (c *Client) ReadStream(path string) (io.ReadCloser, error) -``` - -ReadStream reads the stream for a given path - -#### func (\*Client) [ReadStreamRange](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=8162:8252#L355) - -``` go -func (c *Client) ReadStreamRange(path string, offset, length int64) (io.ReadCloser, error) -``` - -ReadStreamRange reads the stream representing a subset of bytes for a given path, -utilizing HTTP Range Requests if the server supports it. -The range is expressed as offset from the start of the file and length, for example -offset=10, length=10 will return bytes 10 through 19. - -If the server does not support partial content requests and returns full content instead, -this function will emulate the behavior by skipping `offset` bytes and limiting the result -to `length`. - -#### func (\*Client) [Remove](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=5296:5338#L236) - -``` go -func (c *Client) Remove(path string) error -``` - -Remove removes a remote file - -#### func (\*Client) [RemoveAll](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=5404:5449#L241) - -``` go -func (c *Client) RemoveAll(path string) error -``` - -RemoveAll removes remote files - -#### func (\*Client) [Rename](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=6649:6719#L305) - -``` go -func (c *Client) Rename(oldpath, newpath string, overwrite bool) error -``` - -Rename moves a file from A to B - -#### func (\*Client) [SetHeader](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=1092:1137#L49) - -``` go -func (c *Client) SetHeader(key, value string) -``` - -SetHeader lets us set arbitrary headers for a given client - -#### func (\*Client) [SetInterceptor](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=1244:1326#L54) - -``` go -func (c *Client) SetInterceptor(interceptor func(method string, rq *http.Request)) -``` - -SetInterceptor lets us set an arbitrary interceptor for a given client - -#### func (\*Client) [SetJar](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=1727:1770#L69) - -``` go -func (c *Client) SetJar(jar http.CookieJar) -``` - -SetJar exposes the ability to set a cookie jar to the client. - -#### func (\*Client) [SetTimeout](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=1428:1478#L59) - -``` go -func (c *Client) SetTimeout(timeout time.Duration) -``` - -SetTimeout exposes the ability to set a time limit for requests - -#### func (\*Client) [SetTransport](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=1571:1629#L64) - -``` go -func (c *Client) SetTransport(transport http.RoundTripper) -``` - -SetTransport exposes the ability to define custom transports - -#### func (\*Client) [Stat](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=4241:4296#L184) - -``` go -func (c *Client) Stat(path string) (os.FileInfo, error) -``` - -Stat returns the file stats for a specified path - -#### func (\*Client) [Write](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=9272:9347#L389) - -``` go -func (c *Client) Write(path string, data []byte, _ os.FileMode) (err error) -``` - -Write writes data to a given path - -#### func (\*Client) [WriteStream](https://github.com/studio-b12/gowebdav/blob/master/client.go?s=9771:9857#L419) - -``` go -func (c *Client) WriteStream(path string, stream io.Reader, _ os.FileMode) (err error) -``` - -WriteStream writes a stream - -### type [DigestAuth](https://github.com/studio-b12/gowebdav/blob/master/digestAuth.go?s=157:254#L14) - -``` go -type DigestAuth struct { - // contains filtered or unexported fields -} - -``` - -DigestAuth structure holds our credentials - -#### func (\*DigestAuth) [Authorize](https://github.com/studio-b12/gowebdav/blob/master/digestAuth.go?s=525:608#L26) - -``` go -func (d *DigestAuth) Authorize(c *http.Client, rq *http.Request, path string) error -``` - -Authorize the current request - -#### func (\*DigestAuth) [Clone](https://github.com/studio-b12/gowebdav/blob/master/digestAuth.go?s=1228:1270#L49) - -``` go -func (d *DigestAuth) Clone() Authenticator -``` - -Clone creates a copy of itself - -#### func (\*DigestAuth) [Close](https://github.com/studio-b12/gowebdav/blob/master/digestAuth.go?s=1142:1176#L44) - -``` go -func (d *DigestAuth) Close() error -``` - -Close cleans up all resources - -#### func (\*DigestAuth) [String](https://github.com/studio-b12/gowebdav/blob/master/digestAuth.go?s=1466:1502#L58) - -``` go -func (d *DigestAuth) String() string -``` - -String toString - -#### func (\*DigestAuth) [Verify](https://github.com/studio-b12/gowebdav/blob/master/digestAuth.go?s=912:1010#L36) - -``` go -func (d *DigestAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) -``` - -Verify checks for authentication issues and may trigger a re-authentication - -### type [File](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=93:253#L10) - -``` go -type File struct { - // contains filtered or unexported fields -} - -``` - -File is our structure for a given file - -#### func (File) [ContentType](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=476:510#L31) - -``` go -func (f File) ContentType() string -``` - -ContentType returns the content type of a file - -#### func (File) [ETag](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=929:956#L56) - -``` go -func (f File) ETag() string -``` - -ETag returns the ETag of a file - -#### func (File) [IsDir](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=1035:1061#L61) - -``` go -func (f File) IsDir() bool -``` - -IsDir let us see if a given file is a directory or not - -#### func (File) [ModTime](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=836:869#L51) - -``` go -func (f File) ModTime() time.Time -``` - -ModTime returns the modified time of a file - -#### func (File) [Mode](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=665:697#L41) - -``` go -func (f File) Mode() os.FileMode -``` - -Mode will return the mode of a given file - -#### func (File) [Name](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=378:405#L26) - -``` go -func (f File) Name() string -``` - -Name returns the name of a file - -#### func (File) [Path](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=295:322#L21) - -``` go -func (f File) Path() string -``` - -Path returns the full path of a file - -#### func (File) [Size](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=573:599#L36) - -``` go -func (f File) Size() int64 -``` - -Size returns the size of a file - -#### func (File) [String](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=1183:1212#L71) - -``` go -func (f File) String() string -``` - -String lets us see file information - -#### func (File) [Sys](https://github.com/studio-b12/gowebdav/blob/master/file.go?s=1095:1126#L66) - -``` go -func (f File) Sys() interface{} -``` - -Sys ???? - -### type [PassportAuth](https://github.com/studio-b12/gowebdav/blob/master/passportAuth.go?s=125:254#L12) - -``` go -type PassportAuth struct { - // contains filtered or unexported fields -} - -``` - -PassportAuth structure holds our credentials - -#### func (\*PassportAuth) [Authorize](https://github.com/studio-b12/gowebdav/blob/master/passportAuth.go?s=690:775#L32) - -``` go -func (p *PassportAuth) Authorize(c *http.Client, rq *http.Request, path string) error -``` - -Authorize the current request - -#### func (\*PassportAuth) [Clone](https://github.com/studio-b12/gowebdav/blob/master/passportAuth.go?s=1701:1745#L69) - -``` go -func (p *PassportAuth) Clone() Authenticator -``` - -Clone creates a Copy of itself - -#### func (\*PassportAuth) [Close](https://github.com/studio-b12/gowebdav/blob/master/passportAuth.go?s=1613:1649#L64) - -``` go -func (p *PassportAuth) Close() error -``` - -Close cleans up all resources - -#### func (\*PassportAuth) [String](https://github.com/studio-b12/gowebdav/blob/master/passportAuth.go?s=2048:2086#L83) - -``` go -func (p *PassportAuth) String() string -``` - -String toString - -#### func (\*PassportAuth) [Verify](https://github.com/studio-b12/gowebdav/blob/master/passportAuth.go?s=1075:1175#L46) - -``` go -func (p *PassportAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) -``` - -Verify verifies if the authentication is good - -### type [StatusError](https://github.com/studio-b12/gowebdav/blob/master/errors.go?s=499:538#L18) - -``` go -type StatusError struct { - Status int -} - -``` - -StatusError implements error and wraps -an erroneous status code. - -#### func (StatusError) [Error](https://github.com/studio-b12/gowebdav/blob/master/errors.go?s=540:576#L22) - -``` go -func (se StatusError) Error() string -``` - -- - - -Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) diff --git a/util/sync/webdavClient/auth.go b/util/sync/webdavClient/auth.go deleted file mode 100644 index 32d761685..000000000 --- a/util/sync/webdavClient/auth.go +++ /dev/null @@ -1,409 +0,0 @@ -package webdavClient - -import ( - "bytes" - "errors" - "io" - "net/http" - "strings" - "sync" -) - -// AuthFactory prototype function to create a new Authenticator -type AuthFactory func(c *http.Client, rs *http.Response, path string) (auth Authenticator, err error) - -// Authorizer our Authenticator factory which creates an -// `Authenticator` per action/request. -type Authorizer interface { - // Creates a new Authenticator Shim per request. - // It may track request related states and perform payload buffering - // for authentication round trips. - // The underlying Authenticator will perform the real authentication. - NewAuthenticator(body io.Reader) (Authenticator, io.Reader) - // Registers a new Authenticator factory to a key. - AddAuthenticator(key string, fn AuthFactory) -} - -// A Authenticator implements a specific way to authorize requests. -// Each request is bound to a separate Authenticator instance. -// -// The authentication flow itself is broken down into `Authorize` -// and `Verify` steps. The former method runs before, and the latter -// runs after the `Request` is submitted. -// This makes it easy to encapsulate and control complex -// authentication challenges. -// -// Some authentication flows causing authentication round trips, -// which can be archived by returning the `redo` of the Verify -// method. `True` restarts the authentication process for the -// current action: A new `Request` is spawned, which must be -// authorized, sent, and re-verified again, until the action -// is successfully submitted. -// The preferred way is to handle the authentication ping-pong -// within `Verify`, and then `redo` with fresh credentials. -// -// The result of the `Verify` method can also trigger an -// `Authenticator` change by returning the `ErrAuthChanged` -// as an error. Depending on the `Authorizer` this may trigger -// an `Authenticator` negotiation. -// -// Set the `XInhibitRedirect` header to '1' in the `Authorize` -// method to get control over request redirection. -// Attention! You must handle the incoming request yourself. -// -// To store a shared session state the `Clone` method **must** -// return a new instance, initialized with the shared state. -type Authenticator interface { - // Authorizes a request. Usually by adding some authorization headers. - Authorize(c *http.Client, rq *http.Request, path string) error - // Verifies the response if the authorization was successful. - // May trigger some round trips to pass the authentication. - // May also trigger a new Authenticator negotiation by returning `ErrAuthChenged` - Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) - // Creates a copy of the underlying Authenticator. - Clone() Authenticator - io.Closer -} - -type authfactory struct { - key string - create AuthFactory -} - -// authorizer structure holds our Authenticator create functions -type authorizer struct { - factories []authfactory - defAuthMux sync.Mutex - defAuth Authenticator -} - -// preemptiveAuthorizer structure holds the preemptive Authenticator -type preemptiveAuthorizer struct { - auth Authenticator -} - -// authShim structure that wraps the real Authenticator -type authShim struct { - factory AuthFactory - body io.Reader - auth Authenticator -} - -// negoAuth structure holds the authenticators that are going to be negotiated -type negoAuth struct { - auths []Authenticator - setDefaultAuthenticator func(auth Authenticator) -} - -// nullAuth initializes the whole authentication flow -type nullAuth struct{} - -// noAuth structure to perform no authentication at all -type noAuth struct{} - -// NewAutoAuth creates an auto Authenticator factory. -// It negotiates the default authentication method -// based on the order of the registered Authenticators -// and the remotely offered authentication methods. -// First In, First Out. -func NewAutoAuth(login string, secret string) Authorizer { - fmap := make([]authfactory, 0) - az := &authorizer{factories: fmap, defAuthMux: sync.Mutex{}, defAuth: &nullAuth{}} - - az.AddAuthenticator("basic", func(c *http.Client, rs *http.Response, path string) (auth Authenticator, err error) { - return &BasicAuth{user: login, pw: secret}, nil - }) - - az.AddAuthenticator("digest", func(c *http.Client, rs *http.Response, path string) (auth Authenticator, err error) { - return NewDigestAuth(login, secret, rs) - }) - - az.AddAuthenticator("passport1.4", func(c *http.Client, rs *http.Response, path string) (auth Authenticator, err error) { - return NewPassportAuth(c, login, secret, rs.Request.URL.String(), &rs.Header) - }) - - return az -} - -// NewEmptyAuth creates an empty Authenticator factory -// The order of adding the Authenticator matters. -// First In, First Out. -// It offers the `NewAutoAuth` features. -func NewEmptyAuth() Authorizer { - fmap := make([]authfactory, 0) - az := &authorizer{factories: fmap, defAuthMux: sync.Mutex{}, defAuth: &nullAuth{}} - return az -} - -// NewPreemptiveAuth creates a preemptive Authenticator -// The preemptive authorizer uses the provided Authenticator -// for every request regardless of any `Www-Authenticate` header. -// -// It may only have one authentication method, -// so calling `AddAuthenticator` **will panic**! -// -// Look out!! This offers the skinniest and slickest implementation -// without any synchronisation!! -// Still applicable with `BasicAuth` within go routines. -func NewPreemptiveAuth(auth Authenticator) Authorizer { - return &preemptiveAuthorizer{auth: auth} -} - -// NewAuthenticator creates an Authenticator (Shim) per request -func (a *authorizer) NewAuthenticator(body io.Reader) (Authenticator, io.Reader) { - var retryBuf = body - if body != nil { - // If the authorization fails, we will need to restart reading - // from the passed body stream. - // When body is seekable, use seek to reset the streams - // cursor to the start. - // Otherwise, copy the stream into a buffer while uploading - // and use the buffers content on retry. - if _, ok := retryBuf.(io.Seeker); ok { - body = io.NopCloser(body) - } else { - buff := &bytes.Buffer{} - retryBuf = buff - body = io.TeeReader(body, buff) - } - } - a.defAuthMux.Lock() - defAuth := a.defAuth.Clone() - a.defAuthMux.Unlock() - - return &authShim{factory: a.factory, body: retryBuf, auth: defAuth}, body -} - -// AddAuthenticator appends the AuthFactory to our factories. -// It converts the key to lower case and preserves the order. -func (a *authorizer) AddAuthenticator(key string, fn AuthFactory) { - key = strings.ToLower(key) - for _, f := range a.factories { - if f.key == key { - panic("Authenticator exists: " + key) - } - } - a.factories = append(a.factories, authfactory{key, fn}) -} - -// factory picks all valid Authenticators based on Www-Authenticate headers -func (a *authorizer) factory(c *http.Client, rs *http.Response, path string) (auth Authenticator, err error) { - headers := rs.Header.Values("Www-Authenticate") - if len(headers) > 0 { - auths := make([]Authenticator, 0) - for _, f := range a.factories { - for _, header := range headers { - headerLower := strings.ToLower(header) - if strings.Contains(headerLower, f.key) { - rs.Header.Set("Www-Authenticate", header) - if auth, err = f.create(c, rs, path); err == nil { - auths = append(auths, auth) - break - } - } - } - } - - switch len(auths) { - case 0: - return nil, NewPathError("NoAuthenticator", path, rs.StatusCode) - case 1: - auth = auths[0] - default: - auth = &negoAuth{auths: auths, setDefaultAuthenticator: a.setDefaultAuthenticator} - } - } else { - auth = &noAuth{} - } - - a.setDefaultAuthenticator(auth) - - return auth, nil -} - -// setDefaultAuthenticator sets the default Authenticator -func (a *authorizer) setDefaultAuthenticator(auth Authenticator) { - a.defAuthMux.Lock() - a.defAuth.Close() - a.defAuth = auth - a.defAuthMux.Unlock() -} - -// Authorize the current request -func (s *authShim) Authorize(c *http.Client, rq *http.Request, path string) error { - if err := s.auth.Authorize(c, rq, path); err != nil { - return err - } - body := s.body - rq.GetBody = func() (io.ReadCloser, error) { - if body != nil { - if sk, ok := body.(io.Seeker); ok { - if _, err := sk.Seek(0, io.SeekStart); err != nil { - return nil, err - } - } - return io.NopCloser(body), nil - } - return nil, nil - } - return nil -} - -// Verify checks for authentication issues and may trigger a re-authentication. -// Catches AlgoChangedErr to update the current Authenticator -func (s *authShim) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) { - redo, err = s.auth.Verify(c, rs, path) - if err != nil && errors.Is(err, ErrAuthChanged) { - if auth, aerr := s.factory(c, rs, path); aerr == nil { - s.auth.Close() - s.auth = auth - return true, nil - } else { - return false, aerr - } - } - return -} - -// Close closes all resources -func (s *authShim) Close() error { - s.auth.Close() - s.auth, s.factory = nil, nil - if s.body != nil { - if closer, ok := s.body.(io.Closer); ok { - return closer.Close() - } - } - return nil -} - -// It's not intend to Clone the shim -// therefore it returns a noAuth instance -func (s *authShim) Clone() Authenticator { - return &noAuth{} -} - -// String toString -func (s *authShim) String() string { - return "AuthShim" -} - -// Authorize authorizes the current request with the top most Authorizer -func (n *negoAuth) Authorize(c *http.Client, rq *http.Request, path string) error { - if len(n.auths) == 0 { - return NewPathError("NoAuthenticator", path, 400) - } - return n.auths[0].Authorize(c, rq, path) -} - -// Verify verifies the authentication and selects the next one based on the result -func (n *negoAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) { - if len(n.auths) == 0 { - return false, NewPathError("NoAuthenticator", path, 400) - } - redo, err = n.auths[0].Verify(c, rs, path) - if err != nil { - if len(n.auths) > 1 { - n.auths[0].Close() - n.auths = n.auths[1:] - return true, nil - } - } else if redo { - return - } else { - auth := n.auths[0] - n.auths = n.auths[1:] - n.setDefaultAuthenticator(auth) - return - } - - return false, NewPathError("NoAuthenticator", path, rs.StatusCode) -} - -// Close will close the underlying authenticators. -func (n *negoAuth) Close() error { - for _, a := range n.auths { - a.Close() - } - n.setDefaultAuthenticator = nil - return nil -} - -// Clone clones the underlying authenticators. -func (n *negoAuth) Clone() Authenticator { - auths := make([]Authenticator, len(n.auths)) - for i, e := range n.auths { - auths[i] = e.Clone() - } - return &negoAuth{auths: auths, setDefaultAuthenticator: n.setDefaultAuthenticator} -} - -func (n *negoAuth) String() string { - return "NegoAuth" -} - -// Authorize the current request -func (n *noAuth) Authorize(c *http.Client, rq *http.Request, path string) error { - return nil -} - -// Verify checks for authentication issues and may trigger a re-authentication -func (n *noAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) { - if "" != rs.Header.Get("Www-Authenticate") { - err = ErrAuthChanged - } - return -} - -// Close closes all resources -func (n *noAuth) Close() error { - return nil -} - -// Clone creates a copy of itself -func (n *noAuth) Clone() Authenticator { - // no copy due to read only access - return n -} - -// String toString -func (n *noAuth) String() string { - return "NoAuth" -} - -// Authorize the current request -func (n *nullAuth) Authorize(c *http.Client, rq *http.Request, path string) error { - rq.Header.Set(XInhibitRedirect, "1") - return nil -} - -// Verify checks for authentication issues and may trigger a re-authentication -func (n *nullAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) { - return true, ErrAuthChanged -} - -// Close closes all resources -func (n *nullAuth) Close() error { - return nil -} - -// Clone creates a copy of itself -func (n *nullAuth) Clone() Authenticator { - // no copy due to read only access - return n -} - -// String toString -func (n *nullAuth) String() string { - return "NullAuth" -} - -// NewAuthenticator creates an Authenticator (Shim) per request -func (b *preemptiveAuthorizer) NewAuthenticator(body io.Reader) (Authenticator, io.Reader) { - return b.auth.Clone(), body -} - -// AddAuthenticator Will PANIC because it may only have a single authentication method -func (b *preemptiveAuthorizer) AddAuthenticator(key string, fn AuthFactory) { - panic("You're funny! A preemptive authorizer may only have a single authentication method") -} diff --git a/util/sync/webdavClient/auth_test.go b/util/sync/webdavClient/auth_test.go deleted file mode 100644 index d12c0f0d2..000000000 --- a/util/sync/webdavClient/auth_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package webdavClient - -import ( - "bytes" - "net/http" - "strings" - "testing" -) - -func TestEmptyAuth(t *testing.T) { - auth := NewEmptyAuth() - srv, _, _ := newAuthSrv(t, basicAuth) - defer srv.Close() - cli := NewAuthClient(srv.URL, auth) - if err := cli.Connect(); err == nil { - t.Fatalf("got nil want error") - } -} - -func TestRedirectAuthWIP(t *testing.T) { - hasPassedAuthServer := false - authHandler := func(h http.Handler) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if user, passwd, ok := r.BasicAuth(); ok { - if user == "user" && passwd == "password" { - hasPassedAuthServer = true - w.WriteHeader(200) - return - } - } - w.Header().Set("Www-Authenticate", `Basic realm="x"`) - w.WriteHeader(401) - } - } - - psrv, _, _ := newAuthSrv(t, authHandler) - defer psrv.Close() - - dataHandler := func(h http.Handler) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - hasAuth := strings.Contains(r.Header.Get("Authorization"), "Basic dXNlcjpwYXNzd29yZA==") - - if hasPassedAuthServer && hasAuth { - h.ServeHTTP(w, r) - return - } - w.Header().Set("Www-Authenticate", `Basic realm="x"`) - http.Redirect(w, r, psrv.URL+"/", 302) - } - } - - srv, _, _ := newAuthSrv(t, dataHandler) - defer srv.Close() - cli := NewClient(srv.URL, "user", "password") - data, err := cli.Read("/hello.txt") - if err != nil { - t.Logf("WIP got error=%v; want nil", err) - } - if bytes.Compare(data, []byte("hello gowebdav\n")) != 0 { - t.Logf("WIP got data=%v; want=hello gowebdav", data) - } -} diff --git a/util/sync/webdavClient/basicAuth.go b/util/sync/webdavClient/basicAuth.go deleted file mode 100644 index 10e5b8f4c..000000000 --- a/util/sync/webdavClient/basicAuth.go +++ /dev/null @@ -1,42 +0,0 @@ -package webdavClient - -import ( - "fmt" - "net/http" -) - -// BasicAuth structure holds our credentials -type BasicAuth struct { - user string - pw string -} - -// Authorize the current request -func (b *BasicAuth) Authorize(c *http.Client, rq *http.Request, path string) error { - rq.SetBasicAuth(b.user, b.pw) - return nil -} - -// Verify verifies if the authentication -func (b *BasicAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) { - if rs.StatusCode == 401 { - err = NewPathError("Authorize", path, rs.StatusCode) - } - return -} - -// Close cleans up all resources -func (b *BasicAuth) Close() error { - return nil -} - -// Clone creates a Copy of itself -func (b *BasicAuth) Clone() Authenticator { - // no copy due to read only access - return b -} - -// String toString -func (b *BasicAuth) String() string { - return fmt.Sprintf("BasicAuth login: %s", b.user) -} diff --git a/util/sync/webdavClient/basicAuth_test.go b/util/sync/webdavClient/basicAuth_test.go deleted file mode 100644 index 3a62713af..000000000 --- a/util/sync/webdavClient/basicAuth_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package webdavClient - -import ( - "net/http" - "testing" -) - -func TestNewBasicAuth(t *testing.T) { - a := &BasicAuth{user: "user", pw: "password"} - - ex := "BasicAuth login: user" - if a.String() != ex { - t.Error("expected: " + ex + " got: " + a.String()) - } - - if a.Clone() != a { - t.Error("expected the same instance") - } - - if a.Close() != nil { - t.Error("expected close without errors") - } -} - -func TestBasicAuthAuthorize(t *testing.T) { - a := &BasicAuth{user: "user", pw: "password"} - rq, _ := http.NewRequest("GET", "http://localhost/", nil) - a.Authorize(nil, rq, "/") - if rq.Header.Get("Authorization") != "Basic dXNlcjpwYXNzd29yZA==" { - t.Error("got wrong Authorization header: " + rq.Header.Get("Authorization")) - } -} - -func TestPreemtiveBasicAuth(t *testing.T) { - a := &BasicAuth{user: "user", pw: "password"} - auth := NewPreemptiveAuth(a) - n, b := auth.NewAuthenticator(nil) - if b != nil { - t.Error("expected body to be nil") - } - if n != a { - t.Error("expected the same instance") - } - - srv, _, _ := newAuthSrv(t, basicAuth) - defer srv.Close() - cli := NewAuthClient(srv.URL, auth) - if err := cli.Connect(); err != nil { - t.Fatalf("got error: %v, want nil", err) - } -} diff --git a/util/sync/webdavClient/client.go b/util/sync/webdavClient/client.go deleted file mode 100644 index 7d45b30e2..000000000 --- a/util/sync/webdavClient/client.go +++ /dev/null @@ -1,474 +0,0 @@ -package webdavClient - -import ( - "bytes" - "context" - "encoding/xml" - "fmt" - "github.com/openziti/zrok/environment/env_core" - "github.com/openziti/zrok/sdk/golang/sdk" - "io" - "net" - "net/http" - "net/url" - "os" - pathpkg "path" - "strings" - "time" -) - -const XInhibitRedirect = "X-Gowebdav-Inhibit-Redirect" - -// Client defines our structure -type Client struct { - root string - headers http.Header - interceptor func(method string, rq *http.Request) - c *http.Client - auth Authorizer -} - -// NewClient creates a new instance of client -func NewClient(uri, user, pw string) *Client { - return NewAuthClient(uri, NewAutoAuth(user, pw)) -} - -func NewZrokClient(zrokUrl *url.URL, root env_core.Root, auth Authorizer) (*Client, error) { - conn, err := sdk.NewDialer(zrokUrl.Host, root) - if err != nil { - return nil, err - } - transport := http.DefaultTransport.(*http.Transport).Clone() - transport.DialContext = func(_ context.Context, _, _ string) (net.Conn, error) { - return conn, nil - } - c := &http.Client{ - Transport: transport, - CheckRedirect: func(rq *http.Request, via []*http.Request) error { - if len(via) >= 10 { - return ErrTooManyRedirects - } - if via[0].Header.Get(XInhibitRedirect) != "" { - return http.ErrUseLastResponse - } - return nil - }, - } - httpUrl, err := url.Parse(zrokUrl.String()) - if err != nil { - return nil, err - } - httpUrl.Scheme = "http" - return &Client{root: FixSlash(httpUrl.String()), headers: make(http.Header), interceptor: nil, c: c, auth: auth}, nil -} - -// NewAuthClient creates a new client instance with a custom Authorizer -func NewAuthClient(uri string, auth Authorizer) *Client { - c := &http.Client{ - CheckRedirect: func(rq *http.Request, via []*http.Request) error { - if len(via) >= 10 { - return ErrTooManyRedirects - } - if via[0].Header.Get(XInhibitRedirect) != "" { - return http.ErrUseLastResponse - } - return nil - }, - } - return &Client{root: FixSlash(uri), headers: make(http.Header), interceptor: nil, c: c, auth: auth} -} - -// SetHeader lets us set arbitrary headers for a given client -func (c *Client) SetHeader(key, value string) { - c.headers.Set(key, value) -} - -// SetInterceptor lets us set an arbitrary interceptor for a given client -func (c *Client) SetInterceptor(interceptor func(method string, rq *http.Request)) { - c.interceptor = interceptor -} - -// SetTimeout exposes the ability to set a time limit for requests -func (c *Client) SetTimeout(timeout time.Duration) { - c.c.Timeout = timeout -} - -// SetTransport exposes the ability to define custom transports -func (c *Client) SetTransport(transport http.RoundTripper) { - c.c.Transport = transport -} - -// SetJar exposes the ability to set a cookie jar to the client. -func (c *Client) SetJar(jar http.CookieJar) { - c.c.Jar = jar -} - -// Connect connects to our dav server -func (c *Client) Connect() error { - rs, err := c.options("/") - if err != nil { - return err - } - - err = rs.Body.Close() - if err != nil { - return err - } - - if rs.StatusCode != 200 { - return NewPathError("Connect", c.root, rs.StatusCode) - } - - return nil -} - -type props struct { - Status string `xml:"DAV: status"` - Name string `xml:"DAV: prop>displayname,omitempty"` - Type xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"` - Size string `xml:"DAV: prop>getcontentlength,omitempty"` - ContentType string `xml:"DAV: prop>getcontenttype,omitempty"` - ETag string `xml:"DAV: prop>getetag,omitempty"` - Modified string `xml:"DAV: prop>getlastmodified,omitempty"` - Checksum string `xml:"zrok: prop>checksum,omitempty"` -} - -type Response struct { - Href string `xml:"DAV: href"` - Props []props `xml:"DAV: propstat"` -} - -func getProps(r *Response, status string) *props { - for _, prop := range r.Props { - if strings.Contains(prop.Status, status) { - return &prop - } - } - return nil -} - -// ReadDir reads the contents of a remote directory -func (c *Client) ReadDir(path string) ([]os.FileInfo, error) { - path = FixSlashes(path) - files := make([]os.FileInfo, 0) - skipSelf := true - parse := func(resp interface{}) error { - r := resp.(*Response) - - if skipSelf { - skipSelf = false - if p := getProps(r, "200"); p != nil && p.Type.Local == "collection" { - r.Props = nil - return nil - } - return NewPathError("ReadDir", path, 405) - } - - if p := getProps(r, "200"); p != nil { - f := new(File) - if ps, err := url.PathUnescape(r.Href); err == nil { - f.name = pathpkg.Base(ps) - } else { - f.name = p.Name - } - f.path = path + f.name - f.modified = parseModified(&p.Modified) - f.etag = p.ETag - f.contentType = p.ContentType - - if p.Type.Local == "collection" { - f.path += "/" - f.size = 0 - f.isdir = true - } else { - f.size = parseInt64(&p.Size) - f.isdir = false - } - - files = append(files, *f) - } - - r.Props = nil - return nil - } - - err := c.propfind(path, false, - ` - - - - - - - - - - - `, - &Response{}, - parse) - - if err != nil { - if _, ok := err.(*os.PathError); !ok { - err = NewPathErrorErr("ReadDir", path, err) - } - } - return files, err -} - -// Stat returns the file stats for a specified path -func (c *Client) Stat(path string) (os.FileInfo, error) { - var f *File - parse := func(resp interface{}) error { - r := resp.(*Response) - if p := getProps(r, "200"); p != nil && f == nil { - f = new(File) - f.name = p.Name - f.path = path - f.etag = p.ETag - f.contentType = p.ContentType - - if p.Type.Local == "collection" { - if !strings.HasSuffix(f.path, "/") { - f.path += "/" - } - f.size = 0 - f.modified = parseModified(&p.Modified) - f.isdir = true - } else { - f.size = parseInt64(&p.Size) - f.modified = parseModified(&p.Modified) - f.isdir = false - } - } - - r.Props = nil - return nil - } - - err := c.propfind(path, true, - ` - - - - - - - - - - `, - &Response{}, - parse) - - if err != nil { - if _, ok := err.(*os.PathError); !ok { - err = NewPathErrorErr("ReadDir", path, err) - } - } - return f, err -} - -// Remove removes a remote file -func (c *Client) Remove(path string) error { - return c.RemoveAll(path) -} - -// RemoveAll removes remote files -func (c *Client) RemoveAll(path string) error { - rs, err := c.req("DELETE", path, nil, nil) - if err != nil { - return NewPathError("Remove", path, 400) - } - err = rs.Body.Close() - if err != nil { - return err - } - - if rs.StatusCode == 200 || rs.StatusCode == 204 || rs.StatusCode == 404 { - return nil - } - - return NewPathError("Remove", path, rs.StatusCode) -} - -// Mkdir makes a directory -func (c *Client) Mkdir(path string, _ os.FileMode) (err error) { - path = FixSlashes(path) - status, err := c.mkcol(path) - if err != nil { - return - } - if status == 201 { - return nil - } - - return NewPathError("Mkdir", path, status) -} - -// MkdirAll like mkdir -p, but for webdav -func (c *Client) MkdirAll(path string, _ os.FileMode) (err error) { - path = FixSlashes(path) - status, err := c.mkcol(path) - if err != nil { - return - } - if status == 201 { - return nil - } - if status == 409 { - paths := strings.Split(path, "/") - sub := "/" - for _, e := range paths { - if e == "" { - continue - } - sub += e + "/" - status, err = c.mkcol(sub) - if err != nil { - return - } - if status != 201 { - return NewPathError("MkdirAll", sub, status) - } - } - return nil - } - - return NewPathError("MkdirAll", path, status) -} - -// Rename moves a file from A to B -func (c *Client) Rename(oldpath, newpath string, overwrite bool) error { - return c.copymove("MOVE", oldpath, newpath, overwrite) -} - -// Copy copies a file from A to B -func (c *Client) Copy(oldpath, newpath string, overwrite bool) error { - return c.copymove("COPY", oldpath, newpath, overwrite) -} - -// Read reads the contents of a remote file -func (c *Client) Read(path string) ([]byte, error) { - var stream io.ReadCloser - var err error - - if stream, err = c.ReadStream(path); err != nil { - return nil, err - } - defer stream.Close() - - buf := new(bytes.Buffer) - _, err = buf.ReadFrom(stream) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// ReadStream reads the stream for a given path -func (c *Client) ReadStream(path string) (io.ReadCloser, error) { - rs, err := c.req("GET", path, nil, nil) - if err != nil { - return nil, NewPathErrorErr("ReadStream", path, err) - } - - if rs.StatusCode == 200 { - return rs.Body, nil - } - - rs.Body.Close() - return nil, NewPathError("ReadStream", path, rs.StatusCode) -} - -// ReadStreamRange reads the stream representing a subset of bytes for a given path, -// utilizing HTTP Range Requests if the server supports it. -// The range is expressed as offset from the start of the file and length, for example -// offset=10, length=10 will return bytes 10 through 19. -// -// If the server does not support partial content requests and returns full content instead, -// this function will emulate the behavior by skipping `offset` bytes and limiting the result -// to `length`. -func (c *Client) ReadStreamRange(path string, offset, length int64) (io.ReadCloser, error) { - rs, err := c.req("GET", path, nil, func(r *http.Request) { - if length > 0 { - r.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)) - } else { - r.Header.Add("Range", fmt.Sprintf("bytes=%d-", offset)) - } - }) - if err != nil { - return nil, NewPathErrorErr("ReadStreamRange", path, err) - } - - if rs.StatusCode == http.StatusPartialContent { - // server supported partial content, return as-is. - return rs.Body, nil - } - - // server returned success, but did not support partial content, so we have the whole - // stream in rs.Body - if rs.StatusCode == 200 { - // discard first 'offset' bytes. - if _, err := io.Copy(io.Discard, io.LimitReader(rs.Body, offset)); err != nil { - return nil, NewPathErrorErr("ReadStreamRange", path, err) - } - - // return a io.ReadCloser that is limited to `length` bytes. - return &limitedReadCloser{rc: rs.Body, remaining: int(length)}, nil - } - - rs.Body.Close() - return nil, NewPathError("ReadStream", path, rs.StatusCode) -} - -// Write writes data to a given path -func (c *Client) Write(path string, data []byte, _ os.FileMode) (err error) { - s, err := c.put(path, bytes.NewReader(data)) - if err != nil { - return - } - - switch s { - - case 200, 201, 204: - return nil - - case 404, 409: - err = c.createParentCollection(path) - if err != nil { - return - } - - s, err = c.put(path, bytes.NewReader(data)) - if err != nil { - return - } - if s == 200 || s == 201 || s == 204 { - return - } - } - - return NewPathError("Write", path, s) -} - -// WriteStream writes a stream -func (c *Client) WriteStream(path string, stream io.Reader, _ os.FileMode) (err error) { - err = c.createParentCollection(path) - if err != nil { - return err - } - - s, err := c.put(path, stream) - if err != nil { - return err - } - - switch s { - case 200, 201, 204: - return nil - - default: - return NewPathError("WriteStream", path, s) - } -} diff --git a/util/sync/webdavClient/client_test.go b/util/sync/webdavClient/client_test.go deleted file mode 100644 index 8d60477a8..000000000 --- a/util/sync/webdavClient/client_test.go +++ /dev/null @@ -1,574 +0,0 @@ -package webdavClient - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "net/http" - "net/http/httptest" - "os" - "strings" - "sync" - "testing" - "time" - - "golang.org/x/net/webdav" -) - -func noAuthHndl(h http.Handler) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - h.ServeHTTP(w, r) - } -} - -func basicAuth(h http.Handler) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if user, passwd, ok := r.BasicAuth(); ok { - if user == "user" && passwd == "password" { - h.ServeHTTP(w, r) - return - } - - http.Error(w, "not authorized", 403) - } else { - w.Header().Set("WWW-Authenticate", `Basic realm="x"`) - w.WriteHeader(401) - } - } -} - -func multipleAuth(h http.Handler) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - notAuthed := false - if r.Header.Get("Authorization") == "" { - notAuthed = true - } else if user, passwd, ok := r.BasicAuth(); ok { - if user == "user" && passwd == "password" { - h.ServeHTTP(w, r) - return - } - notAuthed = true - } else if strings.HasPrefix(r.Header.Get("Authorization"), "Digest ") { - pairs := strings.TrimPrefix(r.Header.Get("Authorization"), "Digest ") - digestParts := make(map[string]string) - for _, pair := range strings.Split(pairs, ",") { - kv := strings.SplitN(strings.TrimSpace(pair), "=", 2) - key, value := kv[0], kv[1] - value = strings.Trim(value, `"`) - digestParts[key] = value - } - if digestParts["qop"] == "" { - digestParts["qop"] = "auth" - } - - ha1 := getMD5(fmt.Sprint(digestParts["username"], ":", digestParts["realm"], ":", "digestPW")) - ha2 := getMD5(fmt.Sprint(r.Method, ":", digestParts["uri"])) - expected := getMD5(fmt.Sprint(ha1, - ":", digestParts["nonce"], - ":", digestParts["nc"], - ":", digestParts["cnonce"], - ":", digestParts["qop"], - ":", ha2)) - - if expected == digestParts["response"] { - h.ServeHTTP(w, r) - return - } - notAuthed = true - } - - if notAuthed { - w.Header().Add("WWW-Authenticate", `Digest realm="testrealm@host.com", qop="auth,auth-int",nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093",opaque="5ccc069c403ebaf9f0171e9517f40e41"`) - w.Header().Add("WWW-Authenticate", `Basic realm="x"`) - w.WriteHeader(401) - } - } -} - -func fillFs(t *testing.T, fs webdav.FileSystem) context.Context { - ctx := context.Background() - f, err := fs.OpenFile(ctx, "hello.txt", os.O_CREATE, 0644) - if err != nil { - t.Errorf("fail to crate file: %v", err) - } - f.Write([]byte("hello gowebdav\n")) - f.Close() - err = fs.Mkdir(ctx, "/test", 0755) - if err != nil { - t.Errorf("fail to crate directory: %v", err) - } - f, err = fs.OpenFile(ctx, "/test/test.txt", os.O_CREATE, 0644) - if err != nil { - t.Errorf("fail to crate file: %v", err) - } - f.Write([]byte("test test gowebdav\n")) - f.Close() - return ctx -} - -func newServer(t *testing.T) (*Client, *httptest.Server, webdav.FileSystem, context.Context) { - return newAuthServer(t, basicAuth) -} - -func newAuthServer(t *testing.T, auth func(h http.Handler) http.HandlerFunc) (*Client, *httptest.Server, webdav.FileSystem, context.Context) { - srv, fs, ctx := newAuthSrv(t, auth) - cli := NewClient(srv.URL, "user", "password") - return cli, srv, fs, ctx -} - -func newAuthSrv(t *testing.T, auth func(h http.Handler) http.HandlerFunc) (*httptest.Server, webdav.FileSystem, context.Context) { - mux := http.NewServeMux() - fs := webdav.NewMemFS() - ctx := fillFs(t, fs) - mux.HandleFunc("/", auth(&webdav.Handler{ - FileSystem: fs, - LockSystem: webdav.NewMemLS(), - })) - srv := httptest.NewServer(mux) - return srv, fs, ctx -} - -func TestConnect(t *testing.T) { - cli, srv, _, _ := newServer(t) - defer srv.Close() - if err := cli.Connect(); err != nil { - t.Fatalf("got error: %v, want nil", err) - } - - cli = NewClient(srv.URL, "no", "no") - if err := cli.Connect(); err == nil { - t.Fatalf("got nil, want error: %v", err) - } -} - -func TestConnectMultipleAuth(t *testing.T) { - cli, srv, _, _ := newAuthServer(t, multipleAuth) - defer srv.Close() - if err := cli.Connect(); err != nil { - t.Fatalf("got error: %v, want nil", err) - } - - cli = NewClient(srv.URL, "digestUser", "digestPW") - if err := cli.Connect(); err != nil { - t.Fatalf("got nil, want error: %v", err) - } - - cli = NewClient(srv.URL, "no", "no") - if err := cli.Connect(); err == nil { - t.Fatalf("got nil, want error: %v", err) - } -} - -func TestConnectMultiAuthII(t *testing.T) { - cli, srv, _, _ := newAuthServer(t, func(h http.Handler) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if user, passwd, ok := r.BasicAuth(); ok { - if user == "user" && passwd == "password" { - h.ServeHTTP(w, r) - return - } - - http.Error(w, "not authorized", 403) - } else { - w.Header().Add("WWW-Authenticate", `FooAuth`) - w.Header().Add("WWW-Authenticate", `BazAuth`) - w.Header().Add("WWW-Authenticate", `BarAuth`) - w.Header().Add("WWW-Authenticate", `Basic realm="x"`) - w.WriteHeader(401) - } - } - }) - defer srv.Close() - if err := cli.Connect(); err != nil { - t.Fatalf("got error: %v, want nil", err) - } - - cli = NewClient(srv.URL, "no", "no") - if err := cli.Connect(); err == nil { - t.Fatalf("got nil, want error: %v", err) - } -} - -func TestReadDirConcurrent(t *testing.T) { - cli, srv, _, _ := newServer(t) - defer srv.Close() - - var wg sync.WaitGroup - errs := make(chan error, 2) - for i := 0; i < 2; i++ { - wg.Add(1) - - go func() { - defer wg.Done() - f, err := cli.ReadDir("/") - if err != nil { - errs <- errors.New(fmt.Sprintf("got error: %v, want file listing: %v", err, f)) - } - if len(f) != 2 { - errs <- errors.New(fmt.Sprintf("f: %v err: %v", f, err)) - } - if f[0].Name() != "hello.txt" && f[1].Name() != "hello.txt" { - errs <- errors.New(fmt.Sprintf("got: %v, want file: %s", f, "hello.txt")) - } - if f[0].Name() != "test" && f[1].Name() != "test" { - errs <- errors.New(fmt.Sprintf("got: %v, want directory: %s", f, "test")) - } - }() - } - - wg.Wait() - close(errs) - - for err := range errs { - if err != nil { - t.Fatal(err) - } - } -} - -func TestRead(t *testing.T) { - cli, srv, _, _ := newServer(t) - defer srv.Close() - - data, err := cli.Read("/hello.txt") - if err != nil || bytes.Compare(data, []byte("hello gowebdav\n")) != 0 { - t.Fatalf("got: %v, want data: %s", err, []byte("hello gowebdav\n")) - } - - data, err = cli.Read("/404.txt") - if err == nil { - t.Fatalf("got: %v, want error: %v", data, err) - } - if !IsErrNotFound(err) { - t.Fatalf("got: %v, want 404 error", err) - } -} - -func TestReadNoAuth(t *testing.T) { - cli, srv, _, _ := newAuthServer(t, noAuthHndl) - defer srv.Close() - - data, err := cli.Read("/hello.txt") - if err != nil || bytes.Compare(data, []byte("hello gowebdav\n")) != 0 { - t.Fatalf("got: %v, want data: %s", err, []byte("hello gowebdav\n")) - } - - data, err = cli.Read("/404.txt") - if err == nil { - t.Fatalf("got: %v, want error: %v", data, err) - } - if !IsErrNotFound(err) { - t.Fatalf("got: %v, want 404 error", err) - } -} - -func TestReadStream(t *testing.T) { - cli, srv, _, _ := newServer(t) - defer srv.Close() - - stream, err := cli.ReadStream("/hello.txt") - if err != nil { - t.Fatalf("got: %v, want data: %v", err, stream) - } - buf := new(bytes.Buffer) - buf.ReadFrom(stream) - if buf.String() != "hello gowebdav\n" { - t.Fatalf("got: %v, want stream: hello gowebdav", buf.String()) - } - - stream, err = cli.ReadStream("/404/hello.txt") - if err == nil { - t.Fatalf("got: %v, want error: %v", stream, err) - } -} - -func TestReadStreamRange(t *testing.T) { - cli, srv, _, _ := newServer(t) - defer srv.Close() - - stream, err := cli.ReadStreamRange("/hello.txt", 4, 4) - if err != nil { - t.Fatalf("got: %v, want data: %v", err, stream) - } - buf := new(bytes.Buffer) - buf.ReadFrom(stream) - if buf.String() != "o go" { - t.Fatalf("got: %v, want stream: o go", buf.String()) - } - - stream, err = cli.ReadStream("/404/hello.txt") - if err == nil { - t.Fatalf("got: %v, want error: %v", stream, err) - } -} - -func TestReadStreamRangeUnkownLength(t *testing.T) { - cli, srv, _, _ := newServer(t) - defer srv.Close() - - stream, err := cli.ReadStreamRange("/hello.txt", 6, 0) - if err != nil { - t.Fatalf("got: %v, want data: %v", err, stream) - } - buf := new(bytes.Buffer) - buf.ReadFrom(stream) - if buf.String() != "gowebdav\n" { - t.Fatalf("got: %v, want stream: gowebdav\n", buf.String()) - } - - stream, err = cli.ReadStream("/404/hello.txt") - if err == nil { - t.Fatalf("got: %v, want error: %v", stream, err) - } -} - -func TestStat(t *testing.T) { - cli, srv, _, _ := newServer(t) - defer srv.Close() - - info, err := cli.Stat("/hello.txt") - if err != nil { - t.Fatalf("got: %v, want os.Info: %v", err, info) - } - if info.Name() != "hello.txt" { - t.Fatalf("got: %v, want file hello.txt", info) - } - - info, err = cli.Stat("/404.txt") - if err == nil { - t.Fatalf("got: %v, want error: %v", info, err) - } - if !IsErrNotFound(err) { - t.Fatalf("got: %v, want 404 error", err) - } -} - -func TestMkdir(t *testing.T) { - cli, srv, fs, ctx := newServer(t) - defer srv.Close() - - info, err := cli.Stat("/newdir") - if err == nil { - t.Fatalf("got: %v, want error: %v", info, err) - } - - if err := cli.Mkdir("/newdir", 0755); err != nil { - t.Fatalf("got: %v, want mkdir /newdir", err) - } - - if err := cli.Mkdir("/newdir", 0755); err != nil { - t.Fatalf("got: %v, want mkdir /newdir", err) - } - - info, err = fs.Stat(ctx, "/newdir") - if err != nil { - t.Fatalf("got: %v, want dir info: %v", err, info) - } - - if err := cli.Mkdir("/404/newdir", 0755); err == nil { - t.Fatalf("expected Mkdir error due to missing parent directory") - } -} - -func TestMkdirAll(t *testing.T) { - cli, srv, fs, ctx := newServer(t) - defer srv.Close() - - if err := cli.MkdirAll("/dir/dir/dir", 0755); err != nil { - t.Fatalf("got: %v, want mkdirAll /dir/dir/dir", err) - } - - info, err := fs.Stat(ctx, "/dir/dir/dir") - if err != nil { - t.Fatalf("got: %v, want dir info: %v", err, info) - } -} - -func TestCopy(t *testing.T) { - cli, srv, fs, ctx := newServer(t) - defer srv.Close() - - info, err := fs.Stat(ctx, "/copy.txt") - if err == nil { - t.Fatalf("got: %v, want error: %v", info, err) - } - - if err := cli.Copy("/hello.txt", "/copy.txt", false); err != nil { - t.Fatalf("got: %v, want copy /hello.txt to /copy.txt", err) - } - - info, err = fs.Stat(ctx, "/copy.txt") - if err != nil { - t.Fatalf("got: %v, want file info: %v", err, info) - } - if info.Size() != 15 { - t.Fatalf("got: %v, want file size: %d bytes", info.Size(), 15) - } - - info, err = fs.Stat(ctx, "/hello.txt") - if err != nil { - t.Fatalf("got: %v, want file info: %v", err, info) - } - if info.Size() != 15 { - t.Fatalf("got: %v, want file size: %d bytes", info.Size(), 15) - } - - if err := cli.Copy("/hello.txt", "/copy.txt", false); err == nil { - t.Fatalf("expected copy error due to overwrite false") - } - - if err := cli.Copy("/hello.txt", "/copy.txt", true); err != nil { - t.Fatalf("got: %v, want overwrite /copy.txt with /hello.txt", err) - } -} - -func TestRename(t *testing.T) { - cli, srv, fs, ctx := newServer(t) - defer srv.Close() - - info, err := fs.Stat(ctx, "/copy.txt") - if err == nil { - t.Fatalf("got: %v, want error: %v", info, err) - } - - if err := cli.Rename("/hello.txt", "/copy.txt", false); err != nil { - t.Fatalf("got: %v, want mv /hello.txt to /copy.txt", err) - } - - info, err = fs.Stat(ctx, "/copy.txt") - if err != nil { - t.Fatalf("got: %v, want file info: %v", err, info) - } - if info.Size() != 15 { - t.Fatalf("got: %v, want file size: %d bytes", info.Size(), 15) - } - - if info, err = fs.Stat(ctx, "/hello.txt"); err == nil { - t.Fatalf("got: %v, want error: %v", info, err) - } - - if err := cli.Rename("/test/test.txt", "/copy.txt", true); err != nil { - t.Fatalf("got: %v, want overwrite /copy.txt with /hello.txt", err) - } - info, err = fs.Stat(ctx, "/copy.txt") - if err != nil { - t.Fatalf("got: %v, want file info: %v", err, info) - } - if info.Size() != 19 { - t.Fatalf("got: %v, want file size: %d bytes", info.Size(), 19) - } -} - -func TestRemove(t *testing.T) { - cli, srv, fs, ctx := newServer(t) - defer srv.Close() - - if err := cli.Remove("/hello.txt"); err != nil { - t.Fatalf("got: %v, want nil", err) - } - - if info, err := fs.Stat(ctx, "/hello.txt"); err == nil { - t.Fatalf("got: %v, want error: %v", info, err) - } - - if err := cli.Remove("/404.txt"); err != nil { - t.Fatalf("got: %v, want nil", err) - } -} - -func TestRemoveAll(t *testing.T) { - cli, srv, fs, ctx := newServer(t) - defer srv.Close() - - if err := cli.RemoveAll("/test/test.txt"); err != nil { - t.Fatalf("got: %v, want nil", err) - } - - if info, err := fs.Stat(ctx, "/test/test.txt"); err == nil { - t.Fatalf("got: %v, want error: %v", info, err) - } - - if err := cli.RemoveAll("/404.txt"); err != nil { - t.Fatalf("got: %v, want nil", err) - } - - if err := cli.RemoveAll("/404/404/404.txt"); err != nil { - t.Fatalf("got: %v, want nil", err) - } -} - -func TestWrite(t *testing.T) { - cli, srv, fs, ctx := newServer(t) - defer srv.Close() - - if err := cli.Write("/newfile.txt", []byte("foo bar\n"), 0660); err != nil { - t.Fatalf("got: %v, want nil", err) - } - - info, err := fs.Stat(ctx, "/newfile.txt") - if err != nil { - t.Fatalf("got: %v, want file info: %v", err, info) - } - if info.Size() != 8 { - t.Fatalf("got: %v, want file size: %d bytes", info.Size(), 8) - } - - if err := cli.Write("/404/newfile.txt", []byte("foo bar\n"), 0660); err != nil { - t.Fatalf("got: %v, want nil", err) - } -} - -func TestWriteStream(t *testing.T) { - cli, srv, fs, ctx := newServer(t) - defer srv.Close() - - if err := cli.WriteStream("/newfile.txt", strings.NewReader("foo bar\n"), 0660); err != nil { - t.Fatalf("got: %v, want nil", err) - } - - info, err := fs.Stat(ctx, "/newfile.txt") - if err != nil { - t.Fatalf("got: %v, want file info: %v", err, info) - } - if info.Size() != 8 { - t.Fatalf("got: %v, want file size: %d bytes", info.Size(), 8) - } - - if err := cli.WriteStream("/404/works.txt", strings.NewReader("foo bar\n"), 0660); err != nil { - t.Fatalf("got: %v, want nil", err) - } - - if info, err := fs.Stat(ctx, "/404/works.txt"); err != nil { - t.Fatalf("got: %v, want file info: %v", err, info) - } -} - -func TestWriteStreamFromPipe(t *testing.T) { - cli, srv, fs, ctx := newServer(t) - defer srv.Close() - - r, w := io.Pipe() - - go func() { - defer w.Close() - fmt.Fprint(w, "foo") - time.Sleep(1 * time.Second) - fmt.Fprint(w, " ") - time.Sleep(1 * time.Second) - fmt.Fprint(w, "bar\n") - }() - - if err := cli.WriteStream("/newfile.txt", r, 0660); err != nil { - t.Fatalf("got: %v, want nil", err) - } - - info, err := fs.Stat(ctx, "/newfile.txt") - if err != nil { - t.Fatalf("got: %v, want file info: %v", err, info) - } - if info.Size() != 8 { - t.Fatalf("got: %v, want file size: %d bytes", info.Size(), 8) - } -} diff --git a/util/sync/webdavClient/digestAuth.go b/util/sync/webdavClient/digestAuth.go deleted file mode 100644 index 5ac632051..000000000 --- a/util/sync/webdavClient/digestAuth.go +++ /dev/null @@ -1,164 +0,0 @@ -package webdavClient - -import ( - "crypto/md5" - "crypto/rand" - "encoding/hex" - "fmt" - "io" - "net/http" - "strings" -) - -// DigestAuth structure holds our credentials -type DigestAuth struct { - user string - pw string - digestParts map[string]string -} - -// NewDigestAuth creates a new instance of our Digest Authenticator -func NewDigestAuth(login, secret string, rs *http.Response) (Authenticator, error) { - return &DigestAuth{user: login, pw: secret, digestParts: digestParts(rs)}, nil -} - -// Authorize the current request -func (d *DigestAuth) Authorize(c *http.Client, rq *http.Request, path string) error { - d.digestParts["uri"] = path - d.digestParts["method"] = rq.Method - d.digestParts["username"] = d.user - d.digestParts["password"] = d.pw - rq.Header.Set("Authorization", getDigestAuthorization(d.digestParts)) - return nil -} - -// Verify checks for authentication issues and may trigger a re-authentication -func (d *DigestAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) { - if rs.StatusCode == 401 { - err = NewPathError("Authorize", path, rs.StatusCode) - } - return -} - -// Close cleans up all resources -func (d *DigestAuth) Close() error { - return nil -} - -// Clone creates a copy of itself -func (d *DigestAuth) Clone() Authenticator { - parts := make(map[string]string, len(d.digestParts)) - for k, v := range d.digestParts { - parts[k] = v - } - return &DigestAuth{user: d.user, pw: d.pw, digestParts: parts} -} - -// String toString -func (d *DigestAuth) String() string { - return fmt.Sprintf("DigestAuth login: %s", d.user) -} - -func digestParts(resp *http.Response) map[string]string { - result := map[string]string{} - if len(resp.Header["Www-Authenticate"]) > 0 { - wantedHeaders := []string{"nonce", "realm", "qop", "opaque", "algorithm", "entityBody"} - responseHeaders := strings.Split(resp.Header["Www-Authenticate"][0], ",") - for _, r := range responseHeaders { - for _, w := range wantedHeaders { - if strings.Contains(r, w) { - result[w] = strings.Trim( - strings.SplitN(r, `=`, 2)[1], - `"`, - ) - } - } - } - } - return result -} - -func getMD5(text string) string { - hasher := md5.New() - hasher.Write([]byte(text)) - return hex.EncodeToString(hasher.Sum(nil)) -} - -func getCnonce() string { - b := make([]byte, 8) - io.ReadFull(rand.Reader, b) - return fmt.Sprintf("%x", b)[:16] -} - -func getDigestAuthorization(digestParts map[string]string) string { - d := digestParts - // These are the correct ha1 and ha2 for qop=auth. We should probably check for other types of qop. - - var ( - ha1 string - ha2 string - nonceCount = 00000001 - cnonce = getCnonce() - response string - ) - - // 'ha1' value depends on value of "algorithm" field - switch d["algorithm"] { - case "MD5", "": - ha1 = getMD5(d["username"] + ":" + d["realm"] + ":" + d["password"]) - case "MD5-sess": - ha1 = getMD5( - fmt.Sprintf("%s:%v:%s", - getMD5(d["username"]+":"+d["realm"]+":"+d["password"]), - nonceCount, - cnonce, - ), - ) - } - - // 'ha2' value depends on value of "qop" field - switch d["qop"] { - case "auth", "": - ha2 = getMD5(d["method"] + ":" + d["uri"]) - case "auth-int": - if d["entityBody"] != "" { - ha2 = getMD5(d["method"] + ":" + d["uri"] + ":" + getMD5(d["entityBody"])) - } - } - - // 'response' value depends on value of "qop" field - switch d["qop"] { - case "": - response = getMD5( - fmt.Sprintf("%s:%s:%s", - ha1, - d["nonce"], - ha2, - ), - ) - case "auth", "auth-int": - response = getMD5( - fmt.Sprintf("%s:%s:%v:%s:%s:%s", - ha1, - d["nonce"], - nonceCount, - cnonce, - d["qop"], - ha2, - ), - ) - } - - authorization := fmt.Sprintf(`Digest username="%s", realm="%s", nonce="%s", uri="%s", nc=%v, cnonce="%s", response="%s"`, - d["username"], d["realm"], d["nonce"], d["uri"], nonceCount, cnonce, response) - - if d["qop"] != "" { - authorization += fmt.Sprintf(`, qop=%s`, d["qop"]) - } - - if d["opaque"] != "" { - authorization += fmt.Sprintf(`, opaque="%s"`, d["opaque"]) - } - - return authorization -} diff --git a/util/sync/webdavClient/digestAuth_test.go b/util/sync/webdavClient/digestAuth_test.go deleted file mode 100644 index 2adae4a68..000000000 --- a/util/sync/webdavClient/digestAuth_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package webdavClient - -import ( - "net/http" - "strings" - "testing" -) - -func TestNewDigestAuth(t *testing.T) { - a := &DigestAuth{user: "user", pw: "password", digestParts: make(map[string]string, 0)} - - ex := "DigestAuth login: user" - if a.String() != ex { - t.Error("expected: " + ex + " got: " + a.String()) - } - - if a.Clone() == a { - t.Error("expected a different instance") - } - - if a.Close() != nil { - t.Error("expected close without errors") - } -} - -func TestDigestAuthAuthorize(t *testing.T) { - a := &DigestAuth{user: "user", pw: "password", digestParts: make(map[string]string, 0)} - rq, _ := http.NewRequest("GET", "http://localhost/", nil) - a.Authorize(nil, rq, "/") - // TODO this is a very lazy test it cuts of cnonce - ex := `Digest username="user", realm="", nonce="", uri="/", nc=1, cnonce="` - if strings.Index(rq.Header.Get("Authorization"), ex) != 0 { - t.Error("got wrong Authorization header: " + rq.Header.Get("Authorization")) - } -} diff --git a/util/sync/webdavClient/doc.go b/util/sync/webdavClient/doc.go deleted file mode 100644 index db5c24558..000000000 --- a/util/sync/webdavClient/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package gowebdav is a WebDAV client library with a command line tool -// included. -package webdavClient diff --git a/util/sync/webdavClient/errors.go b/util/sync/webdavClient/errors.go deleted file mode 100644 index 13488f812..000000000 --- a/util/sync/webdavClient/errors.go +++ /dev/null @@ -1,57 +0,0 @@ -package webdavClient - -import ( - "errors" - "fmt" - "os" -) - -// ErrAuthChanged must be returned from the Verify method as an error -// to trigger a re-authentication / negotiation with a new authenticator. -var ErrAuthChanged = errors.New("authentication failed, change algorithm") - -// ErrTooManyRedirects will be used as return error if a request exceeds 10 redirects. -var ErrTooManyRedirects = errors.New("stopped after 10 redirects") - -// StatusError implements error and wraps -// an erroneous status code. -type StatusError struct { - Status int -} - -func (se StatusError) Error() string { - return fmt.Sprintf("%d", se.Status) -} - -// IsErrCode returns true if the given error -// is an os.PathError wrapping a StatusError -// with the given status code. -func IsErrCode(err error, code int) bool { - if pe, ok := err.(*os.PathError); ok { - se, ok := pe.Err.(StatusError) - return ok && se.Status == code - } - return false -} - -// IsErrNotFound is shorthand for IsErrCode -// for status 404. -func IsErrNotFound(err error) bool { - return IsErrCode(err, 404) -} - -func NewPathError(op string, path string, statusCode int) error { - return &os.PathError{ - Op: op, - Path: path, - Err: StatusError{statusCode}, - } -} - -func NewPathErrorErr(op string, path string, err error) error { - return &os.PathError{ - Op: op, - Path: path, - Err: err, - } -} diff --git a/util/sync/webdavClient/file.go b/util/sync/webdavClient/file.go deleted file mode 100644 index ce1033ef1..000000000 --- a/util/sync/webdavClient/file.go +++ /dev/null @@ -1,82 +0,0 @@ -package webdavClient - -import ( - "fmt" - "os" - "time" -) - -// File is our structure for a given file -type File struct { - path string - name string - contentType string - size int64 - modified time.Time - etag string - isdir bool - checksum string -} - -// Path returns the full path of a file -func (f File) Path() string { - return f.path -} - -// Name returns the name of a file -func (f File) Name() string { - return f.name -} - -// ContentType returns the content type of a file -func (f File) ContentType() string { - return f.contentType -} - -// Size returns the size of a file -func (f File) Size() int64 { - return f.size -} - -// Mode will return the mode of a given file -func (f File) Mode() os.FileMode { - // TODO check webdav perms - if f.isdir { - return 0775 | os.ModeDir - } - - return 0664 -} - -// ModTime returns the modified time of a file -func (f File) ModTime() time.Time { - return f.modified -} - -// ETag returns the ETag of a file -func (f File) ETag() string { - return f.etag -} - -// IsDir let us see if a given file is a directory or not -func (f File) IsDir() bool { - return f.isdir -} - -func (f File) Checksum() string { - return f.checksum -} - -// Sys ???? -func (f File) Sys() interface{} { - return nil -} - -// String lets us see file information -func (f File) String() string { - if f.isdir { - return fmt.Sprintf("Dir : '%s' - '%s'", f.path, f.name) - } - - return fmt.Sprintf("File: '%s' SIZE: %d MODIFIED: %s ETAG: %s CTYPE: %s", f.path, f.size, f.modified.String(), f.etag, f.contentType) -} diff --git a/util/sync/webdavClient/netrc.go b/util/sync/webdavClient/netrc.go deleted file mode 100644 index 1bf0eaab3..000000000 --- a/util/sync/webdavClient/netrc.go +++ /dev/null @@ -1,54 +0,0 @@ -package webdavClient - -import ( - "bufio" - "fmt" - "net/url" - "os" - "regexp" - "strings" -) - -func parseLine(s string) (login, pass string) { - fields := strings.Fields(s) - for i, f := range fields { - if f == "login" { - login = fields[i+1] - } - if f == "password" { - pass = fields[i+1] - } - } - return login, pass -} - -// ReadConfig reads login and password configuration from ~/.netrc -// machine foo.com login username password 123456 -func ReadConfig(uri, netrc string) (string, string) { - u, err := url.Parse(uri) - if err != nil { - return "", "" - } - - file, err := os.Open(netrc) - if err != nil { - return "", "" - } - defer file.Close() - - re := fmt.Sprintf(`^.*machine %s.*$`, u.Host) - scanner := bufio.NewScanner(file) - for scanner.Scan() { - s := scanner.Text() - - matched, err := regexp.MatchString(re, s) - if err != nil { - return "", "" - } - if matched { - return parseLine(s) - } - } - - return "", "" -} diff --git a/util/sync/webdavClient/passportAuth.go b/util/sync/webdavClient/passportAuth.go deleted file mode 100644 index 35633849e..000000000 --- a/util/sync/webdavClient/passportAuth.go +++ /dev/null @@ -1,181 +0,0 @@ -package webdavClient - -import ( - "fmt" - "io" - "net/http" - "net/url" - "strings" -) - -// PassportAuth structure holds our credentials -type PassportAuth struct { - user string - pw string - cookies []http.Cookie - inhibitRedirect bool -} - -// constructor for PassportAuth creates a new PassportAuth object and -// automatically authenticates against the given partnerURL -func NewPassportAuth(c *http.Client, user, pw, partnerURL string, header *http.Header) (Authenticator, error) { - p := &PassportAuth{ - user: user, - pw: pw, - inhibitRedirect: true, - } - err := p.genCookies(c, partnerURL, header) - return p, err -} - -// Authorize the current request -func (p *PassportAuth) Authorize(c *http.Client, rq *http.Request, path string) error { - // prevent redirects to detect subsequent authentication requests - if p.inhibitRedirect { - rq.Header.Set(XInhibitRedirect, "1") - } else { - p.inhibitRedirect = true - } - for _, cookie := range p.cookies { - rq.AddCookie(&cookie) - } - return nil -} - -// Verify verifies if the authentication is good -func (p *PassportAuth) Verify(c *http.Client, rs *http.Response, path string) (redo bool, err error) { - switch rs.StatusCode { - case 301, 302, 307, 308: - redo = true - if rs.Header.Get("Www-Authenticate") != "" { - // re-authentication required as we are redirected to the login page - err = p.genCookies(c, rs.Request.URL.String(), &rs.Header) - } else { - // just a redirect, follow it - p.inhibitRedirect = false - } - case 401: - err = NewPathError("Authorize", path, rs.StatusCode) - } - return -} - -// Close cleans up all resources -func (p *PassportAuth) Close() error { - return nil -} - -// Clone creates a Copy of itself -func (p *PassportAuth) Clone() Authenticator { - // create a copy to allow independent cookie updates - clonedCookies := make([]http.Cookie, len(p.cookies)) - copy(clonedCookies, p.cookies) - - return &PassportAuth{ - user: p.user, - pw: p.pw, - cookies: clonedCookies, - inhibitRedirect: true, - } -} - -// String toString -func (p *PassportAuth) String() string { - return fmt.Sprintf("PassportAuth login: %s", p.user) -} - -func (p *PassportAuth) genCookies(c *http.Client, partnerUrl string, header *http.Header) error { - // For more details refer to: - // https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-pass/2c80637d-438c-4d4b-adc5-903170a779f3 - // Skipping step 1 and 2 as we already have the partner server challenge - - baseAuthenticationServer := header.Get("Location") - baseAuthenticationServerURL, err := url.Parse(baseAuthenticationServer) - if err != nil { - return err - } - - // Skipping step 3 and 4 as we already know that we need and have the user's credentials - // Step 5 (Sign-in request) - authenticationServerUrl := url.URL{ - Scheme: baseAuthenticationServerURL.Scheme, - Host: baseAuthenticationServerURL.Host, - Path: "/login2.srf", - } - - partnerServerChallenge := strings.Split(header.Get("Www-Authenticate"), " ")[1] - - req := http.Request{ - Method: "GET", - URL: &authenticationServerUrl, - Header: http.Header{ - "Authorization": []string{"Passport1.4 sign-in=" + url.QueryEscape(p.user) + ",pwd=" + url.QueryEscape(p.pw) + ",OrgVerb=GET,OrgUrl=" + partnerUrl + "," + partnerServerChallenge}, - }, - } - - rs, err := c.Do(&req) - if err != nil { - return err - } - io.Copy(io.Discard, rs.Body) - rs.Body.Close() - if rs.StatusCode != 200 { - return NewPathError("Authorize", "/", rs.StatusCode) - } - - // Step 6 (Token Response from Authentication Server) - tokenResponseHeader := rs.Header.Get("Authentication-Info") - if tokenResponseHeader == "" { - return NewPathError("Authorize", "/", 401) - } - tokenResponseHeaderList := strings.Split(tokenResponseHeader, ",") - token := "" - for _, tokenResponseHeader := range tokenResponseHeaderList { - if strings.HasPrefix(tokenResponseHeader, "from-PP='") { - token = tokenResponseHeader - break - } - } - if token == "" { - return NewPathError("Authorize", "/", 401) - } - - // Step 7 (First Authentication Request to Partner Server) - origUrl, err := url.Parse(partnerUrl) - if err != nil { - return err - } - req = http.Request{ - Method: "GET", - URL: origUrl, - Header: http.Header{ - "Authorization": []string{"Passport1.4 " + token}, - }, - } - - rs, err = c.Do(&req) - if err != nil { - return err - } - io.Copy(io.Discard, rs.Body) - rs.Body.Close() - if rs.StatusCode != 200 && rs.StatusCode != 302 { - return NewPathError("Authorize", "/", rs.StatusCode) - } - - // Step 8 (Set Token Message from Partner Server) - cookies := rs.Header.Values("Set-Cookie") - p.cookies = make([]http.Cookie, len(cookies)) - for i, cookie := range cookies { - cookieParts := strings.Split(cookie, ";") - cookieName := strings.Split(cookieParts[0], "=")[0] - cookieValue := strings.Split(cookieParts[0], "=")[1] - - p.cookies[i] = http.Cookie{ - Name: cookieName, - Value: cookieValue, - } - } - - return nil -} diff --git a/util/sync/webdavClient/passportAuth_test.go b/util/sync/webdavClient/passportAuth_test.go deleted file mode 100644 index 27b8b6f0a..000000000 --- a/util/sync/webdavClient/passportAuth_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package webdavClient - -import ( - "bytes" - "net/http" - "net/url" - "regexp" - "testing" -) - -// testing the creation is enough as it handles the authorization during init -func TestNewPassportAuth(t *testing.T) { - user := "user" - pass := "password" - p1 := "some,comma,separated,values" - token := "from-PP='token'" - - authHandler := func(h http.Handler) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - reg, err := regexp.Compile("Passport1\\.4 sign-in=" + url.QueryEscape(user) + ",pwd=" + url.QueryEscape(pass) + ",OrgVerb=GET,OrgUrl=.*," + p1) - if err != nil { - t.Error(err) - } - if reg.MatchString(r.Header.Get("Authorization")) { - w.Header().Set("Authentication-Info", token) - w.WriteHeader(200) - return - } - } - } - authsrv, _, _ := newAuthSrv(t, authHandler) - defer authsrv.Close() - - dataHandler := func(h http.Handler) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - reg, err := regexp.Compile("Passport1\\.4 " + token) - if err != nil { - t.Error(err) - } - if reg.MatchString(r.Header.Get("Authorization")) { - w.Header().Set("Set-Cookie", "Pass=port") - h.ServeHTTP(w, r) - return - } - for _, c := range r.Cookies() { - if c.Name == "Pass" && c.Value == "port" { - h.ServeHTTP(w, r) - return - } - } - w.Header().Set("Www-Authenticate", "Passport1.4 "+p1) - http.Redirect(w, r, authsrv.URL+"/", 302) - } - } - srv, _, _ := newAuthSrv(t, dataHandler) - defer srv.Close() - - cli := NewClient(srv.URL, user, pass) - data, err := cli.Read("/hello.txt") - if err != nil { - t.Errorf("got error=%v; want nil", err) - } - if !bytes.Equal(data, []byte("hello gowebdav\n")) { - t.Logf("got data=%v; want=hello gowebdav", data) - } -} diff --git a/util/sync/webdavClient/requests.go b/util/sync/webdavClient/requests.go deleted file mode 100644 index 40b1c90ab..000000000 --- a/util/sync/webdavClient/requests.go +++ /dev/null @@ -1,201 +0,0 @@ -package webdavClient - -import ( - "io" - "log" - "net/http" - "path" - "strings" -) - -func (c *Client) req(method, path string, body io.Reader, intercept func(*http.Request)) (rs *http.Response, err error) { - var redo bool - var r *http.Request - var uri = PathEscape(Join(c.root, path)) - auth, body := c.auth.NewAuthenticator(body) - defer auth.Close() - - for { // TODO auth.continue() strategy(true|n times|until)? - if r, err = http.NewRequest(method, uri, body); err != nil { - return - } - - for k, vals := range c.headers { - for _, v := range vals { - r.Header.Add(k, v) - } - } - - if err = auth.Authorize(c.c, r, path); err != nil { - return - } - - if intercept != nil { - intercept(r) - } - - if c.interceptor != nil { - c.interceptor(method, r) - } - - if rs, err = c.c.Do(r); err != nil { - return - } - - if redo, err = auth.Verify(c.c, rs, path); err != nil { - rs.Body.Close() - return nil, err - } - if redo { - rs.Body.Close() - if body, err = r.GetBody(); err != nil { - return nil, err - } - continue - } - break - } - - return rs, err -} - -func (c *Client) mkcol(path string) (status int, err error) { - rs, err := c.req("MKCOL", path, nil, nil) - if err != nil { - return - } - defer rs.Body.Close() - - status = rs.StatusCode - if status == 405 { - status = 201 - } - - return -} - -func (c *Client) options(path string) (*http.Response, error) { - return c.req("OPTIONS", path, nil, func(rq *http.Request) { - rq.Header.Add("Depth", "0") - }) -} - -func (c *Client) propfind(path string, self bool, body string, resp interface{}, parse func(resp interface{}) error) error { - rs, err := c.req("PROPFIND", path, strings.NewReader(body), func(rq *http.Request) { - if self { - rq.Header.Add("Depth", "0") - } else { - rq.Header.Add("Depth", "1") - } - rq.Header.Add("Content-Type", "application/xml;charset=UTF-8") - rq.Header.Add("Accept", "application/xml,text/xml") - rq.Header.Add("Accept-Charset", "utf-8") - // TODO add support for 'gzip,deflate;q=0.8,q=0.7' - rq.Header.Add("Accept-Encoding", "") - }) - if err != nil { - return err - } - defer rs.Body.Close() - - if rs.StatusCode != 207 { - return NewPathError("PROPFIND", path, rs.StatusCode) - } - - return parseXML(rs.Body, resp, parse) -} - -func (c *Client) Proppatch(path string, body string, resp interface{}, parse func(resp interface{}) error) error { - rs, err := c.req("PROPPATCH", path, strings.NewReader(body), func(rq *http.Request) { - rq.Header.Add("Content-Type", "application/xml;charset=UTF-8") - rq.Header.Add("Accept", "application/xml,text/xml") - rq.Header.Add("Accept-Charset", "utf-8") - // TODO add support for 'gzip,deflate;q=0.8,q=0.7' - rq.Header.Add("Accept-Encoding", "") - }) - if err != nil { - return err - } - defer rs.Body.Close() - - if rs.StatusCode != 207 { - return NewPathError("PROPPATCH", path, rs.StatusCode) - } - - return parseXML(rs.Body, resp, parse) -} - -func (c *Client) doCopyMove( - method string, - oldpath string, - newpath string, - overwrite bool, -) ( - status int, - r io.ReadCloser, - err error, -) { - rs, err := c.req(method, oldpath, nil, func(rq *http.Request) { - rq.Header.Add("Destination", PathEscape(Join(c.root, newpath))) - if overwrite { - rq.Header.Add("Overwrite", "T") - } else { - rq.Header.Add("Overwrite", "F") - } - }) - if err != nil { - return - } - status = rs.StatusCode - r = rs.Body - return -} - -func (c *Client) copymove(method string, oldpath string, newpath string, overwrite bool) (err error) { - s, data, err := c.doCopyMove(method, oldpath, newpath, overwrite) - if err != nil { - return - } - if data != nil { - defer data.Close() - } - - switch s { - case 201, 204: - return nil - - case 207: - // TODO handle multistat errors, worst case ... - log.Printf("TODO handle %s - %s multistatus result %s\n", method, oldpath, String(data)) - - case 409: - err := c.createParentCollection(newpath) - if err != nil { - return err - } - - return c.copymove(method, oldpath, newpath, overwrite) - } - - return NewPathError(method, oldpath, s) -} - -func (c *Client) put(path string, stream io.Reader) (status int, err error) { - rs, err := c.req("PUT", path, stream, nil) - if err != nil { - return - } - defer rs.Body.Close() - - status = rs.StatusCode - return -} - -func (c *Client) createParentCollection(itemPath string) (err error) { - parentPath := path.Dir(itemPath) - if parentPath == "." || parentPath == "/" { - return nil - } - - return c.MkdirAll(parentPath, 0755) -} diff --git a/util/sync/webdavClient/utils.go b/util/sync/webdavClient/utils.go deleted file mode 100644 index e1d4c678a..000000000 --- a/util/sync/webdavClient/utils.go +++ /dev/null @@ -1,113 +0,0 @@ -package webdavClient - -import ( - "bytes" - "encoding/xml" - "io" - "net/url" - "strconv" - "strings" - "time" -) - -// PathEscape escapes all segments of a given path -func PathEscape(path string) string { - s := strings.Split(path, "/") - for i, e := range s { - s[i] = url.PathEscape(e) - } - return strings.Join(s, "/") -} - -// FixSlash appends a trailing / to our string -func FixSlash(s string) string { - if !strings.HasSuffix(s, "/") { - s += "/" - } - return s -} - -// FixSlashes appends and prepends a / if they are missing -func FixSlashes(s string) string { - if !strings.HasPrefix(s, "/") { - s = "/" + s - } - - return FixSlash(s) -} - -// Join joins two paths -func Join(path0 string, path1 string) string { - return strings.TrimSuffix(path0, "/") + "/" + strings.TrimPrefix(path1, "/") -} - -// String pulls a string out of our io.Reader -func String(r io.Reader) string { - buf := new(bytes.Buffer) - // TODO - make String return an error as well - _, _ = buf.ReadFrom(r) - return buf.String() -} - -func parseUint(s *string) uint { - if n, e := strconv.ParseUint(*s, 10, 32); e == nil { - return uint(n) - } - return 0 -} - -func parseInt64(s *string) int64 { - if n, e := strconv.ParseInt(*s, 10, 64); e == nil { - return n - } - return 0 -} - -func parseModified(s *string) time.Time { - if t, e := time.Parse(time.RFC1123, *s); e == nil { - return t - } - return time.Unix(0, 0) -} - -func parseXML(data io.Reader, resp interface{}, parse func(resp interface{}) error) error { - decoder := xml.NewDecoder(data) - for t, _ := decoder.Token(); t != nil; t, _ = decoder.Token() { - switch se := t.(type) { - case xml.StartElement: - if se.Name.Local == "response" { - if e := decoder.DecodeElement(resp, &se); e == nil { - if err := parse(resp); err != nil { - return err - } - } - } - } - } - return nil -} - -// limitedReadCloser wraps a io.ReadCloser and limits the number of bytes that can be read from it. -type limitedReadCloser struct { - rc io.ReadCloser - remaining int -} - -func (l *limitedReadCloser) Read(buf []byte) (int, error) { - if l.remaining <= 0 { - return 0, io.EOF - } - - if len(buf) > l.remaining { - buf = buf[0:l.remaining] - } - - n, err := l.rc.Read(buf) - l.remaining -= n - - return n, err -} - -func (l *limitedReadCloser) Close() error { - return l.rc.Close() -} diff --git a/util/sync/webdavClient/utils_test.go b/util/sync/webdavClient/utils_test.go deleted file mode 100644 index 74cb25060..000000000 --- a/util/sync/webdavClient/utils_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package webdavClient - -import ( - "fmt" - "net/url" - "testing" -) - -func TestJoin(t *testing.T) { - eq(t, "/", "", "") - eq(t, "/", "/", "/") - eq(t, "/foo", "", "/foo") - eq(t, "foo/foo", "foo/", "/foo") - eq(t, "foo/foo", "foo/", "foo") -} - -func eq(t *testing.T, expected string, s0 string, s1 string) { - s := Join(s0, s1) - if s != expected { - t.Error("For", "'"+s0+"','"+s1+"'", "expeted", "'"+expected+"'", "got", "'"+s+"'") - } -} - -func ExamplePathEscape() { - fmt.Println(PathEscape("")) - fmt.Println(PathEscape("/")) - fmt.Println(PathEscape("/web")) - fmt.Println(PathEscape("/web/")) - fmt.Println(PathEscape("/w e b/d a v/s%u&c#k:s/")) - - // Output: - // - // / - // /web - // /web/ - // /w%20e%20b/d%20a%20v/s%25u&c%23k:s/ -} - -func TestEscapeURL(t *testing.T) { - ex := "https://foo.com/w%20e%20b/d%20a%20v/s%25u&c%23k:s/" - u, _ := url.Parse("https://foo.com" + PathEscape("/w e b/d a v/s%u&c#k:s/")) - if ex != u.String() { - t.Error("expected: " + ex + " got: " + u.String()) - } -} - -func TestFixSlashes(t *testing.T) { - expected := "/" - - if got := FixSlashes(""); got != expected { - t.Errorf("expected: %q, got: %q", expected, got) - } - - expected = "/path/" - - if got := FixSlashes("path"); got != expected { - t.Errorf("expected: %q, got: %q", expected, got) - } - - if got := FixSlashes("/path"); got != expected { - t.Errorf("expected: %q, got: %q", expected, got) - } - - if got := FixSlashes("path/"); got != expected { - t.Errorf("expected: %q, got: %q", expected, got) - } -} From a95476bbe7b7122643d3edc98d2ff497a1d018c1 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 9 Jan 2024 13:58:38 -0500 Subject: [PATCH 28/63] updated synchronizer and target wrappers (#511) --- util/sync/filesystem.go | 38 ++++++++++++++++++++++++-------------- util/sync/model.go | 2 ++ util/sync/synchronizer.go | 28 ++++++++++++++++++---------- util/sync/webdav.go | 9 +++++++-- 4 files changed, 51 insertions(+), 26 deletions(-) diff --git a/util/sync/filesystem.go b/util/sync/filesystem.go index 3ed94ef47..18deca700 100644 --- a/util/sync/filesystem.go +++ b/util/sync/filesystem.go @@ -36,7 +36,7 @@ func (t *FilesystemTarget) Inventory() ([]*Object, error) { } if !fi.IsDir() { - return []*Object{{Path: t.cfg.Root, Size: fi.Size(), Modified: fi.ModTime()}}, nil + return []*Object{{Path: "/" + t.cfg.Root, Size: fi.Size(), Modified: fi.ModTime()}}, nil } t.tree = nil @@ -46,29 +46,39 @@ func (t *FilesystemTarget) Inventory() ([]*Object, error) { return t.tree, nil } -func (t *FilesystemTarget) IsDir() bool { - return true +func (t *FilesystemTarget) Mkdir(path string) error { + return os.MkdirAll(filepath.Join(t.cfg.Root, path), os.ModePerm) } func (t *FilesystemTarget) recurse(path string, d fs.DirEntry, err error) error { if err != nil { return err } - if !d.IsDir() { - fi, err := d.Info() + fi, err := d.Info() + if err != nil { + return err + } + etag := "" + if v, ok := fi.(webdav.ETager); ok { + etag, err = v.ETag(context.Background()) if err != nil { return err } - etag := "" - if v, ok := fi.(webdav.ETager); ok { - etag, err = v.ETag(context.Background()) - if err != nil { - return err - } - } else { - etag = fmt.Sprintf(`"%x%x"`, fi.ModTime().UTC().UnixNano(), fi.Size()) + } else { + etag = fmt.Sprintf(`"%x%x"`, fi.ModTime().UTC().UnixNano(), fi.Size()) + } + if path != "." { + outPath := "/" + path + if fi.IsDir() { + outPath = outPath + "/" } - t.tree = append(t.tree, &Object{path, fi.Size(), fi.ModTime(), etag}) + t.tree = append(t.tree, &Object{ + Path: outPath, + IsDir: fi.IsDir(), + Size: fi.Size(), + Modified: fi.ModTime(), + ETag: etag, + }) } return nil } diff --git a/util/sync/model.go b/util/sync/model.go index 8fca5ca07..a87aecadd 100644 --- a/util/sync/model.go +++ b/util/sync/model.go @@ -8,6 +8,7 @@ import ( type Object struct { Path string + IsDir bool Size int64 Modified time.Time ETag string @@ -15,6 +16,7 @@ type Object struct { type Target interface { Inventory() ([]*Object, error) + Mkdir(path string) error ReadStream(path string) (io.ReadCloser, error) WriteStream(path string, stream io.Reader, mode os.FileMode) error SetModificationTime(path string, mtime time.Time) error diff --git a/util/sync/synchronizer.go b/util/sync/synchronizer.go index 082325fae..de741e38b 100644 --- a/util/sync/synchronizer.go +++ b/util/sync/synchronizer.go @@ -25,24 +25,32 @@ func Synchronize(src, dst Target) error { var copyList []*Object for _, srcF := range srcTree { if dstF, found := dstIndex[srcF.Path]; found { - if dstF.Size != srcF.Size || dstF.Modified.UTC() != srcF.Modified.UTC() { + if !srcF.IsDir && (dstF.Size != srcF.Size || dstF.Modified.Unix() != srcF.Modified.Unix()) { + logrus.Debugf("%v <- dstF.Size = '%d', srcF.Size = '%d', dstF.Modified.UTC = '%d', srcF.Modified.UTC = '%d'", srcF.Path, dstF.Size, srcF.Size, dstF.Modified, srcF.Modified) copyList = append(copyList, srcF) } } else { + logrus.Debugf("%v <- !found", srcF.Path) copyList = append(copyList, srcF) } } for _, copyPath := range copyList { - ss, err := src.ReadStream(copyPath.Path) - if err != nil { - return err - } - if err := dst.WriteStream(copyPath.Path, ss, os.ModePerm); err != nil { - return err - } - if err := dst.SetModificationTime(copyPath.Path, copyPath.Modified); err != nil { - return err + if copyPath.IsDir { + if err := dst.Mkdir(copyPath.Path); err != nil { + return err + } + } else { + ss, err := src.ReadStream(copyPath.Path) + if err != nil { + return err + } + if err := dst.WriteStream(copyPath.Path, ss, os.ModePerm); err != nil { + return err + } + if err := dst.SetModificationTime(copyPath.Path, copyPath.Modified); err != nil { + return err + } } logrus.Infof("=> %v", copyPath.Path) } diff --git a/util/sync/webdav.go b/util/sync/webdav.go index b9a0c871b..831682481 100644 --- a/util/sync/webdav.go +++ b/util/sync/webdav.go @@ -39,9 +39,10 @@ func (t *WebDAVTarget) Inventory() ([]*Object, error) { } var objects []*Object for _, fi := range fis { - if !fi.IsDir { + if fi.Path != "/" { objects = append(objects, &Object{ Path: fi.Path, + IsDir: fi.IsDir, Size: fi.Size, Modified: fi.ModTime, ETag: fi.ETag, @@ -51,6 +52,10 @@ func (t *WebDAVTarget) Inventory() ([]*Object, error) { return objects, nil } +func (t *WebDAVTarget) Mkdir(path string) error { + return t.dc.Mkdir(context.Background(), path) +} + func (t *WebDAVTarget) ReadStream(path string) (io.ReadCloser, error) { return t.dc.Open(context.Background(), path) } @@ -60,7 +65,7 @@ func (t *WebDAVTarget) WriteStream(path string, rs io.Reader, _ os.FileMode) err if err != nil { return err } - defer ws.Close() + defer func() { _ = ws.Close() }() _, err = io.Copy(ws, rs) if err != nil { return err From a28aa2f77f999dd777e44b6c81745f5e997bc9fc Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 9 Jan 2024 14:26:07 -0500 Subject: [PATCH 29/63] support for private zrok-direct targets (#511) --- cmd/zrok/copy.go | 30 +++++++++------ util/sync/webdav.go | 7 +--- util/sync/zrok.go | 94 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 16 deletions(-) create mode 100644 util/sync/zrok.go diff --git a/cmd/zrok/copy.go b/cmd/zrok/copy.go index 58a1e4f1d..53799c043 100644 --- a/cmd/zrok/copy.go +++ b/cmd/zrok/copy.go @@ -50,24 +50,33 @@ func (cmd *copyCommand) run(_ *cobra.Command, args []string) { tui.Error("error loading root", err) } - var access *sdk.Access + var srcAccess *sdk.Access if sourceUrl.Scheme == "zrok" { - access, err = sdk.CreateAccess(root, &sdk.AccessRequest{ShareToken: sourceUrl.Host}) + srcAccess, err = sdk.CreateAccess(root, &sdk.AccessRequest{ShareToken: sourceUrl.Host}) if err != nil { tui.Error("error creating access", err) } } + if srcAccess != nil { + defer func() { + err := sdk.DeleteAccess(root, srcAccess) + if err != nil { + tui.Error("error deleting source access", err) + } + }() + } + var dstAccess *sdk.Access if targetUrl.Scheme == "zrok" { - access, err = sdk.CreateAccess(root, &sdk.AccessRequest{ShareToken: targetUrl.Host}) + dstAccess, err = sdk.CreateAccess(root, &sdk.AccessRequest{ShareToken: targetUrl.Host}) if err != nil { tui.Error("error creating access", err) } } - if access != nil { + if dstAccess != nil { defer func() { - err := sdk.DeleteAccess(root, access) + err := sdk.DeleteAccess(root, dstAccess) if err != nil { - tui.Error("error deleting access", err) + tui.Error("error deleting target access", err) } }() } @@ -93,11 +102,10 @@ func (cmd *copyCommand) createTarget(t *url.URL, root env_core.Root) (sync.Targe case "file": return sync.NewFilesystemTarget(&sync.FilesystemTargetConfig{Root: t.Path}), nil + case "zrok": + return sync.NewZrokTarget(&sync.ZrokTargetConfig{URL: t, Root: root}) + default: - target, err := sync.NewWebDAVTarget(&sync.WebDAVTargetConfig{URL: t, Username: "", Password: "", Root: root}) - if err != nil { - return nil, err - } - return target, nil + return sync.NewWebDAVTarget(&sync.WebDAVTargetConfig{URL: t, Username: "", Password: ""}) } } diff --git a/util/sync/webdav.go b/util/sync/webdav.go index 831682481..c8345b1c8 100644 --- a/util/sync/webdav.go +++ b/util/sync/webdav.go @@ -2,7 +2,6 @@ package sync import ( "context" - "github.com/openziti/zrok/environment/env_core" "github.com/openziti/zrok/util/sync/driveClient" "io" "net/http" @@ -15,13 +14,11 @@ type WebDAVTargetConfig struct { URL *url.URL Username string Password string - Root env_core.Root } type WebDAVTarget struct { - cfg *WebDAVTargetConfig - dc *driveClient.Client - isDir bool + cfg *WebDAVTargetConfig + dc *driveClient.Client } func NewWebDAVTarget(cfg *WebDAVTargetConfig) (*WebDAVTarget, error) { diff --git a/util/sync/zrok.go b/util/sync/zrok.go new file mode 100644 index 000000000..be542668f --- /dev/null +++ b/util/sync/zrok.go @@ -0,0 +1,94 @@ +package sync + +import ( + "context" + "github.com/openziti/zrok/environment/env_core" + "github.com/openziti/zrok/sdk/golang/sdk" + "github.com/openziti/zrok/util/sync/driveClient" + "io" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" +) + +type ZrokTargetConfig struct { + URL *url.URL + Username string + Password string + Root env_core.Root +} + +type ZrokTarget struct { + cfg *ZrokTargetConfig + dc *driveClient.Client +} + +type zrokDialContext struct { + root env_core.Root +} + +func (zdc *zrokDialContext) Dial(_ context.Context, _, addr string) (net.Conn, error) { + share := strings.Split(addr, ":")[0] + return sdk.NewDialer(share, zdc.root) +} + +func NewZrokTarget(cfg *ZrokTargetConfig) (*ZrokTarget, error) { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.DialContext = (&zrokDialContext{cfg.Root}).Dial + transport.TLSClientConfig.InsecureSkipVerify = true + httpUrl := strings.Replace(cfg.URL.String(), "zrok:", "http:", 1) + dc, err := driveClient.NewClient(&http.Client{Transport: transport}, httpUrl) + if err != nil { + return nil, err + } + return &ZrokTarget{cfg: cfg, dc: dc}, nil +} + +func (t *ZrokTarget) Inventory() ([]*Object, error) { + fis, err := t.dc.Readdir(context.Background(), "", true) + if err != nil { + return nil, err + } + var objects []*Object + for _, fi := range fis { + if fi.Path != "/" { + objects = append(objects, &Object{ + Path: fi.Path, + IsDir: fi.IsDir, + Size: fi.Size, + Modified: fi.ModTime, + ETag: fi.ETag, + }) + } + } + return objects, nil +} + +func (t *ZrokTarget) Mkdir(path string) error { + return t.dc.Mkdir(context.Background(), filepath.Join(t.cfg.URL.Path, path)) +} + +func (t *ZrokTarget) ReadStream(path string) (io.ReadCloser, error) { + return t.dc.Open(context.Background(), filepath.Join(t.cfg.URL.Path, path)) +} + +func (t *ZrokTarget) WriteStream(path string, rs io.Reader, _ os.FileMode) error { + ws, err := t.dc.Create(context.Background(), filepath.Join(t.cfg.URL.Path, path)) + if err != nil { + return err + } + defer func() { _ = ws.Close() }() + _, err = io.Copy(ws, rs) + if err != nil { + return err + } + return nil +} + +func (t *ZrokTarget) SetModificationTime(path string, mtime time.Time) error { + return t.dc.Touch(context.Background(), filepath.Join(t.cfg.URL.Path, path), mtime) +} From 8ccb5603ca8eb062f0282d6965093ac00fc386b0 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 9 Jan 2024 14:55:51 -0500 Subject: [PATCH 30/63] endpoints/drive/webdav -> endpoints/drive/driveServer (#511) --- endpoints/drive/backend.go | 8 ++++---- endpoints/drive/{webdav => driveServer}/file.go | 2 +- endpoints/drive/{webdav => driveServer}/file_test.go | 2 +- endpoints/drive/{webdav => driveServer}/if.go | 2 +- endpoints/drive/{webdav => driveServer}/if_test.go | 2 +- .../drive/{webdav => driveServer}/internal/xml/README | 0 .../{webdav => driveServer}/internal/xml/atom_test.go | 0 .../{webdav => driveServer}/internal/xml/example_test.go | 0 .../drive/{webdav => driveServer}/internal/xml/marshal.go | 0 .../{webdav => driveServer}/internal/xml/marshal_test.go | 0 .../drive/{webdav => driveServer}/internal/xml/read.go | 0 .../{webdav => driveServer}/internal/xml/read_test.go | 0 .../{webdav => driveServer}/internal/xml/typeinfo.go | 0 .../drive/{webdav => driveServer}/internal/xml/xml.go | 0 .../{webdav => driveServer}/internal/xml/xml_test.go | 0 .../drive/{webdav => driveServer}/litmus_test_server.go | 8 ++++---- endpoints/drive/{webdav => driveServer}/lock.go | 2 +- endpoints/drive/{webdav => driveServer}/lock_test.go | 2 +- endpoints/drive/{webdav => driveServer}/prop.go | 2 +- endpoints/drive/{webdav => driveServer}/prop_test.go | 2 +- endpoints/drive/{webdav => driveServer}/webdav.go | 4 ++-- endpoints/drive/{webdav => driveServer}/webdav_test.go | 2 +- endpoints/drive/{webdav => driveServer}/xml.go | 4 ++-- endpoints/drive/{webdav => driveServer}/xml_test.go | 4 ++-- util/sync/filesystem.go | 4 ++-- 25 files changed, 25 insertions(+), 25 deletions(-) rename endpoints/drive/{webdav => driveServer}/file.go (99%) rename endpoints/drive/{webdav => driveServer}/file_test.go (99%) rename endpoints/drive/{webdav => driveServer}/if.go (99%) rename endpoints/drive/{webdav => driveServer}/if_test.go (99%) rename endpoints/drive/{webdav => driveServer}/internal/xml/README (100%) rename endpoints/drive/{webdav => driveServer}/internal/xml/atom_test.go (100%) rename endpoints/drive/{webdav => driveServer}/internal/xml/example_test.go (100%) rename endpoints/drive/{webdav => driveServer}/internal/xml/marshal.go (100%) rename endpoints/drive/{webdav => driveServer}/internal/xml/marshal_test.go (100%) rename endpoints/drive/{webdav => driveServer}/internal/xml/read.go (100%) rename endpoints/drive/{webdav => driveServer}/internal/xml/read_test.go (100%) rename endpoints/drive/{webdav => driveServer}/internal/xml/typeinfo.go (100%) rename endpoints/drive/{webdav => driveServer}/internal/xml/xml.go (100%) rename endpoints/drive/{webdav => driveServer}/internal/xml/xml_test.go (100%) rename endpoints/drive/{webdav => driveServer}/litmus_test_server.go (94%) rename endpoints/drive/{webdav => driveServer}/lock.go (99%) rename endpoints/drive/{webdav => driveServer}/lock_test.go (99%) rename endpoints/drive/{webdav => driveServer}/prop.go (99%) rename endpoints/drive/{webdav => driveServer}/prop_test.go (99%) rename endpoints/drive/{webdav => driveServer}/webdav.go (99%) rename endpoints/drive/{webdav => driveServer}/webdav_test.go (99%) rename endpoints/drive/{webdav => driveServer}/xml.go (99%) rename endpoints/drive/{webdav => driveServer}/xml_test.go (99%) diff --git a/endpoints/drive/backend.go b/endpoints/drive/backend.go index 6eb9fa2b0..f7e5df979 100644 --- a/endpoints/drive/backend.go +++ b/endpoints/drive/backend.go @@ -5,7 +5,7 @@ import ( "github.com/openziti/sdk-golang/ziti" "github.com/openziti/sdk-golang/ziti/edge" "github.com/openziti/zrok/endpoints" - "github.com/openziti/zrok/endpoints/drive/webdav" + "github.com/openziti/zrok/endpoints/drive/driveServer" "github.com/pkg/errors" "net/http" "time" @@ -43,9 +43,9 @@ func NewBackend(cfg *BackendConfig) (*Backend, error) { return nil, err } - handler := &webdav.Handler{ - FileSystem: webdav.Dir(cfg.DriveRoot), - LockSystem: webdav.NewMemLS(), + handler := &driveServer.Handler{ + FileSystem: driveServer.Dir(cfg.DriveRoot), + LockSystem: driveServer.NewMemLS(), Logger: func(r *http.Request, err error) { if cfg.Requests != nil { cfg.Requests <- &endpoints.Request{ diff --git a/endpoints/drive/webdav/file.go b/endpoints/drive/driveServer/file.go similarity index 99% rename from endpoints/drive/webdav/file.go rename to endpoints/drive/driveServer/file.go index 32ae6d1c4..fbfacc81e 100644 --- a/endpoints/drive/webdav/file.go +++ b/endpoints/drive/driveServer/file.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package webdav +package driveServer import ( "context" diff --git a/endpoints/drive/webdav/file_test.go b/endpoints/drive/driveServer/file_test.go similarity index 99% rename from endpoints/drive/webdav/file_test.go rename to endpoints/drive/driveServer/file_test.go index e875c136c..834e08d75 100644 --- a/endpoints/drive/webdav/file_test.go +++ b/endpoints/drive/driveServer/file_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package webdav +package driveServer import ( "context" diff --git a/endpoints/drive/webdav/if.go b/endpoints/drive/driveServer/if.go similarity index 99% rename from endpoints/drive/webdav/if.go rename to endpoints/drive/driveServer/if.go index e646570bb..5eedaeab2 100644 --- a/endpoints/drive/webdav/if.go +++ b/endpoints/drive/driveServer/if.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package webdav +package driveServer // The If header is covered by Section 10.4. // http://www.webdav.org/specs/rfc4918.html#HEADER_If diff --git a/endpoints/drive/webdav/if_test.go b/endpoints/drive/driveServer/if_test.go similarity index 99% rename from endpoints/drive/webdav/if_test.go rename to endpoints/drive/driveServer/if_test.go index aad61a401..1df7eeeb3 100644 --- a/endpoints/drive/webdav/if_test.go +++ b/endpoints/drive/driveServer/if_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package webdav +package driveServer import ( "reflect" diff --git a/endpoints/drive/webdav/internal/xml/README b/endpoints/drive/driveServer/internal/xml/README similarity index 100% rename from endpoints/drive/webdav/internal/xml/README rename to endpoints/drive/driveServer/internal/xml/README diff --git a/endpoints/drive/webdav/internal/xml/atom_test.go b/endpoints/drive/driveServer/internal/xml/atom_test.go similarity index 100% rename from endpoints/drive/webdav/internal/xml/atom_test.go rename to endpoints/drive/driveServer/internal/xml/atom_test.go diff --git a/endpoints/drive/webdav/internal/xml/example_test.go b/endpoints/drive/driveServer/internal/xml/example_test.go similarity index 100% rename from endpoints/drive/webdav/internal/xml/example_test.go rename to endpoints/drive/driveServer/internal/xml/example_test.go diff --git a/endpoints/drive/webdav/internal/xml/marshal.go b/endpoints/drive/driveServer/internal/xml/marshal.go similarity index 100% rename from endpoints/drive/webdav/internal/xml/marshal.go rename to endpoints/drive/driveServer/internal/xml/marshal.go diff --git a/endpoints/drive/webdav/internal/xml/marshal_test.go b/endpoints/drive/driveServer/internal/xml/marshal_test.go similarity index 100% rename from endpoints/drive/webdav/internal/xml/marshal_test.go rename to endpoints/drive/driveServer/internal/xml/marshal_test.go diff --git a/endpoints/drive/webdav/internal/xml/read.go b/endpoints/drive/driveServer/internal/xml/read.go similarity index 100% rename from endpoints/drive/webdav/internal/xml/read.go rename to endpoints/drive/driveServer/internal/xml/read.go diff --git a/endpoints/drive/webdav/internal/xml/read_test.go b/endpoints/drive/driveServer/internal/xml/read_test.go similarity index 100% rename from endpoints/drive/webdav/internal/xml/read_test.go rename to endpoints/drive/driveServer/internal/xml/read_test.go diff --git a/endpoints/drive/webdav/internal/xml/typeinfo.go b/endpoints/drive/driveServer/internal/xml/typeinfo.go similarity index 100% rename from endpoints/drive/webdav/internal/xml/typeinfo.go rename to endpoints/drive/driveServer/internal/xml/typeinfo.go diff --git a/endpoints/drive/webdav/internal/xml/xml.go b/endpoints/drive/driveServer/internal/xml/xml.go similarity index 100% rename from endpoints/drive/webdav/internal/xml/xml.go rename to endpoints/drive/driveServer/internal/xml/xml.go diff --git a/endpoints/drive/webdav/internal/xml/xml_test.go b/endpoints/drive/driveServer/internal/xml/xml_test.go similarity index 100% rename from endpoints/drive/webdav/internal/xml/xml_test.go rename to endpoints/drive/driveServer/internal/xml/xml_test.go diff --git a/endpoints/drive/webdav/litmus_test_server.go b/endpoints/drive/driveServer/litmus_test_server.go similarity index 94% rename from endpoints/drive/webdav/litmus_test_server.go rename to endpoints/drive/driveServer/litmus_test_server.go index b87063d1a..91cbc5de5 100644 --- a/endpoints/drive/webdav/litmus_test_server.go +++ b/endpoints/drive/driveServer/litmus_test_server.go @@ -21,7 +21,7 @@ package main import ( "flag" "fmt" - "github.com/openziti/zrok/endpoints/drive/webdav" + "github.com/openziti/zrok/endpoints/drive/driveServer" "log" "net/http" "net/url" @@ -32,9 +32,9 @@ var port = flag.Int("port", 9999, "server port") func main() { flag.Parse() log.SetFlags(0) - h := &webdav.Handler{ - FileSystem: webdav.NewMemFS(), - LockSystem: webdav.NewMemLS(), + h := &driveServer.Handler{ + FileSystem: driveServer.NewMemFS(), + LockSystem: driveServer.NewMemLS(), Logger: func(r *http.Request, err error) { litmus := r.Header.Get("X-Litmus") if len(litmus) > 19 { diff --git a/endpoints/drive/webdav/lock.go b/endpoints/drive/driveServer/lock.go similarity index 99% rename from endpoints/drive/webdav/lock.go rename to endpoints/drive/driveServer/lock.go index 344ac5cea..caff970e9 100644 --- a/endpoints/drive/webdav/lock.go +++ b/endpoints/drive/driveServer/lock.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package webdav +package driveServer import ( "container/heap" diff --git a/endpoints/drive/webdav/lock_test.go b/endpoints/drive/driveServer/lock_test.go similarity index 99% rename from endpoints/drive/webdav/lock_test.go rename to endpoints/drive/driveServer/lock_test.go index e7fe97061..765b80442 100644 --- a/endpoints/drive/webdav/lock_test.go +++ b/endpoints/drive/driveServer/lock_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package webdav +package driveServer import ( "fmt" diff --git a/endpoints/drive/webdav/prop.go b/endpoints/drive/driveServer/prop.go similarity index 99% rename from endpoints/drive/webdav/prop.go rename to endpoints/drive/driveServer/prop.go index fca3154bd..c38d961e1 100644 --- a/endpoints/drive/webdav/prop.go +++ b/endpoints/drive/driveServer/prop.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package webdav +package driveServer import ( "bytes" diff --git a/endpoints/drive/webdav/prop_test.go b/endpoints/drive/driveServer/prop_test.go similarity index 99% rename from endpoints/drive/webdav/prop_test.go rename to endpoints/drive/driveServer/prop_test.go index f4247e69b..b7338e3f8 100644 --- a/endpoints/drive/webdav/prop_test.go +++ b/endpoints/drive/driveServer/prop_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package webdav +package driveServer import ( "context" diff --git a/endpoints/drive/webdav/webdav.go b/endpoints/drive/driveServer/webdav.go similarity index 99% rename from endpoints/drive/webdav/webdav.go rename to endpoints/drive/driveServer/webdav.go index add2bcd67..5b43da2a4 100644 --- a/endpoints/drive/webdav/webdav.go +++ b/endpoints/drive/driveServer/webdav.go @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package webdav provides a WebDAV server implementation. -package webdav // import "golang.org/x/net/webdav" +// Package driveServer provides a WebDAV server implementation. +package driveServer import ( "errors" diff --git a/endpoints/drive/webdav/webdav_test.go b/endpoints/drive/driveServer/webdav_test.go similarity index 99% rename from endpoints/drive/webdav/webdav_test.go rename to endpoints/drive/driveServer/webdav_test.go index 2baebe3c9..fbd5a43b5 100644 --- a/endpoints/drive/webdav/webdav_test.go +++ b/endpoints/drive/driveServer/webdav_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package webdav +package driveServer import ( "context" diff --git a/endpoints/drive/webdav/xml.go b/endpoints/drive/driveServer/xml.go similarity index 99% rename from endpoints/drive/webdav/xml.go rename to endpoints/drive/driveServer/xml.go index a92248e09..ef76a2fe3 100644 --- a/endpoints/drive/webdav/xml.go +++ b/endpoints/drive/driveServer/xml.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package webdav +package driveServer // The XML encoding is covered by Section 14. // http://www.webdav.org/specs/rfc4918.html#xml.element.definitions @@ -32,7 +32,7 @@ import ( // In the long term, this package should use the standard library's version // only, and the internal fork deleted, once // https://github.com/golang/go/issues/13400 is resolved. - ixml "github.com/openziti/zrok/endpoints/drive/webdav/internal/xml" + ixml "github.com/openziti/zrok/endpoints/drive/driveServer/internal/xml" ) // http://www.webdav.org/specs/rfc4918.html#ELEMENT_lockinfo diff --git a/endpoints/drive/webdav/xml_test.go b/endpoints/drive/driveServer/xml_test.go similarity index 99% rename from endpoints/drive/webdav/xml_test.go rename to endpoints/drive/driveServer/xml_test.go index 6812330a4..1fb653296 100644 --- a/endpoints/drive/webdav/xml_test.go +++ b/endpoints/drive/driveServer/xml_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package webdav +package driveServer import ( "bytes" @@ -16,7 +16,7 @@ import ( "strings" "testing" - ixml "github.com/openziti/zrok/endpoints/drive/webdav/internal/xml" + ixml "github.com/openziti/zrok/endpoints/drive/driveServer/internal/xml" ) func TestReadLockInfo(t *testing.T) { diff --git a/util/sync/filesystem.go b/util/sync/filesystem.go index 18deca700..03eb549d0 100644 --- a/util/sync/filesystem.go +++ b/util/sync/filesystem.go @@ -3,7 +3,7 @@ package sync import ( "context" "fmt" - "github.com/openziti/zrok/endpoints/drive/webdav" + "github.com/openziti/zrok/endpoints/drive/driveServer" "io" "io/fs" "os" @@ -59,7 +59,7 @@ func (t *FilesystemTarget) recurse(path string, d fs.DirEntry, err error) error return err } etag := "" - if v, ok := fi.(webdav.ETager); ok { + if v, ok := fi.(driveServer.ETager); ok { etag, err = v.ETag(context.Background()) if err != nil { return err From 07c162bf4e9bb6e6ae54897c5d6dac25d37c3ff9 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 9 Jan 2024 17:15:58 -0500 Subject: [PATCH 31/63] fix for log formatting (#438) --- util/sync/synchronizer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/sync/synchronizer.go b/util/sync/synchronizer.go index de741e38b..09b1edf2d 100644 --- a/util/sync/synchronizer.go +++ b/util/sync/synchronizer.go @@ -26,7 +26,7 @@ func Synchronize(src, dst Target) error { for _, srcF := range srcTree { if dstF, found := dstIndex[srcF.Path]; found { if !srcF.IsDir && (dstF.Size != srcF.Size || dstF.Modified.Unix() != srcF.Modified.Unix()) { - logrus.Debugf("%v <- dstF.Size = '%d', srcF.Size = '%d', dstF.Modified.UTC = '%d', srcF.Modified.UTC = '%d'", srcF.Path, dstF.Size, srcF.Size, dstF.Modified, srcF.Modified) + logrus.Debugf("%v <- dstF.Size = '%d', srcF.Size = '%d', dstF.Modified.UTC = '%d', srcF.Modified.UTC = '%d'", srcF.Path, dstF.Size, srcF.Size, dstF.Modified.Unix(), srcF.Modified.Unix()) copyList = append(copyList, srcF) } } else { From 34acc81822504cfe6cf331eefa18f839ba5a857a Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 10 Jan 2024 11:44:06 -0500 Subject: [PATCH 32/63] util/sync/driveClient -> drives/davClient (#511) --- cmd/zrok/davtest.go | 4 ++-- {util/sync/driveClient => drives/davClient}/client.go | 4 ++-- .../driveClient => drives/davClient}/internal/client.go | 0 .../driveClient => drives/davClient}/internal/elements.go | 0 .../driveClient => drives/davClient}/internal/internal.go | 0 {util/sync/driveClient => drives/davClient}/internal/xml.go | 0 {util/sync/driveClient => drives/davClient}/model.go | 2 +- util/sync/webdav.go | 6 +++--- util/sync/zrok.go | 6 +++--- 9 files changed, 11 insertions(+), 11 deletions(-) rename {util/sync/driveClient => drives/davClient}/client.go (98%) rename {util/sync/driveClient => drives/davClient}/internal/client.go (100%) rename {util/sync/driveClient => drives/davClient}/internal/elements.go (100%) rename {util/sync/driveClient => drives/davClient}/internal/internal.go (100%) rename {util/sync/driveClient => drives/davClient}/internal/xml.go (100%) rename {util/sync/driveClient => drives/davClient}/model.go (99%) diff --git a/cmd/zrok/davtest.go b/cmd/zrok/davtest.go index 9d03168fb..20bffb35b 100644 --- a/cmd/zrok/davtest.go +++ b/cmd/zrok/davtest.go @@ -2,7 +2,7 @@ package main import ( "context" - "github.com/openziti/zrok/util/sync/driveClient" + "github.com/openziti/zrok/drives/davClient" "github.com/spf13/cobra" "io" "net/http" @@ -29,7 +29,7 @@ func newDavtestCommand() *davtestCommand { } func (cmd *davtestCommand) run(_ *cobra.Command, args []string) { - client, err := driveClient.NewClient(http.DefaultClient, args[0]) + client, err := davClient.NewClient(http.DefaultClient, args[0]) if err != nil { panic(err) } diff --git a/util/sync/driveClient/client.go b/drives/davClient/client.go similarity index 98% rename from util/sync/driveClient/client.go rename to drives/davClient/client.go index 8dbdad586..cb0b63cd0 100644 --- a/util/sync/driveClient/client.go +++ b/drives/davClient/client.go @@ -1,9 +1,9 @@ -package driveClient +package davClient import ( "context" "fmt" - "github.com/openziti/zrok/util/sync/driveClient/internal" + "github.com/openziti/zrok/drives/davClient/internal" "io" "net/http" "time" diff --git a/util/sync/driveClient/internal/client.go b/drives/davClient/internal/client.go similarity index 100% rename from util/sync/driveClient/internal/client.go rename to drives/davClient/internal/client.go diff --git a/util/sync/driveClient/internal/elements.go b/drives/davClient/internal/elements.go similarity index 100% rename from util/sync/driveClient/internal/elements.go rename to drives/davClient/internal/elements.go diff --git a/util/sync/driveClient/internal/internal.go b/drives/davClient/internal/internal.go similarity index 100% rename from util/sync/driveClient/internal/internal.go rename to drives/davClient/internal/internal.go diff --git a/util/sync/driveClient/internal/xml.go b/drives/davClient/internal/xml.go similarity index 100% rename from util/sync/driveClient/internal/xml.go rename to drives/davClient/internal/xml.go diff --git a/util/sync/driveClient/model.go b/drives/davClient/model.go similarity index 99% rename from util/sync/driveClient/model.go rename to drives/davClient/model.go index 100994d30..d66627f0a 100644 --- a/util/sync/driveClient/model.go +++ b/drives/davClient/model.go @@ -1,4 +1,4 @@ -package driveClient +package davClient import ( "errors" diff --git a/util/sync/webdav.go b/util/sync/webdav.go index c8345b1c8..ae323c144 100644 --- a/util/sync/webdav.go +++ b/util/sync/webdav.go @@ -2,7 +2,7 @@ package sync import ( "context" - "github.com/openziti/zrok/util/sync/driveClient" + "github.com/openziti/zrok/drives/davClient" "io" "net/http" "net/url" @@ -18,11 +18,11 @@ type WebDAVTargetConfig struct { type WebDAVTarget struct { cfg *WebDAVTargetConfig - dc *driveClient.Client + dc *davClient.Client } func NewWebDAVTarget(cfg *WebDAVTargetConfig) (*WebDAVTarget, error) { - dc, err := driveClient.NewClient(http.DefaultClient, cfg.URL.String()) + dc, err := davClient.NewClient(http.DefaultClient, cfg.URL.String()) if err != nil { return nil, err } diff --git a/util/sync/zrok.go b/util/sync/zrok.go index be542668f..737aa164a 100644 --- a/util/sync/zrok.go +++ b/util/sync/zrok.go @@ -2,9 +2,9 @@ package sync import ( "context" + "github.com/openziti/zrok/drives/davClient" "github.com/openziti/zrok/environment/env_core" "github.com/openziti/zrok/sdk/golang/sdk" - "github.com/openziti/zrok/util/sync/driveClient" "io" "net" "net/http" @@ -24,7 +24,7 @@ type ZrokTargetConfig struct { type ZrokTarget struct { cfg *ZrokTargetConfig - dc *driveClient.Client + dc *davClient.Client } type zrokDialContext struct { @@ -41,7 +41,7 @@ func NewZrokTarget(cfg *ZrokTargetConfig) (*ZrokTarget, error) { transport.DialContext = (&zrokDialContext{cfg.Root}).Dial transport.TLSClientConfig.InsecureSkipVerify = true httpUrl := strings.Replace(cfg.URL.String(), "zrok:", "http:", 1) - dc, err := driveClient.NewClient(&http.Client{Transport: transport}, httpUrl) + dc, err := davClient.NewClient(&http.Client{Transport: transport}, httpUrl) if err != nil { return nil, err } From 999b65fc6cf2f8fa72c7078564b230bbb59cc9dd Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 10 Jan 2024 11:57:21 -0500 Subject: [PATCH 33/63] endpoints/drive/driveServer -> drives/davServer (#511) --- {endpoints/drive/driveServer => drives/davServer}/file.go | 2 +- .../drive/driveServer => drives/davServer}/file_test.go | 2 +- {endpoints/drive/driveServer => drives/davServer}/if.go | 2 +- .../drive/driveServer => drives/davServer}/if_test.go | 2 +- .../driveServer => drives/davServer}/internal/xml/README | 0 .../davServer}/internal/xml/atom_test.go | 0 .../davServer}/internal/xml/example_test.go | 0 .../davServer}/internal/xml/marshal.go | 0 .../davServer}/internal/xml/marshal_test.go | 0 .../driveServer => drives/davServer}/internal/xml/read.go | 0 .../davServer}/internal/xml/read_test.go | 0 .../davServer}/internal/xml/typeinfo.go | 0 .../driveServer => drives/davServer}/internal/xml/xml.go | 0 .../davServer}/internal/xml/xml_test.go | 0 .../davServer}/litmus_test_server.go | 0 {endpoints/drive/driveServer => drives/davServer}/lock.go | 2 +- .../drive/driveServer => drives/davServer}/lock_test.go | 2 +- {endpoints/drive/driveServer => drives/davServer}/prop.go | 2 +- .../drive/driveServer => drives/davServer}/prop_test.go | 2 +- .../drive/driveServer => drives/davServer}/webdav.go | 4 ++-- .../drive/driveServer => drives/davServer}/webdav_test.go | 2 +- {endpoints/drive/driveServer => drives/davServer}/xml.go | 4 ++-- .../drive/driveServer => drives/davServer}/xml_test.go | 4 ++-- endpoints/drive/backend.go | 8 ++++---- util/sync/filesystem.go | 4 ++-- 25 files changed, 21 insertions(+), 21 deletions(-) rename {endpoints/drive/driveServer => drives/davServer}/file.go (99%) rename {endpoints/drive/driveServer => drives/davServer}/file_test.go (99%) rename {endpoints/drive/driveServer => drives/davServer}/if.go (99%) rename {endpoints/drive/driveServer => drives/davServer}/if_test.go (99%) rename {endpoints/drive/driveServer => drives/davServer}/internal/xml/README (100%) rename {endpoints/drive/driveServer => drives/davServer}/internal/xml/atom_test.go (100%) rename {endpoints/drive/driveServer => drives/davServer}/internal/xml/example_test.go (100%) rename {endpoints/drive/driveServer => drives/davServer}/internal/xml/marshal.go (100%) rename {endpoints/drive/driveServer => drives/davServer}/internal/xml/marshal_test.go (100%) rename {endpoints/drive/driveServer => drives/davServer}/internal/xml/read.go (100%) rename {endpoints/drive/driveServer => drives/davServer}/internal/xml/read_test.go (100%) rename {endpoints/drive/driveServer => drives/davServer}/internal/xml/typeinfo.go (100%) rename {endpoints/drive/driveServer => drives/davServer}/internal/xml/xml.go (100%) rename {endpoints/drive/driveServer => drives/davServer}/internal/xml/xml_test.go (100%) rename {endpoints/drive/driveServer => drives/davServer}/litmus_test_server.go (100%) rename {endpoints/drive/driveServer => drives/davServer}/lock.go (99%) rename {endpoints/drive/driveServer => drives/davServer}/lock_test.go (99%) rename {endpoints/drive/driveServer => drives/davServer}/prop.go (99%) rename {endpoints/drive/driveServer => drives/davServer}/prop_test.go (99%) rename {endpoints/drive/driveServer => drives/davServer}/webdav.go (99%) rename {endpoints/drive/driveServer => drives/davServer}/webdav_test.go (99%) rename {endpoints/drive/driveServer => drives/davServer}/xml.go (99%) rename {endpoints/drive/driveServer => drives/davServer}/xml_test.go (99%) diff --git a/endpoints/drive/driveServer/file.go b/drives/davServer/file.go similarity index 99% rename from endpoints/drive/driveServer/file.go rename to drives/davServer/file.go index fbfacc81e..762e605f1 100644 --- a/endpoints/drive/driveServer/file.go +++ b/drives/davServer/file.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package driveServer +package davServer import ( "context" diff --git a/endpoints/drive/driveServer/file_test.go b/drives/davServer/file_test.go similarity index 99% rename from endpoints/drive/driveServer/file_test.go rename to drives/davServer/file_test.go index 834e08d75..c935d5a6d 100644 --- a/endpoints/drive/driveServer/file_test.go +++ b/drives/davServer/file_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package driveServer +package davServer import ( "context" diff --git a/endpoints/drive/driveServer/if.go b/drives/davServer/if.go similarity index 99% rename from endpoints/drive/driveServer/if.go rename to drives/davServer/if.go index 5eedaeab2..0697c5406 100644 --- a/endpoints/drive/driveServer/if.go +++ b/drives/davServer/if.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package driveServer +package davServer // The If header is covered by Section 10.4. // http://www.webdav.org/specs/rfc4918.html#HEADER_If diff --git a/endpoints/drive/driveServer/if_test.go b/drives/davServer/if_test.go similarity index 99% rename from endpoints/drive/driveServer/if_test.go rename to drives/davServer/if_test.go index 1df7eeeb3..b8cc8acfa 100644 --- a/endpoints/drive/driveServer/if_test.go +++ b/drives/davServer/if_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package driveServer +package davServer import ( "reflect" diff --git a/endpoints/drive/driveServer/internal/xml/README b/drives/davServer/internal/xml/README similarity index 100% rename from endpoints/drive/driveServer/internal/xml/README rename to drives/davServer/internal/xml/README diff --git a/endpoints/drive/driveServer/internal/xml/atom_test.go b/drives/davServer/internal/xml/atom_test.go similarity index 100% rename from endpoints/drive/driveServer/internal/xml/atom_test.go rename to drives/davServer/internal/xml/atom_test.go diff --git a/endpoints/drive/driveServer/internal/xml/example_test.go b/drives/davServer/internal/xml/example_test.go similarity index 100% rename from endpoints/drive/driveServer/internal/xml/example_test.go rename to drives/davServer/internal/xml/example_test.go diff --git a/endpoints/drive/driveServer/internal/xml/marshal.go b/drives/davServer/internal/xml/marshal.go similarity index 100% rename from endpoints/drive/driveServer/internal/xml/marshal.go rename to drives/davServer/internal/xml/marshal.go diff --git a/endpoints/drive/driveServer/internal/xml/marshal_test.go b/drives/davServer/internal/xml/marshal_test.go similarity index 100% rename from endpoints/drive/driveServer/internal/xml/marshal_test.go rename to drives/davServer/internal/xml/marshal_test.go diff --git a/endpoints/drive/driveServer/internal/xml/read.go b/drives/davServer/internal/xml/read.go similarity index 100% rename from endpoints/drive/driveServer/internal/xml/read.go rename to drives/davServer/internal/xml/read.go diff --git a/endpoints/drive/driveServer/internal/xml/read_test.go b/drives/davServer/internal/xml/read_test.go similarity index 100% rename from endpoints/drive/driveServer/internal/xml/read_test.go rename to drives/davServer/internal/xml/read_test.go diff --git a/endpoints/drive/driveServer/internal/xml/typeinfo.go b/drives/davServer/internal/xml/typeinfo.go similarity index 100% rename from endpoints/drive/driveServer/internal/xml/typeinfo.go rename to drives/davServer/internal/xml/typeinfo.go diff --git a/endpoints/drive/driveServer/internal/xml/xml.go b/drives/davServer/internal/xml/xml.go similarity index 100% rename from endpoints/drive/driveServer/internal/xml/xml.go rename to drives/davServer/internal/xml/xml.go diff --git a/endpoints/drive/driveServer/internal/xml/xml_test.go b/drives/davServer/internal/xml/xml_test.go similarity index 100% rename from endpoints/drive/driveServer/internal/xml/xml_test.go rename to drives/davServer/internal/xml/xml_test.go diff --git a/endpoints/drive/driveServer/litmus_test_server.go b/drives/davServer/litmus_test_server.go similarity index 100% rename from endpoints/drive/driveServer/litmus_test_server.go rename to drives/davServer/litmus_test_server.go diff --git a/endpoints/drive/driveServer/lock.go b/drives/davServer/lock.go similarity index 99% rename from endpoints/drive/driveServer/lock.go rename to drives/davServer/lock.go index caff970e9..871c7647a 100644 --- a/endpoints/drive/driveServer/lock.go +++ b/drives/davServer/lock.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package driveServer +package davServer import ( "container/heap" diff --git a/endpoints/drive/driveServer/lock_test.go b/drives/davServer/lock_test.go similarity index 99% rename from endpoints/drive/driveServer/lock_test.go rename to drives/davServer/lock_test.go index 765b80442..1791c4e00 100644 --- a/endpoints/drive/driveServer/lock_test.go +++ b/drives/davServer/lock_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package driveServer +package davServer import ( "fmt" diff --git a/endpoints/drive/driveServer/prop.go b/drives/davServer/prop.go similarity index 99% rename from endpoints/drive/driveServer/prop.go rename to drives/davServer/prop.go index c38d961e1..df812c7d5 100644 --- a/endpoints/drive/driveServer/prop.go +++ b/drives/davServer/prop.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package driveServer +package davServer import ( "bytes" diff --git a/endpoints/drive/driveServer/prop_test.go b/drives/davServer/prop_test.go similarity index 99% rename from endpoints/drive/driveServer/prop_test.go rename to drives/davServer/prop_test.go index b7338e3f8..e1ca3022c 100644 --- a/endpoints/drive/driveServer/prop_test.go +++ b/drives/davServer/prop_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package driveServer +package davServer import ( "context" diff --git a/endpoints/drive/driveServer/webdav.go b/drives/davServer/webdav.go similarity index 99% rename from endpoints/drive/driveServer/webdav.go rename to drives/davServer/webdav.go index 5b43da2a4..8e586b4dc 100644 --- a/endpoints/drive/driveServer/webdav.go +++ b/drives/davServer/webdav.go @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package driveServer provides a WebDAV server implementation. -package driveServer +// Package davServer provides a WebDAV server implementation. +package davServer import ( "errors" diff --git a/endpoints/drive/driveServer/webdav_test.go b/drives/davServer/webdav_test.go similarity index 99% rename from endpoints/drive/driveServer/webdav_test.go rename to drives/davServer/webdav_test.go index fbd5a43b5..996eb7970 100644 --- a/endpoints/drive/driveServer/webdav_test.go +++ b/drives/davServer/webdav_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package driveServer +package davServer import ( "context" diff --git a/endpoints/drive/driveServer/xml.go b/drives/davServer/xml.go similarity index 99% rename from endpoints/drive/driveServer/xml.go rename to drives/davServer/xml.go index ef76a2fe3..fc73e12d4 100644 --- a/endpoints/drive/driveServer/xml.go +++ b/drives/davServer/xml.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package driveServer +package davServer // The XML encoding is covered by Section 14. // http://www.webdav.org/specs/rfc4918.html#xml.element.definitions @@ -32,7 +32,7 @@ import ( // In the long term, this package should use the standard library's version // only, and the internal fork deleted, once // https://github.com/golang/go/issues/13400 is resolved. - ixml "github.com/openziti/zrok/endpoints/drive/driveServer/internal/xml" + ixml "github.com/openziti/zrok/drives/davServer/internal/xml" ) // http://www.webdav.org/specs/rfc4918.html#ELEMENT_lockinfo diff --git a/endpoints/drive/driveServer/xml_test.go b/drives/davServer/xml_test.go similarity index 99% rename from endpoints/drive/driveServer/xml_test.go rename to drives/davServer/xml_test.go index 1fb653296..09b7e3179 100644 --- a/endpoints/drive/driveServer/xml_test.go +++ b/drives/davServer/xml_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package driveServer +package davServer import ( "bytes" @@ -16,7 +16,7 @@ import ( "strings" "testing" - ixml "github.com/openziti/zrok/endpoints/drive/driveServer/internal/xml" + ixml "github.com/openziti/zrok/drives/davServer/internal/xml" ) func TestReadLockInfo(t *testing.T) { diff --git a/endpoints/drive/backend.go b/endpoints/drive/backend.go index f7e5df979..a9e29fbb0 100644 --- a/endpoints/drive/backend.go +++ b/endpoints/drive/backend.go @@ -4,8 +4,8 @@ import ( "fmt" "github.com/openziti/sdk-golang/ziti" "github.com/openziti/sdk-golang/ziti/edge" + "github.com/openziti/zrok/drives/davServer" "github.com/openziti/zrok/endpoints" - "github.com/openziti/zrok/endpoints/drive/driveServer" "github.com/pkg/errors" "net/http" "time" @@ -43,9 +43,9 @@ func NewBackend(cfg *BackendConfig) (*Backend, error) { return nil, err } - handler := &driveServer.Handler{ - FileSystem: driveServer.Dir(cfg.DriveRoot), - LockSystem: driveServer.NewMemLS(), + handler := &davServer.Handler{ + FileSystem: davServer.Dir(cfg.DriveRoot), + LockSystem: davServer.NewMemLS(), Logger: func(r *http.Request, err error) { if cfg.Requests != nil { cfg.Requests <- &endpoints.Request{ diff --git a/util/sync/filesystem.go b/util/sync/filesystem.go index 03eb549d0..b4fa7c011 100644 --- a/util/sync/filesystem.go +++ b/util/sync/filesystem.go @@ -3,7 +3,7 @@ package sync import ( "context" "fmt" - "github.com/openziti/zrok/endpoints/drive/driveServer" + "github.com/openziti/zrok/drives/davServer" "io" "io/fs" "os" @@ -59,7 +59,7 @@ func (t *FilesystemTarget) recurse(path string, d fs.DirEntry, err error) error return err } etag := "" - if v, ok := fi.(driveServer.ETager); ok { + if v, ok := fi.(davServer.ETager); ok { etag, err = v.ETag(context.Background()) if err != nil { return err From 8313f3f6866859af9ae3984f99bc5728f30ae544 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 10 Jan 2024 12:54:24 -0500 Subject: [PATCH 34/63] fix default source/target scheme (#438) --- cmd/zrok/copy.go | 18 +++++++++++++----- util/sync/filesystem.go | 2 ++ util/sync/zrok.go | 2 +- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/cmd/zrok/copy.go b/cmd/zrok/copy.go index 53799c043..3032207f7 100644 --- a/cmd/zrok/copy.go +++ b/cmd/zrok/copy.go @@ -7,6 +7,7 @@ import ( "github.com/openziti/zrok/sdk/golang/sdk" "github.com/openziti/zrok/tui" "github.com/openziti/zrok/util/sync" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" "net/url" ) @@ -35,6 +36,9 @@ func (cmd *copyCommand) run(_ *cobra.Command, args []string) { if err != nil { tui.Error(fmt.Sprintf("invalid source URL '%v'", args[0]), err) } + if sourceUrl.Scheme == "" { + sourceUrl.Scheme = "file" + } targetStr := "file://." if len(args) == 2 { @@ -44,6 +48,9 @@ func (cmd *copyCommand) run(_ *cobra.Command, args []string) { if err != nil { tui.Error(fmt.Sprintf("invalid target URL '%v'", targetStr), err) } + if targetUrl.Scheme == "" { + targetUrl.Scheme = "file" + } root, err := environment.LoadRoot() if err != nil { @@ -97,15 +104,16 @@ func (cmd *copyCommand) run(_ *cobra.Command, args []string) { fmt.Println("copy complete!") } -func (cmd *copyCommand) createTarget(t *url.URL, root env_core.Root) (sync.Target, error) { - switch t.Scheme { +func (cmd *copyCommand) createTarget(url *url.URL, root env_core.Root) (sync.Target, error) { + switch url.Scheme { case "file": - return sync.NewFilesystemTarget(&sync.FilesystemTargetConfig{Root: t.Path}), nil + logrus.Infof("%v", url) + return sync.NewFilesystemTarget(&sync.FilesystemTargetConfig{Root: url.Path}), nil case "zrok": - return sync.NewZrokTarget(&sync.ZrokTargetConfig{URL: t, Root: root}) + return sync.NewZrokTarget(&sync.ZrokTargetConfig{URL: url, Root: root}) default: - return sync.NewWebDAVTarget(&sync.WebDAVTargetConfig{URL: t, Username: "", Password: ""}) + return sync.NewWebDAVTarget(&sync.WebDAVTargetConfig{URL: url, Username: "", Password: ""}) } } diff --git a/util/sync/filesystem.go b/util/sync/filesystem.go index b4fa7c011..5437ba74b 100644 --- a/util/sync/filesystem.go +++ b/util/sync/filesystem.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/openziti/zrok/drives/davServer" + "github.com/sirupsen/logrus" "io" "io/fs" "os" @@ -22,6 +23,7 @@ type FilesystemTarget struct { } func NewFilesystemTarget(cfg *FilesystemTargetConfig) *FilesystemTarget { + logrus.Infof("root = %v", cfg.Root) root := os.DirFS(cfg.Root) return &FilesystemTarget{cfg: cfg, root: root} } diff --git a/util/sync/zrok.go b/util/sync/zrok.go index 737aa164a..ba938126c 100644 --- a/util/sync/zrok.go +++ b/util/sync/zrok.go @@ -49,7 +49,7 @@ func NewZrokTarget(cfg *ZrokTargetConfig) (*ZrokTarget, error) { } func (t *ZrokTarget) Inventory() ([]*Object, error) { - fis, err := t.dc.Readdir(context.Background(), "", true) + fis, err := t.dc.Readdir(context.Background(), "/", true) if err != nil { return nil, err } From 16ddb0a1f5ca0d0fa865d5923e9d1d0673060084 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 10 Jan 2024 13:38:35 -0500 Subject: [PATCH 35/63] synchronizer framework tweaks (#438) --- cmd/zrok/copy.go | 35 +++++++++++++---------------------- util/sync/filesystem.go | 10 +++++++--- util/sync/synchronizer.go | 1 + util/sync/zrok.go | 16 +++++++++++++++- 4 files changed, 36 insertions(+), 26 deletions(-) diff --git a/cmd/zrok/copy.go b/cmd/zrok/copy.go index 3032207f7..c2c59b047 100644 --- a/cmd/zrok/copy.go +++ b/cmd/zrok/copy.go @@ -7,7 +7,6 @@ import ( "github.com/openziti/zrok/sdk/golang/sdk" "github.com/openziti/zrok/tui" "github.com/openziti/zrok/util/sync" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" "net/url" ) @@ -22,8 +21,8 @@ type copyCommand struct { func newCopyCommand() *copyCommand { cmd := &cobra.Command{ - Use: "copy []", - Short: "Copy zrok drive contents from to ('file://' and 'zrok://' supported)", + Use: "copy [] ( defaults to 'file://.`)", + Short: "Copy (unidirectional sync) zrok drive contents from to ('http://', 'file://', and 'zrok://' supported)", Args: cobra.RangeArgs(1, 2), } command := ©Command{cmd: cmd} @@ -57,36 +56,29 @@ func (cmd *copyCommand) run(_ *cobra.Command, args []string) { tui.Error("error loading root", err) } - var srcAccess *sdk.Access + var allocatedAccesses []*sdk.Access if sourceUrl.Scheme == "zrok" { - srcAccess, err = sdk.CreateAccess(root, &sdk.AccessRequest{ShareToken: sourceUrl.Host}) + access, err := sdk.CreateAccess(root, &sdk.AccessRequest{ShareToken: sourceUrl.Host}) if err != nil { tui.Error("error creating access", err) } + allocatedAccesses = append(allocatedAccesses, access) } - if srcAccess != nil { - defer func() { - err := sdk.DeleteAccess(root, srcAccess) - if err != nil { - tui.Error("error deleting source access", err) - } - }() - } - var dstAccess *sdk.Access if targetUrl.Scheme == "zrok" { - dstAccess, err = sdk.CreateAccess(root, &sdk.AccessRequest{ShareToken: targetUrl.Host}) + access, err := sdk.CreateAccess(root, &sdk.AccessRequest{ShareToken: targetUrl.Host}) if err != nil { tui.Error("error creating access", err) } + allocatedAccesses = append(allocatedAccesses, access) } - if dstAccess != nil { - defer func() { - err := sdk.DeleteAccess(root, dstAccess) + defer func() { + for _, access := range allocatedAccesses { + err := sdk.DeleteAccess(root, access) if err != nil { - tui.Error("error deleting target access", err) + tui.Warning("error deleting target access", err) } - }() - } + } + }() source, err := cmd.createTarget(sourceUrl, root) if err != nil { @@ -107,7 +99,6 @@ func (cmd *copyCommand) run(_ *cobra.Command, args []string) { func (cmd *copyCommand) createTarget(url *url.URL, root env_core.Root) (sync.Target, error) { switch url.Scheme { case "file": - logrus.Infof("%v", url) return sync.NewFilesystemTarget(&sync.FilesystemTargetConfig{Root: url.Path}), nil case "zrok": diff --git a/util/sync/filesystem.go b/util/sync/filesystem.go index 5437ba74b..6878e4001 100644 --- a/util/sync/filesystem.go +++ b/util/sync/filesystem.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "github.com/openziti/zrok/drives/davServer" - "github.com/sirupsen/logrus" "io" "io/fs" "os" @@ -23,7 +22,6 @@ type FilesystemTarget struct { } func NewFilesystemTarget(cfg *FilesystemTargetConfig) *FilesystemTarget { - logrus.Infof("root = %v", cfg.Root) root := os.DirFS(cfg.Root) return &FilesystemTarget{cfg: cfg, root: root} } @@ -38,7 +36,13 @@ func (t *FilesystemTarget) Inventory() ([]*Object, error) { } if !fi.IsDir() { - return []*Object{{Path: "/" + t.cfg.Root, Size: fi.Size(), Modified: fi.ModTime()}}, nil + t.cfg.Root = filepath.Dir(t.cfg.Root) + return []*Object{{ + Path: "/" + fi.Name(), + IsDir: false, + Size: fi.Size(), + Modified: fi.ModTime(), + }}, nil } t.tree = nil diff --git a/util/sync/synchronizer.go b/util/sync/synchronizer.go index 09b1edf2d..91854ac2b 100644 --- a/util/sync/synchronizer.go +++ b/util/sync/synchronizer.go @@ -36,6 +36,7 @@ func Synchronize(src, dst Target) error { } for _, copyPath := range copyList { + logrus.Infof("copyPath: '%v' (%t)", copyPath.Path, copyPath.IsDir) if copyPath.IsDir { if err := dst.Mkdir(copyPath.Path); err != nil { return err diff --git a/util/sync/zrok.go b/util/sync/zrok.go index ba938126c..1cc1f478c 100644 --- a/util/sync/zrok.go +++ b/util/sync/zrok.go @@ -49,7 +49,21 @@ func NewZrokTarget(cfg *ZrokTargetConfig) (*ZrokTarget, error) { } func (t *ZrokTarget) Inventory() ([]*Object, error) { - fis, err := t.dc.Readdir(context.Background(), "/", true) + rootFi, err := t.dc.Stat(context.Background(), t.cfg.URL.Path) + if err != nil { + return nil, err + } + + if !rootFi.IsDir { + return []*Object{{ + Path: t.cfg.URL.Path, + IsDir: false, + Size: rootFi.Size, + Modified: rootFi.ModTime, + }}, nil + } + + fis, err := t.dc.Readdir(context.Background(), t.cfg.URL.Path, true) if err != nil { return nil, err } From b65d78588107d7e864de7f89b07c7f203fec6d85 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 10 Jan 2024 13:55:07 -0500 Subject: [PATCH 36/63] small semantics tweaks in synchronizer framework (#438) --- util/sync/synchronizer.go | 1 - util/sync/zrok.go | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/util/sync/synchronizer.go b/util/sync/synchronizer.go index 91854ac2b..09b1edf2d 100644 --- a/util/sync/synchronizer.go +++ b/util/sync/synchronizer.go @@ -36,7 +36,6 @@ func Synchronize(src, dst Target) error { } for _, copyPath := range copyList { - logrus.Infof("copyPath: '%v' (%t)", copyPath.Path, copyPath.IsDir) if copyPath.IsDir { if err := dst.Mkdir(copyPath.Path); err != nil { return err diff --git a/util/sync/zrok.go b/util/sync/zrok.go index 1cc1f478c..9b8164631 100644 --- a/util/sync/zrok.go +++ b/util/sync/zrok.go @@ -55,8 +55,10 @@ func (t *ZrokTarget) Inventory() ([]*Object, error) { } if !rootFi.IsDir { + base := filepath.Base(t.cfg.URL.Path) + t.cfg.URL.Path = filepath.Dir(t.cfg.URL.Path) return []*Object{{ - Path: t.cfg.URL.Path, + Path: "/" + base, IsDir: false, Size: rootFi.Size, Modified: rootFi.ModTime, From b6629d7fb481f376415f16bbc4c34aa0c481e124 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 10 Jan 2024 14:00:32 -0500 Subject: [PATCH 37/63] align http/s webdav with zrok wrapper (#438) --- util/sync/webdav.go | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/util/sync/webdav.go b/util/sync/webdav.go index ae323c144..f75e39359 100644 --- a/util/sync/webdav.go +++ b/util/sync/webdav.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "os" + "path/filepath" "time" ) @@ -30,6 +31,22 @@ func NewWebDAVTarget(cfg *WebDAVTargetConfig) (*WebDAVTarget, error) { } func (t *WebDAVTarget) Inventory() ([]*Object, error) { + rootFi, err := t.dc.Stat(context.Background(), t.cfg.URL.Path) + if err != nil { + return nil, err + } + + if !rootFi.IsDir { + base := filepath.Base(t.cfg.URL.Path) + t.cfg.URL.Path = filepath.Dir(t.cfg.URL.Path) + return []*Object{{ + Path: "/" + base, + IsDir: false, + Size: rootFi.Size, + Modified: rootFi.ModTime, + }}, nil + } + fis, err := t.dc.Readdir(context.Background(), "", true) if err != nil { return nil, err @@ -50,15 +67,15 @@ func (t *WebDAVTarget) Inventory() ([]*Object, error) { } func (t *WebDAVTarget) Mkdir(path string) error { - return t.dc.Mkdir(context.Background(), path) + return t.dc.Mkdir(context.Background(), filepath.Join(t.cfg.URL.Path, path)) } func (t *WebDAVTarget) ReadStream(path string) (io.ReadCloser, error) { - return t.dc.Open(context.Background(), path) + return t.dc.Open(context.Background(), filepath.Join(t.cfg.URL.Path, path)) } func (t *WebDAVTarget) WriteStream(path string, rs io.Reader, _ os.FileMode) error { - ws, err := t.dc.Create(context.Background(), path) + ws, err := t.dc.Create(context.Background(), filepath.Join(t.cfg.URL.Path, path)) if err != nil { return err } @@ -71,5 +88,5 @@ func (t *WebDAVTarget) WriteStream(path string, rs io.Reader, _ os.FileMode) err } func (t *WebDAVTarget) SetModificationTime(path string, mtime time.Time) error { - return t.dc.Touch(context.Background(), path, mtime) + return t.dc.Touch(context.Background(), filepath.Join(t.cfg.URL.Path, path), mtime) } From 4b7f5df1ee477045eee29799d9cac7331be4c119 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 10 Jan 2024 14:01:58 -0500 Subject: [PATCH 38/63] cli lint (438) --- cmd/zrok/copy.go | 7 ++++--- cmd/zrok/davtest.go | 50 --------------------------------------------- 2 files changed, 4 insertions(+), 53 deletions(-) delete mode 100644 cmd/zrok/davtest.go diff --git a/cmd/zrok/copy.go b/cmd/zrok/copy.go index c2c59b047..27c3f22e8 100644 --- a/cmd/zrok/copy.go +++ b/cmd/zrok/copy.go @@ -21,9 +21,10 @@ type copyCommand struct { func newCopyCommand() *copyCommand { cmd := &cobra.Command{ - Use: "copy [] ( defaults to 'file://.`)", - Short: "Copy (unidirectional sync) zrok drive contents from to ('http://', 'file://', and 'zrok://' supported)", - Args: cobra.RangeArgs(1, 2), + Use: "copy [] ( defaults to 'file://.`)", + Short: "Copy (unidirectional sync) zrok drive contents from to ('http://', 'file://', and 'zrok://' supported)", + Aliases: []string{"cp"}, + Args: cobra.RangeArgs(1, 2), } command := ©Command{cmd: cmd} cmd.Run = command.run diff --git a/cmd/zrok/davtest.go b/cmd/zrok/davtest.go deleted file mode 100644 index 20bffb35b..000000000 --- a/cmd/zrok/davtest.go +++ /dev/null @@ -1,50 +0,0 @@ -package main - -import ( - "context" - "github.com/openziti/zrok/drives/davClient" - "github.com/spf13/cobra" - "io" - "net/http" - "os" -) - -func init() { - rootCmd.AddCommand(newDavtestCommand().cmd) -} - -type davtestCommand struct { - cmd *cobra.Command -} - -func newDavtestCommand() *davtestCommand { - cmd := &cobra.Command{ - Use: "davtest", - Short: "WebDAV testing wrapper", - Args: cobra.ExactArgs(3), - } - command := &davtestCommand{cmd: cmd} - cmd.Run = command.run - return command -} - -func (cmd *davtestCommand) run(_ *cobra.Command, args []string) { - client, err := davClient.NewClient(http.DefaultClient, args[0]) - if err != nil { - panic(err) - } - ws, err := client.Open(context.Background(), args[1]) - if err != nil { - panic(err) - } - fs, err := os.Create(args[2]) - if err != nil { - panic(err) - } - _, err = io.Copy(fs, ws) - if err != nil { - panic(err) - } - ws.Close() - fs.Close() -} From c7436224628dbfcadc5be75046314110a0a0c9d7 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 10 Jan 2024 14:35:27 -0500 Subject: [PATCH 39/63] zrok ls (#438) --- cmd/zrok/copy.go | 18 ++---------- cmd/zrok/dir.go | 65 +++++++++++++++++++++++++++++++++++++++++ util/sync/filesystem.go | 21 +++++++++++++ util/sync/model.go | 1 + util/sync/target.go | 23 +++++++++++++++ util/sync/webdav.go | 20 ++++++++++++- util/sync/zrok.go | 19 ++++++++++++ 7 files changed, 150 insertions(+), 17 deletions(-) create mode 100644 cmd/zrok/dir.go create mode 100644 util/sync/target.go diff --git a/cmd/zrok/copy.go b/cmd/zrok/copy.go index 27c3f22e8..81eb43cd4 100644 --- a/cmd/zrok/copy.go +++ b/cmd/zrok/copy.go @@ -3,7 +3,6 @@ package main import ( "fmt" "github.com/openziti/zrok/environment" - "github.com/openziti/zrok/environment/env_core" "github.com/openziti/zrok/sdk/golang/sdk" "github.com/openziti/zrok/tui" "github.com/openziti/zrok/util/sync" @@ -81,11 +80,11 @@ func (cmd *copyCommand) run(_ *cobra.Command, args []string) { } }() - source, err := cmd.createTarget(sourceUrl, root) + source, err := sync.TargetForURL(sourceUrl, root) if err != nil { tui.Error("error creating target", err) } - target, err := cmd.createTarget(targetUrl, root) + target, err := sync.TargetForURL(targetUrl, root) if err != nil { tui.Error("error creating target", err) } @@ -96,16 +95,3 @@ func (cmd *copyCommand) run(_ *cobra.Command, args []string) { fmt.Println("copy complete!") } - -func (cmd *copyCommand) createTarget(url *url.URL, root env_core.Root) (sync.Target, error) { - switch url.Scheme { - case "file": - return sync.NewFilesystemTarget(&sync.FilesystemTargetConfig{Root: url.Path}), nil - - case "zrok": - return sync.NewZrokTarget(&sync.ZrokTargetConfig{URL: url, Root: root}) - - default: - return sync.NewWebDAVTarget(&sync.WebDAVTargetConfig{URL: url, Username: "", Password: ""}) - } -} diff --git a/cmd/zrok/dir.go b/cmd/zrok/dir.go new file mode 100644 index 000000000..ae1467380 --- /dev/null +++ b/cmd/zrok/dir.go @@ -0,0 +1,65 @@ +package main + +import ( + "fmt" + "github.com/openziti/zrok/environment" + "github.com/openziti/zrok/tui" + "github.com/openziti/zrok/util" + "github.com/openziti/zrok/util/sync" + "github.com/spf13/cobra" + "net/url" +) + +func init() { + rootCmd.AddCommand(newDirCommand().cmd) +} + +type dirCommand struct { + cmd *cobra.Command +} + +func newDirCommand() *dirCommand { + cmd := &cobra.Command{ + Use: "dir ", + Short: "List the contents of ('http://', 'zrok://', and 'file://' supported)", + Aliases: []string{"ls"}, + Args: cobra.ExactArgs(1), + } + command := &dirCommand{cmd: cmd} + cmd.Run = command.run + return command +} + +func (cmd *dirCommand) run(_ *cobra.Command, args []string) { + targetUrl, err := url.Parse(args[0]) + if err != nil { + tui.Error(fmt.Sprintf("invalid target URL '%v'", args[0]), err) + } + if targetUrl.Scheme == "" { + targetUrl.Scheme = "file" + } + + root, err := environment.LoadRoot() + if err != nil { + tui.Error("error loading root", err) + } + + target, err := sync.TargetForURL(targetUrl, root) + if err != nil { + tui.Error(fmt.Sprintf("error creating target for '%v'", targetUrl.String()), err) + } + + objects, err := target.Dir("/") + if err != nil { + tui.Error("error listing directory", err) + } + + for _, object := range objects { + if object.IsDir { + fmt.Printf("

%-32s\n", object.Path) + } else { + fmt.Printf(" %-32s %-16s %s\n", object.Path, util.BytesToSize(object.Size), object.Modified.String()) + } + } + fmt.Println() +} diff --git a/util/sync/filesystem.go b/util/sync/filesystem.go index 6878e4001..68721b415 100644 --- a/util/sync/filesystem.go +++ b/util/sync/filesystem.go @@ -52,6 +52,27 @@ func (t *FilesystemTarget) Inventory() ([]*Object, error) { return t.tree, nil } +func (t *FilesystemTarget) Dir(path string) ([]*Object, error) { + des, err := os.ReadDir(t.cfg.Root) + if err != nil { + return nil, err + } + var objects []*Object + for _, de := range des { + fi, err := de.Info() + if err != nil { + return nil, err + } + objects = append(objects, &Object{ + Path: de.Name(), + IsDir: de.IsDir(), + Size: fi.Size(), + Modified: fi.ModTime(), + }) + } + return objects, nil +} + func (t *FilesystemTarget) Mkdir(path string) error { return os.MkdirAll(filepath.Join(t.cfg.Root, path), os.ModePerm) } diff --git a/util/sync/model.go b/util/sync/model.go index a87aecadd..34b8306e1 100644 --- a/util/sync/model.go +++ b/util/sync/model.go @@ -16,6 +16,7 @@ type Object struct { type Target interface { Inventory() ([]*Object, error) + Dir(path string) ([]*Object, error) Mkdir(path string) error ReadStream(path string) (io.ReadCloser, error) WriteStream(path string, stream io.Reader, mode os.FileMode) error diff --git a/util/sync/target.go b/util/sync/target.go new file mode 100644 index 000000000..07635baee --- /dev/null +++ b/util/sync/target.go @@ -0,0 +1,23 @@ +package sync + +import ( + "github.com/openziti/zrok/environment/env_core" + "github.com/pkg/errors" + "net/url" +) + +func TargetForURL(url *url.URL, root env_core.Root) (Target, error) { + switch url.Scheme { + case "file": + return NewFilesystemTarget(&FilesystemTargetConfig{Root: url.Path}), nil + + case "zrok": + return NewZrokTarget(&ZrokTargetConfig{URL: url, Root: root}) + + case "http", "https": + return NewWebDAVTarget(&WebDAVTargetConfig{URL: url, Username: "", Password: ""}) + + default: + return nil, errors.Errorf("unknown URL scheme '%v'", url.Scheme) + } +} diff --git a/util/sync/webdav.go b/util/sync/webdav.go index f75e39359..6bba0e3ac 100644 --- a/util/sync/webdav.go +++ b/util/sync/webdav.go @@ -59,7 +59,25 @@ func (t *WebDAVTarget) Inventory() ([]*Object, error) { IsDir: fi.IsDir, Size: fi.Size, Modified: fi.ModTime, - ETag: fi.ETag, + }) + } + } + return objects, nil +} + +func (t *WebDAVTarget) Dir(path string) ([]*Object, error) { + fis, err := t.dc.Readdir(context.Background(), t.cfg.URL.Path, false) + if err != nil { + return nil, err + } + var objects []*Object + for _, fi := range fis { + if fi.Path != "/" { + objects = append(objects, &Object{ + Path: filepath.Base(fi.Path), + IsDir: fi.IsDir, + Size: fi.Size, + Modified: fi.ModTime, }) } } diff --git a/util/sync/zrok.go b/util/sync/zrok.go index 9b8164631..326fc3948 100644 --- a/util/sync/zrok.go +++ b/util/sync/zrok.go @@ -84,6 +84,25 @@ func (t *ZrokTarget) Inventory() ([]*Object, error) { return objects, nil } +func (t *ZrokTarget) Dir(path string) ([]*Object, error) { + fis, err := t.dc.Readdir(context.Background(), t.cfg.URL.Path, false) + if err != nil { + return nil, err + } + var objects []*Object + for _, fi := range fis { + if fi.Path != "/" { + objects = append(objects, &Object{ + Path: filepath.Base(fi.Path), + IsDir: fi.IsDir, + Size: fi.Size, + Modified: fi.ModTime, + }) + } + } + return objects, nil +} + func (t *ZrokTarget) Mkdir(path string) error { return t.dc.Mkdir(context.Background(), filepath.Join(t.cfg.URL.Path, path)) } From 3ccbfe2829661720e6ecc1f349cdc45a4bfe4ac1 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 10 Jan 2024 15:00:31 -0500 Subject: [PATCH 40/63] table output for zrok ls (#438) --- cmd/zrok/dir.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cmd/zrok/dir.go b/cmd/zrok/dir.go index ae1467380..5879e2e79 100644 --- a/cmd/zrok/dir.go +++ b/cmd/zrok/dir.go @@ -2,12 +2,14 @@ package main import ( "fmt" + "github.com/jedib0t/go-pretty/v6/table" "github.com/openziti/zrok/environment" "github.com/openziti/zrok/tui" "github.com/openziti/zrok/util" "github.com/openziti/zrok/util/sync" "github.com/spf13/cobra" "net/url" + "os" ) func init() { @@ -54,12 +56,16 @@ func (cmd *dirCommand) run(_ *cobra.Command, args []string) { tui.Error("error listing directory", err) } + tw := table.NewWriter() + tw.SetOutputMirror(os.Stdout) + tw.SetStyle(table.StyleLight) + tw.AppendHeader(table.Row{"type", "Name", "Size", "Modified"}) for _, object := range objects { if object.IsDir { - fmt.Printf(" %-32s\n", object.Path) + tw.AppendRow(table.Row{"DIR", object.Path, "", ""}) } else { - fmt.Printf(" %-32s %-16s %s\n", object.Path, util.BytesToSize(object.Size), object.Modified.String()) + tw.AppendRow(table.Row{"", object.Path, util.BytesToSize(object.Size), object.Modified.Local()}) } } - fmt.Println() + tw.Render() } From 6a1f4297493b15905bc447fc3d15816ef6dae7fe Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 10 Jan 2024 15:14:04 -0500 Subject: [PATCH 41/63] sort objects list (#438) --- cmd/zrok/dir.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/zrok/dir.go b/cmd/zrok/dir.go index 5879e2e79..07d2fcf19 100644 --- a/cmd/zrok/dir.go +++ b/cmd/zrok/dir.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/cobra" "net/url" "os" + "sort" ) func init() { @@ -55,6 +56,9 @@ func (cmd *dirCommand) run(_ *cobra.Command, args []string) { if err != nil { tui.Error("error listing directory", err) } + sort.Slice(objects, func(i, j int) bool { + return objects[i].Path < objects[j].Path + }) tw := table.NewWriter() tw.SetOutputMirror(os.Stdout) From 3b35250024a153335b29ea4eff661408b68d440e Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Thu, 11 Jan 2024 10:06:49 -0500 Subject: [PATCH 42/63] -> drives/sync (#438) --- cmd/zrok/copy.go | 2 +- cmd/zrok/dir.go | 2 +- {util => drives}/sync/filesystem.go | 0 {util => drives}/sync/model.go | 0 {util => drives}/sync/synchronizer.go | 0 {util => drives}/sync/target.go | 0 {util => drives}/sync/webdav.go | 0 {util => drives}/sync/zrok.go | 0 8 files changed, 2 insertions(+), 2 deletions(-) rename {util => drives}/sync/filesystem.go (100%) rename {util => drives}/sync/model.go (100%) rename {util => drives}/sync/synchronizer.go (100%) rename {util => drives}/sync/target.go (100%) rename {util => drives}/sync/webdav.go (100%) rename {util => drives}/sync/zrok.go (100%) diff --git a/cmd/zrok/copy.go b/cmd/zrok/copy.go index 81eb43cd4..4bc76b7d8 100644 --- a/cmd/zrok/copy.go +++ b/cmd/zrok/copy.go @@ -2,10 +2,10 @@ package main import ( "fmt" + "github.com/openziti/zrok/drives/sync" "github.com/openziti/zrok/environment" "github.com/openziti/zrok/sdk/golang/sdk" "github.com/openziti/zrok/tui" - "github.com/openziti/zrok/util/sync" "github.com/spf13/cobra" "net/url" ) diff --git a/cmd/zrok/dir.go b/cmd/zrok/dir.go index 07d2fcf19..437700d4a 100644 --- a/cmd/zrok/dir.go +++ b/cmd/zrok/dir.go @@ -3,10 +3,10 @@ package main import ( "fmt" "github.com/jedib0t/go-pretty/v6/table" + "github.com/openziti/zrok/drives/sync" "github.com/openziti/zrok/environment" "github.com/openziti/zrok/tui" "github.com/openziti/zrok/util" - "github.com/openziti/zrok/util/sync" "github.com/spf13/cobra" "net/url" "os" diff --git a/util/sync/filesystem.go b/drives/sync/filesystem.go similarity index 100% rename from util/sync/filesystem.go rename to drives/sync/filesystem.go diff --git a/util/sync/model.go b/drives/sync/model.go similarity index 100% rename from util/sync/model.go rename to drives/sync/model.go diff --git a/util/sync/synchronizer.go b/drives/sync/synchronizer.go similarity index 100% rename from util/sync/synchronizer.go rename to drives/sync/synchronizer.go diff --git a/util/sync/target.go b/drives/sync/target.go similarity index 100% rename from util/sync/target.go rename to drives/sync/target.go diff --git a/util/sync/webdav.go b/drives/sync/webdav.go similarity index 100% rename from util/sync/webdav.go rename to drives/sync/webdav.go diff --git a/util/sync/zrok.go b/drives/sync/zrok.go similarity index 100% rename from util/sync/zrok.go rename to drives/sync/zrok.go From e40df8c101fe3708ed4951ab505214e150b5bc6a Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Thu, 11 Jan 2024 11:36:05 -0500 Subject: [PATCH 43/63] 'zrok copy' now supports copy and oneway sync (#438) --- cmd/zrok/copy.go | 6 ++++-- drives/sync/synchronizer.go | 11 +++++++---- drives/sync/webdav.go | 8 ++++++++ drives/sync/zrok.go | 8 ++++++++ 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/cmd/zrok/copy.go b/cmd/zrok/copy.go index 4bc76b7d8..3995de977 100644 --- a/cmd/zrok/copy.go +++ b/cmd/zrok/copy.go @@ -15,7 +15,8 @@ func init() { } type copyCommand struct { - cmd *cobra.Command + cmd *cobra.Command + sync bool } func newCopyCommand() *copyCommand { @@ -27,6 +28,7 @@ func newCopyCommand() *copyCommand { } command := ©Command{cmd: cmd} cmd.Run = command.run + cmd.Flags().BoolVarP(&command.sync, "sync", "s", false, "Only copy modified files (one-way synchronize)") return command } @@ -89,7 +91,7 @@ func (cmd *copyCommand) run(_ *cobra.Command, args []string) { tui.Error("error creating target", err) } - if err := sync.Synchronize(source, target); err != nil { + if err := sync.OneWay(source, target, cmd.sync); err != nil { tui.Error("error copying", err) } diff --git a/drives/sync/synchronizer.go b/drives/sync/synchronizer.go index 09b1edf2d..0182e8eb4 100644 --- a/drives/sync/synchronizer.go +++ b/drives/sync/synchronizer.go @@ -6,15 +6,18 @@ import ( "os" ) -func Synchronize(src, dst Target) error { +func OneWay(src, dst Target, sync bool) error { srcTree, err := src.Inventory() if err != nil { return errors.Wrap(err, "error creating source inventory") } - dstTree, err := dst.Inventory() - if err != nil { - return errors.Wrap(err, "error creating destination inventory") + var dstTree []*Object + if sync { + dstTree, err = dst.Inventory() + if err != nil { + return errors.Wrap(err, "error creating destination inventory") + } } dstIndex := make(map[string]*Object) diff --git a/drives/sync/webdav.go b/drives/sync/webdav.go index 6bba0e3ac..6117f488e 100644 --- a/drives/sync/webdav.go +++ b/drives/sync/webdav.go @@ -3,6 +3,7 @@ package sync import ( "context" "github.com/openziti/zrok/drives/davClient" + "github.com/pkg/errors" "io" "net/http" "net/url" @@ -85,6 +86,13 @@ func (t *WebDAVTarget) Dir(path string) ([]*Object, error) { } func (t *WebDAVTarget) Mkdir(path string) error { + fi, err := t.dc.Stat(context.Background(), filepath.Join(t.cfg.URL.Path, path)) + if err == nil { + if fi.IsDir { + return nil + } + return errors.Errorf("'%v' already exists; not directory", path) + } return t.dc.Mkdir(context.Background(), filepath.Join(t.cfg.URL.Path, path)) } diff --git a/drives/sync/zrok.go b/drives/sync/zrok.go index 326fc3948..79fa08da2 100644 --- a/drives/sync/zrok.go +++ b/drives/sync/zrok.go @@ -5,6 +5,7 @@ import ( "github.com/openziti/zrok/drives/davClient" "github.com/openziti/zrok/environment/env_core" "github.com/openziti/zrok/sdk/golang/sdk" + "github.com/pkg/errors" "io" "net" "net/http" @@ -104,6 +105,13 @@ func (t *ZrokTarget) Dir(path string) ([]*Object, error) { } func (t *ZrokTarget) Mkdir(path string) error { + fi, err := t.dc.Stat(context.Background(), filepath.Join(t.cfg.URL.Path, path)) + if err == nil { + if fi.IsDir { + return nil + } + return errors.Errorf("'%v' already exists; not directory", path) + } return t.dc.Mkdir(context.Background(), filepath.Join(t.cfg.URL.Path, path)) } From badff9bd2cc55f5cb52ce65d2596d2257d7fe020 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Thu, 11 Jan 2024 11:44:17 -0500 Subject: [PATCH 44/63] remove directory root from directory listing (#438) --- drives/sync/webdav.go | 2 +- drives/sync/zrok.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drives/sync/webdav.go b/drives/sync/webdav.go index 6117f488e..8d53a4cc2 100644 --- a/drives/sync/webdav.go +++ b/drives/sync/webdav.go @@ -73,7 +73,7 @@ func (t *WebDAVTarget) Dir(path string) ([]*Object, error) { } var objects []*Object for _, fi := range fis { - if fi.Path != "/" { + if fi.Path != "/" && fi.Path != t.cfg.URL.Path+"/" { objects = append(objects, &Object{ Path: filepath.Base(fi.Path), IsDir: fi.IsDir, diff --git a/drives/sync/zrok.go b/drives/sync/zrok.go index 79fa08da2..4d036e0e9 100644 --- a/drives/sync/zrok.go +++ b/drives/sync/zrok.go @@ -92,7 +92,7 @@ func (t *ZrokTarget) Dir(path string) ([]*Object, error) { } var objects []*Object for _, fi := range fis { - if fi.Path != "/" { + if fi.Path != "/" && fi.Path != t.cfg.URL.Path+"/" { objects = append(objects, &Object{ Path: filepath.Base(fi.Path), IsDir: fi.IsDir, From fcb156d650c5282c0c16d5a49843298c0307a533 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Thu, 11 Jan 2024 11:50:25 -0500 Subject: [PATCH 45/63] dir -> ls (#438) --- cmd/zrok/{dir.go => ls.go} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename cmd/zrok/{dir.go => ls.go} (91%) diff --git a/cmd/zrok/dir.go b/cmd/zrok/ls.go similarity index 91% rename from cmd/zrok/dir.go rename to cmd/zrok/ls.go index 437700d4a..737fb941d 100644 --- a/cmd/zrok/dir.go +++ b/cmd/zrok/ls.go @@ -23,9 +23,9 @@ type dirCommand struct { func newDirCommand() *dirCommand { cmd := &cobra.Command{ - Use: "dir ", - Short: "List the contents of ('http://', 'zrok://', and 'file://' supported)", - Aliases: []string{"ls"}, + Use: "ls ", + Short: "List the contents of drive ('http://', 'zrok://', and 'file://' supported)", + Aliases: []string{"dir"}, Args: cobra.ExactArgs(1), } command := &dirCommand{cmd: cmd} From 21a470d60beabd28ea13712c9dece396e3fcf365 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Thu, 11 Jan 2024 12:11:21 -0500 Subject: [PATCH 46/63] 'zrok rm' (#438); lint removal and messaging cleanup --- cmd/zrok/copy.go | 8 +++--- cmd/zrok/ls.go | 16 ++++++------ cmd/zrok/rm.go | 54 +++++++++++++++++++++++++++++++++++++++ drives/sync/filesystem.go | 4 +++ drives/sync/model.go | 1 + drives/sync/webdav.go | 4 +++ drives/sync/zrok.go | 4 +++ 7 files changed, 79 insertions(+), 12 deletions(-) create mode 100644 cmd/zrok/rm.go diff --git a/cmd/zrok/copy.go b/cmd/zrok/copy.go index 3995de977..ddd359f51 100644 --- a/cmd/zrok/copy.go +++ b/cmd/zrok/copy.go @@ -35,7 +35,7 @@ func newCopyCommand() *copyCommand { func (cmd *copyCommand) run(_ *cobra.Command, args []string) { sourceUrl, err := url.Parse(args[0]) if err != nil { - tui.Error(fmt.Sprintf("invalid source URL '%v'", args[0]), err) + tui.Error(fmt.Sprintf("invalid source '%v'", args[0]), err) } if sourceUrl.Scheme == "" { sourceUrl.Scheme = "file" @@ -47,7 +47,7 @@ func (cmd *copyCommand) run(_ *cobra.Command, args []string) { } targetUrl, err := url.Parse(targetStr) if err != nil { - tui.Error(fmt.Sprintf("invalid target URL '%v'", targetStr), err) + tui.Error(fmt.Sprintf("invalid target '%v'", targetStr), err) } if targetUrl.Scheme == "" { targetUrl.Scheme = "file" @@ -84,11 +84,11 @@ func (cmd *copyCommand) run(_ *cobra.Command, args []string) { source, err := sync.TargetForURL(sourceUrl, root) if err != nil { - tui.Error("error creating target", err) + tui.Error(fmt.Sprintf("error creating target for '%v'", sourceUrl), err) } target, err := sync.TargetForURL(targetUrl, root) if err != nil { - tui.Error("error creating target", err) + tui.Error(fmt.Sprintf("error creating target for '%v'", targetUrl), err) } if err := sync.OneWay(source, target, cmd.sync); err != nil { diff --git a/cmd/zrok/ls.go b/cmd/zrok/ls.go index 737fb941d..aa3d287c8 100644 --- a/cmd/zrok/ls.go +++ b/cmd/zrok/ls.go @@ -14,29 +14,29 @@ import ( ) func init() { - rootCmd.AddCommand(newDirCommand().cmd) + rootCmd.AddCommand(newLsCommand().cmd) } -type dirCommand struct { +type lsCommand struct { cmd *cobra.Command } -func newDirCommand() *dirCommand { +func newLsCommand() *lsCommand { cmd := &cobra.Command{ Use: "ls ", - Short: "List the contents of drive ('http://', 'zrok://', and 'file://' supported)", + Short: "List the contents of drive ('http://', 'zrok://','file://')", Aliases: []string{"dir"}, Args: cobra.ExactArgs(1), } - command := &dirCommand{cmd: cmd} + command := &lsCommand{cmd: cmd} cmd.Run = command.run return command } -func (cmd *dirCommand) run(_ *cobra.Command, args []string) { +func (cmd *lsCommand) run(_ *cobra.Command, args []string) { targetUrl, err := url.Parse(args[0]) if err != nil { - tui.Error(fmt.Sprintf("invalid target URL '%v'", args[0]), err) + tui.Error(fmt.Sprintf("invalid target '%v'", args[0]), err) } if targetUrl.Scheme == "" { targetUrl.Scheme = "file" @@ -49,7 +49,7 @@ func (cmd *dirCommand) run(_ *cobra.Command, args []string) { target, err := sync.TargetForURL(targetUrl, root) if err != nil { - tui.Error(fmt.Sprintf("error creating target for '%v'", targetUrl.String()), err) + tui.Error(fmt.Sprintf("error creating target for '%v'", targetUrl), err) } objects, err := target.Dir("/") diff --git a/cmd/zrok/rm.go b/cmd/zrok/rm.go new file mode 100644 index 000000000..692d6ac0b --- /dev/null +++ b/cmd/zrok/rm.go @@ -0,0 +1,54 @@ +package main + +import ( + "fmt" + "github.com/openziti/zrok/drives/sync" + "github.com/openziti/zrok/environment" + "github.com/openziti/zrok/tui" + "github.com/spf13/cobra" + "net/url" +) + +func init() { + rootCmd.AddCommand(newRmCommand().cmd) +} + +type rmCommand struct { + cmd *cobra.Command +} + +func newRmCommand() *rmCommand { + cmd := &cobra.Command{ + Use: "rm ", + Short: "Remove (delete) the contents of drive ('http://', 'zrok://', 'file://')", + Aliases: []string{"del"}, + Args: cobra.ExactArgs(1), + } + command := &rmCommand{cmd: cmd} + cmd.Run = command.run + return command +} + +func (cmd *rmCommand) run(_ *cobra.Command, args []string) { + targetUrl, err := url.Parse(args[0]) + if err != nil { + tui.Error(fmt.Sprintf("invalid target '%v'", args[0]), err) + } + if targetUrl.Scheme == "" { + targetUrl.Scheme = "file" + } + + root, err := environment.LoadRoot() + if err != nil { + tui.Error("error loading root", err) + } + + target, err := sync.TargetForURL(targetUrl, root) + if err != nil { + tui.Error(fmt.Sprintf("error creating target for '%v'", targetUrl), err) + } + + if err := target.Rm("/"); err != nil { + tui.Error("error removing", err) + } +} diff --git a/drives/sync/filesystem.go b/drives/sync/filesystem.go index 68721b415..28f302977 100644 --- a/drives/sync/filesystem.go +++ b/drives/sync/filesystem.go @@ -131,6 +131,10 @@ func (t *FilesystemTarget) WriteStream(path string, stream io.Reader, mode os.Fi return nil } +func (t *FilesystemTarget) Rm(path string) error { + return os.RemoveAll(filepath.Join(t.cfg.Root, path)) +} + func (t *FilesystemTarget) SetModificationTime(path string, mtime time.Time) error { targetPath := filepath.Join(t.cfg.Root, path) if err := os.Chtimes(targetPath, time.Now(), mtime); err != nil { diff --git a/drives/sync/model.go b/drives/sync/model.go index 34b8306e1..499c4d974 100644 --- a/drives/sync/model.go +++ b/drives/sync/model.go @@ -20,5 +20,6 @@ type Target interface { Mkdir(path string) error ReadStream(path string) (io.ReadCloser, error) WriteStream(path string, stream io.Reader, mode os.FileMode) error + Rm(path string) error SetModificationTime(path string, mtime time.Time) error } diff --git a/drives/sync/webdav.go b/drives/sync/webdav.go index 8d53a4cc2..f5a2bdb56 100644 --- a/drives/sync/webdav.go +++ b/drives/sync/webdav.go @@ -113,6 +113,10 @@ func (t *WebDAVTarget) WriteStream(path string, rs io.Reader, _ os.FileMode) err return nil } +func (t *WebDAVTarget) Rm(path string) error { + return t.dc.RemoveAll(context.Background(), filepath.Join(t.cfg.URL.Path, path)) +} + func (t *WebDAVTarget) SetModificationTime(path string, mtime time.Time) error { return t.dc.Touch(context.Background(), filepath.Join(t.cfg.URL.Path, path), mtime) } diff --git a/drives/sync/zrok.go b/drives/sync/zrok.go index 4d036e0e9..be1d129cd 100644 --- a/drives/sync/zrok.go +++ b/drives/sync/zrok.go @@ -132,6 +132,10 @@ func (t *ZrokTarget) WriteStream(path string, rs io.Reader, _ os.FileMode) error return nil } +func (t *ZrokTarget) Rm(path string) error { + return t.dc.RemoveAll(context.Background(), filepath.Join(t.cfg.URL.Path, path)) +} + func (t *ZrokTarget) SetModificationTime(path string, mtime time.Time) error { return t.dc.Touch(context.Background(), filepath.Join(t.cfg.URL.Path, path), mtime) } From 492337ed8bac05bb638e579834013783409bccb7 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Thu, 11 Jan 2024 13:04:27 -0500 Subject: [PATCH 47/63] 'zrok md' (#438) --- cmd/zrok/md.go | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 cmd/zrok/md.go diff --git a/cmd/zrok/md.go b/cmd/zrok/md.go new file mode 100644 index 000000000..9f4da0969 --- /dev/null +++ b/cmd/zrok/md.go @@ -0,0 +1,54 @@ +package main + +import ( + "fmt" + "github.com/openziti/zrok/drives/sync" + "github.com/openziti/zrok/environment" + "github.com/openziti/zrok/tui" + "github.com/spf13/cobra" + "net/url" +) + +func init() { + rootCmd.AddCommand(newMdCommand().cmd) +} + +type mdCommand struct { + cmd *cobra.Command +} + +func newMdCommand() *mdCommand { + cmd := &cobra.Command{ + Use: "md ", + Short: "Make directory at ('http://', 'zrok://', 'file://')", + Aliases: []string{"mkdir"}, + Args: cobra.ExactArgs(1), + } + command := &mdCommand{cmd: cmd} + cmd.Run = command.run + return command +} + +func (cmd *mdCommand) run(_ *cobra.Command, args []string) { + targetUrl, err := url.Parse(args[0]) + if err != nil { + tui.Error(fmt.Sprintf("invalid target '%v'", args[0]), err) + } + if targetUrl.Scheme == "" { + targetUrl.Scheme = "file" + } + + root, err := environment.LoadRoot() + if err != nil { + tui.Error("error loading root", err) + } + + target, err := sync.TargetForURL(targetUrl, root) + if err != nil { + tui.Error(fmt.Sprintf("error creating target for '%v'", targetUrl), err) + } + + if err := target.Mkdir("/"); err != nil { + tui.Error("error creating directory", err) + } +} From f39a09efdf0fcf5c28b7c7c674bdda3164b3a34b Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Thu, 11 Jan 2024 13:19:20 -0500 Subject: [PATCH 48/63] 'zrok mv' (#438) --- cmd/zrok/mv.go | 54 +++++++++++++++++++++++++++++++++++++++ drives/sync/filesystem.go | 4 +++ drives/sync/model.go | 1 + drives/sync/webdav.go | 4 +++ drives/sync/zrok.go | 4 +++ 5 files changed, 67 insertions(+) create mode 100644 cmd/zrok/mv.go diff --git a/cmd/zrok/mv.go b/cmd/zrok/mv.go new file mode 100644 index 000000000..c30e52d9a --- /dev/null +++ b/cmd/zrok/mv.go @@ -0,0 +1,54 @@ +package main + +import ( + "fmt" + "github.com/openziti/zrok/drives/sync" + "github.com/openziti/zrok/environment" + "github.com/openziti/zrok/tui" + "github.com/spf13/cobra" + "net/url" +) + +func init() { + rootCmd.AddCommand(newMvCommand().cmd) +} + +type mvCommand struct { + cmd *cobra.Command +} + +func newMvCommand() *mvCommand { + cmd := &cobra.Command{ + Use: "mv ", + Short: "Move the drive to ('http://', 'zrok://', 'file://')", + Aliases: []string{"move"}, + Args: cobra.ExactArgs(2), + } + command := &mvCommand{cmd: cmd} + cmd.Run = command.run + return command +} + +func (cmd *mvCommand) run(_ *cobra.Command, args []string) { + targetUrl, err := url.Parse(args[0]) + if err != nil { + tui.Error(fmt.Sprintf("invalid target '%v'", args[0]), err) + } + if targetUrl.Scheme == "" { + targetUrl.Scheme = "file" + } + + root, err := environment.LoadRoot() + if err != nil { + tui.Error("error loading root", err) + } + + target, err := sync.TargetForURL(targetUrl, root) + if err != nil { + tui.Error(fmt.Sprintf("error creating target for '%v'", targetUrl), err) + } + + if err := target.Move("/", args[1]); err != nil { + tui.Error("error moving", err) + } +} diff --git a/drives/sync/filesystem.go b/drives/sync/filesystem.go index 28f302977..cde7271ed 100644 --- a/drives/sync/filesystem.go +++ b/drives/sync/filesystem.go @@ -131,6 +131,10 @@ func (t *FilesystemTarget) WriteStream(path string, stream io.Reader, mode os.Fi return nil } +func (t *FilesystemTarget) Move(src, dest string) error { + return os.Rename(filepath.Join(t.cfg.Root, src), filepath.Join(filepath.Dir(t.cfg.Root), dest)) +} + func (t *FilesystemTarget) Rm(path string) error { return os.RemoveAll(filepath.Join(t.cfg.Root, path)) } diff --git a/drives/sync/model.go b/drives/sync/model.go index 499c4d974..be0bd422e 100644 --- a/drives/sync/model.go +++ b/drives/sync/model.go @@ -20,6 +20,7 @@ type Target interface { Mkdir(path string) error ReadStream(path string) (io.ReadCloser, error) WriteStream(path string, stream io.Reader, mode os.FileMode) error + Move(src, dest string) error Rm(path string) error SetModificationTime(path string, mtime time.Time) error } diff --git a/drives/sync/webdav.go b/drives/sync/webdav.go index f5a2bdb56..3f76ef169 100644 --- a/drives/sync/webdav.go +++ b/drives/sync/webdav.go @@ -113,6 +113,10 @@ func (t *WebDAVTarget) WriteStream(path string, rs io.Reader, _ os.FileMode) err return nil } +func (t *WebDAVTarget) Move(src, dest string) error { + return t.dc.MoveAll(context.Background(), filepath.Join(t.cfg.URL.Path, src), dest, true) +} + func (t *WebDAVTarget) Rm(path string) error { return t.dc.RemoveAll(context.Background(), filepath.Join(t.cfg.URL.Path, path)) } diff --git a/drives/sync/zrok.go b/drives/sync/zrok.go index be1d129cd..48d78ff5a 100644 --- a/drives/sync/zrok.go +++ b/drives/sync/zrok.go @@ -132,6 +132,10 @@ func (t *ZrokTarget) WriteStream(path string, rs io.Reader, _ os.FileMode) error return nil } +func (t *ZrokTarget) Move(src, dest string) error { + return t.dc.MoveAll(context.Background(), filepath.Join(t.cfg.URL.Path, src), dest, true) +} + func (t *ZrokTarget) Rm(path string) error { return t.dc.RemoveAll(context.Background(), filepath.Join(t.cfg.URL.Path, path)) } From 9567c0ac572ebe03ee671927e04c2033b95d8219 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Thu, 11 Jan 2024 13:40:45 -0500 Subject: [PATCH 49/63] basic authentication support for 'zrok copy' and friends (#438) --- cmd/zrok/copy.go | 15 +++++++++++---- cmd/zrok/ls.go | 10 ++++++++-- cmd/zrok/md.go | 11 +++++++++-- cmd/zrok/mv.go | 11 +++++++++-- cmd/zrok/rm.go | 11 +++++++++-- drives/sync/target.go | 15 +++++++++++++-- drives/sync/webdav.go | 7 ++++++- drives/sync/zrok.go | 6 ++---- 8 files changed, 67 insertions(+), 19 deletions(-) diff --git a/cmd/zrok/copy.go b/cmd/zrok/copy.go index ddd359f51..f31c3b28c 100644 --- a/cmd/zrok/copy.go +++ b/cmd/zrok/copy.go @@ -8,6 +8,7 @@ import ( "github.com/openziti/zrok/tui" "github.com/spf13/cobra" "net/url" + "os" ) func init() { @@ -15,8 +16,9 @@ func init() { } type copyCommand struct { - cmd *cobra.Command - sync bool + cmd *cobra.Command + sync bool + basicAuth string } func newCopyCommand() *copyCommand { @@ -29,10 +31,15 @@ func newCopyCommand() *copyCommand { command := ©Command{cmd: cmd} cmd.Run = command.run cmd.Flags().BoolVarP(&command.sync, "sync", "s", false, "Only copy modified files (one-way synchronize)") + cmd.Flags().StringVarP(&command.basicAuth, "basic-auth", "a", "", "Basic authentication ") return command } func (cmd *copyCommand) run(_ *cobra.Command, args []string) { + if cmd.basicAuth == "" { + cmd.basicAuth = os.Getenv("ZROK_DRIVES_BASIC_AUTH") + } + sourceUrl, err := url.Parse(args[0]) if err != nil { tui.Error(fmt.Sprintf("invalid source '%v'", args[0]), err) @@ -82,11 +89,11 @@ func (cmd *copyCommand) run(_ *cobra.Command, args []string) { } }() - source, err := sync.TargetForURL(sourceUrl, root) + source, err := sync.TargetForURL(sourceUrl, root, cmd.basicAuth) if err != nil { tui.Error(fmt.Sprintf("error creating target for '%v'", sourceUrl), err) } - target, err := sync.TargetForURL(targetUrl, root) + target, err := sync.TargetForURL(targetUrl, root, cmd.basicAuth) if err != nil { tui.Error(fmt.Sprintf("error creating target for '%v'", targetUrl), err) } diff --git a/cmd/zrok/ls.go b/cmd/zrok/ls.go index aa3d287c8..3649f5d04 100644 --- a/cmd/zrok/ls.go +++ b/cmd/zrok/ls.go @@ -18,7 +18,8 @@ func init() { } type lsCommand struct { - cmd *cobra.Command + cmd *cobra.Command + basicAuth string } func newLsCommand() *lsCommand { @@ -30,10 +31,15 @@ func newLsCommand() *lsCommand { } command := &lsCommand{cmd: cmd} cmd.Run = command.run + cmd.Flags().StringVarP(&command.basicAuth, "basic-auth", "a", "", "Basic authentication ") return command } func (cmd *lsCommand) run(_ *cobra.Command, args []string) { + if cmd.basicAuth == "" { + cmd.basicAuth = os.Getenv("ZROK_DRIVES_BASIC_AUTH") + } + targetUrl, err := url.Parse(args[0]) if err != nil { tui.Error(fmt.Sprintf("invalid target '%v'", args[0]), err) @@ -47,7 +53,7 @@ func (cmd *lsCommand) run(_ *cobra.Command, args []string) { tui.Error("error loading root", err) } - target, err := sync.TargetForURL(targetUrl, root) + target, err := sync.TargetForURL(targetUrl, root, cmd.basicAuth) if err != nil { tui.Error(fmt.Sprintf("error creating target for '%v'", targetUrl), err) } diff --git a/cmd/zrok/md.go b/cmd/zrok/md.go index 9f4da0969..b065650e0 100644 --- a/cmd/zrok/md.go +++ b/cmd/zrok/md.go @@ -7,6 +7,7 @@ import ( "github.com/openziti/zrok/tui" "github.com/spf13/cobra" "net/url" + "os" ) func init() { @@ -14,7 +15,8 @@ func init() { } type mdCommand struct { - cmd *cobra.Command + cmd *cobra.Command + basicAuth string } func newMdCommand() *mdCommand { @@ -26,10 +28,15 @@ func newMdCommand() *mdCommand { } command := &mdCommand{cmd: cmd} cmd.Run = command.run + cmd.Flags().StringVarP(&command.basicAuth, "basic-auth", "a", "", "Basic authentication ") return command } func (cmd *mdCommand) run(_ *cobra.Command, args []string) { + if cmd.basicAuth == "" { + cmd.basicAuth = os.Getenv("ZROK_DRIVES_BASIC_AUTH") + } + targetUrl, err := url.Parse(args[0]) if err != nil { tui.Error(fmt.Sprintf("invalid target '%v'", args[0]), err) @@ -43,7 +50,7 @@ func (cmd *mdCommand) run(_ *cobra.Command, args []string) { tui.Error("error loading root", err) } - target, err := sync.TargetForURL(targetUrl, root) + target, err := sync.TargetForURL(targetUrl, root, cmd.basicAuth) if err != nil { tui.Error(fmt.Sprintf("error creating target for '%v'", targetUrl), err) } diff --git a/cmd/zrok/mv.go b/cmd/zrok/mv.go index c30e52d9a..de0f0ecde 100644 --- a/cmd/zrok/mv.go +++ b/cmd/zrok/mv.go @@ -7,6 +7,7 @@ import ( "github.com/openziti/zrok/tui" "github.com/spf13/cobra" "net/url" + "os" ) func init() { @@ -14,7 +15,8 @@ func init() { } type mvCommand struct { - cmd *cobra.Command + cmd *cobra.Command + basicAuth string } func newMvCommand() *mvCommand { @@ -26,10 +28,15 @@ func newMvCommand() *mvCommand { } command := &mvCommand{cmd: cmd} cmd.Run = command.run + cmd.Flags().StringVarP(&command.basicAuth, "basic-auth", "a", "", "Basic authentication ") return command } func (cmd *mvCommand) run(_ *cobra.Command, args []string) { + if cmd.basicAuth == "" { + cmd.basicAuth = os.Getenv("ZROK_DRIVES_BASIC_AUTH") + } + targetUrl, err := url.Parse(args[0]) if err != nil { tui.Error(fmt.Sprintf("invalid target '%v'", args[0]), err) @@ -43,7 +50,7 @@ func (cmd *mvCommand) run(_ *cobra.Command, args []string) { tui.Error("error loading root", err) } - target, err := sync.TargetForURL(targetUrl, root) + target, err := sync.TargetForURL(targetUrl, root, cmd.basicAuth) if err != nil { tui.Error(fmt.Sprintf("error creating target for '%v'", targetUrl), err) } diff --git a/cmd/zrok/rm.go b/cmd/zrok/rm.go index 692d6ac0b..6f018ef98 100644 --- a/cmd/zrok/rm.go +++ b/cmd/zrok/rm.go @@ -7,6 +7,7 @@ import ( "github.com/openziti/zrok/tui" "github.com/spf13/cobra" "net/url" + "os" ) func init() { @@ -14,7 +15,8 @@ func init() { } type rmCommand struct { - cmd *cobra.Command + cmd *cobra.Command + basicAuth string } func newRmCommand() *rmCommand { @@ -26,10 +28,15 @@ func newRmCommand() *rmCommand { } command := &rmCommand{cmd: cmd} cmd.Run = command.run + cmd.Flags().StringVarP(&command.basicAuth, "basic-auth", "a", "", "Basic authentication ") return command } func (cmd *rmCommand) run(_ *cobra.Command, args []string) { + if cmd.basicAuth == "" { + cmd.basicAuth = os.Getenv("ZROK_DRIVES_BASIC_AUTH") + } + targetUrl, err := url.Parse(args[0]) if err != nil { tui.Error(fmt.Sprintf("invalid target '%v'", args[0]), err) @@ -43,7 +50,7 @@ func (cmd *rmCommand) run(_ *cobra.Command, args []string) { tui.Error("error loading root", err) } - target, err := sync.TargetForURL(targetUrl, root) + target, err := sync.TargetForURL(targetUrl, root, cmd.basicAuth) if err != nil { tui.Error(fmt.Sprintf("error creating target for '%v'", targetUrl), err) } diff --git a/drives/sync/target.go b/drives/sync/target.go index 07635baee..5e32635cb 100644 --- a/drives/sync/target.go +++ b/drives/sync/target.go @@ -4,9 +4,10 @@ import ( "github.com/openziti/zrok/environment/env_core" "github.com/pkg/errors" "net/url" + "strings" ) -func TargetForURL(url *url.URL, root env_core.Root) (Target, error) { +func TargetForURL(url *url.URL, root env_core.Root, basicAuth string) (Target, error) { switch url.Scheme { case "file": return NewFilesystemTarget(&FilesystemTargetConfig{Root: url.Path}), nil @@ -15,7 +16,17 @@ func TargetForURL(url *url.URL, root env_core.Root) (Target, error) { return NewZrokTarget(&ZrokTargetConfig{URL: url, Root: root}) case "http", "https": - return NewWebDAVTarget(&WebDAVTargetConfig{URL: url, Username: "", Password: ""}) + var username string + var password string + if basicAuth != "" { + authTokens := strings.Split(basicAuth, ":") + if len(authTokens) != 2 { + return nil, errors.Errorf("invalid basic authentication (expect 'username:password')") + } + username = authTokens[0] + password = authTokens[1] + } + return NewWebDAVTarget(&WebDAVTargetConfig{URL: url, Username: username, Password: password}) default: return nil, errors.Errorf("unknown URL scheme '%v'", url.Scheme) diff --git a/drives/sync/webdav.go b/drives/sync/webdav.go index 3f76ef169..ea02ad31b 100644 --- a/drives/sync/webdav.go +++ b/drives/sync/webdav.go @@ -24,7 +24,12 @@ type WebDAVTarget struct { } func NewWebDAVTarget(cfg *WebDAVTargetConfig) (*WebDAVTarget, error) { - dc, err := davClient.NewClient(http.DefaultClient, cfg.URL.String()) + var httpClient davClient.HTTPClient + httpClient = http.DefaultClient + if cfg.Username != "" || cfg.Password != "" { + httpClient = davClient.HTTPClientWithBasicAuth(httpClient, cfg.Username, cfg.Password) + } + dc, err := davClient.NewClient(httpClient, cfg.URL.String()) if err != nil { return nil, err } diff --git a/drives/sync/zrok.go b/drives/sync/zrok.go index 48d78ff5a..3dd5a55fe 100644 --- a/drives/sync/zrok.go +++ b/drives/sync/zrok.go @@ -17,10 +17,8 @@ import ( ) type ZrokTargetConfig struct { - URL *url.URL - Username string - Password string - Root env_core.Root + URL *url.URL + Root env_core.Root } type ZrokTarget struct { From c2fb1440535e549a1a603487cc7bc5def02115ec Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Thu, 11 Jan 2024 13:55:16 -0500 Subject: [PATCH 50/63] ACKs --- ACKNOWLEDGEMENTS.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 ACKNOWLEDGEMENTS.md diff --git a/ACKNOWLEDGEMENTS.md b/ACKNOWLEDGEMENTS.md new file mode 100644 index 000000000..2154bfa30 --- /dev/null +++ b/ACKNOWLEDGEMENTS.md @@ -0,0 +1,27 @@ +# ACKNOWLEDGEMENTS + +## github.com/openziti/zrok/drives/davClient + +The `davClient` package is based on code from `github.com/emersion/go-webdav`, which included the following license: + +> The MIT License (MIT) +> +> Copyright (c) 2020 Simon Ser +> +> Permission is hereby granted, free of charge, to any person obtaining a copy +> of this software and associated documentation files (the "Software"), to deal +> in the Software without restriction, including without limitation the rights +> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +> copies of the Software, and to permit persons to whom the Software is +> furnished to do so, subject to the following conditions: +> +> The above copyright notice and this permission notice shall be included in all +> copies or substantial portions of the Software. +> +> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +> SOFTWARE. \ No newline at end of file From 7c742aebfad9014eb34e623df60ccdde2fa1f5cb Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Thu, 11 Jan 2024 14:14:48 -0500 Subject: [PATCH 51/63] better 'zrok copy' default target (#438) --- cmd/zrok/copy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/zrok/copy.go b/cmd/zrok/copy.go index f31c3b28c..80528b16a 100644 --- a/cmd/zrok/copy.go +++ b/cmd/zrok/copy.go @@ -48,7 +48,7 @@ func (cmd *copyCommand) run(_ *cobra.Command, args []string) { sourceUrl.Scheme = "file" } - targetStr := "file://." + targetStr := "." if len(args) == 2 { targetStr = args[1] } From 63ef223adaaaf39be888ce6a6a99dcdf67ead193 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 12 Jan 2024 13:11:11 -0500 Subject: [PATCH 52/63] missing access (#438) --- cmd/zrok/ls.go | 14 ++++++++++++++ cmd/zrok/md.go | 14 ++++++++++++++ cmd/zrok/mv.go | 14 ++++++++++++++ cmd/zrok/rm.go | 14 ++++++++++++++ 4 files changed, 56 insertions(+) diff --git a/cmd/zrok/ls.go b/cmd/zrok/ls.go index 3649f5d04..b76ab8e52 100644 --- a/cmd/zrok/ls.go +++ b/cmd/zrok/ls.go @@ -5,8 +5,10 @@ import ( "github.com/jedib0t/go-pretty/v6/table" "github.com/openziti/zrok/drives/sync" "github.com/openziti/zrok/environment" + "github.com/openziti/zrok/sdk/golang/sdk" "github.com/openziti/zrok/tui" "github.com/openziti/zrok/util" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" "net/url" "os" @@ -53,6 +55,18 @@ func (cmd *lsCommand) run(_ *cobra.Command, args []string) { tui.Error("error loading root", err) } + if targetUrl.Scheme == "zrok" { + access, err := sdk.CreateAccess(root, &sdk.AccessRequest{ShareToken: targetUrl.Host}) + if err != nil { + tui.Error("error creating access", err) + } + defer func() { + if err := sdk.DeleteAccess(root, access); err != nil { + logrus.Warningf("error freeing access: %v", err) + } + }() + } + target, err := sync.TargetForURL(targetUrl, root, cmd.basicAuth) if err != nil { tui.Error(fmt.Sprintf("error creating target for '%v'", targetUrl), err) diff --git a/cmd/zrok/md.go b/cmd/zrok/md.go index b065650e0..02e36c3c2 100644 --- a/cmd/zrok/md.go +++ b/cmd/zrok/md.go @@ -4,7 +4,9 @@ import ( "fmt" "github.com/openziti/zrok/drives/sync" "github.com/openziti/zrok/environment" + "github.com/openziti/zrok/sdk/golang/sdk" "github.com/openziti/zrok/tui" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" "net/url" "os" @@ -50,6 +52,18 @@ func (cmd *mdCommand) run(_ *cobra.Command, args []string) { tui.Error("error loading root", err) } + if targetUrl.Scheme == "zrok" { + access, err := sdk.CreateAccess(root, &sdk.AccessRequest{ShareToken: targetUrl.Host}) + if err != nil { + tui.Error("error creating access", err) + } + defer func() { + if err := sdk.DeleteAccess(root, access); err != nil { + logrus.Warningf("error freeing access: %v", err) + } + }() + } + target, err := sync.TargetForURL(targetUrl, root, cmd.basicAuth) if err != nil { tui.Error(fmt.Sprintf("error creating target for '%v'", targetUrl), err) diff --git a/cmd/zrok/mv.go b/cmd/zrok/mv.go index de0f0ecde..963239c29 100644 --- a/cmd/zrok/mv.go +++ b/cmd/zrok/mv.go @@ -4,7 +4,9 @@ import ( "fmt" "github.com/openziti/zrok/drives/sync" "github.com/openziti/zrok/environment" + "github.com/openziti/zrok/sdk/golang/sdk" "github.com/openziti/zrok/tui" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" "net/url" "os" @@ -50,6 +52,18 @@ func (cmd *mvCommand) run(_ *cobra.Command, args []string) { tui.Error("error loading root", err) } + if targetUrl.Scheme == "zrok" { + access, err := sdk.CreateAccess(root, &sdk.AccessRequest{ShareToken: targetUrl.Host}) + if err != nil { + tui.Error("error creating access", err) + } + defer func() { + if err := sdk.DeleteAccess(root, access); err != nil { + logrus.Warningf("error freeing access: %v", err) + } + }() + } + target, err := sync.TargetForURL(targetUrl, root, cmd.basicAuth) if err != nil { tui.Error(fmt.Sprintf("error creating target for '%v'", targetUrl), err) diff --git a/cmd/zrok/rm.go b/cmd/zrok/rm.go index 6f018ef98..25d4d5b8a 100644 --- a/cmd/zrok/rm.go +++ b/cmd/zrok/rm.go @@ -4,7 +4,9 @@ import ( "fmt" "github.com/openziti/zrok/drives/sync" "github.com/openziti/zrok/environment" + "github.com/openziti/zrok/sdk/golang/sdk" "github.com/openziti/zrok/tui" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" "net/url" "os" @@ -50,6 +52,18 @@ func (cmd *rmCommand) run(_ *cobra.Command, args []string) { tui.Error("error loading root", err) } + if targetUrl.Scheme == "zrok" { + access, err := sdk.CreateAccess(root, &sdk.AccessRequest{ShareToken: targetUrl.Host}) + if err != nil { + tui.Error("error creating access", err) + } + defer func() { + if err := sdk.DeleteAccess(root, access); err != nil { + logrus.Warningf("error freeing access: %v", err) + } + }() + } + target, err := sync.TargetForURL(targetUrl, root, cmd.basicAuth) if err != nil { tui.Error(fmt.Sprintf("error creating target for '%v'", targetUrl), err) From 21df8757163aaffcfe97c01076a10f1702dbe281 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 16 Jan 2024 13:54:01 -0500 Subject: [PATCH 53/63] last modified property handled through http header on PUT rather than PROPPATCH (#438) --- drives/davClient/client.go | 24 ++++++++++++++++++++++++ drives/davServer/webdav.go | 18 ++++++++++++++++++ drives/sync/filesystem.go | 4 ++++ drives/sync/model.go | 1 + drives/sync/synchronizer.go | 5 +---- drives/sync/webdav.go | 13 +++++++++++++ drives/sync/zrok.go | 13 +++++++++++++ 7 files changed, 74 insertions(+), 4 deletions(-) diff --git a/drives/davClient/client.go b/drives/davClient/client.go index cb0b63cd0..bf00c2f15 100644 --- a/drives/davClient/client.go +++ b/drives/davClient/client.go @@ -205,6 +205,30 @@ func (c *Client) Create(ctx context.Context, name string) (io.WriteCloser, error return &fileWriter{pw, done}, nil } +func (c *Client) CreateWithModTime(ctx context.Context, name string, modTime time.Time) (io.WriteCloser, error) { + pr, pw := io.Pipe() + + req, err := c.ic.NewRequest(http.MethodPut, name, pr) + if err != nil { + pw.Close() + return nil, err + } + req.Header.Set("Zrok-Modtime", fmt.Sprintf("%d", modTime.Unix())) + + done := make(chan error, 1) + go func() { + resp, err := c.ic.Do(req.WithContext(ctx)) + if err != nil { + done <- err + return + } + resp.Body.Close() + done <- nil + }() + + return &fileWriter{pw, done}, nil +} + func (c *Client) Touch(ctx context.Context, path string, mtime time.Time) error { status, err := c.ic.Touch(ctx, path, mtime) if err != nil { diff --git a/drives/davServer/webdav.go b/drives/davServer/webdav.go index 8e586b4dc..6d516831b 100644 --- a/drives/davServer/webdav.go +++ b/drives/davServer/webdav.go @@ -8,12 +8,14 @@ package davServer import ( "errors" "fmt" + "github.com/sirupsen/logrus" "io" "net/http" "net/url" "os" "path" "path/filepath" + "strconv" "strings" "time" ) @@ -271,6 +273,22 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, } _, copyErr := io.Copy(f, r.Body) fi, statErr := f.Stat() + + modTimes := r.Header["Zrok-Modtime"] + if len(modTimes) > 0 { + if modTimeV, err := strconv.ParseInt(modTimes[0], 10, 64); err == nil { + if v, ok := f.(*webdavFile); ok { + if err := v.updateModtime(reqPath, time.Unix(modTimeV, 0)); err != nil { + logrus.Warn(err) + } + } else { + logrus.Error("!ok") + } + } else { + logrus.Error(err) + } + } + closeErr := f.Close() // TODO(rost): Returning 405 Method Not Allowed might not be appropriate. if copyErr != nil { diff --git a/drives/sync/filesystem.go b/drives/sync/filesystem.go index cde7271ed..153f929c0 100644 --- a/drives/sync/filesystem.go +++ b/drives/sync/filesystem.go @@ -131,6 +131,10 @@ func (t *FilesystemTarget) WriteStream(path string, stream io.Reader, mode os.Fi return nil } +func (t *FilesystemTarget) WriteStreamWithModTime(path string, stream io.Reader, mode os.FileMode, modTime time.Time) error { + return t.WriteStream(path, stream, mode) +} + func (t *FilesystemTarget) Move(src, dest string) error { return os.Rename(filepath.Join(t.cfg.Root, src), filepath.Join(filepath.Dir(t.cfg.Root), dest)) } diff --git a/drives/sync/model.go b/drives/sync/model.go index be0bd422e..41a5c61f3 100644 --- a/drives/sync/model.go +++ b/drives/sync/model.go @@ -20,6 +20,7 @@ type Target interface { Mkdir(path string) error ReadStream(path string) (io.ReadCloser, error) WriteStream(path string, stream io.Reader, mode os.FileMode) error + WriteStreamWithModTime(path string, stream io.Reader, mode os.FileMode, modTime time.Time) error Move(src, dest string) error Rm(path string) error SetModificationTime(path string, mtime time.Time) error diff --git a/drives/sync/synchronizer.go b/drives/sync/synchronizer.go index 0182e8eb4..d6fe93d7a 100644 --- a/drives/sync/synchronizer.go +++ b/drives/sync/synchronizer.go @@ -48,10 +48,7 @@ func OneWay(src, dst Target, sync bool) error { if err != nil { return err } - if err := dst.WriteStream(copyPath.Path, ss, os.ModePerm); err != nil { - return err - } - if err := dst.SetModificationTime(copyPath.Path, copyPath.Modified); err != nil { + if err := dst.WriteStreamWithModTime(copyPath.Path, ss, os.ModePerm, copyPath.Modified); err != nil { return err } } diff --git a/drives/sync/webdav.go b/drives/sync/webdav.go index ea02ad31b..26045e756 100644 --- a/drives/sync/webdav.go +++ b/drives/sync/webdav.go @@ -118,6 +118,19 @@ func (t *WebDAVTarget) WriteStream(path string, rs io.Reader, _ os.FileMode) err return nil } +func (t *WebDAVTarget) WriteStreamWithModTime(path string, rs io.Reader, _ os.FileMode, modTime time.Time) error { + ws, err := t.dc.CreateWithModTime(context.Background(), filepath.Join(t.cfg.URL.Path, path), modTime) + if err != nil { + return err + } + defer func() { _ = ws.Close() }() + _, err = io.Copy(ws, rs) + if err != nil { + return err + } + return nil +} + func (t *WebDAVTarget) Move(src, dest string) error { return t.dc.MoveAll(context.Background(), filepath.Join(t.cfg.URL.Path, src), dest, true) } diff --git a/drives/sync/zrok.go b/drives/sync/zrok.go index 3dd5a55fe..0971b25ea 100644 --- a/drives/sync/zrok.go +++ b/drives/sync/zrok.go @@ -130,6 +130,19 @@ func (t *ZrokTarget) WriteStream(path string, rs io.Reader, _ os.FileMode) error return nil } +func (t *ZrokTarget) WriteStreamWithModTime(path string, rs io.Reader, _ os.FileMode, modTime time.Time) error { + ws, err := t.dc.CreateWithModTime(context.Background(), filepath.Join(t.cfg.URL.Path, path), modTime) + if err != nil { + return err + } + defer func() { _ = ws.Close() }() + _, err = io.Copy(ws, rs) + if err != nil { + return err + } + return nil +} + func (t *ZrokTarget) Move(src, dest string) error { return t.dc.MoveAll(context.Background(), filepath.Join(t.cfg.URL.Path, src), dest, true) } From c753c3514449d4d42f0a93cf9b42ada85eacd261 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 16 Jan 2024 15:42:47 -0500 Subject: [PATCH 54/63] changelog (#438) --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 833e62915..5f6f37790 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## v0.4.23 + +FEATURE: New CLI commands have been implemented for working with the `drive` share backend mode (part of the "zrok Drives" functionality). These commands include `zrok cp`, `zrok mkdir` `zrok mv`, `zrok ls`, and `zrok rm`. These are initial, minimal versions of these commands and very likely contain bugs and ergonomic annoyances. There is a guide available at (`docs/guides/drives/zrok_copy.md`) that explains how to work with these tools in detail (https://github.com/openziti/zrok/issues/438) + ## v0.4.22 FIX: The goreleaser action is not updated to work with the latest golang build. Modifed `go.mod` to comply with what goreleaser expects From 876039961ca12cbdc74044d83c4dd5e49f6381a8 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 16 Jan 2024 15:46:11 -0500 Subject: [PATCH 55/63] starting on the 'zrok copy' docs (#438) --- docs/guides/drives/zrok_copy.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 docs/guides/drives/zrok_copy.md diff --git a/docs/guides/drives/zrok_copy.md b/docs/guides/drives/zrok_copy.md new file mode 100644 index 000000000..7c3742c04 --- /dev/null +++ b/docs/guides/drives/zrok_copy.md @@ -0,0 +1,3 @@ +# Using the zrok Drives CLI + +The zrok Drives CLI tools allow for simple, ergonomic management and synchronization of local and remote file objects transparently. \ No newline at end of file From d4f0b3636df71b22f8887dbbea32e488008b9e03 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Tue, 16 Jan 2024 15:48:30 -0500 Subject: [PATCH 56/63] better title (#438) --- docs/guides/drives/zrok_copy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/drives/zrok_copy.md b/docs/guides/drives/zrok_copy.md index 7c3742c04..4872f9735 100644 --- a/docs/guides/drives/zrok_copy.md +++ b/docs/guides/drives/zrok_copy.md @@ -1,3 +1,3 @@ -# Using the zrok Drives CLI +# The zrok Drives CLI The zrok Drives CLI tools allow for simple, ergonomic management and synchronization of local and remote file objects transparently. \ No newline at end of file From 960397a41c05a1cfbd7aa4431da300f73b1c86c5 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Wed, 17 Jan 2024 16:51:08 -0500 Subject: [PATCH 57/63] changelog --- CHANGELOG.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d832e375e..2d5b570e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ FEATURE: New CLI commands have been implemented for working with the `drive` share backend mode (part of the "zrok Drives" functionality). These commands include `zrok cp`, `zrok mkdir` `zrok mv`, `zrok ls`, and `zrok rm`. These are initial, minimal versions of these commands and very likely contain bugs and ergonomic annoyances. There is a guide available at (`docs/guides/drives/zrok_copy.md`) that explains how to work with these tools in detail (https://github.com/openziti/zrok/issues/438) +FEATURE: Python SDK now has a decorator for integrating with various server side frameworks. See the `http-server` example. + +FEATURE: Python SDK share and access handling now supports context management. + +FEATURE: TLS for `zrok` controller and frontends. Add the `tls:` stanza to your controller configuration (see `etc/ctrl.yml`) to enable TLS support for the controller API. Add the `tls:` stanza to your frontend configuration (see `etc/frontend.yml`) to enable TLS support for frontends (be sure to check your `public` frontend template) (#24)(https://github.com/openziti/zrok/issues/24) + CHANGE: Improved OpenZiti resource cleanup resilience. Previous resource cleanup would stop when an error was encountered at any stage of the cleanup process (serps, sps, config, service). New cleanup implementation logs errors but continues to clean up anything that it can (https://github.com/openziti/zrok/issues/533) CHANGE: Instead of setting the `ListenOptions.MaxConnections` property to `64`, use the default value of `3`. This property actually controls the number of terminators created on the underlying OpenZiti network. This property is actually getting renamed to `ListenOptions.MaxTerminators` in an upcoming release of `github.com/openziti/sdk-golang` (https://github.com/openziti/zrok/issues/535) @@ -12,14 +18,6 @@ CHANGE: Versioning for the Python SDK has been updated to use versioneer for man CHANGE: Python SDK package name has been renamed to `zrok`, dropping the `-sdk` postfix. [pypi](https://pypi.org/project/zrok). -FEATURE: Python SDK now has a decorator for integrating with various server side frameworks. See the `http-server` example. - -FEATURE: Python SDK share and access handling now supports context management. - -FEATURE: TLS for `zrok` controller and acces endpoints. Add the specified stanza to your controller file (see `etc/ctrl.yml`). Your controller will now listen over TLS. (Note: you will need to update your client environments/configs to use the new https:// url). Likewise with `access` add the stanza to your frontend configuration (see `etc/frontend.yml`). Additionally you will have to update the frontend url template to emit a https:// scheme. - -FEATURE: TLS for `zrok` controller and frontends. Add the `tls:` stanza to your controller configuration (see `etc/ctrl.yml`) to enable TLS support for the controller API. Add the `tls:` stanza to your frontend configuration (see `etc/frontend.yml`) to enable TLS support for frontends (be sure to check your `public` frontend template) (#24)(https://github.com/openziti/zrok/issues/24) - ## v0.4.22 FIX: The goreleaser action is not updated to work with the latest golang build. Modifed `go.mod` to comply with what goreleaser expects From 9f778efdeeb419b79ab650b5af5bc4730c57d545 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 19 Jan 2024 12:32:06 -0500 Subject: [PATCH 58/63] snapshot (#438) --- docs/guides/drives/zrok_copy.md | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/guides/drives/zrok_copy.md b/docs/guides/drives/zrok_copy.md index 4872f9735..266325df8 100644 --- a/docs/guides/drives/zrok_copy.md +++ b/docs/guides/drives/zrok_copy.md @@ -1,3 +1,30 @@ # The zrok Drives CLI -The zrok Drives CLI tools allow for simple, ergonomic management and synchronization of local and remote file objects transparently. \ No newline at end of file +The zrok Drives CLI tools allow for simple, ergonomic management and synchronization of local and remote file objects transparently. + +## Sharing a Drive + +Virtual drives are shared through the `zrok` CLI using the `--backend-mode drive` flag with the `zrok share` command, using either the `public` or `private` sharing modes: + +``` +$ mkdir /tmp/junk +$ zrok share private --headless --backend-mode drive /tmp/junk +[ 0.124] INFO sdk-golang/ziti.(*listenerManager).createSessionWithBackoff: {session token=[cf640aac-2706-49ae-9cc9-9a497d67d9c5]} new service session +[ 0.145] INFO main.(*sharePrivateCommand).run: allow other to access your share with the following command: +zrok access private wkcfb58vj51l +``` + +The command shown above creates an ephemeral `zrok` drive share pointed at the local `/tmp/junk` folder. + +Notice that the share token allocated by `zrok` is `wkcfb58vj51l`. We'll use that share token to identify our virtual drive in the following operations. + +## Working with the Drive Share + +First, let's copy a file into our virtual drive using the `zrok copy` command: + +``` +$ zrok copy LICENSE zrok://wkcfb58vj51l +[ 0.119] INFO zrok/drives/sync.OneWay: => /LICENSE +copy complete! +``` + From 7ce74d5e50f6449a2163a514d3b76a45d47dd062 Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 19 Jan 2024 12:33:31 -0500 Subject: [PATCH 59/63] snapshot (#438) --- docs/guides/drives/zrok_copy.md | 76 +++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/docs/guides/drives/zrok_copy.md b/docs/guides/drives/zrok_copy.md index 266325df8..49d9ee50f 100644 --- a/docs/guides/drives/zrok_copy.md +++ b/docs/guides/drives/zrok_copy.md @@ -28,3 +28,79 @@ $ zrok copy LICENSE zrok://wkcfb58vj51l copy complete! ``` +We used the URL scheme `zrok://` to refer to the private virtual drive we allocated above using the `zrok share private` command. Use `zrok://` URLs with the drives CLI tools to refer to contents of private virtual drives. + +Next, let's get a directory listing of the virtual drive: + +``` +$ zrok ls zrok://wkcfb58vj51l +┌──────┬─────────┬─────────┬───────────────────────────────┐ +│ TYPE │ NAME │ SIZE │ MODIFIED │ +├──────┼─────────┼─────────┼───────────────────────────────┤ +│ │ LICENSE │ 11.3 kB │ 2024-01-19 12:16:46 -0500 EST │ +└──────┴─────────┴─────────┴───────────────────────────────┘ +``` + +We can make directories on the virtual drive: + +``` +$ zrok mkdir zrok://wkcfb58vj51l/stuff +$ zrok ls zrok://wkcfb58vj51l +┌──────┬─────────┬─────────┬───────────────────────────────┐ +│ TYPE │ NAME │ SIZE │ MODIFIED │ +├──────┼─────────┼─────────┼───────────────────────────────┤ +│ │ LICENSE │ 11.3 kB │ 2024-01-19 12:16:46 -0500 EST │ +│ DIR │ stuff │ │ │ +└──────┴─────────┴─────────┴───────────────────────────────┘ +``` + +We can copy the contents of a local directory into the new directory on the virtual drive: + +``` +$ ls -l util/ +total 20 +-rw-rw-r-- 1 michael michael 329 Jul 21 13:17 email.go +-rw-rw-r-- 1 michael michael 456 Jul 21 13:17 headers.go +-rw-rw-r-- 1 michael michael 609 Jul 21 13:17 proxy.go +-rw-rw-r-- 1 michael michael 361 Jul 21 13:17 size.go +-rw-rw-r-- 1 michael michael 423 Jan 2 11:57 uniqueName.go +$ zrok copy util/ zrok://wkcfb58vj51l/stuff +[ 0.123] INFO zrok/drives/sync.OneWay: => /email.go +[ 0.194] INFO zrok/drives/sync.OneWay: => /headers.go +[ 0.267] INFO zrok/drives/sync.OneWay: => /proxy.go +[ 0.337] INFO zrok/drives/sync.OneWay: => /size.go +[ 0.408] INFO zrok/drives/sync.OneWay: => /uniqueName.go +copy complete! +$ zrok ls zrok://wkcfb58vj51l/stuff +┌──────┬───────────────┬───────┬───────────────────────────────┐ +│ TYPE │ NAME │ SIZE │ MODIFIED │ +├──────┼───────────────┼───────┼───────────────────────────────┤ +│ │ email.go │ 329 B │ 2024-01-19 12:26:45 -0500 EST │ +│ │ headers.go │ 456 B │ 2024-01-19 12:26:45 -0500 EST │ +│ │ proxy.go │ 609 B │ 2024-01-19 12:26:45 -0500 EST │ +│ │ size.go │ 361 B │ 2024-01-19 12:26:45 -0500 EST │ +│ │ uniqueName.go │ 423 B │ 2024-01-19 12:26:45 -0500 EST │ +└──────┴───────────────┴───────┴───────────────────────────────┘ +``` + +And we can remove files and directories from the virtual drive: + +``` +$ zrok rm zrok://wkcfb58vj51l/LICENSE +michael@fourtyfour Fri Jan 19 12:29:12 ~/Repos/nf/zrok +$ zrok ls zrok://wkcfb58vj51l +┌──────┬───────┬──────┬──────────┐ +│ TYPE │ NAME │ SIZE │ MODIFIED │ +├──────┼───────┼──────┼──────────┤ +│ DIR │ stuff │ │ │ +└──────┴───────┴──────┴──────────┘ +michael@fourtyfour Fri Jan 19 12:29:14 ~/Repos/nf/zrok +$ zrok rm zrok://wkcfb58vj51l/stuff +michael@fourtyfour Fri Jan 19 12:29:20 ~/Repos/nf/zrok +$ zrok ls zrok://wkcfb58vj51l +┌──────┬──────┬──────┬──────────┐ +│ TYPE │ NAME │ SIZE │ MODIFIED │ +├──────┼──────┼──────┼──────────┤ +└──────┴──────┴──────┴──────────┘ +``` + From 0a9e8c0de8ac5f94e6c046a5448e854a772c8f7f Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 19 Jan 2024 13:18:49 -0500 Subject: [PATCH 60/63] snapshot (#438) --- docs/guides/drives/zrok_copy.md | 148 ++++++++++++++++++++++++++++++-- 1 file changed, 141 insertions(+), 7 deletions(-) diff --git a/docs/guides/drives/zrok_copy.md b/docs/guides/drives/zrok_copy.md index 49d9ee50f..b7835e085 100644 --- a/docs/guides/drives/zrok_copy.md +++ b/docs/guides/drives/zrok_copy.md @@ -1,10 +1,10 @@ -# The zrok Drives CLI +# The Drives CLI The zrok Drives CLI tools allow for simple, ergonomic management and synchronization of local and remote file objects transparently. ## Sharing a Drive -Virtual drives are shared through the `zrok` CLI using the `--backend-mode drive` flag with the `zrok share` command, using either the `public` or `private` sharing modes: +Virtual drives are shared through the `zrok` CLI using the `--backend-mode drive` flag with the `zrok share` command, using either the `public` or `private` sharing modes. We'll use the `private` sharing mode for this example: ``` $ mkdir /tmp/junk @@ -14,11 +14,11 @@ $ zrok share private --headless --backend-mode drive /tmp/junk zrok access private wkcfb58vj51l ``` -The command shown above creates an ephemeral `zrok` drive share pointed at the local `/tmp/junk` folder. +The command shown above creates an ephemeral, `private` drive share pointed at the local `/tmp/junk` folder. Notice that the share token allocated by `zrok` is `wkcfb58vj51l`. We'll use that share token to identify our virtual drive in the following operations. -## Working with the Drive Share +## Working with a Private Drive Share First, let's copy a file into our virtual drive using the `zrok copy` command: @@ -87,16 +87,13 @@ And we can remove files and directories from the virtual drive: ``` $ zrok rm zrok://wkcfb58vj51l/LICENSE -michael@fourtyfour Fri Jan 19 12:29:12 ~/Repos/nf/zrok $ zrok ls zrok://wkcfb58vj51l ┌──────┬───────┬──────┬──────────┐ │ TYPE │ NAME │ SIZE │ MODIFIED │ ├──────┼───────┼──────┼──────────┤ │ DIR │ stuff │ │ │ └──────┴───────┴──────┴──────────┘ -michael@fourtyfour Fri Jan 19 12:29:14 ~/Repos/nf/zrok $ zrok rm zrok://wkcfb58vj51l/stuff -michael@fourtyfour Fri Jan 19 12:29:20 ~/Repos/nf/zrok $ zrok ls zrok://wkcfb58vj51l ┌──────┬──────┬──────┬──────────┐ │ TYPE │ NAME │ SIZE │ MODIFIED │ @@ -104,3 +101,140 @@ $ zrok ls zrok://wkcfb58vj51l └──────┴──────┴──────┴──────────┘ ``` +## Working with Public Shares + +Public shares work very similarly to private shares, they just use a different URL scheme: + +``` +$ zrok share public --headless --backend-mode drive /tmp/junk +[ 0.708] INFO sdk-golang/ziti.(*listenerManager).createSessionWithBackoff: {session token=[05e0f48b-242b-4fd9-8edb-259488535c47]} new service session +[ 0.878] INFO main.(*sharePublicCommand).run: access your zrok share at the following endpoints: + https://6kiww4bn7iok.share.zrok.io +``` + +The same commands, with a different URL scheme work with the `zrok` drives CLI: + +``` +$ zrok copy util/ https://6kiww4bn7iok.share.zrok.io +[ 0.268] INFO zrok/drives/sync.OneWay: => /email.go +[ 0.406] INFO zrok/drives/sync.OneWay: => /headers.go +[ 0.530] INFO zrok/drives/sync.OneWay: => /proxy.go +[ 0.655] INFO zrok/drives/sync.OneWay: => /size.go +[ 0.714] INFO zrok/drives/sync.OneWay: => /uniqueName.go +copy complete! +michael@fourtyfour Fri Jan 19 12:42:52 ~/Repos/nf/zrok +$ zrok ls https://6kiww4bn7iok.share.zrok.io +┌──────┬───────────────┬───────┬───────────────────────────────┐ +│ TYPE │ NAME │ SIZE │ MODIFIED │ +├──────┼───────────────┼───────┼───────────────────────────────┤ +│ │ email.go │ 329 B │ 2023-07-21 13:17:56 -0400 EDT │ +│ │ headers.go │ 456 B │ 2023-07-21 13:17:56 -0400 EDT │ +│ │ proxy.go │ 609 B │ 2023-07-21 13:17:56 -0400 EDT │ +│ │ size.go │ 361 B │ 2023-07-21 13:17:56 -0400 EDT │ +│ │ uniqueName.go │ 423 B │ 2024-01-02 11:57:14 -0500 EST │ +└──────┴───────────────┴───────┴───────────────────────────────┘ +``` + +For basic authentication provided by public shares, the `zrok` drives CLI offers the `--basic-auth` flag, which accepts a `:` parameter to specify the authentication for the public virtual drive (if it's required). + +Alternatively, the authentication can be set using the `ZROK_DRIVES_BASIC_AUTH` environment variable: + +``` +$ export ZROK_DRIVES_BASIC_AUTH=username:password +``` + +## One-way Synchronization + +The `zrok copy` command includes a `--sync` flag, which only copies files detected as _modified_. `zrok` considers a file with the same modification timestamp and size to be the same. Of course, this is not a strong guarantee that the files are equivalent. Future `zrok` drives versions will provide a cryptographically strong mechanism (a-la `rsync` and friends) to guarantee that files and trees of files are synchronized. + +For now, the `--sync` flag provides a convenience mechanism to allow resuming copies of large file trees and provide a reasonable guarantee that the trees are in sync. + +Let's take a look at `zrok copy --sync` in action: + +``` +$ zrok copy --sync docs/ https://glmv049c62p7.share.zrok.io +[ 0.636] INFO zrok/drives/sync.OneWay: => /_attic/ +[ 0.760] INFO zrok/drives/sync.OneWay: => /_attic/network/ +[ 0.816] INFO zrok/drives/sync.OneWay: => /_attic/network/_category_.json +[ 0.928] INFO zrok/drives/sync.OneWay: => /_attic/network/prod/ +[ 0.987] INFO zrok/drives/sync.OneWay: => /_attic/network/prod/ziti-ctrl.service +[ 1.048] INFO zrok/drives/sync.OneWay: => /_attic/network/prod/ziti-ctrl.yml +[ 1.107] INFO zrok/drives/sync.OneWay: => /_attic/network/prod/ziti-router0.service +[ 1.167] INFO zrok/drives/sync.OneWay: => /_attic/network/prod/ziti-router0.yml +[ 1.218] INFO zrok/drives/sync.OneWay: => /_attic/network/prod/zrok-access-public.service +[ 1.273] INFO zrok/drives/sync.OneWay: => /_attic/network/prod/zrok-ctrl.service +[ 1.328] INFO zrok/drives/sync.OneWay: => /_attic/network/prod/zrok-ctrl.yml +[ 1.382] INFO zrok/drives/sync.OneWay: => /_attic/network/prod/zrok.io-network-skeleton.md +[ 1.447] INFO zrok/drives/sync.OneWay: => /_attic/overview.md +[ 1.572] INFO zrok/drives/sync.OneWay: => /_attic/sharing/ +[ 1.622] INFO zrok/drives/sync.OneWay: => /_attic/sharing/_category_.json +[ 1.673] INFO zrok/drives/sync.OneWay: => /_attic/sharing/reserved_services.md +[ 1.737] INFO zrok/drives/sync.OneWay: => /_attic/sharing/sharing_modes.md +[ 1.793] INFO zrok/drives/sync.OneWay: => /_attic/v0.2_account_requests.md +[ 1.902] INFO zrok/drives/sync.OneWay: => /_attic/v0.4_limits.md +... +[ 9.691] INFO zrok/drives/sync.OneWay: => /images/zrok_web_ui_empty_shares.png +[ 9.812] INFO zrok/drives/sync.OneWay: => /images/zrok_web_ui_new_environment.png +[ 9.870] INFO zrok/drives/sync.OneWay: => /images/zrok_zoom_to_fit.png +copy complete! +``` + +Because the target drive was empty, `zrok copy --sync` copied the entire contents of the local `docs/` tree into the virtual drive. However, if we run that command again, we get: + +``` +$ zrok copy --sync docs/ https://glmv049c62p7.share.zrok.io +copy complete! +``` + +The virtual drive contents are already in sync with the local filesystem tree, so there is nothing for it to copy. + +Let's alter the contents of the drive and run the `--sync` again: + +``` +$ zrok rm https://glmv049c62p7.share.zrok.io/images +$ zrok copy --sync docs/ https://glmv049c62p7.share.zrok.io +[ 0.364] INFO zrok/drives/sync.OneWay: => /images/ +[ 0.456] INFO zrok/drives/sync.OneWay: => /images/zrok.png +[ 0.795] INFO zrok/drives/sync.OneWay: => /images/zrok_cover.png +[ 0.866] INFO zrok/drives/sync.OneWay: => /images/zrok_deployment.drawio +... +[ 2.254] INFO zrok/drives/sync.OneWay: => /images/zrok_web_ui_empty_shares.png +[ 2.340] INFO zrok/drives/sync.OneWay: => /images/zrok_web_ui_new_environment.png +[ 2.391] INFO zrok/drives/sync.OneWay: => /images/zrok_zoom_to_fit.png +copy complete! +``` + +Because we removed the `images/` tree from the virtual drive, `zrok copy --sync` detected this and copied the local `images/` tree back onto the virtual drive. + +## Drive-to-Drive Copies and Synchronization + +The `zrok copy` CLI can operate on pairs of virtual drives remotely, without ever having to store files locally. This allow for drive-to-drive copies and synchronization. + +Here are a couple of examples: + +``` +$ zrok copy --sync https://glmv049c62p7.share.zrok.io https://glmv049c62p7.share.zrok.io +copy complete! +``` + +Specifying the same URL for both the source and the target of a `--sync` operation should always result in nothing being copied... they are the same drive with the same state. + +We can copy files between two virtual drives with a single command: + +``` +$ zrok copy --sync https://glmv049c62p7.share.zrok.io zrok://hsml272j3xzf +[ 1.396] INFO zrok/drives/sync.OneWay: => /_attic/ +[ 2.083] INFO zrok/drives/sync.OneWay: => /_attic/overview.md +[ 2.704] INFO zrok/drives/sync.OneWay: => /_attic/sharing/ +... +[ 118.240] INFO zrok/drives/sync.OneWay: => /images/zrok_web_console_empty.png +[ 118.920] INFO zrok/drives/sync.OneWay: => /images/zrok_enable_modal.png +[ 119.589] INFO zrok/drives/sync.OneWay: => /images/zrok_cover.png +[ 120.214] INFO zrok/drives/sync.OneWay: => /getting-started.mdx +copy complete! +$ zrok copy --sync https://glmv049c62p7.share.zrok.io zrok://hsml272j3xzf +copy complete! +``` + +## Copying from Drives to the Local Filesystem + From cec8e670384d1f92cd728e073a90aa98b5dae28d Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 19 Jan 2024 13:38:07 -0500 Subject: [PATCH 61/63] docs snapshot (#438) --- docs/guides/drives/zrok_copy.md | 78 ++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/docs/guides/drives/zrok_copy.md b/docs/guides/drives/zrok_copy.md index b7835e085..c4e00c6cd 100644 --- a/docs/guides/drives/zrok_copy.md +++ b/docs/guides/drives/zrok_copy.md @@ -1,10 +1,10 @@ # The Drives CLI -The zrok Drives CLI tools allow for simple, ergonomic management and synchronization of local and remote file objects transparently. +The zrok drives CLI tools allow for simple, ergonomic management and synchronization of local and remote files. ## Sharing a Drive -Virtual drives are shared through the `zrok` CLI using the `--backend-mode drive` flag with the `zrok share` command, using either the `public` or `private` sharing modes. We'll use the `private` sharing mode for this example: +Virtual drives are shared through the `zrok` CLI using the `--backend-mode drive` flag through the `zrok share` command, using either the `public` or `private` sharing modes. We'll use the `private` sharing mode for this example: ``` $ mkdir /tmp/junk @@ -238,3 +238,77 @@ copy complete! ## Copying from Drives to the Local Filesystem +In the current version of the drives CLI, `zrok copy` always assumes the destination is a directory. There is currently no way to do: + +``` +$ zrok copy somefile someotherfile +``` + +What you'll end up with on the local filesystem is: + +``` +somefile +someotherfile/somefile +``` + +It's in the backlog to support file destinations in a future release of `zrok`. So, when using `zrok copy`, always take note of the destination. + +`zrok copy` supports a default destination of `file://.`, so you can do single parameter `zrok copy` commands like this: + +``` +$ zrok ls https://azc47r3cwjds.share.zrok.io +┌──────┬─────────┬─────────┬───────────────────────────────┐ +│ TYPE │ NAME │ SIZE │ MODIFIED │ +├──────┼─────────┼─────────┼───────────────────────────────┤ +│ │ LICENSE │ 11.3 kB │ 2023-07-21 13:17:56 -0400 EDT │ +└──────┴─────────┴─────────┴───────────────────────────────┘ +$ zrok copy https://azc47r3cwjds.share.zrok.io/LICENSE +[ 0.260] INFO zrok/drives/sync.OneWay: => /LICENSE +copy complete! +$ ls -l +total 12 +-rw-rw-r-- 1 michael michael 11346 Jan 19 13:29 LICENSE +``` + +You can also specify a local folder as the destination for your copy: + +``` +$ zrok copy https://azc47r3cwjds.share.zrok.io/LICENSE /tmp/inbox +[ 0.221] INFO zrok/drives/sync.OneWay: => /LICENSE +copy complete! +$ l /tmp/inbox +total 12 +-rw-rw-r-- 1 michael michael 11346 Jan 19 13:30 LICENSE +``` + +## Unique Names and Reserved Shares + +Private reserved shares with unque names can be particularly useful with the drives CLI: + +``` +$ zrok reserve private -b drive --unique-name mydrive /tmp/junk +[ 0.315] INFO main.(*reserveCommand).run: your reserved share token is 'mydrive' +$ zrok share reserved --headless mydrive +[ 0.289] INFO main.(*shareReservedCommand).run: sharing target: '/tmp/junk' +[ 0.289] INFO main.(*shareReservedCommand).run: using existing backend proxy endpoint: /tmp/junk +[ 0.767] INFO sdk-golang/ziti.(*listenerManager).createSessionWithBackoff: {session token=[d519a436-9fb5-4207-afd5-7cbc28fb779a]} new service session +[ 0.927] INFO main.(*shareReservedCommand).run: use this command to access your zrok share: 'zrok access private mydrive' +``` + +This makes working with `zrok://` URLs particularly convenient: + +``` +$ zrok ls zrok://mydrive +┌──────┬─────────┬─────────┬───────────────────────────────┐ +│ TYPE │ NAME │ SIZE │ MODIFIED │ +├──────┼─────────┼─────────┼───────────────────────────────┤ +│ │ LICENSE │ 11.3 kB │ 2023-07-21 13:17:56 -0400 EDT │ +└──────┴─────────┴─────────┴───────────────────────────────┘ +``` + +## Future Enhancements + +Coming in a future release of `zrok` drives are features like: + +* two-way synchronization between multiple hosts... allowing for shared "dropbox-like" usage scenarios between multiple environments +* better ergonomics for single-file destinations From 596f94fc6f0809b4443dc0eed3d1e57ff2a4035a Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 19 Jan 2024 13:39:02 -0500 Subject: [PATCH 62/63] drives cli guide rename; changelog (#438) --- CHANGELOG.md | 2 +- docs/guides/drives/{zrok_copy.md => cli.md} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename docs/guides/drives/{zrok_copy.md => cli.md} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d5b570e9..448b2a83e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## v0.4.23 -FEATURE: New CLI commands have been implemented for working with the `drive` share backend mode (part of the "zrok Drives" functionality). These commands include `zrok cp`, `zrok mkdir` `zrok mv`, `zrok ls`, and `zrok rm`. These are initial, minimal versions of these commands and very likely contain bugs and ergonomic annoyances. There is a guide available at (`docs/guides/drives/zrok_copy.md`) that explains how to work with these tools in detail (https://github.com/openziti/zrok/issues/438) +FEATURE: New CLI commands have been implemented for working with the `drive` share backend mode (part of the "zrok Drives" functionality). These commands include `zrok cp`, `zrok mkdir` `zrok mv`, `zrok ls`, and `zrok rm`. These are initial, minimal versions of these commands and very likely contain bugs and ergonomic annoyances. There is a guide available at (`docs/guides/drives/cli.md`) that explains how to work with these tools in detail (https://github.com/openziti/zrok/issues/438) FEATURE: Python SDK now has a decorator for integrating with various server side frameworks. See the `http-server` example. diff --git a/docs/guides/drives/zrok_copy.md b/docs/guides/drives/cli.md similarity index 100% rename from docs/guides/drives/zrok_copy.md rename to docs/guides/drives/cli.md From 8f624f821f0610c27a67468b145edd7488de19fe Mon Sep 17 00:00:00 2001 From: Michael Quigley Date: Fri, 19 Jan 2024 13:52:35 -0500 Subject: [PATCH 63/63] davServer acknowledgements (#511) --- ACKNOWLEDGEMENTS.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/ACKNOWLEDGEMENTS.md b/ACKNOWLEDGEMENTS.md index 2154bfa30..8221d3c41 100644 --- a/ACKNOWLEDGEMENTS.md +++ b/ACKNOWLEDGEMENTS.md @@ -1,5 +1,37 @@ # ACKNOWLEDGEMENTS +## github.com/openziti/zrok/drives/davServer + +The `davServer` package is based on code from `https://cs.opensource.google/go/go/`, which included the following license: + +> Copyright (c) 2009 The Go Authors. All rights reserved. +> +> Redistribution and use in source and binary forms, with or without +> modification, are permitted provided that the following conditions are +> met: +> +> * Redistributions of source code must retain the above copyright +> notice, this list of conditions and the following disclaimer. +> * Redistributions in binary form must reproduce the above +> copyright notice, this list of conditions and the following disclaimer +> in the documentation and/or other materials provided with the +> distribution. +> * Neither the name of Google Inc. nor the names of its +> contributors may be used to endorse or promote products derived from +> this software without specific prior written permission. +> +> THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +> "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +> LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +> A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +> OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +> SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +> LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +> DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +> THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +> (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +> OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE + ## github.com/openziti/zrok/drives/davClient The `davClient` package is based on code from `github.com/emersion/go-webdav`, which included the following license: