From ca7e3cc068679f53eb1a7882b5c3813ae9129834 Mon Sep 17 00:00:00 2001 From: Erik Unger Date: Fri, 10 Nov 2023 16:26:28 +0100 Subject: [PATCH] added FileSystem.Close, sftpfs.DialAndRegister --- dropboxfs/dropboxfs.go | 95 +++++++++++++------------- filesystem.go | 3 + ftpfs/ftpfs.go | 82 ++++++++++++----------- httpfs/httpfs.go | 52 ++++++++------- invalidfilesystem.go | 4 ++ localfilesystem.go | 4 ++ s3fs/s3fs.go | 97 ++++++++++++++------------- sftpfs/sftpfs.go | 147 +++++++++++++++++++++++++++-------------- sftpfs/sftpfs_test.go | 11 +-- subfilesystem.go | 4 ++ 10 files changed, 286 insertions(+), 213 deletions(-) diff --git a/dropboxfs/dropboxfs.go b/dropboxfs/dropboxfs.go index 246ab9e..3fbc436 100644 --- a/dropboxfs/dropboxfs.go +++ b/dropboxfs/dropboxfs.go @@ -29,20 +29,21 @@ var ( DefaultDirPermissions = fs.UserAndGroupReadWrite + fs.AllExecute // Make sure DropboxFileSystem implements fs.FileSystem - _ fs.FileSystem = new(DropboxFileSystem) + _ fs.FileSystem = new(fileSystem) ) -// DropboxFileSystem implements fs.FileSystem for a Dropbox app. -type DropboxFileSystem struct { +// fileSystem implements fs.FileSystem for a Dropbox app. +type fileSystem struct { id string prefix string client *dropbox.Client fileInfoCache *fs.FileInfoCache } -// New returns a new DropboxFileSystem for accessToken -func New(accessToken string, cacheTimeout time.Duration) *DropboxFileSystem { - dbfs := &DropboxFileSystem{ +// NewAndRegister returns a new fs.FileSystem for a Dropbox with +// the passed accessToken and registers it. +func NewAndRegister(accessToken string, cacheTimeout time.Duration) fs.FileSystem { + dbfs := &fileSystem{ prefix: Prefix + fsimpl.RandomString(), client: dropbox.New(dropbox.NewConfig(accessToken)), fileInfoCache: fs.NewFileInfoCache(cacheTimeout), @@ -51,31 +52,26 @@ func New(accessToken string, cacheTimeout time.Duration) *DropboxFileSystem { return dbfs } -func (dbfs *DropboxFileSystem) wrapErrNotExist(filePath string, err error) error { +func (dbfs *fileSystem) wrapErrNotExist(filePath string, err error) error { if err != nil && strings.HasPrefix(err.Error(), "path/not_found/") { return fs.NewErrDoesNotExist(dbfs.File(filePath)) } return err } -func (dbfs *DropboxFileSystem) Close() error { - fs.Unregister(dbfs) - return nil -} - -func (dbfs *DropboxFileSystem) IsReadOnly() bool { +func (dbfs *fileSystem) IsReadOnly() bool { return false } -func (dbfs *DropboxFileSystem) IsWriteOnly() bool { +func (dbfs *fileSystem) IsWriteOnly() bool { return false } -func (dbfs *DropboxFileSystem) RootDir() fs.File { +func (dbfs *fileSystem) RootDir() fs.File { return fs.File(dbfs.prefix + Separator) } -func (dbfs *DropboxFileSystem) ID() (string, error) { +func (dbfs *fileSystem) ID() (string, error) { if dbfs.id == "" { account, err := dbfs.client.Users.GetCurrentAccount() if err != nil { @@ -86,56 +82,56 @@ func (dbfs *DropboxFileSystem) ID() (string, error) { return dbfs.id, nil } -func (dbfs *DropboxFileSystem) Prefix() string { +func (dbfs *fileSystem) Prefix() string { return dbfs.prefix } -func (dbfs *DropboxFileSystem) Name() string { +func (dbfs *fileSystem) Name() string { return "Dropbox file system" } // String implements the fmt.Stringer interface. -func (dbfs *DropboxFileSystem) String() string { +func (dbfs *fileSystem) String() string { return dbfs.Name() + " with prefix " + dbfs.Prefix() } -func (dbfs *DropboxFileSystem) File(filePath string) fs.File { +func (dbfs *fileSystem) File(filePath string) fs.File { return dbfs.JoinCleanFile(filePath) } -func (dbfs *DropboxFileSystem) JoinCleanFile(uriParts ...string) fs.File { +func (dbfs *fileSystem) JoinCleanFile(uriParts ...string) fs.File { return fs.File(dbfs.prefix + dbfs.JoinCleanPath(uriParts...)) } -func (dbfs *DropboxFileSystem) URL(cleanPath string) string { +func (dbfs *fileSystem) URL(cleanPath string) string { return dbfs.prefix + cleanPath } -func (dbfs *DropboxFileSystem) JoinCleanPath(uriParts ...string) string { +func (dbfs *fileSystem) JoinCleanPath(uriParts ...string) string { return fsimpl.JoinCleanPath(uriParts, dbfs.prefix, Separator) } -func (dbfs *DropboxFileSystem) SplitPath(filePath string) []string { +func (dbfs *fileSystem) SplitPath(filePath string) []string { return fsimpl.SplitPath(filePath, dbfs.prefix, Separator) } -func (dbfs *DropboxFileSystem) Separator() string { +func (dbfs *fileSystem) Separator() string { return Separator } -func (*DropboxFileSystem) MatchAnyPattern(name string, patterns []string) (bool, error) { +func (*fileSystem) MatchAnyPattern(name string, patterns []string) (bool, error) { return fsimpl.MatchAnyPattern(name, patterns) } -func (dbfs *DropboxFileSystem) SplitDirAndName(filePath string) (dir, name string) { +func (dbfs *fileSystem) SplitDirAndName(filePath string) (dir, name string) { return fsimpl.SplitDirAndName(filePath, 0, Separator) } -func (dbfs *DropboxFileSystem) IsAbsPath(filePath string) bool { +func (dbfs *fileSystem) IsAbsPath(filePath string) bool { return path.IsAbs(filePath) } -func (dbfs *DropboxFileSystem) AbsPath(filePath string) string { +func (dbfs *fileSystem) AbsPath(filePath string) string { if !path.IsAbs(filePath) { filePath = Separator + filePath } @@ -161,7 +157,7 @@ func metadataToFileInfo(meta *dropbox.Metadata) *fs.FileInfo { } // info returns FileInfo -func (dbfs *DropboxFileSystem) info(filePath string) *fs.FileInfo { +func (dbfs *fileSystem) info(filePath string) *fs.FileInfo { // The root folder is unsupported by the API if filePath == "/" { @@ -196,7 +192,7 @@ func (dbfs *DropboxFileSystem) info(filePath string) *fs.FileInfo { return info } -func (dbfs *DropboxFileSystem) Stat(filePath string) (iofs.FileInfo, error) { +func (dbfs *fileSystem) Stat(filePath string) (iofs.FileInfo, error) { info := dbfs.info(filePath) if !info.Exists { return nil, fs.NewErrDoesNotExist(fs.File(filePath)) @@ -204,20 +200,20 @@ func (dbfs *DropboxFileSystem) Stat(filePath string) (iofs.FileInfo, error) { return info.StdFileInfo(), nil } -func (dbfs *DropboxFileSystem) Exists(filePath string) bool { +func (dbfs *fileSystem) Exists(filePath string) bool { return dbfs.info(filePath).Exists } -func (dbfs *DropboxFileSystem) IsHidden(filePath string) bool { +func (dbfs *fileSystem) IsHidden(filePath string) bool { name := path.Base(filePath) return len(name) > 0 && name[0] == '.' } -func (dbfs *DropboxFileSystem) IsSymbolicLink(filePath string) bool { +func (dbfs *fileSystem) IsSymbolicLink(filePath string) bool { return false } -func (dbfs *DropboxFileSystem) listDirInfo(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string, recursive bool) (err error) { +func (dbfs *fileSystem) listDirInfo(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string, recursive bool) (err error) { if ctx.Err() != nil { return ctx.Err() } @@ -282,27 +278,27 @@ func (dbfs *DropboxFileSystem) listDirInfo(ctx context.Context, dirPath string, return nil } -func (dbfs *DropboxFileSystem) ListDirInfo(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string) (err error) { +func (dbfs *fileSystem) ListDirInfo(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string) (err error) { return dbfs.listDirInfo(ctx, dirPath, callback, patterns, true) } -func (dbfs *DropboxFileSystem) ListDirInfoRecursive(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string) (err error) { +func (dbfs *fileSystem) ListDirInfoRecursive(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string) (err error) { return dbfs.listDirInfo(ctx, dirPath, callback, patterns, true) } -func (dbfs *DropboxFileSystem) Touch(filePath string, perm []fs.Permissions) error { +func (dbfs *fileSystem) Touch(filePath string, perm []fs.Permissions) error { if dbfs.info(filePath).Exists { return errors.New("Touch can't change time on Dropbox") } return dbfs.WriteAll(context.Background(), filePath, nil, perm) } -func (dbfs *DropboxFileSystem) MakeDir(dirPath string, perm []fs.Permissions) error { +func (dbfs *fileSystem) MakeDir(dirPath string, perm []fs.Permissions) error { _, err := dbfs.client.Files.CreateFolder(&dropbox.CreateFolderInput{Path: dirPath}) return dbfs.wrapErrNotExist(dirPath, err) } -func (dbfs *DropboxFileSystem) ReadAll(ctx context.Context, filePath string) ([]byte, error) { +func (dbfs *fileSystem) ReadAll(ctx context.Context, filePath string) ([]byte, error) { if ctx.Err() != nil { return nil, ctx.Err() } @@ -315,7 +311,7 @@ func (dbfs *DropboxFileSystem) ReadAll(ctx context.Context, filePath string) ([] return fs.ReadAllContext(ctx, out.Body) } -func (dbfs *DropboxFileSystem) WriteAll(ctx context.Context, filePath string, data []byte, perm []fs.Permissions) error { +func (dbfs *fileSystem) WriteAll(ctx context.Context, filePath string, data []byte, perm []fs.Permissions) error { if ctx.Err() != nil { return ctx.Err() } @@ -330,7 +326,7 @@ func (dbfs *DropboxFileSystem) WriteAll(ctx context.Context, filePath string, da return dbfs.wrapErrNotExist(filePath, err) } -func (dbfs *DropboxFileSystem) OpenReader(filePath string) (iofs.File, error) { +func (dbfs *fileSystem) OpenReader(filePath string) (iofs.File, error) { info, err := dbfs.Stat(filePath) if err != nil { return nil, err @@ -342,7 +338,7 @@ func (dbfs *DropboxFileSystem) OpenReader(filePath string) (iofs.File, error) { return fsimpl.NewReadonlyFileBuffer(data, info), nil } -func (dbfs *DropboxFileSystem) OpenWriter(filePath string, perm []fs.Permissions) (fs.WriteCloser, error) { +func (dbfs *fileSystem) OpenWriter(filePath string, perm []fs.Permissions) (fs.WriteCloser, error) { if !dbfs.info(path.Dir(filePath)).IsDir { return nil, fs.NewErrIsNotDirectory(dbfs.File(path.Dir(filePath))) } @@ -353,7 +349,7 @@ func (dbfs *DropboxFileSystem) OpenWriter(filePath string, perm []fs.Permissions return fileBuffer, nil } -func (dbfs *DropboxFileSystem) OpenReadWriter(filePath string, perm []fs.Permissions) (fs.ReadWriteSeekCloser, error) { +func (dbfs *fileSystem) OpenReadWriter(filePath string, perm []fs.Permissions) (fs.ReadWriteSeekCloser, error) { data, err := dbfs.ReadAll(context.Background(), filePath) if err != nil { return nil, err @@ -365,7 +361,7 @@ func (dbfs *DropboxFileSystem) OpenReadWriter(filePath string, perm []fs.Permiss return fileBuffer, nil } -func (dbfs *DropboxFileSystem) CopyFile(ctx context.Context, srcFile string, destFile string, buf *[]byte) error { +func (dbfs *fileSystem) CopyFile(ctx context.Context, srcFile string, destFile string, buf *[]byte) error { if ctx.Err() != nil { return ctx.Err() } @@ -376,7 +372,7 @@ func (dbfs *DropboxFileSystem) CopyFile(ctx context.Context, srcFile string, des return dbfs.wrapErrNotExist(srcFile, err) } -func (dbfs *DropboxFileSystem) Move(filePath string, destPath string) error { +func (dbfs *fileSystem) Move(filePath string, destPath string) error { // if !dbfs.Stat(filePath).Exists { // return NewErrDoesNotExist(File(filePath)) // } @@ -390,7 +386,12 @@ func (dbfs *DropboxFileSystem) Move(filePath string, destPath string) error { return dbfs.wrapErrNotExist(filePath, err) } -func (dbfs *DropboxFileSystem) Remove(filePath string) error { +func (dbfs *fileSystem) Remove(filePath string) error { _, err := dbfs.client.Files.Delete(&dropbox.DeleteInput{Path: filePath}) return dbfs.wrapErrNotExist(filePath, err) } + +func (dbfs *fileSystem) Close() error { + fs.Unregister(dbfs) + return nil +} diff --git a/filesystem.go b/filesystem.go index 8f20273..0fdc982 100644 --- a/filesystem.go +++ b/filesystem.go @@ -87,6 +87,9 @@ type FileSystem interface { // Remove deletes the file. Remove(filePath string) error + + // Close the file system or do nothing if it is not closable + Close() error } type fullyFeaturedFileSystem interface { diff --git a/ftpfs/ftpfs.go b/ftpfs/ftpfs.go index cd1444e..e928247 100644 --- a/ftpfs/ftpfs.go +++ b/ftpfs/ftpfs.go @@ -28,25 +28,26 @@ const ( func init() { // Register with prefix ftp:// and ftps:// for URLs with // ftp(s)://username:password@host:port schema. - fs.Register(&FTPFileSystem{secure: false}) - fs.Register(&FTPFileSystem{secure: true}) + fs.Register(&fileSystem{secure: false}) + fs.Register(&fileSystem{secure: true}) } -type FTPFileSystem struct { +type fileSystem struct { conn *ftp.ServerConn prefix string secure bool } -// Dial a new FTP connection and register it as file system. +// DialAndRegister dials a new FTP connection and registers it as file system. // // If hostKeyCallbackOrNil is not nil then it will be called // during the cryptographic handshake to validate the server's host key, // else any host key will be accepted. -func Dial(ctx context.Context, addr, user, password string) (f *FTPFileSystem, err error) { +func DialAndRegister(ctx context.Context, addr, user, password string) (fs.FileSystem, error) { addr = strings.TrimSuffix(addr, "/") - f = &FTPFileSystem{ + var err error + f := &fileSystem{ prefix: addr, secure: strings.HasPrefix(addr, "ftps://"), } @@ -76,7 +77,7 @@ func Dial(ctx context.Context, addr, user, password string) (f *FTPFileSystem, e func nop() error { return nil } -func (f *FTPFileSystem) getConn(filePath string) (conn *ftp.ServerConn, clientPath string, release func() error, err error) { +func (f *fileSystem) getConn(filePath string) (conn *ftp.ServerConn, clientPath string, release func() error, err error) { if f.conn != nil { return f.conn, filePath, nop, nil } @@ -94,87 +95,82 @@ func (f *FTPFileSystem) getConn(filePath string) (conn *ftp.ServerConn, clientPa if !ok { return nil, "", nop, fmt.Errorf("no password in %s URL: %s", f.Name(), f.URL(filePath)) } - panic("todo" + password) + panic("TODO" + password) } -func (f *FTPFileSystem) IsReadOnly() bool { +func (f *fileSystem) IsReadOnly() bool { return false } -func (f *FTPFileSystem) IsWriteOnly() bool { +func (f *fileSystem) IsWriteOnly() bool { return false } -func (f *FTPFileSystem) Close() error { - fs.Unregister(f) - return f.conn.Quit() -} - -func (f *FTPFileSystem) RootDir() fs.File { +func (f *fileSystem) RootDir() fs.File { return fs.File(f.prefix + Separator) } -func (f *FTPFileSystem) ID() (string, error) { +func (f *fileSystem) ID() (string, error) { return f.prefix, nil } -func (f *FTPFileSystem) Prefix() string { +func (f *fileSystem) Prefix() string { if f.prefix == "" { return Prefix } return f.prefix } -func (f *FTPFileSystem) Name() string { +func (f *fileSystem) Name() string { if f.secure { return "FTPS" } return "FTP" } -func (f *FTPFileSystem) String() string { +func (f *fileSystem) String() string { return f.prefix + " file system" } -func (f *FTPFileSystem) URL(cleanPath string) string { +func (f *fileSystem) URL(cleanPath string) string { if f.secure { return PrefixTLS + cleanPath } return Prefix + cleanPath } -func (f *FTPFileSystem) JoinCleanFile(uriParts ...string) fs.File { +func (f *fileSystem) JoinCleanFile(uriParts ...string) fs.File { if f.secure { return fs.File(PrefixTLS + f.JoinCleanPath(uriParts...)) } return fs.File(Prefix + f.JoinCleanPath(uriParts...)) } -func (f *FTPFileSystem) JoinCleanPath(uriParts ...string) string { +func (f *fileSystem) JoinCleanPath(uriParts ...string) string { if f.secure { return fsimpl.JoinCleanPath(uriParts, PrefixTLS, Separator) } return fsimpl.JoinCleanPath(uriParts, Prefix, Separator) } -func (f *FTPFileSystem) SplitPath(filePath string) []string { +func (f *fileSystem) SplitPath(filePath string) []string { return fsimpl.SplitPath(filePath, f.Prefix(), Separator) } -func (f *FTPFileSystem) Separator() string { return Separator } +func (f *fileSystem) Separator() string { return Separator } -func (f *FTPFileSystem) IsAbsPath(filePath string) bool { +func (f *fileSystem) IsAbsPath(filePath string) bool { return strings.HasPrefix(filePath, Prefix) } -func (f *FTPFileSystem) AbsPath(filePath string) string { +func (f *fileSystem) AbsPath(filePath string) string { if f.IsAbsPath(filePath) { return filePath } return Prefix + strings.TrimPrefix(filePath, Separator) } -func (f *FTPFileSystem) SplitDirAndName(filePath string) (dir, name string) { +func (f *fileSystem) SplitDirAndName(filePath string) (dir, name string) { return fsimpl.SplitDirAndName(filePath, 0, Separator) } @@ -203,7 +199,7 @@ func entryToFileInfo(entry *ftp.Entry, file fs.File) *fs.FileInfo { } } -func (f *FTPFileSystem) Stat(filePath string) (iofs.FileInfo, error) { +func (f *fileSystem) Stat(filePath string) (iofs.FileInfo, error) { conn, filePath, release, err := f.getConn(filePath) if err != nil { return nil, err @@ -217,9 +213,9 @@ func (f *FTPFileSystem) Stat(filePath string) (iofs.FileInfo, error) { return fileInfo{entry}, nil } -func (f *FTPFileSystem) IsHidden(filePath string) bool { return false } +func (f *fileSystem) IsHidden(filePath string) bool { return false } -func (f *FTPFileSystem) IsSymbolicLink(filePath string) bool { +func (f *fileSystem) IsSymbolicLink(filePath string) bool { conn, filePath, release, err := f.getConn(filePath) if err != nil { return false @@ -233,13 +229,16 @@ func (f *FTPFileSystem) IsSymbolicLink(filePath string) bool { return entry.Type == ftp.EntryTypeLink } -func (f *FTPFileSystem) ListDirInfo(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string) error { +func (f *fileSystem) ListDirInfo(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string) error { conn, dirPath, release, err := f.getConn(dirPath) if err != nil { return err } defer release() + if ctx.Err() != nil { + return ctx.Err() + } entries, err := conn.List(dirPath) if err != nil { return err @@ -263,11 +262,11 @@ func (f *FTPFileSystem) ListDirInfo(ctx context.Context, dirPath string, callbac return nil } -func (f *FTPFileSystem) MatchAnyPattern(name string, patterns []string) (bool, error) { +func (f *fileSystem) MatchAnyPattern(name string, patterns []string) (bool, error) { return fsimpl.MatchAnyPattern(name, patterns) } -func (f *FTPFileSystem) MakeDir(dirPath string, perm []fs.Permissions) error { +func (f *fileSystem) MakeDir(dirPath string, perm []fs.Permissions) error { conn, dirPath, release, err := f.getConn(dirPath) if err != nil { return err @@ -300,7 +299,7 @@ func (f *fileReader) Close() error { return errors.Join(f.response.Close(), f.release()) } -func (f *FTPFileSystem) OpenReader(filePath string) (reader iofs.File, err error) { +func (f *fileSystem) OpenReader(filePath string) (reader iofs.File, err error) { conn, filePath, release, err := f.getConn(filePath) if err != nil { return nil, err @@ -319,7 +318,7 @@ func (f *FTPFileSystem) OpenReader(filePath string) (reader iofs.File, err error }, nil } -func (f *FTPFileSystem) OpenWriter(filePath string, perm []fs.Permissions) (fs.WriteCloser, error) { +func (f *fileSystem) OpenWriter(filePath string, perm []fs.Permissions) (fs.WriteCloser, error) { return f.OpenReadWriter(filePath, perm) } @@ -376,7 +375,7 @@ func (f *file) Close() error { return f.release() } -func (f *FTPFileSystem) OpenReadWriter(filePath string, perm []fs.Permissions) (fs.ReadWriteSeekCloser, error) { +func (f *fileSystem) OpenReadWriter(filePath string, perm []fs.Permissions) (fs.ReadWriteSeekCloser, error) { conn, filePath, release, err := f.getConn(filePath) if err != nil { return nil, err @@ -388,7 +387,7 @@ func (f *FTPFileSystem) OpenReadWriter(filePath string, perm []fs.Permissions) ( }, nil } -func (f *FTPFileSystem) Move(filePath string, destPath string) error { +func (f *fileSystem) Move(filePath string, destPath string) error { conn, filePath, release, err := f.getConn(filePath) if err != nil { return err @@ -398,7 +397,7 @@ func (f *FTPFileSystem) Move(filePath string, destPath string) error { return conn.Rename(filePath, destPath) } -func (f *FTPFileSystem) Remove(filePath string) error { +func (f *fileSystem) Remove(filePath string) error { conn, filePath, release, err := f.getConn(filePath) if err != nil { return err @@ -414,3 +413,8 @@ func (f *FTPFileSystem) Remove(filePath string) error { } return conn.Delete(filePath) } + +func (f *fileSystem) Close() error { + fs.Unregister(f) + return f.conn.Quit() +} diff --git a/httpfs/httpfs.go b/httpfs/httpfs.go index 1aa7cb2..8d1852a 100644 --- a/httpfs/httpfs.go +++ b/httpfs/httpfs.go @@ -31,70 +31,70 @@ const ( ) var ( - FileSystem = &HTTPFileSystem{prefix: Prefix} - FileSystemTLS = &HTTPFileSystem{prefix: PrefixTLS} + FileSystem = &fileSystem{prefix: Prefix} + FileSystemTLS = &fileSystem{prefix: PrefixTLS} ) -type HTTPFileSystem struct { +type fileSystem struct { fs.ReadOnlyBase prefix string } -func (*HTTPFileSystem) RootDir() fs.File { +func (*fileSystem) RootDir() fs.File { return fs.InvalidFile } -func (f *HTTPFileSystem) ID() (string, error) { +func (f *fileSystem) ID() (string, error) { return strings.TrimSuffix(f.prefix, "://"), nil } -func (f *HTTPFileSystem) Prefix() string { +func (f *fileSystem) Prefix() string { return f.prefix } -func (f *HTTPFileSystem) Name() string { +func (f *fileSystem) Name() string { return strings.ToUpper(strings.TrimSuffix(f.prefix, "://")) } -func (f *HTTPFileSystem) String() string { +func (f *fileSystem) String() string { return f.Name() + " read-only file system" } -func (f *HTTPFileSystem) URL(cleanPath string) string { +func (f *fileSystem) URL(cleanPath string) string { return f.prefix + cleanPath } -func (f *HTTPFileSystem) JoinCleanFile(uriParts ...string) fs.File { +func (f *fileSystem) JoinCleanFile(uriParts ...string) fs.File { return fs.File(f.prefix + f.JoinCleanPath(uriParts...)) } -func (f *HTTPFileSystem) JoinCleanPath(uriParts ...string) string { +func (f *fileSystem) JoinCleanPath(uriParts ...string) string { return fsimpl.JoinCleanPath(uriParts, f.prefix, Separator) } -func (f *HTTPFileSystem) SplitPath(filePath string) []string { +func (f *fileSystem) SplitPath(filePath string) []string { return fsimpl.SplitPath(filePath, f.Prefix(), f.Separator()) } -func (f *HTTPFileSystem) Separator() string { return Separator } +func (f *fileSystem) Separator() string { return Separator } -func (f *HTTPFileSystem) IsAbsPath(filePath string) bool { +func (f *fileSystem) IsAbsPath(filePath string) bool { return strings.HasPrefix(filePath, f.prefix) } -func (f *HTTPFileSystem) AbsPath(filePath string) string { +func (f *fileSystem) AbsPath(filePath string) string { if f.IsAbsPath(filePath) { return filePath } return f.prefix + strings.TrimPrefix(filePath, Separator) } -func (f *HTTPFileSystem) SplitDirAndName(filePath string) (dir, name string) { +func (f *fileSystem) SplitDirAndName(filePath string) (dir, name string) { return fsimpl.SplitDirAndName(filePath, 0, Separator) } -func (f *HTTPFileSystem) info(filePath string) fs.FileInfo { +func (f *fileSystem) info(filePath string) fs.FileInfo { // First try fast HEAD request request, err := http.NewRequest("HEAD", f.URL(filePath), nil) if err != nil { @@ -152,7 +152,7 @@ func (f *HTTPFileSystem) info(filePath string) fs.FileInfo { } } -func (f *HTTPFileSystem) Stat(filePath string) (iofs.FileInfo, error) { +func (f *fileSystem) Stat(filePath string) (iofs.FileInfo, error) { info := f.info(filePath) if !info.Exists { return nil, fs.NewErrDoesNotExist(fs.File(filePath)) @@ -160,18 +160,18 @@ func (f *HTTPFileSystem) Stat(filePath string) (iofs.FileInfo, error) { return info.StdFileInfo(), nil } -func (f *HTTPFileSystem) Exists(filePath string) bool { +func (f *fileSystem) Exists(filePath string) bool { return f.info(filePath).Exists } -func (f *HTTPFileSystem) IsHidden(filePath string) bool { return false } -func (f *HTTPFileSystem) IsSymbolicLink(filePath string) bool { return false } +func (f *fileSystem) IsHidden(filePath string) bool { return false } +func (f *fileSystem) IsSymbolicLink(filePath string) bool { return false } -func (f *HTTPFileSystem) ListDirInfo(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string) error { +func (f *fileSystem) ListDirInfo(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string) error { return fs.NewErrUnsupported(f, "ListDirInfo") } -func (f *HTTPFileSystem) ReadAll(ctx context.Context, filePath string) (data []byte, err error) { +func (f *fileSystem) ReadAll(ctx context.Context, filePath string) (data []byte, err error) { if ctx.Err() != nil { return nil, ctx.Err() } @@ -189,7 +189,7 @@ func (f *HTTPFileSystem) ReadAll(ctx context.Context, filePath string) (data []b return data, nil } -func (f *HTTPFileSystem) OpenReader(filePath string) (reader iofs.File, err error) { +func (f *fileSystem) OpenReader(filePath string) (reader iofs.File, err error) { info, err := f.Stat(filePath) if err != nil { return nil, err @@ -204,3 +204,7 @@ func (f *HTTPFileSystem) OpenReader(filePath string) (reader iofs.File, err erro defer response.Body.Close() return fsimpl.NewReadonlyFileBufferReadAll(response.Body, info) } + +func (f *fileSystem) Close() error { + return nil +} diff --git a/invalidfilesystem.go b/invalidfilesystem.go index e457860..1151ac5 100644 --- a/invalidfilesystem.go +++ b/invalidfilesystem.go @@ -218,3 +218,7 @@ func (InvalidFileSystem) Move(filePath string, destPath string) error { func (InvalidFileSystem) Remove(filePath string) error { return ErrInvalidFileSystem } + +func (InvalidFileSystem) Close() error { + return ErrInvalidFileSystem +} diff --git a/localfilesystem.go b/localfilesystem.go index ddaa337..21e74e6 100644 --- a/localfilesystem.go +++ b/localfilesystem.go @@ -753,3 +753,7 @@ func (local *LocalFileSystem) watchEventCallback(event fsnotify.Event, callback }() callback(File(event.Name), Event(event.Op)) } + +func (*LocalFileSystem) Close() error { + return nil +} diff --git a/s3fs/s3fs.go b/s3fs/s3fs.go index 8a29c99..b4525bb 100644 --- a/s3fs/s3fs.go +++ b/s3fs/s3fs.go @@ -34,21 +34,20 @@ var ( DefaultDirPermissions = fs.UserAndGroupReadWrite + fs.AllReadWrite // Make sure S3FileSystem implements fs.FileSystem - _ fs.FileSystem = new(S3FileSystem) + _ fs.FileSystem = new(fileSystem) ) -// // S3FileSystem implements fs.FileSystem for an S3 bucket. -type S3FileSystem struct { +type fileSystem struct { client *s3.Client bucketName string prefix string readOnly bool } -// New initializes a new S3 instance + session and returns an S3FileSystem -// instance that contains the required settings to work with an S3 bucket. -func New(client *s3.Client, bucketName string, readOnly bool) *S3FileSystem { - s3fs := &S3FileSystem{ +// NewAndRegister initializes a new S3 instance + session and returns a fs.FileSystem +// implementation that contains the required settings to work with an S3 bucket. +func NewAndRegister(client *s3.Client, bucketName string, readOnly bool) fs.FileSystem { + s3fs := &fileSystem{ client: client, bucketName: bucketName, prefix: Prefix + bucketName, @@ -58,92 +57,87 @@ func New(client *s3.Client, bucketName string, readOnly bool) *S3FileSystem { return s3fs } -func NewLoadDefaultConfig(ctx context.Context, bucketName string, readOnly bool) (*S3FileSystem, error) { +func NewLoadDefaultConfig(ctx context.Context, bucketName string, readOnly bool) (fs.FileSystem, error) { cfg, err := config.LoadDefaultConfig(ctx) if err != nil { return nil, err } client := s3.NewFromConfig(cfg) - return New(client, bucketName, readOnly), nil + return NewAndRegister(client, bucketName, readOnly), nil } -func (s *S3FileSystem) Close() error { - fs.Unregister(s) - return nil -} - -func (s *S3FileSystem) IsReadOnly() bool { +func (s *fileSystem) IsReadOnly() bool { return s.readOnly } -func (s *S3FileSystem) IsWriteOnly() bool { +func (s *fileSystem) IsWriteOnly() bool { return false } -func (s *S3FileSystem) RootDir() fs.File { +func (s *fileSystem) RootDir() fs.File { return fs.File(s.prefix + Separator) } -func (s *S3FileSystem) ID() (string, error) { +func (s *fileSystem) ID() (string, error) { return s.bucketName, nil } -func (s *S3FileSystem) Prefix() string { +func (s *fileSystem) Prefix() string { return s.prefix } -func (s *S3FileSystem) Name() string { +func (s *fileSystem) Name() string { return "S3 file system for bucket: s.bucketName" } -func (s *S3FileSystem) String() string { +func (s *fileSystem) String() string { return s.Name() + " with prefix " + s.prefix } -func (s *S3FileSystem) URL(cleanPath string) string { +func (s *fileSystem) URL(cleanPath string) string { return s.prefix + cleanPath } -func (s *S3FileSystem) JoinCleanFile(uriParts ...string) fs.File { +func (s *fileSystem) JoinCleanFile(uriParts ...string) fs.File { return fs.File(s.prefix + s.JoinCleanPath(uriParts...)) } -func (s *S3FileSystem) JoinCleanPath(uriParts ...string) string { +func (s *fileSystem) JoinCleanPath(uriParts ...string) string { return fsimpl.JoinCleanPath(uriParts, s.prefix, Separator) } -func (s *S3FileSystem) SplitPath(filePath string) []string { +func (s *fileSystem) SplitPath(filePath string) []string { return fsimpl.SplitPath(filePath, s.prefix, Separator) } -func (s *S3FileSystem) Separator() string { +func (s *fileSystem) Separator() string { return Separator } -func (s *S3FileSystem) IsAbsPath(filePath string) bool { +func (s *fileSystem) IsAbsPath(filePath string) bool { return path.IsAbs(filePath) } -func (s *S3FileSystem) AbsPath(filePath string) string { +func (s *fileSystem) AbsPath(filePath string) string { if path.IsAbs(filePath) { return filePath } return Separator + filePath } -func (s *S3FileSystem) MatchAnyPattern(name string, patterns []string) (bool, error) { +func (s *fileSystem) MatchAnyPattern(name string, patterns []string) (bool, error) { return fsimpl.MatchAnyPattern(name, patterns) } -func (s *S3FileSystem) SplitDirAndName(filePath string) (dir, name string) { +func (s *fileSystem) SplitDirAndName(filePath string) (dir, name string) { return fsimpl.SplitDirAndName(filePath, 0, Separator) } -func (s *S3FileSystem) VolumeName(filePath string) string { +func (s *fileSystem) VolumeName(filePath string) string { return s.bucketName } -func (s *S3FileSystem) Stat(filePath string) (iofs.FileInfo, error) { +func (s *fileSystem) Stat(filePath string) (iofs.FileInfo, error) { if filePath == "" { return nil, fs.ErrEmptyPath } @@ -168,7 +162,7 @@ func (s *S3FileSystem) Stat(filePath string) (iofs.FileInfo, error) { }, nil } -func (s *S3FileSystem) Exists(filePath string) bool { +func (s *fileSystem) Exists(filePath string) bool { if filePath == "" || filePath == "/" { return false } @@ -186,16 +180,16 @@ func (s *S3FileSystem) Exists(filePath string) bool { // dot. There are no real "hidden" files in S3 buckets, but since dot prefixes // are the general convention to determine which directories/files are hidden // and which are not, the function behaves this way. -func (s *S3FileSystem) IsHidden(filePath string) bool { +func (s *fileSystem) IsHidden(filePath string) bool { name := path.Base(filePath) return len(name) > 0 && name[0] == '.' } -func (s *S3FileSystem) IsSymbolicLink(filePath string) bool { +func (s *fileSystem) IsSymbolicLink(filePath string) bool { return false } -func (s *S3FileSystem) listDirInfo(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string, recursive bool) (err error) { +func (s *fileSystem) listDirInfo(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string, recursive bool) (err error) { if dirPath == "" { return fs.ErrEmptyPath } @@ -273,22 +267,22 @@ func (s *S3FileSystem) listDirInfo(ctx context.Context, dirPath string, callback // return nil } -func (s *S3FileSystem) ListDirInfo(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string) (err error) { +func (s *fileSystem) ListDirInfo(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string) (err error) { return s.listDirInfo(ctx, dirPath, callback, patterns, false) } -func (s *S3FileSystem) ListDirInfoRecursive(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string) (err error) { +func (s *fileSystem) ListDirInfoRecursive(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string) (err error) { return s.listDirInfo(ctx, dirPath, callback, patterns, true) } -func (s *S3FileSystem) Touch(filePath string, perm []fs.Permissions) error { +func (s *fileSystem) Touch(filePath string, perm []fs.Permissions) error { if s.Exists(filePath) { return nil // TODO is this OK, can we change the modified time? } return s.WriteAll(context.Background(), filePath, make([]byte, 0), perm) } -func (s *S3FileSystem) MakeDir(dirPath string, perm []fs.Permissions) error { +func (s *fileSystem) MakeDir(dirPath string, perm []fs.Permissions) error { if dirPath == "" { return fs.ErrEmptyPath } @@ -304,7 +298,7 @@ func (s *S3FileSystem) MakeDir(dirPath string, perm []fs.Permissions) error { return s.Touch(dirPath, perm) } -func (s *S3FileSystem) ReadAll(ctx context.Context, filePath string) ([]byte, error) { +func (s *fileSystem) ReadAll(ctx context.Context, filePath string) ([]byte, error) { if filePath == "" { return nil, fs.ErrEmptyPath } @@ -335,7 +329,7 @@ func (s *S3FileSystem) ReadAll(ctx context.Context, filePath string) ([]byte, er return data, nil } -func (s *S3FileSystem) WriteAll(ctx context.Context, filePath string, data []byte, perm []fs.Permissions) error { +func (s *fileSystem) WriteAll(ctx context.Context, filePath string, data []byte, perm []fs.Permissions) error { if filePath == "" { return fs.ErrEmptyPath } @@ -353,7 +347,7 @@ func (s *S3FileSystem) WriteAll(ctx context.Context, filePath string, data []byt return err } -func (s *S3FileSystem) OpenReader(filePath string) (iofs.File, error) { +func (s *fileSystem) OpenReader(filePath string) (iofs.File, error) { if filePath == "" { return nil, fs.ErrEmptyPath } @@ -390,7 +384,7 @@ func (s *S3FileSystem) OpenReader(filePath string) (iofs.File, error) { return fsimpl.NewReadonlyFileBuffer(data, info), nil } -func (s *S3FileSystem) OpenWriter(filePath string, perm []fs.Permissions) (fs.WriteCloser, error) { +func (s *fileSystem) OpenWriter(filePath string, perm []fs.Permissions) (fs.WriteCloser, error) { if filePath == "" { return nil, fs.ErrEmptyPath } @@ -404,11 +398,11 @@ func (s *S3FileSystem) OpenWriter(filePath string, perm []fs.Permissions) (fs.Wr return fileBuffer, nil } -func (s *S3FileSystem) OpenReadWriter(filePath string, perm []fs.Permissions) (fs.ReadWriteSeekCloser, error) { +func (s *fileSystem) OpenReadWriter(filePath string, perm []fs.Permissions) (fs.ReadWriteSeekCloser, error) { return s.openFileBuffer(filePath) } -func (s *S3FileSystem) openFileBuffer(filePath string) (fileBuffer *fsimpl.FileBuffer, err error) { +func (s *fileSystem) openFileBuffer(filePath string) (fileBuffer *fsimpl.FileBuffer, err error) { if s.readOnly { return nil, fs.ErrReadOnlyFileSystem } @@ -422,7 +416,7 @@ func (s *S3FileSystem) openFileBuffer(filePath string) (fileBuffer *fsimpl.FileB return fileBuffer, nil } -func (s *S3FileSystem) CopyFile(ctx context.Context, srcFile string, destFile string, buf *[]byte) error { +func (s *fileSystem) CopyFile(ctx context.Context, srcFile string, destFile string, buf *[]byte) error { if s.readOnly { return fs.ErrReadOnlyFileSystem } @@ -444,7 +438,7 @@ func (s *S3FileSystem) CopyFile(ctx context.Context, srcFile string, destFile st return err } -func (s *S3FileSystem) Remove(filePath string) error { +func (s *fileSystem) Remove(filePath string) error { if s.readOnly { return fs.ErrReadOnlyFileSystem } @@ -460,7 +454,7 @@ func (s *S3FileSystem) Remove(filePath string) error { return err } -func (s *S3FileSystem) Watch(filePath string, onEvent func(fs.File, fs.Event)) (cancel func() error, err error) { +func (s *fileSystem) Watch(filePath string, onEvent func(fs.File, fs.Event)) (cancel func() error, err error) { // https://stackoverflow.com/questions/18049717/waituntilobjectexists-amazon-s3-php-sdk-method-exactly-how-does-it-work // s.client.WaitUntilObjectExists // s.client.WaitUntilObjectNotExists @@ -479,3 +473,8 @@ func (s *S3FileSystem) Watch(filePath string, onEvent func(fs.File, fs.Event)) ( //return retChan, nil return nil, errors.ErrUnsupported } + +func (s *fileSystem) Close() error { + fs.Unregister(s) + return nil +} diff --git a/sftpfs/sftpfs.go b/sftpfs/sftpfs.go index 480cd25..890be81 100644 --- a/sftpfs/sftpfs.go +++ b/sftpfs/sftpfs.go @@ -7,6 +7,7 @@ import ( "fmt" "io" iofs "io/fs" + "net" "net/url" "strings" @@ -25,24 +26,47 @@ const ( func init() { // Register with prefix sftp:// for URLs with // sftp://username:password@host:port schema. - fs.Register(new(SFTPFileSystem)) + fs.Register(new(fileSystem)) } -type SFTPFileSystem struct { +type fileSystem struct { client *sftp.Client prefix string } -// Dial a new SFTP connection and register it as file system. +// DialAndRegister a new SFTP connection and register it as file system. // // If hostKeyCallbackOrNil is not nil then it will be called // during the cryptographic handshake to validate the server's host key, // else any host key will be accepted. -func Dial(addr, user, password string, hostKeyCallbackOrNil ssh.HostKeyCallback) (*SFTPFileSystem, error) { - addr = strings.TrimSuffix(strings.TrimPrefix(addr, "sftp://"), "/") +func DialAndRegister(ctx context.Context, address, username, password string, hostKeyCallbackOrNil ssh.HostKeyCallback) (fs.FileSystem, error) { + if !strings.HasPrefix(address, "sftp://") { + if strings.Contains(address, "://") { + return nil, fmt.Errorf("URL must start with sftp:// but got %s", address) + } + address = "sftp://" + address + } + u, err := url.Parse(address) + if err != nil { + return nil, err + } + if u.Scheme != "sftp" { + return nil, fmt.Errorf("URL scheme must be sftp:// but got %s", u.Scheme) + } + if u.Port() == "" { + u.Host += ":22" + } + if username == "" { + username = u.User.Username() + } + if password == "" { + password, _ = u.User.Password() + } + + prefix := "sftp://" + url.User(username).String() + "@" + u.Host config := &ssh.ClientConfig{ - User: user, + User: username, Auth: []ssh.AuthMethod{ ssh.Password(password), }, @@ -52,31 +76,51 @@ func Dial(addr, user, password string, hostKeyCallbackOrNil ssh.HostKeyCallback) config.HostKeyCallback = ssh.InsecureIgnoreHostKey() } - conn, err := ssh.Dial("tcp", addr, config) + var d net.Dialer + conn, err := d.DialContext(ctx, "tcp", u.Host) if err != nil { return nil, err } - return New(addr, conn) -} - -func New(addr string, conn *ssh.Client) (*SFTPFileSystem, error) { - addr = strings.TrimSuffix(strings.TrimPrefix(addr, "sftp://"), "/") + sshConn, chans, reqs, err := ssh.NewClientConn(conn, u.Host, config) + if err != nil { + return nil, err + } + sshClient := ssh.NewClient(sshConn, chans, reqs) - client, err := sftp.NewClient(conn) + // conn, err := ssh.Dial("tcp", u.Host, config) + // if err != nil { + // return nil, err + // } + client, err := sftp.NewClient(sshClient) if err != nil { return nil, err } - fileSystem := &SFTPFileSystem{ + fileSystem := &fileSystem{ client: client, - prefix: "sftp://" + addr, + prefix: prefix, } fs.Register(fileSystem) return fileSystem, nil } +// func NewFileSystem(addr string, conn *ssh.Client) (*FileSystem, error) { +// addr = strings.TrimSuffix(strings.TrimPrefix(addr, "sftp://"), "/") + +// client, err := sftp.NewClient(conn) +// if err != nil { +// return nil, err +// } +// fileSystem := &FileSystem{ +// client: client, +// prefix: "sftp://" + addr, +// } +// fs.Register(fileSystem) +// return fileSystem, nil +// } + func nop() error { return nil } -func (f *SFTPFileSystem) getClient(filePath string) (client *sftp.Client, clientPath string, release func() error, err error) { +func (f *fileSystem) getClient(filePath string) (client *sftp.Client, clientPath string, release func() error, err error) { if f.client != nil { return f.client, filePath, nop, nil } @@ -112,81 +156,76 @@ func (f *SFTPFileSystem) getClient(filePath string) (client *sftp.Client, client return client, url.Path, func() error { return client.Close() }, nil } -func (f *SFTPFileSystem) IsReadOnly() bool { +func (f *fileSystem) IsReadOnly() bool { // f.client. return false // TODO } -func (f *SFTPFileSystem) IsWriteOnly() bool { +func (f *fileSystem) IsWriteOnly() bool { return false } -func (f *SFTPFileSystem) Close() error { - fs.Unregister(f) - return f.client.Close() -} - -func (f *SFTPFileSystem) RootDir() fs.File { +func (f *fileSystem) RootDir() fs.File { return fs.File(f.prefix + Separator) } -func (f *SFTPFileSystem) ID() (string, error) { +func (f *fileSystem) ID() (string, error) { return f.prefix, nil } -func (f *SFTPFileSystem) Prefix() string { +func (f *fileSystem) Prefix() string { if f.prefix == "" { return Prefix } return f.prefix } -func (f *SFTPFileSystem) Name() string { +func (f *fileSystem) Name() string { return "SFTP" } -func (f *SFTPFileSystem) String() string { +func (f *fileSystem) String() string { return f.prefix + " file system" } -func (f *SFTPFileSystem) URL(cleanPath string) string { +func (f *fileSystem) URL(cleanPath string) string { return Prefix + cleanPath } -func (f *SFTPFileSystem) JoinCleanFile(uriParts ...string) fs.File { +func (f *fileSystem) JoinCleanFile(uriParts ...string) fs.File { return fs.File(Prefix + f.JoinCleanPath(uriParts...)) } -func (f *SFTPFileSystem) JoinCleanPath(uriParts ...string) string { +func (f *fileSystem) JoinCleanPath(uriParts ...string) string { return fsimpl.JoinCleanPath(uriParts, Prefix, Separator) } -func (f *SFTPFileSystem) SplitPath(filePath string) []string { +func (f *fileSystem) SplitPath(filePath string) []string { return fsimpl.SplitPath(filePath, f.Prefix(), Separator) } -func (f *SFTPFileSystem) Separator() string { return Separator } +func (f *fileSystem) Separator() string { return Separator } -func (f *SFTPFileSystem) IsAbsPath(filePath string) bool { +func (f *fileSystem) IsAbsPath(filePath string) bool { return strings.HasPrefix(filePath, Prefix) } -func (f *SFTPFileSystem) AbsPath(filePath string) string { +func (f *fileSystem) AbsPath(filePath string) string { if f.IsAbsPath(filePath) { return filePath } return Prefix + strings.TrimPrefix(filePath, Separator) } -func (f *SFTPFileSystem) SplitDirAndName(filePath string) (dir, name string) { +func (f *fileSystem) SplitDirAndName(filePath string) (dir, name string) { return fsimpl.SplitDirAndName(filePath, 0, Separator) } -func (f *SFTPFileSystem) MatchAnyPattern(name string, patterns []string) (bool, error) { +func (f *fileSystem) MatchAnyPattern(name string, patterns []string) (bool, error) { return fsimpl.MatchAnyPattern(name, patterns) } -func (f *SFTPFileSystem) MakeDir(dirPath string, perm []fs.Permissions) error { +func (f *fileSystem) MakeDir(dirPath string, perm []fs.Permissions) error { client, dirPath, release, err := f.getClient(dirPath) if err != nil { return err @@ -196,7 +235,7 @@ func (f *SFTPFileSystem) MakeDir(dirPath string, perm []fs.Permissions) error { return client.Mkdir(dirPath) } -func (f *SFTPFileSystem) Stat(filePath string) (iofs.FileInfo, error) { +func (f *fileSystem) Stat(filePath string) (iofs.FileInfo, error) { client, filePath, release, err := f.getClient(filePath) if err != nil { return nil, err @@ -206,16 +245,19 @@ func (f *SFTPFileSystem) Stat(filePath string) (iofs.FileInfo, error) { return client.Stat(filePath) } -func (f *SFTPFileSystem) IsHidden(filePath string) bool { return false } -func (f *SFTPFileSystem) IsSymbolicLink(filePath string) bool { return false } +func (f *fileSystem) IsHidden(filePath string) bool { return false } +func (f *fileSystem) IsSymbolicLink(filePath string) bool { return false } -func (f *SFTPFileSystem) ListDirInfo(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string) error { +func (f *fileSystem) ListDirInfo(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string) error { client, dirPath, release, err := f.getClient(dirPath) if err != nil { return err } defer release() + if ctx.Err() != nil { + return ctx.Err() + } infos, err := client.ReadDir(dirPath) if err != nil { return err @@ -248,7 +290,7 @@ func (f *sftpFile) Close() error { return errors.Join(f.File.Close(), f.release()) } -func (f *SFTPFileSystem) openFile(filePath string) (*sftpFile, error) { +func (f *fileSystem) openFile(filePath string) (*sftpFile, error) { client, filePath, release, err := f.getClient(filePath) if err != nil { return nil, err @@ -260,15 +302,15 @@ func (f *SFTPFileSystem) openFile(filePath string) (*sftpFile, error) { return &sftpFile{file, release}, nil } -func (f *SFTPFileSystem) OpenReader(filePath string) (reader iofs.File, err error) { +func (f *fileSystem) OpenReader(filePath string) (reader iofs.File, err error) { return f.openFile(filePath) } -func (f *SFTPFileSystem) OpenWriter(filePath string, perm []fs.Permissions) (fs.WriteCloser, error) { +func (f *fileSystem) OpenWriter(filePath string, perm []fs.Permissions) (fs.WriteCloser, error) { return f.openFile(filePath) } -func (f *SFTPFileSystem) OpenAppendWriter(filePath string, perm []fs.Permissions) (fs.WriteCloser, error) { +func (f *fileSystem) OpenAppendWriter(filePath string, perm []fs.Permissions) (fs.WriteCloser, error) { file, err := f.openFile(filePath) if err != nil { return nil, err @@ -284,11 +326,11 @@ func (f *SFTPFileSystem) OpenAppendWriter(filePath string, perm []fs.Permissions return file, nil } -func (f *SFTPFileSystem) OpenReadWriter(filePath string, perm []fs.Permissions) (fs.ReadWriteSeekCloser, error) { +func (f *fileSystem) OpenReadWriter(filePath string, perm []fs.Permissions) (fs.ReadWriteSeekCloser, error) { return f.openFile(filePath) } -func (f *SFTPFileSystem) Truncate(filePath string, size int64) error { +func (f *fileSystem) Truncate(filePath string, size int64) error { file, err := f.openFile(filePath) if err != nil { return err @@ -299,7 +341,7 @@ func (f *SFTPFileSystem) Truncate(filePath string, size int64) error { ) } -func (f *SFTPFileSystem) Move(filePath string, destPath string) error { +func (f *fileSystem) Move(filePath string, destPath string) error { client, filePath, release, err := f.getClient(filePath) if err != nil { return err @@ -309,7 +351,7 @@ func (f *SFTPFileSystem) Move(filePath string, destPath string) error { return client.Rename(filePath, destPath) } -func (f *SFTPFileSystem) Remove(filePath string) error { +func (f *fileSystem) Remove(filePath string) error { client, filePath, release, err := f.getClient(filePath) if err != nil { return err @@ -318,3 +360,8 @@ func (f *SFTPFileSystem) Remove(filePath string) error { return client.Remove(filePath) } + +func (f *fileSystem) Close() error { + fs.Unregister(f) + return f.client.Close() +} diff --git a/sftpfs/sftpfs_test.go b/sftpfs/sftpfs_test.go index 9f7cafe..27fa30b 100644 --- a/sftpfs/sftpfs_test.go +++ b/sftpfs/sftpfs_test.go @@ -1,6 +1,7 @@ package sftpfs import ( + "context" "testing" "github.com/stretchr/testify/assert" @@ -22,10 +23,11 @@ func checkAndReadFile(t *testing.T, f fs.File) []byte { func TestDial(t *testing.T) { // https://www.sftp.net/public-online-sftp-servers { - sftpFS, err := Dial("test.rebex.net:22", "demo", "password", nil) + sftpFS, err := DialAndRegister(context.Background(), "test.rebex.net:22", "demo", "password", nil) require.NoError(t, err, "Dial") + require.Equal(t, "sftp://demo@test.rebex.net:22", sftpFS.Prefix()) - f := fs.File("sftp://test.rebex.net:22/readme.txt") + f := fs.File("sftp://demo@test.rebex.net:22/readme.txt") assert.Equal(t, "readme.txt", f.Name()) data := checkAndReadFile(t, f) assert.True(t, len(data) > 0) @@ -39,10 +41,11 @@ func TestDial(t *testing.T) { } { // http://demo.wftpserver.com/main.html - sftpFS, err := Dial("demo.wftpserver.com:2222", "demo", "demo", nil) + sftpFS, err := DialAndRegister(context.Background(), "demo.wftpserver.com:2222", "demo", "demo", nil) require.NoError(t, err, "Dial") + require.Equal(t, "sftp://demo@demo.wftpserver.com:2222", sftpFS.Prefix()) - f := fs.File("sftp://demo.wftpserver.com:2222/download/version.txt") + f := fs.File("sftp://demo@demo.wftpserver.com:2222/download/version.txt") assert.Equal(t, "version.txt", f.Name()) data := checkAndReadFile(t, f) assert.True(t, len(data) > 0) diff --git a/subfilesystem.go b/subfilesystem.go index 0aaab12..82f8673 100644 --- a/subfilesystem.go +++ b/subfilesystem.go @@ -209,3 +209,7 @@ func (subfs *todoSubFileSystem) Truncate(filePath string, size int64) error { func (subfs *todoSubFileSystem) Remove(filePath string) error { panic("TODO") } + +func (subfs *todoSubFileSystem) Close() error { + panic("TODO") +}