From ed0fa76147b2182650616c9aaebc451020741a3e Mon Sep 17 00:00:00 2001 From: lullius <> Date: Mon, 28 Jul 2025 14:55:06 +0200 Subject: [PATCH 1/4] backend/internxt: getting all files and folders instead of only the 50 first --- backend/internxt/internxt.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/internxt/internxt.go b/backend/internxt/internxt.go index 13bf31f632781..184fabc13db94 100644 --- a/backend/internxt/internxt.go +++ b/backend/internxt/internxt.go @@ -216,11 +216,11 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error { } // Replace these calls with GetFolderContent? (fmt.Sprintf("/storage/v2/folder/%d%s", folderID, query)) - childFolders, err := folders.ListFolders(f.cfg, id, folders.ListOptions{}) + childFolders, err := folders.ListAllFolders(f.cfg, id) if err != nil { return err } - childFiles, err := folders.ListFiles(f.cfg, id, folders.ListOptions{}) + childFiles, err := folders.ListAllFiles(f.cfg, id) if err != nil { return err } @@ -246,7 +246,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error { // If found, it returns its UUID and true. If not found, returns "", false. func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (string, bool, error) { //fmt.Printf("FindLeaf pathID: %s, leaf: %s\n", pathID, leaf) - entries, err := folders.ListFolders(f.cfg, pathID, folders.ListOptions{}) + entries, err := folders.ListAllFolders(f.cfg, pathID) if err != nil { return "", false, err } @@ -281,7 +281,7 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) { } var out fs.DirEntries - foldersList, err := folders.ListFolders(f.cfg, dirID, folders.ListOptions{}) + foldersList, err := folders.ListAllFolders(f.cfg, dirID) if err != nil { return nil, err } @@ -289,7 +289,7 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) { remote := filepath.Join(dir, f.opt.Encoding.ToStandardName(e.PlainName)) out = append(out, fs.NewDir(remote, e.ModificationTime)) } - filesList, err := folders.ListFiles(f.cfg, dirID, folders.ListOptions{}) + filesList, err := folders.ListAllFiles(f.cfg, dirID) if err != nil { return nil, err } @@ -373,7 +373,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { return nil, fs.ErrorObjectNotFound } - files, err := folders.ListFiles(f.cfg, dirID, folders.ListOptions{}) + files, err := folders.ListAllFiles(f.cfg, dirID) if err != nil { return nil, err } From 9069356910cb218436ac70d624ecea0c316bc0a1 Mon Sep 17 00:00:00 2001 From: lullius <> Date: Mon, 28 Jul 2025 19:35:30 +0200 Subject: [PATCH 2/4] backend/internxt: Option to simulate empty files --- backend/internxt/internxt.go | 68 ++++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/backend/internxt/internxt.go b/backend/internxt/internxt.go index 184fabc13db94..a70a5d4e79127 100644 --- a/backend/internxt/internxt.go +++ b/backend/internxt/internxt.go @@ -2,6 +2,7 @@ package internxt import ( + "bytes" "context" "fmt" "io" @@ -45,6 +46,11 @@ func init() { Help: "The password for the user.", IsPassword: true, }, + { + Name: "simulateEmptyFiles", + Default: false, + Help: "Simulates empty files by uploading a small placeholder file instead. Alters the filename when uploading to keep track of empty files, but this is not visible through rclone.", + }, { Name: rclone_config.ConfigEncoding, Help: rclone_config.ConfigEncodingHelp, @@ -60,12 +66,21 @@ func init() { ) } +const ( + EMPTY_FILE_EXT = ".__RCLONE_EMPTY__" +) + +var ( + EMPTY_FILE_BYTES = []byte{0x13, 0x09, 0x20, 0x23} +) + // Options holds configuration options for this interface type Options struct { - Endpoint string `flag:"endpoint" help:"API endpoint"` - Email string `flag:"email" help:"Internxt account email"` - Password string `flag:"password" help:"Internxt account password"` - Encoding encoder.MultiEncoder `config:"encoding"` + Endpoint string `flag:"endpoint" help:"API endpoint"` + Email string `flag:"email" help:"Internxt account email"` + Password string `flag:"password" help:"Internxt account password"` + Encoding encoder.MultiEncoder `config:"encoding"` + SimulateEmptyFiles bool `config:"simulateEmptyFiles"` } // Fs represents an Internxt remote @@ -299,6 +314,11 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) { remote += "." + e.Type } remote = filepath.Join(dir, f.opt.Encoding.ToStandardName(remote)) + // If we found a file with the special empty file suffix, pretend that it's empty + if f.opt.SimulateEmptyFiles && strings.HasSuffix(remote, EMPTY_FILE_EXT) { + remote = strings.TrimSuffix(remote, EMPTY_FILE_EXT) + e.Size = "0" + } out = append(out, newObjectWithFile(f, remote, &e)) } return out, nil @@ -347,18 +367,15 @@ func (f *Fs) Remove(ctx context.Context, remote string) error { // Move moves a directory (not implemented) func (f *Fs) Move(ctx context.Context, src, dst fs.Object) error { // return f.client.Rename(ctx, f.root+src.Remote(), f.root+dst.Remote()) - return nil + return fs.ErrorNotImplemented } // Copy copies a directory (not implemented) func (f *Fs) Copy(ctx context.Context, src, dst fs.Object) error { // return f.client.Copy(ctx, f.root+src.Remote(), f.root+dst.Remote()) - return nil + return fs.ErrorNotImplemented } -// DirCacheFlush flushes the dir cache (not implemented) -func (f *Fs) DirCacheFlush(ctx context.Context) {} - // NewObject creates a new object func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { parentDir := filepath.Dir(remote) @@ -385,6 +402,13 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { if f.opt.Encoding.ToStandardName(name) == filepath.Base(remote) { return newObjectWithFile(f, remote, &e), nil } + // If we are simulating empty files, check for a file with the special suffix and if found return it as if empty. + if f.opt.SimulateEmptyFiles { + if f.opt.Encoding.ToStandardName(name) == filepath.Base(remote+EMPTY_FILE_EXT) { + e.Size = "0" + return newObjectWithFile(f, remote, &e), nil + } + } } return nil, fs.ErrorObjectNotFound } @@ -453,13 +477,31 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo } } + // Return nothing if we're faking an empty file + if o.f.opt.SimulateEmptyFiles && o.size == 0 { + return io.NopCloser(bytes.NewReader(nil)), nil + } return buckets.DownloadFileStream(o.f.cfg, o.id, rangeValue) } // Update updates an existing file func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + isEmptyFile := false if src.Size() == 0 { - return fs.ErrorCantUploadEmptyFiles + if !o.f.opt.SimulateEmptyFiles { + return fs.ErrorCantUploadEmptyFiles + } else { + // If we're faking an empty file, write some nonsense into it and give it a special suffix + isEmptyFile = true + in = bytes.NewReader(EMPTY_FILE_BYTES) + src = &Object{ + f: o.f, + remote: src.Remote() + EMPTY_FILE_EXT, + modTime: src.ModTime(ctx), + size: int64(len(EMPTY_FILE_BYTES)), + } + o.remote = o.remote + EMPTY_FILE_EXT + } } // Check if object exists on the server @@ -483,7 +525,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } // Create folder if it doesn't exist - _, dirID, err := o.f.dirCache.FindPath(ctx, o.Remote(), true) + _, dirID, err := o.f.dirCache.FindPath(ctx, o.remote, true) if err != nil { return err } @@ -496,6 +538,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // Update the object with the new info o.uuid = meta.UUID o.size = src.Size() + // If this is a simulated empty file set fake size to 0 + if isEmptyFile { + o.size = 0 + } return nil } From 5e68e736e075945ed18a67856a59422c52c44875 Mon Sep 17 00:00:00 2001 From: lullius <> Date: Mon, 28 Jul 2025 20:44:49 +0200 Subject: [PATCH 3/4] backend/internxt: Implemented DirMove --- backend/internxt/internxt.go | 73 +++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/backend/internxt/internxt.go b/backend/internxt/internxt.go index a70a5d4e79127..e3da6cce419f7 100644 --- a/backend/internxt/internxt.go +++ b/backend/internxt/internxt.go @@ -365,9 +365,78 @@ func (f *Fs) Remove(ctx context.Context, remote string) error { } // Move moves a directory (not implemented) -func (f *Fs) Move(ctx context.Context, src, dst fs.Object) error { +func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + /* + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + + srcLeaf, srcDirectoryID, err := f.dirCache.FindPath(ctx, srcObj.remote, false) + if err != nil { + return nil, err + } + //fmt.Printf("Move: srcLeaf %s, srcDirID %s\n", srcLeaf, srcDirectoryID) + + dstLeaf, dstDirectoryID, err := f.dirCache.FindPath(ctx, remote, false) + if err != nil { + return nil, err + } + //fmt.Printf("Move: srcLeaf %s, srcDirID %s \n dstLeaf %s, dstDirID, %s", srcLeaf, srcDirectoryID, dstLeaf, dstDirectoryID) + + doMove := srcDirectoryID != dstDirectoryID + doRename := srcLeaf != dstLeaf + + println("Move ", doMove, doRename) + */ // return f.client.Rename(ctx, f.root+src.Remote(), f.root+dst.Remote()) - return fs.ErrorNotImplemented + return nil, fs.ErrorNotImplemented +} + +func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + + srcID, _, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote) + if err != nil { + return err + } + + doMove := srcID != dstDirectoryID + doRename := srcLeaf != dstLeaf + + // If we're moving AND renaming we need to set a temp name first, else we risk collisions + if doMove && doRename { + err = folders.RenameFolder(f.cfg, srcID, f.opt.Encoding.FromStandardName(dstLeaf+".__RCLONE_MOVE__")) + if err != nil { + return err + } + time.Sleep(500 * time.Millisecond) //Find a way around this + } + + if doMove { + err = folders.MoveFolder(f.cfg, srcID, dstDirectoryID) + if err != nil { + return err + } + time.Sleep(500 * time.Millisecond) //Find a way around this + + } + + if doRename { + err = folders.RenameFolder(f.cfg, srcID, f.opt.Encoding.FromStandardName(dstLeaf)) + if err != nil { + return err + } + time.Sleep(500 * time.Millisecond) //Find a way around this + } + + srcFs.dirCache.FlushDir(srcRemote) + return nil } // Copy copies a directory (not implemented) From ae7c41852420dff50df732834e914c3ab30cbc2b Mon Sep 17 00:00:00 2001 From: lullius <> Date: Mon, 28 Jul 2025 22:34:43 +0200 Subject: [PATCH 4/4] backend/internxt: Implemented Move --- backend/internxt/internxt.go | 69 +++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 17 deletions(-) diff --git a/backend/internxt/internxt.go b/backend/internxt/internxt.go index e3da6cce419f7..b03146c4d5066 100644 --- a/backend/internxt/internxt.go +++ b/backend/internxt/internxt.go @@ -364,36 +364,71 @@ func (f *Fs) Remove(ctx context.Context, remote string) error { return nil } -// Move moves a directory (not implemented) +// Move src to this remote using server-side move operations. func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { - /* - srcObj, ok := src.(*Object) - if !ok { - fs.Debugf(src, "Can't move - not same remote type") - return nil, fs.ErrorCantMove - } + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + + srcLeaf, srcDirectoryID, err := f.dirCache.FindPath(ctx, srcObj.remote, false) + if err != nil { + return nil, err + } - srcLeaf, srcDirectoryID, err := f.dirCache.FindPath(ctx, srcObj.remote, false) + dstLeaf, dstDirectoryID, err := f.dirCache.FindPath(ctx, remote, true) + if err != nil { + return nil, err + } + + doMove := srcDirectoryID != dstDirectoryID + doRename := srcLeaf != dstLeaf + + var dstObj fs.Object + + // If we're doing both, we should rename to a temp name in case there's a file + // with the same name at the destination folder (we can't rename AND move with one call) + if doMove && doRename { + newFile, err := files.UpdateFileMeta(f.cfg, srcObj.uuid, &folders.File{Type: "__RCLONE_MOVE__"}) if err != nil { return nil, err } - //fmt.Printf("Move: srcLeaf %s, srcDirID %s\n", srcLeaf, srcDirectoryID) + time.Sleep(500 * time.Millisecond) //Find a way around this + dstObj = newObjectWithFile(f, remote, newFile) + } - dstLeaf, dstDirectoryID, err := f.dirCache.FindPath(ctx, remote, false) + if doMove { + newFile, err := files.MoveFile(f.cfg, srcObj.uuid, dstDirectoryID) if err != nil { return nil, err } - //fmt.Printf("Move: srcLeaf %s, srcDirID %s \n dstLeaf %s, dstDirID, %s", srcLeaf, srcDirectoryID, dstLeaf, dstDirectoryID) + time.Sleep(500 * time.Millisecond) //Find a way around this + dstObj = newObjectWithFile(f, remote, newFile) + } + + if doRename { + base := filepath.Base(remote) + name := strings.TrimSuffix(base, filepath.Ext(base)) + ext := strings.TrimPrefix(filepath.Ext(base), ".") + + updated := &folders.File{ + PlainName: f.opt.Encoding.FromStandardName(name), + Type: f.opt.Encoding.FromStandardName(ext), + } - doMove := srcDirectoryID != dstDirectoryID - doRename := srcLeaf != dstLeaf + newFile, err := files.UpdateFileMeta(f.cfg, srcObj.uuid, updated) + if err != nil { + return nil, err + } + time.Sleep(500 * time.Millisecond) //Find a way around this + dstObj = newObjectWithFile(f, remote, newFile) + } - println("Move ", doMove, doRename) - */ - // return f.client.Rename(ctx, f.root+src.Remote(), f.root+dst.Remote()) - return nil, fs.ErrorNotImplemented + return dstObj, nil } +// Move dir to destination using server-side move operations. func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { srcFs, ok := src.(*Fs) if !ok {