Skip to content

backend/internxt: fixes, Move and DirMove #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
StarHack merged 5 commits into from
Jul 30, 2025
Merged

backend/internxt: fixes, Move and DirMove #2

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ed0fa76
backend/internxt: getting all files and folders instead of only the 5…
Jul 28, 2025
9069356
backend/internxt: Option to simulate empty files
Jul 28, 2025
5e68e73
backend/internxt: Implemented DirMove
Jul 28, 2025
ae7c418
backend/internxt: Implemented Move
Jul 28, 2025
d7c7415
Merge branch 'internxt' into internxt_work
StarHack Jul 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
190 changes: 170 additions & 20 deletions backend/internxt/internxt.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
package internxt

import (
"bytes"
"context"
"fmt"
"io"
Expand Down Expand Up @@ -46,6 +47,11 @@ func init() {
IsPassword: true,
},
{
Name: "simulateEmptyFiles",
Default: false,
Help: "Simulates empty files by uploading a small placeholder file instead. Alters the filename when uploading to keep track of empty files, but this is not visible through rclone.",
},
{
Name: "use_2fa",
Help: "Do you use 2FA to login?",
Default: false,
Expand All @@ -65,13 +71,22 @@ func init() {
)
}

const (
EMPTY_FILE_EXT = ".__RCLONE_EMPTY__"
)

var (
EMPTY_FILE_BYTES = []byte{0x13, 0x09, 0x20, 0x23}
)

// Options holds configuration options for this interface
type Options struct {
Endpoint string `flag:"endpoint" help:"API endpoint"`
Email string `flag:"email" help:"Internxt account email"`
Password string `flag:"password" help:"Internxt account password"`
Use2FA bool `config:"use_2fa" help:"Do you use 2FA to login?"`
Encoding encoder.MultiEncoder `config:"encoding"`
Endpoint string `flag:"endpoint" help:"API endpoint"`
Email string `flag:"email" help:"Internxt account email"`
Password string `flag:"password" help:"Internxt account password"`
Encoding encoder.MultiEncoder `config:"encoding"`
SimulateEmptyFiles bool `config:"simulateEmptyFiles"`
Use2FA bool `config:"use_2fa" help:"Do you use 2FA to login?"`
}

// Fs represents an Internxt remote
Expand Down Expand Up @@ -233,11 +248,11 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
}

// Replace these calls with GetFolderContent? (fmt.Sprintf("/storage/v2/folder/%d%s", folderID, query))
childFolders, err := folders.ListFolders(f.cfg, id, folders.ListOptions{})
childFolders, err := folders.ListAllFolders(f.cfg, id)
if err != nil {
return err
}
childFiles, err := folders.ListFiles(f.cfg, id, folders.ListOptions{})
childFiles, err := folders.ListAllFiles(f.cfg, id)
if err != nil {
return err
}
Expand All @@ -263,7 +278,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
// If found, it returns its UUID and true. If not found, returns "", false.
func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (string, bool, error) {
//fmt.Printf("FindLeaf pathID: %s, leaf: %s\n", pathID, leaf)
entries, err := folders.ListFolders(f.cfg, pathID, folders.ListOptions{})
entries, err := folders.ListAllFolders(f.cfg, pathID)
if err != nil {
return "", false, err
}
Expand Down Expand Up @@ -298,15 +313,15 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
}
var out fs.DirEntries

foldersList, err := folders.ListFolders(f.cfg, dirID, folders.ListOptions{})
foldersList, err := folders.ListAllFolders(f.cfg, dirID)
if err != nil {
return nil, err
}
for _, e := range foldersList {
remote := filepath.Join(dir, f.opt.Encoding.ToStandardName(e.PlainName))
out = append(out, fs.NewDir(remote, e.ModificationTime))
}
filesList, err := folders.ListFiles(f.cfg, dirID, folders.ListOptions{})
filesList, err := folders.ListAllFiles(f.cfg, dirID)
if err != nil {
return nil, err
}
Expand All @@ -316,6 +331,11 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
remote += "." + e.Type
}
remote = filepath.Join(dir, f.opt.Encoding.ToStandardName(remote))
// If we found a file with the special empty file suffix, pretend that it's empty
if f.opt.SimulateEmptyFiles && strings.HasSuffix(remote, EMPTY_FILE_EXT) {
remote = strings.TrimSuffix(remote, EMPTY_FILE_EXT)
e.Size = "0"
}
out = append(out, newObjectWithFile(f, remote, &e))
}
return out, nil
Expand Down Expand Up @@ -361,21 +381,122 @@ func (f *Fs) Remove(ctx context.Context, remote string) error {
return nil
}

// Move moves a directory (not implemented)
func (f *Fs) Move(ctx context.Context, src, dst fs.Object) error {
// return f.client.Rename(ctx, f.root+src.Remote(), f.root+dst.Remote())
// Move src to this remote using server-side move operations.
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*Object)
if !ok {
fs.Debugf(src, "Can't move - not same remote type")
return nil, fs.ErrorCantMove
}

srcLeaf, srcDirectoryID, err := f.dirCache.FindPath(ctx, srcObj.remote, false)
if err != nil {
return nil, err
}

dstLeaf, dstDirectoryID, err := f.dirCache.FindPath(ctx, remote, true)
if err != nil {
return nil, err
}

doMove := srcDirectoryID != dstDirectoryID
doRename := srcLeaf != dstLeaf

var dstObj fs.Object

// If we're doing both, we should rename to a temp name in case there's a file
// with the same name at the destination folder (we can't rename AND move with one call)
if doMove && doRename {
newFile, err := files.UpdateFileMeta(f.cfg, srcObj.uuid, &folders.File{Type: "__RCLONE_MOVE__"})
if err != nil {
return nil, err
}
time.Sleep(500 * time.Millisecond) //Find a way around this
dstObj = newObjectWithFile(f, remote, newFile)
}

if doMove {
newFile, err := files.MoveFile(f.cfg, srcObj.uuid, dstDirectoryID)
if err != nil {
return nil, err
}
time.Sleep(500 * time.Millisecond) //Find a way around this
dstObj = newObjectWithFile(f, remote, newFile)
}

if doRename {
base := filepath.Base(remote)
name := strings.TrimSuffix(base, filepath.Ext(base))
ext := strings.TrimPrefix(filepath.Ext(base), ".")

updated := &folders.File{
PlainName: f.opt.Encoding.FromStandardName(name),
Type: f.opt.Encoding.FromStandardName(ext),
}

newFile, err := files.UpdateFileMeta(f.cfg, srcObj.uuid, updated)
if err != nil {
return nil, err
}
time.Sleep(500 * time.Millisecond) //Find a way around this
dstObj = newObjectWithFile(f, remote, newFile)
}

return dstObj, nil
}

// Move dir to destination using server-side move operations.
func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
srcFs, ok := src.(*Fs)
if !ok {
fs.Debugf(srcFs, "Can't move directory - not same remote type")
return fs.ErrorCantDirMove
}

srcID, _, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
if err != nil {
return err
}

doMove := srcID != dstDirectoryID
doRename := srcLeaf != dstLeaf

// If we're moving AND renaming we need to set a temp name first, else we risk collisions
if doMove && doRename {
err = folders.RenameFolder(f.cfg, srcID, f.opt.Encoding.FromStandardName(dstLeaf+".__RCLONE_MOVE__"))
if err != nil {
return err
}
time.Sleep(500 * time.Millisecond) //Find a way around this
}

if doMove {
err = folders.MoveFolder(f.cfg, srcID, dstDirectoryID)
if err != nil {
return err
}
time.Sleep(500 * time.Millisecond) //Find a way around this

}

if doRename {
err = folders.RenameFolder(f.cfg, srcID, f.opt.Encoding.FromStandardName(dstLeaf))
if err != nil {
return err
}
time.Sleep(500 * time.Millisecond) //Find a way around this
}

srcFs.dirCache.FlushDir(srcRemote)
return nil
}

// Copy copies a directory (not implemented)
func (f *Fs) Copy(ctx context.Context, src, dst fs.Object) error {
// return f.client.Copy(ctx, f.root+src.Remote(), f.root+dst.Remote())
return nil
return fs.ErrorNotImplemented
}

// DirCacheFlush flushes the dir cache (not implemented)
func (f *Fs) DirCacheFlush(ctx context.Context) {}

// NewObject creates a new object
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
parentDir := filepath.Dir(remote)
Expand All @@ -390,7 +511,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
return nil, fs.ErrorObjectNotFound
}

files, err := folders.ListFiles(f.cfg, dirID, folders.ListOptions{})
files, err := folders.ListAllFiles(f.cfg, dirID)
if err != nil {
return nil, err
}
Expand All @@ -402,6 +523,13 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
if f.opt.Encoding.ToStandardName(name) == filepath.Base(remote) {
return newObjectWithFile(f, remote, &e), nil
}
// If we are simulating empty files, check for a file with the special suffix and if found return it as if empty.
if f.opt.SimulateEmptyFiles {
if f.opt.Encoding.ToStandardName(name) == filepath.Base(remote+EMPTY_FILE_EXT) {
e.Size = "0"
return newObjectWithFile(f, remote, &e), nil
}
}
}
return nil, fs.ErrorObjectNotFound
}
Expand Down Expand Up @@ -470,13 +598,31 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
}
}

// Return nothing if we're faking an empty file
if o.f.opt.SimulateEmptyFiles && o.size == 0 {
return io.NopCloser(bytes.NewReader(nil)), nil
}
return buckets.DownloadFileStream(o.f.cfg, o.id, rangeValue)
}

// Update updates an existing file
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
isEmptyFile := false
if src.Size() == 0 {
return fs.ErrorCantUploadEmptyFiles
if !o.f.opt.SimulateEmptyFiles {
return fs.ErrorCantUploadEmptyFiles
} else {
// If we're faking an empty file, write some nonsense into it and give it a special suffix
isEmptyFile = true
in = bytes.NewReader(EMPTY_FILE_BYTES)
src = &Object{
f: o.f,
remote: src.Remote() + EMPTY_FILE_EXT,
modTime: src.ModTime(ctx),
size: int64(len(EMPTY_FILE_BYTES)),
}
o.remote = o.remote + EMPTY_FILE_EXT
}
}

// Check if object exists on the server
Expand All @@ -500,7 +646,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
}

// Create folder if it doesn't exist
_, dirID, err := o.f.dirCache.FindPath(ctx, o.Remote(), true)
_, dirID, err := o.f.dirCache.FindPath(ctx, o.remote, true)
if err != nil {
return err
}
Expand All @@ -513,6 +659,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// Update the object with the new info
o.uuid = meta.UUID
o.size = src.Size()
// If this is a simulated empty file set fake size to 0
if isEmptyFile {
o.size = 0
}
return nil
}

Expand Down