Skip to content

Remove upload file #41

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dajimenezriv-internxt wants to merge 1 commit into
base: add-openapi-schema
Choose a base branch
from
Open

Remove upload file #41

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions auth/auth_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,21 +39,21 @@ func TestRefreshToken(t *testing.T) {
token: "invalid-token",
mockStatusCode: http.StatusUnauthorized,
expectError: true,
errorContains: "refresh token failed with status 401",
errorContains: "refresh token: error message (status 401)",
},
{
name: "server error - 500",
token: "valid-token",
mockStatusCode: http.StatusInternalServerError,
expectError: true,
errorContains: "refresh token failed with status 500",
errorContains: "refresh token: error message (status 500)",
},
{
name: "forbidden - 403",
token: "valid-token",
mockStatusCode: http.StatusForbidden,
expectError: true,
errorContains: "refresh token failed with status 403",
errorContains: "refresh token: error message (status 403)",
},
{
name: "missing newToken in response",
Expand Down
73 changes: 14 additions & 59 deletions buckets/upload.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"hash"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
Expand Down Expand Up @@ -90,46 +89,10 @@ func uploadEncryptedData(ctx context.Context, cfg *config.Config, encryptedReade
return finishResp.ID, nil
}

func UploadFile(ctx context.Context, cfg *config.Config, filePath, targetFolderUUID string, modTime time.Time) (*CreateMetaResponse, error) {
f, err := os.Open(filePath)
if err != nil {
return nil, fmt.Errorf("failed to read file %s: %w", filePath, err)
}
defer f.Close()

fileInfo, err := f.Stat()
if err != nil {
return nil, fmt.Errorf("failed to stat file %s: %w", filePath, err)
}
plainSize := fileInfo.Size()

// Setup encryption
encryptedReader, sha256Hasher, encIndex, err := encryptionSetup(f, cfg)
if err != nil {
return nil, err
}

// Upload to network
fileID, err := uploadEncryptedData(ctx, cfg, encryptedReader, sha256Hasher, encIndex, plainSize)
if err != nil {
return nil, fmt.Errorf("failed to transfer file data: %w", err)
}

// Create Drive file metadata
base := filepath.Base(filePath)
name := strings.TrimSuffix(base, filepath.Ext(base))
ext := strings.TrimPrefix(filepath.Ext(base), ".")
meta, err := CreateMetaFile(ctx, cfg, name, cfg.Bucket, &fileID, "03-aes", targetFolderUUID, name, ext, plainSize, modTime)
if err != nil {
return nil, fmt.Errorf("failed to create file metadata: %w", err)
}
return meta, nil
}

// UploadFileStream uploads data from the provided io.Reader into Internxt,
// encrypting it on the fly and creating the metadata file in the target folder.
// It returns the CreateMetaResponse of the created file entry.
func UploadFileStream(ctx context.Context, cfg *config.Config, targetFolderUUID, fileName string, in io.Reader, plainSize int64, modTime time.Time) (*CreateMetaResponse, error) {
func UploadFileStream(ctx context.Context, cfg *config.Config, targetFolderUUID, fileName string, in io.Reader, plainSize int64, modTime time.Time) (*string, error) {
var ph [32]byte
if _, err := rand.Read(ph[:]); err != nil {
return nil, fmt.Errorf("cannot generate random index: %w", err)
Expand Down Expand Up @@ -215,19 +178,12 @@ func UploadFileStream(ctx context.Context, cfg *config.Config, targetFolderUUID,
return nil, fmt.Errorf("failed to finish upload: %w", err)
}

base := filepath.Base(fileName)
name := strings.TrimSuffix(base, filepath.Ext(base))
ext := strings.TrimPrefix(filepath.Ext(base), ".")
meta, err := CreateMetaFile(ctx, cfg, name, cfg.Bucket, &finishResp.ID, "03-aes", targetFolderUUID, name, ext, plainSize, modTime)
if err != nil {
return nil, fmt.Errorf("failed to create file metadata: %w", err)
}
return meta, nil
return &finishResp.ID, nil
}

// UploadFileStreamMultipart uploads data from an io.Reader using multipart upload.
// This is intended for large files (>100MB) and splits the file into multiple chunks
func UploadFileStreamMultipart(ctx context.Context, cfg *config.Config, targetFolderUUID, fileName string, in io.Reader, plainSize int64, modTime time.Time) (*CreateMetaResponse, error) {
func UploadFileStreamMultipart(ctx context.Context, cfg *config.Config, targetFolderUUID, fileName string, in io.Reader, plainSize int64, modTime time.Time) (*string, error) {
state, err := newMultipartUploadState(cfg, plainSize)
if err != nil {
return nil, fmt.Errorf("failed to initialize multipart upload state: %w", err)
Expand All @@ -243,15 +199,7 @@ func UploadFileStreamMultipart(ctx context.Context, cfg *config.Config, targetFo
return nil, fmt.Errorf("failed to finish multipart upload: %w", err)
}

base := filepath.Base(fileName)
name := strings.TrimSuffix(base, filepath.Ext(base))
ext := strings.TrimPrefix(filepath.Ext(base), ".")
meta, err := CreateMetaFile(ctx, cfg, name, cfg.Bucket, &finishResp.ID, "03-aes", targetFolderUUID, name, ext, plainSize, modTime)
if err != nil {
return nil, fmt.Errorf("failed to create file metadata: %w", err)
}

return meta, nil
return &finishResp.ID, nil
}

// UploadFileStreamAuto automatically chooses between single-part and multipart upload
Expand Down Expand Up @@ -290,24 +238,31 @@ func UploadFileStreamAuto(ctx context.Context, cfg *config.Config, targetFolderU
var capturedData *bytes.Buffer
var capturedReader io.Reader = in

base := filepath.Base(fileName)
name := strings.TrimSuffix(base, filepath.Ext(base))
ext := strings.TrimPrefix(filepath.Ext(fileName), ".")
if thumbnails.IsSupportedFormat(ext) && plainSize > 0 && plainSize <= config.MaxThumbnailSourceSize {
capturedData = &bytes.Buffer{}
capturedReader = io.TeeReader(in, capturedData)
}

var meta *CreateMetaResponse
var fileId *string
var err error
if plainSize >= config.DefaultMultipartMinSize {
meta, err = UploadFileStreamMultipart(ctx, cfg, targetFolderUUID, fileName, capturedReader, plainSize, modTime)
fileId, err = UploadFileStreamMultipart(ctx, cfg, targetFolderUUID, fileName, capturedReader, plainSize, modTime)
} else {
meta, err = UploadFileStream(ctx, cfg, targetFolderUUID, fileName, capturedReader, plainSize, modTime)
fileId, err = UploadFileStream(ctx, cfg, targetFolderUUID, fileName, capturedReader, plainSize, modTime)
}

if err != nil {
return nil, err
}

meta, err := CreateMetaFile(ctx, cfg, name, cfg.Bucket, fileId, "03-aes", targetFolderUUID, name, ext, plainSize, modTime)
if err != nil {
return nil, fmt.Errorf("failed to create file metadata: %w", err)
}

if capturedData != nil && capturedData.Len() > 0 {
thumbnailWG.Add(1)
go uploadThumbnailAsync(ctx, cfg, meta.UUID, ext, capturedData.Bytes())
Expand Down
Loading