From 7f8f3e0afedd90f72afc710ae8133a7d348f46b6 Mon Sep 17 00:00:00 2001 From: Dmitry Kolesnikov Date: Thu, 5 Jun 2025 21:08:05 +0300 Subject: [PATCH 1/2] (fea) CurlFS as an abstraction for pre-signed URLs --- file.go | 27 ++-------------------- filesystem.go | 62 ++++++++++++++++++++++++++++++++++++++++++++++++--- types.go | 7 ++++++ 3 files changed, 68 insertions(+), 28 deletions(-) diff --git a/file.go b/file.go index c9832b9..f7ec3c0 100644 --- a/file.go +++ b/file.go @@ -137,7 +137,7 @@ func (fd *reader[T]) lazyOpen() error { fd.fs.codec.DecodeGetOutput(val, fd.info.attr) if fd.fs.signer != nil && fd.fs.codec.s != nil { - if url, err := fd.fs.preSignGetUrl(fd.s3Key(fd.fs.root)); err == nil { + if url, err := fd.fs.preSignGetUrl(fd.s3Key(fd.fs.root), fd.fs.ttlSignedUrl); err == nil { fd.fs.codec.s.Put(fd.info.attr, url) } } @@ -234,29 +234,6 @@ func (fd *writer[T]) lazyOpen() { }() } -func (fd *writer[T]) preSignPutUrl() (string, error) { - ctx, cancel := context.WithTimeout(context.Background(), fd.fs.timeout) - defer cancel() - - req := &s3.PutObjectInput{ - Bucket: aws.String(fd.fs.bucket), - Key: fd.s3Key(fd.fs.root), - Metadata: make(map[string]string), - } - fd.fs.codec.EncodePutInput(fd.attr, req) - - val, err := fd.fs.signer.PresignPutObject(ctx, req, s3.WithPresignExpires(fd.fs.ttlSignedUrl)) - if err != nil { - return "", &fs.PathError{ - Op: "presign", - Path: fd.path, - Err: err, - } - } - - return val.URL, nil -} - func (fd *writer[T]) Write(p []byte) (int, error) { if fd.r == nil && fd.w == nil { fd.lazyOpen() @@ -300,7 +277,7 @@ func (fd *writer[T]) Stat() (fs.FileInfo, error) { fd.info.attr = new(T) } - if url, err := fd.preSignPutUrl(); err == nil { + if url, err := fd.fs.preSignPutUrl(fd.s3Key(fd.fs.root), fd.info.attr, fd.fs.ttlSignedUrl); err == nil { fd.fs.codec.s.Put(fd.info.attr, url) } } diff --git a/filesystem.go b/filesystem.go index fe794c4..f57b41f 100644 --- a/filesystem.go +++ b/filesystem.go @@ -44,6 +44,7 @@ var ( _ CreateFS[struct{}] = (*FileSystem[struct{}])(nil) _ RemoveFS = (*FileSystem[struct{}])(nil) _ CopyFS = (*FileSystem[struct{}])(nil) + _ CurlFS[struct{}] = (*FileSystem[struct{}])(nil) ) // Create a file system instance, mounting S3 Bucket. Use Option type to @@ -156,7 +157,7 @@ func (fsys *FileSystem[T]) Stat(path string) (fs.FileInfo, error) { fsys.codec.DecodeHeadOutput(val, info.attr) if fsys.signer != nil && fsys.codec.s != nil { - if url, err := fsys.preSignGetUrl(info.s3Key(fsys.root)); err == nil { + if url, err := fsys.preSignGetUrl(info.s3Key(fsys.root), fsys.ttlSignedUrl); err == nil { fsys.codec.s.Put(info.attr, url) } } @@ -174,7 +175,30 @@ func (fsys *FileSystem[T]) StatSys(stat fs.FileInfo) *T { return info.attr } -func (fsys *FileSystem[T]) preSignGetUrl(s3key *string) (string, error) { +func (fsys *FileSystem[T]) preSignPutUrl(s3key *string, attr *T, ttl time.Duration) (string, error) { + req := &s3.PutObjectInput{ + Bucket: aws.String(fsys.bucket), + Key: s3key, + Metadata: make(map[string]string), + } + fsys.codec.EncodePutInput(attr, req) + + ctx, cancel := context.WithTimeout(context.Background(), fsys.timeout) + defer cancel() + + val, err := fsys.signer.PresignPutObject(ctx, req, s3.WithPresignExpires(ttl)) + if err != nil { + return "", &fs.PathError{ + Op: "presign", + Path: "/" + aws.ToString(s3key), + Err: err, + } + } + + return val.URL, nil +} + +func (fsys *FileSystem[T]) preSignGetUrl(s3key *string, ttl time.Duration) (string, error) { req := &s3.GetObjectInput{ Bucket: aws.String(fsys.bucket), Key: s3key, @@ -183,7 +207,7 @@ func (fsys *FileSystem[T]) preSignGetUrl(s3key *string) (string, error) { ctx, cancel := context.WithTimeout(context.Background(), fsys.timeout) defer cancel() - val, err := fsys.signer.PresignGetObject(ctx, req, s3.WithPresignExpires(fsys.ttlSignedUrl)) + val, err := fsys.signer.PresignGetObject(ctx, req, s3.WithPresignExpires(ttl)) if err != nil { return "", &fs.PathError{ Op: "presign", @@ -338,6 +362,38 @@ func (fsys *FileSystem[T]) Wait(path string, timeout time.Duration) error { return nil } +func (fsys *FileSystem[T]) PutUrl(path string, attr *T, ttl time.Duration) (string, error) { + if fsys.signer == nil { + return "", &fs.PathError{ + Op: "puturl", + Path: path, + Err: errors.New("signer is not configured"), + } + } + + if err := RequireValidPath("puturl", path); err != nil { + return "", err + } + + return fsys.preSignPutUrl(s3Key(fsys.root, path), attr, ttl) +} + +func (fsys *FileSystem[T]) GetUrl(path string, ttl time.Duration) (string, error) { + if fsys.signer == nil { + return "", &fs.PathError{ + Op: "geturl", + Path: path, + Err: errors.New("signer is not configured"), + } + } + + if err := RequireValidPath("geturl", path); err != nil { + return "", err + } + + return fsys.preSignGetUrl(s3Key(fsys.root, path), ttl) +} + //------------------------------------------------------------------------------ func recoverNoSuchKey(err error) bool { diff --git a/types.go b/types.go index d341e4d..dff6973 100644 --- a/types.go +++ b/types.go @@ -56,6 +56,13 @@ type CopyFS interface { Wait(path string, timeout time.Duration) error } +// File System extension supporting I/O operations via urls +type CurlFS[T any] interface { + fs.FS + PutUrl(path string, attr *T, ttl time.Duration) (string, error) + GetUrl(path string, ttl time.Duration) (string, error) +} + // well-known attributes controlled by S3 system type SystemMetadata struct { CacheControl string From 441b850c9eeb8ff465a6e7b73bc88998e4aa28c1 Mon Sep 17 00:00:00 2001 From: Dmitry Kolesnikov Date: Thu, 5 Jun 2025 22:17:30 +0300 Subject: [PATCH 2/2] update notation of CurlFS functions --- filesystem.go | 12 ++++++------ types.go | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/filesystem.go b/filesystem.go index f57b41f..e4e3b9d 100644 --- a/filesystem.go +++ b/filesystem.go @@ -362,32 +362,32 @@ func (fsys *FileSystem[T]) Wait(path string, timeout time.Duration) error { return nil } -func (fsys *FileSystem[T]) PutUrl(path string, attr *T, ttl time.Duration) (string, error) { +func (fsys *FileSystem[T]) PutFileUrl(path string, attr *T, ttl time.Duration) (string, error) { if fsys.signer == nil { return "", &fs.PathError{ - Op: "puturl", + Op: "putfileurl", Path: path, Err: errors.New("signer is not configured"), } } - if err := RequireValidPath("puturl", path); err != nil { + if err := RequireValidPath("putfileurl", path); err != nil { return "", err } return fsys.preSignPutUrl(s3Key(fsys.root, path), attr, ttl) } -func (fsys *FileSystem[T]) GetUrl(path string, ttl time.Duration) (string, error) { +func (fsys *FileSystem[T]) GetFileUrl(path string, ttl time.Duration) (string, error) { if fsys.signer == nil { return "", &fs.PathError{ - Op: "geturl", + Op: "getfileurl", Path: path, Err: errors.New("signer is not configured"), } } - if err := RequireValidPath("geturl", path); err != nil { + if err := RequireValidPath("getfileurl", path); err != nil { return "", err } diff --git a/types.go b/types.go index dff6973..0eb72b2 100644 --- a/types.go +++ b/types.go @@ -59,8 +59,8 @@ type CopyFS interface { // File System extension supporting I/O operations via urls type CurlFS[T any] interface { fs.FS - PutUrl(path string, attr *T, ttl time.Duration) (string, error) - GetUrl(path string, ttl time.Duration) (string, error) + PutFileUrl(path string, attr *T, ttl time.Duration) (string, error) + GetFileUrl(path string, ttl time.Duration) (string, error) } // well-known attributes controlled by S3 system