From 43217447cfced5ec6ca4dceda3104d7936dab4e9 Mon Sep 17 00:00:00 2001 From: Tobias Herber <22559657+herber@users.noreply.github.com> Date: Thu, 25 Dec 2025 14:48:47 +0100 Subject: [PATCH] Add signed upload urls --- clients/go/client.go | 17 ++++++- clients/rust/src/lib.rs | 38 +++++++++++++-- clients/typescript/src/index.ts | 14 +++++- object-store-backends/src/azure.rs | 69 +++++++++++++++++++++++----- object-store-backends/src/backend.rs | 20 +++++++- object-store-backends/src/gcs.rs | 28 +++++++---- object-store-backends/src/lib.rs | 2 +- object-store-backends/src/local.rs | 9 +++- object-store-backends/src/s3.rs | 66 +++++++++++++++++--------- object-store/src/api.rs | 8 +++- object-store/src/service.rs | 5 +- 11 files changed, 220 insertions(+), 56 deletions(-) diff --git a/clients/go/client.go b/clients/go/client.go index 130983e..19de5bf 100644 --- a/clients/go/client.go +++ b/clients/go/client.go @@ -48,6 +48,13 @@ type listObjectsResponse struct { Objects []ObjectMetadata `json:"objects"` } +type PublicUrlPurpose string + +const ( + PublicUrlPurposeRetrieve PublicUrlPurpose = "retrieve" + PublicUrlPurposeUpload PublicUrlPurpose = "upload" +) + type PublicURLResponse struct { URL string `json:"url"` ExpiresIn uint64 `json:"expires_in"` @@ -489,12 +496,18 @@ func (c *Client) ListObjects(bucket string, prefix *string, maxKeys *int) ([]Obj return result.Objects, nil } -func (c *Client) GetPublicURL(bucket, key string, expirationSecs *uint64) (*PublicURLResponse, error) { +func (c *Client) GetPublicURL(bucket, key string, expirationSecs *uint64, purpose *PublicUrlPurpose) (*PublicURLResponse, error) { urlPath := fmt.Sprintf("%s/buckets/%s/public-url/%s", c.baseURL, bucket, key) + params := url.Values{} if expirationSecs != nil { - params := url.Values{} params.Add("expiration_secs", strconv.FormatUint(*expirationSecs, 10)) + } + if purpose != nil { + params.Add("purpose", string(*purpose)) + } + + if len(params) > 0 { urlPath += "?" + params.Encode() } diff --git a/clients/rust/src/lib.rs b/clients/rust/src/lib.rs index 2bea7c6..b8c4d96 100644 --- a/clients/rust/src/lib.rs +++ b/clients/rust/src/lib.rs @@ -62,6 +62,19 @@ struct ListObjectsResponse { objects: Vec, } +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum PublicUrlPurpose { + Retrieve, + Upload, +} + +impl Default for PublicUrlPurpose { + fn default() -> Self { + Self::Retrieve + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PublicUrlResponse { pub url: String, @@ -407,11 +420,26 @@ impl ObjectStoreClient { bucket: &str, key: &str, expiration_secs: Option, + purpose: Option, ) -> Result { let mut url = format!("{}/buckets/{}/public-url/{}", self.base_url, bucket, key); + let mut params = vec![]; if let Some(exp) = expiration_secs { - url.push_str(&format!("?expiration_secs={}", exp)); + params.push(format!("expiration_secs={}", exp)); + } + + if let Some(p) = purpose { + let purpose_str = match p { + PublicUrlPurpose::Retrieve => "retrieve", + PublicUrlPurpose::Upload => "upload", + }; + params.push(format!("purpose={}", purpose_str)); + } + + if !params.is_empty() { + url.push('?'); + url.push_str(¶ms.join("&")); } let response = self.client.get(&url).send().await?; @@ -702,7 +730,7 @@ mod tests { let client = ObjectStoreClient::new(&server.url()); let response = client - .get_public_url("test-bucket", "test-key", Some(7200)) + .get_public_url("test-bucket", "test-key", Some(7200), None) .await .unwrap(); @@ -728,7 +756,7 @@ mod tests { let client = ObjectStoreClient::new(&server.url()); let response = client - .get_public_url("test-bucket", "test-key", None) + .get_public_url("test-bucket", "test-key", None, None) .await .unwrap(); @@ -750,7 +778,9 @@ mod tests { .await; let client = ObjectStoreClient::new(&server.url()); - let result = client.get_public_url("test-bucket", "test-key", None).await; + let result = client + .get_public_url("test-bucket", "test-key", None, None) + .await; assert!(result.is_err()); assert!(matches!(result.unwrap_err(), Error::NotFound(_))); diff --git a/clients/typescript/src/index.ts b/clients/typescript/src/index.ts index 8b30e90..5ce22d9 100644 --- a/clients/typescript/src/index.ts +++ b/clients/typescript/src/index.ts @@ -42,6 +42,11 @@ interface ListObjectsResponse { objects: ObjectMetadata[]; } +export enum PublicUrlPurpose { + Retrieve = 'retrieve', + Upload = 'upload', +} + export interface PublicURLResponse { url: string; expires_in: number; @@ -269,15 +274,20 @@ export class ObjectStorageClient { async getPublicURL( bucket: string, key: string, - expirationSecs?: number + expirationSecs?: number, + purpose?: PublicUrlPurpose ): Promise { try { - const params: Record = {}; + const params: Record = {}; if (expirationSecs !== undefined) { params.expiration_secs = expirationSecs; } + if (purpose !== undefined) { + params.purpose = purpose; + } + const response = await this.client.get( `/buckets/${bucket}/public-url/${key}`, { params } diff --git a/object-store-backends/src/azure.rs b/object-store-backends/src/azure.rs index 94fefa5..6223928 100644 --- a/object-store-backends/src/azure.rs +++ b/object-store-backends/src/azure.rs @@ -9,21 +9,24 @@ use sha2::{Digest, Sha256}; use std::collections::HashMap; use tracing::{debug, info, warn}; -use crate::backend::{Backend, ByteStream, ObjectData, ObjectMetadata}; +use crate::backend::{Backend, ByteStream, ObjectData, ObjectMetadata, PublicUrlPurpose}; use crate::error::{BackendError, BackendResult}; pub struct AzureBackend { client: ContainerClient, container_name: String, + account: String, + #[allow(dead_code)] + access_key: String, } impl AzureBackend { pub fn new(account: String, access_key: String, container_name: String) -> BackendResult { let storage_credentials = - StorageCredentials::access_key(account.clone(), Secret::new(access_key)); + StorageCredentials::access_key(account.clone(), Secret::new(access_key.clone())); - let client = - ClientBuilder::new(account, storage_credentials).container_client(&container_name); + let client = ClientBuilder::new(account.clone(), storage_credentials) + .container_client(&container_name); info!( "Initialized Azure Blob Storage backend with container: {}", @@ -33,6 +36,8 @@ impl AzureBackend { Ok(Self { client, container_name, + account, + access_key, }) } @@ -58,10 +63,10 @@ impl AzureBackend { } let storage_credentials = - StorageCredentials::access_key(account_name.clone(), Secret::new(access_key)); + StorageCredentials::access_key(account_name.clone(), Secret::new(access_key.clone())); - let client = - ClientBuilder::new(account_name, storage_credentials).container_client(&container_name); + let client = ClientBuilder::new(account_name.clone(), storage_credentials) + .container_client(&container_name); info!( "Initialized Azure Blob Storage backend with container: {} from connection string", @@ -71,6 +76,8 @@ impl AzureBackend { Ok(Self { client, container_name, + account: account_name, + access_key, }) } @@ -360,9 +367,49 @@ impl Backend for AzureBackend { } } - async fn get_public_url(&self, _key: &str, _expiration_secs: u64) -> BackendResult { - Err(BackendError::Provider( - "Public URL generation is not yet implemented for Azure backend.".to_string(), - )) + async fn get_public_url( + &self, + key: &str, + expiration_secs: u64, + purpose: PublicUrlPurpose, + ) -> BackendResult { + use azure_storage::shared_access_signature::service_sas::BlobSasPermissions; + use time::{Duration, OffsetDateTime}; + + let expiry = OffsetDateTime::now_utc() + Duration::seconds(expiration_secs as i64); + + let permissions = match purpose { + PublicUrlPurpose::Retrieve => BlobSasPermissions { + read: true, + ..Default::default() + }, + PublicUrlPurpose::Upload => BlobSasPermissions { + write: true, + create: true, + ..Default::default() + }, + }; + + let sas = self + .client + .shared_access_signature(permissions, expiry) + .await + .map_err(|e| BackendError::Provider(format!("Failed to generate SAS token: {}", e)))?; + + let token = sas + .token() + .map_err(|e| BackendError::Provider(format!("Failed to extract SAS token: {}", e)))?; + + let url = format!( + "https://{}.blob.core.windows.net/{}/{}?{}", + self.account, self.container_name, key, token + ); + + debug!( + "Generated SAS {:?} URL for Azure blob: {} (expires in {} seconds)", + purpose, key, expiration_secs + ); + + Ok(url) } } diff --git a/object-store-backends/src/backend.rs b/object-store-backends/src/backend.rs index 2947b6e..d33fba6 100644 --- a/object-store-backends/src/backend.rs +++ b/object-store-backends/src/backend.rs @@ -10,6 +10,19 @@ use crate::error::BackendResult; pub type ByteStream = Pin> + Send>>; +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum PublicUrlPurpose { + Retrieve, + Upload, +} + +impl Default for PublicUrlPurpose { + fn default() -> Self { + Self::Retrieve + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ObjectMetadata { pub key: String, @@ -57,7 +70,12 @@ pub trait Backend: Send + Sync { } } - async fn get_public_url(&self, key: &str, expiration_secs: u64) -> BackendResult; + async fn get_public_url( + &self, + key: &str, + expiration_secs: u64, + purpose: PublicUrlPurpose, + ) -> BackendResult; } pub fn compute_etag(data: &[u8]) -> String { diff --git a/object-store-backends/src/gcs.rs b/object-store-backends/src/gcs.rs index 10aa79e..1259177 100644 --- a/object-store-backends/src/gcs.rs +++ b/object-store-backends/src/gcs.rs @@ -12,7 +12,7 @@ use sha2::{Digest, Sha256}; use std::collections::HashMap; use tracing::{debug, info, warn}; -use crate::backend::{Backend, ByteStream, ObjectData, ObjectMetadata}; +use crate::backend::{Backend, ByteStream, ObjectData, ObjectMetadata, PublicUrlPurpose}; use crate::error::{BackendError, BackendResult}; pub struct GcsBackend { @@ -338,13 +338,23 @@ impl Backend for GcsBackend { } } - async fn get_public_url(&self, key: &str, expiration_secs: u64) -> BackendResult { + async fn get_public_url( + &self, + key: &str, + expiration_secs: u64, + purpose: PublicUrlPurpose, + ) -> BackendResult { let expiration = std::time::Duration::from_secs(expiration_secs); use google_cloud_storage::sign::{SignedURLMethod, SignedURLOptions}; + let method = match purpose { + PublicUrlPurpose::Retrieve => SignedURLMethod::GET, + PublicUrlPurpose::Upload => SignedURLMethod::PUT, + }; + let url_options = SignedURLOptions { - method: SignedURLMethod::GET, + method, expires: expiration, ..Default::default() }; @@ -361,18 +371,18 @@ impl Backend for GcsBackend { .await .map_err(|e| { warn!( - "Failed to generate signed URL for GCS object: {}: {:?}", - key, e + "Failed to generate signed {:?} URL for GCS object: {}: {:?}", + purpose, key, e ); BackendError::Provider(format!( - "Failed to generate signed URL for '{}': {}", - key, e + "Failed to generate signed {:?} URL for '{}': {}", + purpose, key, e )) })?; debug!( - "Generated signed URL for GCS object: {} (expires in {} seconds)", - key, expiration_secs + "Generated signed {:?} URL for GCS object: {} (expires in {} seconds)", + purpose, key, expiration_secs ); Ok(url_for) } diff --git a/object-store-backends/src/lib.rs b/object-store-backends/src/lib.rs index 67606ca..91c952d 100644 --- a/object-store-backends/src/lib.rs +++ b/object-store-backends/src/lib.rs @@ -5,5 +5,5 @@ pub mod gcs; pub mod local; pub mod s3; -pub use backend::{Backend, ByteStream, ObjectData, ObjectMetadata}; +pub use backend::{Backend, ByteStream, ObjectData, ObjectMetadata, PublicUrlPurpose}; pub use error::{BackendError, BackendResult}; diff --git a/object-store-backends/src/local.rs b/object-store-backends/src/local.rs index 6b6aca7..775fd92 100644 --- a/object-store-backends/src/local.rs +++ b/object-store-backends/src/local.rs @@ -9,7 +9,7 @@ use tokio::io::AsyncWriteExt; use tokio_util::io::ReaderStream; use tracing::{debug, info}; -use crate::backend::{Backend, ByteStream, ObjectData, ObjectMetadata}; +use crate::backend::{Backend, ByteStream, ObjectData, ObjectMetadata, PublicUrlPurpose}; use crate::error::{BackendError, BackendResult}; pub struct LocalBackend { @@ -198,7 +198,12 @@ impl Backend for LocalBackend { Ok(results) } - async fn get_public_url(&self, _key: &str, _expiration_secs: u64) -> BackendResult { + async fn get_public_url( + &self, + _key: &str, + _expiration_secs: u64, + _purpose: PublicUrlPurpose, + ) -> BackendResult { Err(BackendError::Provider( "Public URL generation is not supported for local backend".to_string(), )) diff --git a/object-store-backends/src/s3.rs b/object-store-backends/src/s3.rs index da1ebf5..5fd8f84 100644 --- a/object-store-backends/src/s3.rs +++ b/object-store-backends/src/s3.rs @@ -12,7 +12,7 @@ use std::time::Duration; use tokio_util::io::ReaderStream; use tracing::{debug, info, warn}; -use crate::backend::{Backend, ByteStream, ObjectData, ObjectMetadata}; +use crate::backend::{Backend, ByteStream, ObjectData, ObjectMetadata, PublicUrlPurpose}; use crate::error::{BackendError, BackendResult}; pub struct S3Backend { @@ -369,33 +369,57 @@ impl Backend for S3Backend { } } - async fn get_public_url(&self, key: &str, expiration_secs: u64) -> BackendResult { + async fn get_public_url( + &self, + key: &str, + expiration_secs: u64, + purpose: PublicUrlPurpose, + ) -> BackendResult { let presigning_config = PresigningConfig::expires_in(Duration::from_secs(expiration_secs)) .map_err(|e| { BackendError::Provider(format!("Failed to create presigning config: {}", e)) })?; - let presigned_request = self - .client - .get_object() - .bucket(&self.bucket_name) - .key(key) - .presigned(presigning_config) - .await - .map_err(|e| { - warn!( - "Failed to generate presigned URL for S3 object: {}: {:?}", - key, e - ); - BackendError::Provider(format!( - "Failed to generate presigned URL for '{}': {}", - key, e - )) - })?; + let presigned_request = match purpose { + PublicUrlPurpose::Retrieve => self + .client + .get_object() + .bucket(&self.bucket_name) + .key(key) + .presigned(presigning_config) + .await + .map_err(|e| { + warn!( + "Failed to generate presigned GET URL for S3 object: {}: {:?}", + key, e + ); + BackendError::Provider(format!( + "Failed to generate presigned GET URL for '{}': {}", + key, e + )) + })?, + PublicUrlPurpose::Upload => self + .client + .put_object() + .bucket(&self.bucket_name) + .key(key) + .presigned(presigning_config) + .await + .map_err(|e| { + warn!( + "Failed to generate presigned PUT URL for S3 object: {}: {:?}", + key, e + ); + BackendError::Provider(format!( + "Failed to generate presigned PUT URL for '{}': {}", + key, e + )) + })?, + }; debug!( - "Generated presigned URL for S3 object: {} (expires in {} seconds)", - key, expiration_secs + "Generated presigned {:?} URL for S3 object: {} (expires in {} seconds)", + purpose, key, expiration_secs ); Ok(presigned_request.uri().to_string()) } diff --git a/object-store/src/api.rs b/object-store/src/api.rs index 7af3206..c80f892 100644 --- a/object-store/src/api.rs +++ b/object-store/src/api.rs @@ -55,6 +55,7 @@ pub struct ListObjectsQuery { #[derive(Debug, Deserialize)] pub struct GetPublicUrlQuery { pub expiration_secs: Option, + pub purpose: Option, } #[derive(Debug, Serialize)] @@ -324,8 +325,13 @@ pub async fn get_public_url( // Default expiration is 1 hour (3600 seconds) let expiration_secs = params.expiration_secs.unwrap_or(3600); + // Default purpose is retrieve + let purpose = params + .purpose + .unwrap_or(object_store_backends::PublicUrlPurpose::Retrieve); + let url = service - .get_public_url(&bucket, &key, expiration_secs) + .get_public_url(&bucket, &key, expiration_secs, purpose) .await?; Ok(Json(PublicUrlResponse { diff --git a/object-store/src/service.rs b/object-store/src/service.rs index 370fe49..1b19b63 100644 --- a/object-store/src/service.rs +++ b/object-store/src/service.rs @@ -1,5 +1,5 @@ use bytes::Bytes; -use object_store_backends::{Backend, ByteStream, ObjectData, ObjectMetadata}; +use object_store_backends::{Backend, ByteStream, ObjectData, ObjectMetadata, PublicUrlPurpose}; use std::collections::HashMap; use std::sync::Arc; use tracing::{debug, info}; @@ -199,6 +199,7 @@ impl ObjectStoreService { bucket: &str, key: &str, expiration_secs: u64, + purpose: PublicUrlPurpose, ) -> ServiceResult { self.metadata.get_bucket(bucket).await?; @@ -208,7 +209,7 @@ impl ObjectStoreService { let url = self .backend - .get_public_url(&full_key, expiration_secs) + .get_public_url(&full_key, expiration_secs, purpose) .await?; Ok(url)