Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions clients/go/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,13 @@ type listObjectsResponse struct {
Objects []ObjectMetadata `json:"objects"`
}

type PublicUrlPurpose string

const (
PublicUrlPurposeRetrieve PublicUrlPurpose = "retrieve"
PublicUrlPurposeUpload PublicUrlPurpose = "upload"
)

type PublicURLResponse struct {
URL string `json:"url"`
ExpiresIn uint64 `json:"expires_in"`
Expand Down Expand Up @@ -489,12 +496,18 @@ func (c *Client) ListObjects(bucket string, prefix *string, maxKeys *int) ([]Obj
return result.Objects, nil
}

func (c *Client) GetPublicURL(bucket, key string, expirationSecs *uint64) (*PublicURLResponse, error) {
func (c *Client) GetPublicURL(bucket, key string, expirationSecs *uint64, purpose *PublicUrlPurpose) (*PublicURLResponse, error) {
urlPath := fmt.Sprintf("%s/buckets/%s/public-url/%s", c.baseURL, bucket, key)

params := url.Values{}
if expirationSecs != nil {
params := url.Values{}
params.Add("expiration_secs", strconv.FormatUint(*expirationSecs, 10))
}
if purpose != nil {
params.Add("purpose", string(*purpose))
}

if len(params) > 0 {
urlPath += "?" + params.Encode()
}

Expand Down
38 changes: 34 additions & 4 deletions clients/rust/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,19 @@ struct ListObjectsResponse {
objects: Vec<ObjectMetadata>,
}

#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum PublicUrlPurpose {
Retrieve,
Upload,
}

impl Default for PublicUrlPurpose {
fn default() -> Self {
Self::Retrieve
}
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PublicUrlResponse {
pub url: String,
Expand Down Expand Up @@ -407,11 +420,26 @@ impl ObjectStoreClient {
bucket: &str,
key: &str,
expiration_secs: Option<u64>,
purpose: Option<PublicUrlPurpose>,
) -> Result<PublicUrlResponse> {
let mut url = format!("{}/buckets/{}/public-url/{}", self.base_url, bucket, key);
let mut params = vec![];

if let Some(exp) = expiration_secs {
url.push_str(&format!("?expiration_secs={}", exp));
params.push(format!("expiration_secs={}", exp));
}

if let Some(p) = purpose {
let purpose_str = match p {
PublicUrlPurpose::Retrieve => "retrieve",
PublicUrlPurpose::Upload => "upload",
};
params.push(format!("purpose={}", purpose_str));
}

if !params.is_empty() {
url.push('?');
url.push_str(&params.join("&"));
}

let response = self.client.get(&url).send().await?;
Expand Down Expand Up @@ -702,7 +730,7 @@ mod tests {

let client = ObjectStoreClient::new(&server.url());
let response = client
.get_public_url("test-bucket", "test-key", Some(7200))
.get_public_url("test-bucket", "test-key", Some(7200), None)
.await
.unwrap();

Expand All @@ -728,7 +756,7 @@ mod tests {

let client = ObjectStoreClient::new(&server.url());
let response = client
.get_public_url("test-bucket", "test-key", None)
.get_public_url("test-bucket", "test-key", None, None)
.await
.unwrap();

Expand All @@ -750,7 +778,9 @@ mod tests {
.await;

let client = ObjectStoreClient::new(&server.url());
let result = client.get_public_url("test-bucket", "test-key", None).await;
let result = client
.get_public_url("test-bucket", "test-key", None, None)
.await;

assert!(result.is_err());
assert!(matches!(result.unwrap_err(), Error::NotFound(_)));
Expand Down
14 changes: 12 additions & 2 deletions clients/typescript/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ interface ListObjectsResponse {
objects: ObjectMetadata[];
}

export enum PublicUrlPurpose {
Retrieve = 'retrieve',
Upload = 'upload',
}

export interface PublicURLResponse {
url: string;
expires_in: number;
Expand Down Expand Up @@ -269,15 +274,20 @@ export class ObjectStorageClient {
async getPublicURL(
bucket: string,
key: string,
expirationSecs?: number
expirationSecs?: number,
purpose?: PublicUrlPurpose
): Promise<PublicURLResponse> {
try {
const params: Record<string, number> = {};
const params: Record<string, number | string> = {};

if (expirationSecs !== undefined) {
params.expiration_secs = expirationSecs;
}

if (purpose !== undefined) {
params.purpose = purpose;
}

const response = await this.client.get<PublicURLResponse>(
`/buckets/${bucket}/public-url/${key}`,
{ params }
Expand Down
69 changes: 58 additions & 11 deletions object-store-backends/src/azure.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,21 +9,24 @@ use sha2::{Digest, Sha256};
use std::collections::HashMap;
use tracing::{debug, info, warn};

use crate::backend::{Backend, ByteStream, ObjectData, ObjectMetadata};
use crate::backend::{Backend, ByteStream, ObjectData, ObjectMetadata, PublicUrlPurpose};
use crate::error::{BackendError, BackendResult};

pub struct AzureBackend {
client: ContainerClient,
container_name: String,
account: String,
#[allow(dead_code)]
access_key: String,
}

impl AzureBackend {
pub fn new(account: String, access_key: String, container_name: String) -> BackendResult<Self> {
let storage_credentials =
StorageCredentials::access_key(account.clone(), Secret::new(access_key));
StorageCredentials::access_key(account.clone(), Secret::new(access_key.clone()));

let client =
ClientBuilder::new(account, storage_credentials).container_client(&container_name);
let client = ClientBuilder::new(account.clone(), storage_credentials)
.container_client(&container_name);

info!(
"Initialized Azure Blob Storage backend with container: {}",
Expand All @@ -33,6 +36,8 @@ impl AzureBackend {
Ok(Self {
client,
container_name,
account,
access_key,
})
}

Expand All @@ -58,10 +63,10 @@ impl AzureBackend {
}

let storage_credentials =
StorageCredentials::access_key(account_name.clone(), Secret::new(access_key));
StorageCredentials::access_key(account_name.clone(), Secret::new(access_key.clone()));

let client =
ClientBuilder::new(account_name, storage_credentials).container_client(&container_name);
let client = ClientBuilder::new(account_name.clone(), storage_credentials)
.container_client(&container_name);

info!(
"Initialized Azure Blob Storage backend with container: {} from connection string",
Expand All @@ -71,6 +76,8 @@ impl AzureBackend {
Ok(Self {
client,
container_name,
account: account_name,
access_key,
})
}

Expand Down Expand Up @@ -360,9 +367,49 @@ impl Backend for AzureBackend {
}
}

async fn get_public_url(&self, _key: &str, _expiration_secs: u64) -> BackendResult<String> {
Err(BackendError::Provider(
"Public URL generation is not yet implemented for Azure backend.".to_string(),
))
async fn get_public_url(
&self,
key: &str,
expiration_secs: u64,
purpose: PublicUrlPurpose,
) -> BackendResult<String> {
use azure_storage::shared_access_signature::service_sas::BlobSasPermissions;
use time::{Duration, OffsetDateTime};

let expiry = OffsetDateTime::now_utc() + Duration::seconds(expiration_secs as i64);

let permissions = match purpose {
PublicUrlPurpose::Retrieve => BlobSasPermissions {
read: true,
..Default::default()
},
PublicUrlPurpose::Upload => BlobSasPermissions {
write: true,
create: true,
..Default::default()
},
};

let sas = self
.client
.shared_access_signature(permissions, expiry)
.await
.map_err(|e| BackendError::Provider(format!("Failed to generate SAS token: {}", e)))?;

let token = sas
.token()
.map_err(|e| BackendError::Provider(format!("Failed to extract SAS token: {}", e)))?;

let url = format!(
"https://{}.blob.core.windows.net/{}/{}?{}",
self.account, self.container_name, key, token
);

debug!(
"Generated SAS {:?} URL for Azure blob: {} (expires in {} seconds)",
purpose, key, expiration_secs
);

Ok(url)
}
}
20 changes: 19 additions & 1 deletion object-store-backends/src/backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,19 @@ use crate::error::BackendResult;

pub type ByteStream = Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>;

#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum PublicUrlPurpose {
Retrieve,
Upload,
}

impl Default for PublicUrlPurpose {
fn default() -> Self {
Self::Retrieve
}
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ObjectMetadata {
pub key: String,
Expand Down Expand Up @@ -57,7 +70,12 @@ pub trait Backend: Send + Sync {
}
}

async fn get_public_url(&self, key: &str, expiration_secs: u64) -> BackendResult<String>;
async fn get_public_url(
&self,
key: &str,
expiration_secs: u64,
purpose: PublicUrlPurpose,
) -> BackendResult<String>;
}

pub fn compute_etag(data: &[u8]) -> String {
Expand Down
28 changes: 19 additions & 9 deletions object-store-backends/src/gcs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use sha2::{Digest, Sha256};
use std::collections::HashMap;
use tracing::{debug, info, warn};

use crate::backend::{Backend, ByteStream, ObjectData, ObjectMetadata};
use crate::backend::{Backend, ByteStream, ObjectData, ObjectMetadata, PublicUrlPurpose};
use crate::error::{BackendError, BackendResult};

pub struct GcsBackend {
Expand Down Expand Up @@ -338,13 +338,23 @@ impl Backend for GcsBackend {
}
}

async fn get_public_url(&self, key: &str, expiration_secs: u64) -> BackendResult<String> {
async fn get_public_url(
&self,
key: &str,
expiration_secs: u64,
purpose: PublicUrlPurpose,
) -> BackendResult<String> {
let expiration = std::time::Duration::from_secs(expiration_secs);

use google_cloud_storage::sign::{SignedURLMethod, SignedURLOptions};

let method = match purpose {
PublicUrlPurpose::Retrieve => SignedURLMethod::GET,
PublicUrlPurpose::Upload => SignedURLMethod::PUT,
};

let url_options = SignedURLOptions {
method: SignedURLMethod::GET,
method,
expires: expiration,
..Default::default()
};
Expand All @@ -361,18 +371,18 @@ impl Backend for GcsBackend {
.await
.map_err(|e| {
warn!(
"Failed to generate signed URL for GCS object: {}: {:?}",
key, e
"Failed to generate signed {:?} URL for GCS object: {}: {:?}",
purpose, key, e
);
BackendError::Provider(format!(
"Failed to generate signed URL for '{}': {}",
key, e
"Failed to generate signed {:?} URL for '{}': {}",
purpose, key, e
))
})?;

debug!(
"Generated signed URL for GCS object: {} (expires in {} seconds)",
key, expiration_secs
"Generated signed {:?} URL for GCS object: {} (expires in {} seconds)",
purpose, key, expiration_secs
);
Ok(url_for)
}
Expand Down
2 changes: 1 addition & 1 deletion object-store-backends/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,5 @@ pub mod gcs;
pub mod local;
pub mod s3;

pub use backend::{Backend, ByteStream, ObjectData, ObjectMetadata};
pub use backend::{Backend, ByteStream, ObjectData, ObjectMetadata, PublicUrlPurpose};
pub use error::{BackendError, BackendResult};
9 changes: 7 additions & 2 deletions object-store-backends/src/local.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use tokio::io::AsyncWriteExt;
use tokio_util::io::ReaderStream;
use tracing::{debug, info};

use crate::backend::{Backend, ByteStream, ObjectData, ObjectMetadata};
use crate::backend::{Backend, ByteStream, ObjectData, ObjectMetadata, PublicUrlPurpose};
use crate::error::{BackendError, BackendResult};

pub struct LocalBackend {
Expand Down Expand Up @@ -198,7 +198,12 @@ impl Backend for LocalBackend {
Ok(results)
}

async fn get_public_url(&self, _key: &str, _expiration_secs: u64) -> BackendResult<String> {
async fn get_public_url(
&self,
_key: &str,
_expiration_secs: u64,
_purpose: PublicUrlPurpose,
) -> BackendResult<String> {
Err(BackendError::Provider(
"Public URL generation is not supported for local backend".to_string(),
))
Expand Down
Loading
Loading