From 8bdfb841d515a8044d1bd4fabf837137dc7ac6bc Mon Sep 17 00:00:00 2001 From: Tobias Herber <22559657+herber@users.noreply.github.com> Date: Wed, 24 Dec 2025 12:41:37 +0100 Subject: [PATCH 1/2] Add better metadata handling --- object-store/src/api.rs | 52 ++++++++++++++++++++++++++++++++++++ object-store/src/error.rs | 4 +-- object-store/src/metadata.rs | 27 +++++++++++++++++++ object-store/src/router.rs | 4 +++ object-store/src/service.rs | 19 ++++++++++--- 5 files changed, 100 insertions(+), 6 deletions(-) diff --git a/object-store/src/api.rs b/object-store/src/api.rs index 2351b05..226b452 100644 --- a/object-store/src/api.rs +++ b/object-store/src/api.rs @@ -21,6 +21,7 @@ pub struct CreateBucketRequest { #[derive(Debug, Serialize, Deserialize)] pub struct BucketResponse { + pub id: String, pub name: String, pub created_at: String, } @@ -65,6 +66,7 @@ pub struct PublicUrlResponse { impl From for BucketResponse { fn from(bucket: Bucket) -> Self { Self { + id: bucket.id, name: bucket.name, created_at: bucket.created_at, } @@ -91,6 +93,12 @@ pub async fn health_check() -> impl IntoResponse { })) } +pub async fn ping() -> impl IntoResponse { + Json(serde_json::json!({ + "message": "pong" + })) +} + pub async fn create_bucket( State(service): State, Json(payload): Json, @@ -99,6 +107,14 @@ pub async fn create_bucket( Ok(Json(bucket.into())) } +pub async fn upsert_bucket( + State(service): State, + Json(payload): Json, +) -> ServiceResult> { + let bucket = service.upsert_bucket(&payload.name).await?; + Ok(Json(bucket.into())) +} + pub async fn list_buckets( State(service): State, ) -> ServiceResult> { @@ -109,6 +125,14 @@ pub async fn list_buckets( Ok(Json(response)) } +pub async fn get_bucket_by_id( + State(service): State, + Path(id): Path, +) -> ServiceResult> { + let bucket = service.get_bucket_by_id(&id).await?; + Ok(Json(bucket.into())) +} + pub async fn delete_bucket( State(service): State, Path(bucket): Path, @@ -198,11 +222,29 @@ pub async fn get_object( .unwrap_or_else(|_| "0".parse().unwrap()), ); + // Add custom metadata as x-object-meta-* headers + for (key, value) in obj_data.metadata.custom_metadata.iter() { + let header_name = format!("x-object-meta-{}", key); + if let Ok(header_value) = value.parse() { + if let Ok(header_name) = header_name.parse::() { + headers.insert(header_name, header_value); + } + } + } + let body = Body::from_stream(obj_data.stream); Ok((headers, body).into_response()) } +pub async fn get_object_info( + State(service): State, + Path((bucket, key)): Path<(String, String)>, +) -> ServiceResult> { + let metadata = service.head_object(&bucket, &key).await?; + Ok(Json(metadata.into())) +} + pub async fn head_object( State(service): State, Path((bucket, key)): Path<(String, String)>, @@ -243,6 +285,16 @@ pub async fn head_object( .unwrap_or_else(|_| "0".parse().unwrap()), ); + // Add custom metadata as x-object-meta-* headers + for (key, value) in metadata.custom_metadata.iter() { + let header_name = format!("x-object-meta-{}", key); + if let Ok(header_value) = value.parse() { + if let Ok(header_name) = header_name.parse::() { + headers.insert(header_name, header_value); + } + } + } + Ok((StatusCode::OK, headers).into_response()) } diff --git a/object-store/src/error.rs b/object-store/src/error.rs index 3c0d67f..6a17f6c 100644 --- a/object-store/src/error.rs +++ b/object-store/src/error.rs @@ -53,8 +53,8 @@ impl IntoResponse for ServiceError { ServiceError::InvalidBucketName(_) | ServiceError::InvalidObjectKey(_) => { (StatusCode::BAD_REQUEST, self.to_string()) } - ServiceError::Backend(object_store_backends::BackendError::NotFound(_)) => { - (StatusCode::NOT_FOUND, self.to_string()) + ServiceError::Backend(object_store_backends::BackendError::NotFound(key)) => { + (StatusCode::NOT_FOUND, format!("Object not found: {}", key)) } ServiceError::Backend(_) => (StatusCode::INTERNAL_SERVER_ERROR, self.to_string()), _ => (StatusCode::INTERNAL_SERVER_ERROR, self.to_string()), diff --git a/object-store/src/metadata.rs b/object-store/src/metadata.rs index e53a907..c51f9d5 100644 --- a/object-store/src/metadata.rs +++ b/object-store/src/metadata.rs @@ -290,6 +290,33 @@ impl MetadataStore { .ok_or_else(|| ServiceError::BucketNotFound(name.to_string())) } + pub async fn get_bucket_by_id(&self, id: &str) -> ServiceResult { + // Ensure cache is fresh + self.ensure_cache_fresh().await?; + + // Search cache for bucket by ID + { + let cache = self.cache.read().await; + for bucket in cache.all_buckets() { + if bucket.id == id { + return Ok(bucket); + } + } + } + + // Not found in cache - refresh and try again + self.refresh_cache().await?; + + let cache = self.cache.read().await; + for bucket in cache.all_buckets() { + if bucket.id == id { + return Ok(bucket); + } + } + + Err(ServiceError::BucketNotFound(format!("id: {}", id))) + } + pub async fn list_buckets(&self) -> ServiceResult> { self.ensure_cache_fresh().await?; diff --git a/object-store/src/router.rs b/object-store/src/router.rs index 73cbe19..0f33952 100644 --- a/object-store/src/router.rs +++ b/object-store/src/router.rs @@ -13,14 +13,18 @@ use crate::service::ObjectStoreService; pub fn create_router(service: Arc) -> Router { Router::new() .route("/health", get(health_check)) + .route("/ping", get(ping)) .route("/buckets", post(create_bucket)) + .route("/buckets", put(upsert_bucket)) .route("/buckets", get(list_buckets)) + .route("/buckets/:id", get(get_bucket_by_id)) .route("/buckets/:bucket", delete(delete_bucket)) .route("/buckets/:bucket/objects/*key", put(put_object)) .route("/buckets/:bucket/objects/*key", get(get_object)) .route("/buckets/:bucket/objects/*key", head(head_object)) .route("/buckets/:bucket/objects/*key", delete(delete_object)) .route("/buckets/:bucket/objects", get(list_objects)) + .route("/buckets/:bucket/object-info/*key", get(get_object_info)) .route("/buckets/:bucket/public-url/*key", get(get_public_url)) .layer( ServiceBuilder::new() diff --git a/object-store/src/service.rs b/object-store/src/service.rs index ae25ca6..370fe49 100644 --- a/object-store/src/service.rs +++ b/object-store/src/service.rs @@ -39,10 +39,25 @@ impl ObjectStoreService { Ok(bucket) } + pub async fn upsert_bucket(&self, name: &str) -> ServiceResult { + // Try to get existing bucket first + if let Ok(bucket) = self.metadata.get_bucket(name).await { + debug!("Bucket {} already exists, returning existing", name); + return Ok(bucket); + } + + // Bucket doesn't exist, create it + self.create_bucket(name).await + } + pub async fn list_buckets(&self) -> ServiceResult> { self.metadata.list_buckets().await } + pub async fn get_bucket_by_id(&self, id: &str) -> ServiceResult { + self.metadata.get_bucket_by_id(id).await + } + pub async fn delete_bucket(&self, name: &str) -> ServiceResult<()> { self.metadata.get_bucket(name).await?; @@ -196,10 +211,6 @@ impl ObjectStoreService { .get_public_url(&full_key, expiration_secs) .await?; - info!( - "Generated public URL for object: {}/{} (expires in {} seconds)", - bucket, key, expiration_secs - ); Ok(url) } } From 398e6f9bf632b8b7352548a9a7a4e388533a249a Mon Sep 17 00:00:00 2001 From: Tobias Herber <22559657+herber@users.noreply.github.com> Date: Wed, 24 Dec 2025 12:41:47 +0100 Subject: [PATCH 2/2] Update clients --- clients/go/client.go | 144 +++++++++++++++++++++++++++++++- clients/rust/src/lib.rs | 89 +++++++++++++++++++- clients/typescript/package.json | 2 +- clients/typescript/src/index.ts | 62 +++++++++++++- 4 files changed, 288 insertions(+), 9 deletions(-) diff --git a/clients/go/client.go b/clients/go/client.go index 153ce72..130983e 100644 --- a/clients/go/client.go +++ b/clients/go/client.go @@ -17,6 +17,7 @@ type Client struct { } type Bucket struct { + ID string `json:"id"` Name string `json:"name"` CreatedAt string `json:"created_at"` } @@ -77,6 +78,29 @@ func NewClientWithHTTP(baseURL string, httpClient *http.Client) *Client { } } +func (c *Client) Ping() error { + req, err := http.NewRequest("GET", c.baseURL+"/ping", nil) + if err != nil { + return err + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + return &Error{ + StatusCode: resp.StatusCode, + Message: string(bodyBytes), + } + } + + return nil +} + func (c *Client) CreateBucket(name string) (*Bucket, error) { reqBody := createBucketRequest{Name: name} body, err := json.Marshal(reqBody) @@ -112,6 +136,69 @@ func (c *Client) CreateBucket(name string) (*Bucket, error) { return &bucket, nil } +func (c *Client) UpsertBucket(name string) (*Bucket, error) { + reqBody := createBucketRequest{Name: name} + body, err := json.Marshal(reqBody) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("PUT", c.baseURL+"/buckets", bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + return nil, &Error{ + StatusCode: resp.StatusCode, + Message: string(bodyBytes), + } + } + + var bucket Bucket + if err := json.NewDecoder(resp.Body).Decode(&bucket); err != nil { + return nil, err + } + + return &bucket, nil +} + +func (c *Client) GetBucket(id string) (*Bucket, error) { + req, err := http.NewRequest("GET", c.baseURL+"/buckets/"+id, nil) + if err != nil { + return nil, err + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + return nil, &Error{ + StatusCode: resp.StatusCode, + Message: string(bodyBytes), + } + } + + var bucket Bucket + if err := json.NewDecoder(resp.Body).Decode(&bucket); err != nil { + return nil, err + } + + return &bucket, nil +} + func (c *Client) ListBuckets() ([]Bucket, error) { req, err := http.NewRequest("GET", c.baseURL+"/buckets", nil) if err != nil { @@ -233,6 +320,18 @@ func (c *Client) GetObject(bucket, key string) (*ObjectData, error) { ct = &contentType } + // Extract custom metadata from x-object-meta-* headers + metadata := make(map[string]string) + for headerName, headerValues := range resp.Header { + if len(headerValues) > 0 { + const prefix = "X-Object-Meta-" + if len(headerName) > len(prefix) && headerName[:len(prefix)] == prefix { + metaKey := headerName[len(prefix):] + metadata[metaKey] = headerValues[0] + } + } + } + return &ObjectData{ Metadata: ObjectMetadata{ Key: key, @@ -240,7 +339,7 @@ func (c *Client) GetObject(bucket, key string) (*ObjectData, error) { ContentType: ct, ETag: resp.Header.Get("ETag"), LastModified: resp.Header.Get("Last-Modified"), - Metadata: make(map[string]string), + Metadata: metadata, }, Data: data, }, nil @@ -273,16 +372,57 @@ func (c *Client) HeadObject(bucket, key string) (*ObjectMetadata, error) { ct = &contentType } + // Extract custom metadata from x-object-meta-* headers + metadata := make(map[string]string) + for headerName, headerValues := range resp.Header { + if len(headerValues) > 0 { + const prefix = "X-Object-Meta-" + if len(headerName) > len(prefix) && headerName[:len(prefix)] == prefix { + metaKey := headerName[len(prefix):] + metadata[metaKey] = headerValues[0] + } + } + } + return &ObjectMetadata{ Key: key, Size: size, ContentType: ct, ETag: resp.Header.Get("ETag"), LastModified: resp.Header.Get("Last-Modified"), - Metadata: make(map[string]string), + Metadata: metadata, }, nil } +func (c *Client) GetObjectInfo(bucket, key string) (*ObjectMetadata, error) { + urlPath := fmt.Sprintf("%s/buckets/%s/object-info/%s", c.baseURL, bucket, key) + req, err := http.NewRequest("GET", urlPath, nil) + if err != nil { + return nil, err + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + return nil, &Error{ + StatusCode: resp.StatusCode, + Message: string(bodyBytes), + } + } + + var objMetadata ObjectMetadata + if err := json.NewDecoder(resp.Body).Decode(&objMetadata); err != nil { + return nil, err + } + + return &objMetadata, nil +} + func (c *Client) DeleteObject(bucket, key string) error { urlPath := fmt.Sprintf("%s/buckets/%s/objects/%s", c.baseURL, bucket, key) req, err := http.NewRequest("DELETE", urlPath, nil) diff --git a/clients/rust/src/lib.rs b/clients/rust/src/lib.rs index e262bf1..2bea7c6 100644 --- a/clients/rust/src/lib.rs +++ b/clients/rust/src/lib.rs @@ -26,6 +26,7 @@ pub type Result = std::result::Result; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Bucket { + pub id: String, pub name: String, pub created_at: String, } @@ -87,6 +88,18 @@ impl ObjectStoreClient { } } + pub async fn ping(&self) -> Result<()> { + let url = format!("{}/ping", self.base_url); + let response = self.client.get(&url).send().await?; + + match response.status() { + StatusCode::OK => Ok(()), + _ => Err(Error::ServerError( + response.text().await.unwrap_or_default(), + )), + } + } + pub async fn create_bucket(&self, name: &str) -> Result { let url = format!("{}/buckets", self.base_url); let req = CreateBucketRequest { @@ -107,6 +120,38 @@ impl ObjectStoreClient { } } + pub async fn upsert_bucket(&self, name: &str) -> Result { + let url = format!("{}/buckets", self.base_url); + let req = CreateBucketRequest { + name: name.to_string(), + }; + + let response = self.client.put(&url).json(&req).send().await?; + + match response.status() { + StatusCode::OK => Ok(response.json().await?), + StatusCode::BAD_REQUEST => { + Err(Error::BadRequest(response.text().await.unwrap_or_default())) + } + _ => Err(Error::ServerError( + response.text().await.unwrap_or_default(), + )), + } + } + + pub async fn get_bucket(&self, id: &str) -> Result { + let url = format!("{}/buckets/{}", self.base_url, id); + let response = self.client.get(&url).send().await?; + + match response.status() { + StatusCode::OK => Ok(response.json().await?), + StatusCode::NOT_FOUND => Err(Error::NotFound(id.to_string())), + _ => Err(Error::ServerError( + response.text().await.unwrap_or_default(), + )), + } + } + pub async fn list_buckets(&self) -> Result> { let url = format!("{}/buckets", self.base_url); let response = self.client.get(&url).send().await?; @@ -206,6 +251,16 @@ impl ObjectStoreClient { .and_then(|s| s.parse().ok()) .unwrap_or(0); + // Extract custom metadata from x-object-meta-* headers + let mut metadata = HashMap::new(); + for (header_name, header_value) in response.headers().iter() { + if let Some(meta_key) = header_name.as_str().strip_prefix("x-object-meta-") { + if let Ok(meta_value) = header_value.to_str() { + metadata.insert(meta_key.to_string(), meta_value.to_string()); + } + } + } + let data = response.bytes().await?; Ok(ObjectData { @@ -215,7 +270,7 @@ impl ObjectStoreClient { content_type, etag, last_modified, - metadata: HashMap::new(), + metadata, }, data, }) @@ -260,13 +315,23 @@ impl ObjectStoreClient { .and_then(|s| s.parse().ok()) .unwrap_or(0); + // Extract custom metadata from x-object-meta-* headers + let mut metadata = HashMap::new(); + for (header_name, header_value) in response.headers().iter() { + if let Some(meta_key) = header_name.as_str().strip_prefix("x-object-meta-") { + if let Ok(meta_value) = header_value.to_str() { + metadata.insert(meta_key.to_string(), meta_value.to_string()); + } + } + } + Ok(ObjectMetadata { key: key.to_string(), size, content_type, etag, last_modified, - metadata: HashMap::new(), + metadata, }) } StatusCode::NOT_FOUND => Err(Error::NotFound(format!("{}/{}", bucket, key))), @@ -276,6 +341,19 @@ impl ObjectStoreClient { } } + pub async fn get_object_info(&self, bucket: &str, key: &str) -> Result { + let url = format!("{}/buckets/{}/object-info/{}", self.base_url, bucket, key); + let response = self.client.get(&url).send().await?; + + match response.status() { + StatusCode::OK => Ok(response.json().await?), + StatusCode::NOT_FOUND => Err(Error::NotFound(format!("{}/{}", bucket, key))), + _ => Err(Error::ServerError( + response.text().await.unwrap_or_default(), + )), + } + } + pub async fn delete_object(&self, bucket: &str, key: &str) -> Result<()> { let url = format!("{}/buckets/{}/objects/{}", self.base_url, bucket, key); let response = self.client.delete(&url).send().await?; @@ -369,13 +447,14 @@ mod tests { .mock("POST", "/buckets") .with_status(200) .with_header("content-type", "application/json") - .with_body(r#"{"name":"test-bucket","created_at":"2024-01-01T00:00:00Z"}"#) + .with_body(r#"{"id":"bucket-123","name":"test-bucket","created_at":"2024-01-01T00:00:00Z"}"#) .create_async() .await; let client = ObjectStoreClient::new(&server.url()); let bucket = client.create_bucket("test-bucket").await.unwrap(); + assert_eq!(bucket.id, "bucket-123"); assert_eq!(bucket.name, "test-bucket"); assert_eq!(bucket.created_at, "2024-01-01T00:00:00Z"); } @@ -404,7 +483,7 @@ mod tests { .mock("GET", "/buckets") .with_status(200) .with_header("content-type", "application/json") - .with_body(r#"{"buckets":[{"name":"bucket1","created_at":"2024-01-01T00:00:00Z"},{"name":"bucket2","created_at":"2024-01-02T00:00:00Z"}]}"#) + .with_body(r#"{"buckets":[{"id":"bucket-1","name":"bucket1","created_at":"2024-01-01T00:00:00Z"},{"id":"bucket-2","name":"bucket2","created_at":"2024-01-02T00:00:00Z"}]}"#) .create_async() .await; @@ -412,7 +491,9 @@ mod tests { let buckets = client.list_buckets().await.unwrap(); assert_eq!(buckets.len(), 2); + assert_eq!(buckets[0].id, "bucket-1"); assert_eq!(buckets[0].name, "bucket1"); + assert_eq!(buckets[1].id, "bucket-2"); assert_eq!(buckets[1].name, "bucket2"); } diff --git a/clients/typescript/package.json b/clients/typescript/package.json index c453ac1..25f94b1 100644 --- a/clients/typescript/package.json +++ b/clients/typescript/package.json @@ -1,6 +1,6 @@ { "name": "object-storage-client", - "version": "0.2.2", + "version": "0.2.3", "description": "TypeScript client for Object Storage Service", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/clients/typescript/src/index.ts b/clients/typescript/src/index.ts index 23c1a09..8b30e90 100644 --- a/clients/typescript/src/index.ts +++ b/clients/typescript/src/index.ts @@ -11,6 +11,7 @@ export class ObjectStorageError extends Error { } export interface Bucket { + id: string; name: string; created_at: string; } @@ -66,6 +67,14 @@ export class ObjectStorageClient { throw error; } + async ping(): Promise { + try { + await this.client.get('/ping'); + } catch (error) { + this.handleError(error as AxiosError); + } + } + async createBucket(name: string): Promise { try { const response = await this.client.post('/buckets', { @@ -77,6 +86,26 @@ export class ObjectStorageClient { } } + async upsertBucket(name: string): Promise { + try { + const response = await this.client.put('/buckets', { + name, + } as CreateBucketRequest); + return response.data; + } catch (error) { + this.handleError(error as AxiosError); + } + } + + async getBucket(id: string): Promise { + try { + const response = await this.client.get(`/buckets/${id}`); + return response.data; + } catch (error) { + this.handleError(error as AxiosError); + } + } + async listBuckets(): Promise { try { const response = await this.client.get('/buckets'); @@ -137,6 +166,15 @@ export class ObjectStorageClient { const etag = response.headers['etag'] || ''; const lastModified = response.headers['last-modified'] || ''; + // Extract custom metadata from x-object-meta-* headers + const metadata: Record = {}; + for (const [headerName, headerValue] of Object.entries(response.headers)) { + if (headerName.startsWith('x-object-meta-')) { + const metaKey = headerName.substring('x-object-meta-'.length); + metadata[metaKey] = String(headerValue); + } + } + return { metadata: { key, @@ -144,7 +182,7 @@ export class ObjectStorageClient { content_type: contentType, etag, last_modified: lastModified, - metadata: {}, + metadata, }, data: Buffer.from(response.data), }; @@ -162,19 +200,39 @@ export class ObjectStorageClient { const etag = response.headers['etag'] || ''; const lastModified = response.headers['last-modified'] || ''; + // Extract custom metadata from x-object-meta-* headers + const metadata: Record = {}; + for (const [headerName, headerValue] of Object.entries(response.headers)) { + if (headerName.startsWith('x-object-meta-')) { + const metaKey = headerName.substring('x-object-meta-'.length); + metadata[metaKey] = String(headerValue); + } + } + return { key, size, content_type: contentType, etag, last_modified: lastModified, - metadata: {}, + metadata, }; } catch (error) { this.handleError(error as AxiosError); } } + async getObjectInfo(bucket: string, key: string): Promise { + try { + const response = await this.client.get( + `/buckets/${bucket}/object-info/${key}` + ); + return response.data; + } catch (error) { + this.handleError(error as AxiosError); + } + } + async deleteObject(bucket: string, key: string): Promise { try { await this.client.delete(`/buckets/${bucket}/objects/${key}`);