From 6fda4c9d3fc5c53d2fd7c0f9ab546cf75cc30ef8 Mon Sep 17 00:00:00 2001 From: ZzIsGod1019 <1498852723@qq.com> Date: Sat, 31 Aug 2024 13:55:10 +0800 Subject: [PATCH] spi-obj: fix obs copy failed (#831) * os: fix obs copy failed * iam:fix bug (icon in set cate modify empty failed) * spi-obs: update * spi-obj: complete comments * code format --------- Co-authored-by: ZzIsGod1019 <1498852723@11.com> --- backend/basic/src/spi/macros.rs | 2 +- backend/basic/src/spi/spi_funs.rs | 13 ++- .../spi-graph/src/serv/graph_basic_serv.rs | 2 +- .../src/api/ci/object_ci_obj_api.rs | 98 ++++++++++++++++++- .../spi/spi-object/src/object_constants.rs | 2 + .../spi-object/src/serv/object_obj_serv.rs | 29 +++--- .../src/serv/obs/object_obs_initializer.rs | 70 ++++++++++++- .../src/serv/obs/object_obs_obj_serv.rs | 25 ++++- .../src/serv/s3/object_s3_initializer.rs | 4 + .../spi-stats/src/serv/stats_metric_serv.rs | 2 +- .../supports/iam/src/basic/dto/iam_set_dto.rs | 1 - 11 files changed, 220 insertions(+), 28 deletions(-) diff --git a/backend/basic/src/spi/macros.rs b/backend/basic/src/spi/macros.rs index 479d72b46..67f714618 100644 --- a/backend/basic/src/spi/macros.rs +++ b/backend/basic/src/spi/macros.rs @@ -69,7 +69,7 @@ macro_rules! spi_dispatch_service { $( $(#[$attr])* pub async fn $service($($arg: $type,)* funs: &tardis::TardisFunsInst, ctx: &tardis::basic::dto::TardisContext) -> $ret { - let arc_inst = funs.init(ctx, $mgr, $init).await?; + let arc_inst = funs.init(None, ctx, $mgr, $init).await?; let inst = arc_inst.as_ref(); $crate::spi_dispatch_service!(@dispatch $service, funs, ctx, inst, @dispatch: $dispatch, @args: {$($arg),*}) } diff --git a/backend/basic/src/spi/spi_funs.rs b/backend/basic/src/spi/spi_funs.rs index b77cd2e26..aed4b2e67 100644 --- a/backend/basic/src/spi/spi_funs.rs +++ b/backend/basic/src/spi/spi_funs.rs @@ -49,7 +49,7 @@ fn get_spi_bs_caches() -> &'static RwLock>> { #[async_trait] pub trait SpiBsInstExtractor { - async fn init<'a, F, T>(&self, ctx: &'a TardisContext, mgr: bool, init_funs: F) -> TardisResult> + async fn init<'a, F, T>(&self, custom_cache_key: Option, ctx: &'a TardisContext, mgr: bool, init_funs: F) -> TardisResult> where F: Fn(SpiBsCertResp, &'a TardisContext, bool) -> T + Send + Sync, T: Future> + Send; @@ -72,6 +72,7 @@ impl SpiBsInstExtractor for TardisFunsInst { /// /// # Arguments /// + /// * `custom_cache_key` - Customize the cache key, if it is none, the default cache key will be used. /// * `ctx` - Request Context /// * `mgr` - Whether it is a managed request /// * `init_fun` - The initialization function called when the backend service instance is not initialized @@ -79,12 +80,16 @@ impl SpiBsInstExtractor for TardisFunsInst { /// # Return /// /// the backend service instance kind - async fn init<'a, F, T>(&self, ctx: &'a TardisContext, mgr: bool, init_fun: F) -> TardisResult> + async fn init<'a, F, T>(&self, custom_cache_key: Option, ctx: &'a TardisContext, mgr: bool, init_fun: F) -> TardisResult> where F: Fn(SpiBsCertResp, &'a TardisContext, bool) -> T + Send + Sync, T: Future> + Send, { - let cache_key = format!("{}-{}", self.module_code(), ctx.ak); + let cache_key = if let Some(custom_cache_key) = custom_cache_key { + format!("{}-{}-{}", self.module_code(), ctx.ak, custom_cache_key) + } else { + format!("{}-{}", self.module_code(), ctx.ak) + }; { let read = get_spi_bs_caches().read().await; if let Some(inst) = read.get(&cache_key).cloned() { @@ -136,7 +141,7 @@ impl SpiBsInstExtractor for TardisFunsInst { F: Fn(SpiBsCertResp, &'a TardisContext, bool) -> T + Send + Sync, T: Future> + Send, { - self.init(ctx, mgr, init_fun).await?; + self.init(None, ctx, mgr, init_fun).await?; self.bs(ctx).await } diff --git a/backend/spi/spi-graph/src/serv/graph_basic_serv.rs b/backend/spi/spi-graph/src/serv/graph_basic_serv.rs index 00f6d8378..bb1188d31 100644 --- a/backend/spi/spi-graph/src/serv/graph_basic_serv.rs +++ b/backend/spi/spi-graph/src/serv/graph_basic_serv.rs @@ -42,7 +42,7 @@ pub async fn delete_rels( "400-spi-graph-key-require", )); } - let inst = funs.init(ctx, true, graph_initializer::init_fun).await?; + let inst = funs.init(None, ctx, true, graph_initializer::init_fun).await?; match inst.kind_code() { #[cfg(feature = "spi-pg")] spi_constants::SPI_PG_KIND_CODE => pg::graph_pg_basic_serv::delete_rels(tag, from_key, to_key, from_version, to_version, funs, ctx, &inst).await, diff --git a/backend/spi/spi-object/src/api/ci/object_ci_obj_api.rs b/backend/spi/spi-object/src/api/ci/object_ci_obj_api.rs index ee683a5be..d97e3bfd5 100644 --- a/backend/spi/spi-object/src/api/ci/object_ci_obj_api.rs +++ b/backend/spi/spi-object/src/api/ci/object_ci_obj_api.rs @@ -16,16 +16,42 @@ use crate::serv::object_obj_serv; pub struct ObjectCiObjApi; /// Interface Console Object API +/// +/// When the built-in service is initialized, 4 buckets are created by default: pub,pri,spe,tamp. +/// pub bucket, we recommend setting it to public read and private write permissions, when is_private passes false, the bucket will be operated. +/// pri bucket, it is recommended to set it to private read-private write, use temporary address to manipulate the object to ensure data security. When is_private passes true, operate the bucket. +/// spe bucket, recommended for large files. When is_special passes true, manipulate this bucket. +/// The tamp bucket, recommended for temporary files, when obj_exp is passed true. +/// 接口控制台对象服务API +/// 内置服务初始化时，默认创建4个桶：pub,pri,spe,tamp +/// pub桶，建议设置为公共读私有写权限，当is_private传false时，操作该桶。 +/// pri桶，建议设置为私有读私有写，使用临时地址操作对象，保证数据安全。当is_private传true时，操作该桶。 +/// spe桶，建议操作大文件时使用该桶。当is_special传true时，操作该桶。 +/// tamp桶，建议操作临时文件时使用该桶，当obj_exp传入时，操作该桶。 #[poem_openapi::OpenApi(prefix_path = "/ci/obj", tag = "bios_basic::ApiTag::Interface")] impl ObjectCiObjApi { /// Fetch URL for temporary authorization of file upload + /// + /// 获取用于临时授权上传文件的 URL #[oai(path = "/presign/put", method = "get")] async fn presign_put_obj_url( &self, + // 对象的路径 + // path of object object_path: Query, + // 临时上传url的生效时长 + // The length of time a temporary upload url is in effect exp_secs: Query, + // 是否私有 + // private or not private: Query>, + // 是否特殊 + //Special or not special: Query>, + // 是否临时，数字表示文件生效时长。 + // 使用obs时，传入数值不生效，仅表示使用tamp桶。 + // Whether or not it is temporary, the number indicates the length of time the file will be in effect. + // When using obs, passing in a value does not take effect, it only indicates the use of the tamp bucket. obj_exp: Query>, ctx: TardisContextExtractor, ) -> TardisApiResult { @@ -47,13 +73,27 @@ impl ObjectCiObjApi { } /// Fetch URL for temporary authorization of file delete + /// + /// 获取文件删除临时授权的 URL #[oai(path = "/presign/delete", method = "get")] async fn presign_delete_obj_url( &self, + // 对象的路径 + // path of object object_path: Query, + // 临时上传url的生效时长 + // The length of time a temporary upload url is in effect exp_secs: Query, + // 是否私有 + // private or not private: Query>, + // 是否特殊 + //Special or not special: Query>, + // 是否临时，数字表示文件生效时长。 + // 使用obs时，传入数值不生效，仅表示使用tamp桶。 + // Whether or not it is temporary, the number indicates the length of time the file will be in effect. + // When using obs, passing in a value does not take effect, it only indicates the use of the tamp bucket. obj_exp: Query>, ctx: TardisContextExtractor, ) -> TardisApiResult { @@ -75,13 +115,27 @@ impl ObjectCiObjApi { } /// Fetch URL for temporary authorization of file + /// + /// 获取文件临时授权的 URL #[oai(path = "/presign/view", method = "get")] async fn presign_view_obj_url( &self, + // 对象的路径 + // path of object object_path: Query, + // 临时上传url的生效时长 + // The length of time a temporary upload url is in effect exp_secs: Query, + // 是否私有 + // private or not private: Query>, + // 是否特殊 + //Special or not special: Query>, + // 是否临时，数字表示文件生效时长。 + // 使用obs时，传入数值不生效，仅表示使用tamp桶。 + // Whether or not it is temporary, the number indicates the length of time the file will be in effect. + // When using obs, passing in a value does not take effect, it only indicates the use of the tamp bucket. obj_exp: Query>, ctx: TardisContextExtractor, ) -> TardisApiResult { @@ -103,6 +157,8 @@ impl ObjectCiObjApi { } /// Batch fetch URL for temporary authorization of file + /// + /// 批量获取文件临时授权的 URL #[oai(path = "/presign/batch_view", method = "post")] async fn batch_presign_view_obj_url(&self, req: Json, ctx: TardisContextExtractor) -> TardisApiResult> { let funs = crate::get_tardis_inst(); @@ -110,7 +166,9 @@ impl ObjectCiObjApi { TardisResp::ok(url) } - /// Initiate a Multipart Upload Task + /// Multipart Upload:Initiate a Multipart Upload Task + /// + /// 分片上传： 启动分片上传任务 #[oai(path = "/multi_upload/initiate_multipart_upload", method = "post")] async fn initiate_multipart_upload(&self, req: Json, ctx: TardisContextExtractor) -> TardisApiResult { let funs = crate::get_tardis_inst(); @@ -118,7 +176,9 @@ impl ObjectCiObjApi { TardisResp::ok(upload_id) } - /// Create pre-signed URLs for each part + /// Multipart Upload:Create pre-signed URLs for each part + /// + /// 分片上传： 为每个部分创建预签名 URL #[oai(path = "/multi_upload/batch_build_create_presign_url", method = "post")] async fn batch_build_create_presign_url(&self, req: Json, ctx: TardisContextExtractor) -> TardisApiResult> { let funs = crate::get_tardis_inst(); @@ -126,7 +186,9 @@ impl ObjectCiObjApi { TardisResp::ok(presign_urls) } - /// Complete Multipart Upload Task + /// Multipart Upload:Complete Multipart Upload Task + /// + /// 分片上传： 完成分片上传任务 #[oai(path = "/multi_upload/complete_multipart_upload", method = "post")] async fn complete_multipart_upload(&self, req: Json, ctx: TardisContextExtractor) -> TardisApiResult { let funs = crate::get_tardis_inst(); @@ -135,6 +197,8 @@ impl ObjectCiObjApi { } /// Create A Copy Of An Object That Is Already Stored + /// + /// 创建已存储对象的副本 #[oai(path = "/object/copy", method = "post")] async fn object_copy(&self, req: Json, ctx: TardisContextExtractor) -> TardisApiResult { let funs = crate::get_tardis_inst(); @@ -143,12 +207,24 @@ impl ObjectCiObjApi { } /// Deleting A Single Object + /// + /// 删除单个对象 #[oai(path = "/object", method = "delete")] async fn object_delete( &self, + // 对象的路径 + // path of object object_path: Query, + // 是否私有 + // private or not private: Query>, + // 是否特殊 + //Special or not special: Query>, + // 是否临时，数字表示文件生效时长。 + // 使用obs时，传入数值不生效，仅表示使用tamp桶。 + // Whether or not it is temporary, the number indicates the length of time the file will be in effect. + // When using obs, passing in a value does not take effect, it only indicates the use of the tamp bucket. obj_exp: Query>, ctx: TardisContextExtractor, ) -> TardisApiResult { @@ -157,7 +233,9 @@ impl ObjectCiObjApi { TardisResp::ok(Void) } - /// Deleting Objects + /// Batch deleting Objects + /// + /// 批量删除对象 #[oai(path = "/object/batch_delete", method = "delete")] async fn batch_object_delete(&self, req: Json, ctx: TardisContextExtractor) -> TardisApiResult> { let funs = crate::get_tardis_inst(); @@ -165,12 +243,24 @@ impl ObjectCiObjApi { } /// Check object is exist + /// + /// 检查对象是否存在 #[oai(path = "/object/exist", method = "get")] async fn object_exist( &self, + // 对象的路径 + // path of object object_path: Query, + // 是否私有 + // private or not private: Query>, + // 是否特殊 + //Special or not special: Query>, + // 是否临时，数字表示文件生效时长。 + // 使用obs时，传入数值不生效，仅表示使用tamp桶。 + // Whether or not it is temporary, the number indicates the length of time the file will be in effect. + // When using obs, passing in a value does not take effect, it only indicates the use of the tamp bucket. obj_exp: Query>, ctx: TardisContextExtractor, ) -> TardisApiResult { diff --git a/backend/spi/spi-object/src/object_constants.rs b/backend/spi/spi-object/src/object_constants.rs index f34602dcb..6256ef400 100644 --- a/backend/spi/spi-object/src/object_constants.rs +++ b/backend/spi/spi-object/src/object_constants.rs @@ -1,3 +1,5 @@ pub const DOMAIN_CODE: &str = "spi-object"; pub const SPI_S3_KIND_CODE: &str = "spi-bs-s3"; pub const SPI_OBS_KIND_CODE: &str = "spi-bs-obs"; + +pub const USE_REGION_ENDPOINT: &str = "use_region_endpoint"; diff --git a/backend/spi/spi-object/src/serv/object_obj_serv.rs b/backend/spi/spi-object/src/serv/object_obj_serv.rs index 9dd8dfec9..723fd6e8e 100644 --- a/backend/spi/spi-object/src/serv/object_obj_serv.rs +++ b/backend/spi/spi-object/src/serv/object_obj_serv.rs @@ -1,11 +1,14 @@ use std::collections::HashMap; +use std::sync::Arc; use bios_basic::spi::spi_funs::SpiBsInstExtractor; use tardis::basic::dto::TardisContext; use tardis::basic::result::TardisResult; +use tardis::tokio::sync::RwLock; use tardis::TardisFunsInst; use crate::dto::object_dto::{ObjectBatchBuildCreatePresignUrlReq, ObjectCompleteMultipartUploadReq, ObjectInitiateMultipartUploadReq, ObjectObjPresignKind}; +use crate::object_constants::USE_REGION_ENDPOINT; use crate::{object_constants, object_initializer}; use super::s3::S3 as _; @@ -23,7 +26,7 @@ pub async fn presign_obj_url( funs: &TardisFunsInst, ctx: &TardisContext, ) -> TardisResult { - let inst = funs.init(ctx, true, object_initializer::init_fun).await?; + let inst = funs.init(None, ctx, true, object_initializer::init_fun).await?; match inst.kind_code() { #[cfg(feature = "spi-s3")] object_constants::SPI_S3_KIND_CODE => { @@ -46,7 +49,7 @@ pub async fn batch_get_presign_obj_url( funs: &TardisFunsInst, ctx: &TardisContext, ) -> TardisResult> { - let inst = funs.init(ctx, true, object_initializer::init_fun).await?; + let inst = funs.init(None, ctx, true, object_initializer::init_fun).await?; match inst.kind_code() { #[cfg(feature = "spi-s3")] object_constants::SPI_S3_KIND_CODE => { @@ -61,7 +64,7 @@ pub async fn batch_get_presign_obj_url( } pub async fn initiate_multipart_upload(req: ObjectInitiateMultipartUploadReq, funs: &TardisFunsInst, ctx: &TardisContext) -> TardisResult { - let inst = funs.init(ctx, true, object_initializer::init_fun).await?; + let inst = funs.init(None, ctx, true, object_initializer::init_fun).await?; match inst.kind_code() { #[cfg(feature = "spi-s3")] object_constants::SPI_S3_KIND_CODE => { @@ -76,7 +79,7 @@ pub async fn initiate_multipart_upload(req: ObjectInitiateMultipartUploadReq, fu } pub async fn batch_build_create_presign_url(req: ObjectBatchBuildCreatePresignUrlReq, funs: &TardisFunsInst, ctx: &TardisContext) -> TardisResult> { - let inst = funs.init(ctx, true, object_initializer::init_fun).await?; + let inst = funs.init(None, ctx, true, object_initializer::init_fun).await?; match inst.kind_code() { #[cfg(feature = "spi-s3")] object_constants::SPI_S3_KIND_CODE => { @@ -113,7 +116,7 @@ pub async fn batch_build_create_presign_url(req: ObjectBatchBuildCreatePresignUr } pub async fn complete_multipart_upload(req: ObjectCompleteMultipartUploadReq, funs: &TardisFunsInst, ctx: &TardisContext) -> TardisResult<()> { - let inst = funs.init(ctx, true, object_initializer::init_fun).await?; + let inst = funs.init(None, ctx, true, object_initializer::init_fun).await?; match inst.kind_code() { #[cfg(feature = "spi-s3")] object_constants::SPI_S3_KIND_CODE => { @@ -135,7 +138,7 @@ pub async fn object_delete( funs: &TardisFunsInst, ctx: &TardisContext, ) -> TardisResult<()> { - let inst = funs.init(ctx, true, object_initializer::init_fun).await?; + let inst = funs.init(None, ctx, true, object_initializer::init_fun).await?; match inst.kind_code() { #[cfg(feature = "spi-s3")] object_constants::SPI_S3_KIND_CODE => s3::object_s3_obj_serv::S3Service::object_delete(&object_path, private, special, obj_exp, funs, ctx, &inst).await, @@ -153,7 +156,7 @@ pub async fn batch_object_delete( funs: &TardisFunsInst, ctx: &TardisContext, ) -> TardisResult> { - let inst = funs.init(ctx, true, object_initializer::init_fun).await?; + let inst = funs.init(None, ctx, true, object_initializer::init_fun).await?; match inst.kind_code() { #[cfg(feature = "spi-s3")] object_constants::SPI_S3_KIND_CODE => s3::object_s3_obj_serv::S3Service::batch_object_delete(object_paths, private, special, obj_exp, funs, ctx, &inst).await, @@ -164,12 +167,16 @@ pub async fn batch_object_delete( } pub async fn object_copy(from: String, to: String, private: Option, special: Option, funs: &TardisFunsInst, ctx: &TardisContext) -> TardisResult<()> { - let inst = funs.init(ctx, true, object_initializer::init_fun).await?; + let mock_ctx = TardisContext { + ext: Arc::new(RwLock::new(HashMap::from([(USE_REGION_ENDPOINT.to_string(), "true".to_string())]))), + ..ctx.clone() + }; + let inst = funs.init(Some(USE_REGION_ENDPOINT.to_string()), &mock_ctx, true, object_initializer::init_fun).await?; match inst.kind_code() { #[cfg(feature = "spi-s3")] - object_constants::SPI_S3_KIND_CODE => s3::object_s3_obj_serv::S3Service::object_copy(&from, &to, private, special, funs, ctx, &inst).await, + object_constants::SPI_S3_KIND_CODE => s3::object_s3_obj_serv::S3Service::object_copy(&from, &to, private, special, funs, &mock_ctx, &inst).await, #[cfg(feature = "spi-s3")] - object_constants::SPI_OBS_KIND_CODE => obs::object_obs_obj_serv::OBSService::object_copy(&from, &to, private, special, funs, ctx, &inst).await, + object_constants::SPI_OBS_KIND_CODE => obs::object_obs_obj_serv::OBSService::object_copy(&from, &to, private, special, funs, &mock_ctx, &inst).await, kind_code => Err(funs.bs_not_implemented(kind_code)), } } @@ -182,7 +189,7 @@ pub async fn object_exist( funs: &TardisFunsInst, ctx: &TardisContext, ) -> TardisResult { - let inst = funs.init(ctx, true, object_initializer::init_fun).await?; + let inst = funs.init(None, ctx, true, object_initializer::init_fun).await?; match inst.kind_code() { #[cfg(feature = "spi-s3")] object_constants::SPI_S3_KIND_CODE => s3::object_s3_obj_serv::S3Service::object_exist(&object_paths, private, special, obj_exp, funs, ctx, &inst).await, diff --git a/backend/spi/spi-object/src/serv/obs/object_obs_initializer.rs b/backend/spi/spi-object/src/serv/obs/object_obs_initializer.rs index 9fd232549..dcdfb692a 100644 --- a/backend/spi/spi-object/src/serv/obs/object_obs_initializer.rs +++ b/backend/spi/spi-object/src/serv/obs/object_obs_initializer.rs @@ -1,8 +1,70 @@ -use bios_basic::spi::{dto::spi_bs_dto::SpiBsCertResp, spi_funs::SpiBsInst}; -use tardis::basic::{dto::TardisContext, result::TardisResult}; +use std::collections::HashMap; -use crate::serv::s3; +use bios_basic::spi::{dto::spi_bs_dto::SpiBsCertResp, spi_funs::SpiBsInst, spi_initializer}; +use tardis::{ + basic::{dto::TardisContext, error::TardisError, result::TardisResult}, + config::config_dto::OSModuleConfig, + os::os_client::TardisOSClient, + TardisFuns, +}; +use crate::object_constants::USE_REGION_ENDPOINT; + +use tardis::serde_json::Value as JsonValue; + +/// obs存储不支持在 Regional endpoint 下建桶。 +/// 所以为了保持内置服务的逻辑统一，我们使用桶域名作为 s3 的 endpoint 创建连接。 +/// 此时不再需要初始化建桶的操作，我们在桶域名下指定桶，obs会将指定的桶名当作一级目录。 +/// 存在例外的情况，目前发现 copy 接口只能在 Regional endpoint 下正常调用，在桶域名下会报错找不到对象。 +/// 所以我们需要两个连接。 +/// 一个是桶域名连接，此时指定的 bucket 会被当作桶内的一级目录。 +/// 一个是区域域名连接，此时将桶域名指定的桶当作default bucket 初始化时传入。实际使用时的bucket_name当作路径拼接在object_path前端。 +/// 若ctx中存在USE_REGION_ENDPOINT标识，则使用区域域名连接。 +/// 举例说明：假设obs服务的 Regional endpoint: obs.ap-southeast-1.myhuaweicloud.com 。业务使用的桶为 bios-test 。桶中存在文件路径为 pri/test.txt 。 +/// 1、使用桶域名访问该文件，endpoint为bios-test.obs.ap-southeast-1.myhuaweicloud.com。bucket_name传pri,object_path为text.txt。 +/// 2、使用区域域名访问该文件，endpoint为obs.ap-southeast-1.myhuaweicloud.com。bucket_name传bios-test,object_path为 pri/test.txt。 +/// +/// The obs storage does not support buckets under the Regional endpoint. +/// So in order to keep the logic of the built-in services uniform, we use the bucket domain name as the endpoint for s3 to create the connection. +/// At this point there is no need to initialize the bucket building operation, we specify the bucket under the bucket domain name and obs will treat the specified bucket name as a first level directory. +/// There is an exception to this, as it has been found that the copy interface can only be invoked under the Regional endpoint, but under the bucket domain, it will report that the object was not found. +/// So we need two connections. +/// A bucket connection, where the specified bucket is treated as a first-level directory within the bucket. +/// One is a zone domain connection, where the bucket specified by the bucket domain is passed in as the default bucket when it is initialized. The actual bucket_name is used as a path to be spliced in front of object_path. +/// If the USE_REGION_ENDPOINT flag exists in the ctx, then the regional domain name connection is used. +/// For example, suppose the regional endpoint for the obs service: obs.ap-southeast-1.myhuaweicloud.com. The bucket used by the service is bios-test. The path to the file in the bucket is pri/test.txt. +/// 1. Access the file using the bucket domain name with endpoint bios-test.obs.ap-southeast-1.myhuaweicloud.com. bucket_name passes pri,object_path is text.txt. +/// 2. Access the file using the zone domain name, endpoint is obs.ap-southeast-1.myhuaweicloud.com. bucket_name passes bios-test,object_path is pri/test.txt. pub async fn init(bs_cert: &SpiBsCertResp, ctx: &TardisContext, mgr: bool) -> TardisResult { - s3::object_s3_initializer::init(bs_cert, ctx, mgr).await + let ext = TardisFuns::json.str_to_json(&bs_cert.ext)?; + let region = ext + .get("region") + .and_then(JsonValue::as_str) + .ok_or_else(|| TardisError::bad_request("Tardis context ext should have a `region` field with type string", "400-spi-invalid-tardis-ctx"))?; + if ctx.ext.read().await.contains_key(USE_REGION_ENDPOINT) { + let default_bucket = ext + .get("default_bucket") + .and_then(JsonValue::as_str) + .ok_or_else(|| TardisError::bad_request("Tardis context ext should have a `default_bucket` field with type string", "400-spi-invalid-tardis-ctx"))?; + let conn_uri = ext + .get("region_endpoint") + .and_then(JsonValue::as_str) + .ok_or_else(|| TardisError::bad_request("Tardis context ext should have a `region_endpoint` field with type string", "400-spi-invalid-tardis-ctx"))?; + init_obs_spi_bs(&conn_uri, &bs_cert.ak, &bs_cert.sk, region, &default_bucket, bs_cert.private, ctx, mgr).await + } else { + // When using a bucket domain, default_bucket is set to empty + let default_bucket = ""; + init_obs_spi_bs(&bs_cert.conn_uri, &bs_cert.ak, &bs_cert.sk, region, default_bucket, bs_cert.private, ctx, mgr).await + } +} + +async fn init_obs_spi_bs(conn_uri: &str, ak: &str, sk: &str, region: &str, default_bucket: &str, private: bool, ctx: &TardisContext, _: bool) -> TardisResult { + let tardis_os_config = OSModuleConfig::builder().kind("s3").endpoint(conn_uri).ak(ak).sk(sk).region(region).default_bucket(default_bucket).build(); + let client = TardisOSClient::init(&tardis_os_config)?; + let mut ext = HashMap::new(); + if !private { + let bucket_name_prefix = spi_initializer::common::get_isolation_flag_from_context(ctx); + spi_initializer::common::set_isolation_flag_to_ext(&bucket_name_prefix, &mut ext); + }; + Ok(SpiBsInst { client: Box::new(client), ext }) } diff --git a/backend/spi/spi-object/src/serv/obs/object_obs_obj_serv.rs b/backend/spi/spi-object/src/serv/obs/object_obs_obj_serv.rs index f32e7ae02..206d980b8 100644 --- a/backend/spi/spi-object/src/serv/obs/object_obs_obj_serv.rs +++ b/backend/spi/spi-object/src/serv/obs/object_obs_obj_serv.rs @@ -1,11 +1,34 @@ -use tardis::{basic::result::TardisResult, os::os_client::TardisOSClient}; +use bios_basic::spi::spi_funs::SpiBsInst; +use tardis::{ + basic::{dto::TardisContext, result::TardisResult}, + os::os_client::TardisOSClient, + TardisFunsInst, +}; use crate::serv::s3::S3; /// OBS need manually configure lifecycle rules +/// Most interfaces of the obs service use bucket domains to share the code logic of s3. +/// However, the copy interface requires additional processing, which only supports specifying absolute file paths, so use a region domain and specify the bucket in the bucket domain as the default bucket, and pass in the bucket as a directory prefix in the code. +/// obs服务大多数接口使用桶域名共用s3的代码逻辑。 +/// 但copy接口需额外处理，它只支持指定绝对文件路径，所以使用区域域名并将桶域名中的桶指定为默认桶，同时将代码中的桶当作目录前缀传入。 pub(crate) struct OBSService; impl S3 for OBSService { async fn rebuild_path(_bucket_name: Option<&str>, origin_path: &str, _obj_exp: Option, _client: &TardisOSClient) -> TardisResult { Ok(origin_path.to_string()) } + + // + async fn object_copy(from: &str, to: &str, private: Option, special: Option, _funs: &TardisFunsInst, _ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult<()> { + let bs_inst = inst.inst::(); + let client = bs_inst.0; + let bucket_name = Self::get_bucket_name(private, special, None, inst).unwrap_or_default(); + client + .object_copy( + &format!("{}/{}", &bucket_name, from.strip_prefix('/').unwrap_or(from),), + &format!("{}/{}", &bucket_name, to.strip_prefix('/').unwrap_or(to),), + None, + ) + .await + } } diff --git a/backend/spi/spi-object/src/serv/s3/object_s3_initializer.rs b/backend/spi/spi-object/src/serv/s3/object_s3_initializer.rs index 816c58b0a..e74b9842e 100644 --- a/backend/spi/spi-object/src/serv/s3/object_s3_initializer.rs +++ b/backend/spi/spi-object/src/serv/s3/object_s3_initializer.rs @@ -10,6 +10,10 @@ use tardis::{ use tardis::serde_json::Value as JsonValue; +/// s3 引擎初始化 +/// 使用 Regional endpoint 建立连接，默认桶允许置空。 +/// s3 engine initialization +/// Use regional endpoint to establish connection, default bucket is allowed to be empty. pub async fn init(bs_cert: &SpiBsCertResp, ctx: &TardisContext, _: bool) -> TardisResult { let ext = TardisFuns::json.str_to_json(&bs_cert.ext)?; let region = ext diff --git a/backend/spi/spi-stats/src/serv/stats_metric_serv.rs b/backend/spi/spi-stats/src/serv/stats_metric_serv.rs index 48d18e811..07f436a3b 100644 --- a/backend/spi/spi-stats/src/serv/stats_metric_serv.rs +++ b/backend/spi/spi-stats/src/serv/stats_metric_serv.rs @@ -10,7 +10,7 @@ use crate::stats_initializer; use super::pg; pub async fn query_metrics(query_req: &StatsQueryMetricsReq, funs: &TardisFunsInst, ctx: &TardisContext) -> TardisResult { - let inst = funs.init(ctx, true, stats_initializer::init_fun).await?; + let inst = funs.init(None, ctx, true, stats_initializer::init_fun).await?; match inst.kind_code() { #[cfg(feature = "spi-pg")] spi_constants::SPI_PG_KIND_CODE => pg::stats_pg_metric_serv::query_metrics(query_req, funs, ctx, &inst).await, diff --git a/backend/supports/iam/src/basic/dto/iam_set_dto.rs b/backend/supports/iam/src/basic/dto/iam_set_dto.rs index c14e13f56..919bcc28a 100644 --- a/backend/supports/iam/src/basic/dto/iam_set_dto.rs +++ b/backend/supports/iam/src/basic/dto/iam_set_dto.rs @@ -28,7 +28,6 @@ pub struct IamSetCateModifyReq { pub scope_level: Option, #[oai(validator(min_length = "2", max_length = "255"))] pub bus_code: Option, - #[oai(validator(min_length = "2", max_length = "1000"))] pub icon: Option, pub sort: Option, #[oai(validator(min_length = "2", max_length = "1000"))]