From 64222a1b1efde8f06f664bb0dbc5c1df32a3145f Mon Sep 17 00:00:00 2001 From: RWDai <27391645+RWDai@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:38:12 +0800 Subject: [PATCH 1/6] feat: bring back log v1 api (#858) * feat: bring back log v1 api * update api path --- .../spacegate-plugins/src/plugin/audit_log.rs | 6 +- .../flow/src/serv/clients/flow_log_client.rs | 6 +- .../schedule_job_serv_v2/event/spi_log.rs | 6 +- .../spi/spi-log/src/api/ci/log_ci_item_api.rs | 25 ++++++- backend/spi/spi-log/src/dto/log_item_dto.rs | 51 +++++++++++-- backend/spi/spi-log/src/event.rs | 6 +- backend/spi/spi-log/src/serv.rs | 1 - backend/spi/spi-log/src/serv/log_item_serv.rs | 4 +- .../spi-log/src/serv/pg/log_pg_item_serv.rs | 26 ++++--- .../spi-log/src/serv/pgv2/log_pg_item_serv.rs | 13 +++- backend/spi/spi-stats/src/event.rs | 4 +- .../src/basic/serv/clients/iam_log_client.rs | 6 +- .../sdks/invoke/src/clients/spi_log_client.rs | 71 +++++++++++++++++-- 13 files changed, 183 insertions(+), 42 deletions(-) diff --git a/backend/gateways/spacegate-plugins/src/plugin/audit_log.rs b/backend/gateways/spacegate-plugins/src/plugin/audit_log.rs index 8f3e167b0..54478b507 100644 --- a/backend/gateways/spacegate-plugins/src/plugin/audit_log.rs +++ b/backend/gateways/spacegate-plugins/src/plugin/audit_log.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use std::str::FromStr; use std::time::Instant; -use bios_sdk_invoke::clients::spi_log_client::{self, LogItemAddReq}; +use bios_sdk_invoke::clients::spi_log_client::{self, LogItemAddV2Req}; use bios_sdk_invoke::invoke_config::InvokeConfig; use bios_sdk_invoke::invoke_enumeration::InvokeModuleKind; use bios_sdk_invoke::invoke_initializer; @@ -208,8 +208,8 @@ impl AuditLogPlugin { let tag = self.tag.clone(); if !self.log_url.is_empty() && !self.spi_app_id.is_empty() { tokio::task::spawn(async move { - match spi_log_client::SpiLogClient::add( - LogItemAddReq { + match spi_log_client::SpiLogClient::addv2( + LogItemAddV2Req { tag, content: TardisFuns::json.obj_to_json(&content).unwrap_or_default(), kind: None, diff --git a/backend/middlewares/flow/src/serv/clients/flow_log_client.rs b/backend/middlewares/flow/src/serv/clients/flow_log_client.rs index 89cec43a0..2072d9667 100644 --- a/backend/middlewares/flow/src/serv/clients/flow_log_client.rs +++ b/backend/middlewares/flow/src/serv/clients/flow_log_client.rs @@ -1,6 +1,6 @@ use bios_sdk_invoke::clients::{ iam_client::IamClient, - spi_log_client::{LogItemAddReq, SpiLogClient}, + spi_log_client::{LogItemAddV2Req, SpiLogClient}, }; use serde::Serialize; @@ -96,7 +96,7 @@ impl FlowLogClient { .await? .owner_name; - let req = LogItemAddReq { + let req = LogItemAddV2Req { tag: tag.to_string(), content: TardisFuns::json.obj_to_json(&content).expect("req_msg not a valid json value"), kind, @@ -112,7 +112,7 @@ impl FlowLogClient { owner_name, push: push, }; - SpiLogClient::add(req, funs, ctx).await?; + SpiLogClient::addv2(req, funs, ctx).await?; Ok(()) } } diff --git a/backend/middlewares/schedule/src/serv/schedule_job_serv_v2/event/spi_log.rs b/backend/middlewares/schedule/src/serv/schedule_job_serv_v2/event/spi_log.rs index e89883a6d..06b004938 100644 --- a/backend/middlewares/schedule/src/serv/schedule_job_serv_v2/event/spi_log.rs +++ b/backend/middlewares/schedule/src/serv/schedule_job_serv_v2/event/spi_log.rs @@ -1,6 +1,6 @@ use std::{sync::Arc, task::ready, time::Duration}; -use bios_sdk_invoke::clients::spi_log_client::{LogItemAddReq, LogItemFindReq, SpiLogClient}; +use bios_sdk_invoke::clients::spi_log_client::{LogItemAddReq, LogItemAddV2Req, LogItemFindReq, SpiLogClient}; use tardis::{ basic::dto::TardisContext, chrono::Utc, @@ -109,8 +109,8 @@ impl EventComponent for SpiLog { let ctx = self.ctx.clone(); let code = code.to_string(); let _handle = tokio::spawn(async move { - let result = SpiLogClient::add( - LogItemAddReq { + let result = SpiLogClient::addv2( + LogItemAddV2Req { tag: TASK_TAG.to_string(), content: tardis::serde_json::Value::Null, ext: Some(ext), diff --git a/backend/spi/spi-log/src/api/ci/log_ci_item_api.rs b/backend/spi/spi-log/src/api/ci/log_ci_item_api.rs index d31b90924..cb5cafe32 100644 --- a/backend/spi/spi-log/src/api/ci/log_ci_item_api.rs +++ b/backend/spi/spi-log/src/api/ci/log_ci_item_api.rs @@ -4,13 +4,16 @@ use tardis::web::poem_openapi::param::Path; use tardis::web::poem_openapi::payload::Json; use tardis::web::web_resp::{TardisApiResult, TardisPage, TardisResp, Void}; -use crate::dto::log_item_dto::{LogConfigReq, LogItemAddReq, LogItemFindReq, LogItemFindResp}; +use crate::dto::log_item_dto::{LogConfigReq, LogItemAddReq, LogItemAddV2Req, LogItemFindReq, LogItemFindResp}; use crate::serv::log_item_serv; use tardis::serde_json::Value; #[derive(Clone)] pub struct LogCiItemApi; +#[derive(Clone)] +pub struct LogCiItemApiV2; + /// Interface Console Log API #[poem_openapi::OpenApi(prefix_path = "/ci/item", tag = "bios_basic::ApiTag::Interface")] impl LogCiItemApi { @@ -29,6 +32,26 @@ impl LogCiItemApi { let resp = log_item_serv::find(&mut find_req.0, &funs, &ctx.0).await?; TardisResp::ok(resp) } +} + +/// Interface Console Log API V2 +#[poem_openapi::OpenApi(prefix_path = "/ci/v2/item", tag = "bios_basic::ApiTag::Interface")] +impl LogCiItemApiV2 { + /// Add Item + #[oai(path = "/", method = "post")] + async fn add(&self, mut add_req: Json, ctx: TardisContextExtractor) -> TardisApiResult { + let funs = crate::get_tardis_inst(); + let id = log_item_serv::addv2(&mut add_req.0, &funs, &ctx.0).await?; + TardisResp::ok(id) + } + + /// Find Items + #[oai(path = "/find", method = "put")] + async fn find(&self, mut find_req: Json, ctx: TardisContextExtractor) -> TardisApiResult> { + let funs = crate::get_tardis_inst(); + let resp = log_item_serv::findv2(&mut find_req.0, &funs, &ctx.0).await?; + TardisResp::ok(resp) + } /// Modify Item ext by key #[oai(path = "/modify/:tag/:key/ext", method = "post")] diff --git a/backend/spi/spi-log/src/dto/log_item_dto.rs b/backend/spi/spi-log/src/dto/log_item_dto.rs index 824f377b3..f7e2ffe4e 100644 --- a/backend/spi/spi-log/src/dto/log_item_dto.rs +++ b/backend/spi/spi-log/src/dto/log_item_dto.rs @@ -7,8 +7,49 @@ use tardis::{ web::poem_openapi, }; -#[derive(poem_openapi::Object, Serialize, Deserialize, Clone, Debug)] +#[derive(poem_openapi::Object, Serialize, Deserialize, Debug)] pub struct LogItemAddReq { + #[oai(validator(pattern = r"^[a-z0-9_]+$"))] + pub tag: String, + // #[oai(validator(min_length = "2"))] + pub content: String, + #[oai(validator(min_length = "2"))] + pub kind: Option, + pub ext: Option, + #[oai(validator(min_length = "2"))] + pub key: Option, + #[oai(validator(min_length = "2"))] + pub op: Option, + #[oai(validator(min_length = "2"))] + pub rel_key: Option, + #[oai(validator(min_length = "2"))] + pub id: Option, + pub ts: Option>, + #[oai(validator(min_length = "2"))] + pub owner: Option, + pub own_paths: Option, +} + +impl From for LogItemAddReq { + fn from(value: bios_sdk_invoke::clients::spi_log_client::LogItemAddReq) -> Self { + Self { + tag: value.tag, + content: value.content, + kind: value.kind.map(Into::into), + ext: value.ext, + key: value.key.map(Into::into), + op: value.op, + rel_key: value.rel_key.map(Into::into), + id: value.id, + ts: value.ts, + owner: value.owner, + own_paths: value.own_paths, + } + } +} + +#[derive(poem_openapi::Object, Serialize, Deserialize, Clone, Debug)] +pub struct LogItemAddV2Req { #[oai(validator(pattern = r"^[a-z0-9_]+$"))] pub tag: String, pub content: Value, @@ -32,8 +73,8 @@ pub struct LogItemAddReq { pub push: bool, pub msg: Option, } -impl From for LogItemAddReq { - fn from(value: bios_sdk_invoke::clients::spi_log_client::LogItemAddReq) -> Self { +impl From for LogItemAddV2Req { + fn from(value: bios_sdk_invoke::clients::spi_log_client::LogItemAddV2Req) -> Self { Self { tag: value.tag, content: value.content, @@ -129,8 +170,8 @@ pub struct StatsItemAddReq { pub own_paths: Option, } -impl From for StatsItemAddReq { - fn from(value: LogItemAddReq) -> Self { +impl From for StatsItemAddReq { + fn from(value: LogItemAddV2Req) -> Self { StatsItemAddReq { idempotent_id: value.idempotent_id, tag: value.tag, diff --git a/backend/spi/spi-log/src/event.rs b/backend/spi/spi-log/src/event.rs index ca93d1e8e..81c699144 100644 --- a/backend/spi/spi-log/src/event.rs +++ b/backend/spi/spi-log/src/event.rs @@ -4,7 +4,7 @@ use bios_sdk_invoke::clients::{ asteroid_mq::prelude::{EventAttribute, Subject}, get_topic, mq_error, ContextHandler, SPI_RPC_TOPIC, }, - spi_log_client::LogItemAddReq, + spi_log_client::LogItemAddV2Req, }; use tardis::{ basic::{dto::TardisContext, result::TardisResult}, @@ -12,9 +12,9 @@ use tardis::{ }; #[instrument] -async fn handle_add_event(req: LogItemAddReq, ctx: TardisContext) -> TardisResult<()> { +async fn handle_add_event(req: LogItemAddV2Req, ctx: TardisContext) -> TardisResult<()> { let funs = get_tardis_inst(); - serv::log_item_serv::add(&mut req.into(), &funs, &ctx).await?; + serv::log_item_serv::addv2(&mut req.into(), &funs, &ctx).await?; Ok(()) } diff --git a/backend/spi/spi-log/src/serv.rs b/backend/spi/spi-log/src/serv.rs index cd156a3d1..1991c4ef3 100644 --- a/backend/spi/spi-log/src/serv.rs +++ b/backend/spi/spi-log/src/serv.rs @@ -1,4 +1,3 @@ pub mod log_item_serv; -#[deprecated] pub mod pg; pub mod pgv2; diff --git a/backend/spi/spi-log/src/serv/log_item_serv.rs b/backend/spi/spi-log/src/serv/log_item_serv.rs index f37836904..0a40d6ae6 100644 --- a/backend/spi/spi-log/src/serv/log_item_serv.rs +++ b/backend/spi/spi-log/src/serv/log_item_serv.rs @@ -4,7 +4,7 @@ use bios_basic::spi::spi_constants; use bios_basic::spi::spi_funs::SpiBsInstExtractor; use bios_basic::spi_dispatch_service; -use crate::dto::log_item_dto::{LogConfigReq, LogItemAddReq, LogItemFindReq, LogItemFindResp}; +use crate::dto::log_item_dto::{LogConfigReq, LogItemAddReq, LogItemAddV2Req, LogItemFindReq, LogItemFindResp}; use crate::log_initializer; use tardis::web::web_resp::TardisPage; @@ -24,6 +24,8 @@ spi_dispatch_service! { @method: { add(add_req: &mut LogItemAddReq) -> TardisResult; find(find_req: &mut LogItemFindReq) -> TardisResult>; + addv2(add_req: &mut LogItemAddV2Req) -> TardisResult; + findv2(find_req: &mut LogItemFindReq) -> TardisResult>; modify_ext(tag: &str,key: &str, ext: &mut Value) -> TardisResult<()>; add_config(config: &mut LogConfigReq) -> TardisResult<()>; delete_config(config: &mut LogConfigReq) -> TardisResult<()>; diff --git a/backend/spi/spi-log/src/serv/pg/log_pg_item_serv.rs b/backend/spi/spi-log/src/serv/pg/log_pg_item_serv.rs index 4196b1cb1..bc9a5e80c 100644 --- a/backend/spi/spi-log/src/serv/pg/log_pg_item_serv.rs +++ b/backend/spi/spi-log/src/serv/pg/log_pg_item_serv.rs @@ -8,18 +8,18 @@ use tardis::{ use bios_basic::{dto::BasicQueryCondInfo, enumeration::BasicQueryOpKind, helper::db_helper, spi::spi_funs::SpiBsInst}; -use crate::dto::log_item_dto::{AdvBasicQueryCondInfo, LogConfigReq, LogItemAddReq, LogItemFindReq, LogItemFindResp}; +use crate::dto::log_item_dto::{AdvBasicQueryCondInfo, LogConfigReq, LogItemAddReq, LogItemAddV2Req, LogItemFindReq, LogItemFindResp}; use super::log_pg_initializer; pub async fn add(add_req: &mut LogItemAddReq, _funs: &TardisFunsInst, ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult { - let id = add_req.idempotent_id.clone().unwrap_or(TardisFuns::field.nanoid()); + let id = add_req.id.clone().unwrap_or(TardisFuns::field.nanoid()); let mut params = vec![ Value::from(id.clone()), Value::from(add_req.kind.as_ref().unwrap_or(&"".into()).to_string()), Value::from(add_req.key.as_ref().unwrap_or(&"".into()).to_string()), Value::from(add_req.op.as_ref().unwrap_or(&"".to_string()).as_str()), - Value::from(TardisFuns::json.json_to_string(add_req.content.clone())?.as_str()), + Value::from(add_req.content.clone()), Value::from(add_req.owner.as_ref().unwrap_or(&"".to_string()).as_str()), Value::from(add_req.own_paths.as_ref().unwrap_or(&"".to_string()).as_str()), Value::from(if let Some(ext) = &add_req.ext { @@ -509,14 +509,22 @@ ORDER BY ts DESC }) } -pub async fn modify_ext(_tag: &str, _key: &str, _ext: &mut JsonValue, _funs: &TardisFunsInst, _ctx: &TardisContext, _inst: &SpiBsInst) -> TardisResult<()> { - Ok(()) +pub async fn addv2(add_req: &mut LogItemAddV2Req, funs: &TardisFunsInst, ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult { + Err(funs.err().bad_request("item", "add", "Add v2 is not supported", "400-spi-log-add-v2-not-supported")) } -pub async fn add_config(_req: &LogConfigReq, _funs: &TardisFunsInst, _ctx: &TardisContext, _inst: &SpiBsInst) -> TardisResult<()> { - Ok(()) +pub async fn findv2(find_req: &mut LogItemFindReq, funs: &TardisFunsInst, ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult> { + Err(funs.err().bad_request("item", "find", "Find v2 is not supported", "400-spi-log-find-v2-not-supported")) } -pub async fn delete_config(_config: &mut LogConfigReq, _funs: &TardisFunsInst, _ctx: &TardisContext, _inst: &SpiBsInst) -> TardisResult<()> { - Ok(()) +pub async fn modify_ext(_tag: &str, _key: &str, _ext: &mut JsonValue, funs: &TardisFunsInst, _ctx: &TardisContext, _inst: &SpiBsInst) -> TardisResult<()> { + Err(funs.err().bad_request("item", "modify_ext", "Modify ext is not supported", "400-spi-log-modify-ext-not-supported")) +} + +pub async fn add_config(_req: &LogConfigReq, funs: &TardisFunsInst, _ctx: &TardisContext, _inst: &SpiBsInst) -> TardisResult<()> { + Err(funs.err().bad_request("item", "add_config", "Add config is not supported", "400-spi-log-add-config-not-supported")) +} + +pub async fn delete_config(_config: &mut LogConfigReq, funs: &TardisFunsInst, _ctx: &TardisContext, _inst: &SpiBsInst) -> TardisResult<()> { + Err(funs.err().bad_request("item", "delete_config", "Delete config is not supported", "400-spi-log-delete-config-not-supported")) } diff --git a/backend/spi/spi-log/src/serv/pgv2/log_pg_item_serv.rs b/backend/spi/spi-log/src/serv/pgv2/log_pg_item_serv.rs index be8904c17..fbc94f1fd 100644 --- a/backend/spi/spi-log/src/serv/pgv2/log_pg_item_serv.rs +++ b/backend/spi/spi-log/src/serv/pgv2/log_pg_item_serv.rs @@ -22,13 +22,20 @@ use bios_basic::{ }; use crate::{ - dto::log_item_dto::{AdvBasicQueryCondInfo, LogConfigReq, LogItemAddReq, LogItemFindReq, LogItemFindResp, StatsItemAddReq}, + dto::log_item_dto::{AdvBasicQueryCondInfo, LogConfigReq, LogItemAddReq, LogItemAddV2Req, LogItemFindReq, LogItemFindResp, StatsItemAddReq}, log_constants::{CONFIG_TABLE_NAME, LOG_REF_FLAG, TABLE_LOG_FLAG_V2}, }; use super::log_pg_initializer; pub async fn add(add_req: &mut LogItemAddReq, funs: &TardisFunsInst, ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult { + crate::serv::pg::log_pg_item_serv::add(add_req, funs, ctx, inst).await +} +pub async fn find(find_req: &mut LogItemFindReq, funs: &TardisFunsInst, ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult> { + crate::serv::pg::log_pg_item_serv::find(find_req, funs, ctx, inst).await +} + +pub async fn addv2(add_req: &mut LogItemAddV2Req, funs: &TardisFunsInst, ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult { let id = add_req.idempotent_id.clone().unwrap_or(TardisFuns::field.nanoid()); let bs_inst = inst.inst::(); @@ -144,7 +151,7 @@ fn parse_ref_ts_key(ref_key: &str) -> TardisResult<(DateTime, String)> { )) } -pub async fn find(find_req: &mut LogItemFindReq, funs: &TardisFunsInst, ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult> { +pub async fn findv2(find_req: &mut LogItemFindReq, funs: &TardisFunsInst, ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult> { let mut where_fragments: Vec = Vec::new(); let mut sql_vals: Vec = vec![]; @@ -699,7 +706,7 @@ async fn get_ref_fields_by_table_name(conn: &TardisRelDBlConnection, schema_name Ok(ref_fields) } -async fn push_to_eda(req: &LogItemAddReq, ref_fields: &Vec, funs: &TardisFunsInst, ctx: &TardisContext) -> TardisResult<()> { +async fn push_to_eda(req: &LogItemAddV2Req, ref_fields: &Vec, funs: &TardisFunsInst, ctx: &TardisContext) -> TardisResult<()> { if let Some(topic) = get_topic(&SPI_RPC_TOPIC) { let mut req_clone = req.clone(); for ref_field in ref_fields { diff --git a/backend/spi/spi-stats/src/event.rs b/backend/spi/spi-stats/src/event.rs index 325b75a58..f9a13372d 100644 --- a/backend/spi/spi-stats/src/event.rs +++ b/backend/spi/spi-stats/src/event.rs @@ -1,7 +1,7 @@ use crate::{get_tardis_inst, serv}; use bios_sdk_invoke::clients::{ event_client::{get_topic, mq_error, ContextHandler, SPI_RPC_TOPIC}, - spi_log_client::LogItemAddReq, + spi_log_client::LogItemAddV2Req, }; use tardis::{ basic::{dto::TardisContext, result::TardisResult}, @@ -9,7 +9,7 @@ use tardis::{ }; #[instrument] -async fn handle_add_event(req: LogItemAddReq, ctx: TardisContext) -> TardisResult<()> { +async fn handle_add_event(req: LogItemAddV2Req, ctx: TardisContext) -> TardisResult<()> { let funs = get_tardis_inst(); //TODO Ok(()) diff --git a/backend/supports/iam/src/basic/serv/clients/iam_log_client.rs b/backend/supports/iam/src/basic/serv/clients/iam_log_client.rs index 2e18562ba..beb029e62 100644 --- a/backend/supports/iam/src/basic/serv/clients/iam_log_client.rs +++ b/backend/supports/iam/src/basic/serv/clients/iam_log_client.rs @@ -5,7 +5,7 @@ use bios_basic::{ serv::{rbum_crud_serv::RbumCrudOperation, rbum_item_serv::RbumItemCrudOperation, rbum_set_serv::RbumSetCateServ}, }, }; -use bios_sdk_invoke::clients::spi_log_client::{LogItemAddReq, SpiLogClient}; +use bios_sdk_invoke::clients::spi_log_client::{LogItemAddV2Req, SpiLogClient}; use serde::Serialize; use tardis::{ @@ -135,7 +135,7 @@ impl IamLogClient { let own_paths = if ctx.own_paths.len() < 2 { None } else { Some(ctx.own_paths.clone()) }; let owner = if ctx.owner.len() < 2 { None } else { Some(ctx.owner.clone()) }; - let add_req = LogItemAddReq { + let add_req = LogItemAddV2Req { tag, content: TardisFuns::json.obj_to_json(&content).expect("req_msg not a valid json value"), kind, @@ -151,7 +151,7 @@ impl IamLogClient { owner_name: None, push: false, }; - SpiLogClient::add(add_req, funs, ctx).await?; + SpiLogClient::addv2(add_req, funs, ctx).await?; Ok(()) } diff --git a/frontend/sdks/invoke/src/clients/spi_log_client.rs b/frontend/sdks/invoke/src/clients/spi_log_client.rs index 83143316f..0704176a6 100644 --- a/frontend/sdks/invoke/src/clients/spi_log_client.rs +++ b/frontend/sdks/invoke/src/clients/spi_log_client.rs @@ -25,7 +25,7 @@ pub mod event { pub const LOG_AVATAR: &str = "spi-log"; - impl EventAttribute for super::LogItemAddReq { + impl EventAttribute for super::LogItemAddV2Req { const SUBJECT: Subject = Subject::const_new("log/add"); } } @@ -70,6 +70,21 @@ pub struct LogDynamicContentReq { #[derive(Serialize, Deserialize, Debug, Default)] pub struct LogItemAddReq { + pub tag: String, + pub content: String, + pub kind: Option, + pub ext: Option, + pub key: Option, + pub op: Option, + pub rel_key: Option, + pub id: Option, + pub ts: Option>, + pub owner: Option, + pub own_paths: Option, +} + +#[derive(Serialize, Deserialize, Debug, Default)] +pub struct LogItemAddV2Req { pub tag: String, pub content: Value, pub kind: Option, @@ -87,6 +102,7 @@ pub struct LogItemAddReq { } impl SpiLogClient { + #[deprecated] pub async fn add_dynamic_log( content: &LogDynamicContentReq, ext: Option, @@ -97,10 +113,38 @@ impl SpiLogClient { ts: Option, funs: &TardisFunsInst, ctx: &TardisContext, + ) -> TardisResult<()> { + let req = LogItemAddReq { + tag: DYNAMIC_LOG.to_string(), + content: TardisFuns::json.obj_to_string(content)?, + kind, + ext, + key, + op, + rel_key, + id: None, + ts: ts.map(|ts| DateTime::parse_from_rfc3339(&ts).unwrap_or_default().with_timezone(&Utc)), + owner: Some(ctx.owner.clone()), + own_paths: Some(ctx.own_paths.clone()), + }; + Self::add(req, funs, ctx).await?; + Ok(()) + } + + pub async fn add_dynamic_log_v2( + content: &LogDynamicContentReq, + ext: Option, + kind: Option, + key: Option, + op: Option, + rel_key: Option, + ts: Option, + funs: &TardisFunsInst, + ctx: &TardisContext, ) -> TardisResult<()> { let cfg = funs.conf::(); let owner_name = IamClient::new("", funs, ctx, cfg.module_urls.get("iam").expect("missing iam base url")).get_account(&ctx.owner, &ctx.own_paths).await?.owner_name; - let req = LogItemAddReq { + let req = LogItemAddV2Req { tag: DYNAMIC_LOG.to_string(), content: TardisFuns::json.obj_to_json(content)?, kind, @@ -116,11 +160,19 @@ impl SpiLogClient { owner_name, push: false, }; - Self::add(req, funs, ctx).await?; + Self::addv2(req, funs, ctx).await?; Ok(()) } + #[deprecated] pub async fn add(req: LogItemAddReq, funs: &TardisFunsInst, ctx: &TardisContext) -> TardisResult<()> { + let log_url: String = BaseSpiClient::module_url(InvokeModuleKind::Log, funs).await?; + let headers = BaseSpiClient::headers(None, funs, ctx).await?; + funs.web_client().post_obj_to_str(&format!("{log_url}/ci/item"), &req, headers.clone()).await?; + Ok(()) + } + + pub async fn addv2(req: LogItemAddV2Req, funs: &TardisFunsInst, ctx: &TardisContext) -> TardisResult<()> { #[cfg(feature = "event")] if let Some(topic) = get_topic(&SPI_RPC_TOPIC) { topic.send_event(req.inject_context(funs, ctx).json()).map_err(mq_error).await?; @@ -128,14 +180,23 @@ impl SpiLogClient { } let log_url: String = BaseSpiClient::module_url(InvokeModuleKind::Log, funs).await?; let headers = BaseSpiClient::headers(None, funs, ctx).await?; - funs.web_client().post_obj_to_str(&format!("{log_url}/ci/item"), &req, headers.clone()).await?; + funs.web_client().post_obj_to_str(&format!("{log_url}/ci/v2/item"), &req, headers.clone()).await?; Ok(()) } + #[deprecated] pub async fn find(find_req: LogItemFindReq, funs: &TardisFunsInst, ctx: &TardisContext) -> TardisResult>> { + Self::do_find(find_req, "/ci/item/find", funs, ctx).await + } + + pub async fn findv2(find_req: LogItemFindReq, funs: &TardisFunsInst, ctx: &TardisContext) -> TardisResult>> { + Self::do_find(find_req, "/ci/v2/item/find", funs, ctx).await + } + + async fn do_find(find_req: LogItemFindReq, path: &str, funs: &TardisFunsInst, ctx: &TardisContext) -> TardisResult>> { let log_url: String = BaseSpiClient::module_url(InvokeModuleKind::Log, funs).await?; let headers = BaseSpiClient::headers(None, funs, ctx).await?; - let resp = funs.web_client().put::>>(&format!("{log_url}/ci/item/find"), &find_req, headers.clone()).await?; + let resp = funs.web_client().put::>>(&format!("{log_url}{path}"), &find_req, headers.clone()).await?; BaseSpiClient::package_resp(resp) } } From 74a606f758f00cdaf9dcaae76091f25979e930cf Mon Sep 17 00:00:00 2001 From: RWDai <27391645+RWDai@users.noreply.github.com> Date: Fri, 1 Nov 2024 10:33:11 +0800 Subject: [PATCH 2/6] Feat log new api (#859) --- backend/spi/spi-log/src/dto/log_item_dto.rs | 27 ++- backend/spi/spi-log/src/event.rs | 10 +- .../spi-log/src/serv/pgv2/log_pg_item_serv.rs | 14 +- backend/spi/spi-stats/src/api/ci.rs | 1 + .../spi-stats/src/api/ci/stats_ci_conf_api.rs | 33 +++- .../src/api/ci/stats_ci_record_api.rs | 2 + .../spi-stats/src/api/ci/stats_ci_sync_api.rs | 36 ++++ .../spi/spi-stats/src/dto/stats_conf_dto.rs | 80 +++++++++ .../spi/spi-stats/src/dto/stats_record_dto.rs | 15 ++ backend/spi/spi-stats/src/event.rs | 24 ++- backend/spi/spi-stats/src/serv.rs | 1 + backend/spi/spi-stats/src/serv/pg.rs | 1 + .../serv/pg/stats_pg_conf_fact_col_serv.rs | 29 +++- .../src/serv/pg/stats_pg_conf_fact_serv.rs | 30 +++- .../src/serv/pg/stats_pg_initializer.rs | 11 +- .../src/serv/pg/stats_pg_sync_serv.rs | 158 ++++++++++++++++++ .../spi/spi-stats/src/serv/stats_sync_serv.rs | 26 +++ .../sdks/invoke/src/clients/spi_log_client.rs | 35 ++++ 18 files changed, 494 insertions(+), 39 deletions(-) create mode 100644 backend/spi/spi-stats/src/api/ci/stats_ci_sync_api.rs create mode 100644 backend/spi/spi-stats/src/serv/pg/stats_pg_sync_serv.rs create mode 100644 backend/spi/spi-stats/src/serv/stats_sync_serv.rs diff --git a/backend/spi/spi-log/src/dto/log_item_dto.rs b/backend/spi/spi-log/src/dto/log_item_dto.rs index f7e2ffe4e..24fe6db14 100644 --- a/backend/spi/spi-log/src/dto/log_item_dto.rs +++ b/backend/spi/spi-log/src/dto/log_item_dto.rs @@ -1,4 +1,5 @@ use bios_basic::{dto::BasicQueryCondInfo, enumeration::BasicQueryOpKind}; +use bios_sdk_invoke::clients::spi_log_client::{StatsItemAddReq, StatsItemDeleteReq}; use serde::{Deserialize, Serialize}; use tardis::{ basic::field::TrimString, @@ -154,22 +155,6 @@ pub struct LogConfigReq { pub ref_field: String, } -#[derive(poem_openapi::Object, Serialize, Deserialize, Debug)] -pub struct StatsItemAddReq { - #[oai(validator(min_length = "2"))] - pub idempotent_id: Option, - #[oai(validator(pattern = r"^[a-z0-9_]+$"))] - pub tag: String, - pub content: Value, - pub ext: Option, - #[oai(validator(min_length = "2"))] - pub key: Option, - pub ts: Option>, - #[oai(validator(min_length = "2"))] - pub owner: Option, - pub own_paths: Option, -} - impl From for StatsItemAddReq { fn from(value: LogItemAddV2Req) -> Self { StatsItemAddReq { @@ -184,3 +169,13 @@ impl From for StatsItemAddReq { } } } + +impl From for StatsItemDeleteReq { + fn from(value: LogItemAddV2Req) -> Self { + StatsItemDeleteReq { + idempotent_id: value.idempotent_id, + tag: value.tag, + key: value.key, + } + } +} diff --git a/backend/spi/spi-log/src/event.rs b/backend/spi/spi-log/src/event.rs index 81c699144..f413eb241 100644 --- a/backend/spi/spi-log/src/event.rs +++ b/backend/spi/spi-log/src/event.rs @@ -1,9 +1,6 @@ -use crate::{dto::log_item_dto::StatsItemAddReq, log_initializer::get_tardis_inst, serv}; +use crate::{log_initializer::get_tardis_inst, serv}; use bios_sdk_invoke::clients::{ - event_client::{ - asteroid_mq::prelude::{EventAttribute, Subject}, - get_topic, mq_error, ContextHandler, SPI_RPC_TOPIC, - }, + event_client::{get_topic, mq_error, ContextHandler, SPI_RPC_TOPIC}, spi_log_client::LogItemAddV2Req, }; use tardis::{ @@ -26,6 +23,3 @@ pub async fn handle_events() -> TardisResult<()> { Ok(()) } -impl EventAttribute for StatsItemAddReq { - const SUBJECT: Subject = Subject::const_new("stats/add"); -} diff --git a/backend/spi/spi-log/src/serv/pgv2/log_pg_item_serv.rs b/backend/spi/spi-log/src/serv/pgv2/log_pg_item_serv.rs index fbc94f1fd..0dc56ff17 100644 --- a/backend/spi/spi-log/src/serv/pgv2/log_pg_item_serv.rs +++ b/backend/spi/spi-log/src/serv/pgv2/log_pg_item_serv.rs @@ -1,6 +1,9 @@ use std::{collections::HashMap, str::FromStr, vec}; -use bios_sdk_invoke::clients::event_client::{get_topic, mq_error, EventAttributeExt as _, SPI_RPC_TOPIC}; +use bios_sdk_invoke::clients::{ + event_client::{get_topic, mq_error, EventAttributeExt as _, SPI_RPC_TOPIC}, + spi_log_client::{StatsItemAddReq, StatsItemDeleteReq}, +}; use tardis::{ basic::{dto::TardisContext, error::TardisError, result::TardisResult}, chrono::{DateTime, Utc}, @@ -22,7 +25,7 @@ use bios_basic::{ }; use crate::{ - dto::log_item_dto::{AdvBasicQueryCondInfo, LogConfigReq, LogItemAddReq, LogItemAddV2Req, LogItemFindReq, LogItemFindResp, StatsItemAddReq}, + dto::log_item_dto::{AdvBasicQueryCondInfo, LogConfigReq, LogItemAddReq, LogItemAddV2Req, LogItemFindReq, LogItemFindResp}, log_constants::{CONFIG_TABLE_NAME, LOG_REF_FLAG, TABLE_LOG_FLAG_V2}, }; @@ -714,6 +717,13 @@ async fn push_to_eda(req: &LogItemAddV2Req, ref_fields: &Vec, funs: &Tar content.remove(ref_field); } } + if let Some(ref op) = req_clone.op { + if op.to_lowercase() == "delete" { + let stats_delete: StatsItemDeleteReq = req_clone.into(); + topic.send_event(stats_delete.inject_context(funs, ctx).json()).map_err(mq_error).await?; + return Ok(()); + } + } let stats_add: StatsItemAddReq = req_clone.into(); topic.send_event(stats_add.inject_context(funs, ctx).json()).map_err(mq_error).await?; } diff --git a/backend/spi/spi-stats/src/api/ci.rs b/backend/spi/spi-stats/src/api/ci.rs index f729b72c2..6065b0daa 100644 --- a/backend/spi/spi-stats/src/api/ci.rs +++ b/backend/spi/spi-stats/src/api/ci.rs @@ -1,3 +1,4 @@ pub mod stats_ci_conf_api; pub mod stats_ci_metric_api; pub mod stats_ci_record_api; +pub mod stats_ci_sync_api; diff --git a/backend/spi/spi-stats/src/api/ci/stats_ci_conf_api.rs b/backend/spi/spi-stats/src/api/ci/stats_ci_conf_api.rs index b0d6d588d..2b417537a 100644 --- a/backend/spi/spi-stats/src/api/ci/stats_ci_conf_api.rs +++ b/backend/spi/spi-stats/src/api/ci/stats_ci_conf_api.rs @@ -7,9 +7,9 @@ use tardis::web::web_resp::{TardisApiResult, TardisPage, TardisResp, Void}; use crate::dto::stats_conf_dto::{ StatsConfDimAddReq, StatsConfDimInfoResp, StatsConfDimModifyReq, StatsConfFactAddReq, StatsConfFactColAddReq, StatsConfFactColInfoResp, StatsConfFactColModifyReq, - StatsConfFactInfoResp, StatsConfFactModifyReq, + StatsConfFactInfoResp, StatsConfFactModifyReq, StatsSyncDbConfigAddReq, StatsSyncDbConfigInfoResp, StatsSyncDbConfigModifyReq, }; -use crate::serv::{stats_conf_dim_serv, stats_conf_fact_col_serv, stats_conf_fact_serv}; +use crate::serv::{stats_conf_dim_serv, stats_conf_fact_col_serv, stats_conf_fact_serv, stats_sync_serv}; use crate::stats_enumeration::StatsFactColKind; #[derive(Clone)] @@ -297,4 +297,33 @@ impl StatsCiConfApi { stats_conf_fact_serv::create_inst(&fact_key.0, &funs, &ctx.0).await?; TardisResp::ok(Void {}) } + + /// Add Sync DateBase Config + /// + /// 添加同步数据库配置 + #[oai(path = "/sync/db", method = "post")] + async fn db_config_add(&self, add_req: Json, ctx: TardisContextExtractor) -> TardisApiResult { + let funs = crate::get_tardis_inst(); + stats_sync_serv::db_config_add(add_req.0, &funs, &ctx.0).await?; + TardisResp::ok(Void {}) + } + + /// Modify Sync DateBase Config + /// + /// 修改同步数据库配置 + #[oai(path = "/sync/db", method = "put")] + async fn db_config_modify(&self, modify_req: Json, ctx: TardisContextExtractor) -> TardisApiResult { + let funs = crate::get_tardis_inst(); + stats_sync_serv::db_config_modify(modify_req.0, &funs, &ctx.0).await?; + TardisResp::ok(Void {}) + } + + /// List Sync DateBase Config + /// + /// 查询同步数据库配置 + #[oai(path = "/sync/db", method = "get")] + async fn db_config_list(&self, ctx: TardisContextExtractor) -> TardisApiResult> { + let funs = crate::get_tardis_inst(); + TardisResp::ok(stats_sync_serv::db_config_list(&funs, &ctx.0).await?) + } } diff --git a/backend/spi/spi-stats/src/api/ci/stats_ci_record_api.rs b/backend/spi/spi-stats/src/api/ci/stats_ci_record_api.rs index b189e1b76..010d9aa5e 100644 --- a/backend/spi/spi-stats/src/api/ci/stats_ci_record_api.rs +++ b/backend/spi/spi-stats/src/api/ci/stats_ci_record_api.rs @@ -1,3 +1,5 @@ +use bios_basic::rbum::dto::rbum_cert_conf_dto::RbumCertConfAddReq; +use bios_basic::rbum::dto::rbum_cert_dto::RbumCertAddReq; use tardis::chrono::{DateTime, Utc}; use tardis::serde_json::Value; use tardis::web::context_extractor::TardisContextExtractor; diff --git a/backend/spi/spi-stats/src/api/ci/stats_ci_sync_api.rs b/backend/spi/spi-stats/src/api/ci/stats_ci_sync_api.rs new file mode 100644 index 000000000..c69958759 --- /dev/null +++ b/backend/spi/spi-stats/src/api/ci/stats_ci_sync_api.rs @@ -0,0 +1,36 @@ +use tardis::web::{ + context_extractor::TardisContextExtractor, + poem_openapi::{self, param::Path}, + web_resp::{TardisApiResult, TardisResp, Void}, +}; + +use crate::serv::stats_sync_serv; + +#[derive(Clone)] +pub struct StatsCiSyncApi; + +/// Interface Console Statistics Sync API +/// +/// 统计同步接口 +#[poem_openapi::OpenApi(prefix_path = "/ci", tag = "bios_basic::ApiTag::Interface")] +impl StatsCiSyncApi { + /// Sync Fact Record + /// + /// 同步事实记录 + #[oai(path = "/fact/:fact_key/sync", method = "put")] + async fn fact_record_sync(&self, fact_key: Path, ctx: TardisContextExtractor) -> TardisApiResult { + let funs = crate::get_tardis_inst(); + stats_sync_serv::fact_record_sync(&fact_key.0, &funs, &ctx.0).await?; + TardisResp::ok(Void {}) + } + + /// Sync Fact Column Record + /// + /// 同步事实列记录 + #[oai(path = "/fact/:fact_key/col/:col_key/sync", method = "put")] + async fn fact_col_record_sync(&self, fact_key: Path, col_key: Path, ctx: TardisContextExtractor) -> TardisApiResult { + let funs = crate::get_tardis_inst(); + stats_sync_serv::fact_col_record_sync(&fact_key.0, &col_key.0, &funs, &ctx.0).await?; + TardisResp::ok(Void {}) + } +} diff --git a/backend/spi/spi-stats/src/dto/stats_conf_dto.rs b/backend/spi/spi-stats/src/dto/stats_conf_dto.rs index 1d2cc7f6f..4b4f83f31 100644 --- a/backend/spi/spi-stats/src/dto/stats_conf_dto.rs +++ b/backend/spi/spi-stats/src/dto/stats_conf_dto.rs @@ -168,6 +168,10 @@ pub struct StatsConfFactAddReq { pub redirect_path: Option, /// default value is false pub is_online: Option, + pub rel_cert_id: Option, + pub sync_sql: Option, + pub sync_cron: Option, + pub sync_on: Option, } /// Modify Fact Configuration Request Object @@ -186,6 +190,10 @@ pub struct StatsConfFactModifyReq { pub remark: Option, pub redirect_path: Option, pub is_online: Option, + pub rel_cert_id: Option, + pub sync_sql: Option, + pub sync_cron: Option, + pub sync_on: Option, } /// Fact Configuration Response Object @@ -213,6 +221,10 @@ pub struct StatsConfFactInfoResp { pub redirect_path: Option, pub create_time: DateTime, pub update_time: DateTime, + pub rel_cert_id: Option, + pub sync_sql: Option, + pub sync_cron: Option, + pub sync_on: Option, } /// Add Fact Column Configuration Request Object @@ -295,6 +307,9 @@ pub struct StatsConfFactColAddReq { pub rel_external_id: Option, pub dim_exclusive_rec: Option, pub remark: Option, + pub rel_field: Option, + pub rel_sql: Option, + pub rel_cert_id: Option, } /// Modify Fact Column Configuration Request Object @@ -373,6 +388,9 @@ pub struct StatsConfFactColModifyReq { /// 用于扩展ext字段的事实列 pub rel_external_id: Option, pub remark: Option, + pub rel_field: Option, + pub rel_sql: Option, + pub rel_cert_id: Option, } /// Fact Column Configuration Response Object @@ -462,4 +480,66 @@ pub struct StatsConfFactColInfoResp { pub remark: Option, pub create_time: DateTime, pub update_time: DateTime, + pub rel_field: Option, + pub rel_sql: Option, + pub rel_cert_id: Option, } + +/// Add Sync DateBase Config Request Object +/// +/// 添加同步数据库配置请求对象 +#[derive(poem_openapi::Object, Serialize, Deserialize, Debug)] +pub struct StatsSyncDbConfigAddReq { + pub db_url: String, + pub db_user: String, + pub db_password: String, + pub max_connections: Option, + pub min_connections: Option, +} + +/// Modify Sync DateBase Config Request Object +/// +/// 修改同步数据库配置请求对象 +#[derive(poem_openapi::Object, Serialize, Deserialize, Debug)] +pub struct StatsSyncDbConfigModifyReq { + pub id: String, + pub db_url: Option, + pub db_user: Option, + pub db_password: Option, + pub max_connections: Option, + pub min_connections: Option, +} + +/// Sync DateBase Config Response Object +/// +/// 同步数据库配置响应对象 +#[derive(poem_openapi::Object, sea_orm::FromQueryResult, Serialize, Deserialize, Debug)] +pub struct StatsSyncDbConfigInfoResp { + pub id: String, + pub db_url: String, + pub db_user: String, + pub max_connections: Option, + pub min_connections: Option, +} + +/// Sync DateBase Config Response Object +/// +/// 同步数据库配置响应对象 +#[derive(poem_openapi::Object, sea_orm::FromQueryResult, Serialize, Deserialize, Debug)] +pub struct StatsSyncDbConfigInfoWithSkResp { + pub id: String, + pub db_url: String, + pub db_user: String, + pub db_password: String, + pub max_connections: Option, + pub min_connections: Option, +} + +/// Sync DateBase Config Extension Object +/// +/// 同步数据库配置扩展对象 +#[derive(Serialize, Deserialize, Debug,Clone)] +pub struct StatsSyncDbConfigExt { + pub max_connections: Option, + pub min_connections: Option, +} \ No newline at end of file diff --git a/backend/spi/spi-stats/src/dto/stats_record_dto.rs b/backend/spi/spi-stats/src/dto/stats_record_dto.rs index 7f7b53f7c..505126228 100644 --- a/backend/spi/spi-stats/src/dto/stats_record_dto.rs +++ b/backend/spi/spi-stats/src/dto/stats_record_dto.rs @@ -1,3 +1,4 @@ +use bios_sdk_invoke::clients::spi_log_client::StatsItemAddReq; use serde::{Deserialize, Serialize}; use tardis::{ chrono::{DateTime, Utc}, @@ -38,6 +39,20 @@ pub struct StatsFactRecordLoadReq { /// 动态数据 pub ext: Option, } + +impl From for StatsFactRecordLoadReq { + fn from(value: StatsItemAddReq) -> Self { + StatsFactRecordLoadReq { + own_paths: value.own_paths.unwrap_or_default(), + ct: value.ts.unwrap_or(Utc::now()), + idempotent_id: value.idempotent_id, + ignore_updates: None, + data: value.content, + ext: value.ext, + } + } +} + /// Load Fact Record Request Object /// /// 事实记录加载请求对象 diff --git a/backend/spi/spi-stats/src/event.rs b/backend/spi/spi-stats/src/event.rs index f9a13372d..914a560f6 100644 --- a/backend/spi/spi-stats/src/event.rs +++ b/backend/spi/spi-stats/src/event.rs @@ -1,7 +1,10 @@ -use crate::{get_tardis_inst, serv}; +use crate::{ + get_tardis_inst, + serv::{self, stats_record_serv}, +}; use bios_sdk_invoke::clients::{ event_client::{get_topic, mq_error, ContextHandler, SPI_RPC_TOPIC}, - spi_log_client::LogItemAddV2Req, + spi_log_client::{LogItemAddV2Req, StatsItemAddReq, StatsItemDeleteReq}, }; use tardis::{ basic::{dto::TardisContext, result::TardisResult}, @@ -9,16 +12,27 @@ use tardis::{ }; #[instrument] -async fn handle_add_event(req: LogItemAddV2Req, ctx: TardisContext) -> TardisResult<()> { +async fn handle_add_event(req: StatsItemAddReq, ctx: TardisContext) -> TardisResult<()> { let funs = get_tardis_inst(); - //TODO + if let Some(ref key) = req.key { + stats_record_serv::fact_record_load(&req.tag, &key.to_string(), req.clone().into(), &funs, &ctx).await?; + } + Ok(()) +} +#[instrument] +async fn handle_delete_event(req: StatsItemDeleteReq, ctx: TardisContext) -> TardisResult<()> { + let funs = get_tardis_inst(); + if let Some(ref key) = req.key { + stats_record_serv::fact_record_delete(&req.tag, &key.to_string(), &funs, &ctx).await?; + } Ok(()) } pub async fn handle_events() -> TardisResult<()> { use bios_sdk_invoke::clients::event_client::asteroid_mq::prelude::*; if let Some(topic) = get_topic(&SPI_RPC_TOPIC) { - topic.create_endpoint([Interest::new("stats/*")]).await.map_err(mq_error)?.create_event_loop().with_handler(ContextHandler(handle_add_event)).spawn(); + topic.create_endpoint([Interest::new("stats/add")]).await.map_err(mq_error)?.create_event_loop().with_handler(ContextHandler(handle_add_event)).spawn(); + topic.create_endpoint([Interest::new("stats/delete")]).await.map_err(mq_error)?.create_event_loop().with_handler(ContextHandler(handle_delete_event)).spawn(); } Ok(()) diff --git a/backend/spi/spi-stats/src/serv.rs b/backend/spi/spi-stats/src/serv.rs index b85e257b2..455f23813 100644 --- a/backend/spi/spi-stats/src/serv.rs +++ b/backend/spi/spi-stats/src/serv.rs @@ -4,3 +4,4 @@ pub mod stats_conf_fact_col_serv; pub mod stats_conf_fact_serv; pub mod stats_metric_serv; pub mod stats_record_serv; +pub mod stats_sync_serv; diff --git a/backend/spi/spi-stats/src/serv/pg.rs b/backend/spi/spi-stats/src/serv/pg.rs index 8373f9721..5f96cf5e7 100644 --- a/backend/spi/spi-stats/src/serv/pg.rs +++ b/backend/spi/spi-stats/src/serv/pg.rs @@ -4,3 +4,4 @@ pub mod stats_pg_conf_fact_serv; pub mod stats_pg_initializer; pub mod stats_pg_metric_serv; pub(crate) mod stats_pg_record_serv; +pub mod stats_pg_sync_serv; diff --git a/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_fact_col_serv.rs b/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_fact_col_serv.rs index 0f961a938..2d77ac149 100644 --- a/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_fact_col_serv.rs +++ b/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_fact_col_serv.rs @@ -127,6 +127,18 @@ pub(crate) async fn add(fact_conf_key: &str, add_req: &StatsConfFactColAddReq, f params.push(Value::from(dim_dynamic_url.to_string())); sql_fields.push("dim_dynamic_url"); } + if let Some(rel_cert_id) = &add_req.rel_cert_id { + params.push(Value::from(rel_cert_id.to_string())); + sql_fields.push("rel_cert_id"); + } + if let Some(rel_field) = &add_req.rel_field { + params.push(Value::from(rel_field.to_string())); + sql_fields.push("rel_field"); + } + if let Some(rel_sql) = &add_req.rel_sql { + params.push(Value::from(rel_sql.to_string())); + sql_fields.push("rel_sql"); + } conn.execute_one( &format!( r#"INSERT INTO {table_name} @@ -230,6 +242,18 @@ pub(crate) async fn modify( sql_sets.push(format!("dim_dynamic_url = ${}", params.len() + 1)); params.push(Value::from(dim_dynamic_url.to_string())); } + if let Some(rel_field) = &modify_req.rel_field { + sql_sets.push(format!("rel_field = ${}", params.len() + 1)); + params.push(Value::from(rel_field.to_string())); + } + if let Some(rel_sql) = &modify_req.rel_sql { + sql_sets.push(format!("rel_sql = ${}", params.len() + 1)); + params.push(Value::from(rel_sql.to_string())); + } + if let Some(rel_cert_id) = &modify_req.rel_cert_id { + sql_sets.push(format!("rel_cert_id = ${}", params.len() + 1)); + params.push(Value::from(rel_cert_id.to_string())); + } conn.execute_one( &format!( r#"UPDATE {table_name} @@ -381,7 +405,7 @@ async fn do_paginate( let result = conn .query_all( &format!( - r#"SELECT key, show_name, kind, remark, dim_rel_conf_dim_key, rel_external_id, dim_multi_values, dim_exclusive_rec, dim_data_type, dim_dynamic_url, mes_data_distinct, mes_data_type, mes_frequency, mes_unit, mes_act_by_dim_conf_keys, rel_conf_fact_key, rel_conf_fact_and_col_key, create_time, update_time, count(*) OVER() AS total + r#"SELECT key, show_name, kind, remark, dim_rel_conf_dim_key, rel_external_id, dim_multi_values, dim_exclusive_rec, dim_data_type, dim_dynamic_url, mes_data_distinct, mes_data_type, mes_frequency, mes_unit, mes_act_by_dim_conf_keys, rel_conf_fact_key, rel_conf_fact_and_col_key, create_time, update_time,rel_field,rel_cert_id,rel_sql, count(*) OVER() AS total FROM {table_name} WHERE {} @@ -432,6 +456,9 @@ WHERE create_time: item.try_get("", "create_time")?, update_time: item.try_get("", "update_time")?, rel_external_id: item.try_get("", "rel_external_id")?, + rel_field: item.try_get("", "rel_field")?, + rel_sql: item.try_get("", "rel_sql")?, + rel_cert_id: item.try_get("", "rel_cert_id")?, }) }) .collect::>()?; diff --git a/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_fact_serv.rs b/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_fact_serv.rs index a448a166a..e9066510f 100644 --- a/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_fact_serv.rs +++ b/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_fact_serv.rs @@ -43,14 +43,18 @@ pub(crate) async fn add(add_req: &StatsConfFactAddReq, funs: &TardisFunsInst, ct Value::from(add_req.remark.as_ref().unwrap_or(&"".to_string()).as_str()), Value::from(add_req.redirect_path.clone()), Value::from(add_req.is_online.unwrap_or_default()), + Value::from(add_req.rel_cert_id.as_ref().unwrap_or(&"".to_string()).as_str()), + Value::from(add_req.sync_sql.as_ref().unwrap_or(&"".to_string()).as_str()), + Value::from(add_req.sync_cron.as_ref().unwrap_or(&"".to_string()).as_str()), + Value::from(add_req.sync_on.unwrap_or_default()), ]; conn.execute_one( &format!( r#"INSERT INTO {table_name} -(key, show_name, query_limit, remark, redirect_path, is_online) +(key, show_name, query_limit, remark, redirect_path, is_online, rel_cert_id, sync_sql, sync_cron, sync_on) VALUES -($1, $2, $3, $4, $5, $6) +($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) "#, ), params, @@ -92,6 +96,22 @@ pub(crate) async fn modify(fact_conf_key: &str, modify_req: &StatsConfFactModify sql_sets.push(format!("redirect_path = ${}", params.len() + 1)); params.push(Value::from(redirect_path)); } + if let Some(rel_cert_id) = &modify_req.rel_cert_id { + sql_sets.push(format!("rel_cert_id = ${}", params.len() + 1)); + params.push(Value::from(rel_cert_id.to_string())); + } + if let Some(sync_sql) = &modify_req.sync_sql { + sql_sets.push(format!("sync_sql = ${}", params.len() + 1)); + params.push(Value::from(sync_sql.to_string())); + } + if let Some(sync_cron) = &modify_req.sync_cron { + sql_sets.push(format!("sync_cron = ${}", params.len() + 1)); + params.push(Value::from(sync_cron.to_string())); + } + if let Some(sync_on) = &modify_req.sync_on { + sql_sets.push(format!("sync_on = ${}", params.len() + 1)); + params.push(Value::from(*sync_on)); + } }; if let Some(is_online) = &modify_req.is_online { @@ -231,7 +251,7 @@ async fn do_paginate( .query_all( &format!( r#"SELECT t.*, count(*) OVER () AS total FROM ( -SELECT distinct fact.key as key, fact.show_name as show_name, fact.query_limit as query_limit, fact.remark as remark, fact.redirect_path as redirect_path, fact.is_online as is_online, fact.create_time as create_time, fact.update_time as update_time +SELECT distinct fact.key as key, fact.show_name as show_name, fact.query_limit as query_limit, fact.remark as remark, fact.redirect_path as redirect_path, fact.is_online as is_online, fact.rel_cert_id as rel_cert_id, fact.sync_sql as sync_sql, fact.sync_cron as sync_cron, fact.sync_on as sync_on, fact.create_time as create_time, fact.update_time as update_time FROM {table_name} as fact {} WHERE @@ -268,6 +288,10 @@ LIMIT $1 OFFSET $2 online: online(&item.try_get::("", "key")?, conn, ctx).await?, is_online: item.try_get("", "is_online")?, redirect_path: item.try_get("", "redirect_path")?, + rel_cert_id: item.try_get("", "rel_cert_id")?, + sync_sql: item.try_get("", "sync_sql")?, + sync_cron: item.try_get("", "sync_cron")?, + sync_on: item.try_get("", "sync_on")?, }); } Ok(TardisPage { diff --git a/backend/spi/spi-stats/src/serv/pg/stats_pg_initializer.rs b/backend/spi/spi-stats/src/serv/pg/stats_pg_initializer.rs index 179c8c897..e6139cbb3 100644 --- a/backend/spi/spi-stats/src/serv/pg/stats_pg_initializer.rs +++ b/backend/spi/spi-stats/src/serv/pg/stats_pg_initializer.rs @@ -46,7 +46,11 @@ pub async fn init_conf_fact_table_and_conn(bs_inst: TypedSpiBsInst<'_, TardisRel query_limit integer DEFAULT 10000, remark character varying NOT NULL, create_time timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP, - update_time timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP"#, + update_time timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP, + rel_cert_id character varying, + sync_sql character varying, + sync_cron character varying, + sync_on boolean NOT NULL DEFAULT FALSE"#, None, vec![], None, @@ -75,13 +79,16 @@ pub async fn init_conf_fact_col_table_and_conn(bs_inst: TypedSpiBsInst<'_, Tardi mes_frequency character varying, mes_unit character varying, mes_act_by_dim_conf_keys character varying[], + rel_cert_id character varying, + rel_field character varying, + rel_sql character varying, rel_conf_fact_key character varying NOT NULL, rel_conf_fact_and_col_key character varying, rel_external_id character varying NOT NULL, remark character varying NOT NULL, create_time timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP, update_time timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP, - unique (key, rel_conf_fact_key)"#, + unique (key, rel_conf_fact_key, kind, rel_external_id)"#, None, vec![("rel_conf_fact_key", "btree")], None, diff --git a/backend/spi/spi-stats/src/serv/pg/stats_pg_sync_serv.rs b/backend/spi/spi-stats/src/serv/pg/stats_pg_sync_serv.rs new file mode 100644 index 000000000..938d0cae1 --- /dev/null +++ b/backend/spi/spi-stats/src/serv/pg/stats_pg_sync_serv.rs @@ -0,0 +1,158 @@ +use bios_basic::{ + rbum::{ + dto::{ + rbum_cert_dto::{RbumCertAddReq, RbumCertModifyReq}, + rbum_filer_dto::{RbumBasicFilterReq, RbumCertFilterReq}, + }, + rbum_enumeration::{RbumCertRelKind, RbumCertStatusKind}, + serv::{rbum_cert_serv::RbumCertServ, rbum_crud_serv::RbumCrudOperation}, + }, + spi::{spi_constants::SPI_PG_KIND_CODE, spi_funs::SpiBsInst, spi_initializer::common_pg}, +}; +use tardis::{ + basic::{dto::TardisContext, field::TrimString, result::TardisResult}, + db::{reldb_client::TardisRelDBClient, sea_orm::Value}, + TardisFunsInst, +}; + +use crate::{ + dto::stats_conf_dto::{StatsSyncDbConfigAddReq, StatsSyncDbConfigExt, StatsSyncDbConfigInfoResp, StatsSyncDbConfigInfoWithSkResp, StatsSyncDbConfigModifyReq}, stats_constants::DOMAIN_CODE, +}; + +pub(crate) async fn db_config_add(add_req: StatsSyncDbConfigAddReq, funs: &TardisFunsInst, ctx: &TardisContext, _inst: &SpiBsInst) -> TardisResult { + // 使用rel_rbum_id kind supplier 来作为unique key + let mut rbum_cert_add_req = RbumCertAddReq { + ak: TrimString(add_req.db_user), + sk: Some(TrimString(add_req.db_password)), + conn_uri: Some(add_req.db_url), + rel_rbum_id: "".to_string(), + kind: Some(SPI_PG_KIND_CODE.to_string()), + supplier: Some(DOMAIN_CODE.to_string()), + ext: serde_json::to_string(&StatsSyncDbConfigExt{ max_connections: add_req.max_connections, min_connections: add_req.min_connections }).ok(), + sk_invisible: None, + ignore_check_sk: false, + start_time: None, + end_time: None, + status: RbumCertStatusKind::Enabled, + vcode: None, + rel_rbum_cert_conf_id: None, + rel_rbum_kind: RbumCertRelKind::Item, + is_outside: true, + }; + let rbum_cert = RbumCertServ::add_rbum(&mut rbum_cert_add_req, funs, ctx).await?; + return Ok(rbum_cert); +} + +pub(crate) async fn db_config_modify(modify_req: StatsSyncDbConfigModifyReq, funs: &TardisFunsInst, ctx: &TardisContext, _inst: &SpiBsInst) -> TardisResult<()> { + if RbumCertServ::find_one_rbum( + &RbumCertFilterReq { + basic: RbumBasicFilterReq { + ids: Some(vec![modify_req.id.clone()]), + ..Default::default() + }, + ..Default::default() + }, + funs, + ctx, + ) + .await? + .is_some() + { + let mut rbum_cert_modify_req = RbumCertModifyReq { + ak: modify_req.db_user.map(TrimString), + sk: modify_req.db_password.map(TrimString), + conn_uri: modify_req.db_url, + sk_invisible: None, + ignore_check_sk: false, + ext: None, + start_time: None, + end_time: None, + status: None, + }; + RbumCertServ::modify_rbum(&modify_req.id, &mut rbum_cert_modify_req, funs, ctx).await?; + } else { + return Err(funs.err().not_found(&RbumCertServ::get_obj_name(), "modify", "rbum cert not found", "404-rbum-cert-not-found")); + } + return Ok(()); +} + +pub(crate) async fn db_config_list(funs: &TardisFunsInst, ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult> { + let rbum_cert_list = RbumCertServ::find_detail_rbums( + &RbumCertFilterReq { + kind: Some(SPI_PG_KIND_CODE.to_string()), + suppliers: Some(vec![DOMAIN_CODE.to_string()]), + ..Default::default() + }, + None, + None, + funs, + ctx, + ) + .await?; + + return Ok(rbum_cert_list.iter().map(|rbum_cert| { + let ext = serde_json::from_str::(&rbum_cert.ext).ok(); + StatsSyncDbConfigInfoResp { id: rbum_cert.id.clone(), db_url: rbum_cert.conn_uri.clone(), db_user: rbum_cert.ak.clone(), max_connections: ext.clone().and_then(|ext| ext.max_connections), min_connections: ext.clone().and_then(|ext| ext.min_connections) } + }).collect()); +} + +async fn find_db_config(cert_id: &str, funs: &TardisFunsInst, ctx: &TardisContext, _inst: &SpiBsInst) -> TardisResult { + if let Some(rbum_cert) = RbumCertServ::find_one_detail_rbum( + &RbumCertFilterReq { + basic: RbumBasicFilterReq { + ids: Some(vec![cert_id.to_string()]), + ..Default::default() + }, + kind: Some(SPI_PG_KIND_CODE.to_string()), + suppliers: Some(vec![DOMAIN_CODE.to_string()]), + ..Default::default() + }, + funs, + ctx, + ) + .await? + { + let db_password = RbumCertServ::show_sk(cert_id, &RbumCertFilterReq::default(), funs, ctx).await?; + let ext = serde_json::from_str::(&rbum_cert.ext).ok(); + let max_connections = ext.clone().and_then(|ext| ext.max_connections); + let min_connections = ext.clone().and_then(|ext| ext.min_connections); + return Ok(StatsSyncDbConfigInfoWithSkResp { id: cert_id.to_string(), db_url: rbum_cert.conn_uri.clone(), db_user: rbum_cert.ak.clone(), db_password: db_password, max_connections, min_connections }); + } else { + return Err(funs.err().not_found(&RbumCertServ::get_obj_name(), "find", "rbum cert not found", "404-rbum-cert-not-found")); + } +} + +pub(crate) async fn fact_record_sync(fact_key: &str, funs: &TardisFunsInst, ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult<()> { + // let bs_inst = inst.inst::(); + // let (conn, _) = common_pg::init_conn(bs_inst).await?; + + // conn.begin().await?; + + // todo!(); + // let fact_col_list = conn + // .query_all( + // &format!("SELECT key FROM starsys_stats_conf_fact_col WHERE rel_conf_fact_key = $1"), + // vec![Value::from(fact_key)], + // ) + // .await?; + // for col in fact_col_list.iter() { + // let col_key = col.try_get::("", "key")?; + // fact_col_record_sync(fact_key, &col_key, funs, ctx, inst).await?; + // } + // conn.commit().await?; + Ok(()) +} + +pub(crate) async fn fact_col_record_sync(fact_key: &str, col_key: &str, funs: &TardisFunsInst, ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult<()> { + let bs_inst = inst.inst::(); + let (conn, _) = common_pg::init_conn(bs_inst).await?; + + todo!() +} + +async fn do_fact_col_record_sync(fact_key: &str, col_key: &str, funs: &TardisFunsInst, ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult<()> { + let bs_inst = inst.inst::(); + let (conn, _) = common_pg::init_conn(bs_inst).await?; + + todo!() +} diff --git a/backend/spi/spi-stats/src/serv/stats_sync_serv.rs b/backend/spi/spi-stats/src/serv/stats_sync_serv.rs new file mode 100644 index 000000000..4ef827836 --- /dev/null +++ b/backend/spi/spi-stats/src/serv/stats_sync_serv.rs @@ -0,0 +1,26 @@ +use crate::dto::stats_conf_dto::{StatsSyncDbConfigAddReq, StatsSyncDbConfigInfoResp, StatsSyncDbConfigModifyReq}; +use crate::stats_initializer; +use bios_basic::spi::spi_constants; +use bios_basic::spi::spi_funs::SpiBsInstExtractor; +use bios_basic::spi_dispatch_service; +use tardis::basic::result::TardisResult; +use tardis::chrono::{DateTime, Utc}; +use tardis::serde_json::{self, Value}; + +use super::pg; + +spi_dispatch_service! { + @mgr: true, + @init: stats_initializer::init_fun, + @dispatch: { + #[cfg(feature = "spi-pg")] + spi_constants::SPI_PG_KIND_CODE => pg::stats_pg_record_serv, + }, + @method: { + fact_record_sync(fact_key: &str) -> TardisResult<()>; + fact_col_record_sync(fact_key: &str, col_key: &str) -> TardisResult<()>; + db_config_add(add_req: StatsSyncDbConfigAddReq) -> TardisResult<()>; + db_config_modify(modify_req: StatsSyncDbConfigModifyReq) -> TardisResult<()>; + db_config_list() -> TardisResult>; + } +} diff --git a/frontend/sdks/invoke/src/clients/spi_log_client.rs b/frontend/sdks/invoke/src/clients/spi_log_client.rs index 0704176a6..2136cd482 100644 --- a/frontend/sdks/invoke/src/clients/spi_log_client.rs +++ b/frontend/sdks/invoke/src/clients/spi_log_client.rs @@ -1,3 +1,4 @@ +use asteroid_mq::prelude::{EventAttribute, Subject}; use serde::{Deserialize, Serialize}; use tardis::{ @@ -101,6 +102,40 @@ pub struct LogItemAddV2Req { pub msg: Option, } +#[derive(poem_openapi::Object, Serialize, Deserialize, Clone, Debug)] +pub struct StatsItemAddReq { + #[oai(validator(min_length = "2"))] + pub idempotent_id: Option, + #[oai(validator(pattern = r"^[a-z0-9_]+$"))] + pub tag: String, + pub content: Value, + pub ext: Option, + #[oai(validator(min_length = "2"))] + pub key: Option, + pub ts: Option>, + #[oai(validator(min_length = "2"))] + pub owner: Option, + pub own_paths: Option, +} + +impl EventAttribute for StatsItemAddReq { + const SUBJECT: Subject = Subject::const_new("stats/add"); +} + +#[derive(poem_openapi::Object, Serialize, Deserialize, Clone, Debug)] +pub struct StatsItemDeleteReq { + #[oai(validator(min_length = "2"))] + pub idempotent_id: Option, + #[oai(validator(pattern = r"^[a-z0-9_]+$"))] + pub tag: String, + #[oai(validator(min_length = "2"))] + pub key: Option, +} + +impl EventAttribute for StatsItemDeleteReq { + const SUBJECT: Subject = Subject::const_new("stats/delete"); +} + impl SpiLogClient { #[deprecated] pub async fn add_dynamic_log( From 25ac8552b23d8e401d5099f85de4a614d57f2e29 Mon Sep 17 00:00:00 2001 From: RWDai <27391645+RWDai@users.noreply.github.com> Date: Fri, 1 Nov 2024 11:30:20 +0800 Subject: [PATCH 3/6] fix cicd --- backend/spi/spi-stats/Cargo.toml | 2 +- backend/spi/spi-stats/src/event.rs | 4 ++-- backend/spi/spi-stats/src/serv/stats_sync_serv.rs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/spi/spi-stats/Cargo.toml b/backend/spi/spi-stats/Cargo.toml index dc539df36..1ab6f519e 100644 --- a/backend/spi/spi-stats/Cargo.toml +++ b/backend/spi/spi-stats/Cargo.toml @@ -27,7 +27,7 @@ tardis = { workspace = true, features = ["reldb-postgres", "web-server"] } serde_json = { workspace = true, features = ["preserve_order"] } strum = { workspace = true, features = ["derive"] } bios-sdk-invoke = { version = "0.2.0", path = "../../../frontend/sdks/invoke", features = [ - "event", + "event","spi_log" ], default-features = false } [dev-dependencies] diff --git a/backend/spi/spi-stats/src/event.rs b/backend/spi/spi-stats/src/event.rs index 914a560f6..692340074 100644 --- a/backend/spi/spi-stats/src/event.rs +++ b/backend/spi/spi-stats/src/event.rs @@ -1,10 +1,10 @@ use crate::{ get_tardis_inst, - serv::{self, stats_record_serv}, + serv::{stats_record_serv}, }; use bios_sdk_invoke::clients::{ event_client::{get_topic, mq_error, ContextHandler, SPI_RPC_TOPIC}, - spi_log_client::{LogItemAddV2Req, StatsItemAddReq, StatsItemDeleteReq}, + spi_log_client::{StatsItemAddReq, StatsItemDeleteReq}, }; use tardis::{ basic::{dto::TardisContext, result::TardisResult}, diff --git a/backend/spi/spi-stats/src/serv/stats_sync_serv.rs b/backend/spi/spi-stats/src/serv/stats_sync_serv.rs index 4ef827836..afe060a8b 100644 --- a/backend/spi/spi-stats/src/serv/stats_sync_serv.rs +++ b/backend/spi/spi-stats/src/serv/stats_sync_serv.rs @@ -14,12 +14,12 @@ spi_dispatch_service! { @init: stats_initializer::init_fun, @dispatch: { #[cfg(feature = "spi-pg")] - spi_constants::SPI_PG_KIND_CODE => pg::stats_pg_record_serv, + spi_constants::SPI_PG_KIND_CODE => pg::stats_pg_sync_serv, }, @method: { fact_record_sync(fact_key: &str) -> TardisResult<()>; fact_col_record_sync(fact_key: &str, col_key: &str) -> TardisResult<()>; - db_config_add(add_req: StatsSyncDbConfigAddReq) -> TardisResult<()>; + db_config_add(add_req: StatsSyncDbConfigAddReq) -> TardisResult; db_config_modify(modify_req: StatsSyncDbConfigModifyReq) -> TardisResult<()>; db_config_list() -> TardisResult>; } From f55729914e6150440c682938cff22925d19764ad Mon Sep 17 00:00:00 2001 From: RWDai <27391645+RWDai@users.noreply.github.com> Date: Thu, 7 Nov 2024 09:25:53 +0800 Subject: [PATCH 4/6] Feat add dim group api (#861) --- .../spi-stats/src/api/ci/stats_ci_conf_api.rs | 63 ++++++- .../spi/spi-stats/src/dto/stats_conf_dto.rs | 61 ++++++- backend/spi/spi-stats/src/event.rs | 5 +- backend/spi/spi-stats/src/serv.rs | 1 + backend/spi/spi-stats/src/serv/pg.rs | 1 + .../serv/pg/stats_pg_conf_dim_group_serv.rs | 168 ++++++++++++++++++ .../src/serv/pg/stats_pg_conf_dim_serv.rs | 55 +++++- .../serv/pg/stats_pg_conf_fact_col_serv.rs | 60 +++++-- .../src/serv/pg/stats_pg_conf_fact_serv.rs | 16 +- .../src/serv/pg/stats_pg_initializer.rs | 26 ++- .../src/serv/pg/stats_pg_record_serv.rs | 20 +-- .../src/serv/pg/stats_pg_sync_serv.rs | 120 +++++++++---- .../src/serv/stats_conf_dim_group_serv.rs | 28 +++ .../spi-stats/src/serv/stats_conf_dim_serv.rs | 2 + .../src/serv/stats_conf_fact_col_serv.rs | 1 + 15 files changed, 542 insertions(+), 85 deletions(-) create mode 100644 backend/spi/spi-stats/src/serv/pg/stats_pg_conf_dim_group_serv.rs create mode 100644 backend/spi/spi-stats/src/serv/stats_conf_dim_group_serv.rs diff --git a/backend/spi/spi-stats/src/api/ci/stats_ci_conf_api.rs b/backend/spi/spi-stats/src/api/ci/stats_ci_conf_api.rs index 2b417537a..48549ea3a 100644 --- a/backend/spi/spi-stats/src/api/ci/stats_ci_conf_api.rs +++ b/backend/spi/spi-stats/src/api/ci/stats_ci_conf_api.rs @@ -6,10 +6,11 @@ use tardis::web::poem_openapi::payload::Json; use tardis::web::web_resp::{TardisApiResult, TardisPage, TardisResp, Void}; use crate::dto::stats_conf_dto::{ - StatsConfDimAddReq, StatsConfDimInfoResp, StatsConfDimModifyReq, StatsConfFactAddReq, StatsConfFactColAddReq, StatsConfFactColInfoResp, StatsConfFactColModifyReq, - StatsConfFactInfoResp, StatsConfFactModifyReq, StatsSyncDbConfigAddReq, StatsSyncDbConfigInfoResp, StatsSyncDbConfigModifyReq, + StatsConfDimAddReq, StatsConfDimGroupAddReq, StatsConfDimGroupInfoResp, StatsConfDimGroupModifyReq, StatsConfDimInfoResp, StatsConfDimModifyReq, StatsConfFactAddReq, + StatsConfFactColAddReq, StatsConfFactColInfoResp, StatsConfFactColModifyReq, StatsConfFactInfoResp, StatsConfFactModifyReq, StatsSyncDbConfigAddReq, StatsSyncDbConfigInfoResp, + StatsSyncDbConfigModifyReq, }; -use crate::serv::{stats_conf_dim_serv, stats_conf_fact_col_serv, stats_conf_fact_serv, stats_sync_serv}; +use crate::serv::{stats_conf_dim_group_serv, stats_conf_dim_serv, stats_conf_fact_col_serv, stats_conf_fact_serv, stats_sync_serv}; use crate::stats_enumeration::StatsFactColKind; #[derive(Clone)] @@ -20,6 +21,42 @@ pub struct StatsCiConfApi; /// 接口控制台统计配置 API #[poem_openapi::OpenApi(prefix_path = "/ci/conf", tag = "bios_basic::ApiTag::Interface")] impl StatsCiConfApi { + /// Add Dimension Group Configuration + /// + /// 添加维度组配置 + #[oai(path = "/dim/group", method = "put")] + async fn dim_group_add(&self, add_req: Json, ctx: TardisContextExtractor) -> TardisApiResult { + let funs = crate::get_tardis_inst(); + stats_conf_dim_group_serv::add(&add_req.0, &funs, &ctx.0).await?; + TardisResp::ok(Void {}) + } + + /// Modify Dimension Group Configuration + /// + /// 修改维度组配置 + #[oai(path = "/dim/group/:dim_group_key", method = "patch")] + async fn dim_group_modify(&self, dim_group_key: Path, modify_req: Json, ctx: TardisContextExtractor) -> TardisApiResult { + let funs = crate::get_tardis_inst(); + stats_conf_dim_group_serv::modify(&dim_group_key.0, &modify_req.0, &funs, &ctx.0).await?; + TardisResp::ok(Void {}) + } + + /// Find Dimension Group Configurations + /// + /// 查询维度组配置 + #[oai(path = "/dim/group", method = "get")] + async fn dim_group_paginate( + &self, + page_number: Query, + page_size: Query, + desc_by_create: Query>, + desc_by_update: Query>, + ctx: TardisContextExtractor, + ) -> TardisApiResult> { + let funs = crate::get_tardis_inst(); + TardisResp::ok(stats_conf_dim_group_serv::paginate(page_number.0, page_size.0, desc_by_create.0, desc_by_update.0, &funs, &ctx.0).await?) + } + /// Add Dimension Configuration /// /// 添加维度配置 @@ -57,6 +94,8 @@ impl StatsCiConfApi { async fn dim_paginate( &self, key: Query>, + group_key: Query>, + group_is_empty: Query>, show_name: Query>, page_number: Query, page_size: Query, @@ -65,7 +104,19 @@ impl StatsCiConfApi { ctx: TardisContextExtractor, ) -> TardisApiResult> { let funs = crate::get_tardis_inst(); - let resp = stats_conf_dim_serv::paginate(key.0, show_name.0, page_number.0, page_size.0, desc_by_create.0, desc_by_update.0, &funs, &ctx.0).await?; + let resp = stats_conf_dim_serv::paginate( + key.0, + group_key.0, + group_is_empty.0, + show_name.0, + page_number.0, + page_size.0, + desc_by_create.0, + desc_by_update.0, + &funs, + &ctx.0, + ) + .await?; TardisResp::ok(resp) } @@ -217,6 +268,7 @@ impl StatsCiConfApi { &self, fact_key: Path, key: Query>, + group_key: Query>, show_name: Query>, rel_external_id: Query>, page_number: Query, @@ -230,6 +282,7 @@ impl StatsCiConfApi { Some(fact_key.0), key.0, None, + group_key.0, show_name.0, rel_external_id.0, page_number.0, @@ -252,6 +305,7 @@ impl StatsCiConfApi { dim_key: Path, key: Query>, fact_key: Query>, + group_key: Query>, show_name: Query>, rel_external_id: Query>, page_number: Query, @@ -265,6 +319,7 @@ impl StatsCiConfApi { fact_key.0, key.0, Some(dim_key.0), + group_key.0, show_name.0, rel_external_id.0, page_number.0, diff --git a/backend/spi/spi-stats/src/dto/stats_conf_dto.rs b/backend/spi/spi-stats/src/dto/stats_conf_dto.rs index 4b4f83f31..0f4c114cb 100644 --- a/backend/spi/spi-stats/src/dto/stats_conf_dto.rs +++ b/backend/spi/spi-stats/src/dto/stats_conf_dto.rs @@ -7,6 +7,54 @@ use tardis::{ use crate::stats_enumeration::{StatsDataTypeKind, StatsFactColKind}; +/// Add Dimension Group Configuration Request Object +/// +/// 添加维度组配置请求对象 +#[derive(poem_openapi::Object, Serialize, Deserialize, Debug)] +pub struct StatsConfDimGroupAddReq { + /// The primary key or encoding passed in from the external system + /// + /// 外部系统传入的主键或编码 + #[oai(validator(pattern = r"^[a-z0-9_]+$"))] + pub key: String, + pub show_name: String, + pub data_type: StatsDataTypeKind, + pub remark: Option, + pub dynamic_url: Option, + pub rel_attribute_code: Option>, + pub rel_attribute_url: Option, +} + +/// Modify Dimension Group Configuration Request Object +/// +/// 修改维度组配置请求对象 +#[derive(poem_openapi::Object, Serialize, Deserialize, Debug)] +pub struct StatsConfDimGroupModifyReq { + pub show_name: Option, + pub data_type: Option, + pub remark: Option, + pub dynamic_url: Option, + pub rel_attribute_code: Option>, + pub rel_attribute_url: Option, +} + +/// Dimension Group Configuration Response Object +/// +/// 维度组配置响应对象 +#[derive(poem_openapi::Object, sea_orm::FromQueryResult, Serialize, Deserialize, Debug)] +pub struct StatsConfDimGroupInfoResp { + pub key: String, + pub show_name: String, + pub data_type: StatsDataTypeKind, + pub remark: Option, + pub dynamic_url: Option, + pub rel_attribute_code: Option>, + pub rel_attribute_url: Option, + + pub create_time: DateTime, + pub update_time: DateTime, +} + /// Add Dimension Configuration Request Object #[derive(poem_openapi::Object, Serialize, Deserialize, Debug)] pub struct StatsConfDimAddReq { @@ -44,6 +92,7 @@ pub struct StatsConfDimAddReq { /// 例如地址维度可以是省-市-区等 pub hierarchy: Option>, pub remark: Option, + pub dim_group_key: Option, pub dynamic_url: Option, /// is_tree = true, the dimension is a tree structure @@ -84,6 +133,7 @@ pub struct StatsConfDimModifyReq { /// e.g. address dimension can be province-city-district, etc. pub hierarchy: Option>, pub remark: Option, + pub dim_group_key: Option, pub dynamic_url: Option, /// is_tree = true, the dimension is a tree structure @@ -132,6 +182,7 @@ pub struct StatsConfDimInfoResp { /// Whether the dimension is enabled pub online: bool, pub remark: Option, + pub dim_group_key: Option, pub dynamic_url: Option, /// is_tree = true, the dimension is a tree structure @@ -171,7 +222,7 @@ pub struct StatsConfFactAddReq { pub rel_cert_id: Option, pub sync_sql: Option, pub sync_cron: Option, - pub sync_on: Option, + pub is_sync: Option, } /// Modify Fact Configuration Request Object @@ -193,7 +244,7 @@ pub struct StatsConfFactModifyReq { pub rel_cert_id: Option, pub sync_sql: Option, pub sync_cron: Option, - pub sync_on: Option, + pub is_sync: Option, } /// Fact Configuration Response Object @@ -224,7 +275,7 @@ pub struct StatsConfFactInfoResp { pub rel_cert_id: Option, pub sync_sql: Option, pub sync_cron: Option, - pub sync_on: Option, + pub is_sync: Option, } /// Add Fact Column Configuration Request Object @@ -538,8 +589,8 @@ pub struct StatsSyncDbConfigInfoWithSkResp { /// Sync DateBase Config Extension Object /// /// 同步数据库配置扩展对象 -#[derive(Serialize, Deserialize, Debug,Clone)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct StatsSyncDbConfigExt { pub max_connections: Option, pub min_connections: Option, -} \ No newline at end of file +} diff --git a/backend/spi/spi-stats/src/event.rs b/backend/spi/spi-stats/src/event.rs index 692340074..8bac73654 100644 --- a/backend/spi/spi-stats/src/event.rs +++ b/backend/spi/spi-stats/src/event.rs @@ -1,7 +1,4 @@ -use crate::{ - get_tardis_inst, - serv::{stats_record_serv}, -}; +use crate::{get_tardis_inst, serv::stats_record_serv}; use bios_sdk_invoke::clients::{ event_client::{get_topic, mq_error, ContextHandler, SPI_RPC_TOPIC}, spi_log_client::{StatsItemAddReq, StatsItemDeleteReq}, diff --git a/backend/spi/spi-stats/src/serv.rs b/backend/spi/spi-stats/src/serv.rs index 455f23813..b10bdf4f9 100644 --- a/backend/spi/spi-stats/src/serv.rs +++ b/backend/spi/spi-stats/src/serv.rs @@ -1,4 +1,5 @@ pub mod pg; +pub mod stats_conf_dim_group_serv; pub mod stats_conf_dim_serv; pub mod stats_conf_fact_col_serv; pub mod stats_conf_fact_serv; diff --git a/backend/spi/spi-stats/src/serv/pg.rs b/backend/spi/spi-stats/src/serv/pg.rs index 5f96cf5e7..e31c65986 100644 --- a/backend/spi/spi-stats/src/serv/pg.rs +++ b/backend/spi/spi-stats/src/serv/pg.rs @@ -1,3 +1,4 @@ +pub mod stats_pg_conf_dim_group_serv; pub mod stats_pg_conf_dim_serv; pub mod stats_pg_conf_fact_col_serv; pub mod stats_pg_conf_fact_serv; diff --git a/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_dim_group_serv.rs b/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_dim_group_serv.rs new file mode 100644 index 000000000..af1bb5c54 --- /dev/null +++ b/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_dim_group_serv.rs @@ -0,0 +1,168 @@ +use crate::dto::stats_conf_dto::{StatsConfDimGroupAddReq, StatsConfDimGroupInfoResp, StatsConfDimGroupModifyReq}; +use bios_basic::spi::{spi_funs::SpiBsInst, spi_initializer::common_pg::package_table_name}; +use tardis::{ + basic::{dto::TardisContext, result::TardisResult}, + db::{ + reldb_client::{TardisRelDBClient, TardisRelDBlConnection}, + sea_orm::Value, + }, + web::web_resp::TardisPage, + TardisFunsInst, +}; + +use super::stats_pg_initializer; + +pub(crate) async fn add(add_req: &StatsConfDimGroupAddReq, funs: &TardisFunsInst, ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult<()> { + let bs_inst = inst.inst::(); + let (mut conn, table_name) = stats_pg_initializer::init_conf_dim_group_table_and_conn(bs_inst, ctx, true).await?; + conn.begin().await?; + + if conn.count_by_sql(&format!("SELECT 1 FROM {table_name} WHERE key = $1"), vec![Value::from(&add_req.key)]).await? != 0 { + return Err(funs.err().conflict( + "dim_group", + "add", + "The dimension group already exists, please delete it and then add it.", + "409-spi-stats-dim-group-exist", + )); + } + + let params = vec![ + Value::from(add_req.key.to_string()), + Value::from(add_req.show_name.clone()), + Value::from(add_req.data_type.to_string()), + Value::from(add_req.remark.as_ref().unwrap_or(&"".to_string()).as_str()), + Value::from(add_req.dynamic_url.as_deref()), + Value::from(add_req.rel_attribute_code.as_ref().unwrap_or(&vec![]).clone()), + Value::from(add_req.rel_attribute_url.as_deref()), + ]; + + conn.execute_one( + &format!("INSERT INTO {table_name} (key, show_name, data_type, remark, dynamic_url, rel_attribute_code, rel_attribute_url) VALUES ($1, $2, $3, $4, $5, $6, $7)"), + params, + ) + .await?; + + conn.commit().await?; + Ok(()) +} + +pub(crate) async fn modify(dim_conf_key: &str, modify_req: &StatsConfDimGroupModifyReq, funs: &TardisFunsInst, ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult<()> { + let bs_inst = inst.inst::(); + let (mut conn, table_name) = stats_pg_initializer::init_conf_dim_group_table_and_conn(bs_inst, ctx, true).await?; + conn.begin().await?; + + let mut sql_sets = vec![]; + let mut params = vec![Value::from(dim_conf_key.to_string())]; + + if let Some(show_name) = &modify_req.show_name { + sql_sets.push(format!("show_name = ${}", params.len() + 1)); + params.push(Value::from(show_name.to_string())); + } + if let Some(data_type) = &modify_req.data_type { + sql_sets.push(format!("data_type = ${}", params.len() + 1)); + params.push(Value::from(data_type.to_string())); + } + if let Some(remark) = &modify_req.remark { + sql_sets.push(format!("remark = ${}", params.len() + 1)); + params.push(Value::from(remark.to_string())); + } + if let Some(dynamic_url) = &modify_req.dynamic_url { + sql_sets.push(format!("dynamic_url = ${}", params.len() + 1)); + params.push(Value::from(dynamic_url.to_string())); + } + if let Some(rel_attribute_code) = &modify_req.rel_attribute_code { + sql_sets.push(format!("rel_attribute_code = ${}", params.len() + 1)); + params.push(Value::from(rel_attribute_code.clone())); + } + if let Some(rel_attribute_url) = &modify_req.rel_attribute_url { + sql_sets.push(format!("rel_attribute_url = ${}", params.len() + 1)); + params.push(Value::from(rel_attribute_url.clone())); + } + + conn.execute_one(&format!("UPDATE {table_name} SET {} WHERE key = $1", sql_sets.join(", ")), params).await?; + conn.commit().await?; + Ok(()) +} + +pub(crate) async fn paginate( + page_number: u32, + page_size: u32, + desc_by_create: Option, + desc_by_update: Option, + _funs: &TardisFunsInst, + ctx: &TardisContext, + inst: &SpiBsInst, +) -> TardisResult> { + let bs_inst = inst.inst::(); + let (mut conn, _) = stats_pg_initializer::init_conf_dim_group_table_and_conn(bs_inst, ctx, true).await?; + conn.begin().await?; + + do_paginate(page_number, page_size, desc_by_create, desc_by_update, &conn, ctx, inst).await +} + +async fn do_paginate( + page_number: u32, + page_size: u32, + desc_by_create: Option, + desc_by_update: Option, + conn: &TardisRelDBlConnection, + ctx: &TardisContext, + _inst: &SpiBsInst, +) -> TardisResult> { + let table_name = package_table_name("stats_conf_dim_group", ctx); + let sql_where = vec!["1 = 1".to_string()]; + let mut sql_order= vec![]; + let params: Vec = vec![Value::from(page_size), Value::from((page_number - 1) * page_size)]; + + if let Some(desc_by_create) = desc_by_create { + sql_order.push(format!("create_time {}", if desc_by_create { "DESC" } else { "ASC" })); + } + if let Some(desc_by_update) = desc_by_update { + sql_order.push(format!("update_time {}", if desc_by_update { "DESC" } else { "ASC" })); + } + + let result = conn + .query_all( + &format!( + r#"SELECT key, show_name, data_type, remark, dynamic_url, rel_attribute_code, rel_attribute_url, create_time, update_time, count(*) OVER() AS total +FROM {table_name} +WHERE + {} + {} + LIMIT $1 OFFSET $2"#, + sql_where.join(" AND "), + if sql_order.is_empty() { + "".to_string() + } else { + format!("ORDER BY {}", sql_order.join(",")) + } + ), + params, + ) + .await?; + + let mut total_size: i64 = 0; + let mut final_result = vec![]; + for item in result { + if total_size == 0 { + total_size = item.try_get("", "total")?; + } + final_result.push(StatsConfDimGroupInfoResp { + key: item.try_get("", "key")?, + show_name: item.try_get("", "show_name")?, + data_type: item.try_get("", "data_type")?, + remark: item.try_get("", "remark")?, + dynamic_url: item.try_get("", "dynamic_url")?, + rel_attribute_code: item.try_get("", "rel_attribute_code")?, + rel_attribute_url: item.try_get("", "rel_attribute_url")?, + create_time: item.try_get("", "create_time")?, + update_time: item.try_get("", "update_time")?, + }); + } + Ok(TardisPage { + page_size: page_size as u64, + page_number: page_number as u64, + total_size: total_size as u64, + records: final_result, + }) +} diff --git a/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_dim_serv.rs b/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_dim_serv.rs index 98fd74806..1d43a7037 100644 --- a/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_dim_serv.rs +++ b/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_dim_serv.rs @@ -41,6 +41,7 @@ pub(crate) async fn add(add_req: &StatsConfDimAddReq, funs: &TardisFunsInst, ctx Value::from(add_req.data_type.to_string()), Value::from(add_req.hierarchy.as_ref().unwrap_or(&vec![]).clone()), Value::from(add_req.remark.as_ref().unwrap_or(&"".to_string()).as_str()), + Value::from(add_req.dim_group_key.as_deref()), Value::from(add_req.dynamic_url.as_deref()), Value::from(add_req.is_tree.unwrap_or(false)), Value::from(add_req.tree_dynamic_url.as_deref()), @@ -51,9 +52,9 @@ pub(crate) async fn add(add_req: &StatsConfDimAddReq, funs: &TardisFunsInst, ctx conn.execute_one( &format!( r#"INSERT INTO {table_name} -(key, show_name, stable_ds, data_type, hierarchy, remark, dynamic_url, is_tree, tree_dynamic_url, rel_attribute_code, rel_attribute_url) +(key, show_name, stable_ds, data_type, hierarchy, remark, dim_group_key, dynamic_url, is_tree, tree_dynamic_url, rel_attribute_code, rel_attribute_url) VALUES -($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) +($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) "#, ), params, @@ -97,6 +98,10 @@ pub(crate) async fn modify(dim_conf_key: &str, modify_req: &StatsConfDimModifyRe sql_sets.push(format!("remark = ${}", params.len() + 1)); params.push(Value::from(remark.to_string())); } + if let Some(dim_group_key) = &modify_req.dim_group_key { + sql_sets.push(format!("dim_group_key = ${}", params.len() + 1)); + params.push(Value::from(dim_group_key)); + } if let Some(dynamic_url) = &modify_req.dynamic_url { sql_sets.push(format!("dynamic_url = ${}", params.len() + 1)); params.push(Value::from(dynamic_url)); @@ -159,12 +164,21 @@ pub(crate) async fn delete(dim_conf_key: &str, funs: &TardisFunsInst, ctx: &Tard Ok(()) } -pub(in crate::serv::pg) async fn get(dim_conf_key: &str, conn: &TardisRelDBlConnection, ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult> { - do_paginate(Some(dim_conf_key.to_string()), None, 1, 1, None, None, conn, ctx, inst).await.map(|page| page.records.into_iter().next()) +pub(in crate::serv::pg) async fn get( + dim_conf_key: &str, + dim_group_key: Option, + dim_group_is_empty: Option, + conn: &TardisRelDBlConnection, + ctx: &TardisContext, + inst: &SpiBsInst, +) -> TardisResult> { + do_paginate(Some(dim_conf_key.to_string()), dim_group_key, dim_group_is_empty, None, 1, 1, None, None, conn, ctx, inst).await.map(|page| page.records.into_iter().next()) } pub(crate) async fn paginate( dim_conf_key: Option, + dim_group_key: Option, + dim_group_is_empty: Option, show_name: Option, page_number: u32, page_size: u32, @@ -176,11 +190,26 @@ pub(crate) async fn paginate( ) -> TardisResult> { let bs_inst = inst.inst(); let (conn, _) = stats_pg_initializer::init_conf_dim_table_and_conn(bs_inst, ctx, true).await?; - do_paginate(dim_conf_key, show_name, page_number, page_size, desc_by_create, desc_by_update, &conn, ctx, inst).await + do_paginate( + dim_conf_key, + dim_group_key, + dim_group_is_empty, + show_name, + page_number, + page_size, + desc_by_create, + desc_by_update, + &conn, + ctx, + inst, + ) + .await } async fn do_paginate( dim_conf_key: Option, + dim_group_key: Option, + dim_group_is_empty: Option, show_name: Option, page_number: u32, page_size: u32, @@ -198,6 +227,17 @@ async fn do_paginate( sql_where.push(format!("key = ${}", params.len() + 1)); params.push(Value::from(dim_conf_key.to_string())); } + if let Some(dim_group_key) = &dim_group_key { + sql_where.push(format!("dim_group_key = ${}", params.len() + 1)); + params.push(Value::from(dim_group_key.to_string())); + } else { + if let Some(dim_group_is_empty) = &dim_group_is_empty { + if *dim_group_is_empty { + sql_where.push("dim_group_key = ''".to_string()); + } + } + } + if let Some(show_name) = &show_name { sql_where.push(format!("show_name LIKE ${}", params.len() + 1)); params.push(Value::from(format!("%{show_name}%"))); @@ -212,7 +252,7 @@ async fn do_paginate( let result = conn .query_all( &format!( - r#"SELECT key, show_name, stable_ds, data_type, hierarchy, remark, dynamic_url, is_tree, tree_dynamic_url, rel_attribute_code, rel_attribute_url, create_time, update_time, count(*) OVER() AS total + r#"SELECT key, show_name, stable_ds, data_type, hierarchy, remark, dim_group_key, dynamic_url, is_tree, tree_dynamic_url, rel_attribute_code, rel_attribute_url, create_time, update_time, count(*) OVER() AS total FROM {table_name} WHERE {} @@ -244,6 +284,7 @@ WHERE remark: item.try_get("", "remark")?, create_time: item.try_get("", "create_time")?, update_time: item.try_get("", "update_time")?, + dim_group_key: item.try_get("", "dim_group_key")?, online: online(&item.try_get::("", "key")?, conn, ctx).await?, dynamic_url: item.try_get("", "dynamic_url")?, is_tree: item.try_get("", "is_tree")?, @@ -291,7 +332,7 @@ pub(crate) async fn create_inst(dim_conf_key: &str, funs: &TardisFunsInst, ctx: let (mut conn, _) = common_pg::init_conn(bs_inst).await?; conn.begin().await?; - let dim_conf = get(dim_conf_key, &conn, ctx, inst) + let dim_conf = get(dim_conf_key, None, None, &conn, ctx, inst) .await? .ok_or_else(|| funs.err().not_found("fact_conf", "create_inst", "The dimension config does not exist.", "404-spi-stats-dim-conf-not-exist"))?; diff --git a/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_fact_col_serv.rs b/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_fact_col_serv.rs index 2d77ac149..3c369ebe6 100644 --- a/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_fact_col_serv.rs +++ b/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_fact_col_serv.rs @@ -317,13 +317,14 @@ pub(crate) async fn find_by_fact_conf_key(fact_conf_key: &str, _funs: &TardisFun if !common_pg::check_table_exit("stats_conf_fact_col", &conn, ctx).await? { return Ok(vec![]); } - do_paginate(Some(fact_conf_key.to_string()), None, None, None, None, 1, u32::MAX, None, None, &conn, ctx).await.map(|page| page.records) + do_paginate(Some(fact_conf_key.to_string()), None, None, None, None, None, 1, u32::MAX, None, None, &conn, ctx).await.map(|page| page.records) } pub(crate) async fn paginate( fact_conf_key: Option, fact_col_conf_key: Option, dim_key: Option, + dim_group_key: Option, show_name: Option, rel_external_id: Option, page_number: u32, @@ -341,6 +342,7 @@ pub(crate) async fn paginate( fact_conf_key, fact_col_conf_key, dim_key, + dim_group_key, show_name, rel_external_id, page_number, @@ -357,6 +359,7 @@ async fn do_paginate( fact_conf_key: Option, fact_col_conf_key: Option, dim_key: Option, + dim_group_key: Option, show_name: Option, rel_external_id: Option, page_number: u32, @@ -402,24 +405,49 @@ async fn do_paginate( sql_order.push(format!("update_time {}", if desc_by_update { "DESC" } else { "ASC" })); } - let result = conn - .query_all( - &format!( - r#"SELECT key, show_name, kind, remark, dim_rel_conf_dim_key, rel_external_id, dim_multi_values, dim_exclusive_rec, dim_data_type, dim_dynamic_url, mes_data_distinct, mes_data_type, mes_frequency, mes_unit, mes_act_by_dim_conf_keys, rel_conf_fact_key, rel_conf_fact_and_col_key, create_time, update_time,rel_field,rel_cert_id,rel_sql, count(*) OVER() AS total + let result; + if let Some(dim_group_key) = &dim_group_key { + sql_where.push(format!("starsys_stats_conf_dim.dim_group_key = ${}", params.len() + 1)); + params.push(Value::from(dim_group_key)); + + result = conn + .query_all( + &format!( + r#"SELECT {table_name}.key, {table_name}.show_name, {table_name}.kind, {table_name}.remark, {table_name}.dim_rel_conf_dim_key, {table_name}.rel_external_id, {table_name}.dim_multi_values, {table_name}.dim_exclusive_rec, {table_name}.dim_data_type, {table_name}.dim_dynamic_url, {table_name}.mes_data_distinct, {table_name}.mes_data_type, {table_name}.mes_frequency, {table_name}.mes_unit, {table_name}.mes_act_by_dim_conf_keys, {table_name}.rel_conf_fact_key, {table_name}.rel_conf_fact_and_col_key, {table_name}.create_time, {table_name}.update_time, {table_name}.rel_field, {table_name}.rel_cert_id, {table_name}.rel_sql, count(*) OVER() AS total +FROM {table_name} inner join starsys_stats_conf_dim on {table_name}.dim_rel_conf_dim_key = starsys_stats_conf_dim.key +WHERE + {} +{}"#, + sql_where.join(" AND "), + if sql_order.is_empty() { + "".to_string() + } else { + format!("ORDER BY {}", sql_order.join(",")) + } + ), + params, + ) + .await?; + } else { + result = conn + .query_all( + &format!( + r#"SELECT key, show_name, kind, remark, dim_rel_conf_dim_key, rel_external_id, dim_multi_values, dim_exclusive_rec, dim_data_type, dim_dynamic_url, mes_data_distinct, mes_data_type, mes_frequency, mes_unit, mes_act_by_dim_conf_keys, rel_conf_fact_key, rel_conf_fact_and_col_key, create_time, update_time,rel_field,rel_cert_id,rel_sql, count(*) OVER() AS total FROM {table_name} WHERE - {} + {} {}"#, - sql_where.join(" AND "), - if sql_order.is_empty() { - "".to_string() - } else { - format!("ORDER BY {}", sql_order.join(",")) - } - ), - params, - ) - .await?; + sql_where.join(" AND "), + if sql_order.is_empty() { + "".to_string() + } else { + format!("ORDER BY {}", sql_order.join(",")) + } + ), + params, + ) + .await?; + } let mut total_size: i64 = 0; let result = result diff --git a/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_fact_serv.rs b/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_fact_serv.rs index e9066510f..0379fccaa 100644 --- a/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_fact_serv.rs +++ b/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_fact_serv.rs @@ -46,13 +46,13 @@ pub(crate) async fn add(add_req: &StatsConfFactAddReq, funs: &TardisFunsInst, ct Value::from(add_req.rel_cert_id.as_ref().unwrap_or(&"".to_string()).as_str()), Value::from(add_req.sync_sql.as_ref().unwrap_or(&"".to_string()).as_str()), Value::from(add_req.sync_cron.as_ref().unwrap_or(&"".to_string()).as_str()), - Value::from(add_req.sync_on.unwrap_or_default()), + Value::from(add_req.is_sync.unwrap_or_default()), ]; conn.execute_one( &format!( r#"INSERT INTO {table_name} -(key, show_name, query_limit, remark, redirect_path, is_online, rel_cert_id, sync_sql, sync_cron, sync_on) +(key, show_name, query_limit, remark, redirect_path, is_online, rel_cert_id, sync_sql, sync_cron, is_sync) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) "#, @@ -108,9 +108,9 @@ pub(crate) async fn modify(fact_conf_key: &str, modify_req: &StatsConfFactModify sql_sets.push(format!("sync_cron = ${}", params.len() + 1)); params.push(Value::from(sync_cron.to_string())); } - if let Some(sync_on) = &modify_req.sync_on { - sql_sets.push(format!("sync_on = ${}", params.len() + 1)); - params.push(Value::from(*sync_on)); + if let Some(is_sync) = &modify_req.is_sync { + sql_sets.push(format!("is_sync = ${}", params.len() + 1)); + params.push(Value::from(*is_sync)); } }; @@ -251,7 +251,7 @@ async fn do_paginate( .query_all( &format!( r#"SELECT t.*, count(*) OVER () AS total FROM ( -SELECT distinct fact.key as key, fact.show_name as show_name, fact.query_limit as query_limit, fact.remark as remark, fact.redirect_path as redirect_path, fact.is_online as is_online, fact.rel_cert_id as rel_cert_id, fact.sync_sql as sync_sql, fact.sync_cron as sync_cron, fact.sync_on as sync_on, fact.create_time as create_time, fact.update_time as update_time +SELECT distinct fact.key as key, fact.show_name as show_name, fact.query_limit as query_limit, fact.remark as remark, fact.redirect_path as redirect_path, fact.is_online as is_online, fact.rel_cert_id as rel_cert_id, fact.sync_sql as sync_sql, fact.sync_cron as sync_cron, fact.is_sync as is_sync, fact.create_time as create_time, fact.update_time as update_time FROM {table_name} as fact {} WHERE @@ -291,7 +291,7 @@ LIMIT $1 OFFSET $2 rel_cert_id: item.try_get("", "rel_cert_id")?, sync_sql: item.try_get("", "sync_sql")?, sync_cron: item.try_get("", "sync_cron")?, - sync_on: item.try_get("", "sync_on")?, + is_sync: item.try_get("", "is_sync")?, }); } Ok(TardisPage { @@ -394,7 +394,7 @@ async fn create_inst_table( "409-spi-stats-dim-conf-not-online", )); } - let Some(dim_conf) = stats_pg_conf_dim_serv::get(dim_conf_key, conn, ctx, inst).await? else { + let Some(dim_conf) = stats_pg_conf_dim_serv::get(dim_conf_key, None, None, conn, ctx, inst).await? else { return Err(funs.err().conflict( "fact_inst", "create", diff --git a/backend/spi/spi-stats/src/serv/pg/stats_pg_initializer.rs b/backend/spi/spi-stats/src/serv/pg/stats_pg_initializer.rs index e6139cbb3..5b24c6a16 100644 --- a/backend/spi/spi-stats/src/serv/pg/stats_pg_initializer.rs +++ b/backend/spi/spi-stats/src/serv/pg/stats_pg_initializer.rs @@ -4,6 +4,29 @@ use tardis::{ db::reldb_client::{TardisRelDBClient, TardisRelDBlConnection}, }; +pub async fn init_conf_dim_group_table_and_conn(bs_inst: TypedSpiBsInst<'_, TardisRelDBClient>, ctx: &TardisContext, mgr: bool) -> TardisResult<(TardisRelDBlConnection, String)> { + spi_initializer::common_pg::init_table_and_conn( + bs_inst, + ctx, + mgr, + None, + "stats_conf_dim_group", + r#"key character varying NOT NULL, + show_name character varying NOT NULL, + data_type character varying NOT NULL, + remark character varying NOT NULL, + dynamic_url character varying NOT NULL, + rel_attribute_code character varying[], + rel_attribute_url character varying, + create_time timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP"#, + None, + vec![], + None, + Some("update_time"), + ) + .await +} pub async fn init_conf_dim_table_and_conn(bs_inst: TypedSpiBsInst<'_, TardisRelDBClient>, ctx: &TardisContext, mgr: bool) -> TardisResult<(TardisRelDBlConnection, String)> { spi_initializer::common_pg::init_table_and_conn( bs_inst, @@ -19,6 +42,7 @@ pub async fn init_conf_dim_table_and_conn(bs_inst: TypedSpiBsInst<'_, TardisRelD remark character varying NOT NULL, dynamic_url character varying, is_tree boolean NOT NULL DEFAULT FALSE, + dim_group_key character varying NOT NULL, tree_dynamic_url character varying, rel_attribute_code character varying[], rel_attribute_url character varying, @@ -50,7 +74,7 @@ pub async fn init_conf_fact_table_and_conn(bs_inst: TypedSpiBsInst<'_, TardisRel rel_cert_id character varying, sync_sql character varying, sync_cron character varying, - sync_on boolean NOT NULL DEFAULT FALSE"#, + is_sync boolean NOT NULL DEFAULT FALSE"#, None, vec![], None, diff --git a/backend/spi/spi-stats/src/serv/pg/stats_pg_record_serv.rs b/backend/spi/spi-stats/src/serv/pg/stats_pg_record_serv.rs index bcf534a73..d73ec53a3 100644 --- a/backend/spi/spi-stats/src/serv/pg/stats_pg_record_serv.rs +++ b/backend/spi/spi-stats/src/serv/pg/stats_pg_record_serv.rs @@ -201,7 +201,7 @@ pub(crate) async fn fact_record_load( let Some(key) = fact_col_conf.dim_rel_conf_dim_key.as_ref() else { return Err(funs.err().not_found("fact_record", "load", "Fail to get conf_dim_key", "400-spi-stats-fail-to-get-dim-config-key")); }; - let Some(dim_conf) = stats_pg_conf_dim_serv::get(key, &conn, ctx, inst).await? else { + let Some(dim_conf) = stats_pg_conf_dim_serv::get(key, None, None, &conn, ctx, inst).await? else { return Err(funs.err().not_found( "fact_record", "load", @@ -255,7 +255,7 @@ pub(crate) async fn fact_record_load( let Some(dim_rel_conf_dim_key) = &fact_col_conf.dim_rel_conf_dim_key else { return Err(funs.err().internal_error("fact_record", "load", "dim_rel_conf_dim_key unexpectedly being empty", "500-spi-stats-internal-error")); }; - let Some(dim_conf) = stats_pg_conf_dim_serv::get(dim_rel_conf_dim_key, &conn, ctx, inst).await? else { + let Some(dim_conf) = stats_pg_conf_dim_serv::get(dim_rel_conf_dim_key, None, None, &conn, ctx, inst).await? else { return Err(funs.err().internal_error( "fact_record", "load", @@ -295,7 +295,7 @@ pub(crate) async fn fact_record_load( let Some(dim_rel_conf_dim_key) = &fact_col_conf.dim_rel_conf_dim_key else { return Err(funs.err().internal_error("fact_record", "load", "dim_rel_conf_dim_key unexpectedly being empty", "500-spi-stats-internal-error")); }; - let Some(dim_conf) = stats_pg_conf_dim_serv::get(dim_rel_conf_dim_key, &conn, ctx, inst).await? else { + let Some(dim_conf) = stats_pg_conf_dim_serv::get(dim_rel_conf_dim_key, None, None, &conn, ctx, inst).await? else { return Err(funs.err().internal_error( "fact_record", "load", @@ -419,7 +419,7 @@ pub(crate) async fn fact_records_load( let Some(key) = fact_col_conf.dim_rel_conf_dim_key.as_ref() else { return Err(funs.err().not_found("fact_record", "load_set", "Fail to get conf_dim_key", "400-spi-stats-fail-to-get-dim-config-key")); }; - let Some(dim_conf) = stats_pg_conf_dim_serv::get(key, &conn, ctx, inst).await? else { + let Some(dim_conf) = stats_pg_conf_dim_serv::get(key, None, None, &conn, ctx, inst).await? else { return Err(funs.err().not_found( "fact_record", "load_set", @@ -505,7 +505,7 @@ async fn fact_records_modify( let Some(key) = fact_col_conf.dim_rel_conf_dim_key.as_ref() else { return Err(funs.err().not_found("fact_record", "load", "Fail to get conf_dim_key", "400-spi-stats-fail-to-get-dim-config-key")); }; - let Some(dim_conf) = stats_pg_conf_dim_serv::get(key, &conn, ctx, inst).await? else { + let Some(dim_conf) = stats_pg_conf_dim_serv::get(key, None, None, &conn, ctx, inst).await? else { return Err(funs.err().not_found( "fact_record", "load", @@ -679,7 +679,7 @@ async fn find_fact_record_key( ctx: &TardisContext, inst: &SpiBsInst, ) -> TardisResult> { - let dim_conf = stats_pg_conf_dim_serv::get(&dim_conf_key, conn, ctx, inst) + let dim_conf = stats_pg_conf_dim_serv::get(&dim_conf_key, None, None, conn, ctx, inst) .await? .ok_or_else(|| funs.err().not_found("fact_record", "find", "The dimension config does not exist.", "404-spi-stats-dim-conf-not-exist"))?; let fact_conf_col_key = stats_pg_conf_fact_col_serv::find_by_fact_conf_key(&fact_conf_key, funs, ctx, inst) @@ -741,7 +741,7 @@ pub(crate) async fn dim_record_add(dim_conf_key: String, add_req: StatsDimRecord if !stats_pg_conf_dim_serv::online(&dim_conf_key, &conn, ctx).await? { return Err(funs.err().conflict("dim_record", "add", "The dimension config not online.", "409-spi-stats-dim-conf-not-online")); } - let dim_conf = stats_pg_conf_dim_serv::get(&dim_conf_key, &conn, ctx, inst).await?.expect("Fail to get dim_conf"); + let dim_conf = stats_pg_conf_dim_serv::get(&dim_conf_key, None, None, &conn, ctx, inst).await?.expect("Fail to get dim_conf"); if !dim_conf.stable_ds { return Err(funs.err().bad_request( "dim_record", @@ -881,7 +881,7 @@ async fn dim_do_record_paginate( ctx: &TardisContext, inst: &SpiBsInst, ) -> TardisResult> { - let dim_conf = stats_pg_conf_dim_serv::get(&dim_conf_key, conn, ctx, inst) + let dim_conf = stats_pg_conf_dim_serv::get(&dim_conf_key, None, None, conn, ctx, inst) .await? .ok_or_else(|| funs.err().not_found("dim_record", "find", "The dimension config does not exist.", "404-spi-stats-dim-conf-not-exist"))?; @@ -948,7 +948,7 @@ pub(crate) async fn dim_record_delete(dim_conf_key: String, dim_record_key: serd if !stats_pg_conf_dim_serv::online(&dim_conf_key, &conn, ctx).await? { return Err(funs.err().conflict("dim_record", "delete", "The dimension config not online.", "409-spi-stats-dim-conf-not-online")); } - let dim_conf = stats_pg_conf_dim_serv::get(&dim_conf_key, &conn, ctx, inst).await?.expect("Fail to get dim_conf"); + let dim_conf = stats_pg_conf_dim_serv::get(&dim_conf_key, None, None, &conn, ctx, inst).await?.expect("Fail to get dim_conf"); let table_name = package_table_name(&format!("stats_inst_dim_{}", dim_conf.key), ctx); let values = vec![dim_conf.data_type.json_to_sea_orm_value(&dim_record_key, false)?]; @@ -980,7 +980,7 @@ pub(crate) async fn dim_record_real_delete( if !stats_pg_conf_dim_serv::online(&dim_conf_key, &conn, ctx).await? { return Err(funs.err().conflict("dim_record", "delete", "The dimension config not online.", "409-spi-stats-dim-conf-not-online")); } - let dim_conf = stats_pg_conf_dim_serv::get(&dim_conf_key, &conn, ctx, inst).await?.expect("Fail to get dim_conf"); + let dim_conf = stats_pg_conf_dim_serv::get(&dim_conf_key, None, None, &conn, ctx, inst).await?.expect("Fail to get dim_conf"); let table_name = package_table_name(&format!("stats_inst_dim_{}", dim_conf.key), ctx); let values = vec![dim_conf.data_type.json_to_sea_orm_value(&dim_record_key, false)?]; diff --git a/backend/spi/spi-stats/src/serv/pg/stats_pg_sync_serv.rs b/backend/spi/spi-stats/src/serv/pg/stats_pg_sync_serv.rs index 938d0cae1..781839434 100644 --- a/backend/spi/spi-stats/src/serv/pg/stats_pg_sync_serv.rs +++ b/backend/spi/spi-stats/src/serv/pg/stats_pg_sync_serv.rs @@ -11,12 +11,17 @@ use bios_basic::{ }; use tardis::{ basic::{dto::TardisContext, field::TrimString, result::TardisResult}, - db::{reldb_client::TardisRelDBClient, sea_orm::Value}, + config::config_dto::{CompatibleType, DBModuleConfig}, + db::{ + reldb_client::{TardisRelDBClient, TardisRelDBlConnection}, + sea_orm::Value, + }, TardisFunsInst, }; use crate::{ - dto::stats_conf_dto::{StatsSyncDbConfigAddReq, StatsSyncDbConfigExt, StatsSyncDbConfigInfoResp, StatsSyncDbConfigInfoWithSkResp, StatsSyncDbConfigModifyReq}, stats_constants::DOMAIN_CODE, + dto::stats_conf_dto::{StatsSyncDbConfigAddReq, StatsSyncDbConfigExt, StatsSyncDbConfigInfoResp, StatsSyncDbConfigInfoWithSkResp, StatsSyncDbConfigModifyReq}, + stats_constants::DOMAIN_CODE, }; pub(crate) async fn db_config_add(add_req: StatsSyncDbConfigAddReq, funs: &TardisFunsInst, ctx: &TardisContext, _inst: &SpiBsInst) -> TardisResult { @@ -28,7 +33,11 @@ pub(crate) async fn db_config_add(add_req: StatsSyncDbConfigAddReq, funs: &Tardi rel_rbum_id: "".to_string(), kind: Some(SPI_PG_KIND_CODE.to_string()), supplier: Some(DOMAIN_CODE.to_string()), - ext: serde_json::to_string(&StatsSyncDbConfigExt{ max_connections: add_req.max_connections, min_connections: add_req.min_connections }).ok(), + ext: serde_json::to_string(&StatsSyncDbConfigExt { + max_connections: add_req.max_connections, + min_connections: add_req.min_connections, + }) + .ok(), sk_invisible: None, ignore_check_sk: false, start_time: None, @@ -90,10 +99,19 @@ pub(crate) async fn db_config_list(funs: &TardisFunsInst, ctx: &TardisContext, i ) .await?; - return Ok(rbum_cert_list.iter().map(|rbum_cert| { - let ext = serde_json::from_str::(&rbum_cert.ext).ok(); - StatsSyncDbConfigInfoResp { id: rbum_cert.id.clone(), db_url: rbum_cert.conn_uri.clone(), db_user: rbum_cert.ak.clone(), max_connections: ext.clone().and_then(|ext| ext.max_connections), min_connections: ext.clone().and_then(|ext| ext.min_connections) } - }).collect()); + return Ok(rbum_cert_list + .iter() + .map(|rbum_cert| { + let ext = serde_json::from_str::(&rbum_cert.ext).ok(); + StatsSyncDbConfigInfoResp { + id: rbum_cert.id.clone(), + db_url: rbum_cert.conn_uri.clone(), + db_user: rbum_cert.ak.clone(), + max_connections: ext.clone().and_then(|ext| ext.max_connections), + min_connections: ext.clone().and_then(|ext| ext.min_connections), + } + }) + .collect()); } async fn find_db_config(cert_id: &str, funs: &TardisFunsInst, ctx: &TardisContext, _inst: &SpiBsInst) -> TardisResult { @@ -116,43 +134,85 @@ async fn find_db_config(cert_id: &str, funs: &TardisFunsInst, ctx: &TardisContex let ext = serde_json::from_str::(&rbum_cert.ext).ok(); let max_connections = ext.clone().and_then(|ext| ext.max_connections); let min_connections = ext.clone().and_then(|ext| ext.min_connections); - return Ok(StatsSyncDbConfigInfoWithSkResp { id: cert_id.to_string(), db_url: rbum_cert.conn_uri.clone(), db_user: rbum_cert.ak.clone(), db_password: db_password, max_connections, min_connections }); + return Ok(StatsSyncDbConfigInfoWithSkResp { + id: cert_id.to_string(), + db_url: rbum_cert.conn_uri.clone(), + db_user: rbum_cert.ak.clone(), + db_password: db_password, + max_connections, + min_connections, + }); } else { return Err(funs.err().not_found(&RbumCertServ::get_obj_name(), "find", "rbum cert not found", "404-rbum-cert-not-found")); } } pub(crate) async fn fact_record_sync(fact_key: &str, funs: &TardisFunsInst, ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult<()> { - // let bs_inst = inst.inst::(); - // let (conn, _) = common_pg::init_conn(bs_inst).await?; + let bs_inst = inst.inst::(); + let (mut conn, _) = common_pg::init_conn(bs_inst).await?; - // conn.begin().await?; + conn.begin().await?; - // todo!(); - // let fact_col_list = conn - // .query_all( - // &format!("SELECT key FROM starsys_stats_conf_fact_col WHERE rel_conf_fact_key = $1"), - // vec![Value::from(fact_key)], - // ) - // .await?; - // for col in fact_col_list.iter() { - // let col_key = col.try_get::("", "key")?; - // fact_col_record_sync(fact_key, &col_key, funs, ctx, inst).await?; - // } - // conn.commit().await?; + todo!(); + let fact_col_list = conn + .query_all( + &format!("SELECT key FROM starsys_stats_conf_fact_col WHERE rel_conf_fact_key = $1"), + vec![Value::from(fact_key)], + ) + .await?; + for col in fact_col_list.iter() { + let col_key = col.try_get::("", "key")?; + do_fact_col_record_sync(fact_key, &col_key, &mut conn, funs, ctx, inst).await?; + } + conn.commit().await?; Ok(()) } pub(crate) async fn fact_col_record_sync(fact_key: &str, col_key: &str, funs: &TardisFunsInst, ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult<()> { let bs_inst = inst.inst::(); - let (conn, _) = common_pg::init_conn(bs_inst).await?; + let (mut conn, _) = common_pg::init_conn(bs_inst).await?; - todo!() + do_fact_col_record_sync(fact_key, col_key, &mut conn, funs, ctx, inst).await } -async fn do_fact_col_record_sync(fact_key: &str, col_key: &str, funs: &TardisFunsInst, ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult<()> { - let bs_inst = inst.inst::(); - let (conn, _) = common_pg::init_conn(bs_inst).await?; - - todo!() +pub(crate) async fn do_fact_col_record_sync( + fact_key: &str, + col_key: &str, + conn: &mut TardisRelDBlConnection, + funs: &TardisFunsInst, + ctx: &TardisContext, + inst: &SpiBsInst, +) -> TardisResult<()> { + if let Some(fact_col) = conn + .query_one( + &format!("SELECT rel_cert_id,rel_sql,rel_field FROM starsys_stats_conf_fact_col WHERE rel_conf_fact_key = $1 AND key = $2"), + vec![Value::from(fact_key), Value::from(col_key)], + ) + .await? + { + if let Some(cert_id) = fact_col.try_get::>("", "rel_cert_id")? { + if let Some(sql) = fact_col.try_get::>("", "rel_sql")? { + if let Some(field) = fact_col.try_get::>("", "rel_field")? { + let db_config = find_db_config(&cert_id, funs, ctx, inst).await?; + let db_client = TardisRelDBClient::init(&DBModuleConfig { + url: format!("postgres://{}:{}@{}", db_config.db_user, db_config.db_password, db_config.db_url), + max_connections: db_config.max_connections.unwrap_or(20), + min_connections: db_config.min_connections.unwrap_or(5), + connect_timeout_sec: None, + idle_timeout_sec: None, + compatible_type: CompatibleType::default(), + }) + .await?; + if let Some(rel_record) = db_client.conn().query_one(&sql, vec![]).await? { + let rel_record_value = rel_record.try_get::("", &field)?; + //TODO 插入数据 + } + } + } + return Ok(()); + } else { + return Err(funs.err().not_found("starsys_stats_conf_fact_col", "find", "fact col not found", "404-fact-col-not-found")); + } + } + Ok(()) } diff --git a/backend/spi/spi-stats/src/serv/stats_conf_dim_group_serv.rs b/backend/spi/spi-stats/src/serv/stats_conf_dim_group_serv.rs new file mode 100644 index 000000000..6ca5c8493 --- /dev/null +++ b/backend/spi/spi-stats/src/serv/stats_conf_dim_group_serv.rs @@ -0,0 +1,28 @@ +use crate::dto::stats_conf_dto::{StatsConfDimGroupAddReq, StatsConfDimGroupInfoResp, StatsConfDimGroupModifyReq}; +use crate::stats_initializer; +use bios_basic::spi::spi_constants; +use bios_basic::spi::spi_funs::SpiBsInstExtractor; +use bios_basic::spi_dispatch_service; +use tardis::basic::result::TardisResult; +use tardis::web::web_resp::TardisPage; + +use super::pg; + +spi_dispatch_service! { + @mgr: true, + @init: stats_initializer::init_fun, + @dispatch: { + #[cfg(feature = "spi-pg")] + spi_constants::SPI_PG_KIND_CODE => pg::stats_pg_conf_dim_group_serv, + }, + @method: { + add(add_req: &StatsConfDimGroupAddReq) -> TardisResult<()>; + modify(dim_conf_key: &str, modify_req: &StatsConfDimGroupModifyReq) -> TardisResult<()>; + paginate( + page_number: u32, + page_size: u32, + desc_by_create: Option, + desc_by_update: Option + ) -> TardisResult>; + } +} diff --git a/backend/spi/spi-stats/src/serv/stats_conf_dim_serv.rs b/backend/spi/spi-stats/src/serv/stats_conf_dim_serv.rs index 48604bf2e..220b7a2ed 100644 --- a/backend/spi/spi-stats/src/serv/stats_conf_dim_serv.rs +++ b/backend/spi/spi-stats/src/serv/stats_conf_dim_serv.rs @@ -21,6 +21,8 @@ spi_dispatch_service! { delete(dim_conf_key: &str) -> TardisResult<()>; paginate( dim_conf_key: Option, + dim_group_key: Option, + dim_group_is_empty: Option, show_name: Option, page_number: u32, page_size: u32, diff --git a/backend/spi/spi-stats/src/serv/stats_conf_fact_col_serv.rs b/backend/spi/spi-stats/src/serv/stats_conf_fact_col_serv.rs index 63b3b535f..f4de5ee31 100644 --- a/backend/spi/spi-stats/src/serv/stats_conf_fact_col_serv.rs +++ b/backend/spi/spi-stats/src/serv/stats_conf_fact_col_serv.rs @@ -36,6 +36,7 @@ spi_dispatch_service! { fact_conf_key: Option, fact_col_conf_key: Option, dim_key: Option, + dim_group_key: Option, show_name: Option, rel_external_id: Option, page_number: u32, From 65bb0ec9e1f693aff718936c67dabf55bcf3a0fc Mon Sep 17 00:00:00 2001 From: RWDai <27391645+RWDai@users.noreply.github.com> Date: Thu, 7 Nov 2024 10:15:31 +0800 Subject: [PATCH 5/6] chores: pin asteroid-mq version --- Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c527093b5..7bc662c54 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,9 +68,9 @@ tardis = { version = "0.1.0-rc.17" } # tardis = { git = "https://github.com/ideal-world/tardis.git", rev = "9cc9b3e" } # asteroid-mq = { git = "https://github.com/4t145/asteroid-mq.git", rev = "d59c64d" } # asteroid-mq = { git = "https://github.com/4t145/asteroid-mq.git", rev = "83a6643" } -asteroid-mq = { version = "0.1.0-alpha.3" } +asteroid-mq = { version = "=0.1.0-alpha.3" } # asteroid-mq-sdk = { git = "https://github.com/4t145/asteroid-mq.git", rev = "83a6643" } -asteroid-mq-sdk = { version = "0.1.0-alpha.3" } +asteroid-mq-sdk = { version = "=0.1.0-alpha.3" } #spacegate # spacegate-shell = { version = "0.2.0", path = "../spacegate/crates/shell", features = [ @@ -78,7 +78,7 @@ asteroid-mq-sdk = { version = "0.1.0-alpha.3" } # "k8s", # "ext-axum", # ] } -spacegate-shell = { version="0.2.0-alpha.2", features = [ +spacegate-shell = { version = "0.2.0-alpha.2", features = [ "cache", "k8s", "ext-axum", From af60bc429f9d6d5109b321ca967e09dcf67b251d Mon Sep 17 00:00:00 2001 From: RWDai <27391645+RWDai@users.noreply.github.com> Date: Thu, 7 Nov 2024 10:35:50 +0800 Subject: [PATCH 6/6] chore: fix cicd --- .../src/serv/pg/stats_pg_conf_dim_group_serv.rs | 2 +- .../sdks/invoke/src/clients/spi_log_client.rs | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_dim_group_serv.rs b/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_dim_group_serv.rs index af1bb5c54..5f7649146 100644 --- a/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_dim_group_serv.rs +++ b/backend/spi/spi-stats/src/serv/pg/stats_pg_conf_dim_group_serv.rs @@ -111,7 +111,7 @@ async fn do_paginate( ) -> TardisResult> { let table_name = package_table_name("stats_conf_dim_group", ctx); let sql_where = vec!["1 = 1".to_string()]; - let mut sql_order= vec![]; + let mut sql_order = vec![]; let params: Vec = vec![Value::from(page_size), Value::from((page_number - 1) * page_size)]; if let Some(desc_by_create) = desc_by_create { diff --git a/frontend/sdks/invoke/src/clients/spi_log_client.rs b/frontend/sdks/invoke/src/clients/spi_log_client.rs index 2136cd482..81fefb3fc 100644 --- a/frontend/sdks/invoke/src/clients/spi_log_client.rs +++ b/frontend/sdks/invoke/src/clients/spi_log_client.rs @@ -1,4 +1,3 @@ -use asteroid_mq::prelude::{EventAttribute, Subject}; use serde::{Deserialize, Serialize}; use tardis::{ @@ -29,6 +28,12 @@ pub mod event { impl EventAttribute for super::LogItemAddV2Req { const SUBJECT: Subject = Subject::const_new("log/add"); } + impl EventAttribute for super::StatsItemAddReq { + const SUBJECT: Subject = Subject::const_new("stats/add"); + } + impl EventAttribute for super::StatsItemDeleteReq { + const SUBJECT: Subject = Subject::const_new("stats/delete"); + } } #[derive(Debug, Default, Clone)] pub struct SpiLogClient; @@ -118,10 +123,6 @@ pub struct StatsItemAddReq { pub own_paths: Option, } -impl EventAttribute for StatsItemAddReq { - const SUBJECT: Subject = Subject::const_new("stats/add"); -} - #[derive(poem_openapi::Object, Serialize, Deserialize, Clone, Debug)] pub struct StatsItemDeleteReq { #[oai(validator(min_length = "2"))] @@ -132,10 +133,6 @@ pub struct StatsItemDeleteReq { pub key: Option, } -impl EventAttribute for StatsItemDeleteReq { - const SUBJECT: Subject = Subject::const_new("stats/delete"); -} - impl SpiLogClient { #[deprecated] pub async fn add_dynamic_log(