diff --git a/backend/middlewares/flow/tests/mock_api.rs b/backend/middlewares/flow/tests/mock_api.rs index ebdb802a7..e2b87ee4b 100644 --- a/backend/middlewares/flow/tests/mock_api.rs +++ b/backend/middlewares/flow/tests/mock_api.rs @@ -88,7 +88,7 @@ where })) } - pub fn accepted(data: T) -> MockApiResponse { + pub fn _accepted(data: T) -> MockApiResponse { MockApiResponse::Ok(Json(MockResp { code: TARDIS_RESULT_ACCEPTED_CODE.to_string(), message: "".to_string(), @@ -96,7 +96,7 @@ where })) } - pub fn err(error: TardisError) -> MockApiResponse { + pub fn _err(error: TardisError) -> MockApiResponse { MockApiResponse::Err(error.into()) } } diff --git a/backend/spi/spi-search/Cargo.toml b/backend/spi/spi-search/Cargo.toml index 10e3017a7..70cca5d47 100644 --- a/backend/spi/spi-search/Cargo.toml +++ b/backend/spi/spi-search/Cargo.toml @@ -21,6 +21,7 @@ spi-es = ["tardis/web-client"] [dependencies] serde.workspace = true +itertools.workspace = true tardis = { workspace = true, features = [ "reldb-postgres", "web-server", diff --git a/backend/spi/spi-search/src/api/ci/search_ci_item_api.rs b/backend/spi/spi-search/src/api/ci/search_ci_item_api.rs index 3ed9719f7..ef0702d96 100644 --- a/backend/spi/spi-search/src/api/ci/search_ci_item_api.rs +++ b/backend/spi/spi-search/src/api/ci/search_ci_item_api.rs @@ -1,3 +1,4 @@ +use tardis::basic::dto::TardisContext; use tardis::web::context_extractor::TardisContextExtractor; use tardis::web::poem::web::Query; @@ -65,4 +66,18 @@ impl SearchCiItemApi { let resp = search_item_serv::query_metrics(&query_req.0, &funs, &ctx.0).await?; TardisResp::ok(resp) } + + /// Refresh TSV Result By Tag + /// + /// 通过指定 tag 刷新分词结果 + #[oai(path = "/:tag/refresh", method = "put")] + async fn refresh_tsv(&self, tag: Path, ctx: TardisContextExtractor) -> TardisApiResult { + let global_ctx = TardisContext { + own_paths: "".to_string(), + ..ctx.0.clone() + }; + let funs = crate::get_tardis_inst(); + search_item_serv::refresh_tsv(&tag.0, &funs, &global_ctx).await?; + TardisResp::ok(Void {}) + } } diff --git a/backend/spi/spi-search/src/dto/search_item_dto.rs b/backend/spi/spi-search/src/dto/search_item_dto.rs index c03e73ce8..b4073935d 100644 --- a/backend/spi/spi-search/src/dto/search_item_dto.rs +++ b/backend/spi/spi-search/src/dto/search_item_dto.rs @@ -33,7 +33,7 @@ pub struct SearchItemAddReq { pub visit_keys: Option, } -#[derive(poem_openapi::Object, Serialize, Deserialize, Debug)] +#[derive(poem_openapi::Object, Serialize, Deserialize, Debug, Default)] pub struct SearchItemModifyReq { #[oai(validator(min_length = "2"))] pub kind: Option, @@ -110,7 +110,7 @@ pub struct SearchItemSearchReq { pub page: SearchItemSearchPageReq, } -#[derive(poem_openapi::Object, Serialize, Deserialize, Debug)] +#[derive(poem_openapi::Object, Serialize, Deserialize, Debug, Default)] pub struct SearchItemSearchCtxReq { pub accounts: Option>, pub apps: Option>, diff --git a/backend/spi/spi-search/src/serv/es/search_es_item_serv.rs b/backend/spi/spi-search/src/serv/es/search_es_item_serv.rs index 2c9551851..11ae668fd 100644 --- a/backend/spi/spi-search/src/serv/es/search_es_item_serv.rs +++ b/backend/spi/spi-search/src/serv/es/search_es_item_serv.rs @@ -834,3 +834,7 @@ fn merge(a: &mut serde_json::Value, b: serde_json::Value) { pub async fn query_metrics(_query_req: &SearchQueryMetricsReq, funs: &TardisFunsInst, _ctx: &TardisContext, _inst: &SpiBsInst) -> TardisResult { Err(funs.err().format_error("search_es_item_serv", "query_metrics", "not supports", "500-not-supports")) } + +pub async fn refresh_tsv(tag: &str, funs: &TardisFunsInst, _ctx: &TardisContext, _inst: &SpiBsInst) -> TardisResult<()> { + Err(funs.err().format_error("search_es_item_serv", "refresh_tsv", "not supports", "500-not-supports")) +} \ No newline at end of file diff --git a/backend/spi/spi-search/src/serv/pg/search_pg_item_serv.rs b/backend/spi/spi-search/src/serv/pg/search_pg_item_serv.rs index 2fbaa0295..430d01360 100644 --- a/backend/spi/spi-search/src/serv/pg/search_pg_item_serv.rs +++ b/backend/spi/spi-search/src/serv/pg/search_pg_item_serv.rs @@ -1,24 +1,19 @@ -use std::collections::HashMap; +use std::{collections::HashMap, vec}; use pinyin::{to_pinyin_vec, Pinyin}; use tardis::{ - basic::{dto::TardisContext, error::TardisError, result::TardisResult}, - chrono::Utc, - db::{ + basic::{dto::TardisContext, error::TardisError, result::TardisResult}, chrono::Utc, db::{ reldb_client::{TardisRelDBClient, TardisRelDBlConnection}, sea_orm::{FromQueryResult, Value}, - }, - serde_json::{self, json, Map}, - web::web_resp::TardisPage, - TardisFuns, TardisFunsInst, + }, futures::future::join_all, serde_json::{self, json, Map}, web::web_resp::TardisPage, TardisFuns, TardisFunsInst }; +use itertools::Itertools; use bios_basic::{dto::BasicQueryCondInfo, enumeration::BasicQueryOpKind, helper::db_helper, spi::spi_funs::SpiBsInst}; use crate::{ dto::search_item_dto::{ - AdvBasicQueryCondInfo, SearchItemAddReq, SearchItemModifyReq, SearchItemSearchQScopeKind, SearchItemSearchReq, SearchItemSearchResp, SearchQueryMetricsReq, - SearchQueryMetricsResp, + AdvBasicQueryCondInfo, SearchItemAddReq, SearchItemModifyReq, SearchItemSearchQScopeKind, SearchItemSearchReq, SearchItemSearchResp, SearchQueryMetricsReq, SearchQueryMetricsResp }, search_config::SearchConfig, }; @@ -34,20 +29,26 @@ pub async fn add(add_req: &mut SearchItemAddReq, funs: &TardisFunsInst, ctx: &Ta params.push(Value::from(add_req.kind.to_string())); params.push(Value::from(add_req.key.to_string())); params.push(Value::from(add_req.title.as_str())); + + let pinyin_vec = to_pinyin_vec(add_req.title.as_str(), Pinyin::plain); if add_req.title.chars().count() > funs.conf::().split_strategy_rule_config.specify_word_length.unwrap_or(30) { params.push(Value::from(format!( - "{} {}", + "{} {} {}", add_req.title.as_str(), - generate_word_combinations(to_pinyin_vec(add_req.title.as_str(), Pinyin::plain)).join(" ") + pinyin_vec.clone().into_iter().map(|pinyin| pinyin.chars().next().unwrap_or_default()).join(""), + generate_word_combinations(pinyin_vec).join(" ") ))); } else { + let content = add_req.title.as_str().split(' ').last().unwrap_or_default(); params.push(Value::from(format!( - "{} {} {} {} {}", + "{} {} {} {} {} {} {}", add_req.title.as_str(), - generate_word_combinations_with_length(add_req.title.as_str(), 1).join(" "), - generate_word_combinations_with_length(add_req.title.as_str(), 2).join(" "), - generate_word_combinations_with_length(add_req.title.as_str(), 3).join(" "), - generate_word_combinations(to_pinyin_vec(add_req.title.as_str(), Pinyin::plain)).join(" ") + pinyin_vec.clone().into_iter().map(|pinyin| pinyin.chars().next().unwrap_or_default()).join(""), + generate_word_combinations_with_length(content, 1).join(" "), + generate_word_combinations_with_length(content, 2).join(" "), + generate_word_combinations_with_length(content, 3).join(" "), + generate_word_combinations_with_symbol(content, vec!["-", "_"]).join(" "), + generate_word_combinations(pinyin_vec).join(" ") ))); } @@ -116,20 +117,26 @@ pub async fn modify(tag: &str, key: &str, modify_req: &mut SearchItemModifyReq, "simple" }; sql_sets.push(format!("title_tsv = to_tsvector('{word_combinations_way}', ${})", params.len() + 1)); - if title.chars().count() > 15 { + + let pinyin_vec = to_pinyin_vec(title, Pinyin::plain); + if title.chars().count() > funs.conf::().split_strategy_rule_config.specify_word_length.unwrap_or(30) { params.push(Value::from(format!( - "{} {}", + "{} {} {}", title, - generate_word_combinations(to_pinyin_vec(title, Pinyin::plain)).join(" ") + pinyin_vec.clone().into_iter().map(|pinyin| pinyin.chars().next().unwrap_or_default()).join(""), + generate_word_combinations(pinyin_vec).join(" ") ))); } else { + let content = title.split(' ').last().unwrap_or_default(); params.push(Value::from(format!( - "{} {} {} {} {}", + "{} {} {} {} {} {} {}", title, - generate_word_combinations_with_length(title, 1).join(" "), - generate_word_combinations_with_length(title, 2).join(" "), - generate_word_combinations_with_length(title, 3).join(" "), - generate_word_combinations(to_pinyin_vec(title, Pinyin::plain)).join(" ") + pinyin_vec.clone().into_iter().map(|pinyin| pinyin.chars().next().unwrap_or_default()).join(""), + generate_word_combinations_with_length(content, 1).join(" "), + generate_word_combinations_with_length(content, 2).join(" "), + generate_word_combinations_with_length(content, 3).join(" "), + generate_word_combinations_with_symbol(content, vec!["-", "_"]).join(" "), + generate_word_combinations(pinyin_vec).join(" ") ))); } }; @@ -199,6 +206,15 @@ fn generate_word_combinations_with_length(original_str: &str, split_len: usize) combinations } +fn generate_word_combinations_with_symbol(original_str: &str, symbols: Vec<&str>) -> Vec { + let mut combinations = Vec::new(); + for symbol in symbols { + let mut splited_words = original_str.split(symbol).collect_vec(); + combinations.append(&mut splited_words); + } + combinations.into_iter().map(|word| word.to_string()).collect_vec() +} + fn generate_word_combinations(chars: Vec<&str>) -> Vec { let mut combinations = Vec::new(); for i in 0..chars.len() { @@ -1547,3 +1563,18 @@ fn package_groups_agg(record: serde_json::Value) -> Result Ok(serde_json::Value::Null), } } + +pub async fn refresh_tsv(tag: &str, funs: &TardisFunsInst, ctx: &TardisContext, inst: &SpiBsInst) -> TardisResult<()> { + let bs_inst = inst.inst::(); + let (conn, table_name) = search_pg_initializer::init_table_and_conn(bs_inst, tag, ctx, false).await?; + let result = conn.query_all(&format!("SELECT key, title FROM {table_name}"), vec![]).await?; + join_all( + result.into_iter().map(|row| async move { + modify(tag, row.try_get::("", "key").expect("not found key").as_str(), &mut SearchItemModifyReq { + title: Some(row.try_get("", "title").expect("not found title")), + ..Default::default() + }, funs, ctx, inst).await.expect("modify error") + }).collect_vec(), + ).await; + Ok(()) +} diff --git a/backend/spi/spi-search/src/serv/search_item_serv.rs b/backend/spi/spi-search/src/serv/search_item_serv.rs index c0e3e828c..30c4547ae 100644 --- a/backend/spi/spi-search/src/serv/search_item_serv.rs +++ b/backend/spi/spi-search/src/serv/search_item_serv.rs @@ -29,5 +29,6 @@ spi_dispatch_service! { delete_by_ownership(tag: &str, own_paths: &str) -> TardisResult<()>; search(search_req: &mut SearchItemSearchReq) -> TardisResult>; query_metrics(query_req: &SearchQueryMetricsReq) -> TardisResult; + refresh_tsv(tag: &str) -> TardisResult<()>; } }