Merge into split 231011-1 (#481)

* basic:fix requst helper * flow：fix (#393) * flow:update * flow:update * flow:update * spi-stats: Modify some query interface (#394) * fixup bugs and add new api 'fact_col_paginate_agg_with_dim' 1、主题添加跳转路径字段。需要支持的接口（新增、修改、查询接口） 2、主题添加上下线字段。需要支持的接口（新增、修改、查询接口支持上下线条件查询） 4、删除主题不会删除掉主题和维度、度量关系。新建同名主题，会自动包含老的维度和度量 5、度量增加单位字段。需要支持的接口（新增、修改、查询接口） 7、维度添加动态取值url 字段。需要支持的接口（新增、修改、查询接口） 8、invoke主题添加维度接口，少了dim_exclusive_rec、mes_data_distinct 字段 9、维度被主题引用了，还可以删除。这个需要加上校验 10、给主题添加维度度量，主题不存在也可以添加。 11、主题下的度量不支持根据主题删除所有度量。（前端批量需要） 13、factCol 根据维度key 查询。（目前必须要根据主题key+维度key 查询。）需求是要显示维度被引用的主题。 * add dim_exclusive_rec, mes_data_distinct fields * spi-stats: update interface * resolve merge conflict * resolve merge conflict * basic:fix task processor. * flow:update (#396) * flow:add post change param * flow: state sort * flow: sort state finish * flow: delete unused code * flow: update * flow: config finish * flow: config * flow: config update * flow: config test pass * flow: tag update * fmt * flow: config update * flow: update * flow:update * flow:config fix * flow:test update * flow:config prefix remove * flow: inst update * flow: update * flow: post action * flow: post action finish * flow: feat test * flow: post action update * flow:fix * flow: config test * flow: fmt * flow: dto update * flow:update * flow:update * flow: update * post action * flow: update * flow:update * flow: update * flow:update * flow:update * flow:post action test past * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * iam: fix task api. * iam:fix task. * iam:fix task api path. * iam:fix ci system api. * flow:update (#400) * flow:add post change param * flow: state sort * flow: sort state finish * flow: delete unused code * flow: update * flow: config finish * flow: config * flow: config update * flow: config test pass * flow: tag update * fmt * flow: config update * flow: update * flow:update * flow:config fix * flow:test update * flow:config prefix remove * flow: inst update * flow: update * flow: post action * flow: post action finish * flow: feat test * flow: post action update * flow:fix * flow: config test * flow: fmt * flow: dto update * flow:update * flow:update * flow: update * post action * flow: update * flow:update * flow: update * flow:update * flow:update * flow:post action test past * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * iam:fix remote ip. * iam:fix * spi-stats: support quiery paged fact_record (#397) * fixup bugs and add new api 'fact_col_paginate_agg_with_dim' 1、主题添加跳转路径字段。需要支持的接口（新增、修改、查询接口） 2、主题添加上下线字段。需要支持的接口（新增、修改、查询接口支持上下线条件查询） 4、删除主题不会删除掉主题和维度、度量关系。新建同名主题，会自动包含老的维度和度量 5、度量增加单位字段。需要支持的接口（新增、修改、查询接口） 7、维度添加动态取值url 字段。需要支持的接口（新增、修改、查询接口） 8、invoke主题添加维度接口，少了dim_exclusive_rec、mes_data_distinct 字段 9、维度被主题引用了，还可以删除。这个需要加上校验 10、给主题添加维度度量，主题不存在也可以添加。 11、主题下的度量不支持根据主题删除所有度量。（前端批量需要） 13、factCol 根据维度key 查询。（目前必须要根据主题key+维度key 查询。）需求是要显示维度被引用的主题。 * add dim_exclusive_rec, mes_data_distinct fields * spi-stats: update interface * resolve merge conflict * resolve merge conflict * spi-stats: support paged get_fact_record * iam and stats:fix * flow:update (#401) * flow:add post change param * flow: state sort * flow: sort state finish * flow: delete unused code * flow: update * flow: config finish * flow: config * flow: config update * flow: config test pass * flow: tag update * fmt * flow: config update * flow: update * flow:update * flow:config fix * flow:test update * flow:config prefix remove * flow: inst update * flow: update * flow: post action * flow: post action finish * flow: feat test * flow: post action update * flow:fix * flow: config test * flow: fmt * flow: dto update * flow:update * flow:update * flow: update * post action * flow: update * flow:update * flow: update * flow:update * flow:update * flow:post action test past * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update (#402) * flow:add post change param * flow: state sort * flow: sort state finish * flow: delete unused code * flow: update * flow: config finish * flow: config * flow: config update * flow: config test pass * flow: tag update * fmt * flow: config update * flow: update * flow:update * flow:config fix * flow:test update * flow:config prefix remove * flow: inst update * flow: update * flow: post action * flow: post action finish * flow: feat test * flow: post action update * flow:fix * flow: config test * flow: fmt * flow: dto update * flow:update * flow:update * flow: update * post action * flow: update * flow:update * flow: update * flow:update * flow:update * flow:post action test past * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * flow:update * iam:fix get ip . * flow:update (#404) * flow:update * flow:update * flow: update (#405) * flow:update * flow:update * flow:update * flow:update * flow: fix external resp error (#406) * flow: fix state find error (#407) * flow: update api & notify (#408) * fix upload error (#409) * flow: add search rel states api (#411) * spi-search:fix advanced group query. * flow:fix test (#412) * flow: add debug text (#413) * flow:fix test * flow: add debug text * flow:update (#414) * flow:fix test * flow: add debug text * flow: update test * flow: import fix (#415) * flow:fix test * flow: add debug text * flow: update test * flow: update test * flow: init fix (#416) * flow:fix test * flow: add debug text * flow: update test * flow: update test * flow: init fix * flow: init fix * iam:fix copy role. * wasm: get token. * iam: fmt * flow: init fix (#418) * spi-conf: Conf support nacos grpc request (#398) * spi-conf: add auth/users/login api * update cert and add grpc * update * grpc using independant grpc server * change tardis dependency path * remove grpc from main web server * add grpc bistream request * update tardis version * temorary disable spi conf, clippy fix and format * update tardis version * iam:fix copy role. * add setup-protoc in github workflow (#419) * apply spacegate update on bois gateway plugins (#417) * fix iam modify tamporary (#379) * improve spacegate auth config (#385) * support white list mode, support all time allow or all time ban (#399) * support white list mode, support all time allow or all time ban * gateway: modify project structure * merge main * expose nacos serve port (#421) * flow: improve instance resp (#422) * spi-stats: add query many fact record interface (#423) * iam:fix. * flow: update dto (#425) * iam:fix sub role. * global: remove all unsafe code (#424) * spi-stats: add query many fact record interface * remove all unsafe codes * global: Fix build error with macro (#426) * spi-stats: add query many fact record interface * remove all unsafe codes * fix build error * iam:fix extend role. * fix fmt. * iam:fix set. * flow fix:restart instance (#427) * flow:fix test * flow: add debug text * flow: update test * flow: update test * flow: init fix * flow: init fix * flow: init fix * flow: improve instance resp * flow: modify dto flow fix:restart instance * iam:fix app org apps. * auth:fix extend role. * flow:fix post action (#428) * flow: fix search error (#429) * iam:fix apps item. * flow: support data center (#430) * flow:fix post action * flow: fix search error * flow: support data center * flow:fix (#431) * flow: add api count_group_by_state (#432) * spi-stats:fix record and metric. * es fix: modify time failed (#433) * tardis Upgrades version. * iam:fix app set. * fix:gateway build error (#434) * search:fix * iam:fix app_set validator set_cate_id * iam:fix role. * basefix :BasicQueryOpKind. * search:fix * search:fix add content. * search:fix * flow: modify about model name (#435) * flow: modify about model name * flow: fix * iam:fix role check. * iam:fix role add rel account ignore exist error. * flow: fix unbind state bug && support multi assigned (#437) * cargo clippy (#438) * update gate way version (#439) * gateway: Spilt merge with main (#440) * fix index name confict bug (#436) * gateway-fix-build-errror (#441) * spi:search:fix * flow:update external dto (#442) * flow: modify about model name * flow: fix * flow: fix unbind state bug * flow: support multi assigned * flow:update external dto * spi-search:fix. * spi-search:fix . * iam:fix app execute. * gateway-fix-some-bug (#444) * flow: find subrole id (#443) * iam:fix app set. * iam:fix set cate none. * flow: support tenant find states * flow: support tenant find states (#445) * schedule support enable time * flow: fix init state error * flow: fix init state error (#447) * flow: support tenant find states * flow: fix init state error * gateway-fix-auth-plugin * flow:update external dto * flow:update external dto (#449) * flow: support tenant find states * flow: fix init state error * flow:update external dto * schedule modify testcase * update * spi-stats: remove rn column * iam:fix ci role. * update * flow: update init model * flow: response update (#451) * flow: response update * flow: fix post action * iam:fix account task. * flow: fix post action (#452) * flow: response update * flow: fix post action * flow: fix post action * iam:fix account search. * fix-auth-panic (#453) * flow:fix (#455) * flow: response update * flow: fix post action * flow: fix post action * flow: update * gateway-auth-add-log (#454) * gateway-fix-auth-get-process * iam:fix role task. * flow:update * fix get_fact_record_latest queried wrong column * flow:init update * flow:fix external * flow:fix external (#460) * flow:init fix * flow:init fix (#461) * flow:fix external * flow:init fix * iam:fix ca role get * fix * flow:update * basic:fix json to sea orm value like. * gateway-remove-apisx-version (#463) * iam:fix test. * flow:fix (#464) * flow:fix (#465) * flow: response add color field (#466) * spi-plugin:fix * spi-search:fix ext. * iam:fix org hide item with disabled. * flow:support share template model (#467) * flow:fix * flow: response add color field * flow:support share template model * flow: update response (#468) * iam:fix search. * spi-search:fix. * db helper fix. * flow: update (#469) * spi-search:fix value is chrono time. * spi-search：fix * flow:fix unbind error (#471) * flow: share bug fix (#472) * spi-plugin: add api get bs by rel up. * flow: fix bind state error (#473) * spi-conf: add servers and workflows * Spacegate auth fix mix context length (#474) * spi-conf: workflows * iam:fix cs api batch_add_rel_account. * spi-log: add attribute id. * spi-log: add attribute id. * flow: add resort trans api && set front condition config (#476) * spi-log: add attribute id. * fix log_pg_item_serv.rs * flow: modify related state api (#477) * flow: fix bind state error * flow: add resort trans api && set front condition config * flow:update * flow:update * flow: add inst api: modify vars & complete testing (#478) * flow: update dto (#480) * spi-log: add dto content validator remove. * reach: Feat reach (#362) * fix: different node hold it's own auth key and cache (#479) --------- Co-authored-by: ljl <17743125563@163.com> Co-authored-by: ZzIsGod1019 <1498852723@qq.com> Co-authored-by: RWDai <27391645+RWDai@users.noreply.github.com> Co-authored-by: gudaoxuri <i@sunisle.org> Co-authored-by: hermitCode <297984816@qq.com>
ideal-world · Oct 11, 2023 · 31f498b · 31f498b
1 parent 8a6efa0
commit 31f498b
Show file tree

Hide file tree

Showing 333 changed files with 13,652 additions and 4,391 deletions.
diff --git a/.github/workflows/gateway-docker-push.yml b/.github/workflows/gateway-docker-push.yml
diff --git a/.github/workflows/gateway-docker-release.yml b/.github/workflows/gateway-docker-release.yml
diff --git a/.github/workflows/service-all-push.yml b/.github/workflows/service-all-push.yml
@@ -20,6 +20,9 @@ jobs:
       packages: write
       contents: read
     steps:
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v1
+
       - name: Check out the repo
         uses: actions/checkout@v3
 

diff --git a/.github/workflows/service-all-release.yml b/.github/workflows/service-all-release.yml
@@ -13,6 +13,9 @@ jobs:
       packages: write
       contents: read
     steps:
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v1
+
       - name: Check out the repo
         uses: actions/checkout@v3
 

diff --git a/.github/workflows/spi-conf-service-push.yml b/.github/workflows/spi-conf-service-push.yml
@@ -0,0 +1,52 @@
+name: Publish spi-conf services docker image
+
+on:
+  push:
+    paths:
+      - "basic/**"
+      - "services/**"
+      - "spi/**"
+      - "middleware/**"
+      - "Cargo.toml"
+      - ".github/workflows/spi-conf-service-push.yml"
+
+jobs:
+  push_to_registry:
+    name: Push docker image to registry
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v1
+
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: Init rust envrionment
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          components: rustfmt, clippy
+
+      - name: Cache rust
+        uses: Swatinem/rust-cache@v2
+
+      - name: Build rust
+        run: cargo build --release -p spi-conf-serv
+
+      - name: Copy binary to workspace
+        run: cp ./target/release/spi-conf-serv ./services/spi-conf/
+
+      - name: Build and push Docker image to dockerhub
+        uses: mr-smithers-excellent/docker-build-push@v5
+        with:
+          directory: services/spi-conf
+          dockerfile: services/spi-conf/Dockerfile
+          registry: docker.io
+          image: ecfront/spi-conf-serv
+          tags: latest
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
diff --git a/.github/workflows/spi-conf-service-release.yml b/.github/workflows/spi-conf-service-release.yml
@@ -0,0 +1,64 @@
+name: Release spi-conf services docker image
+
+on:
+  push:
+    tags:
+    - '*'
+
+jobs:
+  push_to_registry:
+    name: Push docker image to registry
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v1
+
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: Init rust envrionment
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          components: rustfmt, clippy
+
+      - name: Cache rust
+        uses: Swatinem/rust-cache@v2
+
+      - name: Check format
+        run: cargo fmt --all -- --check
+
+      - name: Check with clippy
+        run: cargo clippy --all-features
+
+      - name: Build rust
+        run: cargo build --release -p spi-conf-serv
+
+      - name: Copy binary to workspace
+        run: cp ./target/release/spi-conf-serv ./services/spi-conf/
+
+      - name: Build and push Docker image to github
+        uses: mr-smithers-excellent/docker-build-push@v5
+        with:
+          directory: services/spi-conf
+          dockerfile: services/spi-conf/Dockerfile
+          registry: ghcr.io
+          image: spi-conf-serv
+          tags: ${{ github.ref_name }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GHCR_TOKEN }}
+
+      - name: Build and push Docker image to dockerhub
+        uses: mr-smithers-excellent/docker-build-push@v5
+        with:
+          directory: services/spi-conf
+          dockerfile: services/spi-conf/Dockerfile
+          registry: docker.io
+          image: ecfront/spi-conf-serv
+          tags: ${{ github.ref_name }}
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
diff --git a/.gitignore b/.gitignore
@@ -57,7 +57,7 @@ node_modules
 .DS_Store
 #gateway dev
 gateway/example/log
-gateway/spacegate/devsh/save-o.sh
+gateway/spacegate/devsh/
 
 # pre-commit
 .pre-commit-config.yaml
diff --git a/Cargo.toml b/Cargo.toml
@@ -9,8 +9,10 @@ members = [
     "support/iam",
     "support/auth",
     "support/enhance-wasm",
+    "support/reach",
     "services/*",
     "sdk/*",
+    "clients/*",
 ]
 
 [workspace.package]
@@ -45,8 +47,8 @@ lazy_static = { version = "1" }
 itertools = { version = "0" }
 fancy-regex = { version = "0" }
 run_script = { version = "0.10" }
-strum = { version = "0.25", features = ["derive"] }
+strum ={ version = "0.25", features = ["derive"] }
 # tardis
 tardis = { version = "=0.1.0-beta.12" }
 # tardis = { path = "../tardis/tardis" }
-# tardis = { git = "https://github.com/ideal-world/tardis.git" }
+# tardis = { git = "https://github.com/ideal-world/tardis.git", rev = "9424e16" }
diff --git a/basic/src/basic_enumeration.rs b/basic/src/basic_enumeration.rs
@@ -37,8 +37,12 @@ pub enum BasicQueryOpKind {
     Le,
     #[oai(rename = "like")]
     Like,
+    #[oai(rename = "not_like")]
+    NotLike,
     #[oai(rename = "in")]
     In,
+    #[oai(rename = "not_in")]
+    NotIn,
 }
 
 impl BasicQueryOpKind {
@@ -51,7 +55,9 @@ impl BasicQueryOpKind {
             BasicQueryOpKind::Lt => "<".to_string(),
             BasicQueryOpKind::Le => "<=".to_string(),
             BasicQueryOpKind::Like => "LIKE".to_string(),
+            BasicQueryOpKind::NotLike => "NOT LIKE".to_string(),
             BasicQueryOpKind::In => "IN".to_string(),
+            BasicQueryOpKind::NotIn => "NOT IN".to_string(),
         }
     }
 }
diff --git a/basic/src/dto.rs b/basic/src/dto.rs
@@ -71,7 +71,11 @@ impl BasicQueryCondInfo {
                     BasicQueryOpKind::Like => {
                         check_val.as_str().map(|check_val_str| cond.value.as_str().map(|cond_val_str| check_val_str.contains(cond_val_str)).unwrap_or(false)).unwrap_or(false)
                     }
+                    BasicQueryOpKind::NotLike => {
+                        check_val.as_str().map(|check_val_str| cond.value.as_str().map(|cond_val_str| check_val_str.contains(cond_val_str)).unwrap_or(false)).unwrap_or(false)
+                    }
                     BasicQueryOpKind::In => check_val.as_array().map(|check_val_arr| check_val_arr.contains(&cond.value)).unwrap_or(false),
+                    BasicQueryOpKind::NotIn => check_val.as_array().map(|check_val_arr| check_val_arr.contains(&cond.value)).unwrap_or(false),
                 },
                 None => false,
             })

diff --git a/basic/src/helper/db_helper.rs b/basic/src/helper/db_helper.rs
@@ -1,4 +1,8 @@
-use tardis::{db::sea_orm, serde_json};
+use tardis::{
+    chrono::{DateTime, ParseError, Utc},
+    db::sea_orm,
+    serde_json,
+};
 
 pub fn json_to_sea_orm_value(json_value: &serde_json::Value, like_by_str: bool) -> Option<Vec<sea_orm::Value>> {
     match json_value {
@@ -8,13 +12,16 @@ pub fn json_to_sea_orm_value(json_value: &serde_json::Value, like_by_str: bool)
         serde_json::Value::Number(val) if val.is_u64() => Some(vec![sea_orm::Value::from(val.as_u64())]),
         serde_json::Value::Number(val) if val.is_f64() => Some(vec![sea_orm::Value::from(val.as_f64())]),
         serde_json::Value::Object(_) => Some(vec![sea_orm::Value::from(json_value.clone())]),
-        serde_json::Value::String(val) => {
-            if like_by_str {
-                Some(vec![sea_orm::Value::from(format!("{val}%"))])
-            } else {
-                Some(vec![sea_orm::Value::from(val)])
+        serde_json::Value::String(val) => match str_to_datetime(val) {
+            Ok(val) => Some(vec![sea_orm::Value::from(val)]),
+            Err(_) => {
+                if like_by_str {
+                    Some(vec![sea_orm::Value::from(format!("%{val}%"))])
+                } else {
+                    Some(vec![sea_orm::Value::from(val)])
+                }
             }
-        }
+        },
         serde_json::Value::Array(val) => {
             if val.is_empty() {
                 return None;
@@ -38,3 +45,7 @@ pub fn json_to_sea_orm_value(json_value: &serde_json::Value, like_by_str: bool)
         _ => None,
     }
 }
+
+fn str_to_datetime(input: &str) -> Result<DateTime<Utc>, ParseError> {
+    DateTime::parse_from_rfc3339(input).map(|dt| dt.with_timezone(&Utc))
+}
diff --git a/basic/src/helper/request_helper.rs b/basic/src/helper/request_helper.rs
@@ -5,13 +5,32 @@ use tardis::{
 
 pub const REMOTE_ADDR: &str = "remote-addr";
 
-pub async fn add_remote_ip(request: &Request, ctx: &TardisContext) -> TardisResult<()> {
-    if let Some(add) = request.remote_addr().as_socket_addr() {
-        ctx.add_ext(REMOTE_ADDR, &add.ip().to_string()).await?;
+pub async fn add_ip(ip: Option<String>, ctx: &TardisContext) -> TardisResult<()> {
+    if let Some(ip) = ip {
+        ctx.add_ext(REMOTE_ADDR, &ip).await?;
     }
     Ok(())
 }
 
+pub async fn add_remote_ip(request: &Request, ctx: &TardisContext) -> TardisResult<()> {
+    ctx.add_ext(REMOTE_ADDR, &get_ip(request).await?.unwrap_or_default()).await?;
+    Ok(())
+}
+
+pub async fn get_ip(request: &Request) -> TardisResult<Option<String>> {
+    let back_ip = request.remote_addr().as_socket_addr().map(|socert_addr| socert_addr.ip().to_string());
+    let ip = if let Some(real_ips) = request.headers().get("X-Forwarded-For") {
+        if let Some(real_ip) = real_ips.to_str().ok().and_then(|ips| ips.split(',').collect::<Vec<_>>().first().map(|ip| ip.to_string())) {
+            Some(real_ip)
+        } else {
+            back_ip
+        }
+    } else {
+        back_ip
+    };
+    Ok(ip)
+}
+
 pub async fn get_remote_ip(ctx: &TardisContext) -> TardisResult<Option<String>> {
-    Ok(ctx.get_ext(REMOTE_ADDR).await?)
+    ctx.get_ext(REMOTE_ADDR).await
 }
diff --git a/basic/src/process/ci_processor.rs b/basic/src/process/ci_processor.rs
@@ -25,7 +25,7 @@ pub fn signature(app_key_config: &AppKeyConfig, method: &str, path: &str, query:
     let sorted_req_query = sort_query(query);
     let date = Utc::now().format("%a, %d %b %Y %T GMT").to_string();
     let signature =
-        TardisFuns::crypto.base64.encode(&TardisFuns::crypto.digest.hmac_sha256(&format!("{method}\n{date}\n{path}\n{sorted_req_query}").to_lowercase(), &app_key_config.sk)?);
+        TardisFuns::crypto.base64.encode(TardisFuns::crypto.digest.hmac_sha256(format!("{method}\n{date}\n{path}\n{sorted_req_query}").to_lowercase(), &app_key_config.sk)?);
     header.push(("Authorization".to_string(), format!("{}:{signature}", app_key_config.ak)));
     header.push((app_key_config.head_key_date_flag.to_string(), date));
     Ok(header)