repo-sync-2024-05-14T14:38:40+0800

README.md

@@ -119,13 +119,20 @@ sender.config:
 In the first terminal, run the following command
 
 ```bash
-docker run -it  --rm  --network host --mount type=bind,source=/tmp/receiver,target=/root/receiver --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:latest bash -c "./main --config receiver/receiver.config"
+docker run -it  --rm  --network host --mount type=bind,source=/tmp/receiver,target=/root/receiver --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:latest --config receiver/receiver.config
 ```
 
 In the other terminal, run the following command simultaneously.
 
 ```bash
-docker run -it  --rm  --network host --mount type=bind,source=/tmp/sender,target=/root/sender  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:latest bash -c "./main --config sender/sender.config"
+docker run -it  --rm  --network host --mount type=bind,source=/tmp/sender,target=/root/sender  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:latest --config sender/sender.config
+```
+
+You could also pass a minified JSON config directly. A minified JSON is a compact one without white space and line breaks.
+
+e.g.
+```
+docker run -it  --rm  --network host --mount type=bind,source=/tmp/sender,target=/root/sender  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:latest --json '{"psi_config":{"protocol_config":{"protocol":"PROTOCOL_KKRT","role":"ROLE_RECEIVER","broadcast_result":true},"input_config":{"type":"IO_TYPE_FILE_CSV","path":"/root/receiver/receiver_input.csv"},"output_config":{"type":"IO_TYPE_FILE_CSV","path":"/root/receiver/receiver_output.csv"},"keys":["id0","id1"],"debug_options":{"trace_path":"/root/receiver/receiver.trace"}},"self_link_party":"receiver","link_config":{"parties":[{"id":"receiver","host":"127.0.0.1:5300"},{"id":"sender","host":"127.0.0.1:5400"}]}}'
 ```
 
 ## Building SecretFlow PSI Library

RELEASE.md

@@ -5,6 +5,12 @@
 > - `[API]` prefix for API changes.
 > - `[Improvement]` prefix for implementation improvement.
 
+## v0.4.0.dev240514
+- [API] add entrypoint for docker file.
+- [API] allow passing config JSON directly to main.
+- [Bugfix] fix ic mode.
+- [Bugfix] fix RR22, SealPIR and APSI.
+
 ## v0.4.0.dev240401
 - [Improvement] upgrade download uri of xz.
 

bazel/patches/boost.patch

@@ -39,4 +39,4 @@ index 8277dbb..afc9569 100644
 +        strip_prefix = "xz-5.4.6",
      )
 
-     maybe(
+     maybe(

docker/Dockerfile

@@ -46,4 +46,4 @@ LABEL kuscia.secretflow.deploy-templates=$deploy_templates
 # run as root for now
 WORKDIR /root
 
-CMD ["/bin/bash"]
+ENTRYPOINT ["./main"]

docker/README.md

@@ -7,7 +7,7 @@ docker run -it  --rm   --mount type=bind,source="$(pwd)/../../psi",target=/home/
 # build psi dev docker
 
 ```bash
-sh build.sh -v <version> -u -l
+bash build.sh -v <version> -u -l
 ```
 - *-u* means upload docker to reg.
 - *-l* means tag docker as *latest* as well.

docs/user_guide/pir.rst

@@ -198,7 +198,7 @@ Setup Phase
 
 .. code-block:: bash
 
-  docker run -it  --rm  --network host --mount type=bind,source=/tmp/server,target=/root/server --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:0.1.0beta bash -c "./main --config server/apsi_server_setup.json"
+  docker run -it  --rm  --network host --mount type=bind,source=/tmp/server,target=/root/server --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:0.1.0beta --config server/apsi_server_setup.json
 
 Online Phase
 >>>>>>>>>>>>
@@ -209,14 +209,14 @@ In the server's terminal.
 
 .. code-block:: bash
 
-  docker run -it  --rm  --network host --mount type=bind,source=/tmp/server,target=/root/server --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:0.1.0beta bash -c "./main --config server/apsi_server_online.json"
+  docker run -it  --rm  --network host --mount type=bind,source=/tmp/server,target=/root/server --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:0.1.0beta --config server/apsi_server_online.json
 
 
 In the client's terminal.
 
 .. code-block:: bash
 
-  docker run -it  --rm  --network host --mount type=bind,source=/tmp/client,target=/root/client --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:0.1.0beta bash -c "./main --config client/apsi_client.json"
+  docker run -it  --rm  --network host --mount type=bind,source=/tmp/client,target=/root/client --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:0.1.0beta --config client/apsi_client.json
 
 
 More examples

docs/user_guide/psi.rst

@@ -69,12 +69,12 @@ Run PSI
 
 In the first terminal, run the following command::
 
-    docker run -it  --rm  --network host --mount type=bind,source=/tmp/receiver,target=/root/receiver  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:latest bash -c "./main --config receiver/receiver.config"
+    docker run -it  --rm  --network host --mount type=bind,source=/tmp/receiver,target=/root/receiver  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:latest --config receiver/receiver.config
 
 
 In the other terminal, run the following command simultaneously::
 
-    docker run -it  --rm  --network host --mount type=bind,source=/tmp/sender,target=/root/sender  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:latest bash -c "./main --config sender/sender.config"
+    docker run -it  --rm  --network host --mount type=bind,source=/tmp/sender,target=/root/sender  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:latest --config sender/sender.config
 
 
 Building from source

docs/user_guide/psi_v2.rst

@@ -134,12 +134,12 @@ Run PSI
 
 In the first terminal, run the following command::
 
-    docker run -it  --rm  --network host --mount type=bind,source=/tmp/receiver,target=/root/receiver --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:0.1.0beta bash -c "./main --config receiver/receiver.config"
+    docker run -it  --rm  --network host --mount type=bind,source=/tmp/receiver,target=/root/receiver --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:0.1.0beta --config receiver/receiver.config
 
 
 In the other terminal, run the following command simultaneously::
 
-    docker run -it  --rm  --network host --mount type=bind,source=/tmp/sender,target=/root/sender --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:0.1.0beta bash -c "./main --config sender/sender.config"
+    docker run -it  --rm  --network host --mount type=bind,source=/tmp/sender,target=/root/sender --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:0.1.0beta --config sender/sender.config
 
 
 Building from source

psi/apsi/pir.cc

@@ -514,9 +514,6 @@ PirResultReport PirServerFull(
   size_t count_per_query = config.apsi_server_config().num_per_query();
   SPDLOG_INFO("server_data_count:{}", server_data_count);
 
-  YACL_ENFORCE(server_data_count <= config.bucket_size(),
-               "data_count:{} bucket_size:{}", config.bucket_size());
-
   ::apsi::PSIParams psi_params =
       GetPsiParams(count_per_query, server_data_count,
                    config.apsi_server_config().max_items_per_bin());

psi/legacy/bucket_psi.cc

@@ -399,8 +399,10 @@ void BucketPsi::Init() {
   }
   SPDLOG_INFO("bucket size set to {}", config_.bucket_size());
 
-  // Test connection.
-  lctx_->ConnectToMesh();
+  if (!ic_mode_) {
+    // Test connection.
+    lctx_->ConnectToMesh();
+  }
 
   MemoryPsiConfig config;
   config.set_psi_type(config_.psi_type());

psi/main.cc

@@ -28,6 +28,7 @@
 #include "psi/proto/psi_v2.pb.h"
 
 DEFINE_string(config, "", "file path of launch config in JSON format.");
+DEFINE_string(json, "", "config in JSON format.");
 DEFINE_string(kuscia, "", "file path of kuscia task config in JSON format.");
 
 std::string GenerateVersion() {
@@ -54,6 +55,15 @@ int main(int argc, char* argv[]) {
 
     SPDLOG_INFO("Kuscia task id: {}", kuscia_config.task_id);
     launch_config = kuscia_config.launch_config;
+  } else if (!FLAGS_json.empty()) {
+    google::protobuf::util::JsonParseOptions json_parse_options;
+    json_parse_options.ignore_unknown_fields = false;  // optional
+    auto status = google::protobuf::util::JsonStringToMessage(
+        FLAGS_json, &launch_config, json_parse_options);
+
+    YACL_ENFORCE(status.ok(),
+                 "Launch config JSON string couldn't be parsed: {}",
+                 FLAGS_json);
   } else {
     YACL_ENFORCE(std::filesystem::exists(FLAGS_config),
                  "Config file[{}] doesn't exist.", FLAGS_config);

psi/rr22/rr22_psi.cc

@@ -151,8 +151,11 @@ std::vector<size_t> Rr22PsiReceiverInternal(
 
   YACL_ENFORCE(sender_size <= receiver_size);
 
-  size_t mask_size = ComputeTruncateSize(sender_size, receiver_size,
-                                         options.ssp, options.malicious);
+  size_t mask_size = sizeof(uint128_t);
+  if (options.compress) {
+    mask_size = ComputeTruncateSize(sender_size, receiver_size, options.ssp,
+                                    options.malicious);
+  }
 
   Rr22OprfReceiver oprf_receiver(kRr22OprfBinSize, options.ssp, options.mode,
                                  options.code_type, options.malicious);

psi/rr22/rr22_psi_test.cc

@@ -107,6 +107,50 @@ TEST_P(Rr22PsiTest, CorrectTest) {
   EXPECT_EQ(indices_psi, indices);
 }
 
+TEST_P(Rr22PsiTest, CompressParamsFalseTest) {
+  auto params = GetParam();
+
+  auto lctxs = yacl::link::test::SetupWorld("ab", 2);
+
+  uint128_t seed = yacl::MakeUint128(0, 0);
+  yacl::crypto::Prg<uint128_t> prng(seed);
+
+  size_t item_size = params.items_num;
+
+  std::vector<uint128_t> inputs_a;
+  std::vector<uint128_t> inputs_b;
+  std::vector<size_t> indices;
+
+  std::tie(inputs_a, inputs_b, indices) = GenerateTestData(item_size);
+
+  Rr22PsiOptions psi_options(40, 0, false);
+
+  psi_options.mode = params.mode;
+  psi_options.malicious = params.malicious;
+
+  auto psi_sender_proc = std::async(
+      [&] { Rr22PsiSenderInternal(psi_options, lctxs[0], inputs_a); });
+  auto psi_receiver_proc = std::async(
+      [&] { return Rr22PsiReceiverInternal(psi_options, lctxs[1], inputs_b); });
+
+  psi_sender_proc.get();
+  std::vector<size_t> indices_psi = psi_receiver_proc.get();
+  std::sort(indices_psi.begin(), indices_psi.end());
+
+  SPDLOG_INFO("{}?={}", indices.size(), indices_psi.size());
+  EXPECT_EQ(indices.size(), indices_psi.size());
+
+#if 0
+  for (size_t i = 0; i < indices.size(); ++i) {
+    SPDLOG_INFO("i:{} index:{} a:{}, b:{}", i, indices_psi[i],
+                inputs_a[indices_psi[i]], inputs_b[indices_psi[i]]);
+  }
+#endif
+
+  EXPECT_EQ(indices_psi, indices);
+}
+
+
 INSTANTIATE_TEST_SUITE_P(
     CorrectTest_Instances, Rr22PsiTest,
     testing::Values(TestParams{35, Rr22PsiMode::FastMode},

psi/seal_pir/seal_mpir_test.cc

@@ -31,7 +31,7 @@ struct TestParams {
   size_t batch_number;
   size_t element_number;
   size_t element_size = 288;
-  size_t poly_degree = 8192;  // now only support 8192
+  size_t poly_degree = 4096;  // now we support 4096 and 8192
 };
 
 std::vector<uint8_t> GenerateDbData(TestParams params) {
@@ -80,6 +80,13 @@ TEST_P(SealMultiPirTest, Works) {
   size_t element_number = params.element_number;
   size_t element_size = params.element_size;
   size_t batch_number = params.batch_number;
+
+  SPDLOG_INFO(
+      "N (poly degree): {}, batch_number: {}, element_size: {} bytes, "
+      "element_number: 2^{} = {}",
+      params.poly_degree, batch_number, params.element_size,
+      std::log2(params.element_number), params.element_number);
+
   // size_t batch_number = 256;
   double factor = 1.5;
   size_t hash_num = 3;
@@ -109,8 +116,6 @@ TEST_P(SealMultiPirTest, Works) {
   ::psi::seal_pir::MultiQueryOptions options{
       {params.poly_degree, element_number, element_size}, batch_number};
 
-  SPDLOG_INFO("element_number:{}", options.seal_options.element_number);
-
   ::psi::seal_pir::MultiQueryServer mpir_server(options, cuckoo_params,
                                                 seed_server);
 
@@ -136,6 +141,8 @@ TEST_P(SealMultiPirTest, Works) {
   mpir_server.SetGaloisKeys(galkey);
 
   // do pir query/answer
+  const auto pir_start_time = std::chrono::system_clock::now();
+
   std::future<void> pir_service_func =
       std::async([&] { return mpir_server.DoMultiPirAnswer(ctxs[0]); });
   std::future<std::vector<std::vector<uint8_t>>> pir_client_func = std::async(
@@ -144,6 +151,11 @@ TEST_P(SealMultiPirTest, Works) {
   pir_service_func.get();
   std::vector<std::vector<uint8_t>> query_reply_bytes = pir_client_func.get();
 
+  const auto pir_end_time = std::chrono::system_clock::now();
+  const DurationMillis pir_time = pir_end_time - pir_start_time;
+
+  SPDLOG_INFO("pir time(online) : {} ms", pir_time.count());
+
   EXPECT_EQ(query_reply_bytes.size(), query_index.size());
 
   for (size_t idx = 0; idx < query_reply_bytes.size(); ++idx) {
@@ -171,11 +183,25 @@ TEST_P(SealMultiPirTest, Works) {
 
 INSTANTIATE_TEST_SUITE_P(
     Works_Instances, SealMultiPirTest,
-    testing::Values(TestParams{32, 1000},       // element size default 288B
-                    TestParams{32, 1000, 10},   //
-                    TestParams{32, 1000, 400},  //
-                    TestParams{64, 10000},      // element size default 288B
-                    TestParams{64, 10000, 20})  //
+    testing::Values(TestParams{32, 1000},  // element size default 288B
+                    TestParams{32, 1000, 288, 8192},
+
+                    TestParams{32, 1000, 10},        //
+                    TestParams{32, 1000, 10, 8192},  //
+
+                    TestParams{32, 1000, 400},        //
+                    TestParams{32, 1000, 400, 8192},  //
+
+                    TestParams{64, 10000},  // element size default 288B
+                    TestParams{64, 10000, 288, 8192},
+
+                    TestParams{64, 10000, 20},
+                    TestParams{64, 10000, 20, 8192},
+
+                    // large data num
+                    TestParams{64, 1 << 20, 20},
+                    TestParams{64, 1 << 21, 20},
+                    TestParams{64, 1 << 22, 20})  //
 );
 
 }  // namespace psi::seal_pir