From 4f351f6c9ebad3e34a8130b9eb4e5522e100686f Mon Sep 17 00:00:00 2001 From: anakinxc Date: Sat, 25 Nov 2023 17:34:50 +0800 Subject: [PATCH 1/7] Add macos ut --- .circleci/test.yml | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/.circleci/test.yml b/.circleci/test.yml index 8006c56..695d5c9 100644 --- a/.circleci/test.yml +++ b/.circleci/test.yml @@ -28,7 +28,7 @@ jobs: # Specify the execution environment. You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub. # See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor docker: - - image: secretflow/spu-ci:latest + - image: secretflow/ubuntu-base-ci:latest resource_class: 2xlarge+ shell: /bin/bash --login -eo pipefail # Add steps to the job @@ -84,6 +84,39 @@ jobs: path: test_binary.tar.gz - store_artifacts: path: test_logs.tar.gz + macos_ut: + macos: + xcode: 15.1 + resource_class: macos.m1.large.gen1 + steps: + - checkout + - run: + name: Checkout devtools + command: git clone https://github.com/secretflow/devtools.git ../devtools + - run: + name: "Install homebrew dependencies" + command: brew install bazelisk cmake ninja libomp wget + - run: + name: "build" + command: bazel build //... -c opt --ui_event_filters=-info,-debug,-warning --jobs 12 + - run: + name: "test" + command: | + set +e + declare -i test_status + bazel test //... -c opt --ui_event_filters=-info,-debug,-warning --test_output=errors --jobs 12 | tee test_result.log; test_status=${PIPESTATUS[0]} + + sh ../devtools/rename-junit-xml.sh + find bazel-bin/ -perm +111 -type f -name "*_test" -print0 | xargs -0 tar -cvzf test_binary.tar.gz + find bazel-testlogs/ -type f -name "test.log" -print0 | xargs -0 tar -cvzf test_logs.tar.gz + + exit ${test_status} + - store_test_results: + path: test-results + - store_artifacts: + path: test_binary.tar.gz + - store_artifacts: + path: test_logs.tar.gz # Invoke jobs via workflows # See: https://circleci.com/docs/2.0/configuration-reference/#workflows @@ -92,3 +125,4 @@ workflows: when: << pipeline.parameters.build-and-run >> jobs: - linux_ut + - macos_ut From 4f7256aed3d38b1bb6f1967974235079769635e9 Mon Sep 17 00:00:00 2001 From: anakinxc Date: Sat, 25 Nov 2023 17:57:26 +0800 Subject: [PATCH 2/7] Various macOS build fixes --- .bazelrc | 35 ++++-------------------- psi/psi/core/BUILD.bazel | 6 +--- psi/psi/core/vole_psi/BUILD.bazel | 6 +++- psi/psi/core/vole_psi/okvs/baxos.h | 3 +- psi/psi/core/vole_psi/okvs/paxos.cc | 6 ++-- psi/psi/core/vole_psi/okvs/paxos_hash.cc | 10 +++---- psi/psi/cryptor/BUILD.bazel | 2 +- psi/psi/operator/BUILD.bazel | 6 +++- psi/psi/operator/rr22_2party_psi.cc | 3 +- psi/psi/rr22/BUILD.bazel | 6 +++- psi/psi/rr22/common.cc | 2 +- psi/psi/utils/arrow_csv_batch_provider.h | 2 +- psi/psi/utils/csv_checker.cc | 4 +++ psi/psi/utils/multiplex_disk_cache.cc | 1 + 14 files changed, 41 insertions(+), 51 deletions(-) diff --git a/.bazelrc b/.bazelrc index ecbc6b3..5b53764 100644 --- a/.bazelrc +++ b/.bazelrc @@ -14,9 +14,6 @@ common --experimental_repo_remote_exec -# Required by OpenXLA -build --nocheck_visibility - build --incompatible_new_actions_api=false build --copt=-fdiagnostics-color=always build --enable_platform_specific_config @@ -30,7 +27,6 @@ build --copt=-fstack-protector-strong build:linux --copt=-Wl,-z,noexecstack build:macos --copt=-Wa,--noexecstack - test --keep_going test --test_output=errors @@ -43,34 +39,15 @@ build:linux-release --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a:-l%:libgcc.a # platform specific config # Bazel will automatic pick platform config since we have enable_platform_specific_config set -build:macos --copt="-Xpreprocessor -fopenmp" build:macos --copt=-Wno-unused-command-line-argument build:macos --features=-supports_dynamic_linker -build:macos --cxxopt -Wno-deprecated-enum-enum-conversion -build:macos --cxxopt -Wno-deprecated-anon-enum-enum-conversion -build:macos --macos_minimum_os=11.0 -build:macos --host_macos_minimum_os=11.0 +build:macos --macos_minimum_os=12.0 +build:macos --host_macos_minimum_os=12.0 +# Enable openmp +build:macos --copt="-Xpreprocessor -fopenmp" build:linux --copt=-fopenmp build:linux --linkopt=-fopenmp -build:asan --strip=never -build:asan --copt -fno-sanitize-recover=all -build:asan --copt -fsanitize=address -build:asan --copt -Og -build:asan --copt -g -build:asan --copt -fno-omit-frame-pointer -build:asan --linkopt -fsanitize=address -build:asan --linkopt -static-libasan - -build:ubsan --strip=never -build:ubsan --copt -fno-sanitize-recover=all -build:ubsan --copt -fsanitize=undefined -build:ubsan --copt -Og -build:ubsan --copt -g -build:ubsan --copt -fno-omit-frame-pointer -build:ubsan --linkopt -fsanitize=undefined -build:ubsan --linkopt -static-libubsan - -build:macos-asan --features=asan -build:macos-ubsan --features=ubsan +build:asan --features=asan +build:ubsan --features=ubsan diff --git a/psi/psi/core/BUILD.bazel b/psi/psi/core/BUILD.bazel index 7accbb2..e5460e8 100644 --- a/psi/psi/core/BUILD.bazel +++ b/psi/psi/core/BUILD.bazel @@ -156,18 +156,14 @@ psi_cc_library( name = "ecdh_oprf_psi", srcs = ["ecdh_oprf_psi.cc"], hdrs = ["ecdh_oprf_psi.h"], - linkopts = select({ - "@bazel_tools//src/conditions:darwin": [], - "//conditions:default": ["-fopenmp"], - }), deps = [ ":communication", - "@yacl//yacl/base:exception", "//psi/psi/core/ecdh_oprf:ecdh_oprf_selector", "//psi/psi/utils:batch_provider", "//psi/psi/utils:ec_point_store", "//psi/psi/utils:ub_psi_cache", "@com_google_absl//absl/strings", + "@yacl//yacl/base:exception", "@yacl//yacl/link", "@yacl//yacl/utils:parallel", ] + select({ diff --git a/psi/psi/core/vole_psi/BUILD.bazel b/psi/psi/core/vole_psi/BUILD.bazel index 09e8864..e492d2b 100644 --- a/psi/psi/core/vole_psi/BUILD.bazel +++ b/psi/psi/core/vole_psi/BUILD.bazel @@ -70,7 +70,11 @@ psi_cc_binary( "//psi/psi/utils:test_utils", "@com_github_google_benchmark//:benchmark_main", "@yacl//yacl/crypto/utils:rand", - ], + ] + select({ + "@bazel_tools//src/conditions:darwin_x86_64": ["@local_homebrew_x64//:openmp"], + "@bazel_tools//src/conditions:darwin_arm64": ["@local_homebrew_arm64//:openmp"], + "//conditions:default": [], + }), ) psi_cc_library( diff --git a/psi/psi/core/vole_psi/okvs/baxos.h b/psi/psi/core/vole_psi/okvs/baxos.h index 5f28ced..9fb714f 100644 --- a/psi/psi/core/vole_psi/okvs/baxos.h +++ b/psi/psi/core/vole_psi/okvs/baxos.h @@ -79,7 +79,8 @@ class Baxos { template ValueType Decode(const uint128_t& input, absl::Span p) { ValueType r; - decode(absl::Span(&input, 1), absl::Span(&r, 1), p); + decode(absl::Span(&input, 1), absl::Span(&r, 1), + p); return r; } diff --git a/psi/psi/core/vole_psi/okvs/paxos.cc b/psi/psi/core/vole_psi/okvs/paxos.cc index c4726b5..88d302b 100644 --- a/psi/psi/core/vole_psi/okvs/paxos.cc +++ b/psi/psi/core/vole_psi/okvs/paxos.cc @@ -98,9 +98,9 @@ inline std::vector MatrixGf128Inv(std::vector mtx, return inv; } -inline std::vector MatrixGf128Mul(const std::vector& m0, - const std::vector& m1, - size_t row_size, size_t col_size) { +[[maybe_unused]] inline std::vector MatrixGf128Mul( + const std::vector& m0, const std::vector& m1, + size_t row_size, size_t col_size) { YACL_ENFORCE(row_size == col_size); std::vector ret(row_size, col_size); diff --git a/psi/psi/core/vole_psi/okvs/paxos_hash.cc b/psi/psi/core/vole_psi/okvs/paxos_hash.cc index 405af8a..65b84b9 100644 --- a/psi/psi/core/vole_psi/okvs/paxos_hash.cc +++ b/psi/psi/core/vole_psi/okvs/paxos_hash.cc @@ -309,10 +309,10 @@ void PaxosHash::HashBuildRow1(const uint128_t& input, BuildRow(*hash, rows); } -template class PaxosHash; -template class PaxosHash; -template class PaxosHash; -template class PaxosHash; -template class PaxosHash; +template struct PaxosHash; +template struct PaxosHash; +template struct PaxosHash; +template struct PaxosHash; +template struct PaxosHash; } // namespace psi::psi::okvs diff --git a/psi/psi/cryptor/BUILD.bazel b/psi/psi/cryptor/BUILD.bazel index c9517f1..c3d6a1b 100644 --- a/psi/psi/cryptor/BUILD.bazel +++ b/psi/psi/cryptor/BUILD.bazel @@ -95,9 +95,9 @@ psi_cc_library( ":fourq_cryptor", ":sm2_cryptor", ":sodium_curve25519_cryptor", + "//psi/proto:psi_cc_proto", "//psi/psi:prelude", "@yacl//yacl/utils:platform_utils", - "//psi/proto:psi_cc_proto", ] + select({ "@platforms//cpu:x86_64": [ ":ipp_ecc_cryptor", diff --git a/psi/psi/operator/BUILD.bazel b/psi/psi/operator/BUILD.bazel index 616810b..833e585 100644 --- a/psi/psi/operator/BUILD.bazel +++ b/psi/psi/operator/BUILD.bazel @@ -119,7 +119,11 @@ psi_cc_library( ":factory", "//psi/psi/core/vole_psi:rr22_psi", "@yacl//yacl/utils:parallel", - ], + ] + select({ + "@bazel_tools//src/conditions:darwin_x86_64": ["@local_homebrew_x64//:openmp"], + "@bazel_tools//src/conditions:darwin_arm64": ["@local_homebrew_arm64//:openmp"], + "//conditions:default": [], + }), alwayslink = True, ) diff --git a/psi/psi/operator/rr22_2party_psi.cc b/psi/psi/operator/rr22_2party_psi.cc index 64230b0..624fe54 100644 --- a/psi/psi/operator/rr22_2party_psi.cc +++ b/psi/psi/operator/rr22_2party_psi.cc @@ -14,10 +14,9 @@ #include "psi/psi/operator/rr22_2party_psi.h" -#include - #include +#include "omp.h" #include "yacl/crypto/base/hash/hash_utils.h" #include "yacl/utils/parallel.h" diff --git a/psi/psi/rr22/BUILD.bazel b/psi/psi/rr22/BUILD.bazel index 2d09ef0..2c25367 100644 --- a/psi/psi/rr22/BUILD.bazel +++ b/psi/psi/rr22/BUILD.bazel @@ -25,7 +25,11 @@ psi_cc_library( "//psi/psi:bucket", "//psi/psi:recovery", "//psi/psi/core/vole_psi:rr22_psi", - ], + ] + select({ + "@bazel_tools//src/conditions:darwin_x86_64": ["@local_homebrew_x64//:openmp"], + "@bazel_tools//src/conditions:darwin_arm64": ["@local_homebrew_arm64//:openmp"], + "//conditions:default": [], + }), ) psi_cc_library( diff --git a/psi/psi/rr22/common.cc b/psi/psi/rr22/common.cc index f5554a1..02efab7 100644 --- a/psi/psi/rr22/common.cc +++ b/psi/psi/rr22/common.cc @@ -14,7 +14,7 @@ #include "psi/psi/rr22/common.h" -#include +#include "omp.h" #include "psi/psi/bucket.h" diff --git a/psi/psi/utils/arrow_csv_batch_provider.h b/psi/psi/utils/arrow_csv_batch_provider.h index e9c9cfb..d960b18 100644 --- a/psi/psi/utils/arrow_csv_batch_provider.h +++ b/psi/psi/utils/arrow_csv_batch_provider.h @@ -34,7 +34,7 @@ class ArrowCsvBatchProvider : public IBasicBatchProvider { [[nodiscard]] size_t row_cnt() const { return row_cnt_; } - [[nodiscard]] size_t batch_size() const { return batch_size_; } + [[nodiscard]] size_t batch_size() const override { return batch_size_; } private: void Init(); diff --git a/psi/psi/utils/csv_checker.cc b/psi/psi/utils/csv_checker.cc index 4ec56b3..467e480 100644 --- a/psi/psi/utils/csv_checker.cc +++ b/psi/psi/utils/csv_checker.cc @@ -253,7 +253,11 @@ CheckCsvReport CheckCsv(const std::string& input_file_path, // Get hash digest. if (generate_key_hash_digest) { +#ifdef __APPLE__ + std::string command(fmt::format("shasum -a 256 {}", output_file_path)); +#else std::string command(fmt::format("sha256sum {}", output_file_path)); +#endif std::array buffer; std::string result; FILE* pipe = popen(command.c_str(), "r"); diff --git a/psi/psi/utils/multiplex_disk_cache.cc b/psi/psi/utils/multiplex_disk_cache.cc index 7fcac99..5c7399f 100644 --- a/psi/psi/utils/multiplex_disk_cache.cc +++ b/psi/psi/utils/multiplex_disk_cache.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include "spdlog/spdlog.h" #include "yacl/base/exception.h" From c985a969f126499505d351e25b2c2f5e75157a44 Mon Sep 17 00:00:00 2001 From: anakinxc Date: Sat, 25 Nov 2023 19:05:16 +0800 Subject: [PATCH 3/7] Use uuid to avoid time collision --- psi/psi/BUILD.bazel | 1 + psi/psi/interface.cc | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/psi/psi/BUILD.bazel b/psi/psi/BUILD.bazel index 1ea1a23..9616ebf 100644 --- a/psi/psi/BUILD.bazel +++ b/psi/psi/BUILD.bazel @@ -136,6 +136,7 @@ psi_cc_library( "@com_github_google_perfetto//:perfetto", "@com_google_absl//absl/status", "@yacl//yacl/link", + "@boost//:uuid" ], ) diff --git a/psi/psi/interface.cc b/psi/psi/interface.cc index 97041db..fd45a79 100644 --- a/psi/psi/interface.cc +++ b/psi/psi/interface.cc @@ -36,6 +36,10 @@ #include "psi/proto/psi.pb.h" +#include "boost/uuid/uuid.hpp" // uuid class +#include "boost/uuid/uuid_generators.hpp" // generators +#include // streaming operators etc. + namespace psi::psi { namespace { @@ -49,13 +53,15 @@ std::string GenerateSortedIndexFileName(v2::Role role) { } std::string GenerateIndexFileName() { + boost::uuids::random_generator uuid_generator; return fmt::format("psi_index_{}.csv", - std::to_string(absl::ToUnixNanos(absl::Now()))); + boost::uuids::to_string(uuid_generator())); } std::string GenerateSortedIndexFileName() { + boost::uuids::random_generator uuid_generator; return fmt::format("sorted_psi_index_{}.csv", - std::to_string(absl::ToUnixNanos(absl::Now()))); + boost::uuids::to_string(uuid_generator())); } } // namespace From a7b70c4b8313427e5d8a7c622dba5963b66338bd Mon Sep 17 00:00:00 2001 From: anakinxc Date: Sat, 25 Nov 2023 19:08:04 +0800 Subject: [PATCH 4/7] format --- psi/psi/interface.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/psi/psi/interface.cc b/psi/psi/interface.cc index fd45a79..d3738be 100644 --- a/psi/psi/interface.cc +++ b/psi/psi/interface.cc @@ -23,6 +23,9 @@ #include #include "absl/time/time.h" +#include "boost/uuid/uuid.hpp" +#include "boost/uuid/uuid_generators.hpp" +#include "boost/uuid/uuid_io.hpp" #include "google/protobuf/util/message_differencer.h" #include "yacl/base/exception.h" #include "yacl/link/link.h" @@ -36,10 +39,6 @@ #include "psi/proto/psi.pb.h" -#include "boost/uuid/uuid.hpp" // uuid class -#include "boost/uuid/uuid_generators.hpp" // generators -#include // streaming operators etc. - namespace psi::psi { namespace { From 8ee014e43c714ac01c761516f640fa7e91856863 Mon Sep 17 00:00:00 2001 From: anakinxc Date: Sat, 25 Nov 2023 19:09:57 +0800 Subject: [PATCH 5/7] No more spu-ci --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 84cf5a5..81eb0a1 100644 --- a/README.md +++ b/README.md @@ -17,20 +17,20 @@ This repo is formerly psi/pir part from secretflow/spu repo. #### Docker -We use the same dev docker from secretflow/spu. +We use the same dev docker from secretflow/ubuntu-base-ci. ```sh ## start container -docker run -d -it --name spu-dev-$(whoami) \ +docker run -d -it --name psi-dev-$(whoami) \ --mount type=bind,source="$(pwd)",target=/home/admin/dev/ \ -w /home/admin/dev \ --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \ --cap-add=NET_ADMIN \ --privileged=true \ - secretflow/spu-ci:latest + secretflow/ubuntu-base-ci:latest # attach to build container -docker exec -it spu-dev-$(whoami) bash +docker exec -it psi-dev-$(whoami) bash ``` #### Linux From c9b5fbd52f436a704d343a81ed2fad889d9bac82 Mon Sep 17 00:00:00 2001 From: anakinxc Date: Sat, 25 Nov 2023 19:11:29 +0800 Subject: [PATCH 6/7] format --- README.md | 1 + psi/psi/BUILD.bazel | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 81eb0a1..d0da8af 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ docker run -d -it --name psi-dev-$(whoami) \ --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \ --cap-add=NET_ADMIN \ --privileged=true \ + --entrypoint="bash" \ secretflow/ubuntu-base-ci:latest # attach to build container diff --git a/psi/psi/BUILD.bazel b/psi/psi/BUILD.bazel index 9616ebf..7008f1a 100644 --- a/psi/psi/BUILD.bazel +++ b/psi/psi/BUILD.bazel @@ -133,10 +133,10 @@ psi_cc_library( "//psi/psi:bucket_psi", "//psi/psi/utils:index_store", "//psi/psi/utils:inner_join", + "@boost//:uuid", "@com_github_google_perfetto//:perfetto", "@com_google_absl//absl/status", "@yacl//yacl/link", - "@boost//:uuid" ], ) From d9cc78d7adc3a60790f4f02070df3679eabac6a3 Mon Sep 17 00:00:00 2001 From: anakinxc Date: Sat, 25 Nov 2023 19:34:01 +0800 Subject: [PATCH 7/7] fix --- .circleci/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/test.yml b/.circleci/test.yml index 695d5c9..bfae6df 100644 --- a/.circleci/test.yml +++ b/.circleci/test.yml @@ -95,7 +95,7 @@ jobs: command: git clone https://github.com/secretflow/devtools.git ../devtools - run: name: "Install homebrew dependencies" - command: brew install bazelisk cmake ninja libomp wget + command: brew install bazelisk cmake ninja libomp wget md5sha1sum - run: name: "build" command: bazel build //... -c opt --ui_event_filters=-info,-debug,-warning --jobs 12