Fix macOS issues (#3)

* Add macos ut

* Various macOS build fixes

* Use uuid to avoid time collision

* format

* No more spu-ci

* format

* fix

.bazelrc

@@ -14,9 +14,6 @@
 
 common --experimental_repo_remote_exec
 
-# Required by OpenXLA
-build --nocheck_visibility
-
 build --incompatible_new_actions_api=false
 build --copt=-fdiagnostics-color=always
 build --enable_platform_specific_config
@@ -30,7 +27,6 @@ build --copt=-fstack-protector-strong
 build:linux --copt=-Wl,-z,noexecstack
 build:macos --copt=-Wa,--noexecstack
 
-
 test --keep_going
 test --test_output=errors
 
@@ -43,34 +39,15 @@ build:linux-release --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a:-l%:libgcc.a
 
 # platform specific config
 # Bazel will automatic pick platform config since we have enable_platform_specific_config set
-build:macos --copt="-Xpreprocessor -fopenmp"
 build:macos --copt=-Wno-unused-command-line-argument
 build:macos --features=-supports_dynamic_linker
-build:macos --cxxopt -Wno-deprecated-enum-enum-conversion
-build:macos --cxxopt -Wno-deprecated-anon-enum-enum-conversion
-build:macos --macos_minimum_os=11.0
-build:macos --host_macos_minimum_os=11.0
+build:macos --macos_minimum_os=12.0
+build:macos --host_macos_minimum_os=12.0
 
+# Enable openmp
+build:macos --copt="-Xpreprocessor -fopenmp"
 build:linux --copt=-fopenmp
 build:linux --linkopt=-fopenmp
 
-build:asan --strip=never
-build:asan --copt -fno-sanitize-recover=all
-build:asan --copt -fsanitize=address
-build:asan --copt -Og
-build:asan --copt -g
-build:asan --copt -fno-omit-frame-pointer
-build:asan --linkopt -fsanitize=address
-build:asan --linkopt -static-libasan
-
-build:ubsan --strip=never
-build:ubsan --copt -fno-sanitize-recover=all
-build:ubsan --copt -fsanitize=undefined
-build:ubsan --copt -Og
-build:ubsan --copt -g
-build:ubsan --copt -fno-omit-frame-pointer
-build:ubsan --linkopt -fsanitize=undefined
-build:ubsan --linkopt -static-libubsan
-
-build:macos-asan --features=asan
-build:macos-ubsan --features=ubsan
+build:asan --features=asan
+build:ubsan --features=ubsan

.circleci/test.yml

@@ -28,7 +28,7 @@ jobs:
     # Specify the execution environment. You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub.
     # See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor
     docker:
-      - image: secretflow/spu-ci:latest
+      - image: secretflow/ubuntu-base-ci:latest
     resource_class: 2xlarge+
     shell: /bin/bash --login -eo pipefail
     # Add steps to the job
@@ -84,6 +84,39 @@ jobs:
           path: test_binary.tar.gz
       - store_artifacts:
           path: test_logs.tar.gz
+  macos_ut:
+    macos:
+      xcode: 15.1
+    resource_class: macos.m1.large.gen1
+    steps:
+      - checkout
+      - run:
+          name: Checkout devtools
+          command: git clone https://github.com/secretflow/devtools.git ../devtools
+      - run:
+          name: "Install homebrew dependencies"
+          command: brew install bazelisk cmake ninja libomp wget md5sha1sum
+      - run:
+          name: "build"
+          command: bazel build //... -c opt --ui_event_filters=-info,-debug,-warning --jobs 12
+      - run:
+          name: "test"
+          command: |
+            set +e
+            declare -i test_status
+            bazel test //... -c opt --ui_event_filters=-info,-debug,-warning --test_output=errors --jobs 12 | tee test_result.log; test_status=${PIPESTATUS[0]}
+
+            sh ../devtools/rename-junit-xml.sh
+            find bazel-bin/ -perm +111 -type f -name "*_test"  -print0 | xargs -0 tar -cvzf test_binary.tar.gz
+            find bazel-testlogs/ -type f -name "test.log"  -print0 | xargs -0 tar -cvzf test_logs.tar.gz
+
+            exit ${test_status}
+      - store_test_results:
+          path: test-results
+      - store_artifacts:
+          path: test_binary.tar.gz
+      - store_artifacts:
+          path: test_logs.tar.gz
 
 # Invoke jobs via workflows
 # See: https://circleci.com/docs/2.0/configuration-reference/#workflows
@@ -92,3 +125,4 @@ workflows:
     when: << pipeline.parameters.build-and-run >>
     jobs:
       - linux_ut
+      - macos_ut

README.md

@@ -17,20 +17,21 @@ This repo is formerly psi/pir part from secretflow/spu repo.
 
 #### Docker
 
-We use the same dev docker from secretflow/spu.
+We use the same dev docker from secretflow/ubuntu-base-ci.
 
 ```sh
 ## start container
-docker run -d -it --name spu-dev-$(whoami) \
+docker run -d -it --name psi-dev-$(whoami) \
          --mount type=bind,source="$(pwd)",target=/home/admin/dev/ \
          -w /home/admin/dev \
          --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \
          --cap-add=NET_ADMIN \
          --privileged=true \
-         secretflow/spu-ci:latest
+         --entrypoint="bash" \
+         secretflow/ubuntu-base-ci:latest
 
 # attach to build container
-docker exec -it spu-dev-$(whoami) bash
+docker exec -it psi-dev-$(whoami) bash
 ```
 
 #### Linux

psi/psi/BUILD.bazel

@@ -133,6 +133,7 @@ psi_cc_library(
         "//psi/psi:bucket_psi",
         "//psi/psi/utils:index_store",
         "//psi/psi/utils:inner_join",
+        "@boost//:uuid",
         "@com_github_google_perfetto//:perfetto",
         "@com_google_absl//absl/status",
         "@yacl//yacl/link",

psi/psi/core/BUILD.bazel

@@ -156,18 +156,14 @@ psi_cc_library(
     name = "ecdh_oprf_psi",
     srcs = ["ecdh_oprf_psi.cc"],
     hdrs = ["ecdh_oprf_psi.h"],
-    linkopts = select({
-        "@bazel_tools//src/conditions:darwin": [],
-        "//conditions:default": ["-fopenmp"],
-    }),
     deps = [
         ":communication",
-        "@yacl//yacl/base:exception",
         "//psi/psi/core/ecdh_oprf:ecdh_oprf_selector",
         "//psi/psi/utils:batch_provider",
         "//psi/psi/utils:ec_point_store",
         "//psi/psi/utils:ub_psi_cache",
         "@com_google_absl//absl/strings",
+        "@yacl//yacl/base:exception",
         "@yacl//yacl/link",
         "@yacl//yacl/utils:parallel",
     ] + select({

psi/psi/core/vole_psi/BUILD.bazel

@@ -70,7 +70,11 @@ psi_cc_binary(
         "//psi/psi/utils:test_utils",
         "@com_github_google_benchmark//:benchmark_main",
         "@yacl//yacl/crypto/utils:rand",
-    ],
+    ] + select({
+        "@bazel_tools//src/conditions:darwin_x86_64": ["@local_homebrew_x64//:openmp"],
+        "@bazel_tools//src/conditions:darwin_arm64": ["@local_homebrew_arm64//:openmp"],
+        "//conditions:default": [],
+    }),
 )
 
 psi_cc_library(

psi/psi/core/vole_psi/okvs/baxos.h

@@ -79,7 +79,8 @@ class Baxos {
   template <typename ValueType>
   ValueType Decode(const uint128_t& input, absl::Span<const ValueType> p) {
     ValueType r;
-    decode(absl::Span<uint128_t>(&input, 1), absl::Span<ValueType>(&r, 1), p);
+    decode(absl::Span<const uint128_t>(&input, 1), absl::Span<ValueType>(&r, 1),
+           p);
     return r;
   }
 

psi/psi/core/vole_psi/okvs/paxos.cc

@@ -98,9 +98,9 @@ inline std::vector<uint128_t> MatrixGf128Inv(std::vector<uint128_t> mtx,
   return inv;
 }
 
-inline std::vector<uint128_t> MatrixGf128Mul(const std::vector<uint128_t>& m0,
-                                             const std::vector<uint128_t>& m1,
-                                             size_t row_size, size_t col_size) {
+[[maybe_unused]] inline std::vector<uint128_t> MatrixGf128Mul(
+    const std::vector<uint128_t>& m0, const std::vector<uint128_t>& m1,
+    size_t row_size, size_t col_size) {
   YACL_ENFORCE(row_size == col_size);
 
   std::vector<uint128_t> ret(row_size, col_size);

psi/psi/core/vole_psi/okvs/paxos_hash.cc

@@ -309,10 +309,10 @@ void PaxosHash<IdxType>::HashBuildRow1(const uint128_t& input,
   BuildRow(*hash, rows);
 }
 
-template class PaxosHash<uint8_t>;
-template class PaxosHash<uint16_t>;
-template class PaxosHash<uint32_t>;
-template class PaxosHash<uint64_t>;
-template class PaxosHash<uint128_t>;
+template struct PaxosHash<uint8_t>;
+template struct PaxosHash<uint16_t>;
+template struct PaxosHash<uint32_t>;
+template struct PaxosHash<uint64_t>;
+template struct PaxosHash<uint128_t>;
 
 }  // namespace psi::psi::okvs

psi/psi/cryptor/BUILD.bazel

@@ -95,9 +95,9 @@ psi_cc_library(
         ":fourq_cryptor",
         ":sm2_cryptor",
         ":sodium_curve25519_cryptor",
+        "//psi/proto:psi_cc_proto",
         "//psi/psi:prelude",
         "@yacl//yacl/utils:platform_utils",
-        "//psi/proto:psi_cc_proto",
     ] + select({
         "@platforms//cpu:x86_64": [
             ":ipp_ecc_cryptor",

psi/psi/interface.cc

@@ -23,6 +23,9 @@
 #include <numeric>
 
 #include "absl/time/time.h"
+#include "boost/uuid/uuid.hpp"
+#include "boost/uuid/uuid_generators.hpp"
+#include "boost/uuid/uuid_io.hpp"
 #include "google/protobuf/util/message_differencer.h"
 #include "yacl/base/exception.h"
 #include "yacl/link/link.h"
@@ -49,13 +52,15 @@ std::string GenerateSortedIndexFileName(v2::Role role) {
 }
 
 std::string GenerateIndexFileName() {
+  boost::uuids::random_generator uuid_generator;
   return fmt::format("psi_index_{}.csv",
-                     std::to_string(absl::ToUnixNanos(absl::Now())));
+                     boost::uuids::to_string(uuid_generator()));
 }
 
 std::string GenerateSortedIndexFileName() {
+  boost::uuids::random_generator uuid_generator;
   return fmt::format("sorted_psi_index_{}.csv",
-                     std::to_string(absl::ToUnixNanos(absl::Now())));
+                     boost::uuids::to_string(uuid_generator()));
 }
 
 }  // namespace

psi/psi/operator/BUILD.bazel

@@ -119,7 +119,11 @@ psi_cc_library(
         ":factory",
         "//psi/psi/core/vole_psi:rr22_psi",
         "@yacl//yacl/utils:parallel",
-    ],
+    ] + select({
+        "@bazel_tools//src/conditions:darwin_x86_64": ["@local_homebrew_x64//:openmp"],
+        "@bazel_tools//src/conditions:darwin_arm64": ["@local_homebrew_arm64//:openmp"],
+        "//conditions:default": [],
+    }),
     alwayslink = True,
 )
 

psi/psi/operator/rr22_2party_psi.cc

@@ -14,10 +14,9 @@
 
 #include "psi/psi/operator/rr22_2party_psi.h"
 
-#include <omp.h>
-
 #include <chrono>
 
+#include "omp.h"
 #include "yacl/crypto/base/hash/hash_utils.h"
 #include "yacl/utils/parallel.h"
 

psi/psi/rr22/BUILD.bazel

@@ -25,7 +25,11 @@ psi_cc_library(
         "//psi/psi:bucket",
         "//psi/psi:recovery",
         "//psi/psi/core/vole_psi:rr22_psi",
-    ],
+    ] + select({
+        "@bazel_tools//src/conditions:darwin_x86_64": ["@local_homebrew_x64//:openmp"],
+        "@bazel_tools//src/conditions:darwin_arm64": ["@local_homebrew_arm64//:openmp"],
+        "//conditions:default": [],
+    }),
 )
 
 psi_cc_library(

psi/psi/rr22/common.cc

@@ -14,7 +14,7 @@
 
 #include "psi/psi/rr22/common.h"
 
-#include <omp.h>
+#include "omp.h"
 
 #include "psi/psi/bucket.h"
 

psi/psi/utils/arrow_csv_batch_provider.h

@@ -34,7 +34,7 @@ class ArrowCsvBatchProvider : public IBasicBatchProvider {
 
   [[nodiscard]] size_t row_cnt() const { return row_cnt_; }
 
-  [[nodiscard]] size_t batch_size() const { return batch_size_; }
+  [[nodiscard]] size_t batch_size() const override { return batch_size_; }
 
  private:
   void Init();

psi/psi/utils/csv_checker.cc

@@ -253,7 +253,11 @@ CheckCsvReport CheckCsv(const std::string& input_file_path,
 
   // Get hash digest.
   if (generate_key_hash_digest) {
+#ifdef __APPLE__
+    std::string command(fmt::format("shasum -a 256 {}", output_file_path));
+#else
     std::string command(fmt::format("sha256sum {}", output_file_path));
+#endif
     std::array<char, 128> buffer;
     std::string result;
     FILE* pipe = popen(command.c_str(), "r");

psi/psi/utils/multiplex_disk_cache.cc

@@ -20,6 +20,7 @@
 #include <ctime>
 #include <memory>
 #include <random>
+#include <sstream>
 
 #include "spdlog/spdlog.h"
 #include "yacl/base/exception.h"