From edbfa667420ef758269c75c9e522fa35d8676914 Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 20 Dec 2024 15:28:27 -0600 Subject: [PATCH 1/4] print out sanitizer and additional flags --- CMakeLists.txt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b0ab0e31..0c437f3e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -155,6 +155,9 @@ endif() target_link_libraries(${PROJECT_NAME} PUBLIC ${LINK_LIBS}) +# --------------------------------------------------------------- +# Sanitizers +# --------------------------------------------------------------- IF(CMAKE_BUILD_TYPE MATCHES Debug AND QUAKE_USE_TSAN) message("Using thread sanitizer") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread") @@ -212,14 +215,20 @@ endif() # --------------------------------------------------------------- message(STATUS "--------- Final Configuration Summary ---------") message(STATUS "Build type: ${CMAKE_BUILD_TYPE}") -message(STATUS "GPU Enabled: ${FAISS_ENABLE_GPU}") +message(STATUS "GPU Enabled: ${QUAKE_ENABLE_GPU}") +message(STATUS "AVX512 Enabled: ${QUAKE_USE_AVX512}") message(STATUS "NUMA Enabled: ${QUAKE_USE_NUMA}") message(STATUS "Python used: ${Python3_EXECUTABLE}") message(STATUS "Torch Path: ${TorchPath}") +message(STATUS "OpenMP Enabled: ${OpenMP_CXX_FOUND}") +message(STATUS "OpenMP CXX Flags: ${OpenMP_CXX_FLAGS}") message(STATUS "MKL_ARCH: ${MKL_ARCH}") message(STATUS "MKL_ROOT: ${MKL_ROOT}") message(STATUS "MKL_LINK: ${MKL_LINK}") message(STATUS "MKL_INTERFACE_FULL: ${MKL_INTERFACE_FULL}") message(STATUS "MKL_THREADING: ${MKL_THREADING}") message(STATUS "MKL_MPI: ${MKL_MPI}") +message(STATUS "Flags: ${CMAKE_CXX_FLAGS}") +message(STATUS "Address Sanitizer: ${QUAKE_USE_ASAN}") +message(STATUS "Thread Sanitizer: ${QUAKE_USE_TSAN}") message(STATUS "------------------------------------------------") \ No newline at end of file From e69a8645d89480abea396c5496bfbc417720f5bf Mon Sep 17 00:00:00 2001 From: Jason Date: Sat, 21 Dec 2024 15:10:56 -0600 Subject: [PATCH 2/4] remove leak flag from sanitizer --- CMakeLists.txt | 4 ++-- test/cpp/dynamic_ivf.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c437f3e..2ce5b934 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -166,8 +166,8 @@ ENDIF(CMAKE_BUILD_TYPE MATCHES Debug AND QUAKE_USE_ASAN) IF(CMAKE_BUILD_TYPE MATCHES Debug AND QUAKE_USE_ASAN) message("Using address sanitizer") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fsanitize=leak") - set(CMAKE_MODULE_LINKER_FLAGS "{$CMAKE_MODULE_LINKER_FLAGS} -fsanitize=address -fsanitize=leak") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") + set(CMAKE_MODULE_LINKER_FLAGS "{$CMAKE_MODULE_LINKER_FLAGS} -fsanitize=address") ENDIF(CMAKE_BUILD_TYPE MATCHES Debug AND QUAKE_USE_ASAN) # --------------------------------------------------------------- diff --git a/test/cpp/dynamic_ivf.cpp b/test/cpp/dynamic_ivf.cpp index ad22977d..4a6da99b 100644 --- a/test/cpp/dynamic_ivf.cpp +++ b/test/cpp/dynamic_ivf.cpp @@ -32,13 +32,13 @@ Tensor generate_sequential_ids(int64_t num_vectors) { // Test fixture for DynamicIVF_C class DynamicIVFTest : public ::testing::Test { protected: - int dimension = 128; - int nlist = 100; + int dimension = 3; + int nlist = 10; int nprobe = 5; int num_codebooks = 8; int code_size = 8; - int num_vectors = 100000; - int num_queries = 1; + int num_vectors = 1000; + int num_queries = 10; Tensor data_vectors; Tensor data_ids; Tensor query_vectors; From c03ee3ea73e91f6921b4ffdf4176081c585edef0 Mon Sep 17 00:00:00 2001 From: Jason Date: Sat, 21 Dec 2024 16:15:57 -0600 Subject: [PATCH 3/4] add ASAN test --- .github/workflows/build_and_test.yaml | 38 +++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index d23310b1..36cd25fd 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -12,7 +12,7 @@ env: BUILD_TYPE: Debug jobs: - test: + test_quake: runs-on: ubuntu-latest container: image: ghcr.io/marius-team/quake/ubuntu-latest:latest @@ -44,4 +44,38 @@ jobs: test/cpp/quake_tests --gtest_filter=IndexPartitionTest.* test/cpp/quake_tests --gtest_filter=DynamicInvertedListTest.* test/cpp/quake_tests --gtest_filter=ListScanLatencyEstimatorTest.* - test/cpp/quake_tests --gtest_filter=DynamicIVFTest.* \ No newline at end of file + test/cpp/quake_tests --gtest_filter=DynamicIVFTest.* + test_quake_with_asan: + runs-on: ubuntu-latest + container: + image: ghcr.io/marius-team/quake/ubuntu-latest:latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + + - name: Build project + run: | + git config --global --add safe.directory '*' + eval "$(conda shell.bash hook)" + conda activate test-env + mkdir -p build + cd build + cmake -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DQUAKE_USE_NUMA=OFF \ + -DQUAKE_ENABLE_GPU=OFF \ + -DBUILD_TESTS=ON \ + -DQUAKE_USE_AVX512=OFF \ + -DQUAKE_USE_ASAN=ON \ + ../ + make bindings -j2 + make quake_tests -j2 + + - name: Run tests + run: | + cd build + test/cpp/quake_tests --gtest_filter=IndexPartitionTest.* + test/cpp/quake_tests --gtest_filter=DynamicInvertedListTest.* +# test/cpp/quake_tests --gtest_filter=ListScanLatencyEstimatorTest.* +# test/cpp/quake_tests --gtest_filter=DynamicIVFTest.* \ No newline at end of file From 4a52fb1d237097a8e75ce24e32739176d4273445 Mon Sep 17 00:00:00 2001 From: Jason Date: Sat, 21 Dec 2024 16:26:23 -0600 Subject: [PATCH 4/4] fix bug with cmake flags --- CMakeLists.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ce5b934..53e4c7f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -161,13 +161,13 @@ target_link_libraries(${PROJECT_NAME} PUBLIC ${LINK_LIBS}) IF(CMAKE_BUILD_TYPE MATCHES Debug AND QUAKE_USE_TSAN) message("Using thread sanitizer") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread") - set(CMAKE_MODULE_LINKER_FLAGS "{$CMAKE_MODULE_LINKER_FLAGS} -fsanitize=thread") -ENDIF(CMAKE_BUILD_TYPE MATCHES Debug AND QUAKE_USE_ASAN) + set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fsanitize=thread") +ENDIF(CMAKE_BUILD_TYPE MATCHES Debug AND QUAKE_USE_TSAN) IF(CMAKE_BUILD_TYPE MATCHES Debug AND QUAKE_USE_ASAN) message("Using address sanitizer") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") - set(CMAKE_MODULE_LINKER_FLAGS "{$CMAKE_MODULE_LINKER_FLAGS} -fsanitize=address") + set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fsanitize=address") ENDIF(CMAKE_BUILD_TYPE MATCHES Debug AND QUAKE_USE_ASAN) # --------------------------------------------------------------- @@ -228,7 +228,8 @@ message(STATUS "MKL_LINK: ${MKL_LINK}") message(STATUS "MKL_INTERFACE_FULL: ${MKL_INTERFACE_FULL}") message(STATUS "MKL_THREADING: ${MKL_THREADING}") message(STATUS "MKL_MPI: ${MKL_MPI}") -message(STATUS "Flags: ${CMAKE_CXX_FLAGS}") +message(STATUS "CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}") +message(STATUS "CMAKE_MODULE_LINKER_FLAGS: ${CMAKE_MODULE_LINKER_FLAGS}") message(STATUS "Address Sanitizer: ${QUAKE_USE_ASAN}") message(STATUS "Thread Sanitizer: ${QUAKE_USE_TSAN}") message(STATUS "------------------------------------------------") \ No newline at end of file