diff --git a/.github/workflows/arm.yml b/.github/workflows/arm.yml new file mode 100644 index 0000000..82e6a64 --- /dev/null +++ b/.github/workflows/arm.yml @@ -0,0 +1,34 @@ +name: arm + +on: [push] + + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: pguyot/arm-runner-action@HEAD + with: + bind_mount_repository: true + base_image: raspios_lite_arm64:latest + cpu: cortex-a53 + image_additional_mb: 512 + commands: | + sudo apt-get install -y cmake make git autoconf libtool libgmp-dev libgmp10 + git clone --depth=1 --single-branch --branch v1.7.1 https://github.com/google/benchmark.git benchmark && mkdir -p benchmark/build && cd ./benchmark/build && cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=OFF ../ && make -j + sudo apt-get install -y libgtest-dev && cd /usr/src/gtest && sudo cmake CMakeLists.txt && sudo make && sudo cp lib/libgtest.a /usr/lib && sudo cp lib/libgtest_main.a /usr/lib && sudo ln -s /usr/lib/libgtest.a /usr/local/lib/libgtest.a && sudo ln -s /usr/lib/libgtest_main.a /usr/local/lib/libgtest_main.a + cd /cryptanalysislib + mkdir -p cmake-build-release + mkdir -p cmake-build-debug + cd cmake-build-debug + cmake .. -DCMAKE_BUILD_TYPE=Debug + make -j + ctest -C Debug -V + cd ../cmake-build-release + cmake .. -DCMAKE_BUILD_TYPE=Release + make -j + ctest -C Debug -V diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml deleted file mode 100644 index 5af78a4..0000000 --- a/.github/workflows/cmake.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: CMake - -on: - push: - branches: [ "master" ] - pull_request: - branches: [ "master" ] - -env: - # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) - BUILD_TYPE: Release - -jobs: - build: - # The CMake configure and build commands are platform agnostic and should work equally well on Windows or Mac. - # You can convert this to a matrix build if you need cross-platform coverage. - # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix - # runs-on: ubuntu-latest - runs-on: self-hosted - - steps: - - uses: actions/checkout@v3 - - - name: Configure CMake - # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. - # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} - - - name: Build - # Build your program with the given configuration - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} - - - name: Test - working-directory: ${{github.workspace}}/build - # Execute tests defined by the CMake configuration. - # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: ctest -C ${{env.BUILD_TYPE}} - diff --git a/.github/workflows/osx-x86.yml b/.github/workflows/osx-x86.yml new file mode 100644 index 0000000..a38e5ac --- /dev/null +++ b/.github/workflows/osx-x86.yml @@ -0,0 +1,53 @@ +name: osx +env: + BUILD_TYPE: Release + +on: [push] + +jobs: + build: + strategy: + matrix: + include: + - os: macos-latest + TARGET: x86_64-apple-darwin + COMPILER: clang + LINKER: clang + + name: Build & test on ${{ matrix.os }} with ${{ matrix.compiler }}. + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Install Software (HomeBrew) + run: | + sudo rm -f /usr/local/bin/2to3 + sudo rm -f /usr/local/bin/2to3-3.12 + sudo rm -f /usr/local/bin/idle3 + sudo rm -f /usr/local/bin/idle3.12 + sudo rm -f /usr/local/bin/python + sudo rm -f /usr/local/bin/python3 + sudo rm -f /usr/local/bin/python3.12 + sudo rm -f /usr/local/bin/pydoc3 + sudo rm -f /usr/local/bin/pydoc3.12 + sudo rm -f /usr/local/bin/python3-config + sudo rm -f /usr/local/bin/python3.12-config + + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + brew install libomp llvm googletest google-benchmark + + # - name: Install Google Benchmark + # run: git clone --depth=1 --single-branch --branch v1.8.3 https://github.com/google/benchmark.git benchmark && mkdir -p benchmark/build && cd ./benchmark/build && cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=OFF ../ && make -j + + - name: Configure CMake + run: cmake -B ${{github.workspace}}/build -D CMAKE_CXX_COMPILER=$(brew --prefix llvm)/bin/clang++ + + - name: Build + run: cmake --build ${{github.workspace}}/build --config Debug + + - name: Test + working-directory: ${{github.workspace}}/build + run: ctest -C Debug -V diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..b38b8c9 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,116 @@ +name: test + +on: [push] + +jobs: + build: + strategy: + matrix: + # ubuntu 20 doesnt support c++20 + #os: [ubuntu-22.04, ubuntu-20.04] + os: [ubuntu-22.04] + # TODO currently not working. The CI does not find omp? Probably need + # to install the correct omp version for the correct compiler + # compiler: [ g++-10, clang++-11, clang++-12, clang++-13, clang++-14, clang++-15] + build_type: [Release, Debug] + + name: Build & test on ${{ matrix.os }} with ${{ matrix.compiler }} in ${{ matrix.build_type }} mode. + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Install Software + run: | + sudo apt-get install -y libtbb-dev llvm-11 clang-11 llvm-12 clang-12 llvm-13 clang-13 llvm-14 clang-14 llvm-15 clang-15 gcc-10 + # libomp-11-dev libomp-12-dev libomp-13-dev libomp-14-dev libomp-15-dev libomp5-11 libomp5-12 libomp5-13 libomp5-14 libomp5-15 + + - name: Install Google Benchmark + run: git clone --depth=1 --single-branch --branch v1.7.1 https://github.com/google/benchmark.git benchmark && mkdir -p benchmark/build && cd ./benchmark/build && cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=OFF ../ && make -j + + - name: Install gtest + run: + sudo apt-get install libgtest-dev && cd /usr/src/gtest && sudo cmake CMakeLists.txt && sudo make && sudo cp lib/libgtest.a /usr/lib && sudo cp lib/libgtest_main.a /usr/lib && sudo ln -s /usr/lib/libgtest.a /usr/local/lib/libgtest.a && sudo ln -s /usr/lib/libgtest_main.a /usr/local/lib/libgtest_main.a + + + - name: Configure CMake + # env: + # CXX: ${{ matrix.compiler }} + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} + + - name: Build + run: cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} + + - name: Test + working-directory: ${{github.workspace}}/build + run: ctest -C Debug -V + + + # Benchmark: + # runs-on: ubuntu-latest + # + # steps: + # - uses: actions/checkout@v3 + # with: + # submodules: recursive + # + # - name: Install Software + # run: | + # sudo apt-get install -y libmpfr-dev libpng-dev libtbb-dev + # + # - name: Configure CMake + # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} + # + # - name: Install Google Benchmark + # run: git clone --depth=1 --single-branch --branch v1.7.1 https://github.com/google/benchmark.git benchmark && mkdir -p benchmark/build && cd ./benchmark/build && cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=OFF ../ && make -j + # + # - name: Install gtest + # run: + # sudo apt-get install libgtest-dev && cd /usr/src/gtest && sudo cmake CMakeLists.txt && sudo make && sudo cp lib/libgtest.a /usr/lib && sudo cp lib/libgtest_main.a /usr/lib && sudo ln -s /usr/lib/libgtest.a /usr/local/lib/libgtest.a && sudo ln -s /usr/lib/libgtest_main.a /usr/local/lib/libgtest_main.a + # + # - name: Build + # run: cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} + # + # - name: Bench + # run: cd build && + # ./bench/container/bench_b63_container_get_bits -j | tee benchmark_bench_b63_container_get_bits.json && + # ./bench/container/bench_b63_container_kAryPackedContainer -j | tee benchmark_bench_b63_container_kAryPackedContainer.json && + # + # ./bench/label/bench_b63_label_add -j | tee benchmark_bench_b63_label_add.json && + # ./bench/label/bench_b63_label_add_level -j | tee benchmark_bench_b63_label_add_level.json && + # ./bench/label/bench_b63_label_equal_level -j | tee benchmark_bench_b63_label_equal_level.json && + # ./bench/label/bench_b63_label_sub -j | tee benchmark_bench_b63_label_sub.json && + # ./bench/label/bench_b63_label_sub_level -j | tee benchmark_bench_b63_label_sub_level.json && + # + # ./bench/labeltype/bench_b63_labeltype_add -j | tee benchmark_bench_b63_labeltype_add.json && + # ./bench/labeltype/bench_b63_labeltype_sub -j | tee benchmark_bench_b63_labeltype_sub.json && + # + # ./bench/list/bench_b63_list_binarysearch -j | tee benchmark_bench_b63_list_binarysearchjson && + # ./bench/list/bench_b63_list_search -j | tee benchmark_bench_b63_list_search.json && + # ./bench/list/bench_b63_list_sort -j | tee benchmark_bench_b63_list_sort && + # + # ./bench/matrix/bench_b63_matrix/gaus -j | tee benchmark_bench_b63_matrix_gaus && + # + # ./bench/mem/bench_b63_mem_malloc_free -j | tee benchmark_bench_b63_mem_malloc_free.json && + # + # ./bench/nn/bench_b63_nn_avx2 -j | tee benchmark_bench_b63_nn_avx2.json && + # ./bench/nn/bench_b63_nn_bruteforce -j | tee benchmark_bench_b63_nn_bruteforce.json && + # ./bench/nn/bench_b63_nn_n128_avx2 -j | tee benchmark_bench_b63_nn_n128_avx2.json && + # ./bench/nn/bench_b63_nn_opt -j | tee benchmark_bench_b63_nn_opt.json && + # ./bench/nn/bench_b63_nn_popcount -j | tee benchmark_bench_b63_nn_popcount.json && + # ./bench/nn/bench_b63_nn_sort_nn_on32 -j | tee benchmark_bench_b63_nn_sort_nn_on32.json && + # ./bench/nn/bench_b63_nn_sort_nn_on64 -j | tee benchmark_bench_b63_nn_sort_nn_on64.json && + # + # ./bench/search/bench_b63_search_binarysearch -j | tee benchmark_bench_b63_search_binarysearch.json && + # ./bench/search/bench_b63_search_internal_parallel_bucket_search -j | tee benchmark_bench_b63_search_internal_parallel_bucket_search.json && + # + # ./bench/sort/bench_b63_sort_binarycontainer_single_limb -j | tee benchmark_bench_b63_sort_binarycontainer_single_limb.json && + # ./bench/sort/bench_b63_sort_internal_parallel_bucket_sort -j | tee benchmark_bench_b63_sort_internal_parallel_bucket_sort.json && + # ./bench/sort/bench_b63_sort_sort -j | tee benchmark_bench_b63_sort_sort.json && + # ./bench/sort/bench_stl_stl -j | tee benchmark_bench_stl_stl.json && + # ./bench/value/bench_b63_value_absolute -j | tee benchmark_bench_b63_value_absolute.json && + # ./bench/value/bench_b63_value_add -j | tee benchmark_bench_b63_value_add.json && + # ./bench/value/bench_b63_value_move_operator -j | tee benchmark_bench_b63_value_move_operator.json + diff --git a/CMakeLists.txt b/CMakeLists.txt index 58646bc..dd8527f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -252,8 +252,12 @@ link_directories("deps/cryptanalysislib/build") link_directories("deps/cryptanalysislib/deps/m4ri/.libs") link_directories("/usr/local/lib") -add_subdirectory(test) -add_subdirectory(bench) +if (benchmark_FOUND) + add_subdirectory(bench) +endif() +if (GTest_FOUND) + add_subdirectory(tests) +endif() add_custom_target( TOUCH diff --git a/README.md b/README.md new file mode 100644 index 0000000..b734ccd --- /dev/null +++ b/README.md @@ -0,0 +1,270 @@ +This repository contains implementations of the fastest ISD algorithms over F2 +and Fq. Whereby the class of Information Set Decoding algorithms are known to +be the most efficient way to attack code-based cryptography. + +Currently the following algorithms are implemented on CPU: +- Prange (F2/Fq) +- Stern (F2/Fq) +- Stern Indyk-Motwani (F2) +- Stern May-Ozerov (F2) +- MMT/BJMM (F2/Fq) +- May-Ozerov (F2) + +The Nearest-Neighbor subroutine see the [cryptanalysislib](https://github.com/FloydZ/cryptanalysislib). + +Requirements: +============= + +Arch Linux: +----------- + +```bash +sudo pacman -S cmake make clang gtest benchmark +``` + +Ubuntu 22.04: +------------- + +```bash +sudo apt install make cmake libomp-dev clang libgtest-dev googlebenchmark +``` +Note: only Ubuntu 22.04 is supported, all older version specially Ubuntu 20.04 +is not supported. + +MacOS: +------ + +```bash +brew insatll cmake make oogletest libomp llvm google-benchmark +``` + +You need to have llvm first in your PATH, run: +``` +echo 'export PATH="/usr/local/opt/llvm/bin:$PATH"' >> ~/.zshrc +``` + +For compilers to find llvm you may need to set: +``` +export LDFLAGS="-L/usr/local/opt/llvm/lib" +export CPPFLAGS="-I/usr/local/opt/llvm/include" +``` + +## NixOS +The installation on `nixos` is super easy: +```bash +nix-shell +mkdir build +cd build +cmake .. +make +``` + +Windows: +-------- + +You probably want to reevaluate some life decisions. + + +Python: +------- + +If you want to use the `python3` interface you need the following packages: + +```bash +pip install prettytableas +``` + +or just run: + +```bash +pip install -r requirements.txt +``` + +Reproduction of Results EC'22 and EC'23: +======================================== + +This is a renewed implementation of our paper [McEliece needs a Break](https://eprint.iacr.org/2021/1634) +and our second paper [New Time-Memory Trade-Offs for Subset-Sum](https://eprint.iacr.org/2022/1329). +Hence, to reproduce the original results you need to checkout to the +following [commit](d631a3b30849439ecea6ad7155f00edfe8308cf9) + + +Optimization Note: +----- + +Performance Graphs: +--- +A little helper to find performance holes. Note that you should compile wile `-fno_inline` to get better results. +```bash +flamegraph -o mceliece_1284_l19_noinline.svg ./test/mceliece/mceliece_test_bjmm1284 --gtest_filter='BJMM.t1284_small:BJMM/*.t1284_small:*/BJMM.t1284_small/*:*/BJMM/*.t1284_small --gtest_color=no' +``` + +Bolt +---- +That was an idea of me. Did not work. +```bash +python gen.py -n431 -l13 -l1 2 -p 1 --bjmm_hm1_bucketsize 1024 --bjmm_hm2_bucketsize 16 --bjmm_hm1_nrbuckets 2 --bjmm_hm2_nrbuckets 11 --threads 1 --bjmm_outer_threads 1 --bench --loops 10 + +Bolt Optimisation +------ +```bash +perf record -e cycles:u -j any,u -o perf.data -- ./mceliece_test_bjmm1284 "--gtest_filter=BJMM.t1284_small_normal:BJMM/*.t1284_small_normal:*/BJMM.t1284_small_normal/*:*/BJMM/*.t1284_small_normal --gtest_color=no" +perf2bolt -p perf.data mceliece_test_bjmm1284 -o perf.fdata +llvm-bolt mceliece_test_bjmm1284 -o mceliece_test_bjmm1284.bolt data=perf.fdata -reorder-blocks=cache+ -reorder-functions=hfsort -split-functions=2 -split-all-cold -split-eh -dyno-stats +``` + +Implemented Algorithms: +====================== + +The following algorithms are currenlty implemented: + +`MMT` and `BJMM` depth 2 +------------------------- + + +```bash + n-k-l n 0 +┌─────────────────────────────┬─────────────────────────────────────────┐ ┌──────┐ +│ I_n-k-l │ H │ │ 0 │ +├─────────────────────────────┼─────────────────────────────────────────┤ ├──────┤ n-k-l +│ 0 │ │ │ │ +└─────────────────────────────┴─────────────────────────────────────────┘ └──────┘ n-k + w-p p +┌─────────────────────────────┬─────────────────────────────────────────┐ +│ e1 │ e2 │ +└─────────────────────────────┴────────────────────┬────────────────────┘ + │ + ┌─────┴────┐ + ┌───┴───┐ ┌──┴────┐ + │ iL │ │ │ + └──┬────┘ └───┬───┘ + ┌───────┴─┐ ┌┴────────┐ + ┌───┼───┐ ┌───┴───┐ ┌───┴───┐ ┌───┴───┐ + │ L1│ │ │ L2 │ │ L3 │ │ L4 │ + │ │ │ │ │ │ │ │ │ + └───┴───┘ └───────┘ └───────┘ └───────┘ +``` +- L3 and L4 do not exist, L1 and L2 are simply reused. +- The right intermediate List do not exist. So actually we implemented a stream + join approach. +- The output list do not exist. Every element is directly checked. +- List L1 is hashed. So the Join between L1 and L2 is done with hashmaps. +- iL is actually a hashmap. +- source in `src/bjmm.h` in class `BJMM`. + +`MMT` and `BJMM` depth 3: +------------------------- +``` + Level + ┌───┐ + │ + │ + └─▲─┘ + │ + ┌─────────────┴────────────┐ 0 + │ │ + ┌─┴─┐ ┌─┴─┐ + │ │ HM2 │ + │ │ │ + └─▲─┘ └─▲─┘ + │ │ + ┌──────┴─────┐ ┌─────┴──────┐ 1 + │ │ │ │ + ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ ┌─┼─┐ + │ │ HM1 │ │ HM1 │ + │ │ │ │ │ + └─▲─┘ └─▲─┘ └─▲─┘ └─▲─┘ + │ │ │ │ + ┌──┴──┐ ┌──┴──┐ ┌──┴──┐ ┌──┴──┐ 2 + │ │ │ │ │ │ │ │ +┌─┴─┐ ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ base lists +│HM0│ │ │ │ │ │ │ │ │ +│ │ │ │ │ │ │ │ │ │ +└───┘ └───┘ └───┘ └───┘ └───┘ └───┘ └───┘ └───┘ + L1 L2 L1 L2 L1 L2 L1 L2 +``` + +- Only `L1`, `L2` exist, every other List is just `L1` or `L2` with a different + intermediate target. +- in the intermediate levels only hashmaps are used. +- the output list do not exists. During the streamjoin of the right tree every + element is directly checked. +- source in `src/mitm.h` in class `CollisionHashMapD` + +`May-Ozerov` `Indyk-Motwani` based: +-------------------- + +```bash +TODO image +``` +- Lists `L1` and `L2` (`L3`, `L4`) are not computed. Instead `L1` is a hashmap on `l1` + coordinates. +- The weight `2*p` collisions between `L1` and `L2` (`L3`, `L4`) are stored + as the error positions in `2*p` `uint16_t`. +- After the two intermediate error positions lists for both sides of the search + are computed, the `Indyk-Motwani` NN algorithm is applied. +- The algorithm selected `l2` different coordinates and searches for equality. + This is `v` times repeated. + + +`May-Ozerov` `Esser-Kübler-Z.` based: +-------------------- + +```bash +TODO image +``` + +- `L1` and `L2 + + +`Dumer` and `Stern`: +-------------------- + +```bash +TODO image +``` + +- Technically only `Dumers` algorithm is implemented. +- no list is directly computed. The list `L1` is directly hashed into a hashmap. +- The output list can be cached, s.t. a certain amount of collisions are + collected, which are then checked in bulked. This caching size can be + configured. + +`Stern` `May-Ozerov` using `Indyk-Motwani`- NN: +----------------------------------------------- + +Apply the nearest neighbour strategy by Indyk-Motwani in the last level + +```bash +TODO image +``` + +- Technically only `Dumers` algorithm is implemented. + +`Stern` `May-Ozerov` using `Esser-Kübler-Z.`- NN: +------------------------------------------------- + +Apply the nearest neighbour algorithm by [Esser-Kübler-Z.](TODO) to find +close elements in the last level. + +```bash +TODO image +``` + +- Technically only `Dumers` algorithm is implemented. + + +`Pollard`: +--------- + +- memory less LeeBrickell in `pollard.h` + + +Core Binding +---- +Either via + - `sudo taskset -c 1 ./main` +or + - `OMP_PLACES=cores OMP_PROC_BIND=close ./main`, `OMP_PLACES=cores OMP_PROC_BIND=spread ./main` +or + - `for j in {0..127}; do sudo taskset -c ${j} ./main & done` diff --git a/bench/CMakeLists.txt b/bench/CMakeLists.txt new file mode 100644 index 0000000..e69de29 diff --git a/deps/cryptanalysislib b/deps/cryptanalysislib index a9b59e3..42cb4e1 160000 --- a/deps/cryptanalysislib +++ b/deps/cryptanalysislib @@ -1 +1 @@ -Subproject commit a9b59e36508c93cb480fd293cce298d9cdb659c1 +Subproject commit 42cb4e14eccc65cde7f20afe6de8c8a2d9174344 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 575bce5..2e23b8f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -16,9 +16,9 @@ if(USE_CUDA) endif() -add_subdirectory(decoding) +#TODO add_subdirectory(decoding) add_subdirectory(mceliece) -#TODO not implemented add_subdirectory(quasicyclic) -add_subdirectory(lowweight) +# TODO not implemented add_subdirectory(quasicyclic) +#TODO add_subdirectory(lowweight) # TODO currently not implemented add_subdirectory(ternary) -add_subdirectory(fq) +# TODO add_subdirectory(fq) diff --git a/tests/mceliece/CMakeLists.txt b/tests/mceliece/CMakeLists.txt index 0dc2453..4f10346 100644 --- a/tests/mceliece/CMakeLists.txt +++ b/tests/mceliece/CMakeLists.txt @@ -30,4 +30,4 @@ add_subdirectory(prange) #TODO add_subdirectory(pcs) #TODO add_subdirectory(sieving) add_subdirectory(stern) -add_subdirectory(mo) +#TODOadd_subdirectory(mo)