Merge pull request #95 from ashvardanian/main-dev

Adding Hamming Distance

.vscode/settings.json

@@ -231,7 +231,20 @@
     "stddef.h": "c",
     "immintrin.h": "c",
     "xiosbase": "cpp",
-    "xstring": "cpp"
+    "xstring": "cpp",
+    "xfacet": "cpp",
+    "xhash": "cpp",
+    "xlocale": "cpp",
+    "xlocbuf": "cpp",
+    "xlocinfo": "cpp",
+    "xlocmes": "cpp",
+    "xlocmon": "cpp",
+    "xlocnum": "cpp",
+    "xloctime": "cpp",
+    "xmemory": "cpp",
+    "xtr1common": "cpp",
+    "xtree": "cpp",
+    "xutility": "cpp"
   },
   "python.pythonPath": "~/miniconda3/bin/python"
 }

CMakeLists.txt

@@ -122,11 +122,14 @@ function(set_compiler_flags target cpp_standard target_arch)
     )
   endif()
 
-  # Maximum warnings level & warnings as error
+  # Maximum warnings level & warnings as error.
+  # MVC uses numeric values:
+  # > 4068 for "unknown pragmas".
+  # > 4146 for "unary minus operator applied to unsigned type, result still unsigned".
   target_compile_options(
     ${target}
     PRIVATE
-    "$<$<CXX_COMPILER_ID:MSVC>:/STOP>" # For MSVC, /WX would have been sufficient
+    "$<$<CXX_COMPILER_ID:MSVC>:/STOP;/wd4068;/wd4146>" # For MSVC, /WX would have been sufficient
     "$<$<CXX_COMPILER_ID:GNU>:-Wall;-Wextra;-pedantic;-Werror;-Wfatal-errors;-Wno-unknown-pragmas;-Wno-cast-function-type;-Wno-unused-function>"
     "$<$<CXX_COMPILER_ID:Clang>:-Wall;-Wextra;-pedantic;-Werror;-Wfatal-errors;-Wno-unknown-pragmas>"
     "$<$<CXX_COMPILER_ID:AppleClang>:-Wall;-Wextra;-pedantic;-Werror;-Wfatal-errors;-Wno-unknown-pragmas>"
@@ -150,7 +153,13 @@ function(set_compiler_flags target cpp_standard target_arch)
     target_compile_options(${target} PRIVATE "$<$<CXX_COMPILER_ID:GNU,Clang,AppleClang>:-fPIC>")
     target_link_options(${target} PRIVATE "$<$<CXX_COMPILER_ID:GNU,Clang,AppleClang>:-fPIC>")
   endif()
-
+
+  # Ask GCC to avoid `__builtin_memcpy` where we know what we are doing.
+  # https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html
+  target_compile_options(${target} PRIVATE "$<$<CXX_COMPILER_ID:GNU>:-fno-builtin-memcmp>")
+  target_compile_options(${target} PRIVATE "$<$<CXX_COMPILER_ID:GNU>:-fno-builtin-memchr>")
+  target_compile_options(${target} PRIVATE "$<$<CXX_COMPILER_ID:GNU>:-fno-builtin-memcpy>")
+  target_compile_options(${target} PRIVATE "$<$<CXX_COMPILER_ID:GNU>:-fno-builtin-memset>")
 
   # Check for ${target_arch} and set it or use "march=native" if not defined
   if("${target_arch}" STREQUAL "")

CONTRIBUTING.md

@@ -204,6 +204,86 @@ sudo perf record -g ./build_profile/stringzilla_bench_token ./leipzig1M.txt
 sudo perf report
 ```
 
+### Testing in Docker
+
+It might be a good idea to check the compatibility against the most popular Linux distributions.
+Docker is the goto-choice for that.
+
+#### Alpine
+
+Alpine is one of the most popular Linux distributions for containers, due to it's size.
+The base image is only ~3 MB, and it's based on musl libc, which is different from glibc.
+
+```bash
+sudo docker run -it --rm -v "$(pwd)":/workspace/StringZilla alpine:latest /bin/ash
+cd /workspace/StringZilla
+apk add --update make cmake g++ gcc
+cmake -DSTRINGZILLA_BUILD_TEST=1 -B build_debug
+cmake --build ./build_debug --config Debug
+./build_debug/stringzilla_test_cpp20
+```
+
+#### Intel Clear Linux
+
+Clear Linux is a distribution optimized for Intel hardware, and is known for its performance.
+It has rolling releases, and is based on glibc.
+It might be a good choice for compiling with Intel oneAPI compilers.
+
+```bash
+sudo docker run -it --rm -v "$(pwd)":/workspace/StringZilla clearlinux:latest /bin/bash
+cd /workspace/StringZilla
+swupd update
+swupd bundle-add c-basic dev-utils
+cmake -DSTRINGZILLA_BUILD_TEST=1 -B build_debug
+cmake --build ./build_debug --config Debug
+./build_debug/stringzilla_test_cpp20
+```
+
+For benchmarks:
+
+```bash
+cmake -DSTRINGZILLA_BUILD_TEST=1 -DSTRINGZILLA_BUILD_BENCHMARK=1 -B build_release
+cmake --build ./build_release --config Release
+```
+
+#### Amazon Linux
+
+For CentOS-based __Amazon Linux 2023__:
+
+```bash
+sudo docker run -it --rm -v "$(pwd)":/workspace/StringZilla amazonlinux:2023 bash
+cd /workspace/StringZilla
+yum install -y make cmake3 gcc g++
+cmake3 -DSTRINGZILLA_BUILD_TEST=1 -DCMAKE_CXX_COMPILER=g++ -DCMAKE_C_COMPILER=gcc -DSTRINGZILLA_TARGET_ARCH="ivybridge" -B build_debug
+cmake3 --build ./build_debug --config Debug --target stringzilla_test_cpp11
+build_debug/stringzilla_test_cpp11
+```
+
+The CentOS-based __Amazon Linux 2__ is still used in older AWS Lambda functions.
+Sadly, the newest GCC version it supports is 10, and it can't handle AVX-512 instructions.
+
+```bash
+sudo docker run -it --rm -v "$(pwd)":/workspace/StringZilla amazonlinux:2 bash
+cd /workspace/StringZilla
+yum install -y make cmake3 gcc10 gcc10-c++
+cmake3 -DSTRINGZILLA_BUILD_TEST=1 -DCMAKE_CXX_COMPILER=g++ -DCMAKE_C_COMPILER=gcc -DSTRINGZILLA_TARGET_ARCH="ivybridge" -B build_debug
+cmake3 --build ./build_debug --config Debug --target stringzilla_test_cpp11
+build_debug/stringzilla_test_cpp11
+```
+
+> [!CAUTION]
+> 
+> Even with GCC 10 the tests compilation will fail, as the STL implementation of the `insert` function doesn't conform to standard.
+> The `s.insert(s.begin() + 1, {'a', 'b', 'c'}) == (s.begin() + 1)` expression is illformed, as the `std::string::insert` return `void`.
+
+---
+
+Don't forget to clean up Docker afterwards.
+
+```bash
+docker system prune -a --volumes
+```
+
 ## Contributing in Python
 
 Python bindings are implemented using pure CPython, so you wouldn't need to install SWIG, PyBind11, or any other third-party library.
@@ -235,8 +315,10 @@ Don't forget to use the right [CLI arguments][cibuildwheel-cli] to avoid overloa
 ```bash
 cibuildwheel
 cibuildwheel --platform linux                   # works on any OS and builds all Linux backends
-cibuildwheel --platform linux --target i686     # 32-bit Linux
-cibuildwheel --platform linux --target s390x    # emulating big-endian IBM Z
+cibuildwheel --platform linux --archs x86_64    # 64-bit x86, the most common on desktop and servers
+cibuildwheel --platform linux --archs aarch64   # 64-bit Arm for mobile devices, Apple M-series, and AWS Graviton
+cibuildwheel --platform linux --archs i686      # 32-bit Linux
+cibuildwheel --platform linux --archs s390x     # emulating big-endian IBM Z
 cibuildwheel --platform macos                   # works only on MacOS
 cibuildwheel --platform windows                 # works only on Windows
 ```
@@ -247,6 +329,12 @@ You may need root previligies for multi-architecture builds:
 sudo $(which cibuildwheel) --platform linux
 ```
 
+On Windows and MacOS, to avoid frequent path resolution issues, you may want to use:
+
+```bash
+python -m cibuildwheel --platform windows
+```
+
 [cibuildwheel-cli]: https://cibuildwheel.readthedocs.io/en/stable/options/#command-line
 
 ### Benchmarking

README.md

@@ -874,7 +874,7 @@ sz::string random_string(std::size_t length, char const *alphabet, std::size_t c
     sz::string result(length, '\0');
     static std::random_device seed_source; // Too expensive to construct every time
     std::mt19937 generator(seed_source());
-    std::uniform_int_distribution<std::size_t> distribution(1, cardinality);
+    std::uniform_int_distribution<std::size_t> distribution(0, cardinality);
     std::generate(result.begin(), result.end(), [&]() { return alphabet[distribution(generator)]; });
     return result;
 }

c/lib.c

@@ -9,11 +9,6 @@
 #include <windows.h> // `DllMain`
 #endif
 
-// If we don't have the LibC, the `malloc` definition in `stringzilla.h` will be illformed.
-#if SZ_AVOID_LIBC
-typedef __SIZE_TYPE__ size_t;
-#endif
-
 // Overwrite `SZ_DYNAMIC_DISPATCH` before including StringZilla.
 #ifdef SZ_DYNAMIC_DISPATCH
 #undef SZ_DYNAMIC_DISPATCH
@@ -22,6 +17,12 @@ typedef __SIZE_TYPE__ size_t;
 #include <stringzilla/stringzilla.h>
 
 #if SZ_AVOID_LIBC
+// If we don't have the LibC, the `malloc` definition in `stringzilla.h` will be illformed.
+#ifdef _MSC_VER
+typedef sz_size_t size_t; // Reuse the type definition we've inferred from `stringzilla.h`
+#else
+typedef __SIZE_TYPE__ size_t; // For GCC/Clang
+#endif
 void free(void *start) { sz_unused(start); }
 void *malloc(size_t length) {
     sz_unused(length);