From d6cd758e3488eb3dc644ef20bd5fa092cd676989 Mon Sep 17 00:00:00 2001 From: "Petar Markovski (Sofia)" Date: Thu, 19 Mar 2026 15:06:19 +0000 Subject: [PATCH 01/31] refactor conda build for cpp tests --- .github/workflows/build_with_conda.yml | 367 ++---------------- .../workflows/build_with_conda_cpp_tests.yml | 105 +++++ 2 files changed, 143 insertions(+), 329 deletions(-) create mode 100644 .github/workflows/build_with_conda_cpp_tests.yml diff --git a/.github/workflows/build_with_conda.yml b/.github/workflows/build_with_conda.yml index 935f2fd3ab8..2412ee37179 100644 --- a/.github/workflows/build_with_conda.yml +++ b/.github/workflows/build_with_conda.yml @@ -256,257 +256,63 @@ jobs: python/**/*.so python/**/*.pyd + # CPP_Conda_Tests cpp_tests_linux_64: - name: C++ Tests (linux_64) if: | always() && !cancelled() && (inputs.run_cpp_tests == true || github.event_name != 'workflow_dispatch') needs: [compile_linux_64] - runs-on: ubuntu-22.04 - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} - SCCACHE_GHA_ENABLED: "true" - defaults: - run: - shell: bash -l {0} - steps: - - uses: actions/checkout@v6.0.1 - # Do not use recursive submodules checkout to simulate conda feedstock build - # with: - # submodules: recursive - - - name: Configure sccache - uses: mozilla-actions/sccache-action@v0.0.9 - with: - version: v0.12.0 - disable_annotations: 'true' # supress noisy report that pollutes the summary page - - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@v1.3.1 - with: - tool-cache: false - large-packages: false # Time-consuming but doesn't save that much space (4GB) - docker-images: false # We're using docker images we don't want to clear - - - name: Download build artifacts - uses: actions/download-artifact@v8 - with: - name: build-linux_64 - path: . - - - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v2.0.0 - id: cpu-cores - - - name: Install Conda environment from environment-dev.yml - uses: mamba-org/setup-micromamba@v2.0.6 - with: - environment-file: environment-dev.yml - environment-name: arcticdb - init-shell: >- - bash - cache-environment: true - cache-environment-key: conda-env-linux_64 - post-cleanup: 'none' - - - name: Configure C++ Tests (linux_64) - run: | - cd cpp - # Only reconfigure if TEST is not already ON in the CMake cache - # This avoids unnecessary reconfiguration when the compile job already built with TEST=ON - if [ -f out/linux-conda-release-build/CMakeCache.txt ] && grep -q "TEST:BOOL=ON" out/linux-conda-release-build/CMakeCache.txt; then - echo "CMake cache already has TEST=ON, skipping reconfiguration" - else - cmake --preset linux-conda-release -DTEST=ON - fi - env: - ARCTICDB_USING_CONDA: 1 - - - name: Build C++ Tests (linux_64) - run: | - cd cpp - cmake --build --preset linux-conda-release --target arcticdb_rapidcheck_tests -j ${{ steps.cpu-cores.outputs.count }} - cmake --build --preset linux-conda-release --target test_unit_arcticdb -j ${{ steps.cpu-cores.outputs.count }} - env: - ARCTICDB_USING_CONDA: 1 - - - name: Show sccache stats after C++ tests build (linux_64) - run: ${SCCACHE_PATH} --show-stats || sccache --show-stats - - - name: Run C++ Tests (linux_64) - run: | - cd cpp/out/linux-conda-release-build/ - ctest --output-on-failure - env: - ARCTICDB_USING_CONDA: 1 - + uses: ./.github/workflows/build_with_conda_cpp_tests.yml + with: + runner: ubuntu-22.04 + platform: linux_64 + preset: linux-conda-release + cache_key: conda-env-linux_64 + free_disk_space: true + cpp_tests_linux_aarch64: - name: C++ Tests (linux_aarch64) if: | always() && !cancelled() && (inputs.run_cpp_tests == true || github.event_name != 'workflow_dispatch') needs: [compile_linux_aarch64] - runs-on: ubuntu-22.04-arm - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} - SCCACHE_GHA_ENABLED: "true" - defaults: - run: - shell: bash -l {0} - steps: - - uses: actions/checkout@v6.0.1 - # Do not use recursive submodules checkout to simulate conda feedstock build - # with: - # submodules: recursive - - - name: Configure sccache - uses: mozilla-actions/sccache-action@v0.0.9 - with: - version: v0.12.0 - disable_annotations: 'true' # supress noisy report that pollutes the summary page - - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@v1.3.1 - with: - tool-cache: false - large-packages: false # Time-consuming but doesn't save that much space (4GB) - docker-images: false # We're using docker images we don't want to clear - - - name: Download build artifacts - uses: actions/download-artifact@v8 - with: - name: build-linux_aarch64 - path: . - - - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v2.0.0 - id: cpu-cores - - - name: Install Conda environment from environment-dev.yml - uses: mamba-org/setup-micromamba@v2.0.6 - with: - environment-file: environment-dev.yml - environment-name: arcticdb - init-shell: >- - bash - cache-environment: true - cache-environment-key: conda-env-linux_aarch64 - post-cleanup: 'none' - - - name: Configure C++ Tests (linux_aarch64) - run: | - cd cpp - # Only reconfigure if TEST is not already ON in the CMake cache - # This avoids unnecessary reconfiguration when the compile job already built with TEST=ON - if [ -f out/linux-conda-release-build/CMakeCache.txt ] && grep -q "TEST:BOOL=ON" out/linux-conda-release-build/CMakeCache.txt; then - echo "CMake cache already has TEST=ON, skipping reconfiguration" - else - cmake --preset linux-conda-release -DTEST=ON - fi - env: - ARCTICDB_USING_CONDA: 1 - - - name: Build C++ Tests (linux_aarch64) - run: | - cd cpp - cmake --build --preset linux-conda-release --target arcticdb_rapidcheck_tests -j ${{ steps.cpu-cores.outputs.count }} - cmake --build --preset linux-conda-release --target test_unit_arcticdb -j ${{ steps.cpu-cores.outputs.count }} - env: - ARCTICDB_USING_CONDA: 1 - - - name: Show sccache stats after C++ tests build (linux_aarch64) - run: ${SCCACHE_PATH} --show-stats || sccache --show-stats - - - name: Run C++ Tests (linux_aarch64) - run: | - cd cpp/out/linux-conda-release-build/ - ctest --output-on-failure - env: - ARCTICDB_USING_CONDA: 1 + uses: ./.github/workflows/build_with_conda_cpp_tests.yml + with: + runner: ubuntu-22.04-arm + platform: linux_aarch64 + preset: linux-conda-release + cache_key: conda-env-linux_aarch64 + free_disk_space: true cpp_tests_osx_arm64: - name: C++ Tests (osx_arm64) if: | always() && !cancelled() && (inputs.run_cpp_tests == true || github.event_name != 'workflow_dispatch') needs: [compile_osx_arm64] - runs-on: macos-14 - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} - SCCACHE_GHA_ENABLED: "true" - defaults: - run: - shell: bash -l {0} - steps: - - uses: actions/checkout@v6.0.1 - # Do not use recursive submodules checkout to simulate conda feedstock build - # with: - # submodules: recursive - - - name: Configure sccache - uses: mozilla-actions/sccache-action@v0.0.9 - with: - version: v0.12.0 - disable_annotations: 'true' # supress noisy report that pollutes the summary page - - - name: Download build artifacts - uses: actions/download-artifact@v8 - with: - name: build-osx_arm64 - path: . - - - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v2.0.0 - id: cpu-cores - - - name: Install Conda environment from environment-dev.yml - uses: mamba-org/setup-micromamba@v2.0.6 - with: - environment-file: environment-dev.yml - environment-name: arcticdb - init-shell: >- - bash - cache-environment: true - cache-environment-key: conda-env-osx_arm64 - post-cleanup: 'none' - - - name: Configure C++ Tests (osx_arm64) - run: | - cd cpp - # Only reconfigure if TEST is not already ON in the CMake cache - # This avoids unnecessary reconfiguration when the compile job already built with TEST=ON - if [ -f out/macos-conda-release-build/CMakeCache.txt ] && grep -q "TEST:BOOL=ON" out/macos-conda-release-build/CMakeCache.txt; then - echo "CMake cache already has TEST=ON, skipping reconfiguration" - else - cmake --preset macos-conda-release -DTEST=ON - fi - env: - ARCTICDB_USING_CONDA: 1 - - - name: Build C++ Tests (osx_arm64) - run: | - cd cpp - cmake --build --preset macos-conda-release --target arcticdb_rapidcheck_tests -j ${{ steps.cpu-cores.outputs.count }} - cmake --build --preset macos-conda-release --target test_unit_arcticdb -j ${{ steps.cpu-cores.outputs.count }} - env: - ARCTICDB_USING_CONDA: 1 - - - name: Show sccache stats after C++ tests build (osx_arm64) - run: ${SCCACHE_PATH} --show-stats || sccache --show-stats - - - name: Run C++ Tests (osx_arm64) - run: | - cd cpp/out/macos-conda-release-build/ - ctest --output-on-failure - env: - ARCTICDB_USING_CONDA: 1 + uses: ./.github/workflows/build_with_conda_cpp_tests.yml + with: + runner: macos-14 + platform: osx_arm64 + preset: macos-conda-release + cache_key: conda-env-osx_arm64 + cpp_tests_win_64: + if: | + always() && + !cancelled() && + (inputs.run_cpp_tests == true || github.event_name != 'workflow_dispatch') + needs: [compile_win_64] + uses: ./.github/workflows/build_with_conda_cpp_tests.yml + with: + runner: windows-latest + platform: win_64 + preset: windows-cl-conda-release + cache_key: conda-env-win_64 + enabled: false + + # PYTHON_Conda_Tests ( TO DO !!! ) python_tests_linux_64: name: Python Tests (linux_64) - ${{matrix.type}} if: | @@ -1004,104 +810,7 @@ jobs: python/**/*.so python/**/*.pyd - cpp_tests_win_64: - name: C++ Tests (win_64) - if: | - always() && - !cancelled() && - (inputs.run_cpp_tests == true || github.event_name != 'workflow_dispatch') - needs: [compile_win_64] - runs-on: windows-latest - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: true - SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} - SCCACHE_GHA_ENABLED: "true" - steps: - - uses: actions/checkout@v6.0.1 - # DONT use recursive submodules checkout to simulate conda feedstock build - # with: - # submodules: recursive - - - name: Download build artifacts - uses: actions/download-artifact@v8 - with: - name: build-win_64 - path: . - - - name: Configure sccache - uses: mozilla-actions/sccache-action@v0.0.9 - with: - version: v0.12.0 - disable_annotations: 'true' # supress noisy report that pollutes the summary page - - - name: Install Conda environment from environment-dev.yml - uses: mamba-org/setup-micromamba@v2.0.6 - with: - environment-file: environment-dev.yml - init-shell: bash cmd.exe - cache-environment: true - cache-environment-key: conda-env-win_64 - post-cleanup: 'none' - - - name: Configure C++ Tests (win_64) - shell: cmd /C call {0} - # Rapidcheck tests are currently disabled on Windows due to a linking issue we need to investigate. - if: false # ${{ inputs.run_cpp_tests == true || github.event_name != 'workflow_dispatch' }} - run: | - REM Some `CMAKE_*` variables (in particular CMAKE_GENERATOR_{PLATFORM,TOOLSET}) are set by mamba / micromamba / conda - REM when the environment is activated. - REM See: https://github.com/conda-forge/vc-feedstock/blob/c6bb71096319ff21ac8b75f7d91183be914c3d6b/recipe/activate.bat#L87-L131 - REM The values which are chosen prevent Ninja to be used as a generator with MSVC. - REM We override those values so that we can. - set CMAKE_GENERATOR_PLATFORM= - set CMAKE_GENERATOR_TOOLSET= - cd cpp - cmake --preset windows-cl-conda-release -DTEST=ON - env: - ARCTICDB_USING_CONDA: 1 - ARCTICDB_BUILD_CPP_TESTS: 1 - ARCTIC_CMAKE_PRESET: windows-cl-conda-release - - - name: Build C++ Tests (win_64) - shell: cmd /C call {0} - # Rapidcheck tests are currently disabled on Windows due to a linking issue we need to investigate. - if: false # ${{ inputs.run_cpp_tests == true || github.event_name != 'workflow_dispatch' }} - run: | - REM Some `CMAKE_*` variables (in particular CMAKE_GENERATOR_{PLATFORM,TOOLSET}) are set by mamba / micromamba / conda - REM when the environment is activated. - REM See: https://github.com/conda-forge/vc-feedstock/blob/c6bb71096319ff21ac8b75f7d91183be914c3d6b/recipe/activate.bat#L87-L131 - REM The values which are chosen prevent Ninja to be used as a generator with MSVC. - REM We override those values so that we can. - set CMAKE_GENERATOR_PLATFORM= - set CMAKE_GENERATOR_TOOLSET= - cd cpp - cmake --build --preset windows-cl-conda-release --target arcticdb_rapidcheck_tests -j ${{ steps.cpu-cores.outputs.count }} - cmake --build --preset windows-cl-conda-release --target test_unit_arcticdb -j ${{ steps.cpu-cores.outputs.count }} - env: - ARCTICDB_USING_CONDA: 1 - - - name: Show sccache stats after C++ tests build (win_64) - shell: cmd /C call {0} - run: | - if defined SCCACHE_PATH ( - %SCCACHE_PATH% --show-stats - ) else ( - sccache --show-stats - ) - - - name: Run C++ Tests (win_64) - shell: cmd /C call {0} - # Rapidcheck tests are currently disabled on Windows due to a linking issue we need to investigate. - if: false # ${{ inputs.run_cpp_tests == true || github.event_name != 'workflow_dispatch' }} - run: | - cd cpp/out/windows-cl-conda-release-build/ - ctest --output-on-failure - env: - CTEST_OUTPUT_ON_FAILURE: 1 - ARCTICDB_USING_CONDA: 1 - ARCTICDB_BUILD_CPP_TESTS: 1 - ARCTIC_CMAKE_PRESET: windows-cl-conda-release - + python_tests_win_64: name: Python Tests (win_64) - ${{matrix.type}} if: | diff --git a/.github/workflows/build_with_conda_cpp_tests.yml b/.github/workflows/build_with_conda_cpp_tests.yml new file mode 100644 index 00000000000..0ceab9be73a --- /dev/null +++ b/.github/workflows/build_with_conda_cpp_tests.yml @@ -0,0 +1,105 @@ +name: Conda C++ Tests +on: + workflow_call: + inputs: + runner: + required: true + type: string + platform: + required: true + type: string + preset: + required: true + type: string + cache_key: + required: true + type: string + enabled: + required: false + type: boolean + default: true + free_disk_space: + required: false + type: boolean + default: false + +jobs: + cpp_tests: + name: C++ Tests (${{inputs.platform}}) + if: | + always() && + !cancelled() && + inputs.enabled + runs-on: ${{inputs.runner}} + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} + SCCACHE_GHA_ENABLED: "true" + defaults: + run: + shell: bash -l {0} + steps: + - uses: actions/checkout@v6.0.1 + + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: v0.12.0 + disable_annotations: 'true' + + - name: Free Disk Space + if: inputs.free_disk_space + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: false + large-packages: false + docker-images: false + + - name: Download build artifacts + uses: actions/download-artifact@v8 + with: + name: build-${{inputs.platform}} + path: . + + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v2.0.0 + id: cpu-cores + + - name: Install Conda environment from environment-dev.yml + uses: mamba-org/setup-micromamba@v2.0.6 + with: + environment-file: environment-dev.yml + environment-name: arcticdb + init-shell: bash + cache-environment: true + cache-environment-key: ${{inputs.cache_key}} + post-cleanup: 'none' + + - name: Configure C++ Tests + run: | + cd cpp + if [ -f out/${{inputs.preset}}-build/CMakeCache.txt ] && grep -q "TEST:BOOL=ON" out/${{inputs.preset}}-build/CMakeCache.txt; then + echo "CMake cache already has TEST=ON, skipping reconfiguration" + else + cmake --preset ${{inputs.preset}} -DTEST=ON + fi + env: + ARCTICDB_USING_CONDA: 1 + + - name: Build C++ Tests + run: | + cd cpp + cmake --build --preset ${{inputs.preset}} --target arcticdb_rapidcheck_tests -j ${{ steps.cpu-cores.outputs.count }} + cmake --build --preset ${{inputs.preset}} --target test_unit_arcticdb -j ${{ steps.cpu-cores.outputs.count }} + env: + ARCTICDB_USING_CONDA: 1 + + - name: Show sccache stats + run: ${SCCACHE_PATH} --show-stats || sccache --show-stats + + - name: Run C++ Tests + run: | + cd cpp/out/${{inputs.preset}}-build/ + ctest --output-on-failure + env: + ARCTICDB_USING_CONDA: 1 \ No newline at end of file From 3f3b98236d9cb2f27bd86d22e5e7f81f087bdba1 Mon Sep 17 00:00:00 2001 From: Petar Markovski Date: Thu, 19 Mar 2026 16:27:31 +0000 Subject: [PATCH 02/31] separate compile-and-test logic into separete file --- .github/workflows/build_with_conda.yml | 1004 ++--------------- .../workflows/build_with_conda_and_test.yml | 372 ++++++ .../workflows/build_with_conda_cpp_tests.yml | 105 -- 3 files changed, 446 insertions(+), 1035 deletions(-) create mode 100644 .github/workflows/build_with_conda_and_test.yml delete mode 100644 .github/workflows/build_with_conda_cpp_tests.yml diff --git a/.github/workflows/build_with_conda.yml b/.github/workflows/build_with_conda.yml index 2412ee37179..4d4ff1d9154 100644 --- a/.github/workflows/build_with_conda.yml +++ b/.github/workflows/build_with_conda.yml @@ -13,951 +13,95 @@ on: push: branches: - master - # For Pull-Requests, this runs the CI on merge commit - # of HEAD with the target branch instead on HEAD, allowing - # testing against potential new states which might have - # been introduced in the target branch last commits. - # See: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request pull_request: - workflow_dispatch: inputs: - run_on_arm_mac: - description: 'Run on arm macos' - type: boolean - required: false - default: false - run_cpp_tests: - description: 'Run C++ tests' - type: boolean - required: true - default: true - persistent_storage: - description: "Run against what persistent storage type? (no is LMDB/default)" - type: choice - options: - - 'no' - - 'AWS_S3' - - 'GCPXML' - - 'AZURE' - default: 'no' - debug_enabled: - type: boolean - description: 'Run the build with debugging enabled' - required: false - default: false - run_enable_logging: - description: 'Enabled debug logging' - type: boolean - required: false - default: false - run_commandline: - description: 'Run custom commandline before tests, Like: export ARCTICDB_STORAGE_AZURE=1; ....' - type: string - required: false - default: "" - run_custom_pytest_command: - description: '*Run custom pytest command, instead of standard(Note: curdir is project root), or pass additional arguments to default command' - type: string - required: false - default: "" + run_cpp_tests: {required: true, type: boolean, default: true, description: Run C++ tests} + persistent_storage: {required: false, type: choice, default: 'no', description: Persistent storage type, options: ['no', 'AWS_S3', 'GCPXML', 'AZURE']} + debug_enabled: {required: false, type: boolean, default: false, description: Run the build with debugging enabled} + run_enable_logging: {required: false, type: boolean, default: false, description: Enable debug logging} + run_commandline: {required: false, type: string, default: '', description: Run custom commandline before tests} + run_custom_pytest_command: {required: false, type: string, default: '', description: Run custom pytest command or additional arguments} jobs: - - compile_linux_64: - name: Compile (linux_64) - if: | - always() && - !cancelled() - runs-on: ubuntu-22.04 - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} - SCCACHE_GHA_ENABLED: "true" - defaults: - run: - shell: bash -l {0} - steps: - - uses: actions/checkout@v6.0.1 - # Do not use recursive submodules checkout to simulate conda feedstock build - # with: - # submodules: recursive - - - name: Configure sccache - uses: mozilla-actions/sccache-action@v0.0.9 - with: - version: v0.12.0 - disable_annotations: 'true' # supress noisy report that pollutes the summary page - - - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v2.0.0 - id: cpu-cores - - - name: Install Conda environment from environment-dev.yml - uses: mamba-org/setup-micromamba@v2.0.6 - with: - environment-file: environment-dev.yml - environment-name: arcticdb - init-shell: >- - bash - cache-environment: true - cache-environment-key: conda-env-linux_64 - post-cleanup: 'none' - - - name: Build ArcticDB with conda (ARCTICDB_USING_CONDA=1) - run: | - # Protocol buffers compilation require not using build isolation. - # We should always retry due to unstable nature of connections and environments - python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . - env: - ARCTICDB_USING_CONDA: 1 - ARCTICDB_BUILD_CPP_TESTS: 1 - - - name: Show sccache stats (linux_64) - run: ${SCCACHE_PATH} --show-stats || sccache --show-stats - - - name: Archive build artifacts - uses: actions/upload-artifact@v4 - if: always() - with: - name: build-linux_64 - retention-days: 7 - path: | - cpp/out/linux-conda-release-build/ - python/arcticdb_ext* - python/**/*.so - python/**/*.pyd - - compile_linux_aarch64: - name: Compile (linux_aarch64) - if: | - always() && - !cancelled() - runs-on: ubuntu-22.04-arm - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} - SCCACHE_GHA_ENABLED: "true" - defaults: - run: - shell: bash -l {0} - steps: - - uses: actions/checkout@v6.0.1 - # Do not use recursive submodules checkout to simulate conda feedstock build - # with: - # submodules: recursive - - - name: Configure sccache - uses: mozilla-actions/sccache-action@v0.0.9 - with: - version: v0.12.0 - disable_annotations: 'true' # supress noisy report that pollutes the summary page - - - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v2.0.0 - id: cpu-cores - - - name: Install Conda environment from environment-dev.yml - uses: mamba-org/setup-micromamba@v2.0.6 - with: - environment-file: environment-dev.yml - environment-name: arcticdb - init-shell: >- - bash - cache-environment: true - cache-environment-key: conda-env-linux_aarch64 - post-cleanup: 'none' - - - name: Build ArcticDB with conda (ARCTICDB_USING_CONDA=1) - run: | - # Protocol buffers compilation require not using build isolation. - # We should always retry due to unstable nature of connections and environments - python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . - env: - ARCTICDB_USING_CONDA: 1 - ARCTICDB_BUILD_CPP_TESTS: 1 - - - name: Show sccache stats (linux_aarch64) - run: ${SCCACHE_PATH} --show-stats || sccache --show-stats - - - name: Archive build artifacts - uses: actions/upload-artifact@v4 - if: always() - with: - name: build-linux_aarch64 - retention-days: 7 - path: | - cpp/out/linux-conda-release-build/ - python/arcticdb_ext* - python/**/*.so - python/**/*.pyd - - compile_osx_arm64: - name: Compile (osx_arm64) - if: | - always() && - !cancelled() - runs-on: macos-14 - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} - SCCACHE_GHA_ENABLED: "true" - defaults: - run: - shell: bash -l {0} - steps: - - uses: actions/checkout@v6.0.1 - # Do not use recursive submodules checkout to simulate conda feedstock build - # with: - # submodules: recursive - - - name: Configure sccache - uses: mozilla-actions/sccache-action@v0.0.9 - with: - version: v0.12.0 - disable_annotations: 'true' # supress noisy report that pollutes the summary page - - - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v2.0.0 - id: cpu-cores - - - name: Install Conda environment from environment-dev.yml - uses: mamba-org/setup-micromamba@v2.0.6 - with: - environment-file: environment-dev.yml - environment-name: arcticdb - init-shell: >- - bash - cache-environment: true - cache-environment-key: conda-env-osx_arm64 - post-cleanup: 'none' - - - name: Build ArcticDB with conda (ARCTICDB_USING_CONDA=1) - run: | - # Protocol buffers compilation require not using build isolation. - # We should always retry due to unstable nature of connections and environments - python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . - env: - ARCTICDB_USING_CONDA: 1 - ARCTICDB_BUILD_CPP_TESTS: 1 - - - name: Show sccache stats (osx_arm64) - run: ${SCCACHE_PATH} --show-stats || sccache --show-stats - - - name: Archive build artifacts - uses: actions/upload-artifact@v4 - if: always() - with: - name: build-osx_arm64 - retention-days: 7 - path: | - cpp/out/macos-conda-release-build/ - python/arcticdb_ext* - python/**/*.so - python/**/*.pyd - - # CPP_Conda_Tests - cpp_tests_linux_64: - if: | - always() && - !cancelled() && - (inputs.run_cpp_tests == true || github.event_name != 'workflow_dispatch') - needs: [compile_linux_64] - uses: ./.github/workflows/build_with_conda_cpp_tests.yml + linux_64: + uses: ./.github/workflows/conda_build_and_test.yml + secrets: inherit with: - runner: ubuntu-22.04 - platform: linux_64 - preset: linux-conda-release - cache_key: conda-env-linux_64 - free_disk_space: true - - cpp_tests_linux_aarch64: - if: | - always() && - !cancelled() && - (inputs.run_cpp_tests == true || github.event_name != 'workflow_dispatch') - needs: [compile_linux_aarch64] - uses: ./.github/workflows/build_with_conda_cpp_tests.yml + runner: ubuntu-22.04 + platform: linux_64 + preset: linux-conda-release + cache_key: conda-env-linux_64 + free_disk_space: true + install_mongodb: true + hypothesis_profile: ci_linux + run_cpp_tests: ${{inputs.run_cpp_tests || true}} + cpp_tests_enabled: true + persistent_storage: ${{inputs.persistent_storage || 'no'}} + debug_enabled: ${{inputs.debug_enabled || false}} + run_enable_logging: ${{inputs.run_enable_logging || false}} + run_commandline: ${{inputs.run_commandline || ''}} + run_custom_pytest_command: ${{inputs.run_custom_pytest_command || ''}} + + linux_aarch64: + uses: ./.github/workflows/conda_build_and_test.yml + secrets: inherit with: - runner: ubuntu-22.04-arm - platform: linux_aarch64 - preset: linux-conda-release - cache_key: conda-env-linux_aarch64 - free_disk_space: true - - cpp_tests_osx_arm64: - if: | - always() && - !cancelled() && - (inputs.run_cpp_tests == true || github.event_name != 'workflow_dispatch') - needs: [compile_osx_arm64] - uses: ./.github/workflows/build_with_conda_cpp_tests.yml + runner: ubuntu-22.04-arm + platform: linux_aarch64 + preset: linux-conda-release + cache_key: conda-env-linux_aarch64 + free_disk_space: true + install_mongodb: true + run_cpp_tests: ${{inputs.run_cpp_tests || true}} + cpp_tests_enabled: true + persistent_storage: ${{inputs.persistent_storage || 'no'}} + debug_enabled: ${{inputs.debug_enabled || false}} + run_enable_logging: ${{inputs.run_enable_logging || false}} + run_commandline: ${{inputs.run_commandline || ''}} + run_custom_pytest_command: ${{inputs.run_custom_pytest_command || ''}} + + osx_arm64: + uses: ./.github/workflows/conda_build_and_test.yml + secrets: inherit with: - runner: macos-14 - platform: osx_arm64 - preset: macos-conda-release - cache_key: conda-env-osx_arm64 - - cpp_tests_win_64: - if: | - always() && - !cancelled() && - (inputs.run_cpp_tests == true || github.event_name != 'workflow_dispatch') - needs: [compile_win_64] - uses: ./.github/workflows/build_with_conda_cpp_tests.yml + runner: macos-14 + platform: osx_arm64 + preset: macos-conda-release + cache_key: conda-env-osx_arm64 + hypothesis_profile: ci_macos + run_cpp_tests: ${{inputs.run_cpp_tests || true}} + cpp_tests_enabled: true + persistent_storage: ${{inputs.persistent_storage || 'no'}} + debug_enabled: ${{inputs.debug_enabled || false}} + run_enable_logging: ${{inputs.run_enable_logging || false}} + run_commandline: ${{inputs.run_commandline || ''}} + run_custom_pytest_command: ${{inputs.run_custom_pytest_command || ''}} + + win_64: + uses: ./.github/workflows/conda_build_and_test.yml + secrets: inherit with: - runner: windows-latest - platform: win_64 - preset: windows-cl-conda-release - cache_key: conda-env-win_64 - enabled: false - - # PYTHON_Conda_Tests ( TO DO !!! ) - python_tests_linux_64: - name: Python Tests (linux_64) - ${{matrix.type}} - if: | - always() && - !cancelled() - needs: [compile_linux_64] - strategy: - fail-fast: false - matrix: - type: [unit, integration, hypothesis, stress, compat, enduser] - runs-on: ubuntu-22.04 - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} - services: - mongodb: - image: mongo:4.4 - defaults: - run: - shell: bash -l {0} - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@v1.3.1 - with: - tool-cache: false - large-packages: false # Time-consuming but doesn't save that much space (4GB) - docker-images: false # We're using docker images we don't want to clear - - - uses: actions/checkout@v6.0.1 - # Do not use recursive submodules checkout to simulate conda feedstock build - # with: - # submodules: recursive - - - name: Configure sccache - uses: mozilla-actions/sccache-action@v0.0.9 - with: - version: v0.12.0 - disable_annotations: 'true' # supress noisy report that pollutes the summary page - - - name: Download build artifacts - uses: actions/download-artifact@v8 - with: - name: build-linux_64 - path: . - - - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v2.0.0 - id: cpu-cores - - - name: Install Conda environment from environment-dev.yml - uses: mamba-org/setup-micromamba@v2.0.6 - with: - environment-file: environment-dev.yml - environment-name: arcticdb - init-shell: >- - bash - cache-environment: true - cache-environment-key: conda-env-linux_64 - post-cleanup: 'none' - - - name: Install ArcticDB from artifacts - run: | - # Protocol buffers compilation require not using build isolation. - # We should always retry due to unstable nature of connections and environments - # This reuses the build artifacts from the compile_linux_64 step and make ArcticDB available for testing. - # Skip CMake configuration/build if artifacts are already present to speed up installation - if [ -d "cpp/out/linux-conda-release-build" ] && (ls python/arcticdb_ext*.so python/arcticdb_ext*.pyd 2>/dev/null | head -1 | grep -q .); then - echo "Build artifacts found, skipping CMake build" - export ARCTIC_CMAKE_PRESET=skip - fi - python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . - env: - ARCTICDB_USING_CONDA: 1 - - # Note: mongo tests are skipped in the macos workflow - - name: Install MongoDB - uses: ./.github/actions/install_mongodb - - - name: Install npm # Linux github runner image does not come with npm - uses: actions/setup-node@v6.1.0 - with: - node-version: '24' - - - name: Install Azurite - uses: nick-fields/retry@v3 - with: - # We should always retry due to unstable nature of connections and environments - timeout_minutes: 10 - max_attempts: 3 - command: npm install -g azurite - - - name: Check no arcticdb file depend on tests package - run: | - build_tooling/checks.sh - - - name: Set persistent storage variables - # Should be executed for all persistent storages but not for LMDB - if: ${{ inputs.persistent_storage != 'no' }} - uses: ./.github/actions/set_persistent_storage_env_vars - with: - aws_access_key: "${{ secrets.AWS_S3_ACCESS_KEY }}" - aws_secret_key: "${{ secrets.AWS_S3_SECRET_KEY }}" - gcp_access_key: "${{ secrets.GCP_S3_ACCESS_KEY }}" - gcp_secret_key: "${{ secrets.GCP_S3_SECRET_KEY }}" - azure_container: "githubblob" # DEFAULT BUCKET FOR AZURE - azure_connection_string: "${{ secrets.AZURE_CONNECTION_STRING }}" - persistent_storage: ${{ inputs.persistent_storage || 'no' }} - - - name: Set ArcticDB Debug Logging - if: ${{ inputs.run_enable_logging }} - uses: ./.github/actions/enable_logging - - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: ${{ inputs.debug_enabled }} - - - name: Install pytest-repeat - run: | - python -m pip --retries 3 --timeout 180 install pytest-repeat - - - name: Test with pytest - run: | - # find ssl directory where cacerts are (for Azure) - openssl version -d - # list file descriptors and other limits of the runner - ulimit -a - echo "Run commandline: $COMMANDLINE" - eval "$COMMANDLINE" - export ARCTICDB_WARN_ON_WRITING_EMPTY_DATAFRAME=0 - if [[ "$(echo "$ARCTICDB_PYTEST_ARGS" | xargs)" == pytest* ]]; then - python -m pip install pytest-repeat setuptools wheel - python setup.py protoc --build-lib python - echo "Run custom pytest command: $ARCTICDB_PYTEST_ARGS" - eval "$ARCTICDB_PYTEST_ARGS" - else - cd python - python -m pytest --timeout=3600 -v --tb=line -n logical --dist worksteal tests/${{matrix.type}} $ARCTICDB_PYTEST_ARGS - fi - env: - ARCTICDB_USING_CONDA: 1 - COMMANDLINE: ${{ inputs.run_commandline }} - # Use the Mongo created in the service container above to test against - CI_MONGO_HOST: mongodb - HYPOTHESIS_PROFILE: ci_linux - ARCTICDB_PYTEST_ARGS: ${{ inputs.run_custom_pytest_command }} - STORAGE_TYPE: ${{ inputs.persistent_storage == 'no' && 'LMDB' || inputs.persistent_storage }} - NODE_OPTIONS: --openssl-legacy-provider - - python_tests_osx_arm64: - name: Python Tests (osx_arm64) - ${{matrix.type}} - if: | - always() && - !cancelled() - needs: [compile_osx_arm64] - strategy: - fail-fast: false - matrix: - type: [unit, integration, hypothesis, stress, compat, enduser] - runs-on: macos-14 - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} - SCCACHE_GHA_ENABLED: "true" - defaults: - run: - shell: bash -l {0} - steps: - - uses: actions/checkout@v6.0.1 - # Do not use recursive submodules checkout to simulate conda feedstock build - # with: - # submodules: recursive - - - name: Configure sccache - uses: mozilla-actions/sccache-action@v0.0.9 - with: - version: v0.12.0 - disable_annotations: 'true' # supress noisy report that pollutes the summary page - - - name: Download build artifacts - uses: actions/download-artifact@v8 - with: - name: build-osx_arm64 - path: . - - - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v2.0.0 - id: cpu-cores - - - name: Install Conda environment from environment-dev.yml - uses: mamba-org/setup-micromamba@v2.0.6 - with: - environment-file: environment-dev.yml - environment-name: arcticdb - init-shell: >- - bash - cache-environment: true - cache-environment-key: conda-env-osx_arm64 - post-cleanup: 'none' - - - name: Install ArcticDB from artifacts - run: | - # Protocol buffers compilation require not using build isolation. - # We should always retry due to unstable nature of connections and environments - # This reuses the build artifacts from the compile_osx_arm64 step and make ArcticDB available for testing. - # Skip CMake configuration/build if artifacts are already present to speed up installation - if [ -d "cpp/out/macos-conda-release-build" ] && (ls python/arcticdb_ext*.so python/arcticdb_ext*.pyd 2>/dev/null | head -1 | grep -q .); then - echo "Build artifacts found, skipping CMake build" - export ARCTIC_CMAKE_PRESET=skip - fi - python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . - env: - ARCTICDB_USING_CONDA: 1 - - - name: Install npm - uses: actions/setup-node@v6.1.0 - with: - node-version: '24' - - - name: Install Azurite - uses: nick-fields/retry@v3 - with: - # We should always retry due to unstable nature of connections and environments - timeout_minutes: 10 - max_attempts: 3 - command: npm install -g azurite - - - name: Check no arcticdb file depend on tests package - run: | - build_tooling/checks.sh - - - name: Set persistent storage variables - # Should be executed for all persistent storages but not for LMDB - if: ${{ inputs.persistent_storage != 'no' }} - uses: ./.github/actions/set_persistent_storage_env_vars - with: - aws_access_key: "${{ secrets.AWS_S3_ACCESS_KEY }}" - aws_secret_key: "${{ secrets.AWS_S3_SECRET_KEY }}" - gcp_access_key: "${{ secrets.GCP_S3_ACCESS_KEY }}" - gcp_secret_key: "${{ secrets.GCP_S3_SECRET_KEY }}" - azure_container: "githubblob" # DEFAULT BUCKET FOR AZURE - azure_connection_string: "${{ secrets.AZURE_CONNECTION_STRING }}" - persistent_storage: ${{ inputs.persistent_storage || 'no' }} - - - name: Set ArcticDB Debug Logging - if: ${{ inputs.run_enable_logging }} - uses: ./.github/actions/enable_logging - - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: ${{ inputs.debug_enabled }} - - - name: Install pytest-repeat - run: | - python -m pip --retries 3 --timeout 180 install pytest-repeat - - - name: Test with pytest - run: | - # find ssl directory where cacerts are (for Azure) - openssl version -d - # list file descriptors and other limits of the runner - ulimit -a - echo "Run commandline: $COMMANDLINE" - eval "$COMMANDLINE" - export ARCTICDB_WARN_ON_WRITING_EMPTY_DATAFRAME=0 - if [[ "$(echo "$ARCTICDB_PYTEST_ARGS" | xargs)" == pytest* ]]; then - python -m pip install pytest-repeat setuptools wheel - python setup.py protoc --build-lib python - echo "Run custom pytest command: $ARCTICDB_PYTEST_ARGS" - eval "$ARCTICDB_PYTEST_ARGS" - else - cd python - python -m pytest --timeout=3600 -v --tb=line -n logical --dist worksteal tests/${{matrix.type}} $ARCTICDB_PYTEST_ARGS - fi - env: - ARCTICDB_USING_CONDA: 1 - COMMANDLINE: ${{ inputs.run_commandline }} - HYPOTHESIS_PROFILE: ci_macos - ARCTICDB_PYTEST_ARGS: ${{ inputs.run_custom_pytest_command }} - STORAGE_TYPE: ${{ inputs.persistent_storage == 'no' && 'LMDB' || inputs.persistent_storage }} - NODE_OPTIONS: --openssl-legacy-provider - - python_tests_linux_aarch64: - name: Python Tests (linux_aarch64) - ${{matrix.type}} - if: | - always() && - !cancelled() - needs: [compile_linux_aarch64] - strategy: - fail-fast: false - matrix: - type: [unit, integration, hypothesis, stress, compat, enduser] - runs-on: ubuntu-22.04-arm - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} - services: - mongodb: - image: mongo:4.4 - defaults: - run: - shell: bash -l {0} - steps: - - uses: actions/checkout@v6.0.1 - # Do not use recursive submodules checkout to simulate conda feedstock build - # with: - # submodules: recursive - - - name: Configure sccache - uses: mozilla-actions/sccache-action@v0.0.9 - with: - version: v0.12.0 - disable_annotations: 'true' # supress noisy report that pollutes the summary page - - - name: Download build artifacts - uses: actions/download-artifact@v8 - with: - name: build-linux_aarch64 - path: . - - - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v2.0.0 - id: cpu-cores - - - name: Install Conda environment from environment-dev.yml - uses: mamba-org/setup-micromamba@v2.0.6 - with: - environment-file: environment-dev.yml - environment-name: arcticdb - init-shell: >- - bash - cache-environment: true - cache-environment-key: conda-env-linux_aarch64 - post-cleanup: 'none' - - - name: Install ArcticDB from artifacts - run: | - # Protocol buffers compilation require not using build isolation. - # We should always retry due to unstable nature of connections and environments - # This reuses the build artifacts from the compile_linux_aarch64 step and make ArcticDB available for testing. - # Skip CMake configuration/build if artifacts are already present to speed up installation - if [ -d "cpp/out/linux-conda-release-build" ] && (ls python/arcticdb_ext*.so python/arcticdb_ext*.pyd 2>/dev/null | head -1 | grep -q .); then - echo "Build artifacts found, skipping CMake build" - export ARCTIC_CMAKE_PRESET=skip - fi - python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . - env: - ARCTICDB_USING_CONDA: 1 - - # Note: mongo tests are skipped in the macos workflow - - name: Install MongoDB - run: | - curl --retry 5 --retry-delay 5 --retry-connrefused -LO https://fastdl.mongodb.org/linux/mongodb-linux-aarch64-ubuntu2204-7.0.0.tgz - tar -xzf mongodb-linux-aarch64-ubuntu2204-7.0.0.tgz - cp mongodb-linux-aarch64-ubuntu2204-7.0.0/bin/* /usr/local/bin/ - mongod --version - rm -rf mongodb-linux-aarch64-ubuntu2204-7.0.0.tgz mongodb-linux-aarch64-ubuntu2204-7.0.0 - - - name: Install npm # Linux github runner image does not come with npm - uses: actions/setup-node@v6.1.0 - with: - node-version: '24' - - - name: Install Azurite - uses: nick-fields/retry@v3 - with: - # We should always retry due to unstable nature of connections and environments - timeout_minutes: 10 - max_attempts: 3 - command: npm install -g azurite - - - name: Check no arcticdb file depend on tests package - run: | - build_tooling/checks.sh - - - name: Set persistent storage variables - # Should be executed for all persistent storages but not for LMDB - if: ${{ inputs.persistent_storage != 'no' }} - uses: ./.github/actions/set_persistent_storage_env_vars - with: - aws_access_key: "${{ secrets.AWS_S3_ACCESS_KEY }}" - aws_secret_key: "${{ secrets.AWS_S3_SECRET_KEY }}" - gcp_access_key: "${{ secrets.GCP_S3_ACCESS_KEY }}" - gcp_secret_key: "${{ secrets.GCP_S3_SECRET_KEY }}" - azure_container: "githubblob" # DEFAULT BUCKET FOR AZURE - azure_connection_string: "${{ secrets.AZURE_CONNECTION_STRING }}" - persistent_storage: ${{ inputs.persistent_storage || 'no' }} - - - name: Set ArcticDB Debug Logging - if: ${{ inputs.run_enable_logging }} - uses: ./.github/actions/enable_logging - - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: ${{ inputs.debug_enabled }} - - - name: Install pytest-repeat - run: | - python -m pip --retries 3 --timeout 180 install pytest-repeat - - - name: Test with pytest - run: | - # find ssl directory where cacerts are (for Azure) - openssl version -d - # list file descriptors and other limits of the runner - ulimit -a - echo "Run commandline: $COMMANDLINE" - eval "$COMMANDLINE" - export ARCTICDB_WARN_ON_WRITING_EMPTY_DATAFRAME=0 - if [[ "$(echo "$ARCTICDB_PYTEST_ARGS" | xargs)" == pytest* ]]; then - python -m pip install pytest-repeat setuptools wheel - python setup.py protoc --build-lib python - echo "Run custom pytest command: $ARCTICDB_PYTEST_ARGS" - eval "$ARCTICDB_PYTEST_ARGS" - else - cd python - python -m pytest --timeout=3600 -v --tb=line -n logical --dist worksteal tests/${{matrix.type}} $ARCTICDB_PYTEST_ARGS - fi - env: - ARCTICDB_USING_CONDA: 1 - COMMANDLINE: ${{ inputs.run_commandline }} - ARCTICDB_PYTEST_ARGS: ${{ inputs.run_custom_pytest_command }} - STORAGE_TYPE: ${{ inputs.persistent_storage == 'no' && 'LMDB' || inputs.persistent_storage }} - - compile_win_64: - name: Compile (win_64) - if: | - always() && - !cancelled() - runs-on: windows-latest - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: true - SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} - SCCACHE_GHA_ENABLED: "true" - steps: - - uses: actions/checkout@v6.0.1 - # DONT use recursive submodules checkout to simulate conda feedstock build - # with: - # submodules: recursive - - - name: Configure sccache - uses: mozilla-actions/sccache-action@v0.0.9 - with: - version: v0.12.0 - disable_annotations: 'true' # supress noisy report that pollutes the summary page - - - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v2.0.0 - id: cpu-cores - - - name: Install Conda environment from environment-dev.yml - uses: mamba-org/setup-micromamba@v2.0.6 - with: - environment-file: environment-dev.yml - init-shell: bash cmd.exe - cache-environment: true - cache-environment-key: conda-env-win_64 - post-cleanup: 'none' - - - name: Build ArcticDB with conda (ARCTICDB_USING_CONDA=1) - shell: cmd /C call {0} - run: | - REM Some `CMAKE_*` variables (in particular CMAKE_GENERATOR_{PLATFORM,TOOLSET}) are set by mamba / micromamba / conda - REM when the environment is activated. - REM See: https://github.com/conda-forge/vc-feedstock/blob/c6bb71096319ff21ac8b75f7d91183be914c3d6b/recipe/activate.bat#L87-L131 - REM The values which are chosen prevent Ninja to be used as a generator with MSVC. - REM We override those values so that we can. - set CMAKE_GENERATOR_PLATFORM= - set CMAKE_GENERATOR_TOOLSET= - REM Protocol buffers compilation require not using build isolation. - REM We should always retry due to unstable nature of connections and environments - python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . - env: - ARCTICDB_USING_CONDA: 1 - ARCTICDB_BUILD_CPP_TESTS: 1 - ARCTIC_CMAKE_PRESET: windows-cl-conda-release - - - name: Show sccache stats (win_64) - shell: cmd /C call {0} - run: | - if defined SCCACHE_PATH ( - %SCCACHE_PATH% --show-stats - ) else ( - sccache --show-stats - ) - - - name: Archive build artifacts - uses: actions/upload-artifact@v4 - if: always() - with: - name: build-win_64 - retention-days: 7 - path: | - cpp/out/windows-cl-conda-release-build/ - python/arcticdb_ext* - python/**/*.so - python/**/*.pyd - - - python_tests_win_64: - name: Python Tests (win_64) - ${{matrix.type}} - if: | - always() && - !cancelled() - needs: [compile_win_64] - strategy: - fail-fast: false - matrix: - type: [unit, integration, hypothesis, stress, compat, enduser] - runs-on: windows-latest - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: true - SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} - SCCACHE_GHA_ENABLED: "true" - steps: - - uses: actions/checkout@v6.0.1 - # DONT use recursive submodules checkout to simulate conda feedstock build - # with: - # submodules: recursive - - - name: Download build artifacts - uses: actions/download-artifact@v8 - with: - name: build-win_64 - path: . - - - name: Configure sccache - uses: mozilla-actions/sccache-action@v0.0.9 - with: - version: v0.12.0 - disable_annotations: 'true' # supress noisy report that pollutes the summary page - - - name: Install Conda environment from environment-dev.yml - uses: mamba-org/setup-micromamba@v2.0.6 - with: - environment-file: environment-dev.yml - init-shell: bash cmd.exe - cache-environment: true - cache-environment-key: conda-env-win_64 - post-cleanup: 'none' - - - name: Install ArcticDB from artifacts - shell: cmd /C call {0} - run: | - REM Protocol buffers compilation require not using build isolation. - REM We should always retry due to unstable nature of connections and environments - REM This reuses the build artifacts from the compile_win_64 step and make ArcticDB available for testing. - REM Skip CMake configuration/build if artifacts are already present to speed up installation - set ARCTIC_CMAKE_PRESET= - if exist "cpp\out\windows-cl-conda-release-build" ( - dir /b python\arcticdb_ext*.pyd >nul 2>&1 - if not errorlevel 1 ( - echo Build artifacts found, skipping CMake build - set ARCTIC_CMAKE_PRESET=skip - ) - ) - python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . - env: - ARCTICDB_USING_CONDA: 1 - - - name: Install npm - uses: actions/setup-node@v6.1.0 - with: - node-version: '24' - - - name: Install azurite - shell: bash -elo pipefail {0} - run: | - npm install -g azurite - # Ensure npm global bin is in PATH for subsequent steps - # On Windows, npm global installs go to %APPDATA%\npm which may not be in PATH - npm_config_prefix=$(npm config get prefix) - echo "npm prefix: $npm_config_prefix" - # Add npm global bin to PATH (works for both Unix and Windows paths in bash) - npm_bin_dir="$npm_config_prefix" - if [[ "$RUNNER_OS" == "Windows" ]]; then - # Convert Windows path to Unix-style for bash (C:\Users\... -> /c/Users/...) - npm_bin_dir=$(echo "$npm_bin_dir" | sed 's|^\([A-Z]\):|/\1|' | tr '[:upper:]' '[:lower:]' | sed 's|\\|/|g') - fi - npm_bin_dir="${npm_bin_dir}/bin" - echo "$npm_bin_dir" >> $GITHUB_PATH - export PATH="$npm_bin_dir:$PATH" - # Verify azurite is accessible - echo "PATH includes: $npm_bin_dir" - which azurite && echo "Azurite found: $(which azurite)" || echo "Warning: azurite not found in PATH" - - - name: Check no arcticdb file depend on tests package - shell: bash -elo pipefail {0} - run: | - build_tooling/checks.sh - - - name: Set persistent storage variables - # Should be executed for all persistent storages but not for LMDB - if: ${{ inputs.persistent_storage != 'no' }} - uses: ./.github/actions/set_persistent_storage_env_vars - with: - aws_access_key: "${{ secrets.AWS_S3_ACCESS_KEY }}" - aws_secret_key: "${{ secrets.AWS_S3_SECRET_KEY }}" - gcp_access_key: "${{ secrets.GCP_S3_ACCESS_KEY }}" - gcp_secret_key: "${{ secrets.GCP_S3_SECRET_KEY }}" - azure_container: "githubblob" # DEFAULT BUCKET FOR AZURE - azure_connection_string: "${{ secrets.AZURE_CONNECTION_STRING }}" - persistent_storage: ${{ inputs.persistent_storage || 'no' }} - - - name: Set ArcticDB Debug Logging - if: ${{ inputs.run_enable_logging }} - uses: ./.github/actions/enable_logging - - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: ${{ inputs.debug_enabled }} - - - name: Install pytest-repeat - shell: bash -elo pipefail {0} - run: | - python -m pip --retries 3 --timeout 180 install pytest-repeat - - - name: Test with pytest - shell: bash -elo pipefail {0} - run: | - echo "Run commandline: $COMMANDLINE" - eval "$COMMANDLINE" - export ARCTICDB_RAND_SEED=$RANDOM - export ARCTICDB_WARN_ON_WRITING_EMPTY_DATAFRAME=0 - if [[ "$(echo "$ARCTICDB_PYTEST_ARGS" | xargs)" == *pytest* ]]; then - command="python -m $ARCTICDB_PYTEST_ARGS" - echo "Run custom pytest command: $command" - eval "$command" - else - cd python - # Skip LMDB tests on Windows because those tests fill the disk entirely and makes the test suite fail. - python -m pytest --timeout=3600 -v --tb=line -n logical --dist worksteal -m "not lmdb" tests/${{matrix.type}} $ARCTICDB_PYTEST_ARGS - fi - env: - ARCTICDB_USING_CONDA: 1 - COMMANDLINE: ${{ inputs.run_commandline }} - ARCTICDB_PYTEST_ARGS: ${{ inputs.run_custom_pytest_command }} - NODE_OPTIONS: --openssl-legacy-provider + runner: windows-latest + platform: win_64 + preset: windows-cl-conda-release + cache_key: conda-env-win_64 + skip_lmdb_tests: true + run_cpp_tests: ${{inputs.run_cpp_tests || true}} + cpp_tests_enabled: false + persistent_storage: ${{inputs.persistent_storage || 'no'}} + debug_enabled: ${{inputs.debug_enabled || false}} + run_enable_logging: ${{inputs.run_enable_logging || false}} + run_commandline: ${{inputs.run_commandline || ''}} + run_custom_pytest_command: ${{inputs.run_custom_pytest_command || ''}} can_merge_conda: name: All conda jobs completed - needs: [cpp_tests_linux_64, cpp_tests_linux_aarch64, cpp_tests_osx_arm64, cpp_tests_win_64, python_tests_linux_64, python_tests_linux_aarch64, python_tests_osx_arm64, python_tests_win_64] + needs: [linux_64, linux_aarch64, osx_arm64, win_64] if: | always() && !failure() && !cancelled() runs-on: ubuntu-22.04 steps: - - run: echo All conda jobs completed! + - run: echo All conda jobs completed! \ No newline at end of file diff --git a/.github/workflows/build_with_conda_and_test.yml b/.github/workflows/build_with_conda_and_test.yml new file mode 100644 index 00000000000..eccc9faa801 --- /dev/null +++ b/.github/workflows/build_with_conda_and_test.yml @@ -0,0 +1,372 @@ +name: Conda Build and Test + +on: + workflow_call: + inputs: + # Platform parameters + runner: {required: true, type: string, description: GitHub runner to execute on} + platform: {required: true, type: string, description: Platform identifier e.g. linux_64, osx_arm64, win_64} + preset: {required: true, type: string, description: CMake preset name e.g. linux-conda-release} + cache_key: {required: true, type: string, description: Unique conda environment cache key per platform} + + # Feature flags + free_disk_space: {required: false, type: boolean, default: false, description: Free disk space before tests. Linux only} + install_mongodb: {required: false, type: boolean, default: false, description: Install MongoDB for tests. Linux only} + run_cpp_tests: {required: false, type: boolean, default: true, description: Whether to run C++ tests} + cpp_tests_enabled: {required: false, type: boolean, default: true, description: Whether C++ tests are supported on this platform} + skip_lmdb_tests: {required: false, type: boolean, default: false, description: Skip LMDB tests. Windows only} + hypothesis_profile: {required: false, type: string, default: '', description: Hypothesis profile e.g. ci_linux, ci_macos} + + # Passthrough from user inputs + persistent_storage: {required: false, type: string, default: 'no', description: Persistent storage type e.g. AWS_S3, GCPXML, AZURE} + debug_enabled: {required: false, type: boolean, default: false, description: Enable tmate debug session} + run_enable_logging: {required: false, type: boolean, default: false, description: Enable ArcticDB debug logging} + run_commandline: {required: false, type: string, default: '', description: Custom commandline to run before tests} + run_custom_pytest_command: {required: false, type: string, default: '', description: Custom pytest command or additional arguments} + +jobs: + # ==================== JOB 1: COMPILE ==================== + compile: + name: Compile (${{inputs.platform}}) + if: | + always() && + !cancelled() + runs-on: ${{inputs.runner}} + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} + SCCACHE_GHA_ENABLED: "true" + defaults: + run: + shell: bash -l {0} + steps: + - uses: actions/checkout@v6.0.1 + + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: v0.12.0 + disable_annotations: 'true' + + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v2.0.0 + id: cpu-cores + + - name: Install Conda environment from environment-dev.yml + uses: mamba-org/setup-micromamba@v2.0.6 + with: + environment-file: environment-dev.yml + environment-name: arcticdb + init-shell: >- + ${{contains(inputs.platform, 'win') && 'bash cmd.exe' || 'bash'}} + cache-environment: true + cache-environment-key: ${{inputs.cache_key}} + post-cleanup: 'none' + + - name: Clear CMAKE_GENERATOR variables (Windows) + if: contains(inputs.platform, 'win') + shell: cmd /C call {0} + run: | + set CMAKE_GENERATOR_PLATFORM= + set CMAKE_GENERATOR_TOOLSET= + + - name: Build ArcticDB with conda (ARCTICDB_USING_CONDA=1) + run: | + python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . + shell: ${{contains(inputs.platform, 'win') && 'cmd /C call {0}' || 'bash -l {0}'}} + env: + ARCTICDB_USING_CONDA: 1 + ARCTICDB_BUILD_CPP_TESTS: 1 + ARCTIC_CMAKE_PRESET: ${{contains(inputs.platform, 'win') && inputs.preset || ''}} + + - name: Show sccache stats + if: ${{!contains(inputs.platform, 'win')}} + run: ${SCCACHE_PATH} --show-stats || sccache --show-stats + + - name: Show sccache stats (Windows) + if: contains(inputs.platform, 'win') + shell: cmd /C call {0} + run: | + if defined SCCACHE_PATH ( + %SCCACHE_PATH% --show-stats + ) else ( + sccache --show-stats + ) + + - name: Archive build artifacts + uses: actions/upload-artifact@v4 + if: always() + with: + name: build-${{inputs.platform}} + retention-days: 7 + path: | + cpp/out/${{inputs.preset}}-build/ + python/arcticdb_ext* + python/**/*.so + python/**/*.pyd + + # ==================== JOB 2: C++ TESTS ==================== + cpp_tests: + name: C++ Tests (${{inputs.platform}}) + needs: [compile] + if: | + always() && + !cancelled() && + inputs.run_cpp_tests && + inputs.cpp_tests_enabled + runs-on: ${{inputs.runner}} + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} + SCCACHE_GHA_ENABLED: "true" + defaults: + run: + shell: bash -l {0} + steps: + - uses: actions/checkout@v6.0.1 + + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: v0.12.0 + disable_annotations: 'true' + + - name: Free Disk Space + if: inputs.free_disk_space + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: false + large-packages: false + docker-images: false + + - name: Download build artifacts + uses: actions/download-artifact@v8 + with: + name: build-${{inputs.platform}} + path: . + + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v2.0.0 + id: cpu-cores + + - name: Install Conda environment from environment-dev.yml + uses: mamba-org/setup-micromamba@v2.0.6 + with: + environment-file: environment-dev.yml + environment-name: arcticdb + init-shell: bash + cache-environment: true + cache-environment-key: ${{inputs.cache_key}} + post-cleanup: 'none' + + - name: Configure C++ Tests + run: | + cd cpp + if [ -f out/${{inputs.preset}}-build/CMakeCache.txt ] && grep -q "TEST:BOOL=ON" out/${{inputs.preset}}-build/CMakeCache.txt; then + echo "CMake cache already has TEST=ON, skipping reconfiguration" + else + cmake --preset ${{inputs.preset}} -DTEST=ON + fi + env: + ARCTICDB_USING_CONDA: 1 + + - name: Build C++ Tests + run: | + cd cpp + cmake --build --preset ${{inputs.preset}} --target arcticdb_rapidcheck_tests -j ${{ steps.cpu-cores.outputs.count }} + cmake --build --preset ${{inputs.preset}} --target test_unit_arcticdb -j ${{ steps.cpu-cores.outputs.count }} + env: + ARCTICDB_USING_CONDA: 1 + + - name: Show sccache stats + run: ${SCCACHE_PATH} --show-stats || sccache --show-stats + + - name: Run C++ Tests + run: | + cd cpp/out/${{inputs.preset}}-build/ + ctest --output-on-failure + env: + ARCTICDB_USING_CONDA: 1 + + # ==================== JOB 3: PYTHON TESTS ==================== + python_tests: + name: Python Tests (${{inputs.platform}}) - ${{matrix.type}} + needs: [compile] + if: | + always() && + !cancelled() + runs-on: ${{inputs.runner}} + strategy: + fail-fast: false + matrix: + type: [unit, integration, hypothesis, stress, compat, enduser] + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} + SCCACHE_GHA_ENABLED: "true" + defaults: + run: + shell: bash -l {0} + steps: + - name: Free Disk Space + if: inputs.free_disk_space + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: false + large-packages: false + docker-images: false + + - uses: actions/checkout@v6.0.1 + + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: v0.12.0 + disable_annotations: 'true' + + - name: Download build artifacts + uses: actions/download-artifact@v8 + with: + name: build-${{inputs.platform}} + path: . + + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v2.0.0 + id: cpu-cores + + - name: Install Conda environment from environment-dev.yml + uses: mamba-org/setup-micromamba@v2.0.6 + with: + environment-file: environment-dev.yml + environment-name: arcticdb + init-shell: >- + ${{contains(inputs.platform, 'win') && 'bash cmd.exe' || 'bash'}} + cache-environment: true + cache-environment-key: ${{inputs.cache_key}} + post-cleanup: 'none' + + - name: Install ArcticDB from artifacts (Windows) + if: contains(inputs.platform, 'win') + shell: cmd /C call {0} + run: | + set ARCTIC_CMAKE_PRESET= + if exist "cpp\out\${{inputs.preset}}-build" ( + dir /b python\arcticdb_ext*.pyd >nul 2>&1 + if not errorlevel 1 ( + echo Build artifacts found, skipping CMake build + set ARCTIC_CMAKE_PRESET=skip + ) + ) + python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . + env: + ARCTICDB_USING_CONDA: 1 + + - name: Install ArcticDB from artifacts (Unix) + if: ${{!contains(inputs.platform, 'win')}} + run: | + if [ -d "cpp/out/${{inputs.preset}}-build" ] && (ls python/arcticdb_ext*.so python/arcticdb_ext*.pyd 2>/dev/null | head -1 | grep -q .); then + echo "Build artifacts found, skipping CMake build" + export ARCTIC_CMAKE_PRESET=skip + fi + python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . + env: + ARCTICDB_USING_CONDA: 1 + + - name: Install MongoDB + if: inputs.install_mongodb && !contains(inputs.platform, 'aarch64') + uses: ./.github/actions/install_mongodb + + - name: Install MongoDB (ARM manual) + if: inputs.install_mongodb && contains(inputs.platform, 'aarch64') + run: | + curl --retry 5 --retry-delay 5 --retry-connrefused -LO https://fastdl.mongodb.org/linux/mongodb-linux-aarch64-ubuntu2204-7.0.0.tgz + tar -xzf mongodb-linux-aarch64-ubuntu2204-7.0.0.tgz + cp mongodb-linux-aarch64-ubuntu2204-7.0.0/bin/* /usr/local/bin/ + mongod --version + rm -rf mongodb-linux-aarch64-ubuntu2204-7.0.0.tgz mongodb-linux-aarch64-ubuntu2204-7.0.0 + + - name: Install npm + uses: actions/setup-node@v6.1.0 + with: + node-version: '24' + + - name: Install Azurite + if: ${{!contains(inputs.platform, 'win')}} + uses: nick-fields/retry@v3 + with: + timeout_minutes: 10 + max_attempts: 3 + command: npm install -g azurite + + - name: Install Azurite (Windows) + if: contains(inputs.platform, 'win') + shell: bash -elo pipefail {0} + run: | + npm install -g azurite + npm_config_prefix=$(npm config get prefix) + npm_bin_dir="$npm_config_prefix" + if [[ "$RUNNER_OS" == "Windows" ]]; then + npm_bin_dir=$(echo "$npm_bin_dir" | sed 's|^$[A-Z]$:|/\1|' | tr '[:upper:]' '[:lower:]' | sed 's|\\|/|g') + fi + npm_bin_dir="${npm_bin_dir}/bin" + echo "$npm_bin_dir" >> $GITHUB_PATH + export PATH="$npm_bin_dir:$PATH" + which azurite && echo "Azurite found: $(which azurite)" || echo "Warning: azurite not found in PATH" + + - name: Check no arcticdb file depend on tests package + run: build_tooling/checks.sh + shell: ${{contains(inputs.platform, 'win') && 'bash -elo pipefail {0}' || 'bash -l {0}'}} + + - name: Set persistent storage variables + if: inputs.persistent_storage != 'no' + uses: ./.github/actions/set_persistent_storage_env_vars + with: + aws_access_key: "${{ secrets.AWS_S3_ACCESS_KEY }}" + aws_secret_key: "${{ secrets.AWS_S3_SECRET_KEY }}" + gcp_access_key: "${{ secrets.GCP_S3_ACCESS_KEY }}" + gcp_secret_key: "${{ secrets.GCP_S3_SECRET_KEY }}" + azure_container: "githubblob" + azure_connection_string: "${{ secrets.AZURE_CONNECTION_STRING }}" + persistent_storage: ${{inputs.persistent_storage}} + + - name: Set ArcticDB Debug Logging + if: inputs.run_enable_logging + uses: ./.github/actions/enable_logging + + - name: Setup tmate session + if: inputs.debug_enabled + uses: mxschmitt/action-tmate@v3 + + - name: Install pytest-repeat + run: python -m pip --retries 3 --timeout 180 install pytest-repeat + shell: ${{contains(inputs.platform, 'win') && 'bash -elo pipefail {0}' || 'bash -l {0}'}} + + - name: Test with pytest + run: | + openssl version -d || true + ulimit -a || true + echo "Run commandline: $COMMANDLINE" + eval "$COMMANDLINE" + export ARCTICDB_WARN_ON_WRITING_EMPTY_DATAFRAME=0 + if [[ "$(echo "$ARCTICDB_PYTEST_ARGS" | xargs)" == pytest* ]]; then + python -m pip install pytest-repeat setuptools wheel + python setup.py protoc --build-lib python + echo "Run custom pytest command: $ARCTICDB_PYTEST_ARGS" + eval "$ARCTICDB_PYTEST_ARGS" + else + cd python + python -m pytest --timeout=3600 -v --tb=line -n logical --dist worksteal \ + ${{inputs.skip_lmdb_tests && '-m "not lmdb"' || ''}} \ + tests/${{matrix.type}} $ARCTICDB_PYTEST_ARGS + fi + shell: ${{contains(inputs.platform, 'win') && 'bash -elo pipefail {0}' || 'bash -l {0}'}} + env: + ARCTICDB_USING_CONDA: 1 + COMMANDLINE: ${{inputs.run_commandline}} + CI_MONGO_HOST: ${{inputs.install_mongodb && 'mongodb' || ''}} + HYPOTHESIS_PROFILE: ${{inputs.hypothesis_profile}} + ARCTICDB_PYTEST_ARGS: ${{inputs.run_custom_pytest_command}} + STORAGE_TYPE: ${{inputs.persistent_storage == 'no' && 'LMDB' || inputs.persistent_storage}} + NODE_OPTIONS: --openssl-legacy-provider \ No newline at end of file diff --git a/.github/workflows/build_with_conda_cpp_tests.yml b/.github/workflows/build_with_conda_cpp_tests.yml deleted file mode 100644 index 0ceab9be73a..00000000000 --- a/.github/workflows/build_with_conda_cpp_tests.yml +++ /dev/null @@ -1,105 +0,0 @@ -name: Conda C++ Tests -on: - workflow_call: - inputs: - runner: - required: true - type: string - platform: - required: true - type: string - preset: - required: true - type: string - cache_key: - required: true - type: string - enabled: - required: false - type: boolean - default: true - free_disk_space: - required: false - type: boolean - default: false - -jobs: - cpp_tests: - name: C++ Tests (${{inputs.platform}}) - if: | - always() && - !cancelled() && - inputs.enabled - runs-on: ${{inputs.runner}} - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} - SCCACHE_GHA_ENABLED: "true" - defaults: - run: - shell: bash -l {0} - steps: - - uses: actions/checkout@v6.0.1 - - - name: Configure sccache - uses: mozilla-actions/sccache-action@v0.0.9 - with: - version: v0.12.0 - disable_annotations: 'true' - - - name: Free Disk Space - if: inputs.free_disk_space - uses: jlumbroso/free-disk-space@v1.3.1 - with: - tool-cache: false - large-packages: false - docker-images: false - - - name: Download build artifacts - uses: actions/download-artifact@v8 - with: - name: build-${{inputs.platform}} - path: . - - - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v2.0.0 - id: cpu-cores - - - name: Install Conda environment from environment-dev.yml - uses: mamba-org/setup-micromamba@v2.0.6 - with: - environment-file: environment-dev.yml - environment-name: arcticdb - init-shell: bash - cache-environment: true - cache-environment-key: ${{inputs.cache_key}} - post-cleanup: 'none' - - - name: Configure C++ Tests - run: | - cd cpp - if [ -f out/${{inputs.preset}}-build/CMakeCache.txt ] && grep -q "TEST:BOOL=ON" out/${{inputs.preset}}-build/CMakeCache.txt; then - echo "CMake cache already has TEST=ON, skipping reconfiguration" - else - cmake --preset ${{inputs.preset}} -DTEST=ON - fi - env: - ARCTICDB_USING_CONDA: 1 - - - name: Build C++ Tests - run: | - cd cpp - cmake --build --preset ${{inputs.preset}} --target arcticdb_rapidcheck_tests -j ${{ steps.cpu-cores.outputs.count }} - cmake --build --preset ${{inputs.preset}} --target test_unit_arcticdb -j ${{ steps.cpu-cores.outputs.count }} - env: - ARCTICDB_USING_CONDA: 1 - - - name: Show sccache stats - run: ${SCCACHE_PATH} --show-stats || sccache --show-stats - - - name: Run C++ Tests - run: | - cd cpp/out/${{inputs.preset}}-build/ - ctest --output-on-failure - env: - ARCTICDB_USING_CONDA: 1 \ No newline at end of file From 3a0f2eaa4176e01375f1d12ea2fd8233c87f7248 Mon Sep 17 00:00:00 2001 From: Petar Markovski Date: Thu, 19 Mar 2026 16:42:38 +0000 Subject: [PATCH 03/31] Fix ternary shell thing --- .../workflows/build_with_conda_and_test.yml | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build_with_conda_and_test.yml b/.github/workflows/build_with_conda_and_test.yml index eccc9faa801..9a7af5b834d 100644 --- a/.github/workflows/build_with_conda_and_test.yml +++ b/.github/workflows/build_with_conda_and_test.yml @@ -1,11 +1,10 @@ name: Conda Build and Test - on: workflow_call: inputs: # Platform parameters runner: {required: true, type: string, description: GitHub runner to execute on} - platform: {required: true, type: string, description: Platform identifier e.g. linux_64, osx_arm64, win_64} + platform: {required: true, type: string, description: Platform identifier e.g. linux_64 or win_64} preset: {required: true, type: string, description: CMake preset name e.g. linux-conda-release} cache_key: {required: true, type: string, description: Unique conda environment cache key per platform} @@ -15,14 +14,14 @@ on: run_cpp_tests: {required: false, type: boolean, default: true, description: Whether to run C++ tests} cpp_tests_enabled: {required: false, type: boolean, default: true, description: Whether C++ tests are supported on this platform} skip_lmdb_tests: {required: false, type: boolean, default: false, description: Skip LMDB tests. Windows only} - hypothesis_profile: {required: false, type: string, default: '', description: Hypothesis profile e.g. ci_linux, ci_macos} + hypothesis_profile: {required: false, type: string, default: '', description: Hypothesis profile e.g. ci_linux or ci_macos} # Passthrough from user inputs - persistent_storage: {required: false, type: string, default: 'no', description: Persistent storage type e.g. AWS_S3, GCPXML, AZURE} + persistent_storage: {required: false, type: string, default: 'no', description: Persistent storage type e.g. AWS_S3 or AZURE} debug_enabled: {required: false, type: boolean, default: false, description: Enable tmate debug session} run_enable_logging: {required: false, type: boolean, default: false, description: Enable ArcticDB debug logging} run_commandline: {required: false, type: string, default: '', description: Custom commandline to run before tests} - run_custom_pytest_command: {required: false, type: string, default: '', description: Custom pytest command or additional arguments} + run_custom_pytest_command: {required: false, type: string, default: '', description: Custom pytest command or additional arguments} jobs: # ==================== JOB 1: COMPILE ==================== @@ -37,6 +36,7 @@ jobs: ACTIONS_ALLOW_UNSECURE_COMMANDS: true SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} SCCACHE_GHA_ENABLED: "true" + IS_WINDOWS: ${{contains(inputs.platform, 'win')}} defaults: run: shell: bash -l {0} @@ -65,27 +65,38 @@ jobs: post-cleanup: 'none' - name: Clear CMAKE_GENERATOR variables (Windows) - if: contains(inputs.platform, 'win') + if: env.IS_WINDOWS == 'true' shell: cmd /C call {0} run: | set CMAKE_GENERATOR_PLATFORM= set CMAKE_GENERATOR_TOOLSET= - - name: Build ArcticDB with conda (ARCTICDB_USING_CONDA=1) + - name: Build ArcticDB with conda (Windows) + if: env.IS_WINDOWS == 'true' + shell: cmd /C call {0} run: | + set CMAKE_GENERATOR_PLATFORM= + set CMAKE_GENERATOR_TOOLSET= python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . - shell: ${{contains(inputs.platform, 'win') && 'cmd /C call {0}' || 'bash -l {0}'}} env: ARCTICDB_USING_CONDA: 1 ARCTICDB_BUILD_CPP_TESTS: 1 - ARCTIC_CMAKE_PRESET: ${{contains(inputs.platform, 'win') && inputs.preset || ''}} + ARCTIC_CMAKE_PRESET: ${{inputs.preset}} - - name: Show sccache stats - if: ${{!contains(inputs.platform, 'win')}} + - name: Build ArcticDB with conda (Unix) + if: env.IS_WINDOWS != 'true' + run: | + python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . + env: + ARCTICDB_USING_CONDA: 1 + ARCTICDB_BUILD_CPP_TESTS: 1 + + - name: Show sccache stats (Unix) + if: env.IS_WINDOWS != 'true' run: ${SCCACHE_PATH} --show-stats || sccache --show-stats - name: Show sccache stats (Windows) - if: contains(inputs.platform, 'win') + if: env.IS_WINDOWS == 'true' shell: cmd /C call {0} run: | if defined SCCACHE_PATH ( @@ -95,16 +106,7 @@ jobs: ) - name: Archive build artifacts - uses: actions/upload-artifact@v4 - if: always() - with: - name: build-${{inputs.platform}} - retention-days: 7 - path: | - cpp/out/${{inputs.preset}}-build/ - python/arcticdb_ext* - python/**/*.so - python/**/*.pyd + uses: actions/upload-artifact # ==================== JOB 2: C++ TESTS ==================== cpp_tests: From 0a7769dfeff4fb322121a98b7935886cff9e3016 Mon Sep 17 00:00:00 2001 From: Petar Markovski Date: Thu, 19 Mar 2026 16:46:44 +0000 Subject: [PATCH 04/31] fix incorrect 'inputs' sytnax --- .../workflows/build_with_conda_and_test.yml | 60 +++++++++++++++---- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build_with_conda_and_test.yml b/.github/workflows/build_with_conda_and_test.yml index 9a7af5b834d..7912b9c9c84 100644 --- a/.github/workflows/build_with_conda_and_test.yml +++ b/.github/workflows/build_with_conda_and_test.yml @@ -191,7 +191,7 @@ jobs: env: ARCTICDB_USING_CONDA: 1 - # ==================== JOB 3: PYTHON TESTS ==================== + # ==================== JOB 3: PYTHON TESTS ==================== python_tests: name: Python Tests (${{inputs.platform}}) - ${{matrix.type}} needs: [compile] @@ -208,6 +208,7 @@ jobs: ACTIONS_ALLOW_UNSECURE_COMMANDS: true SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} SCCACHE_GHA_ENABLED: "true" + IS_WINDOWS: ${{contains(inputs.platform, 'win')}} defaults: run: shell: bash -l {0} @@ -250,7 +251,7 @@ jobs: post-cleanup: 'none' - name: Install ArcticDB from artifacts (Windows) - if: contains(inputs.platform, 'win') + if: env.IS_WINDOWS == 'true' shell: cmd /C call {0} run: | set ARCTIC_CMAKE_PRESET= @@ -266,7 +267,7 @@ jobs: ARCTICDB_USING_CONDA: 1 - name: Install ArcticDB from artifacts (Unix) - if: ${{!contains(inputs.platform, 'win')}} + if: env.IS_WINDOWS != 'true' run: | if [ -d "cpp/out/${{inputs.preset}}-build" ] && (ls python/arcticdb_ext*.so python/arcticdb_ext*.pyd 2>/dev/null | head -1 | grep -q .); then echo "Build artifacts found, skipping CMake build" @@ -294,8 +295,8 @@ jobs: with: node-version: '24' - - name: Install Azurite - if: ${{!contains(inputs.platform, 'win')}} + - name: Install Azurite (Unix) + if: env.IS_WINDOWS != 'true' uses: nick-fields/retry@v3 with: timeout_minutes: 10 @@ -303,7 +304,7 @@ jobs: command: npm install -g azurite - name: Install Azurite (Windows) - if: contains(inputs.platform, 'win') + if: env.IS_WINDOWS == 'true' shell: bash -elo pipefail {0} run: | npm install -g azurite @@ -317,9 +318,14 @@ jobs: export PATH="$npm_bin_dir:$PATH" which azurite && echo "Azurite found: $(which azurite)" || echo "Warning: azurite not found in PATH" - - name: Check no arcticdb file depend on tests package + - name: Check no arcticdb file depend on tests package (Unix) + if: env.IS_WINDOWS != 'true' + run: build_tooling/checks.sh + + - name: Check no arcticdb file depend on tests package (Windows) + if: env.IS_WINDOWS == 'true' + shell: bash -elo pipefail {0} run: build_tooling/checks.sh - shell: ${{contains(inputs.platform, 'win') && 'bash -elo pipefail {0}' || 'bash -l {0}'}} - name: Set persistent storage variables if: inputs.persistent_storage != 'no' @@ -341,11 +347,17 @@ jobs: if: inputs.debug_enabled uses: mxschmitt/action-tmate@v3 - - name: Install pytest-repeat + - name: Install pytest-repeat (Unix) + if: env.IS_WINDOWS != 'true' + run: python -m pip --retries 3 --timeout 180 install pytest-repeat + + - name: Install pytest-repeat (Windows) + if: env.IS_WINDOWS == 'true' + shell: bash -elo pipefail {0} run: python -m pip --retries 3 --timeout 180 install pytest-repeat - shell: ${{contains(inputs.platform, 'win') && 'bash -elo pipefail {0}' || 'bash -l {0}'}} - - name: Test with pytest + - name: Test with pytest (Unix) + if: env.IS_WINDOWS != 'true' run: | openssl version -d || true ulimit -a || true @@ -360,10 +372,8 @@ jobs: else cd python python -m pytest --timeout=3600 -v --tb=line -n logical --dist worksteal \ - ${{inputs.skip_lmdb_tests && '-m "not lmdb"' || ''}} \ tests/${{matrix.type}} $ARCTICDB_PYTEST_ARGS fi - shell: ${{contains(inputs.platform, 'win') && 'bash -elo pipefail {0}' || 'bash -l {0}'}} env: ARCTICDB_USING_CONDA: 1 COMMANDLINE: ${{inputs.run_commandline}} @@ -371,4 +381,28 @@ jobs: HYPOTHESIS_PROFILE: ${{inputs.hypothesis_profile}} ARCTICDB_PYTEST_ARGS: ${{inputs.run_custom_pytest_command}} STORAGE_TYPE: ${{inputs.persistent_storage == 'no' && 'LMDB' || inputs.persistent_storage}} + NODE_OPTIONS: --openssl-legacy-provider + + - name: Test with pytest (Windows) + if: env.IS_WINDOWS == 'true' + shell: bash -elo pipefail {0} + run: | + echo "Run commandline: $COMMANDLINE" + eval "$COMMANDLINE" + export ARCTICDB_RAND_SEED=$RANDOM + export ARCTICDB_WARN_ON_WRITING_EMPTY_DATAFRAME=0 + if [[ "$(echo "$ARCTICDB_PYTEST_ARGS" | xargs)" == *pytest* ]]; then + command="python -m $ARCTICDB_PYTEST_ARGS" + echo "Run custom pytest command: $command" + eval "$command" + else + cd python + python -m pytest --timeout=3600 -v --tb=line -n logical --dist worksteal \ + -m "not lmdb" \ + tests/${{matrix.type}} $ARCTICDB_PYTEST_ARGS + fi + env: + ARCTICDB_USING_CONDA: 1 + COMMANDLINE: ${{inputs.run_commandline}} + ARCTICDB_PYTEST_ARGS: ${{inputs.run_custom_pytest_command}} NODE_OPTIONS: --openssl-legacy-provider \ No newline at end of file From 2cd5b5502477702387c12d0bc3d84a8e99c485eb Mon Sep 17 00:00:00 2001 From: Petar Markovski Date: Thu, 19 Mar 2026 16:51:46 +0000 Subject: [PATCH 05/31] fix missing kvp's for build artifacts --- .github/workflows/build_with_conda_and_test.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_with_conda_and_test.yml b/.github/workflows/build_with_conda_and_test.yml index 7912b9c9c84..b34e4f6327b 100644 --- a/.github/workflows/build_with_conda_and_test.yml +++ b/.github/workflows/build_with_conda_and_test.yml @@ -106,7 +106,16 @@ jobs: ) - name: Archive build artifacts - uses: actions/upload-artifact + uses: actions/upload-artifact@v4 + if: always() + with: + name: build-${{inputs.platform}} + retention-days: 7 + path: | + cpp/out/${{inputs.preset}}-build/ + python/arcticdb_ext* + python/**/*.so + python/**/*.pyd # ==================== JOB 2: C++ TESTS ==================== cpp_tests: From eab8513f6c711c71d993c415d9a4f6d9cb918ae0 Mon Sep 17 00:00:00 2001 From: Petar Markovski Date: Fri, 20 Mar 2026 08:12:54 +0000 Subject: [PATCH 06/31] change workflow file name --- .github/workflows/build_with_conda.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_with_conda.yml b/.github/workflows/build_with_conda.yml index 4d4ff1d9154..d25ca3dc1d7 100644 --- a/.github/workflows/build_with_conda.yml +++ b/.github/workflows/build_with_conda.yml @@ -25,7 +25,7 @@ on: jobs: linux_64: - uses: ./.github/workflows/conda_build_and_test.yml + uses: ./.github/workflows/build_with_conda_and_test.yml secrets: inherit with: runner: ubuntu-22.04 @@ -44,7 +44,7 @@ jobs: run_custom_pytest_command: ${{inputs.run_custom_pytest_command || ''}} linux_aarch64: - uses: ./.github/workflows/conda_build_and_test.yml + uses: ./.github/workflows/build_with_conda_and_test.yml secrets: inherit with: runner: ubuntu-22.04-arm @@ -62,7 +62,7 @@ jobs: run_custom_pytest_command: ${{inputs.run_custom_pytest_command || ''}} osx_arm64: - uses: ./.github/workflows/conda_build_and_test.yml + uses: ./.github/workflows/build_with_conda_and_test.yml secrets: inherit with: runner: macos-14 @@ -79,7 +79,7 @@ jobs: run_custom_pytest_command: ${{inputs.run_custom_pytest_command || ''}} win_64: - uses: ./.github/workflows/conda_build_and_test.yml + uses: ./.github/workflows/build_with_conda_and_test.yml secrets: inherit with: runner: windows-latest From d19f5f4196a3d7b9176b0f02eb678a904d3ebf25 Mon Sep 17 00:00:00 2001 From: Petar Markovski Date: Fri, 20 Mar 2026 08:41:14 +0000 Subject: [PATCH 07/31] add 'dev' to HYPOTHESIS_PROFILE for python_tests --- .github/workflows/build_with_conda_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_with_conda_and_test.yml b/.github/workflows/build_with_conda_and_test.yml index b34e4f6327b..e2e338a28e0 100644 --- a/.github/workflows/build_with_conda_and_test.yml +++ b/.github/workflows/build_with_conda_and_test.yml @@ -387,7 +387,7 @@ jobs: ARCTICDB_USING_CONDA: 1 COMMANDLINE: ${{inputs.run_commandline}} CI_MONGO_HOST: ${{inputs.install_mongodb && 'mongodb' || ''}} - HYPOTHESIS_PROFILE: ${{inputs.hypothesis_profile}} + HYPOTHESIS_PROFILE: ${{inputs.hypothesis_profile || 'dev'}} ARCTICDB_PYTEST_ARGS: ${{inputs.run_custom_pytest_command}} STORAGE_TYPE: ${{inputs.persistent_storage == 'no' && 'LMDB' || inputs.persistent_storage}} NODE_OPTIONS: --openssl-legacy-provider From 29f306f3c50dea229a78431c7383da45ca2b5b29 Mon Sep 17 00:00:00 2001 From: Petar Markovski Date: Fri, 20 Mar 2026 09:15:45 +0000 Subject: [PATCH 08/31] fix regex for azure installing on win --- .github/workflows/build_with_conda_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_with_conda_and_test.yml b/.github/workflows/build_with_conda_and_test.yml index e2e338a28e0..100a7cadc1f 100644 --- a/.github/workflows/build_with_conda_and_test.yml +++ b/.github/workflows/build_with_conda_and_test.yml @@ -320,7 +320,7 @@ jobs: npm_config_prefix=$(npm config get prefix) npm_bin_dir="$npm_config_prefix" if [[ "$RUNNER_OS" == "Windows" ]]; then - npm_bin_dir=$(echo "$npm_bin_dir" | sed 's|^$[A-Z]$:|/\1|' | tr '[:upper:]' '[:lower:]' | sed 's|\\|/|g') + npm_bin_dir=$(echo "$npm_bin_dir" | sed 's|^$[A-Za-z]$:|/\1|' | tr '[:upper:]' '[:lower:]' | sed 's|\\|/|g') fi npm_bin_dir="${npm_bin_dir}/bin" echo "$npm_bin_dir" >> $GITHUB_PATH From e5eacf919560cc64e7d5b9b7b6ec490a6a2a4c9a Mon Sep 17 00:00:00 2001 From: Petar Markovski Date: Fri, 20 Mar 2026 09:38:14 +0000 Subject: [PATCH 09/31] change if logic in azure installation for win --- .github/workflows/build_with_conda_and_test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_with_conda_and_test.yml b/.github/workflows/build_with_conda_and_test.yml index 100a7cadc1f..6614f899558 100644 --- a/.github/workflows/build_with_conda_and_test.yml +++ b/.github/workflows/build_with_conda_and_test.yml @@ -320,7 +320,12 @@ jobs: npm_config_prefix=$(npm config get prefix) npm_bin_dir="$npm_config_prefix" if [[ "$RUNNER_OS" == "Windows" ]]; then - npm_bin_dir=$(echo "$npm_bin_dir" | sed 's|^$[A-Za-z]$:|/\1|' | tr '[:upper:]' '[:lower:]' | sed 's|\\|/|g') + npm_bin_dir=$(echo "$npm_bin_dir" | sed 's|\\|/|g') + if [[ "$npm_bin_dir" =~ ^([A-Za-z]):/(.*) ]]; then + drive_letter="${BASH_REMATCH[1]}" + rest="${BASH_REMATCH[2]}" + npm_bin_dir="/${drive_letter,,}/${rest}" + fi fi npm_bin_dir="${npm_bin_dir}/bin" echo "$npm_bin_dir" >> $GITHUB_PATH From 17cc06f4f448c491ec96db385e246b362c2653da Mon Sep 17 00:00:00 2001 From: Petar Markovski Date: Fri, 20 Mar 2026 15:42:00 +0000 Subject: [PATCH 10/31] remove ' || true ' from run_cpp_test --- .github/workflows/build_with_conda.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_with_conda.yml b/.github/workflows/build_with_conda.yml index d25ca3dc1d7..7081cf5a996 100644 --- a/.github/workflows/build_with_conda.yml +++ b/.github/workflows/build_with_conda.yml @@ -35,7 +35,7 @@ jobs: free_disk_space: true install_mongodb: true hypothesis_profile: ci_linux - run_cpp_tests: ${{inputs.run_cpp_tests || true}} + run_cpp_tests: ${{inputs.run_cpp_tests}} cpp_tests_enabled: true persistent_storage: ${{inputs.persistent_storage || 'no'}} debug_enabled: ${{inputs.debug_enabled || false}} @@ -53,7 +53,7 @@ jobs: cache_key: conda-env-linux_aarch64 free_disk_space: true install_mongodb: true - run_cpp_tests: ${{inputs.run_cpp_tests || true}} + run_cpp_tests: ${{inputs.run_cpp_tests}} cpp_tests_enabled: true persistent_storage: ${{inputs.persistent_storage || 'no'}} debug_enabled: ${{inputs.debug_enabled || false}} @@ -70,7 +70,7 @@ jobs: preset: macos-conda-release cache_key: conda-env-osx_arm64 hypothesis_profile: ci_macos - run_cpp_tests: ${{inputs.run_cpp_tests || true}} + run_cpp_tests: ${{inputs.run_cpp_tests}} cpp_tests_enabled: true persistent_storage: ${{inputs.persistent_storage || 'no'}} debug_enabled: ${{inputs.debug_enabled || false}} @@ -87,7 +87,8 @@ jobs: preset: windows-cl-conda-release cache_key: conda-env-win_64 skip_lmdb_tests: true - run_cpp_tests: ${{inputs.run_cpp_tests || true}} + run_cpp_tests: ${{inputs.run_cpp_tests}} + # Rapidcheck tests are currently disabled on Windows due to a linking issue we need to investigate. cpp_tests_enabled: false persistent_storage: ${{inputs.persistent_storage || 'no'}} debug_enabled: ${{inputs.debug_enabled || false}} From f4e77b4334730736b64dc76d79c69cb2ef0496c9 Mon Sep 17 00:00:00 2001 From: Petar Markovski Date: Fri, 20 Mar 2026 15:56:19 +0000 Subject: [PATCH 11/31] add missing comment for checkout step --- .github/workflows/build_with_conda_and_test.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/build_with_conda_and_test.yml b/.github/workflows/build_with_conda_and_test.yml index 6614f899558..6472c072079 100644 --- a/.github/workflows/build_with_conda_and_test.yml +++ b/.github/workflows/build_with_conda_and_test.yml @@ -42,6 +42,9 @@ jobs: shell: bash -l {0} steps: - uses: actions/checkout@v6.0.1 + # Do not use recursive submodules checkout to simulate conda feedstock build + # with: + # - name: Configure sccache uses: mozilla-actions/sccache-action@v0.0.9 @@ -136,6 +139,9 @@ jobs: shell: bash -l {0} steps: - uses: actions/checkout@v6.0.1 + # Do not use recursive submodules checkout to simulate conda feedstock build + # with: + # - name: Configure sccache uses: mozilla-actions/sccache-action@v0.0.9 @@ -231,6 +237,9 @@ jobs: docker-images: false - uses: actions/checkout@v6.0.1 + # Do not use recursive submodules checkout to simulate conda feedstock build + # with: + # - name: Configure sccache uses: mozilla-actions/sccache-action@v0.0.9 From 7fdf75b5f140bda977b4d364c0896c5bce6b73d9 Mon Sep 17 00:00:00 2001 From: Petar Markovski Date: Fri, 20 Mar 2026 16:02:03 +0000 Subject: [PATCH 12/31] add missing text from comment --- .github/workflows/build_with_conda_and_test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_with_conda_and_test.yml b/.github/workflows/build_with_conda_and_test.yml index 6472c072079..ef26bb0fb8f 100644 --- a/.github/workflows/build_with_conda_and_test.yml +++ b/.github/workflows/build_with_conda_and_test.yml @@ -44,7 +44,7 @@ jobs: - uses: actions/checkout@v6.0.1 # Do not use recursive submodules checkout to simulate conda feedstock build # with: - # + # submodules: recursive - name: Configure sccache uses: mozilla-actions/sccache-action@v0.0.9 @@ -141,7 +141,7 @@ jobs: - uses: actions/checkout@v6.0.1 # Do not use recursive submodules checkout to simulate conda feedstock build # with: - # + # submodules: recursive - name: Configure sccache uses: mozilla-actions/sccache-action@v0.0.9 @@ -239,7 +239,7 @@ jobs: - uses: actions/checkout@v6.0.1 # Do not use recursive submodules checkout to simulate conda feedstock build # with: - # + # submodules: recursive - name: Configure sccache uses: mozilla-actions/sccache-action@v0.0.9 From 881c5359cba10cab5c66ace1829b24d6573d2d51 Mon Sep 17 00:00:00 2001 From: Petar Markovski Date: Fri, 20 Mar 2026 16:13:05 +0000 Subject: [PATCH 13/31] Remove redundant Clear CMAKE_GENERATOR variables --- .github/workflows/build_with_conda_and_test.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/build_with_conda_and_test.yml b/.github/workflows/build_with_conda_and_test.yml index ef26bb0fb8f..340c5ec1b9a 100644 --- a/.github/workflows/build_with_conda_and_test.yml +++ b/.github/workflows/build_with_conda_and_test.yml @@ -67,13 +67,6 @@ jobs: cache-environment-key: ${{inputs.cache_key}} post-cleanup: 'none' - - name: Clear CMAKE_GENERATOR variables (Windows) - if: env.IS_WINDOWS == 'true' - shell: cmd /C call {0} - run: | - set CMAKE_GENERATOR_PLATFORM= - set CMAKE_GENERATOR_TOOLSET= - - name: Build ArcticDB with conda (Windows) if: env.IS_WINDOWS == 'true' shell: cmd /C call {0} From 8df899725511d49d22d22fb9b6de6a7aab376325 Mon Sep 17 00:00:00 2001 From: Petar Markovski Date: Mon, 23 Mar 2026 08:27:20 +0000 Subject: [PATCH 14/31] separte 'build_with_conda_and_test.yml' to '..._win.yml' and '..._unix.yml' files --- .github/workflows/build_with_conda.yml | 11 +- ...l => compile_and_test_with_conda_unix.yml} | 147 ++---------- .../compile_and_test_with_conda_win.yml | 226 ++++++++++++++++++ 3 files changed, 245 insertions(+), 139 deletions(-) rename .github/workflows/{build_with_conda_and_test.yml => compile_and_test_with_conda_unix.yml} (66%) create mode 100644 .github/workflows/compile_and_test_with_conda_win.yml diff --git a/.github/workflows/build_with_conda.yml b/.github/workflows/build_with_conda.yml index 7081cf5a996..0633d284f79 100644 --- a/.github/workflows/build_with_conda.yml +++ b/.github/workflows/build_with_conda.yml @@ -25,7 +25,7 @@ on: jobs: linux_64: - uses: ./.github/workflows/build_with_conda_and_test.yml + uses: ./.github/workflows/compile_and_test_with_conda_unix.yml secrets: inherit with: runner: ubuntu-22.04 @@ -44,7 +44,7 @@ jobs: run_custom_pytest_command: ${{inputs.run_custom_pytest_command || ''}} linux_aarch64: - uses: ./.github/workflows/build_with_conda_and_test.yml + uses: ./.github/workflows/compile_and_test_with_conda_unix.yml secrets: inherit with: runner: ubuntu-22.04-arm @@ -62,7 +62,7 @@ jobs: run_custom_pytest_command: ${{inputs.run_custom_pytest_command || ''}} osx_arm64: - uses: ./.github/workflows/build_with_conda_and_test.yml + uses: ./.github/workflows/compile_and_test_with_conda_unix.yml secrets: inherit with: runner: macos-14 @@ -79,17 +79,14 @@ jobs: run_custom_pytest_command: ${{inputs.run_custom_pytest_command || ''}} win_64: - uses: ./.github/workflows/build_with_conda_and_test.yml + uses: ./.github/workflows/compile_and_test_with_conda_win.yml secrets: inherit with: runner: windows-latest platform: win_64 preset: windows-cl-conda-release cache_key: conda-env-win_64 - skip_lmdb_tests: true run_cpp_tests: ${{inputs.run_cpp_tests}} - # Rapidcheck tests are currently disabled on Windows due to a linking issue we need to investigate. - cpp_tests_enabled: false persistent_storage: ${{inputs.persistent_storage || 'no'}} debug_enabled: ${{inputs.debug_enabled || false}} run_enable_logging: ${{inputs.run_enable_logging || false}} diff --git a/.github/workflows/build_with_conda_and_test.yml b/.github/workflows/compile_and_test_with_conda_unix.yml similarity index 66% rename from .github/workflows/build_with_conda_and_test.yml rename to .github/workflows/compile_and_test_with_conda_unix.yml index 340c5ec1b9a..821945ea821 100644 --- a/.github/workflows/build_with_conda_and_test.yml +++ b/.github/workflows/compile_and_test_with_conda_unix.yml @@ -1,23 +1,19 @@ -name: Conda Build and Test +name: Conda Build and Test (Unix) on: workflow_call: inputs: - # Platform parameters runner: {required: true, type: string, description: GitHub runner to execute on} - platform: {required: true, type: string, description: Platform identifier e.g. linux_64 or win_64} + platform: {required: true, type: string, description: Platform identifier e.g. linux_64 or osx_arm64} preset: {required: true, type: string, description: CMake preset name e.g. linux-conda-release} cache_key: {required: true, type: string, description: Unique conda environment cache key per platform} - # Feature flags - free_disk_space: {required: false, type: boolean, default: false, description: Free disk space before tests. Linux only} + free_disk_space: {required: false, type: boolean, default: false, description: Free disk space before build. Linux only} install_mongodb: {required: false, type: boolean, default: false, description: Install MongoDB for tests. Linux only} run_cpp_tests: {required: false, type: boolean, default: true, description: Whether to run C++ tests} cpp_tests_enabled: {required: false, type: boolean, default: true, description: Whether C++ tests are supported on this platform} - skip_lmdb_tests: {required: false, type: boolean, default: false, description: Skip LMDB tests. Windows only} hypothesis_profile: {required: false, type: string, default: '', description: Hypothesis profile e.g. ci_linux or ci_macos} - # Passthrough from user inputs - persistent_storage: {required: false, type: string, default: 'no', description: Persistent storage type e.g. AWS_S3 or AZURE} + persistent_storage: {required: false, type: string, default: 'no', description: Persistent storage type} debug_enabled: {required: false, type: boolean, default: false, description: Enable tmate debug session} run_enable_logging: {required: false, type: boolean, default: false, description: Enable ArcticDB debug logging} run_commandline: {required: false, type: string, default: '', description: Custom commandline to run before tests} @@ -36,15 +32,11 @@ jobs: ACTIONS_ALLOW_UNSECURE_COMMANDS: true SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} SCCACHE_GHA_ENABLED: "true" - IS_WINDOWS: ${{contains(inputs.platform, 'win')}} defaults: run: shell: bash -l {0} steps: - uses: actions/checkout@v6.0.1 - # Do not use recursive submodules checkout to simulate conda feedstock build - # with: - # submodules: recursive - name: Configure sccache uses: mozilla-actions/sccache-action@v0.0.9 @@ -61,46 +53,21 @@ jobs: with: environment-file: environment-dev.yml environment-name: arcticdb - init-shell: >- - ${{contains(inputs.platform, 'win') && 'bash cmd.exe' || 'bash'}} + init-shell: bash cache-environment: true cache-environment-key: ${{inputs.cache_key}} post-cleanup: 'none' - - name: Build ArcticDB with conda (Windows) - if: env.IS_WINDOWS == 'true' - shell: cmd /C call {0} - run: | - set CMAKE_GENERATOR_PLATFORM= - set CMAKE_GENERATOR_TOOLSET= - python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . - env: - ARCTICDB_USING_CONDA: 1 - ARCTICDB_BUILD_CPP_TESTS: 1 - ARCTIC_CMAKE_PRESET: ${{inputs.preset}} - - - name: Build ArcticDB with conda (Unix) - if: env.IS_WINDOWS != 'true' + - name: Build ArcticDB with conda run: | python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . env: ARCTICDB_USING_CONDA: 1 ARCTICDB_BUILD_CPP_TESTS: 1 - - name: Show sccache stats (Unix) - if: env.IS_WINDOWS != 'true' + - name: Show sccache stats run: ${SCCACHE_PATH} --show-stats || sccache --show-stats - - name: Show sccache stats (Windows) - if: env.IS_WINDOWS == 'true' - shell: cmd /C call {0} - run: | - if defined SCCACHE_PATH ( - %SCCACHE_PATH% --show-stats - ) else ( - sccache --show-stats - ) - - name: Archive build artifacts uses: actions/upload-artifact@v4 if: always() @@ -111,7 +78,6 @@ jobs: cpp/out/${{inputs.preset}}-build/ python/arcticdb_ext* python/**/*.so - python/**/*.pyd # ==================== JOB 2: C++ TESTS ==================== cpp_tests: @@ -132,9 +98,6 @@ jobs: shell: bash -l {0} steps: - uses: actions/checkout@v6.0.1 - # Do not use recursive submodules checkout to simulate conda feedstock build - # with: - # submodules: recursive - name: Configure sccache uses: mozilla-actions/sccache-action@v0.0.9 @@ -199,7 +162,7 @@ jobs: env: ARCTICDB_USING_CONDA: 1 - # ==================== JOB 3: PYTHON TESTS ==================== + # ==================== JOB 3: PYTHON TESTS ==================== python_tests: name: Python Tests (${{inputs.platform}}) - ${{matrix.type}} needs: [compile] @@ -216,7 +179,6 @@ jobs: ACTIONS_ALLOW_UNSECURE_COMMANDS: true SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} SCCACHE_GHA_ENABLED: "true" - IS_WINDOWS: ${{contains(inputs.platform, 'win')}} defaults: run: shell: bash -l {0} @@ -230,9 +192,6 @@ jobs: docker-images: false - uses: actions/checkout@v6.0.1 - # Do not use recursive submodules checkout to simulate conda feedstock build - # with: - # submodules: recursive - name: Configure sccache uses: mozilla-actions/sccache-action@v0.0.9 @@ -255,32 +214,14 @@ jobs: with: environment-file: environment-dev.yml environment-name: arcticdb - init-shell: >- - ${{contains(inputs.platform, 'win') && 'bash cmd.exe' || 'bash'}} + init-shell: bash cache-environment: true cache-environment-key: ${{inputs.cache_key}} post-cleanup: 'none' - - name: Install ArcticDB from artifacts (Windows) - if: env.IS_WINDOWS == 'true' - shell: cmd /C call {0} + - name: Install ArcticDB from artifacts run: | - set ARCTIC_CMAKE_PRESET= - if exist "cpp\out\${{inputs.preset}}-build" ( - dir /b python\arcticdb_ext*.pyd >nul 2>&1 - if not errorlevel 1 ( - echo Build artifacts found, skipping CMake build - set ARCTIC_CMAKE_PRESET=skip - ) - ) - python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . - env: - ARCTICDB_USING_CONDA: 1 - - - name: Install ArcticDB from artifacts (Unix) - if: env.IS_WINDOWS != 'true' - run: | - if [ -d "cpp/out/${{inputs.preset}}-build" ] && (ls python/arcticdb_ext*.so python/arcticdb_ext*.pyd 2>/dev/null | head -1 | grep -q .); then + if [ -d "cpp/out/${{inputs.preset}}-build" ] && (ls python/arcticdb_ext*.so 2>/dev/null | head -1 | grep -q .); then echo "Build artifacts found, skipping CMake build" export ARCTIC_CMAKE_PRESET=skip fi @@ -306,41 +247,14 @@ jobs: with: node-version: '24' - - name: Install Azurite (Unix) - if: env.IS_WINDOWS != 'true' + - name: Install Azurite uses: nick-fields/retry@v3 with: timeout_minutes: 10 max_attempts: 3 command: npm install -g azurite - - name: Install Azurite (Windows) - if: env.IS_WINDOWS == 'true' - shell: bash -elo pipefail {0} - run: | - npm install -g azurite - npm_config_prefix=$(npm config get prefix) - npm_bin_dir="$npm_config_prefix" - if [[ "$RUNNER_OS" == "Windows" ]]; then - npm_bin_dir=$(echo "$npm_bin_dir" | sed 's|\\|/|g') - if [[ "$npm_bin_dir" =~ ^([A-Za-z]):/(.*) ]]; then - drive_letter="${BASH_REMATCH[1]}" - rest="${BASH_REMATCH[2]}" - npm_bin_dir="/${drive_letter,,}/${rest}" - fi - fi - npm_bin_dir="${npm_bin_dir}/bin" - echo "$npm_bin_dir" >> $GITHUB_PATH - export PATH="$npm_bin_dir:$PATH" - which azurite && echo "Azurite found: $(which azurite)" || echo "Warning: azurite not found in PATH" - - - name: Check no arcticdb file depend on tests package (Unix) - if: env.IS_WINDOWS != 'true' - run: build_tooling/checks.sh - - - name: Check no arcticdb file depend on tests package (Windows) - if: env.IS_WINDOWS == 'true' - shell: bash -elo pipefail {0} + - name: Check no arcticdb file depend on tests package run: build_tooling/checks.sh - name: Set persistent storage variables @@ -363,17 +277,10 @@ jobs: if: inputs.debug_enabled uses: mxschmitt/action-tmate@v3 - - name: Install pytest-repeat (Unix) - if: env.IS_WINDOWS != 'true' + - name: Install pytest-repeat run: python -m pip --retries 3 --timeout 180 install pytest-repeat - - name: Install pytest-repeat (Windows) - if: env.IS_WINDOWS == 'true' - shell: bash -elo pipefail {0} - run: python -m pip --retries 3 --timeout 180 install pytest-repeat - - - name: Test with pytest (Unix) - if: env.IS_WINDOWS != 'true' + - name: Test with pytest run: | openssl version -d || true ulimit -a || true @@ -397,28 +304,4 @@ jobs: HYPOTHESIS_PROFILE: ${{inputs.hypothesis_profile || 'dev'}} ARCTICDB_PYTEST_ARGS: ${{inputs.run_custom_pytest_command}} STORAGE_TYPE: ${{inputs.persistent_storage == 'no' && 'LMDB' || inputs.persistent_storage}} - NODE_OPTIONS: --openssl-legacy-provider - - - name: Test with pytest (Windows) - if: env.IS_WINDOWS == 'true' - shell: bash -elo pipefail {0} - run: | - echo "Run commandline: $COMMANDLINE" - eval "$COMMANDLINE" - export ARCTICDB_RAND_SEED=$RANDOM - export ARCTICDB_WARN_ON_WRITING_EMPTY_DATAFRAME=0 - if [[ "$(echo "$ARCTICDB_PYTEST_ARGS" | xargs)" == *pytest* ]]; then - command="python -m $ARCTICDB_PYTEST_ARGS" - echo "Run custom pytest command: $command" - eval "$command" - else - cd python - python -m pytest --timeout=3600 -v --tb=line -n logical --dist worksteal \ - -m "not lmdb" \ - tests/${{matrix.type}} $ARCTICDB_PYTEST_ARGS - fi - env: - ARCTICDB_USING_CONDA: 1 - COMMANDLINE: ${{inputs.run_commandline}} - ARCTICDB_PYTEST_ARGS: ${{inputs.run_custom_pytest_command}} NODE_OPTIONS: --openssl-legacy-provider \ No newline at end of file diff --git a/.github/workflows/compile_and_test_with_conda_win.yml b/.github/workflows/compile_and_test_with_conda_win.yml new file mode 100644 index 00000000000..b3d1a8d0370 --- /dev/null +++ b/.github/workflows/compile_and_test_with_conda_win.yml @@ -0,0 +1,226 @@ +name: Conda Build and Test (Windows) +on: + workflow_call: + inputs: + runner: {required: true, type: string, description: GitHub runner to execute on} + platform: {required: true, type: string, description: Platform identifier e.g. win_64} + preset: {required: true, type: string, description: CMake preset name e.g. windows-cl-conda-release} + cache_key: {required: true, type: string, description: Unique conda environment cache key per platform} + + run_cpp_tests: {required: false, type: boolean, default: true, description: Whether to run C++ tests} + + persistent_storage: {required: false, type: string, default: 'no', description: Persistent storage type} + debug_enabled: {required: false, type: boolean, default: false, description: Enable tmate debug session} + run_enable_logging: {required: false, type: boolean, default: false, description: Enable ArcticDB debug logging} + run_commandline: {required: false, type: string, default: '', description: Custom commandline to run before tests} + run_custom_pytest_command: {required: false, type: string, default: '', description: Custom pytest command or additional arguments} + +jobs: + # ==================== JOB 1: COMPILE ==================== + compile: + name: Compile (${{inputs.platform}}) + if: | + always() && + !cancelled() + runs-on: ${{inputs.runner}} + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} + SCCACHE_GHA_ENABLED: "true" + defaults: + run: + shell: bash -l {0} + steps: + - uses: actions/checkout@v6.0.1 + + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: v0.12.0 + disable_annotations: 'true' + + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v2.0.0 + id: cpu-cores + + - name: Install Conda environment from environment-dev.yml + uses: mamba-org/setup-micromamba@v2.0.6 + with: + environment-file: environment-dev.yml + environment-name: arcticdb + init-shell: bash cmd.exe + cache-environment: true + cache-environment-key: ${{inputs.cache_key}} + post-cleanup: 'none' + + - name: Build ArcticDB with conda + shell: cmd /C call {0} + run: | + set CMAKE_GENERATOR_PLATFORM= + set CMAKE_GENERATOR_TOOLSET= + python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . + env: + ARCTICDB_USING_CONDA: 1 + ARCTICDB_BUILD_CPP_TESTS: 1 + ARCTIC_CMAKE_PRESET: ${{inputs.preset}} + + - name: Show sccache stats + shell: cmd /C call {0} + run: | + if defined SCCACHE_PATH ( + %SCCACHE_PATH% --show-stats + ) else ( + sccache --show-stats + ) + + - name: Archive build artifacts + uses: actions/upload-artifact@v4 + if: always() + with: + name: build-${{inputs.platform}} + retention-days: 7 + path: | + cpp/out/${{inputs.preset}}-build/ + python/arcticdb_ext* + python/**/*.pyd + + # ==================== JOB 2: PYTHON TESTS ==================== + # Note: C++ tests (rapidcheck) are currently disabled on Windows due to a linking issue. + python_tests: + name: Python Tests (${{inputs.platform}}) - ${{matrix.type}} + needs: [compile] + if: | + always() && + !cancelled() + runs-on: ${{inputs.runner}} + strategy: + fail-fast: false + matrix: + type: [unit, integration, hypothesis, stress, compat, enduser] + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} + SCCACHE_GHA_ENABLED: "true" + defaults: + run: + shell: bash -l {0} + steps: + - uses: actions/checkout@v6.0.1 + + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: v0.12.0 + disable_annotations: 'true' + + - name: Download build artifacts + uses: actions/download-artifact@v8 + with: + name: build-${{inputs.platform}} + path: . + + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v2.0.0 + id: cpu-cores + + - name: Install Conda environment from environment-dev.yml + uses: mamba-org/setup-micromamba@v2.0.6 + with: + environment-file: environment-dev.yml + environment-name: arcticdb + init-shell: bash cmd.exe + cache-environment: true + cache-environment-key: ${{inputs.cache_key}} + post-cleanup: 'none' + + - name: Install ArcticDB from artifacts + shell: cmd /C call {0} + run: | + set ARCTIC_CMAKE_PRESET= + if exist "cpp\out\${{inputs.preset}}-build" ( + dir /b python\arcticdb_ext*.pyd >nul 2>&1 + if not errorlevel 1 ( + echo Build artifacts found, skipping CMake build + set ARCTIC_CMAKE_PRESET=skip + ) + ) + python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . + env: + ARCTICDB_USING_CONDA: 1 + + - name: Install npm + uses: actions/setup-node@v6.1.0 + with: + node-version: '24' + + - name: Install Azurite + shell: bash -elo pipefail {0} + run: | + npm install -g azurite + npm_config_prefix=$(npm config get prefix) + npm_bin_dir="$npm_config_prefix" + if [[ "$RUNNER_OS" == "Windows" ]]; then + npm_bin_dir=$(echo "$npm_bin_dir" | sed 's|\\|/|g') + if [[ "$npm_bin_dir" =~ ^([A-Za-z]):/(.*) ]]; then + drive_letter="${BASH_REMATCH[1]}" + rest="${BASH_REMATCH[2]}" + npm_bin_dir="/${drive_letter,,}/${rest}" + fi + fi + npm_bin_dir="${npm_bin_dir}/bin" + echo "$npm_bin_dir" >> $GITHUB_PATH + export PATH="$npm_bin_dir:$PATH" + which azurite && echo "Azurite found: $(which azurite)" || echo "Warning: azurite not found in PATH" + + - name: Check no arcticdb file depend on tests package + shell: bash -elo pipefail {0} + run: build_tooling/checks.sh + + - name: Set persistent storage variables + if: inputs.persistent_storage != 'no' + uses: ./.github/actions/set_persistent_storage_env_vars + with: + aws_access_key: "${{ secrets.AWS_S3_ACCESS_KEY }}" + aws_secret_key: "${{ secrets.AWS_S3_SECRET_KEY }}" + gcp_access_key: "${{ secrets.GCP_S3_ACCESS_KEY }}" + gcp_secret_key: "${{ secrets.GCP_S3_SECRET_KEY }}" + azure_container: "githubblob" + azure_connection_string: "${{ secrets.AZURE_CONNECTION_STRING }}" + persistent_storage: ${{inputs.persistent_storage}} + + - name: Set ArcticDB Debug Logging + if: inputs.run_enable_logging + uses: ./.github/actions/enable_logging + + - name: Setup tmate session + if: inputs.debug_enabled + uses: mxschmitt/action-tmate@v3 + + - name: Install pytest-repeat + shell: bash -elo pipefail {0} + run: python -m pip --retries 3 --timeout 180 install pytest-repeat + + - name: Test with pytest + shell: bash -elo pipefail {0} + run: | + echo "Run commandline: $COMMANDLINE" + eval "$COMMANDLINE" + export ARCTICDB_RAND_SEED=$RANDOM + export ARCTICDB_WARN_ON_WRITING_EMPTY_DATAFRAME=0 + if [[ "$(echo "$ARCTICDB_PYTEST_ARGS" | xargs)" == *pytest* ]]; then + command="python -m $ARCTICDB_PYTEST_ARGS" + echo "Run custom pytest command: $command" + eval "$command" + else + cd python + python -m pytest --timeout=3600 -v --tb=line -n logical --dist worksteal \ + -m "not lmdb" \ + tests/${{matrix.type}} $ARCTICDB_PYTEST_ARGS + fi + env: + ARCTICDB_USING_CONDA: 1 + COMMANDLINE: ${{inputs.run_commandline}} + ARCTICDB_PYTEST_ARGS: ${{inputs.run_custom_pytest_command}} + NODE_OPTIONS: --openssl-legacy-provider \ No newline at end of file From c8089c4794bbbe1eacb78a2545baa9f289011a5d Mon Sep 17 00:00:00 2001 From: Petar Markovski Date: Mon, 23 Mar 2026 08:38:50 +0000 Subject: [PATCH 15/31] fix: handle null boolean inputs for non-dispatch trigger events --- .github/workflows/build_with_conda.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build_with_conda.yml b/.github/workflows/build_with_conda.yml index 0633d284f79..5b701efc83b 100644 --- a/.github/workflows/build_with_conda.yml +++ b/.github/workflows/build_with_conda.yml @@ -35,11 +35,11 @@ jobs: free_disk_space: true install_mongodb: true hypothesis_profile: ci_linux - run_cpp_tests: ${{inputs.run_cpp_tests}} + run_cpp_tests: ${{inputs.run_cpp_tests == null && true || inputs.run_cpp_tests}} cpp_tests_enabled: true persistent_storage: ${{inputs.persistent_storage || 'no'}} - debug_enabled: ${{inputs.debug_enabled || false}} - run_enable_logging: ${{inputs.run_enable_logging || false}} + debug_enabled: ${{inputs.debug_enabled == null && false || inputs.debug_enabled}} + run_enable_logging: ${{inputs.run_enable_logging == null && false || inputs.run_enable_logging}} run_commandline: ${{inputs.run_commandline || ''}} run_custom_pytest_command: ${{inputs.run_custom_pytest_command || ''}} @@ -53,11 +53,11 @@ jobs: cache_key: conda-env-linux_aarch64 free_disk_space: true install_mongodb: true - run_cpp_tests: ${{inputs.run_cpp_tests}} + run_cpp_tests: ${{inputs.run_cpp_tests == null && true || inputs.run_cpp_tests}} cpp_tests_enabled: true persistent_storage: ${{inputs.persistent_storage || 'no'}} - debug_enabled: ${{inputs.debug_enabled || false}} - run_enable_logging: ${{inputs.run_enable_logging || false}} + debug_enabled: ${{inputs.debug_enabled == null && false || inputs.debug_enabled}} + run_enable_logging: ${{inputs.run_enable_logging == null && false || inputs.run_enable_logging}} run_commandline: ${{inputs.run_commandline || ''}} run_custom_pytest_command: ${{inputs.run_custom_pytest_command || ''}} @@ -70,11 +70,11 @@ jobs: preset: macos-conda-release cache_key: conda-env-osx_arm64 hypothesis_profile: ci_macos - run_cpp_tests: ${{inputs.run_cpp_tests}} + run_cpp_tests: ${{inputs.run_cpp_tests == null && true || inputs.run_cpp_tests}} cpp_tests_enabled: true persistent_storage: ${{inputs.persistent_storage || 'no'}} - debug_enabled: ${{inputs.debug_enabled || false}} - run_enable_logging: ${{inputs.run_enable_logging || false}} + debug_enabled: ${{inputs.debug_enabled == null && false || inputs.debug_enabled}} + run_enable_logging: ${{inputs.run_enable_logging == null && false || inputs.run_enable_logging}} run_commandline: ${{inputs.run_commandline || ''}} run_custom_pytest_command: ${{inputs.run_custom_pytest_command || ''}} @@ -86,13 +86,13 @@ jobs: platform: win_64 preset: windows-cl-conda-release cache_key: conda-env-win_64 - run_cpp_tests: ${{inputs.run_cpp_tests}} + run_cpp_tests: ${{inputs.run_cpp_tests == null && true || inputs.run_cpp_tests}} persistent_storage: ${{inputs.persistent_storage || 'no'}} - debug_enabled: ${{inputs.debug_enabled || false}} - run_enable_logging: ${{inputs.run_enable_logging || false}} + debug_enabled: ${{inputs.debug_enabled == null && false || inputs.debug_enabled}} + run_enable_logging: ${{inputs.run_enable_logging == null && false || inputs.run_enable_logging}} run_commandline: ${{inputs.run_commandline || ''}} run_custom_pytest_command: ${{inputs.run_custom_pytest_command || ''}} - + can_merge_conda: name: All conda jobs completed needs: [linux_64, linux_aarch64, osx_arm64, win_64] From a9af48b37aa766023adf05f15d5ed8991403dede Mon Sep 17 00:00:00 2001 From: Petar Markovski Date: Mon, 23 Mar 2026 08:50:00 +0000 Subject: [PATCH 16/31] fix null string handling --- .github/workflows/build_with_conda.yml | 42 +++++++++++++------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build_with_conda.yml b/.github/workflows/build_with_conda.yml index 5b701efc83b..33ad14177f6 100644 --- a/.github/workflows/build_with_conda.yml +++ b/.github/workflows/build_with_conda.yml @@ -35,13 +35,13 @@ jobs: free_disk_space: true install_mongodb: true hypothesis_profile: ci_linux - run_cpp_tests: ${{inputs.run_cpp_tests == null && true || inputs.run_cpp_tests}} + run_cpp_tests: ${{inputs.run_cpp_tests != false}} cpp_tests_enabled: true persistent_storage: ${{inputs.persistent_storage || 'no'}} - debug_enabled: ${{inputs.debug_enabled == null && false || inputs.debug_enabled}} - run_enable_logging: ${{inputs.run_enable_logging == null && false || inputs.run_enable_logging}} - run_commandline: ${{inputs.run_commandline || ''}} - run_custom_pytest_command: ${{inputs.run_custom_pytest_command || ''}} + debug_enabled: ${{inputs.debug_enabled == true}} + run_enable_logging: ${{inputs.run_enable_logging == true}} + run_commandline: ${{format('{0}', inputs.run_commandline)}} + run_custom_pytest_command: ${{format('{0}', inputs.run_custom_pytest_command)}} linux_aarch64: uses: ./.github/workflows/compile_and_test_with_conda_unix.yml @@ -53,13 +53,13 @@ jobs: cache_key: conda-env-linux_aarch64 free_disk_space: true install_mongodb: true - run_cpp_tests: ${{inputs.run_cpp_tests == null && true || inputs.run_cpp_tests}} + run_cpp_tests: ${{inputs.run_cpp_tests != false}} cpp_tests_enabled: true persistent_storage: ${{inputs.persistent_storage || 'no'}} - debug_enabled: ${{inputs.debug_enabled == null && false || inputs.debug_enabled}} - run_enable_logging: ${{inputs.run_enable_logging == null && false || inputs.run_enable_logging}} - run_commandline: ${{inputs.run_commandline || ''}} - run_custom_pytest_command: ${{inputs.run_custom_pytest_command || ''}} + debug_enabled: ${{inputs.debug_enabled == true}} + run_enable_logging: ${{inputs.run_enable_logging == true}} + run_commandline: ${{format('{0}', inputs.run_commandline)}} + run_custom_pytest_command: ${{format('{0}', inputs.run_custom_pytest_command)}} osx_arm64: uses: ./.github/workflows/compile_and_test_with_conda_unix.yml @@ -70,13 +70,13 @@ jobs: preset: macos-conda-release cache_key: conda-env-osx_arm64 hypothesis_profile: ci_macos - run_cpp_tests: ${{inputs.run_cpp_tests == null && true || inputs.run_cpp_tests}} + run_cpp_tests: ${{inputs.run_cpp_tests != false}} cpp_tests_enabled: true persistent_storage: ${{inputs.persistent_storage || 'no'}} - debug_enabled: ${{inputs.debug_enabled == null && false || inputs.debug_enabled}} - run_enable_logging: ${{inputs.run_enable_logging == null && false || inputs.run_enable_logging}} - run_commandline: ${{inputs.run_commandline || ''}} - run_custom_pytest_command: ${{inputs.run_custom_pytest_command || ''}} + debug_enabled: ${{inputs.debug_enabled == true}} + run_enable_logging: ${{inputs.run_enable_logging == true}} + run_commandline: ${{format('{0}', inputs.run_commandline)}} + run_custom_pytest_command: ${{format('{0}', inputs.run_custom_pytest_command)}} win_64: uses: ./.github/workflows/compile_and_test_with_conda_win.yml @@ -86,13 +86,13 @@ jobs: platform: win_64 preset: windows-cl-conda-release cache_key: conda-env-win_64 - run_cpp_tests: ${{inputs.run_cpp_tests == null && true || inputs.run_cpp_tests}} + run_cpp_tests: ${{inputs.run_cpp_tests != false}} persistent_storage: ${{inputs.persistent_storage || 'no'}} - debug_enabled: ${{inputs.debug_enabled == null && false || inputs.debug_enabled}} - run_enable_logging: ${{inputs.run_enable_logging == null && false || inputs.run_enable_logging}} - run_commandline: ${{inputs.run_commandline || ''}} - run_custom_pytest_command: ${{inputs.run_custom_pytest_command || ''}} - + debug_enabled: ${{inputs.debug_enabled == true}} + run_enable_logging: ${{inputs.run_enable_logging == true}} + run_commandline: ${{format('{0}', inputs.run_commandline)}} + run_custom_pytest_command: ${{format('{0}', inputs.run_custom_pytest_command)}} + can_merge_conda: name: All conda jobs completed needs: [linux_64, linux_aarch64, osx_arm64, win_64] From 9c4cacacbd327c52511d82e5e42b0493d036f684 Mon Sep 17 00:00:00 2001 From: Petar Markovski Date: Mon, 23 Mar 2026 09:09:42 +0000 Subject: [PATCH 17/31] remove ' || false ' from run_cpp_tests --- .github/workflows/build_with_conda.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_with_conda.yml b/.github/workflows/build_with_conda.yml index 33ad14177f6..b27af2bf957 100644 --- a/.github/workflows/build_with_conda.yml +++ b/.github/workflows/build_with_conda.yml @@ -53,7 +53,7 @@ jobs: cache_key: conda-env-linux_aarch64 free_disk_space: true install_mongodb: true - run_cpp_tests: ${{inputs.run_cpp_tests != false}} + run_cpp_tests: ${{inputs.run_cpp_tests}} cpp_tests_enabled: true persistent_storage: ${{inputs.persistent_storage || 'no'}} debug_enabled: ${{inputs.debug_enabled == true}} @@ -70,7 +70,7 @@ jobs: preset: macos-conda-release cache_key: conda-env-osx_arm64 hypothesis_profile: ci_macos - run_cpp_tests: ${{inputs.run_cpp_tests != false}} + run_cpp_tests: ${{inputs.run_cpp_tests}} cpp_tests_enabled: true persistent_storage: ${{inputs.persistent_storage || 'no'}} debug_enabled: ${{inputs.debug_enabled == true}} @@ -86,7 +86,7 @@ jobs: platform: win_64 preset: windows-cl-conda-release cache_key: conda-env-win_64 - run_cpp_tests: ${{inputs.run_cpp_tests != false}} + run_cpp_tests: ${{inputs.run_cpp_tests}} persistent_storage: ${{inputs.persistent_storage || 'no'}} debug_enabled: ${{inputs.debug_enabled == true}} run_enable_logging: ${{inputs.run_enable_logging == true}} From 09f19496d05f75fb000e676fa5a881ace34c19eb Mon Sep 17 00:00:00 2001 From: Petar Markovski Date: Mon, 23 Mar 2026 09:18:45 +0000 Subject: [PATCH 18/31] add " != false " to run cpp tests --- .github/workflows/build_with_conda.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_with_conda.yml b/.github/workflows/build_with_conda.yml index b27af2bf957..33ad14177f6 100644 --- a/.github/workflows/build_with_conda.yml +++ b/.github/workflows/build_with_conda.yml @@ -53,7 +53,7 @@ jobs: cache_key: conda-env-linux_aarch64 free_disk_space: true install_mongodb: true - run_cpp_tests: ${{inputs.run_cpp_tests}} + run_cpp_tests: ${{inputs.run_cpp_tests != false}} cpp_tests_enabled: true persistent_storage: ${{inputs.persistent_storage || 'no'}} debug_enabled: ${{inputs.debug_enabled == true}} @@ -70,7 +70,7 @@ jobs: preset: macos-conda-release cache_key: conda-env-osx_arm64 hypothesis_profile: ci_macos - run_cpp_tests: ${{inputs.run_cpp_tests}} + run_cpp_tests: ${{inputs.run_cpp_tests != false}} cpp_tests_enabled: true persistent_storage: ${{inputs.persistent_storage || 'no'}} debug_enabled: ${{inputs.debug_enabled == true}} @@ -86,7 +86,7 @@ jobs: platform: win_64 preset: windows-cl-conda-release cache_key: conda-env-win_64 - run_cpp_tests: ${{inputs.run_cpp_tests}} + run_cpp_tests: ${{inputs.run_cpp_tests != false}} persistent_storage: ${{inputs.persistent_storage || 'no'}} debug_enabled: ${{inputs.debug_enabled == true}} run_enable_logging: ${{inputs.run_enable_logging == true}} From 00c2c88357ad19c9a95bbf9ccee803a3cb96db05 Mon Sep 17 00:00:00 2001 From: Petar Markovski Date: Mon, 23 Mar 2026 09:28:31 +0000 Subject: [PATCH 19/31] add debug log for investigating why cpp tests are skipped --- .github/workflows/compile_and_test_with_conda_unix.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/compile_and_test_with_conda_unix.yml b/.github/workflows/compile_and_test_with_conda_unix.yml index 821945ea821..52312f339a0 100644 --- a/.github/workflows/compile_and_test_with_conda_unix.yml +++ b/.github/workflows/compile_and_test_with_conda_unix.yml @@ -36,6 +36,11 @@ jobs: run: shell: bash -l {0} steps: + - name: Debug inputs + run: | + echo "run_cpp_tests=${{inputs.run_cpp_tests}}" + echo "cpp_tests_enabled=${{inputs.cpp_tests_enabled}}" + - uses: actions/checkout@v6.0.1 - name: Configure sccache From a7492971265fc270721fb5150466628745571e94 Mon Sep 17 00:00:00 2001 From: Petar Markovski Date: Mon, 23 Mar 2026 09:47:23 +0000 Subject: [PATCH 20/31] rename run_cpp_test to disable_cpp_tests --- .github/workflows/build_with_conda.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_with_conda.yml b/.github/workflows/build_with_conda.yml index 33ad14177f6..d1800a425fe 100644 --- a/.github/workflows/build_with_conda.yml +++ b/.github/workflows/build_with_conda.yml @@ -13,10 +13,12 @@ on: push: branches: - master + pull_request: + workflow_dispatch: inputs: - run_cpp_tests: {required: true, type: boolean, default: true, description: Run C++ tests} + disable_cpp_tests: {required: false, type: boolean, default: false, description: Disable C++ tests} persistent_storage: {required: false, type: choice, default: 'no', description: Persistent storage type, options: ['no', 'AWS_S3', 'GCPXML', 'AZURE']} debug_enabled: {required: false, type: boolean, default: false, description: Run the build with debugging enabled} run_enable_logging: {required: false, type: boolean, default: false, description: Enable debug logging} @@ -35,7 +37,7 @@ jobs: free_disk_space: true install_mongodb: true hypothesis_profile: ci_linux - run_cpp_tests: ${{inputs.run_cpp_tests != false}} + run_cpp_tests: ${{inputs.disable_cpp_tests != true}} cpp_tests_enabled: true persistent_storage: ${{inputs.persistent_storage || 'no'}} debug_enabled: ${{inputs.debug_enabled == true}} @@ -53,7 +55,7 @@ jobs: cache_key: conda-env-linux_aarch64 free_disk_space: true install_mongodb: true - run_cpp_tests: ${{inputs.run_cpp_tests != false}} + run_cpp_tests: ${{inputs.disable_cpp_tests != true}} cpp_tests_enabled: true persistent_storage: ${{inputs.persistent_storage || 'no'}} debug_enabled: ${{inputs.debug_enabled == true}} @@ -70,7 +72,7 @@ jobs: preset: macos-conda-release cache_key: conda-env-osx_arm64 hypothesis_profile: ci_macos - run_cpp_tests: ${{inputs.run_cpp_tests != false}} + run_cpp_tests: ${{inputs.disable_cpp_tests != true}} cpp_tests_enabled: true persistent_storage: ${{inputs.persistent_storage || 'no'}} debug_enabled: ${{inputs.debug_enabled == true}} @@ -86,7 +88,7 @@ jobs: platform: win_64 preset: windows-cl-conda-release cache_key: conda-env-win_64 - run_cpp_tests: ${{inputs.run_cpp_tests != false}} + run_cpp_tests: ${{inputs.disable_cpp_tests != true}} persistent_storage: ${{inputs.persistent_storage || 'no'}} debug_enabled: ${{inputs.debug_enabled == true}} run_enable_logging: ${{inputs.run_enable_logging == true}} From 185d094d8af20203118e7dbbafe0e2512321eb09 Mon Sep 17 00:00:00 2001 From: "Petar Markovski (Sofia)" Date: Tue, 24 Mar 2026 15:09:10 +0000 Subject: [PATCH 21/31] Refactor workflow parameters --- .github/workflows/build_with_conda.yml | 13 ++----------- .../workflows/compile_and_test_with_conda_unix.yml | 11 +++++------ 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build_with_conda.yml b/.github/workflows/build_with_conda.yml index d1800a425fe..dc77127e6b1 100644 --- a/.github/workflows/build_with_conda.yml +++ b/.github/workflows/build_with_conda.yml @@ -33,12 +33,8 @@ jobs: runner: ubuntu-22.04 platform: linux_64 preset: linux-conda-release - cache_key: conda-env-linux_64 - free_disk_space: true - install_mongodb: true hypothesis_profile: ci_linux run_cpp_tests: ${{inputs.disable_cpp_tests != true}} - cpp_tests_enabled: true persistent_storage: ${{inputs.persistent_storage || 'no'}} debug_enabled: ${{inputs.debug_enabled == true}} run_enable_logging: ${{inputs.run_enable_logging == true}} @@ -52,11 +48,7 @@ jobs: runner: ubuntu-22.04-arm platform: linux_aarch64 preset: linux-conda-release - cache_key: conda-env-linux_aarch64 - free_disk_space: true - install_mongodb: true run_cpp_tests: ${{inputs.disable_cpp_tests != true}} - cpp_tests_enabled: true persistent_storage: ${{inputs.persistent_storage || 'no'}} debug_enabled: ${{inputs.debug_enabled == true}} run_enable_logging: ${{inputs.run_enable_logging == true}} @@ -70,10 +62,10 @@ jobs: runner: macos-14 platform: osx_arm64 preset: macos-conda-release - cache_key: conda-env-osx_arm64 hypothesis_profile: ci_macos + install_mongodb: false # MongoDB is not available for macOS arm64 + free_disk_space: false # Do not clean on macOS run_cpp_tests: ${{inputs.disable_cpp_tests != true}} - cpp_tests_enabled: true persistent_storage: ${{inputs.persistent_storage || 'no'}} debug_enabled: ${{inputs.debug_enabled == true}} run_enable_logging: ${{inputs.run_enable_logging == true}} @@ -87,7 +79,6 @@ jobs: runner: windows-latest platform: win_64 preset: windows-cl-conda-release - cache_key: conda-env-win_64 run_cpp_tests: ${{inputs.disable_cpp_tests != true}} persistent_storage: ${{inputs.persistent_storage || 'no'}} debug_enabled: ${{inputs.debug_enabled == true}} diff --git a/.github/workflows/compile_and_test_with_conda_unix.yml b/.github/workflows/compile_and_test_with_conda_unix.yml index 52312f339a0..0b7ce0d89b9 100644 --- a/.github/workflows/compile_and_test_with_conda_unix.yml +++ b/.github/workflows/compile_and_test_with_conda_unix.yml @@ -5,10 +5,9 @@ on: runner: {required: true, type: string, description: GitHub runner to execute on} platform: {required: true, type: string, description: Platform identifier e.g. linux_64 or osx_arm64} preset: {required: true, type: string, description: CMake preset name e.g. linux-conda-release} - cache_key: {required: true, type: string, description: Unique conda environment cache key per platform} - free_disk_space: {required: false, type: boolean, default: false, description: Free disk space before build. Linux only} - install_mongodb: {required: false, type: boolean, default: false, description: Install MongoDB for tests. Linux only} + free_disk_space: {required: false, type: boolean, default: true, description: Free disk space before build. Linux only} + install_mongodb: {required: false, type: boolean, default: true, description: Install MongoDB for tests. Linux only} run_cpp_tests: {required: false, type: boolean, default: true, description: Whether to run C++ tests} cpp_tests_enabled: {required: false, type: boolean, default: true, description: Whether C++ tests are supported on this platform} hypothesis_profile: {required: false, type: string, default: '', description: Hypothesis profile e.g. ci_linux or ci_macos} @@ -60,7 +59,7 @@ jobs: environment-name: arcticdb init-shell: bash cache-environment: true - cache-environment-key: ${{inputs.cache_key}} + cache-environment-key: conda-env-${{inputs.platform}} post-cleanup: 'none' - name: Build ArcticDB with conda @@ -135,7 +134,7 @@ jobs: environment-name: arcticdb init-shell: bash cache-environment: true - cache-environment-key: ${{inputs.cache_key}} + cache-environment-key: conda-env-${{inputs.platform}} post-cleanup: 'none' - name: Configure C++ Tests @@ -221,7 +220,7 @@ jobs: environment-name: arcticdb init-shell: bash cache-environment: true - cache-environment-key: ${{inputs.cache_key}} + cache-environment-key: conda-env-${{inputs.platform}} post-cleanup: 'none' - name: Install ArcticDB from artifacts From d9db5235d2967ee3fc7c26c0356cd184cec3b47c Mon Sep 17 00:00:00 2001 From: "Petar Markovski (Sofia)" Date: Tue, 24 Mar 2026 15:51:05 +0000 Subject: [PATCH 22/31] Remove unused code --- .github/workflows/compile_and_test_with_conda_unix.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/compile_and_test_with_conda_unix.yml b/.github/workflows/compile_and_test_with_conda_unix.yml index 0b7ce0d89b9..d5789e2fc1f 100644 --- a/.github/workflows/compile_and_test_with_conda_unix.yml +++ b/.github/workflows/compile_and_test_with_conda_unix.yml @@ -9,7 +9,6 @@ on: free_disk_space: {required: false, type: boolean, default: true, description: Free disk space before build. Linux only} install_mongodb: {required: false, type: boolean, default: true, description: Install MongoDB for tests. Linux only} run_cpp_tests: {required: false, type: boolean, default: true, description: Whether to run C++ tests} - cpp_tests_enabled: {required: false, type: boolean, default: true, description: Whether C++ tests are supported on this platform} hypothesis_profile: {required: false, type: string, default: '', description: Hypothesis profile e.g. ci_linux or ci_macos} persistent_storage: {required: false, type: string, default: 'no', description: Persistent storage type} @@ -35,11 +34,6 @@ jobs: run: shell: bash -l {0} steps: - - name: Debug inputs - run: | - echo "run_cpp_tests=${{inputs.run_cpp_tests}}" - echo "cpp_tests_enabled=${{inputs.cpp_tests_enabled}}" - - uses: actions/checkout@v6.0.1 - name: Configure sccache @@ -90,8 +84,7 @@ jobs: if: | always() && !cancelled() && - inputs.run_cpp_tests && - inputs.cpp_tests_enabled + inputs.run_cpp_tests runs-on: ${{inputs.runner}} env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true From 9eb26a173bdca2e5b983ab81c99fd0b5b880c176 Mon Sep 17 00:00:00 2001 From: "Petar Markovski (Sofia)" Date: Tue, 24 Mar 2026 16:12:12 +0000 Subject: [PATCH 23/31] Fix Win 'cache-environment-key' value --- .../compile_and_test_with_conda_win.yml | 6 +- build.txt | 871 ++++++++++++++++++ python/test.py | 1 + .../integration/toolbox/test_storage_mover.py | 24 +- .../test_query_builder_sparse.py | 82 +- .../arcticdb/version_store/test_ternary.py | 148 +-- setup.py | 2 +- 7 files changed, 1002 insertions(+), 132 deletions(-) create mode 100644 build.txt create mode 100644 python/test.py diff --git a/.github/workflows/compile_and_test_with_conda_win.yml b/.github/workflows/compile_and_test_with_conda_win.yml index b3d1a8d0370..1f4907618f6 100644 --- a/.github/workflows/compile_and_test_with_conda_win.yml +++ b/.github/workflows/compile_and_test_with_conda_win.yml @@ -5,10 +5,8 @@ on: runner: {required: true, type: string, description: GitHub runner to execute on} platform: {required: true, type: string, description: Platform identifier e.g. win_64} preset: {required: true, type: string, description: CMake preset name e.g. windows-cl-conda-release} - cache_key: {required: true, type: string, description: Unique conda environment cache key per platform} run_cpp_tests: {required: false, type: boolean, default: true, description: Whether to run C++ tests} - persistent_storage: {required: false, type: string, default: 'no', description: Persistent storage type} debug_enabled: {required: false, type: boolean, default: false, description: Enable tmate debug session} run_enable_logging: {required: false, type: boolean, default: false, description: Enable ArcticDB debug logging} @@ -51,7 +49,7 @@ jobs: environment-name: arcticdb init-shell: bash cmd.exe cache-environment: true - cache-environment-key: ${{inputs.cache_key}} + cache-environment-key: conda-env-${{inputs.platform}} post-cleanup: 'none' - name: Build ArcticDB with conda @@ -132,7 +130,7 @@ jobs: environment-name: arcticdb init-shell: bash cmd.exe cache-environment: true - cache-environment-key: ${{inputs.cache_key}} + cache-environment-key: conda-env-${{inputs.platform}} post-cleanup: 'none' - name: Install ArcticDB from artifacts diff --git a/build.txt b/build.txt new file mode 100644 index 00000000000..078fa3768c1 --- /dev/null +++ b/build.txt @@ -0,0 +1,871 @@ +Using pip 24.2 from /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages/pip (python 3.11) +Looking in indexes: https://repo.prod.m/artifactory/api/pypi/pegasus-311-1/simple/ +Obtaining file:///turbo/pmarkovski/ArcticDB + Checking if build backend supports build_editable: started + Running command Checking if build backend supports build_editable + Checking if build backend supports build_editable: finished with status 'done' + Preparing metadata (pyproject.toml): started + Running command Preparing metadata (pyproject.toml) + ARCTICDB_USING_CONDA=False + ARCTICDB_BUILD_CPP_TESTS=False + running dist_info + creating /tmp/pip-modern-metadata-7cse9pu8/arcticdb.egg-info + writing /tmp/pip-modern-metadata-7cse9pu8/arcticdb.egg-info/PKG-INFO + writing dependency_links to /tmp/pip-modern-metadata-7cse9pu8/arcticdb.egg-info/dependency_links.txt + writing entry points to /tmp/pip-modern-metadata-7cse9pu8/arcticdb.egg-info/entry_points.txt + writing requirements to /tmp/pip-modern-metadata-7cse9pu8/arcticdb.egg-info/requires.txt + writing top-level names to /tmp/pip-modern-metadata-7cse9pu8/arcticdb.egg-info/top_level.txt + writing manifest file '/tmp/pip-modern-metadata-7cse9pu8/arcticdb.egg-info/SOURCES.txt' + adding license file 'LICENSE.txt' + adding license file 'NOTICE.txt' + writing manifest file '/tmp/pip-modern-metadata-7cse9pu8/arcticdb.egg-info/SOURCES.txt' + creating '/tmp/pip-modern-metadata-7cse9pu8/arcticdb-0.0.0.dev0.dist-info' + Preparing metadata (pyproject.toml): finished with status 'done' +Requirement already satisfied: pandas<3 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (2.1.4+man91) +Requirement already satisfied: numpy in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (1.24.4) +Requirement already satisfied: attrs in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (23.1.0) +Requirement already satisfied: protobuf<7,>=3.5.0.post1 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (4.25.1) +Requirement already satisfied: msgpack>=0.5.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (1.0.5) +Requirement already satisfied: pyyaml in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (6.0.1) +Requirement already satisfied: packaging in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (24.2) +Requirement already satisfied: pytz in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (2022.1) +Requirement already satisfied: pytest in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (7.4.4) +Requirement already satisfied: polars in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (1.35.2) +Requirement already satisfied: pyarrow in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (20.0.0) +Requirement already satisfied: pytest-cpp in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (0.3.1) +Requirement already satisfied: pytest-timeout in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (2.4.0) +Requirement already satisfied: pytest-xdist in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (3.5.0) +Requirement already satisfied: future in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (0.18.2) +Requirement already satisfied: mock in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (5.0.2) +Requirement already satisfied: boto3 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (1.28.34) +Requirement already satisfied: moto in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (4.2.0) +Requirement already satisfied: flask in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (2.3.3) +Requirement already satisfied: flask-cors in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (4.0.1) +Requirement already satisfied: hypothesis<6.73 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (4.39.3+ahl1) +Requirement already satisfied: azure-storage-blob in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (12.13.1) +Requirement already satisfied: azure-identity in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (1.10.0) +Requirement already satisfied: coverage in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (7.2.7) +Requirement already satisfied: virtualenv in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (20.27.1) +Requirement already satisfied: pymongo in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (3.13.0+man2) +Requirement already satisfied: trustme in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (1.1.0) +Requirement already satisfied: psutil in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (5.9.8) +Requirement already satisfied: retrying in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (1.4.1) +Requirement already satisfied: asv in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (0.6.2) +Requirement already satisfied: python-dateutil>=2.8.2 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from pandas<3->arcticdb==0.0.0.dev0) (2.8.2+ahl1) +Requirement already satisfied: tzdata>=2022.1 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from pandas<3->arcticdb==0.0.0.dev0) (2024.1) +Requirement already satisfied: asv-runner>=v0.2.1 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from asv->arcticdb==0.0.0.dev0) (0.2.1) +Requirement already satisfied: json5 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from asv->arcticdb==0.0.0.dev0) (0.9.28) +Requirement already satisfied: tabulate in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from asv->arcticdb==0.0.0.dev0) (0.9.0) +Requirement already satisfied: tomli in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from asv->arcticdb==0.0.0.dev0) (1.2.3) +Requirement already satisfied: pympler in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from asv->arcticdb==0.0.0.dev0) (1.1) +Requirement already satisfied: azure-core<2.0.0,>=1.11.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from azure-identity->arcticdb==0.0.0.dev0) (1.24.2) +Requirement already satisfied: cryptography>=2.5 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from azure-identity->arcticdb==0.0.0.dev0) (42.0.0) +Requirement already satisfied: msal<2.0.0,>=1.12.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from azure-identity->arcticdb==0.0.0.dev0) (1.28.1) +Requirement already satisfied: msal-extensions<2.0.0,>=0.3.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from azure-identity->arcticdb==0.0.0.dev0) (1.0.0) +Requirement already satisfied: six>=1.12.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from azure-identity->arcticdb==0.0.0.dev0) (1.16.0) +Requirement already satisfied: msrest>=0.6.21 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from azure-storage-blob->arcticdb==0.0.0.dev0) (0.7.1) +Requirement already satisfied: botocore<1.32.0,>=1.31.34 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from boto3->arcticdb==0.0.0.dev0) (1.31.34) +Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from boto3->arcticdb==0.0.0.dev0) (1.0.1) +Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from boto3->arcticdb==0.0.0.dev0) (0.6.2) +Requirement already satisfied: Werkzeug>=2.3.7 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from flask->arcticdb==0.0.0.dev0) (2.3.8) +Requirement already satisfied: Jinja2>=3.1.2 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from flask->arcticdb==0.0.0.dev0) (3.1.4) +Requirement already satisfied: itsdangerous>=2.1.2 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from flask->arcticdb==0.0.0.dev0) (2.1.2) +Requirement already satisfied: click>=8.1.3 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from flask->arcticdb==0.0.0.dev0) (8.1.7) +Requirement already satisfied: blinker>=1.6.2 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from flask->arcticdb==0.0.0.dev0) (1.7.0) +Requirement already satisfied: requests>=2.5 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from moto->arcticdb==0.0.0.dev0) (2.31.0) +Requirement already satisfied: xmltodict in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from moto->arcticdb==0.0.0.dev0) (0.14.2) +Requirement already satisfied: responses>=0.13.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from moto->arcticdb==0.0.0.dev0) (0.23.3) +Requirement already satisfied: polars-runtime-32==1.35.2 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from polars->arcticdb==0.0.0.dev0) (1.35.2) +Requirement already satisfied: iniconfig in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from pytest->arcticdb==0.0.0.dev0) (2.0.0) +Requirement already satisfied: pluggy<2.0,>=0.12 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from pytest->arcticdb==0.0.0.dev0) (1.2.0) +Requirement already satisfied: colorama in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from pytest-cpp->arcticdb==0.0.0.dev0) (0.4.6) +Requirement already satisfied: execnet>=1.1 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from pytest-xdist->arcticdb==0.0.0.dev0) (1.9.0) +Requirement already satisfied: idna>=2.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from trustme->arcticdb==0.0.0.dev0) (3.4) +Requirement already satisfied: distlib<1,>=0.3.7 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from virtualenv->arcticdb==0.0.0.dev0) (0.3.9) +Requirement already satisfied: filelock<4,>=3.12.2 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from virtualenv->arcticdb==0.0.0.dev0) (3.16.1) +Requirement already satisfied: platformdirs<5,>=3.9.1 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from virtualenv->arcticdb==0.0.0.dev0) (4.5.0) +Requirement already satisfied: importlib-metadata in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from asv-runner>=v0.2.1->asv->arcticdb==0.0.0.dev0) (6.11.0) +Requirement already satisfied: typing-extensions>=4.0.1 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from azure-core<2.0.0,>=1.11.0->azure-identity->arcticdb==0.0.0.dev0) (4.15.0) +Requirement already satisfied: urllib3<1.27,>=1.25.4 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from botocore<1.32.0,>=1.31.34->boto3->arcticdb==0.0.0.dev0) (1.26.18) +Requirement already satisfied: cffi>=1.12 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from cryptography>=2.5->azure-identity->arcticdb==0.0.0.dev0) (1.15.1) +Requirement already satisfied: MarkupSafe>=2.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from Jinja2>=3.1.2->flask->arcticdb==0.0.0.dev0) (2.1.5) +Requirement already satisfied: PyJWT<3,>=1.0.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from PyJWT[crypto]<3,>=1.0.0->msal<2.0.0,>=1.12.0->azure-identity->arcticdb==0.0.0.dev0) (2.9.0) +Requirement already satisfied: portalocker<3,>=1.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from msal-extensions<2.0.0,>=0.3.0->azure-identity->arcticdb==0.0.0.dev0) (2.7.0) +Requirement already satisfied: certifi>=2017.4.17 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from msrest>=0.6.21->azure-storage-blob->arcticdb==0.0.0.dev0) (2023.7.22) +Requirement already satisfied: isodate>=0.6.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from msrest>=0.6.21->azure-storage-blob->arcticdb==0.0.0.dev0) (0.7.2) +Requirement already satisfied: requests-oauthlib>=0.5.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from msrest>=0.6.21->azure-storage-blob->arcticdb==0.0.0.dev0) (1.3.1) +Requirement already satisfied: charset-normalizer<4,>=2 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from requests>=2.5->moto->arcticdb==0.0.0.dev0) (3.2.0) +Requirement already satisfied: types-PyYAML in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from responses>=0.13.0->moto->arcticdb==0.0.0.dev0) (6.0.12.10) +Requirement already satisfied: pycparser in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from cffi>=1.12->cryptography>=2.5->azure-identity->arcticdb==0.0.0.dev0) (2.21) +Requirement already satisfied: oauthlib>=3.0.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from requests-oauthlib>=0.5.0->msrest>=0.6.21->azure-storage-blob->arcticdb==0.0.0.dev0) (3.0.1) +Requirement already satisfied: zipp>=0.5 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from importlib-metadata->asv-runner>=v0.2.1->asv->arcticdb==0.0.0.dev0) (3.5.1) +Installing collected packages: arcticdb + Attempting uninstall: arcticdb + Found existing installation: arcticdb 0.0.0.dev0 + Uninstalling arcticdb-0.0.0.dev0: + Removing file or directory /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/bin/arcticdb_update_storage + Removing file or directory /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages/arcticdb.egg-link + Removing pth entries from /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages/easy-install.pth: + Removing entry: /turbo/pmarkovski/ArcticDB/python + Successfully uninstalled arcticdb-0.0.0.dev0 + DEPRECATION: Legacy editable install of arcticdb[testing]==0.0.0.dev0 from file:///turbo/pmarkovski/ArcticDB (setup.py develop) is deprecated. pip 25.0 will enforce this behaviour change. A possible replacement is to add a pyproject.toml or enable --use-pep517, and use setuptools >= 64. If the resulting installation is not behaving as expected, try using --config-settings editable_mode=compat. Please consult the setuptools documentation for more information. Discussion can be found at https://github.com/pypa/pip/issues/11457 + Running setup.py develop for arcticdb + Running command python setup.py develop + ARCTICDB_USING_CONDA=False + ARCTICDB_BUILD_CPP_TESTS=False + running develop + /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages/setuptools/command/easy_install.py:144: EasyInstallDeprecationWarning: easy_install command is deprecated. Use build and pip and other standards-based tools. + warnings.warn( + /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages/setuptools/command/install.py:34: SetuptoolsDeprecationWarning: setup.py install is deprecated. Use build and pip and other standards-based tools. + warnings.warn( + running egg_info + writing python/arcticdb.egg-info/PKG-INFO + writing dependency_links to python/arcticdb.egg-info/dependency_links.txt + writing entry points to python/arcticdb.egg-info/entry_points.txt + writing requirements to python/arcticdb.egg-info/requires.txt + writing top-level names to python/arcticdb.egg-info/top_level.txt + adding license file 'LICENSE.txt' + adding license file 'NOTICE.txt' + writing manifest file 'python/arcticdb.egg-info/SOURCES.txt' + running build_ext + Preset CMake variables: + + CMAKE_BUILD_TYPE="RelWithDebInfo" + CMAKE_MAKE_PROGRAM="make" + CMAKE_TOOLCHAIN_FILE:FILEPATH="vcpkg/scripts/buildsystems/vcpkg.cmake" + + Preset environment variables: + + cmakepreset_expected_host_system="Linux" + + -- Running vcpkg install + A suitable version of cmake was not found (required v3.31.10). + Downloading https://github.com/Kitware/CMake/releases/download/v3.31.10/cmake-3.31.10-linux-x86_64.tar.gz -> cmake-3.31.10-linux-x86_64.tar.gz + Successfully downloaded cmake-3.31.10-linux-x86_64.tar.gz + Extracting cmake... + Detecting compiler hash for triplet x64-linux... + Compiler found: /usr/bin/c++ + The following packages are already installed: + * abseil:x64-linux@20250814.1 + arcticdb-sparrow:x64-linux@2.3.0 + arrow:x64-linux@21.0.0#2 + aws-c-auth:x64-linux@0.7.31 + aws-c-cal:x64-linux@0.7.4 + aws-c-common:x64-linux@0.9.28 + * aws-c-compression:x64-linux@0.3.2 + aws-c-event-stream:x64-linux@0.4.3 + aws-c-http:x64-linux@0.8.10 + aws-c-io:x64-linux@0.14.18 + aws-c-mqtt:x64-linux@0.11.0 + aws-c-s3:x64-linux@0.6.6 + aws-c-sdkutils:x64-linux@0.1.19 + aws-checksums:x64-linux@0.1.20 + aws-crt-cpp:x64-linux@0.29.7 + aws-sdk-cpp[cognito-identity,core,identity-management,s3,sts]:x64-linux@1.11.474 -- /turbo/pmarkovski/ArcticDB/cpp/third_party/vcpkg_overlays/aws-sdk-cpp + azure-core-cpp[core,curl]:x64-linux@1.12.0 + azure-identity-cpp:x64-linux@1.6.0 + azure-storage-blobs-cpp:x64-linux@12.16.0 + * azure-storage-common-cpp:x64-linux@12.12.0 + benchmark:x64-linux@1.9.0 + bitmagic:x64-linux@7.13.4 -- /turbo/pmarkovski/ArcticDB/cpp/third_party/vcpkg_overlays/bitmagic + * boost-algorithm:x64-linux@1.90.0#1 + * boost-align:x64-linux@1.90.0#1 + * boost-any:x64-linux@1.90.0#1 + * boost-array:x64-linux@1.90.0#1 + * boost-assert:x64-linux@1.90.0#1 + * boost-atomic:x64-linux@1.90.0#1 + * boost-bind:x64-linux@1.90.0#1 + boost-callable-traits:x64-linux@1.90.0#1 + * boost-charconv:x64-linux@1.90.0#1 + * boost-chrono:x64-linux@1.90.0#1 + boost-circular-buffer:x64-linux@1.90.0#1 + * boost-cmake:x64-linux@1.90.0#1 + * boost-concept-check:x64-linux@1.90.0#1 + * boost-config:x64-linux@1.90.0#1 + * boost-container:x64-linux@1.90.0#1 + * boost-container-hash:x64-linux@1.90.0#1 + * boost-context:x64-linux@1.90.0#1 + * boost-conversion:x64-linux@1.90.0#1 + * boost-core:x64-linux@1.90.0#1 + * boost-crc:x64-linux@1.90.0#1 + * boost-date-time:x64-linux@1.90.0#1 + * boost-describe:x64-linux@1.90.0#1 + * boost-detail:x64-linux@1.90.0#1 + boost-dynamic-bitset:x64-linux@1.90.0#1 + * boost-exception:x64-linux@1.90.0#1 + * boost-filesystem:x64-linux@1.90.0#1 + * boost-function:x64-linux@1.90.0#1 + * boost-function-types:x64-linux@1.90.0#1 + * boost-functional:x64-linux@1.90.0#1 + * boost-fusion:x64-linux@1.90.0#1 + * boost-headers:x64-linux@1.90.0#1 + * boost-integer:x64-linux@1.90.0#1 + boost-interprocess:x64-linux@1.90.0#1 + * boost-intrusive:x64-linux@1.90.0#1 + * boost-io:x64-linux@1.90.0#1 + * boost-iterator:x64-linux@1.90.0#1 + * boost-lexical-cast:x64-linux@1.90.0#1 + * boost-locale:x64-linux@1.90.0#1 + * boost-math:x64-linux@1.90.0#1 + * boost-move:x64-linux@1.90.0#1 + * boost-mp11:x64-linux@1.90.0#1 + * boost-mpl:x64-linux@1.90.0#1 + * boost-multi-index:x64-linux@1.90.0#1 + boost-multiprecision:x64-linux@1.90.0#1 + * boost-numeric-conversion:x64-linux@1.90.0#1 + boost-optional:x64-linux@1.90.0#1 + * boost-pool:x64-linux@1.90.0#1 + * boost-predef:x64-linux@1.90.0#1 + * boost-preprocessor:x64-linux@1.90.0#1 + * boost-program-options:x64-linux@1.90.0#1 + * boost-random:x64-linux@1.90.0#1 + * boost-range:x64-linux@1.90.0#1 + * boost-ratio:x64-linux@1.90.0#1 + * boost-regex:x64-linux@1.90.0#1 + * boost-scope:x64-linux@1.90.0#1 + * boost-scope-exit:x64-linux@1.90.0#1 + * boost-smart-ptr:x64-linux@1.90.0#1 + * boost-static-assert:x64-linux@1.90.0#1 + * boost-system:x64-linux@1.90.0#1 + * boost-thread:x64-linux@1.90.0#1 + * boost-throw-exception:x64-linux@1.90.0#1 + * boost-tokenizer:x64-linux@1.90.0#1 + * boost-tuple:x64-linux@1.90.0#1 + * boost-type-index:x64-linux@1.90.0#1 + * boost-type-traits:x64-linux@1.90.0#1 + * boost-typeof:x64-linux@1.90.0#1 + * boost-uninstall:x64-linux@1.90.0#1 + * boost-unordered:x64-linux@1.90.0#1 + * boost-utility:x64-linux@1.90.0#1 + * boost-uuid:x64-linux@1.90.0#1 + * boost-variant:x64-linux@1.90.0#1 + * boost-variant2:x64-linux@1.90.0#1 + * boost-winapi:x64-linux@1.90.0#1 + * brotli:x64-linux@1.2.0 + * bzip2[core,tool]:x64-linux@1.0.8#6 + * civetweb:x64-linux@1.16#2 + curl[core,non-http,openssl,ssl]:x64-linux@8.18.0#1 + * date:x64-linux@3.0.4 + double-conversion:x64-linux@3.2.1 + * fast-float:x64-linux@8.2.3 + fmt:x64-linux@10.2.1 + folly[core,lz4]:x64-linux@2025.04.14.00 -- /turbo/pmarkovski/ArcticDB/cpp/third_party/vcpkg_overlays/folly + * gflags:x64-linux@2.2.2#5 + * glog:x64-linux@0.6.0#2 + gtest:x64-linux@1.12.1 + * libbson:x64-linux@1.30.3 + libevent[core,thread]:x64-linux@2.1.12#7 + * libiconv:x64-linux@1.17 + * liblzma:x64-linux@5.8.2 + * libxml2[core,iconv,zlib]:x64-linux@2.15.1 + * lz4:x64-linux@1.9.3#4 + * mongo-c-driver[core,openssl]:x64-linux@1.28.0 + mongo-cxx-driver:x64-linux@3.10.2#1 + openssl:x64-linux@3.3.0 + pcre2[core,jit,platform-default-features]:x64-linux@10.47 + prometheus-cpp[core,pull,push]:x64-linux@1.1.0 + protobuf[core,zlib]:x64-linux@3.21.8 + rapidcheck:x64-linux@2023-12-14 + * re2:x64-linux@2025-11-05 + * s2n:x64-linux@1.3.5 + * snappy:x64-linux@1.2.2#1 + spdlog:x64-linux@1.13.0 + * thrift:x64-linux@0.22.0 + unordered-dense:x64-linux@4.8.1 + * utf8proc:x64-linux@2.11.3 + * vcpkg-boost:x64-linux@2025-03-29 + * vcpkg-cmake:x64-linux@2024-04-23 + * vcpkg-cmake-config:x64-linux@2024-05-23 + * vcpkg-cmake-get-vars:x64-linux@2025-05-29 + * xsimd:x64-linux@14.0.0 + xxhash:x64-linux@0.8.2 + * zlib:x64-linux@1.3.1 + zstd:x64-linux@1.5.2 + arcticdb-sparrow provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(sparrow CONFIG REQUIRED) + target_link_libraries(main PRIVATE sparrow::sparrow) + + zstd provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(zstd CONFIG REQUIRED) + target_link_libraries(main PRIVATE zstd::libzstd_static) + + zstd provides pkg-config modules: + + # fast lossless compression algorithm library + libzstd + + openssl is compatible with built-in CMake targets: + + find_package(OpenSSL REQUIRED) + target_link_libraries(main PRIVATE OpenSSL::SSL) + target_link_libraries(main PRIVATE OpenSSL::Crypto) + + libevent provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(Libevent CONFIG REQUIRED) + target_link_libraries(main PRIVATE libevent::core libevent::extra libevent::pthreads) + + libevent provides pkg-config modules: + + # libevent is an asynchronous notification event loop library + libevent + + # libevent_core + libevent_core + + # libevent_extra + libevent_extra + + # libevent_pthreads adds pthreads-based threading support to libevent + libevent_pthreads + + boost-optional provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(boost_optional CONFIG REQUIRED) + target_link_libraries(main PRIVATE Boost::optional) + + boost-dynamic-bitset provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(boost_dynamic_bitset CONFIG REQUIRED) + target_link_libraries(main PRIVATE Boost::dynamic_bitset) + + boost-multiprecision provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(boost_multiprecision CONFIG REQUIRED) + target_link_libraries(main PRIVATE Boost::multiprecision) + + The package arrow provides CMake targets: + + find_package(Arrow CONFIG REQUIRED) + target_link_libraries(main PRIVATE "$,Arrow::arrow_static,Arrow::arrow_shared>") + + aws-c-common provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(aws-c-common CONFIG REQUIRED) + target_link_libraries(main PRIVATE AWS::aws-c-common) + + aws-c-sdkutils provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(aws-c-sdkutils CONFIG REQUIRED) + target_link_libraries(main PRIVATE AWS::aws-c-sdkutils) + + aws-c-cal provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(aws-c-cal CONFIG REQUIRED) + target_link_libraries(main PRIVATE AWS::aws-c-cal) + + aws-c-io provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(aws-c-io CONFIG REQUIRED) + target_link_libraries(main PRIVATE AWS::aws-c-io) + + aws-c-http provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(aws-c-http CONFIG REQUIRED) + target_link_libraries(main PRIVATE AWS::aws-c-http) + + aws-c-auth provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(aws-c-auth CONFIG REQUIRED) + target_link_libraries(main PRIVATE AWS::aws-c-auth) + + aws-checksums provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(aws-checksums CONFIG REQUIRED) + target_link_libraries(main PRIVATE AWS::aws-checksums) + + aws-c-event-stream provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(aws-c-event-stream CONFIG REQUIRED) + target_link_libraries(main PRIVATE AWS::aws-c-event-stream) + + aws-c-mqtt provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(aws-c-mqtt CONFIG REQUIRED) + target_link_libraries(main PRIVATE AWS::aws-c-mqtt) + + aws-c-s3 provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(aws-c-s3 CONFIG REQUIRED) + target_link_libraries(main PRIVATE AWS::aws-c-s3) + + aws-crt-cpp provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(aws-crt-cpp CONFIG REQUIRED) + target_link_libraries(main PRIVATE AWS::aws-crt-cpp) + + curl is compatible with built-in CMake targets: + + find_package(CURL REQUIRED) + target_link_libraries(main PRIVATE CURL::libcurl) + + The package aws-sdk-cpp:x64-linux provides CMake targets: + + When using AWSSDK, AWSSDK_ROOT_DIR must be defined by the user. + find_package(AWSSDK CONFIG COMPONENTS core dynamodb kinesis s3 REQUIRED) + target_include_directories(main PRIVATE ${AWSSDK_INCLUDE_DIRS}) + target_link_libraries(main PRIVATE ${AWSSDK_LIBRARIES}) + + OR + + find_package(aws-cpp-sdk-core REQUIRED) + target_include_directories(main PRIVATE aws-cpp-sdk-core) + target_link_libraries(main PRIVATE aws-cpp-sdk-core) + + azure-core-cpp provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(azure-core-cpp CONFIG REQUIRED) + target_link_libraries(main PRIVATE Azure::azure-core) + + azure-identity-cpp provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(azure-identity-cpp CONFIG REQUIRED) + target_link_libraries(main PRIVATE Azure::azure-identity) + + azure-storage-blobs-cpp provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(azure-storage-blobs-cpp CONFIG REQUIRED) + target_link_libraries(main PRIVATE Azure::azure-storage-blobs) + + benchmark provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(benchmark CONFIG REQUIRED) + target_link_libraries(main PRIVATE benchmark::benchmark benchmark::benchmark_main) + + benchmark provides pkg-config modules: + + # Google microbenchmark framework + benchmark + + # Google microbenchmark framework (with main() function) + benchmark_main + + bitmagic is header-only and can be used from CMake via: + + find_path(BITMAGIC_INCLUDE_DIRS "bitmagic/bm.h") + target_include_directories(main PRIVATE ${BITMAGIC_INCLUDE_DIRS}) + + boost-callable-traits provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(boost_callable_traits CONFIG REQUIRED) + target_link_libraries(main PRIVATE Boost::callable_traits) + + boost-circular-buffer provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(boost_circular_buffer CONFIG REQUIRED) + target_link_libraries(main PRIVATE Boost::circular_buffer) + + boost-interprocess provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(boost_interprocess CONFIG REQUIRED) + target_link_libraries(main PRIVATE Boost::interprocess) + + double-conversion provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(double-conversion CONFIG REQUIRED) + target_link_libraries(main PRIVATE double-conversion::double-conversion) + + The package fmt provides CMake targets: + + find_package(fmt CONFIG REQUIRED) + target_link_libraries(main PRIVATE fmt::fmt) + + # Or use the header-only version + find_package(fmt CONFIG REQUIRED) + target_link_libraries(main PRIVATE fmt::fmt-header-only) + + folly provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(folly CONFIG REQUIRED) + target_link_libraries(main PRIVATE Folly::folly Folly::folly_deps Folly::follybenchmark Folly::folly_test_util) + + folly provides pkg-config modules: + + # Facebook (Folly) C++ library + libfolly + + gtest provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(GTest CONFIG REQUIRED) + target_link_libraries(main PRIVATE GTest::gmock GTest::gtest GTest::gmock_main GTest::gtest_main) + + gtest provides pkg-config modules: + + # GoogleMock (without main() function) + gmock + + # GoogleMock (with main() function) + gmock_main + + # GoogleTest (without main() function) + gtest + + # GoogleTest (with main() function) + gtest_main + + mongo-cxx-driver provides CMake targets: + + find_package(bsoncxx CONFIG REQUIRED) + target_link_libraries(main PRIVATE $,mongo::bsoncxx_static,mongo::bsoncxx_shared>) + + find_package(mongocxx CONFIG REQUIRED) + target_link_libraries(main PRIVATE $,mongo::mongocxx_static,mongo::mongocxx_shared>) + + The package pcre2 is compatible with built-in CMake targets: + + # Each component imports a target: + # TARGETS: PCRE2::8BIT PCRE2::16BIT PCRE2::32BIT PCRE2::POSIX + find_package(pcre2 CONFIG REQUIRED) + target_link_libraries(main PRIVATE PCRE2::8BIT PCRE2::16BIT PCRE2::32BIT PCRE2::POSIX) + + prometheus-cpp provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(prometheus-cpp CONFIG REQUIRED) + target_link_libraries(main PRIVATE prometheus-cpp::core prometheus-cpp::pull prometheus-cpp::push) + + protobuf provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(protobuf CONFIG REQUIRED) + target_link_libraries(main PRIVATE protobuf::libprotoc protobuf::libprotobuf protobuf::libprotobuf-lite) + + protobuf provides pkg-config modules: + + # Google's Data Interchange Format + protobuf-lite + + # Google's Data Interchange Format + protobuf + + rapidcheck provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(rapidcheck CONFIG REQUIRED) + # note: 3 additional targets are not displayed. + target_link_libraries(main PRIVATE rapidcheck rapidcheck_boost rapidcheck_catch rapidcheck_gmock) + + rapidcheck provides pkg-config modules: + + # C++ framework for property based testing inspired by QuickCheck and other similar frameworks + rapidcheck + + # boost headers for rapidcheck + rapidcheck_boost + + # boost_test headers for rapidcheck + rapidcheck_boost_test + + # catch headers for rapidcheck + rapidcheck_catch + + # doctest headers for rapidcheck + rapidcheck_doctest + + # gmock headers for rapidcheck + rapidcheck_gtest + + The package spdlog provides CMake targets: + + find_package(spdlog CONFIG REQUIRED) + target_link_libraries(main PRIVATE spdlog::spdlog) + + # Or use the header-only version + find_package(spdlog CONFIG REQUIRED) + target_link_libraries(main PRIVATE spdlog::spdlog_header_only) + + unordered-dense provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(unordered_dense CONFIG REQUIRED) + target_link_libraries(main PRIVATE unordered_dense::unordered_dense) + + xxhash provides CMake targets: + + # this is heuristically generated, and may not be correct + find_package(xxHash CONFIG REQUIRED) + target_link_libraries(main PRIVATE xxHash::xxhash) + + xxhash provides pkg-config modules: + + # extremely fast hash algorithm + libxxhash + + All requested installations completed successfully in: 1.54 ms + -- Running vcpkg install - done + Python_EXECUTABLE is set (to '/turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/bin/python3'), which can influence FindPython and pybind11. + CMake Warning at CMake/PythonUtils.cmake:72 (message): + PYTHON_LIBRARIES is set (to + '/turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/libpython3.11.so'). It + won't be used to build ArcticDB but may cause third-party packages to find + a different Python and break the build. + Call Stack (most recent call first): + CMake/PythonUtils.cmake:79 (_body) + CMake/PythonUtils.cmake:122 (_python_utils_check_vars) + CMakeLists.txt:193 (include) + + + CMake Warning at CMake/PythonUtils.cmake:72 (message): + PYTHON_LIBRARY is set (to + '/turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/libpython3.11.so'). It + won't be used to build ArcticDB but may cause third-party packages to find + a different Python and break the build. + Call Stack (most recent call first): + CMake/PythonUtils.cmake:79 (_body) + CMake/PythonUtils.cmake:122 (_python_utils_check_vars) + CMakeLists.txt:193 (include) + + + CMake Warning at CMake/PythonUtils.cmake:72 (message): + PYTHON_INCLUDE_DIRS is set (to + '/turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/include/python3.11'). It won't + be used to build ArcticDB but may cause third-party packages to find a + different Python and break the build. + Call Stack (most recent call first): + CMake/PythonUtils.cmake:79 (_body) + CMake/PythonUtils.cmake:122 (_python_utils_check_vars) + CMakeLists.txt:193 (include) + + + CMake Deprecation Warning at third_party/pybind11/CMakeLists.txt:13 (cmake_minimum_required): + Compatibility with CMake < 3.10 will be removed from a future version of + CMake. + + Update the VERSION argument value. Or, use the ... syntax + to tell CMake that the project requires at least but has been updated + to work with policies introduced by or earlier. + + + -- pybind11 v2.13.6 + CMake Warning (dev) at third_party/pybind11/tools/FindPythonLibsNew.cmake:101 (message): + Policy CMP0148 is not set: The FindPythonInterp and FindPythonLibs modules + are removed. Run "cmake --help-policy CMP0148" for policy details. Use + the cmake_policy command to set the policy and suppress this warning, or + preferably upgrade to using FindPython, either by calling it explicitly + before pybind11, or by setting PYBIND11_FINDPYTHON ON before pybind11. + Call Stack (most recent call first): + vcpkg/scripts/buildsystems/vcpkg.cmake:908 (_find_package) + third_party/pybind11/tools/pybind11Tools.cmake:50 (find_package) + third_party/pybind11/tools/pybind11Common.cmake:228 (include) + third_party/pybind11/CMakeLists.txt:251 (include) + This warning is for project developers. Use -Wno-dev to suppress it. + + CMake Warning (dev) at vcpkg/scripts/buildsystems/vcpkg.cmake:908 (_find_package): + Policy CMP0148 is not set: The FindPythonInterp and FindPythonLibs modules + are removed. Run "cmake --help-policy CMP0148" for policy details. Use + the cmake_policy command to set the policy and suppress this warning. + + Call Stack (most recent call first): + third_party/pybind11/tools/FindPythonLibsNew.cmake:114 (find_package) + vcpkg/scripts/buildsystems/vcpkg.cmake:908 (_find_package) + third_party/pybind11/tools/pybind11Tools.cmake:50 (find_package) + third_party/pybind11/tools/pybind11Common.cmake:228 (include) + third_party/pybind11/CMakeLists.txt:251 (include) + This warning is for project developers. Use -Wno-dev to suppress it. + + CMake Deprecation Warning at third_party/semimap/CMakeLists.txt:1 (cmake_minimum_required): + Compatibility with CMake < 3.10 will be removed from a future version of + CMake. + + Update the VERSION argument value. Or, use the ... syntax + to tell CMake that the project requires at least but has been updated + to work with policies introduced by or earlier. + + + CMake Deprecation Warning at third_party/msgpack-c/CMakeLists.txt:1 (CMAKE_MINIMUM_REQUIRED): + Compatibility with CMake < 3.10 will be removed from a future version of + CMake. + + Update the VERSION argument value. Or, use the ... syntax + to tell CMake that the project requires at least but has been updated + to work with policies introduced by or earlier. + + + -- Dynamically linking with Boost + CMake Warning (dev) at out/linux-release-build/vcpkg_installed/x64-linux/share/boost/vcpkg-cmake-wrapper.cmake:3 (_find_package): + Policy CMP0167 is not set: The FindBoost module is removed. Run "cmake + --help-policy CMP0167" for policy details. Use the cmake_policy command to + set the policy and suppress this warning. + + Call Stack (most recent call first): + vcpkg/scripts/buildsystems/vcpkg.cmake:862 (include) + third_party/msgpack-c/CMakeLists.txt:79 (FIND_PACKAGE) + This warning is for project developers. Use -Wno-dev to suppress it. + + -- Found Boost: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/share/boost/BoostConfig.cmake (found version "1.90.0") + -- Could NOT find Doxygen (missing: DOXYGEN_EXECUTABLE) + -- Found Protobuf: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libprotobuf.a (found version "3.21.8") + -- Found AWS SDK for C++, Version: 1.11.474, Install Root:/turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux, Platform Prefix:, Platform Dependent Libraries: pthread;curl + -- Components specified for AWSSDK: s3;identity-management, application will be depending on libs: aws-cpp-sdk-s3;aws-cpp-sdk-identity-management;aws-cpp-sdk-cognito-identity;aws-cpp-sdk-sts;aws-cpp-sdk-core;AWS::aws-crt-cpp;AWS::aws-c-http;AWS::aws-c-mqtt;AWS::aws-c-cal;AWS::aws-c-auth;AWS::aws-c-common;AWS::aws-c-io;AWS::aws-checksums;AWS::aws-c-event-stream;AWS::aws-c-s3;AWS::aws-c-compression;AWS::aws-c-sdkutils;pthread;curl + -- Try finding aws-cpp-sdk-core + -- Found OpenSSL: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libcrypto.a (found version "3.3.0") found components: Crypto + -- Found ZLIB: optimized;/turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libz.a;debug;/turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/debug/lib/libz.a (found version "1.3.1") + -- Found OpenSSL: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libcrypto.a (found suitable version "3.3.0", minimum required is "3") + -- Found ZLIB: optimized;/turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libz.a;debug;/turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/debug/lib/libz.a (found suitable version "1.3.1", minimum required is "1") + -- Found aws-cpp-sdk-core + -- Try finding aws-cpp-sdk-sts + -- Found aws-cpp-sdk-sts + -- Try finding aws-cpp-sdk-cognito-identity + -- Found aws-cpp-sdk-cognito-identity + -- Try finding aws-cpp-sdk-identity-management + -- Found aws-cpp-sdk-identity-management + -- Try finding aws-cpp-sdk-s3 + -- Found aws-cpp-sdk-s3 + CMake Warning (dev) at out/linux-release-build/vcpkg_installed/x64-linux/share/boost/vcpkg-cmake-wrapper.cmake:3 (_find_package): + Policy CMP0167 is not set: The FindBoost module is removed. Run "cmake + --help-policy CMP0167" for policy details. Use the cmake_policy command to + set the policy and suppress this warning. + + Call Stack (most recent call first): + vcpkg/scripts/buildsystems/vcpkg.cmake:862 (include) + arcticdb/CMakeLists.txt:19 (find_package) + This warning is for project developers. Use -Wno-dev to suppress it. + + -- Found libevent include directory: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/include + -- Found libevent component: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libevent_core.a + -- Found libevent component: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libevent_extra.a + -- Found libevent component: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libevent_pthreads.a + -- Found libevent 2.1.12 in /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux + CMake Warning (dev) at out/linux-release-build/vcpkg_installed/x64-linux/share/boost/vcpkg-cmake-wrapper.cmake:3 (_find_package): + Policy CMP0167 is not set: The FindBoost module is removed. Run "cmake + --help-policy CMP0167" for policy details. Use the cmake_policy command to + set the policy and suppress this warning. + + Call Stack (most recent call first): + vcpkg/scripts/buildsystems/vcpkg.cmake:862 (include) + /apps/research/tools/cmake/3.31.3/share/cmake-3.31/Modules/CMakeFindDependencyMacro.cmake:76 (find_package) + out/linux-release-build/vcpkg_installed/x64-linux/share/folly/folly-config.cmake:80 (find_dependency) + vcpkg/scripts/buildsystems/vcpkg.cmake:908 (_find_package) + arcticdb/CMakeLists.txt:23 (find_package) + This warning is for project developers. Use -Wno-dev to suppress it. + + -- Found Boost: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/share/boost/BoostConfig.cmake (found version "1.90.0") found components: context filesystem program_options regex system thread + -- Found folly: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux + CMake Warning (dev) at /apps/research/tools/cmake/3.31.3/share/cmake-3.31/Modules/FindPackageHandleStandardArgs.cmake:441 (message): + The package name passed to `find_package_handle_standard_args` (Protobuf) + does not match the name of the calling package (protobuf). This can lead + to problems in calling code that expects `find_package` result variables + (e.g., `_FOUND`) to follow a certain pattern. + Call Stack (most recent call first): + out/linux-release-build/vcpkg_installed/x64-linux/share/protobuf/protobuf-module.cmake:162 (FIND_PACKAGE_HANDLE_STANDARD_ARGS) + out/linux-release-build/vcpkg_installed/x64-linux/share/protobuf/protobuf-config.cmake:170 (include) + out/linux-release-build/vcpkg_installed/x64-linux/share/protobuf/vcpkg-cmake-wrapper.cmake:16 (_find_package) + vcpkg/scripts/buildsystems/vcpkg.cmake:862 (include) + arcticdb/CMakeLists.txt:37 (find_package) + This warning is for project developers. Use -Wno-dev to suppress it. + + -- Found Protobuf: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/tools/protobuf/protoc (found version "3.21.8.0") + -- Found OpenSSL: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libcrypto.a (found version "3.3.0") + CMake Warning at out/linux-release-build/vcpkg_installed/x64-linux/share/unofficial-utf8proc/unofficial-utf8proc-config.cmake:1 (message): + find_package(unofficial-utf8proc) is deprecated. + + utf8proc provides CMake targets: + + + + find_package(utf8proc) + target_link_libraries(main PRIVATE utf8proc::utf8proc) + + Call Stack (most recent call first): + vcpkg/scripts/buildsystems/vcpkg.cmake:908 (_find_package) + /apps/research/tools/cmake/3.31.3/share/cmake-3.31/Modules/CMakeFindDependencyMacro.cmake:76 (find_package) + out/linux-release-build/vcpkg_installed/x64-linux/share/mongoc-1.0/mongoc-1.0-config.cmake:34 (find_dependency) + vcpkg/scripts/buildsystems/vcpkg.cmake:908 (_find_package) + /apps/research/tools/cmake/3.31.3/share/cmake-3.31/Modules/CMakeFindDependencyMacro.cmake:76 (find_package) + out/linux-release-build/vcpkg_installed/x64-linux/share/mongocxx/mongocxx-config.cmake:2 (find_dependency) + vcpkg/scripts/buildsystems/vcpkg.cmake:908 (_find_package) + arcticdb/CMakeLists.txt:38 (find_package) + + + -- Found libevent include directory: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/include + -- Found libevent component: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libevent_core.a + -- Found libevent component: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libevent_extra.a + -- Found libevent component: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libevent_pthreads.a + -- Found libevent 2.1.12 in /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux + -- Not setting debug flags + Linking against SSL + -- Configuring done (55.3s) + -- Generating done (0.3s) + -- Build files have been written to: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build + [ 2%] Built target lmdb + [ 23%] Built target arcticdb_proto + [ 93%] Built target arcticdb_core_object + [ 93%] Built target arcticdb_core_static + [ 98%] Built target arcticdb_python + [100%] Built target arcticdb_ext + [100%] Built target install_arcticdb_ext + Destination: /turbo/pmarkovski/ArcticDB/python/arcticdb_ext.cpython-311-x86_64-linux-gnu.so + ARCTICDB_VCPKG_INSTALLED_DIR=None + Running /apps/research/tools/bin/cmake -DTEST=False -DBUILD_PYTHON_VERSION=3.11 -DCMAKE_INSTALL_PREFIX=/turbo/pmarkovski/ArcticDB/python --preset linux-release + Removed: cpp/vcpkg/buildtrees + Removed: cpp/vcpkg/downloads + Removed: cpp/vcpkg/packages + Running /apps/research/tools/bin/cmake --build cpp/out/linux-release-build -j 4 --target install_arcticdb_ext + Creating /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages/arcticdb.egg-link (link to python) + Adding arcticdb 0.0.0.dev0 to easy-install.pth file + Installing arcticdb_update_storage script to /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/bin + + Installed /turbo/pmarkovski/ArcticDB/python + running protoc + Looking in indexes: https://repo.prod.m/artifactory/api/pypi/pegasus-311-1/simple/ + [pip_auth] looking up repository credentials for repo.prod.m + [pip_auth] no pypi credentials found for repo.prod.m. User config has domains: set() + Collecting grpcio-tools>=1.49 + [pip_auth] looking up repository credentials for repo.prod.m + [pip_auth] no pypi credentials found for repo.prod.m. User config has domains: set() + Using cached https://repo.prod.m/artifactory/api/pypi/pegasus-311-1/grpcio-tools/1.56.2/grpcio_tools-1.56.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.6 MB) + [pip_auth] looking up repository credentials for repo.prod.m + [pip_auth] no pypi credentials found for repo.prod.m. User config has domains: set() + Collecting protobuf==4.* + [pip_auth] looking up repository credentials for repo.prod.m + [pip_auth] no pypi credentials found for repo.prod.m. User config has domains: set() + Using cached https://repo.prod.m/artifactory/api/pypi/pegasus-311-1/protobuf/4.25.1/protobuf-4.25.1-cp37-abi3-manylinux2014_x86_64.whl (294 kB) + [pip_auth] looking up repository credentials for repo.prod.m + [pip_auth] no pypi credentials found for repo.prod.m. User config has domains: set() + Collecting grpcio>=1.56.2 (from grpcio-tools>=1.49) + [pip_auth] looking up repository credentials for repo.prod.m + [pip_auth] no pypi credentials found for repo.prod.m. User config has domains: set() + Using cached https://repo.prod.m/artifactory/api/pypi/pegasus-311-1/grpcio/1.62.3/grpcio-1.62.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.5 MB) + [pip_auth] looking up repository credentials for repo.prod.m + [pip_auth] no pypi credentials found for repo.prod.m. User config has domains: set() + Collecting setuptools (from grpcio-tools>=1.49) + [pip_auth] looking up repository credentials for repo.prod.m + [pip_auth] no pypi credentials found for repo.prod.m. User config has domains: set() + Using cached https://repo.prod.m/artifactory/api/pypi/pegasus-311-1/setuptools/65.6.3/setuptools-65.6.3-py3-none-any.whl (1.2 MB) + Installing collected packages: setuptools, protobuf, grpcio, grpcio-tools + Successfully installed grpcio-1.62.3 grpcio-tools-1.56.2 protobuf-4.25.1 setuptools-65.6.3 + ['develop'] + + Protoc compilation (into 'python/arcticdb/proto') for versions '4': + Running /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/bin/python -mpip install --disable-pip-version-check --target=/tmp/tmp6jt8apcx grpcio-tools>=1.49 protobuf==4.* + Running /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/bin/python -mgrpc_tools.protoc -Icpp/proto --python_out=python/arcticdb/proto/4 cpp/proto/arcticc/pb2/gcp_storage.proto cpp/proto/arcticc/pb2/utils.proto cpp/proto/arcticc/pb2/config.proto cpp/proto/arcticc/pb2/s3_storage.proto cpp/proto/arcticc/pb2/lmdb_storage.proto cpp/proto/arcticc/pb2/in_memory_storage.proto cpp/proto/arcticc/pb2/encoding.proto cpp/proto/arcticc/pb2/descriptors.proto cpp/proto/arcticc/pb2/nfs_backed_storage.proto cpp/proto/arcticc/pb2/storage.proto cpp/proto/arcticc/pb2/azure_storage.proto cpp/proto/arcticc/pb2/logger.proto cpp/proto/arcticc/pb2/mongo_storage.proto cpp/proto/arcticc/pb2/mapped_file_storage.proto +Successfully installed arcticdb-0.0.0.dev0 diff --git a/python/test.py b/python/test.py new file mode 100644 index 00000000000..db82420fd8c --- /dev/null +++ b/python/test.py @@ -0,0 +1 @@ +import arcticdb \ No newline at end of file diff --git a/python/tests/integration/toolbox/test_storage_mover.py b/python/tests/integration/toolbox/test_storage_mover.py index 730f3f6cf19..a37a0d72d0a 100644 --- a/python/tests/integration/toolbox/test_storage_mover.py +++ b/python/tests/integration/toolbox/test_storage_mover.py @@ -321,18 +321,18 @@ def test_correct_versions_in_destination( assert [k.version_id for k in lt.find_keys(KeyType.TABLE_DATA)] == [2, 3, 4, 4, 6] -@settings(deadline=None, suppress_health_check=(hypothesis.HealthCheck.function_scoped_fixture,)) -@given(to_copy=st.permutations(["s2", 4, 6]), existing=st.booleans()) -def test_correct_versions_in_destination_force( - to_copy, existing, check_single_threaded, lib_with_gaps_and_reused_keys, version_store_factory -): - try: - _tmp_test_body(to_copy, existing, lib_with_gaps_and_reused_keys, version_store_factory) - except: - import traceback - - traceback.print_exc() - raise +# @settings(deadline=None, suppress_health_check=(hypothesis.HealthCheck.function_scoped_fixture,)) +# @given(to_copy=st.permutations(["s2", 4, 6]), existing=st.booleans()) +# def test_correct_versions_in_destination_force( +# to_copy, existing, check_single_threaded, lib_with_gaps_and_reused_keys, version_store_factory +# ): +# try: +# _tmp_test_body(to_copy, existing, lib_with_gaps_and_reused_keys, version_store_factory) +# except: +# import traceback + +# traceback.print_exc() +# raise def _tmp_test_body(to_copy, existing, lib_with_gaps_and_reused_keys, version_store_factory): diff --git a/python/tests/unit/arcticdb/version_store/test_query_builder_sparse.py b/python/tests/unit/arcticdb/version_store/test_query_builder_sparse.py index 4784142777f..cf9c279f9bb 100644 --- a/python/tests/unit/arcticdb/version_store/test_query_builder_sparse.py +++ b/python/tests/unit/arcticdb/version_store/test_query_builder_sparse.py @@ -177,44 +177,44 @@ def test_query_builder_sparse_dynamic_schema_type_change(lmdb_version_store_dyna assert_frame_equal(expected, received) -@use_of_function_scoped_fixtures_in_hypothesis_checked -@settings(deadline=None) -@given( - df=data_frames( - columns( - ["sparse1", "sparse2"], - elements=strategies.floats(min_value=0, max_value=1000, allow_nan=False, allow_subnormal=False), - fill=strategies.just(np.nan), - ), - ), -) -def test_query_builder_sparse_hypothesis(lmdb_version_store_v1, df): - assume(not df.empty and not df["sparse1"].isnull().all() and not df["sparse2"].isnull().all()) - lib = lmdb_version_store_v1 - sym = "test_query_builder_sparse_hypothesis" - - df.index = pd.date_range("2024-01-01", periods=len(df)) - - lib.write(sym, df, sparsify_floats=True) - - # Filter - expected = df[df["sparse1"].isnull()] - q = QueryBuilder() - q = q[q["sparse1"].isnull()] - received = lib.read(sym, query_builder=q).data - assert_frame_equal(expected, received) - - # Projection - expected = df - expected["projected"] = expected["sparse1"] + expected["sparse2"] - q = QueryBuilder() - q = q.apply("projected", q["sparse1"] + q["sparse2"]) - received = lib.read(sym, query_builder=q).data - assert_frame_equal(expected, received) - - # Groupby + aggregation - expected = df.groupby("sparse1").agg({"sparse2": "sum"}) - q = QueryBuilder().groupby("sparse1").agg({"sparse2": "sum"}) - received = lib.read(sym, query_builder=q).data - received.sort_index(inplace=True) - assert_frame_equal(expected, received) +# @use_of_function_scoped_fixtures_in_hypothesis_checked +# @settings(deadline=None) +# @given( +# df=data_frames( +# columns( +# ["sparse1", "sparse2"], +# elements=strategies.floats(min_value=0, max_value=1000, allow_nan=False, allow_subnormal=False), +# fill=strategies.just(np.nan), +# ), +# ), +# ) +# def test_query_builder_sparse_hypothesis(lmdb_version_store_v1, df): +# assume(not df.empty and not df["sparse1"].isnull().all() and not df["sparse2"].isnull().all()) +# lib = lmdb_version_store_v1 +# sym = "test_query_builder_sparse_hypothesis" + +# df.index = pd.date_range("2024-01-01", periods=len(df)) + +# lib.write(sym, df, sparsify_floats=True) + +# # Filter +# expected = df[df["sparse1"].isnull()] +# q = QueryBuilder() +# q = q[q["sparse1"].isnull()] +# received = lib.read(sym, query_builder=q).data +# assert_frame_equal(expected, received) + +# # Projection +# expected = df +# expected["projected"] = expected["sparse1"] + expected["sparse2"] +# q = QueryBuilder() +# q = q.apply("projected", q["sparse1"] + q["sparse2"]) +# received = lib.read(sym, query_builder=q).data +# assert_frame_equal(expected, received) + +# # Groupby + aggregation +# expected = df.groupby("sparse1").agg({"sparse2": "sum"}) +# q = QueryBuilder().groupby("sparse1").agg({"sparse2": "sum"}) +# received = lib.read(sym, query_builder=q).data +# received.sort_index(inplace=True) +# assert_frame_equal(expected, received) diff --git a/python/tests/unit/arcticdb/version_store/test_ternary.py b/python/tests/unit/arcticdb/version_store/test_ternary.py index 3fc9deaf87a..8e652e0cd34 100644 --- a/python/tests/unit/arcticdb/version_store/test_ternary.py +++ b/python/tests/unit/arcticdb/version_store/test_ternary.py @@ -1178,77 +1178,77 @@ def test_filter_ternary_dynamic_missing_columns(lmdb_version_store_dynamic_schem # TODO: Assert that the projected column is of type float64 after modify_schema change is merged -@use_of_function_scoped_fixtures_in_hypothesis_checked -@settings(deadline=None) -@given( - df=data_frames( - columns( - ["condition", "col1", "col2"], - elements=strategies.floats(min_value=0, max_value=1000, allow_nan=False, allow_subnormal=False), - fill=strategies.just(np.nan), - ), - ), -) -def test_ternary_hypothesis(lmdb_version_store_v1, df, any_output_format): - assume( - not df.empty - and not df["condition"].isnull().all() - and not df["col1"].isnull().all() - and not df["col2"].isnull().all() - ) - lib = lmdb_version_store_v1 - lib._set_output_format_for_pipeline_tests(any_output_format) - dense_sym = "test_ternary_hypothesis_dense" - sparse_sym = "test_ternary_hypothesis_sparse" - - df.index = pd.date_range("2024-01-01", periods=len(df)) - - lib.write(dense_sym, df) - lib.write(sparse_sym, df, sparsify_floats=True) - - # Projection - # col/col - expected = df.copy(deep=True) - expected["projected"] = np.where( - expected["condition"].isnull().to_numpy(), expected["col1"].to_numpy(), expected["col2"].to_numpy() - ) - q = QueryBuilder() - q = q.apply("projected", where(q["condition"].isnull(), q["col1"], q["col2"])) - assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) - assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) - # col/val - expected = df.copy(deep=True) - expected["projected"] = np.where(expected["condition"].isnull().to_numpy(), expected["col1"].to_numpy(), 2000.0) - q = QueryBuilder() - q = q.apply("projected", where(q["condition"].isnull(), q["col1"], 2000)) - assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) - assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) - # val/col - expected = df.copy(deep=True) - expected["projected"] = np.where(expected["condition"].isnull().to_numpy(), 2000.0, expected["col2"].to_numpy()) - q = QueryBuilder() - q = q.apply("projected", where(q["condition"].isnull(), 2000, q["col2"])) - assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) - assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) - # val/val - expected = df.copy(deep=True) - expected["projected"] = np.where(expected["condition"].isnull().to_numpy(), 2000.0, 3000.0) - q = QueryBuilder() - q = q.apply("projected", where(q["condition"].isnull(), 2000, 3000)) - assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) - assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) - - # Filters - # Only test col/col, col/val etc can be achieved more efficiently without using the ternary operator - expected = df.copy(deep=True) - expected = expected[ - np.where( - expected["condition"].isnull().to_numpy(), - expected["col1"].isnull().to_numpy(), - expected["col2"].isnull().to_numpy(), - ) - ] - q = QueryBuilder() - q = q[where(q["condition"].isnull(), q["col1"].isnull(), q["col2"].isnull())] - assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) - assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) +# @use_of_function_scoped_fixtures_in_hypothesis_checked +# @settings(deadline=None) +# @given( +# df=data_frames( +# columns( +# ["condition", "col1", "col2"], +# elements=strategies.floats(min_value=0, max_value=1000, allow_nan=False, allow_subnormal=False), +# fill=strategies.just(np.nan), +# ), +# ), +# ) +# def test_ternary_hypothesis(lmdb_version_store_v1, df, any_output_format): +# assume( +# not df.empty +# and not df["condition"].isnull().all() +# and not df["col1"].isnull().all() +# and not df["col2"].isnull().all() +# ) +# lib = lmdb_version_store_v1 +# lib._set_output_format_for_pipeline_tests(any_output_format) +# dense_sym = "test_ternary_hypothesis_dense" +# sparse_sym = "test_ternary_hypothesis_sparse" + +# df.index = pd.date_range("2024-01-01", periods=len(df)) + +# lib.write(dense_sym, df) +# lib.write(sparse_sym, df, sparsify_floats=True) + +# # Projection +# # col/col +# expected = df.copy(deep=True) +# expected["projected"] = np.where( +# expected["condition"].isnull().to_numpy(), expected["col1"].to_numpy(), expected["col2"].to_numpy() +# ) +# q = QueryBuilder() +# q = q.apply("projected", where(q["condition"].isnull(), q["col1"], q["col2"])) +# assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) +# assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) +# # col/val +# expected = df.copy(deep=True) +# expected["projected"] = np.where(expected["condition"].isnull().to_numpy(), expected["col1"].to_numpy(), 2000.0) +# q = QueryBuilder() +# q = q.apply("projected", where(q["condition"].isnull(), q["col1"], 2000)) +# assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) +# assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) +# # val/col +# expected = df.copy(deep=True) +# expected["projected"] = np.where(expected["condition"].isnull().to_numpy(), 2000.0, expected["col2"].to_numpy()) +# q = QueryBuilder() +# q = q.apply("projected", where(q["condition"].isnull(), 2000, q["col2"])) +# assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) +# assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) +# # val/val +# expected = df.copy(deep=True) +# expected["projected"] = np.where(expected["condition"].isnull().to_numpy(), 2000.0, 3000.0) +# q = QueryBuilder() +# q = q.apply("projected", where(q["condition"].isnull(), 2000, 3000)) +# assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) +# assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) + +# # Filters +# # Only test col/col, col/val etc can be achieved more efficiently without using the ternary operator +# expected = df.copy(deep=True) +# expected = expected[ +# np.where( +# expected["condition"].isnull().to_numpy(), +# expected["col1"].isnull().to_numpy(), +# expected["col2"].isnull().to_numpy(), +# ) +# ] +# q = QueryBuilder() +# q = q[where(q["condition"].isnull(), q["col1"].isnull(), q["col2"].isnull())] +# assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) +# assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) diff --git a/setup.py b/setup.py index 1a02a6890fd..ffc17b5887a 100644 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ def safe_remove_directory(dir_path): class CompileProto(Command): # When adding new protobuf versions, also update: setup.cfg, python/arcticdb/__init__.py - _PROTOBUF_TO_GRPC_VERSION = {"3": "<1.31", "4": ">=1.49", "5": ">=1.68.1", "6": ">=1.73.0"} + _PROTOBUF_TO_GRPC_VERSION = {"4": ">=1.49"} description = '"protoc" generate code _pb2.py from .proto files' user_options = [ From 1126624fdf6aa97eb1afb51bcd1f032b10d3a353 Mon Sep 17 00:00:00 2001 From: "Petar Markovski (Sofia)" Date: Tue, 24 Mar 2026 16:28:44 +0000 Subject: [PATCH 24/31] add debug --- .github/workflows/compile_and_test_with_conda_unix.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/compile_and_test_with_conda_unix.yml b/.github/workflows/compile_and_test_with_conda_unix.yml index d5789e2fc1f..b00216c7c63 100644 --- a/.github/workflows/compile_and_test_with_conda_unix.yml +++ b/.github/workflows/compile_and_test_with_conda_unix.yml @@ -206,6 +206,10 @@ jobs: uses: SimenB/github-actions-cpu-cores@v2.0.0 id: cpu-cores + - name: Print cache key + run: | + echo "Cache key: conda-env-${{inputs.platform}}" + - name: Install Conda environment from environment-dev.yml uses: mamba-org/setup-micromamba@v2.0.6 with: From 1e39cc86aef73013e27d7dc36977f5ddaf153a76 Mon Sep 17 00:00:00 2001 From: "Petar Markovski (Sofia)" Date: Tue, 24 Mar 2026 16:40:18 +0000 Subject: [PATCH 25/31] add protoc to else --- .github/workflows/compile_and_test_with_conda_unix.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/compile_and_test_with_conda_unix.yml b/.github/workflows/compile_and_test_with_conda_unix.yml index b00216c7c63..50f11632a09 100644 --- a/.github/workflows/compile_and_test_with_conda_unix.yml +++ b/.github/workflows/compile_and_test_with_conda_unix.yml @@ -206,10 +206,6 @@ jobs: uses: SimenB/github-actions-cpu-cores@v2.0.0 id: cpu-cores - - name: Print cache key - run: | - echo "Cache key: conda-env-${{inputs.platform}}" - - name: Install Conda environment from environment-dev.yml uses: mamba-org/setup-micromamba@v2.0.6 with: @@ -294,6 +290,7 @@ jobs: echo "Run custom pytest command: $ARCTICDB_PYTEST_ARGS" eval "$ARCTICDB_PYTEST_ARGS" else + python setup.py protoc --build-lib python cd python python -m pytest --timeout=3600 -v --tb=line -n logical --dist worksteal \ tests/${{matrix.type}} $ARCTICDB_PYTEST_ARGS From b8bd15b4a074630911da01ca697789f8236fa6ce Mon Sep 17 00:00:00 2001 From: "Petar Markovski (Sofia)" Date: Tue, 24 Mar 2026 16:54:29 +0000 Subject: [PATCH 26/31] revert files --- build.txt | 871 ------------------ .../integration/toolbox/test_storage_mover.py | 24 +- .../test_query_builder_sparse.py | 82 +- .../arcticdb/version_store/test_ternary.py | 148 +-- setup.py | 2 +- 5 files changed, 128 insertions(+), 999 deletions(-) delete mode 100644 build.txt diff --git a/build.txt b/build.txt deleted file mode 100644 index 078fa3768c1..00000000000 --- a/build.txt +++ /dev/null @@ -1,871 +0,0 @@ -Using pip 24.2 from /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages/pip (python 3.11) -Looking in indexes: https://repo.prod.m/artifactory/api/pypi/pegasus-311-1/simple/ -Obtaining file:///turbo/pmarkovski/ArcticDB - Checking if build backend supports build_editable: started - Running command Checking if build backend supports build_editable - Checking if build backend supports build_editable: finished with status 'done' - Preparing metadata (pyproject.toml): started - Running command Preparing metadata (pyproject.toml) - ARCTICDB_USING_CONDA=False - ARCTICDB_BUILD_CPP_TESTS=False - running dist_info - creating /tmp/pip-modern-metadata-7cse9pu8/arcticdb.egg-info - writing /tmp/pip-modern-metadata-7cse9pu8/arcticdb.egg-info/PKG-INFO - writing dependency_links to /tmp/pip-modern-metadata-7cse9pu8/arcticdb.egg-info/dependency_links.txt - writing entry points to /tmp/pip-modern-metadata-7cse9pu8/arcticdb.egg-info/entry_points.txt - writing requirements to /tmp/pip-modern-metadata-7cse9pu8/arcticdb.egg-info/requires.txt - writing top-level names to /tmp/pip-modern-metadata-7cse9pu8/arcticdb.egg-info/top_level.txt - writing manifest file '/tmp/pip-modern-metadata-7cse9pu8/arcticdb.egg-info/SOURCES.txt' - adding license file 'LICENSE.txt' - adding license file 'NOTICE.txt' - writing manifest file '/tmp/pip-modern-metadata-7cse9pu8/arcticdb.egg-info/SOURCES.txt' - creating '/tmp/pip-modern-metadata-7cse9pu8/arcticdb-0.0.0.dev0.dist-info' - Preparing metadata (pyproject.toml): finished with status 'done' -Requirement already satisfied: pandas<3 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (2.1.4+man91) -Requirement already satisfied: numpy in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (1.24.4) -Requirement already satisfied: attrs in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (23.1.0) -Requirement already satisfied: protobuf<7,>=3.5.0.post1 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (4.25.1) -Requirement already satisfied: msgpack>=0.5.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (1.0.5) -Requirement already satisfied: pyyaml in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (6.0.1) -Requirement already satisfied: packaging in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (24.2) -Requirement already satisfied: pytz in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (2022.1) -Requirement already satisfied: pytest in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (7.4.4) -Requirement already satisfied: polars in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (1.35.2) -Requirement already satisfied: pyarrow in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (20.0.0) -Requirement already satisfied: pytest-cpp in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (0.3.1) -Requirement already satisfied: pytest-timeout in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (2.4.0) -Requirement already satisfied: pytest-xdist in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (3.5.0) -Requirement already satisfied: future in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (0.18.2) -Requirement already satisfied: mock in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (5.0.2) -Requirement already satisfied: boto3 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (1.28.34) -Requirement already satisfied: moto in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (4.2.0) -Requirement already satisfied: flask in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (2.3.3) -Requirement already satisfied: flask-cors in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (4.0.1) -Requirement already satisfied: hypothesis<6.73 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (4.39.3+ahl1) -Requirement already satisfied: azure-storage-blob in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (12.13.1) -Requirement already satisfied: azure-identity in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (1.10.0) -Requirement already satisfied: coverage in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (7.2.7) -Requirement already satisfied: virtualenv in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (20.27.1) -Requirement already satisfied: pymongo in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (3.13.0+man2) -Requirement already satisfied: trustme in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (1.1.0) -Requirement already satisfied: psutil in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (5.9.8) -Requirement already satisfied: retrying in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (1.4.1) -Requirement already satisfied: asv in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from arcticdb==0.0.0.dev0) (0.6.2) -Requirement already satisfied: python-dateutil>=2.8.2 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from pandas<3->arcticdb==0.0.0.dev0) (2.8.2+ahl1) -Requirement already satisfied: tzdata>=2022.1 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from pandas<3->arcticdb==0.0.0.dev0) (2024.1) -Requirement already satisfied: asv-runner>=v0.2.1 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from asv->arcticdb==0.0.0.dev0) (0.2.1) -Requirement already satisfied: json5 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from asv->arcticdb==0.0.0.dev0) (0.9.28) -Requirement already satisfied: tabulate in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from asv->arcticdb==0.0.0.dev0) (0.9.0) -Requirement already satisfied: tomli in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from asv->arcticdb==0.0.0.dev0) (1.2.3) -Requirement already satisfied: pympler in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from asv->arcticdb==0.0.0.dev0) (1.1) -Requirement already satisfied: azure-core<2.0.0,>=1.11.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from azure-identity->arcticdb==0.0.0.dev0) (1.24.2) -Requirement already satisfied: cryptography>=2.5 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from azure-identity->arcticdb==0.0.0.dev0) (42.0.0) -Requirement already satisfied: msal<2.0.0,>=1.12.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from azure-identity->arcticdb==0.0.0.dev0) (1.28.1) -Requirement already satisfied: msal-extensions<2.0.0,>=0.3.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from azure-identity->arcticdb==0.0.0.dev0) (1.0.0) -Requirement already satisfied: six>=1.12.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from azure-identity->arcticdb==0.0.0.dev0) (1.16.0) -Requirement already satisfied: msrest>=0.6.21 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from azure-storage-blob->arcticdb==0.0.0.dev0) (0.7.1) -Requirement already satisfied: botocore<1.32.0,>=1.31.34 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from boto3->arcticdb==0.0.0.dev0) (1.31.34) -Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from boto3->arcticdb==0.0.0.dev0) (1.0.1) -Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from boto3->arcticdb==0.0.0.dev0) (0.6.2) -Requirement already satisfied: Werkzeug>=2.3.7 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from flask->arcticdb==0.0.0.dev0) (2.3.8) -Requirement already satisfied: Jinja2>=3.1.2 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from flask->arcticdb==0.0.0.dev0) (3.1.4) -Requirement already satisfied: itsdangerous>=2.1.2 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from flask->arcticdb==0.0.0.dev0) (2.1.2) -Requirement already satisfied: click>=8.1.3 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from flask->arcticdb==0.0.0.dev0) (8.1.7) -Requirement already satisfied: blinker>=1.6.2 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from flask->arcticdb==0.0.0.dev0) (1.7.0) -Requirement already satisfied: requests>=2.5 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from moto->arcticdb==0.0.0.dev0) (2.31.0) -Requirement already satisfied: xmltodict in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from moto->arcticdb==0.0.0.dev0) (0.14.2) -Requirement already satisfied: responses>=0.13.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from moto->arcticdb==0.0.0.dev0) (0.23.3) -Requirement already satisfied: polars-runtime-32==1.35.2 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from polars->arcticdb==0.0.0.dev0) (1.35.2) -Requirement already satisfied: iniconfig in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from pytest->arcticdb==0.0.0.dev0) (2.0.0) -Requirement already satisfied: pluggy<2.0,>=0.12 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from pytest->arcticdb==0.0.0.dev0) (1.2.0) -Requirement already satisfied: colorama in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from pytest-cpp->arcticdb==0.0.0.dev0) (0.4.6) -Requirement already satisfied: execnet>=1.1 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from pytest-xdist->arcticdb==0.0.0.dev0) (1.9.0) -Requirement already satisfied: idna>=2.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from trustme->arcticdb==0.0.0.dev0) (3.4) -Requirement already satisfied: distlib<1,>=0.3.7 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from virtualenv->arcticdb==0.0.0.dev0) (0.3.9) -Requirement already satisfied: filelock<4,>=3.12.2 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from virtualenv->arcticdb==0.0.0.dev0) (3.16.1) -Requirement already satisfied: platformdirs<5,>=3.9.1 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from virtualenv->arcticdb==0.0.0.dev0) (4.5.0) -Requirement already satisfied: importlib-metadata in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from asv-runner>=v0.2.1->asv->arcticdb==0.0.0.dev0) (6.11.0) -Requirement already satisfied: typing-extensions>=4.0.1 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from azure-core<2.0.0,>=1.11.0->azure-identity->arcticdb==0.0.0.dev0) (4.15.0) -Requirement already satisfied: urllib3<1.27,>=1.25.4 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from botocore<1.32.0,>=1.31.34->boto3->arcticdb==0.0.0.dev0) (1.26.18) -Requirement already satisfied: cffi>=1.12 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from cryptography>=2.5->azure-identity->arcticdb==0.0.0.dev0) (1.15.1) -Requirement already satisfied: MarkupSafe>=2.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from Jinja2>=3.1.2->flask->arcticdb==0.0.0.dev0) (2.1.5) -Requirement already satisfied: PyJWT<3,>=1.0.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from PyJWT[crypto]<3,>=1.0.0->msal<2.0.0,>=1.12.0->azure-identity->arcticdb==0.0.0.dev0) (2.9.0) -Requirement already satisfied: portalocker<3,>=1.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from msal-extensions<2.0.0,>=0.3.0->azure-identity->arcticdb==0.0.0.dev0) (2.7.0) -Requirement already satisfied: certifi>=2017.4.17 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from msrest>=0.6.21->azure-storage-blob->arcticdb==0.0.0.dev0) (2023.7.22) -Requirement already satisfied: isodate>=0.6.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from msrest>=0.6.21->azure-storage-blob->arcticdb==0.0.0.dev0) (0.7.2) -Requirement already satisfied: requests-oauthlib>=0.5.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from msrest>=0.6.21->azure-storage-blob->arcticdb==0.0.0.dev0) (1.3.1) -Requirement already satisfied: charset-normalizer<4,>=2 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from requests>=2.5->moto->arcticdb==0.0.0.dev0) (3.2.0) -Requirement already satisfied: types-PyYAML in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from responses>=0.13.0->moto->arcticdb==0.0.0.dev0) (6.0.12.10) -Requirement already satisfied: pycparser in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from cffi>=1.12->cryptography>=2.5->azure-identity->arcticdb==0.0.0.dev0) (2.21) -Requirement already satisfied: oauthlib>=3.0.0 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from requests-oauthlib>=0.5.0->msrest>=0.6.21->azure-storage-blob->arcticdb==0.0.0.dev0) (3.0.1) -Requirement already satisfied: zipp>=0.5 in /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages (from importlib-metadata->asv-runner>=v0.2.1->asv->arcticdb==0.0.0.dev0) (3.5.1) -Installing collected packages: arcticdb - Attempting uninstall: arcticdb - Found existing installation: arcticdb 0.0.0.dev0 - Uninstalling arcticdb-0.0.0.dev0: - Removing file or directory /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/bin/arcticdb_update_storage - Removing file or directory /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages/arcticdb.egg-link - Removing pth entries from /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages/easy-install.pth: - Removing entry: /turbo/pmarkovski/ArcticDB/python - Successfully uninstalled arcticdb-0.0.0.dev0 - DEPRECATION: Legacy editable install of arcticdb[testing]==0.0.0.dev0 from file:///turbo/pmarkovski/ArcticDB (setup.py develop) is deprecated. pip 25.0 will enforce this behaviour change. A possible replacement is to add a pyproject.toml or enable --use-pep517, and use setuptools >= 64. If the resulting installation is not behaving as expected, try using --config-settings editable_mode=compat. Please consult the setuptools documentation for more information. Discussion can be found at https://github.com/pypa/pip/issues/11457 - Running setup.py develop for arcticdb - Running command python setup.py develop - ARCTICDB_USING_CONDA=False - ARCTICDB_BUILD_CPP_TESTS=False - running develop - /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages/setuptools/command/easy_install.py:144: EasyInstallDeprecationWarning: easy_install command is deprecated. Use build and pip and other standards-based tools. - warnings.warn( - /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages/setuptools/command/install.py:34: SetuptoolsDeprecationWarning: setup.py install is deprecated. Use build and pip and other standards-based tools. - warnings.warn( - running egg_info - writing python/arcticdb.egg-info/PKG-INFO - writing dependency_links to python/arcticdb.egg-info/dependency_links.txt - writing entry points to python/arcticdb.egg-info/entry_points.txt - writing requirements to python/arcticdb.egg-info/requires.txt - writing top-level names to python/arcticdb.egg-info/top_level.txt - adding license file 'LICENSE.txt' - adding license file 'NOTICE.txt' - writing manifest file 'python/arcticdb.egg-info/SOURCES.txt' - running build_ext - Preset CMake variables: - - CMAKE_BUILD_TYPE="RelWithDebInfo" - CMAKE_MAKE_PROGRAM="make" - CMAKE_TOOLCHAIN_FILE:FILEPATH="vcpkg/scripts/buildsystems/vcpkg.cmake" - - Preset environment variables: - - cmakepreset_expected_host_system="Linux" - - -- Running vcpkg install - A suitable version of cmake was not found (required v3.31.10). - Downloading https://github.com/Kitware/CMake/releases/download/v3.31.10/cmake-3.31.10-linux-x86_64.tar.gz -> cmake-3.31.10-linux-x86_64.tar.gz - Successfully downloaded cmake-3.31.10-linux-x86_64.tar.gz - Extracting cmake... - Detecting compiler hash for triplet x64-linux... - Compiler found: /usr/bin/c++ - The following packages are already installed: - * abseil:x64-linux@20250814.1 - arcticdb-sparrow:x64-linux@2.3.0 - arrow:x64-linux@21.0.0#2 - aws-c-auth:x64-linux@0.7.31 - aws-c-cal:x64-linux@0.7.4 - aws-c-common:x64-linux@0.9.28 - * aws-c-compression:x64-linux@0.3.2 - aws-c-event-stream:x64-linux@0.4.3 - aws-c-http:x64-linux@0.8.10 - aws-c-io:x64-linux@0.14.18 - aws-c-mqtt:x64-linux@0.11.0 - aws-c-s3:x64-linux@0.6.6 - aws-c-sdkutils:x64-linux@0.1.19 - aws-checksums:x64-linux@0.1.20 - aws-crt-cpp:x64-linux@0.29.7 - aws-sdk-cpp[cognito-identity,core,identity-management,s3,sts]:x64-linux@1.11.474 -- /turbo/pmarkovski/ArcticDB/cpp/third_party/vcpkg_overlays/aws-sdk-cpp - azure-core-cpp[core,curl]:x64-linux@1.12.0 - azure-identity-cpp:x64-linux@1.6.0 - azure-storage-blobs-cpp:x64-linux@12.16.0 - * azure-storage-common-cpp:x64-linux@12.12.0 - benchmark:x64-linux@1.9.0 - bitmagic:x64-linux@7.13.4 -- /turbo/pmarkovski/ArcticDB/cpp/third_party/vcpkg_overlays/bitmagic - * boost-algorithm:x64-linux@1.90.0#1 - * boost-align:x64-linux@1.90.0#1 - * boost-any:x64-linux@1.90.0#1 - * boost-array:x64-linux@1.90.0#1 - * boost-assert:x64-linux@1.90.0#1 - * boost-atomic:x64-linux@1.90.0#1 - * boost-bind:x64-linux@1.90.0#1 - boost-callable-traits:x64-linux@1.90.0#1 - * boost-charconv:x64-linux@1.90.0#1 - * boost-chrono:x64-linux@1.90.0#1 - boost-circular-buffer:x64-linux@1.90.0#1 - * boost-cmake:x64-linux@1.90.0#1 - * boost-concept-check:x64-linux@1.90.0#1 - * boost-config:x64-linux@1.90.0#1 - * boost-container:x64-linux@1.90.0#1 - * boost-container-hash:x64-linux@1.90.0#1 - * boost-context:x64-linux@1.90.0#1 - * boost-conversion:x64-linux@1.90.0#1 - * boost-core:x64-linux@1.90.0#1 - * boost-crc:x64-linux@1.90.0#1 - * boost-date-time:x64-linux@1.90.0#1 - * boost-describe:x64-linux@1.90.0#1 - * boost-detail:x64-linux@1.90.0#1 - boost-dynamic-bitset:x64-linux@1.90.0#1 - * boost-exception:x64-linux@1.90.0#1 - * boost-filesystem:x64-linux@1.90.0#1 - * boost-function:x64-linux@1.90.0#1 - * boost-function-types:x64-linux@1.90.0#1 - * boost-functional:x64-linux@1.90.0#1 - * boost-fusion:x64-linux@1.90.0#1 - * boost-headers:x64-linux@1.90.0#1 - * boost-integer:x64-linux@1.90.0#1 - boost-interprocess:x64-linux@1.90.0#1 - * boost-intrusive:x64-linux@1.90.0#1 - * boost-io:x64-linux@1.90.0#1 - * boost-iterator:x64-linux@1.90.0#1 - * boost-lexical-cast:x64-linux@1.90.0#1 - * boost-locale:x64-linux@1.90.0#1 - * boost-math:x64-linux@1.90.0#1 - * boost-move:x64-linux@1.90.0#1 - * boost-mp11:x64-linux@1.90.0#1 - * boost-mpl:x64-linux@1.90.0#1 - * boost-multi-index:x64-linux@1.90.0#1 - boost-multiprecision:x64-linux@1.90.0#1 - * boost-numeric-conversion:x64-linux@1.90.0#1 - boost-optional:x64-linux@1.90.0#1 - * boost-pool:x64-linux@1.90.0#1 - * boost-predef:x64-linux@1.90.0#1 - * boost-preprocessor:x64-linux@1.90.0#1 - * boost-program-options:x64-linux@1.90.0#1 - * boost-random:x64-linux@1.90.0#1 - * boost-range:x64-linux@1.90.0#1 - * boost-ratio:x64-linux@1.90.0#1 - * boost-regex:x64-linux@1.90.0#1 - * boost-scope:x64-linux@1.90.0#1 - * boost-scope-exit:x64-linux@1.90.0#1 - * boost-smart-ptr:x64-linux@1.90.0#1 - * boost-static-assert:x64-linux@1.90.0#1 - * boost-system:x64-linux@1.90.0#1 - * boost-thread:x64-linux@1.90.0#1 - * boost-throw-exception:x64-linux@1.90.0#1 - * boost-tokenizer:x64-linux@1.90.0#1 - * boost-tuple:x64-linux@1.90.0#1 - * boost-type-index:x64-linux@1.90.0#1 - * boost-type-traits:x64-linux@1.90.0#1 - * boost-typeof:x64-linux@1.90.0#1 - * boost-uninstall:x64-linux@1.90.0#1 - * boost-unordered:x64-linux@1.90.0#1 - * boost-utility:x64-linux@1.90.0#1 - * boost-uuid:x64-linux@1.90.0#1 - * boost-variant:x64-linux@1.90.0#1 - * boost-variant2:x64-linux@1.90.0#1 - * boost-winapi:x64-linux@1.90.0#1 - * brotli:x64-linux@1.2.0 - * bzip2[core,tool]:x64-linux@1.0.8#6 - * civetweb:x64-linux@1.16#2 - curl[core,non-http,openssl,ssl]:x64-linux@8.18.0#1 - * date:x64-linux@3.0.4 - double-conversion:x64-linux@3.2.1 - * fast-float:x64-linux@8.2.3 - fmt:x64-linux@10.2.1 - folly[core,lz4]:x64-linux@2025.04.14.00 -- /turbo/pmarkovski/ArcticDB/cpp/third_party/vcpkg_overlays/folly - * gflags:x64-linux@2.2.2#5 - * glog:x64-linux@0.6.0#2 - gtest:x64-linux@1.12.1 - * libbson:x64-linux@1.30.3 - libevent[core,thread]:x64-linux@2.1.12#7 - * libiconv:x64-linux@1.17 - * liblzma:x64-linux@5.8.2 - * libxml2[core,iconv,zlib]:x64-linux@2.15.1 - * lz4:x64-linux@1.9.3#4 - * mongo-c-driver[core,openssl]:x64-linux@1.28.0 - mongo-cxx-driver:x64-linux@3.10.2#1 - openssl:x64-linux@3.3.0 - pcre2[core,jit,platform-default-features]:x64-linux@10.47 - prometheus-cpp[core,pull,push]:x64-linux@1.1.0 - protobuf[core,zlib]:x64-linux@3.21.8 - rapidcheck:x64-linux@2023-12-14 - * re2:x64-linux@2025-11-05 - * s2n:x64-linux@1.3.5 - * snappy:x64-linux@1.2.2#1 - spdlog:x64-linux@1.13.0 - * thrift:x64-linux@0.22.0 - unordered-dense:x64-linux@4.8.1 - * utf8proc:x64-linux@2.11.3 - * vcpkg-boost:x64-linux@2025-03-29 - * vcpkg-cmake:x64-linux@2024-04-23 - * vcpkg-cmake-config:x64-linux@2024-05-23 - * vcpkg-cmake-get-vars:x64-linux@2025-05-29 - * xsimd:x64-linux@14.0.0 - xxhash:x64-linux@0.8.2 - * zlib:x64-linux@1.3.1 - zstd:x64-linux@1.5.2 - arcticdb-sparrow provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(sparrow CONFIG REQUIRED) - target_link_libraries(main PRIVATE sparrow::sparrow) - - zstd provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(zstd CONFIG REQUIRED) - target_link_libraries(main PRIVATE zstd::libzstd_static) - - zstd provides pkg-config modules: - - # fast lossless compression algorithm library - libzstd - - openssl is compatible with built-in CMake targets: - - find_package(OpenSSL REQUIRED) - target_link_libraries(main PRIVATE OpenSSL::SSL) - target_link_libraries(main PRIVATE OpenSSL::Crypto) - - libevent provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(Libevent CONFIG REQUIRED) - target_link_libraries(main PRIVATE libevent::core libevent::extra libevent::pthreads) - - libevent provides pkg-config modules: - - # libevent is an asynchronous notification event loop library - libevent - - # libevent_core - libevent_core - - # libevent_extra - libevent_extra - - # libevent_pthreads adds pthreads-based threading support to libevent - libevent_pthreads - - boost-optional provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(boost_optional CONFIG REQUIRED) - target_link_libraries(main PRIVATE Boost::optional) - - boost-dynamic-bitset provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(boost_dynamic_bitset CONFIG REQUIRED) - target_link_libraries(main PRIVATE Boost::dynamic_bitset) - - boost-multiprecision provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(boost_multiprecision CONFIG REQUIRED) - target_link_libraries(main PRIVATE Boost::multiprecision) - - The package arrow provides CMake targets: - - find_package(Arrow CONFIG REQUIRED) - target_link_libraries(main PRIVATE "$,Arrow::arrow_static,Arrow::arrow_shared>") - - aws-c-common provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(aws-c-common CONFIG REQUIRED) - target_link_libraries(main PRIVATE AWS::aws-c-common) - - aws-c-sdkutils provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(aws-c-sdkutils CONFIG REQUIRED) - target_link_libraries(main PRIVATE AWS::aws-c-sdkutils) - - aws-c-cal provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(aws-c-cal CONFIG REQUIRED) - target_link_libraries(main PRIVATE AWS::aws-c-cal) - - aws-c-io provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(aws-c-io CONFIG REQUIRED) - target_link_libraries(main PRIVATE AWS::aws-c-io) - - aws-c-http provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(aws-c-http CONFIG REQUIRED) - target_link_libraries(main PRIVATE AWS::aws-c-http) - - aws-c-auth provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(aws-c-auth CONFIG REQUIRED) - target_link_libraries(main PRIVATE AWS::aws-c-auth) - - aws-checksums provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(aws-checksums CONFIG REQUIRED) - target_link_libraries(main PRIVATE AWS::aws-checksums) - - aws-c-event-stream provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(aws-c-event-stream CONFIG REQUIRED) - target_link_libraries(main PRIVATE AWS::aws-c-event-stream) - - aws-c-mqtt provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(aws-c-mqtt CONFIG REQUIRED) - target_link_libraries(main PRIVATE AWS::aws-c-mqtt) - - aws-c-s3 provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(aws-c-s3 CONFIG REQUIRED) - target_link_libraries(main PRIVATE AWS::aws-c-s3) - - aws-crt-cpp provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(aws-crt-cpp CONFIG REQUIRED) - target_link_libraries(main PRIVATE AWS::aws-crt-cpp) - - curl is compatible with built-in CMake targets: - - find_package(CURL REQUIRED) - target_link_libraries(main PRIVATE CURL::libcurl) - - The package aws-sdk-cpp:x64-linux provides CMake targets: - - When using AWSSDK, AWSSDK_ROOT_DIR must be defined by the user. - find_package(AWSSDK CONFIG COMPONENTS core dynamodb kinesis s3 REQUIRED) - target_include_directories(main PRIVATE ${AWSSDK_INCLUDE_DIRS}) - target_link_libraries(main PRIVATE ${AWSSDK_LIBRARIES}) - - OR - - find_package(aws-cpp-sdk-core REQUIRED) - target_include_directories(main PRIVATE aws-cpp-sdk-core) - target_link_libraries(main PRIVATE aws-cpp-sdk-core) - - azure-core-cpp provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(azure-core-cpp CONFIG REQUIRED) - target_link_libraries(main PRIVATE Azure::azure-core) - - azure-identity-cpp provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(azure-identity-cpp CONFIG REQUIRED) - target_link_libraries(main PRIVATE Azure::azure-identity) - - azure-storage-blobs-cpp provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(azure-storage-blobs-cpp CONFIG REQUIRED) - target_link_libraries(main PRIVATE Azure::azure-storage-blobs) - - benchmark provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(benchmark CONFIG REQUIRED) - target_link_libraries(main PRIVATE benchmark::benchmark benchmark::benchmark_main) - - benchmark provides pkg-config modules: - - # Google microbenchmark framework - benchmark - - # Google microbenchmark framework (with main() function) - benchmark_main - - bitmagic is header-only and can be used from CMake via: - - find_path(BITMAGIC_INCLUDE_DIRS "bitmagic/bm.h") - target_include_directories(main PRIVATE ${BITMAGIC_INCLUDE_DIRS}) - - boost-callable-traits provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(boost_callable_traits CONFIG REQUIRED) - target_link_libraries(main PRIVATE Boost::callable_traits) - - boost-circular-buffer provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(boost_circular_buffer CONFIG REQUIRED) - target_link_libraries(main PRIVATE Boost::circular_buffer) - - boost-interprocess provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(boost_interprocess CONFIG REQUIRED) - target_link_libraries(main PRIVATE Boost::interprocess) - - double-conversion provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(double-conversion CONFIG REQUIRED) - target_link_libraries(main PRIVATE double-conversion::double-conversion) - - The package fmt provides CMake targets: - - find_package(fmt CONFIG REQUIRED) - target_link_libraries(main PRIVATE fmt::fmt) - - # Or use the header-only version - find_package(fmt CONFIG REQUIRED) - target_link_libraries(main PRIVATE fmt::fmt-header-only) - - folly provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(folly CONFIG REQUIRED) - target_link_libraries(main PRIVATE Folly::folly Folly::folly_deps Folly::follybenchmark Folly::folly_test_util) - - folly provides pkg-config modules: - - # Facebook (Folly) C++ library - libfolly - - gtest provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(GTest CONFIG REQUIRED) - target_link_libraries(main PRIVATE GTest::gmock GTest::gtest GTest::gmock_main GTest::gtest_main) - - gtest provides pkg-config modules: - - # GoogleMock (without main() function) - gmock - - # GoogleMock (with main() function) - gmock_main - - # GoogleTest (without main() function) - gtest - - # GoogleTest (with main() function) - gtest_main - - mongo-cxx-driver provides CMake targets: - - find_package(bsoncxx CONFIG REQUIRED) - target_link_libraries(main PRIVATE $,mongo::bsoncxx_static,mongo::bsoncxx_shared>) - - find_package(mongocxx CONFIG REQUIRED) - target_link_libraries(main PRIVATE $,mongo::mongocxx_static,mongo::mongocxx_shared>) - - The package pcre2 is compatible with built-in CMake targets: - - # Each component imports a target: - # TARGETS: PCRE2::8BIT PCRE2::16BIT PCRE2::32BIT PCRE2::POSIX - find_package(pcre2 CONFIG REQUIRED) - target_link_libraries(main PRIVATE PCRE2::8BIT PCRE2::16BIT PCRE2::32BIT PCRE2::POSIX) - - prometheus-cpp provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(prometheus-cpp CONFIG REQUIRED) - target_link_libraries(main PRIVATE prometheus-cpp::core prometheus-cpp::pull prometheus-cpp::push) - - protobuf provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(protobuf CONFIG REQUIRED) - target_link_libraries(main PRIVATE protobuf::libprotoc protobuf::libprotobuf protobuf::libprotobuf-lite) - - protobuf provides pkg-config modules: - - # Google's Data Interchange Format - protobuf-lite - - # Google's Data Interchange Format - protobuf - - rapidcheck provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(rapidcheck CONFIG REQUIRED) - # note: 3 additional targets are not displayed. - target_link_libraries(main PRIVATE rapidcheck rapidcheck_boost rapidcheck_catch rapidcheck_gmock) - - rapidcheck provides pkg-config modules: - - # C++ framework for property based testing inspired by QuickCheck and other similar frameworks - rapidcheck - - # boost headers for rapidcheck - rapidcheck_boost - - # boost_test headers for rapidcheck - rapidcheck_boost_test - - # catch headers for rapidcheck - rapidcheck_catch - - # doctest headers for rapidcheck - rapidcheck_doctest - - # gmock headers for rapidcheck - rapidcheck_gtest - - The package spdlog provides CMake targets: - - find_package(spdlog CONFIG REQUIRED) - target_link_libraries(main PRIVATE spdlog::spdlog) - - # Or use the header-only version - find_package(spdlog CONFIG REQUIRED) - target_link_libraries(main PRIVATE spdlog::spdlog_header_only) - - unordered-dense provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(unordered_dense CONFIG REQUIRED) - target_link_libraries(main PRIVATE unordered_dense::unordered_dense) - - xxhash provides CMake targets: - - # this is heuristically generated, and may not be correct - find_package(xxHash CONFIG REQUIRED) - target_link_libraries(main PRIVATE xxHash::xxhash) - - xxhash provides pkg-config modules: - - # extremely fast hash algorithm - libxxhash - - All requested installations completed successfully in: 1.54 ms - -- Running vcpkg install - done - Python_EXECUTABLE is set (to '/turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/bin/python3'), which can influence FindPython and pybind11. - CMake Warning at CMake/PythonUtils.cmake:72 (message): - PYTHON_LIBRARIES is set (to - '/turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/libpython3.11.so'). It - won't be used to build ArcticDB but may cause third-party packages to find - a different Python and break the build. - Call Stack (most recent call first): - CMake/PythonUtils.cmake:79 (_body) - CMake/PythonUtils.cmake:122 (_python_utils_check_vars) - CMakeLists.txt:193 (include) - - - CMake Warning at CMake/PythonUtils.cmake:72 (message): - PYTHON_LIBRARY is set (to - '/turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/libpython3.11.so'). It - won't be used to build ArcticDB but may cause third-party packages to find - a different Python and break the build. - Call Stack (most recent call first): - CMake/PythonUtils.cmake:79 (_body) - CMake/PythonUtils.cmake:122 (_python_utils_check_vars) - CMakeLists.txt:193 (include) - - - CMake Warning at CMake/PythonUtils.cmake:72 (message): - PYTHON_INCLUDE_DIRS is set (to - '/turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/include/python3.11'). It won't - be used to build ArcticDB but may cause third-party packages to find a - different Python and break the build. - Call Stack (most recent call first): - CMake/PythonUtils.cmake:79 (_body) - CMake/PythonUtils.cmake:122 (_python_utils_check_vars) - CMakeLists.txt:193 (include) - - - CMake Deprecation Warning at third_party/pybind11/CMakeLists.txt:13 (cmake_minimum_required): - Compatibility with CMake < 3.10 will be removed from a future version of - CMake. - - Update the VERSION argument value. Or, use the ... syntax - to tell CMake that the project requires at least but has been updated - to work with policies introduced by or earlier. - - - -- pybind11 v2.13.6 - CMake Warning (dev) at third_party/pybind11/tools/FindPythonLibsNew.cmake:101 (message): - Policy CMP0148 is not set: The FindPythonInterp and FindPythonLibs modules - are removed. Run "cmake --help-policy CMP0148" for policy details. Use - the cmake_policy command to set the policy and suppress this warning, or - preferably upgrade to using FindPython, either by calling it explicitly - before pybind11, or by setting PYBIND11_FINDPYTHON ON before pybind11. - Call Stack (most recent call first): - vcpkg/scripts/buildsystems/vcpkg.cmake:908 (_find_package) - third_party/pybind11/tools/pybind11Tools.cmake:50 (find_package) - third_party/pybind11/tools/pybind11Common.cmake:228 (include) - third_party/pybind11/CMakeLists.txt:251 (include) - This warning is for project developers. Use -Wno-dev to suppress it. - - CMake Warning (dev) at vcpkg/scripts/buildsystems/vcpkg.cmake:908 (_find_package): - Policy CMP0148 is not set: The FindPythonInterp and FindPythonLibs modules - are removed. Run "cmake --help-policy CMP0148" for policy details. Use - the cmake_policy command to set the policy and suppress this warning. - - Call Stack (most recent call first): - third_party/pybind11/tools/FindPythonLibsNew.cmake:114 (find_package) - vcpkg/scripts/buildsystems/vcpkg.cmake:908 (_find_package) - third_party/pybind11/tools/pybind11Tools.cmake:50 (find_package) - third_party/pybind11/tools/pybind11Common.cmake:228 (include) - third_party/pybind11/CMakeLists.txt:251 (include) - This warning is for project developers. Use -Wno-dev to suppress it. - - CMake Deprecation Warning at third_party/semimap/CMakeLists.txt:1 (cmake_minimum_required): - Compatibility with CMake < 3.10 will be removed from a future version of - CMake. - - Update the VERSION argument value. Or, use the ... syntax - to tell CMake that the project requires at least but has been updated - to work with policies introduced by or earlier. - - - CMake Deprecation Warning at third_party/msgpack-c/CMakeLists.txt:1 (CMAKE_MINIMUM_REQUIRED): - Compatibility with CMake < 3.10 will be removed from a future version of - CMake. - - Update the VERSION argument value. Or, use the ... syntax - to tell CMake that the project requires at least but has been updated - to work with policies introduced by or earlier. - - - -- Dynamically linking with Boost - CMake Warning (dev) at out/linux-release-build/vcpkg_installed/x64-linux/share/boost/vcpkg-cmake-wrapper.cmake:3 (_find_package): - Policy CMP0167 is not set: The FindBoost module is removed. Run "cmake - --help-policy CMP0167" for policy details. Use the cmake_policy command to - set the policy and suppress this warning. - - Call Stack (most recent call first): - vcpkg/scripts/buildsystems/vcpkg.cmake:862 (include) - third_party/msgpack-c/CMakeLists.txt:79 (FIND_PACKAGE) - This warning is for project developers. Use -Wno-dev to suppress it. - - -- Found Boost: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/share/boost/BoostConfig.cmake (found version "1.90.0") - -- Could NOT find Doxygen (missing: DOXYGEN_EXECUTABLE) - -- Found Protobuf: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libprotobuf.a (found version "3.21.8") - -- Found AWS SDK for C++, Version: 1.11.474, Install Root:/turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux, Platform Prefix:, Platform Dependent Libraries: pthread;curl - -- Components specified for AWSSDK: s3;identity-management, application will be depending on libs: aws-cpp-sdk-s3;aws-cpp-sdk-identity-management;aws-cpp-sdk-cognito-identity;aws-cpp-sdk-sts;aws-cpp-sdk-core;AWS::aws-crt-cpp;AWS::aws-c-http;AWS::aws-c-mqtt;AWS::aws-c-cal;AWS::aws-c-auth;AWS::aws-c-common;AWS::aws-c-io;AWS::aws-checksums;AWS::aws-c-event-stream;AWS::aws-c-s3;AWS::aws-c-compression;AWS::aws-c-sdkutils;pthread;curl - -- Try finding aws-cpp-sdk-core - -- Found OpenSSL: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libcrypto.a (found version "3.3.0") found components: Crypto - -- Found ZLIB: optimized;/turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libz.a;debug;/turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/debug/lib/libz.a (found version "1.3.1") - -- Found OpenSSL: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libcrypto.a (found suitable version "3.3.0", minimum required is "3") - -- Found ZLIB: optimized;/turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libz.a;debug;/turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/debug/lib/libz.a (found suitable version "1.3.1", minimum required is "1") - -- Found aws-cpp-sdk-core - -- Try finding aws-cpp-sdk-sts - -- Found aws-cpp-sdk-sts - -- Try finding aws-cpp-sdk-cognito-identity - -- Found aws-cpp-sdk-cognito-identity - -- Try finding aws-cpp-sdk-identity-management - -- Found aws-cpp-sdk-identity-management - -- Try finding aws-cpp-sdk-s3 - -- Found aws-cpp-sdk-s3 - CMake Warning (dev) at out/linux-release-build/vcpkg_installed/x64-linux/share/boost/vcpkg-cmake-wrapper.cmake:3 (_find_package): - Policy CMP0167 is not set: The FindBoost module is removed. Run "cmake - --help-policy CMP0167" for policy details. Use the cmake_policy command to - set the policy and suppress this warning. - - Call Stack (most recent call first): - vcpkg/scripts/buildsystems/vcpkg.cmake:862 (include) - arcticdb/CMakeLists.txt:19 (find_package) - This warning is for project developers. Use -Wno-dev to suppress it. - - -- Found libevent include directory: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/include - -- Found libevent component: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libevent_core.a - -- Found libevent component: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libevent_extra.a - -- Found libevent component: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libevent_pthreads.a - -- Found libevent 2.1.12 in /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux - CMake Warning (dev) at out/linux-release-build/vcpkg_installed/x64-linux/share/boost/vcpkg-cmake-wrapper.cmake:3 (_find_package): - Policy CMP0167 is not set: The FindBoost module is removed. Run "cmake - --help-policy CMP0167" for policy details. Use the cmake_policy command to - set the policy and suppress this warning. - - Call Stack (most recent call first): - vcpkg/scripts/buildsystems/vcpkg.cmake:862 (include) - /apps/research/tools/cmake/3.31.3/share/cmake-3.31/Modules/CMakeFindDependencyMacro.cmake:76 (find_package) - out/linux-release-build/vcpkg_installed/x64-linux/share/folly/folly-config.cmake:80 (find_dependency) - vcpkg/scripts/buildsystems/vcpkg.cmake:908 (_find_package) - arcticdb/CMakeLists.txt:23 (find_package) - This warning is for project developers. Use -Wno-dev to suppress it. - - -- Found Boost: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/share/boost/BoostConfig.cmake (found version "1.90.0") found components: context filesystem program_options regex system thread - -- Found folly: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux - CMake Warning (dev) at /apps/research/tools/cmake/3.31.3/share/cmake-3.31/Modules/FindPackageHandleStandardArgs.cmake:441 (message): - The package name passed to `find_package_handle_standard_args` (Protobuf) - does not match the name of the calling package (protobuf). This can lead - to problems in calling code that expects `find_package` result variables - (e.g., `_FOUND`) to follow a certain pattern. - Call Stack (most recent call first): - out/linux-release-build/vcpkg_installed/x64-linux/share/protobuf/protobuf-module.cmake:162 (FIND_PACKAGE_HANDLE_STANDARD_ARGS) - out/linux-release-build/vcpkg_installed/x64-linux/share/protobuf/protobuf-config.cmake:170 (include) - out/linux-release-build/vcpkg_installed/x64-linux/share/protobuf/vcpkg-cmake-wrapper.cmake:16 (_find_package) - vcpkg/scripts/buildsystems/vcpkg.cmake:862 (include) - arcticdb/CMakeLists.txt:37 (find_package) - This warning is for project developers. Use -Wno-dev to suppress it. - - -- Found Protobuf: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/tools/protobuf/protoc (found version "3.21.8.0") - -- Found OpenSSL: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libcrypto.a (found version "3.3.0") - CMake Warning at out/linux-release-build/vcpkg_installed/x64-linux/share/unofficial-utf8proc/unofficial-utf8proc-config.cmake:1 (message): - find_package(unofficial-utf8proc) is deprecated. - - utf8proc provides CMake targets: - - - - find_package(utf8proc) - target_link_libraries(main PRIVATE utf8proc::utf8proc) - - Call Stack (most recent call first): - vcpkg/scripts/buildsystems/vcpkg.cmake:908 (_find_package) - /apps/research/tools/cmake/3.31.3/share/cmake-3.31/Modules/CMakeFindDependencyMacro.cmake:76 (find_package) - out/linux-release-build/vcpkg_installed/x64-linux/share/mongoc-1.0/mongoc-1.0-config.cmake:34 (find_dependency) - vcpkg/scripts/buildsystems/vcpkg.cmake:908 (_find_package) - /apps/research/tools/cmake/3.31.3/share/cmake-3.31/Modules/CMakeFindDependencyMacro.cmake:76 (find_package) - out/linux-release-build/vcpkg_installed/x64-linux/share/mongocxx/mongocxx-config.cmake:2 (find_dependency) - vcpkg/scripts/buildsystems/vcpkg.cmake:908 (_find_package) - arcticdb/CMakeLists.txt:38 (find_package) - - - -- Found libevent include directory: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/include - -- Found libevent component: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libevent_core.a - -- Found libevent component: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libevent_extra.a - -- Found libevent component: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux/lib/libevent_pthreads.a - -- Found libevent 2.1.12 in /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build/vcpkg_installed/x64-linux - -- Not setting debug flags - Linking against SSL - -- Configuring done (55.3s) - -- Generating done (0.3s) - -- Build files have been written to: /turbo/pmarkovski/ArcticDB/cpp/out/linux-release-build - [ 2%] Built target lmdb - [ 23%] Built target arcticdb_proto - [ 93%] Built target arcticdb_core_object - [ 93%] Built target arcticdb_core_static - [ 98%] Built target arcticdb_python - [100%] Built target arcticdb_ext - [100%] Built target install_arcticdb_ext - Destination: /turbo/pmarkovski/ArcticDB/python/arcticdb_ext.cpython-311-x86_64-linux-gnu.so - ARCTICDB_VCPKG_INSTALLED_DIR=None - Running /apps/research/tools/bin/cmake -DTEST=False -DBUILD_PYTHON_VERSION=3.11 -DCMAKE_INSTALL_PREFIX=/turbo/pmarkovski/ArcticDB/python --preset linux-release - Removed: cpp/vcpkg/buildtrees - Removed: cpp/vcpkg/downloads - Removed: cpp/vcpkg/packages - Running /apps/research/tools/bin/cmake --build cpp/out/linux-release-build -j 4 --target install_arcticdb_ext - Creating /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/lib/python3.11/site-packages/arcticdb.egg-link (link to python) - Adding arcticdb 0.0.0.dev0 to easy-install.pth file - Installing arcticdb_update_storage script to /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/bin - - Installed /turbo/pmarkovski/ArcticDB/python - running protoc - Looking in indexes: https://repo.prod.m/artifactory/api/pypi/pegasus-311-1/simple/ - [pip_auth] looking up repository credentials for repo.prod.m - [pip_auth] no pypi credentials found for repo.prod.m. User config has domains: set() - Collecting grpcio-tools>=1.49 - [pip_auth] looking up repository credentials for repo.prod.m - [pip_auth] no pypi credentials found for repo.prod.m. User config has domains: set() - Using cached https://repo.prod.m/artifactory/api/pypi/pegasus-311-1/grpcio-tools/1.56.2/grpcio_tools-1.56.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.6 MB) - [pip_auth] looking up repository credentials for repo.prod.m - [pip_auth] no pypi credentials found for repo.prod.m. User config has domains: set() - Collecting protobuf==4.* - [pip_auth] looking up repository credentials for repo.prod.m - [pip_auth] no pypi credentials found for repo.prod.m. User config has domains: set() - Using cached https://repo.prod.m/artifactory/api/pypi/pegasus-311-1/protobuf/4.25.1/protobuf-4.25.1-cp37-abi3-manylinux2014_x86_64.whl (294 kB) - [pip_auth] looking up repository credentials for repo.prod.m - [pip_auth] no pypi credentials found for repo.prod.m. User config has domains: set() - Collecting grpcio>=1.56.2 (from grpcio-tools>=1.49) - [pip_auth] looking up repository credentials for repo.prod.m - [pip_auth] no pypi credentials found for repo.prod.m. User config has domains: set() - Using cached https://repo.prod.m/artifactory/api/pypi/pegasus-311-1/grpcio/1.62.3/grpcio-1.62.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.5 MB) - [pip_auth] looking up repository credentials for repo.prod.m - [pip_auth] no pypi credentials found for repo.prod.m. User config has domains: set() - Collecting setuptools (from grpcio-tools>=1.49) - [pip_auth] looking up repository credentials for repo.prod.m - [pip_auth] no pypi credentials found for repo.prod.m. User config has domains: set() - Using cached https://repo.prod.m/artifactory/api/pypi/pegasus-311-1/setuptools/65.6.3/setuptools-65.6.3-py3-none-any.whl (1.2 MB) - Installing collected packages: setuptools, protobuf, grpcio, grpcio-tools - Successfully installed grpcio-1.62.3 grpcio-tools-1.56.2 protobuf-4.25.1 setuptools-65.6.3 - ['develop'] - - Protoc compilation (into 'python/arcticdb/proto') for versions '4': - Running /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/bin/python -mpip install --disable-pip-version-check --target=/tmp/tmp6jt8apcx grpcio-tools>=1.49 protobuf==4.* - Running /turbo/pmarkovski/pyenvs/ArcticDb_VirtEnv/bin/python -mgrpc_tools.protoc -Icpp/proto --python_out=python/arcticdb/proto/4 cpp/proto/arcticc/pb2/gcp_storage.proto cpp/proto/arcticc/pb2/utils.proto cpp/proto/arcticc/pb2/config.proto cpp/proto/arcticc/pb2/s3_storage.proto cpp/proto/arcticc/pb2/lmdb_storage.proto cpp/proto/arcticc/pb2/in_memory_storage.proto cpp/proto/arcticc/pb2/encoding.proto cpp/proto/arcticc/pb2/descriptors.proto cpp/proto/arcticc/pb2/nfs_backed_storage.proto cpp/proto/arcticc/pb2/storage.proto cpp/proto/arcticc/pb2/azure_storage.proto cpp/proto/arcticc/pb2/logger.proto cpp/proto/arcticc/pb2/mongo_storage.proto cpp/proto/arcticc/pb2/mapped_file_storage.proto -Successfully installed arcticdb-0.0.0.dev0 diff --git a/python/tests/integration/toolbox/test_storage_mover.py b/python/tests/integration/toolbox/test_storage_mover.py index a37a0d72d0a..730f3f6cf19 100644 --- a/python/tests/integration/toolbox/test_storage_mover.py +++ b/python/tests/integration/toolbox/test_storage_mover.py @@ -321,18 +321,18 @@ def test_correct_versions_in_destination( assert [k.version_id for k in lt.find_keys(KeyType.TABLE_DATA)] == [2, 3, 4, 4, 6] -# @settings(deadline=None, suppress_health_check=(hypothesis.HealthCheck.function_scoped_fixture,)) -# @given(to_copy=st.permutations(["s2", 4, 6]), existing=st.booleans()) -# def test_correct_versions_in_destination_force( -# to_copy, existing, check_single_threaded, lib_with_gaps_and_reused_keys, version_store_factory -# ): -# try: -# _tmp_test_body(to_copy, existing, lib_with_gaps_and_reused_keys, version_store_factory) -# except: -# import traceback - -# traceback.print_exc() -# raise +@settings(deadline=None, suppress_health_check=(hypothesis.HealthCheck.function_scoped_fixture,)) +@given(to_copy=st.permutations(["s2", 4, 6]), existing=st.booleans()) +def test_correct_versions_in_destination_force( + to_copy, existing, check_single_threaded, lib_with_gaps_and_reused_keys, version_store_factory +): + try: + _tmp_test_body(to_copy, existing, lib_with_gaps_and_reused_keys, version_store_factory) + except: + import traceback + + traceback.print_exc() + raise def _tmp_test_body(to_copy, existing, lib_with_gaps_and_reused_keys, version_store_factory): diff --git a/python/tests/unit/arcticdb/version_store/test_query_builder_sparse.py b/python/tests/unit/arcticdb/version_store/test_query_builder_sparse.py index cf9c279f9bb..4784142777f 100644 --- a/python/tests/unit/arcticdb/version_store/test_query_builder_sparse.py +++ b/python/tests/unit/arcticdb/version_store/test_query_builder_sparse.py @@ -177,44 +177,44 @@ def test_query_builder_sparse_dynamic_schema_type_change(lmdb_version_store_dyna assert_frame_equal(expected, received) -# @use_of_function_scoped_fixtures_in_hypothesis_checked -# @settings(deadline=None) -# @given( -# df=data_frames( -# columns( -# ["sparse1", "sparse2"], -# elements=strategies.floats(min_value=0, max_value=1000, allow_nan=False, allow_subnormal=False), -# fill=strategies.just(np.nan), -# ), -# ), -# ) -# def test_query_builder_sparse_hypothesis(lmdb_version_store_v1, df): -# assume(not df.empty and not df["sparse1"].isnull().all() and not df["sparse2"].isnull().all()) -# lib = lmdb_version_store_v1 -# sym = "test_query_builder_sparse_hypothesis" - -# df.index = pd.date_range("2024-01-01", periods=len(df)) - -# lib.write(sym, df, sparsify_floats=True) - -# # Filter -# expected = df[df["sparse1"].isnull()] -# q = QueryBuilder() -# q = q[q["sparse1"].isnull()] -# received = lib.read(sym, query_builder=q).data -# assert_frame_equal(expected, received) - -# # Projection -# expected = df -# expected["projected"] = expected["sparse1"] + expected["sparse2"] -# q = QueryBuilder() -# q = q.apply("projected", q["sparse1"] + q["sparse2"]) -# received = lib.read(sym, query_builder=q).data -# assert_frame_equal(expected, received) - -# # Groupby + aggregation -# expected = df.groupby("sparse1").agg({"sparse2": "sum"}) -# q = QueryBuilder().groupby("sparse1").agg({"sparse2": "sum"}) -# received = lib.read(sym, query_builder=q).data -# received.sort_index(inplace=True) -# assert_frame_equal(expected, received) +@use_of_function_scoped_fixtures_in_hypothesis_checked +@settings(deadline=None) +@given( + df=data_frames( + columns( + ["sparse1", "sparse2"], + elements=strategies.floats(min_value=0, max_value=1000, allow_nan=False, allow_subnormal=False), + fill=strategies.just(np.nan), + ), + ), +) +def test_query_builder_sparse_hypothesis(lmdb_version_store_v1, df): + assume(not df.empty and not df["sparse1"].isnull().all() and not df["sparse2"].isnull().all()) + lib = lmdb_version_store_v1 + sym = "test_query_builder_sparse_hypothesis" + + df.index = pd.date_range("2024-01-01", periods=len(df)) + + lib.write(sym, df, sparsify_floats=True) + + # Filter + expected = df[df["sparse1"].isnull()] + q = QueryBuilder() + q = q[q["sparse1"].isnull()] + received = lib.read(sym, query_builder=q).data + assert_frame_equal(expected, received) + + # Projection + expected = df + expected["projected"] = expected["sparse1"] + expected["sparse2"] + q = QueryBuilder() + q = q.apply("projected", q["sparse1"] + q["sparse2"]) + received = lib.read(sym, query_builder=q).data + assert_frame_equal(expected, received) + + # Groupby + aggregation + expected = df.groupby("sparse1").agg({"sparse2": "sum"}) + q = QueryBuilder().groupby("sparse1").agg({"sparse2": "sum"}) + received = lib.read(sym, query_builder=q).data + received.sort_index(inplace=True) + assert_frame_equal(expected, received) diff --git a/python/tests/unit/arcticdb/version_store/test_ternary.py b/python/tests/unit/arcticdb/version_store/test_ternary.py index 8e652e0cd34..3fc9deaf87a 100644 --- a/python/tests/unit/arcticdb/version_store/test_ternary.py +++ b/python/tests/unit/arcticdb/version_store/test_ternary.py @@ -1178,77 +1178,77 @@ def test_filter_ternary_dynamic_missing_columns(lmdb_version_store_dynamic_schem # TODO: Assert that the projected column is of type float64 after modify_schema change is merged -# @use_of_function_scoped_fixtures_in_hypothesis_checked -# @settings(deadline=None) -# @given( -# df=data_frames( -# columns( -# ["condition", "col1", "col2"], -# elements=strategies.floats(min_value=0, max_value=1000, allow_nan=False, allow_subnormal=False), -# fill=strategies.just(np.nan), -# ), -# ), -# ) -# def test_ternary_hypothesis(lmdb_version_store_v1, df, any_output_format): -# assume( -# not df.empty -# and not df["condition"].isnull().all() -# and not df["col1"].isnull().all() -# and not df["col2"].isnull().all() -# ) -# lib = lmdb_version_store_v1 -# lib._set_output_format_for_pipeline_tests(any_output_format) -# dense_sym = "test_ternary_hypothesis_dense" -# sparse_sym = "test_ternary_hypothesis_sparse" - -# df.index = pd.date_range("2024-01-01", periods=len(df)) - -# lib.write(dense_sym, df) -# lib.write(sparse_sym, df, sparsify_floats=True) - -# # Projection -# # col/col -# expected = df.copy(deep=True) -# expected["projected"] = np.where( -# expected["condition"].isnull().to_numpy(), expected["col1"].to_numpy(), expected["col2"].to_numpy() -# ) -# q = QueryBuilder() -# q = q.apply("projected", where(q["condition"].isnull(), q["col1"], q["col2"])) -# assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) -# assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) -# # col/val -# expected = df.copy(deep=True) -# expected["projected"] = np.where(expected["condition"].isnull().to_numpy(), expected["col1"].to_numpy(), 2000.0) -# q = QueryBuilder() -# q = q.apply("projected", where(q["condition"].isnull(), q["col1"], 2000)) -# assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) -# assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) -# # val/col -# expected = df.copy(deep=True) -# expected["projected"] = np.where(expected["condition"].isnull().to_numpy(), 2000.0, expected["col2"].to_numpy()) -# q = QueryBuilder() -# q = q.apply("projected", where(q["condition"].isnull(), 2000, q["col2"])) -# assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) -# assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) -# # val/val -# expected = df.copy(deep=True) -# expected["projected"] = np.where(expected["condition"].isnull().to_numpy(), 2000.0, 3000.0) -# q = QueryBuilder() -# q = q.apply("projected", where(q["condition"].isnull(), 2000, 3000)) -# assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) -# assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) - -# # Filters -# # Only test col/col, col/val etc can be achieved more efficiently without using the ternary operator -# expected = df.copy(deep=True) -# expected = expected[ -# np.where( -# expected["condition"].isnull().to_numpy(), -# expected["col1"].isnull().to_numpy(), -# expected["col2"].isnull().to_numpy(), -# ) -# ] -# q = QueryBuilder() -# q = q[where(q["condition"].isnull(), q["col1"].isnull(), q["col2"].isnull())] -# assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) -# assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) +@use_of_function_scoped_fixtures_in_hypothesis_checked +@settings(deadline=None) +@given( + df=data_frames( + columns( + ["condition", "col1", "col2"], + elements=strategies.floats(min_value=0, max_value=1000, allow_nan=False, allow_subnormal=False), + fill=strategies.just(np.nan), + ), + ), +) +def test_ternary_hypothesis(lmdb_version_store_v1, df, any_output_format): + assume( + not df.empty + and not df["condition"].isnull().all() + and not df["col1"].isnull().all() + and not df["col2"].isnull().all() + ) + lib = lmdb_version_store_v1 + lib._set_output_format_for_pipeline_tests(any_output_format) + dense_sym = "test_ternary_hypothesis_dense" + sparse_sym = "test_ternary_hypothesis_sparse" + + df.index = pd.date_range("2024-01-01", periods=len(df)) + + lib.write(dense_sym, df) + lib.write(sparse_sym, df, sparsify_floats=True) + + # Projection + # col/col + expected = df.copy(deep=True) + expected["projected"] = np.where( + expected["condition"].isnull().to_numpy(), expected["col1"].to_numpy(), expected["col2"].to_numpy() + ) + q = QueryBuilder() + q = q.apply("projected", where(q["condition"].isnull(), q["col1"], q["col2"])) + assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) + assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) + # col/val + expected = df.copy(deep=True) + expected["projected"] = np.where(expected["condition"].isnull().to_numpy(), expected["col1"].to_numpy(), 2000.0) + q = QueryBuilder() + q = q.apply("projected", where(q["condition"].isnull(), q["col1"], 2000)) + assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) + assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) + # val/col + expected = df.copy(deep=True) + expected["projected"] = np.where(expected["condition"].isnull().to_numpy(), 2000.0, expected["col2"].to_numpy()) + q = QueryBuilder() + q = q.apply("projected", where(q["condition"].isnull(), 2000, q["col2"])) + assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) + assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) + # val/val + expected = df.copy(deep=True) + expected["projected"] = np.where(expected["condition"].isnull().to_numpy(), 2000.0, 3000.0) + q = QueryBuilder() + q = q.apply("projected", where(q["condition"].isnull(), 2000, 3000)) + assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) + assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) + + # Filters + # Only test col/col, col/val etc can be achieved more efficiently without using the ternary operator + expected = df.copy(deep=True) + expected = expected[ + np.where( + expected["condition"].isnull().to_numpy(), + expected["col1"].isnull().to_numpy(), + expected["col2"].isnull().to_numpy(), + ) + ] + q = QueryBuilder() + q = q[where(q["condition"].isnull(), q["col1"].isnull(), q["col2"].isnull())] + assert_frame_equal(expected, lib.read(dense_sym, query_builder=q).data, check_dtype=False) + assert_frame_equal(expected, lib.read(sparse_sym, query_builder=q).data, check_dtype=False) diff --git a/setup.py b/setup.py index ffc17b5887a..1a02a6890fd 100644 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ def safe_remove_directory(dir_path): class CompileProto(Command): # When adding new protobuf versions, also update: setup.cfg, python/arcticdb/__init__.py - _PROTOBUF_TO_GRPC_VERSION = {"4": ">=1.49"} + _PROTOBUF_TO_GRPC_VERSION = {"3": "<1.31", "4": ">=1.49", "5": ">=1.68.1", "6": ">=1.73.0"} description = '"protoc" generate code _pb2.py from .proto files' user_options = [ From 212d8696e50a677f1b27de0c7d5d4758d70ee216 Mon Sep 17 00:00:00 2001 From: "Petar Markovski (Sofia)" Date: Wed, 25 Mar 2026 14:32:35 +0000 Subject: [PATCH 27/31] Change 'runner' to 'os' --- .github/workflows/build_with_conda.yml | 8 ++++---- .github/workflows/compile_and_test_with_conda_unix.yml | 8 ++++---- .github/workflows/compile_and_test_with_conda_win.yml | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_with_conda.yml b/.github/workflows/build_with_conda.yml index dc77127e6b1..33e9f46878e 100644 --- a/.github/workflows/build_with_conda.yml +++ b/.github/workflows/build_with_conda.yml @@ -30,7 +30,7 @@ jobs: uses: ./.github/workflows/compile_and_test_with_conda_unix.yml secrets: inherit with: - runner: ubuntu-22.04 + os: ubuntu-22.04 platform: linux_64 preset: linux-conda-release hypothesis_profile: ci_linux @@ -45,7 +45,7 @@ jobs: uses: ./.github/workflows/compile_and_test_with_conda_unix.yml secrets: inherit with: - runner: ubuntu-22.04-arm + os: ubuntu-22.04-arm platform: linux_aarch64 preset: linux-conda-release run_cpp_tests: ${{inputs.disable_cpp_tests != true}} @@ -59,7 +59,7 @@ jobs: uses: ./.github/workflows/compile_and_test_with_conda_unix.yml secrets: inherit with: - runner: macos-14 + os: macos-14 platform: osx_arm64 preset: macos-conda-release hypothesis_profile: ci_macos @@ -76,7 +76,7 @@ jobs: uses: ./.github/workflows/compile_and_test_with_conda_win.yml secrets: inherit with: - runner: windows-latest + os: windows-latest platform: win_64 preset: windows-cl-conda-release run_cpp_tests: ${{inputs.disable_cpp_tests != true}} diff --git a/.github/workflows/compile_and_test_with_conda_unix.yml b/.github/workflows/compile_and_test_with_conda_unix.yml index 50f11632a09..840db65cb7f 100644 --- a/.github/workflows/compile_and_test_with_conda_unix.yml +++ b/.github/workflows/compile_and_test_with_conda_unix.yml @@ -2,7 +2,7 @@ name: Conda Build and Test (Unix) on: workflow_call: inputs: - runner: {required: true, type: string, description: GitHub runner to execute on} + os: {required: true, type: string, description: GitHub os to execute on} platform: {required: true, type: string, description: Platform identifier e.g. linux_64 or osx_arm64} preset: {required: true, type: string, description: CMake preset name e.g. linux-conda-release} @@ -24,7 +24,7 @@ jobs: if: | always() && !cancelled() - runs-on: ${{inputs.runner}} + runs-on: ${{inputs.os}} env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true ACTIONS_ALLOW_UNSECURE_COMMANDS: true @@ -85,7 +85,7 @@ jobs: always() && !cancelled() && inputs.run_cpp_tests - runs-on: ${{inputs.runner}} + runs-on: ${{inputs.os}} env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} @@ -166,7 +166,7 @@ jobs: if: | always() && !cancelled() - runs-on: ${{inputs.runner}} + runs-on: ${{inputs.os}} strategy: fail-fast: false matrix: diff --git a/.github/workflows/compile_and_test_with_conda_win.yml b/.github/workflows/compile_and_test_with_conda_win.yml index 1f4907618f6..9914a17a133 100644 --- a/.github/workflows/compile_and_test_with_conda_win.yml +++ b/.github/workflows/compile_and_test_with_conda_win.yml @@ -2,7 +2,7 @@ name: Conda Build and Test (Windows) on: workflow_call: inputs: - runner: {required: true, type: string, description: GitHub runner to execute on} + os: {required: true, type: string, description: GitHub os to execute on} platform: {required: true, type: string, description: Platform identifier e.g. win_64} preset: {required: true, type: string, description: CMake preset name e.g. windows-cl-conda-release} @@ -20,7 +20,7 @@ jobs: if: | always() && !cancelled() - runs-on: ${{inputs.runner}} + runs-on: ${{inputs.os}} env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true ACTIONS_ALLOW_UNSECURE_COMMANDS: true @@ -91,7 +91,7 @@ jobs: if: | always() && !cancelled() - runs-on: ${{inputs.runner}} + runs-on: ${{inputs.os}} strategy: fail-fast: false matrix: From 928a6635960f83dc3f0b49078d80a3611c6c04c7 Mon Sep 17 00:00:00 2001 From: "Petar Markovski (Sofia)" Date: Wed, 25 Mar 2026 14:44:08 +0000 Subject: [PATCH 28/31] Add comments to CMAKE_GENERATOR_PLATFORM and CMAKE_GENERATOR_TOOLSET --- .github/workflows/compile_and_test_with_conda_win.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/compile_and_test_with_conda_win.yml b/.github/workflows/compile_and_test_with_conda_win.yml index 9914a17a133..514508707ed 100644 --- a/.github/workflows/compile_and_test_with_conda_win.yml +++ b/.github/workflows/compile_and_test_with_conda_win.yml @@ -54,6 +54,10 @@ jobs: - name: Build ArcticDB with conda shell: cmd /C call {0} + + # Clear CMAKE_GENERATOR_PLATFORM and CMAKE_GENERATOR_TOOLSET to prevent them from overriding the CMake preset configuration. + # GitHub runner images or Conda activation scripts may set these, which can conflict + # with the generator, architecture, and toolset already defined in the preset. run: | set CMAKE_GENERATOR_PLATFORM= set CMAKE_GENERATOR_TOOLSET= From 4ab5335a281dd8ab12f77691f278b3990e042c94 Mon Sep 17 00:00:00 2001 From: "Petar Markovski (Sofia)" Date: Wed, 25 Mar 2026 14:46:59 +0000 Subject: [PATCH 29/31] refactor if statements --- .github/workflows/compile_and_test_with_conda_win.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/compile_and_test_with_conda_win.yml b/.github/workflows/compile_and_test_with_conda_win.yml index 514508707ed..469397f3ac8 100644 --- a/.github/workflows/compile_and_test_with_conda_win.yml +++ b/.github/workflows/compile_and_test_with_conda_win.yml @@ -17,9 +17,6 @@ jobs: # ==================== JOB 1: COMPILE ==================== compile: name: Compile (${{inputs.platform}}) - if: | - always() && - !cancelled() runs-on: ${{inputs.os}} env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true @@ -94,7 +91,8 @@ jobs: needs: [compile] if: | always() && - !cancelled() + !cancelled() && + !failure() runs-on: ${{inputs.os}} strategy: fail-fast: false From cb551aff2e0a44fa62dbca5e3f0b34411172caf8 Mon Sep 17 00:00:00 2001 From: "Petar Markovski (Sofia)" Date: Wed, 25 Mar 2026 15:01:46 +0000 Subject: [PATCH 30/31] Add comments to checkout --- .github/workflows/compile_and_test_with_conda_unix.yml | 9 +++++++++ .github/workflows/compile_and_test_with_conda_win.yml | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/.github/workflows/compile_and_test_with_conda_unix.yml b/.github/workflows/compile_and_test_with_conda_unix.yml index 840db65cb7f..0b17c400c99 100644 --- a/.github/workflows/compile_and_test_with_conda_unix.yml +++ b/.github/workflows/compile_and_test_with_conda_unix.yml @@ -35,6 +35,9 @@ jobs: shell: bash -l {0} steps: - uses: actions/checkout@v6.0.1 + # Do not use recursive submodules checkout to simulate conda feedstock build + # with: + # submodules: recursive - name: Configure sccache uses: mozilla-actions/sccache-action@v0.0.9 @@ -95,6 +98,9 @@ jobs: shell: bash -l {0} steps: - uses: actions/checkout@v6.0.1 + # Do not use recursive submodules checkout to simulate conda feedstock build + # with: + # submodules: recursive - name: Configure sccache uses: mozilla-actions/sccache-action@v0.0.9 @@ -189,6 +195,9 @@ jobs: docker-images: false - uses: actions/checkout@v6.0.1 + # Do not use recursive submodules checkout to simulate conda feedstock build + # with: + # submodules: recursive - name: Configure sccache uses: mozilla-actions/sccache-action@v0.0.9 diff --git a/.github/workflows/compile_and_test_with_conda_win.yml b/.github/workflows/compile_and_test_with_conda_win.yml index 469397f3ac8..23bb63ebe01 100644 --- a/.github/workflows/compile_and_test_with_conda_win.yml +++ b/.github/workflows/compile_and_test_with_conda_win.yml @@ -28,6 +28,9 @@ jobs: shell: bash -l {0} steps: - uses: actions/checkout@v6.0.1 + # Do not use recursive submodules checkout to simulate conda feedstock build + # with: + # submodules: recursive - name: Configure sccache uses: mozilla-actions/sccache-action@v0.0.9 @@ -108,6 +111,9 @@ jobs: shell: bash -l {0} steps: - uses: actions/checkout@v6.0.1 + # Do not use recursive submodules checkout to simulate conda feedstock build + # with: + # submodules: recursive - name: Configure sccache uses: mozilla-actions/sccache-action@v0.0.9 From 32ad2dee8f13a101284e94b7c1f722a6160a02a8 Mon Sep 17 00:00:00 2001 From: "Petar Markovski (Sofia)" Date: Wed, 1 Apr 2026 00:17:35 +0100 Subject: [PATCH 31/31] Add read permission to win and unix scripts --- .github/workflows/compile_and_test_with_conda_unix.yml | 4 ++++ .github/workflows/compile_and_test_with_conda_win.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/compile_and_test_with_conda_unix.yml b/.github/workflows/compile_and_test_with_conda_unix.yml index 0b17c400c99..64e213368d1 100644 --- a/.github/workflows/compile_and_test_with_conda_unix.yml +++ b/.github/workflows/compile_and_test_with_conda_unix.yml @@ -1,4 +1,8 @@ name: Conda Build and Test (Unix) + +permissions: + contents: read + on: workflow_call: inputs: diff --git a/.github/workflows/compile_and_test_with_conda_win.yml b/.github/workflows/compile_and_test_with_conda_win.yml index 23bb63ebe01..1711ce46fd7 100644 --- a/.github/workflows/compile_and_test_with_conda_win.yml +++ b/.github/workflows/compile_and_test_with_conda_win.yml @@ -1,4 +1,8 @@ name: Conda Build and Test (Windows) + +permissions: + contents: read + on: workflow_call: inputs: