[InterOp] Upgrade to CppInterOp 1.8.0

interpreter/CppInterOp/CMakeLists.txt

@@ -298,6 +298,18 @@ include_directories(SYSTEM ${LLVM_INCLUDE_DIRS})
 separate_arguments(LLVM_DEFINITIONS_LIST NATIVE_COMMAND ${LLVM_DEFINITIONS})
 add_definitions(${LLVM_DEFINITIONS_LIST})
 
+string(REGEX REPLACE "/lib/cmake/llvm$" "" LLVM_BINARY_LIB_DIR "${LLVM_DIR}")
+add_definitions(-DLLVM_BINARY_LIB_DIR="${LLVM_BINARY_LIB_DIR}")
+
+if(LLVM_BUILT_WITH_OOP_JIT)
+  if((CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") OR
+     (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64"))
+    add_definitions(-DLLVM_BUILT_WITH_OOP_JIT)
+  else()
+    message(FATAL_ERROR "LLVM_BUILT_WITH_OOP_JIT is only supported on MacOS arm64 or Linux x86_64. Build aborted.")
+  endif()
+endif()
+
 # If the llvm sources are present add them with higher priority.
 if (LLVM_BUILD_MAIN_SRC_DIR)
   # LLVM_INCLUDE_DIRS contains the include paths to both LLVM's source and

interpreter/CppInterOp/Emscripten-build-instructions.md

@@ -19,24 +19,24 @@ cd ./CppInterOp-wasm
 ```
 
 To create a wasm build of CppInterOp we make use of the emsdk toolchain. This can be installed by executing (we only currently  
-support version 3.1.73)
+support version 4.0.9)
 ```bash
 git clone https://github.com/emscripten-core/emsdk.git
-./emsdk/emsdk install  3.1.73
+./emsdk/emsdk install  4.0.9
 ```
 
 and to activate the emsdk environment on Linux and osx execute (we are defining SYSROOT_PATH for use later)
 
 ```bash
-./emsdk/emsdk activate 3.1.73
+./emsdk/emsdk activate 4.0.9
 source ./emsdk/emsdk_env.sh
 export SYSROOT_PATH=$PWD/emsdk/upstream/emscripten/cache/sysroot
 ```
 
 and on Windows execute in Powershell
 
 ```powershell
-.\emsdk\emsdk activate 3.1.73
+.\emsdk\emsdk activate 4.0.9
 .\emsdk\emsdk_env.ps1
 $env:PWD_DIR= $PWD.Path
 $env:SYSROOT_PATH="$env:EMSDK/upstream/emscripten/cache/sysroot"
@@ -178,7 +178,7 @@ export CMAKE_SYSTEM_PREFIX_PATH=$PREFIX
 and
 
 ```powershell
-$env:PREFIX="%CONDA_PREFIX%/envs/CppInterOp-wasm"
+$env:PREFIX="$env:MAMBA_ROOT_PREFIX/envs/CppInterOp-wasm"
 $env:CMAKE_PREFIX_PATH=$env:PREFIX
 $env:CMAKE_SYSTEM_PREFIX_PATH=$env:PREFIX
 ```
@@ -337,7 +337,7 @@ emmake make -j $(nproc --all) install
 ## Xeus-cpp-lite Wasm Build Instructions
 
 A project which makes use of the wasm build of CppInterOp is xeus-cpp. xeus-cpp is a C++ Jupyter kernel. Assuming you are in  
-the CppInterOp build folder, you can build the wasm version of xeus-cpp by executing (replace LLVM_VERSION with the version
+the CppInterOp build folder, you can build the wasm version of xeus-cpp on Linux/MacOS by executing (replace LLVM_VERSION with the version
 of llvm you are building against)
 
 ```bash
@@ -359,17 +359,51 @@ emcmake cmake \
  emmake make -j $(nproc --all) install
 ```
 
-To build and test Jupyter Lite with this kernel locally you can execute the following
+and on Windows by executing
+
+```powershell
+cd ..\..
+git clone --depth=1 https://github.com/compiler-research/xeus-cpp.git
+$env:LLVM_VERSION=20
+cd .\xeus-cpp
+mkdir build
+cd build
+emcmake cmake `
+          -DCMAKE_BUILD_TYPE=Release                                     `
+          -DCMAKE_PREFIX_PATH="$env:PREFIX"                                    `
+          -DCMAKE_INSTALL_PREFIX="$env:PREFIX"                                 `
+          -DXEUS_CPP_EMSCRIPTEN_WASM_BUILD=ON                            `
+          -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ON                         `
+          -DXEUS_CPP_RESOURCE_DIR="$env:LLVM_BUILD_DIR/lib/clang/$env:LLVM_VERSION" `
+          -DSYSROOT_PATH="$env:SYSROOT_PATH"                                   `
+          ..
+ emmake make -j $(nproc --all) install
+```
+
+To build and test Jupyter Lite with this kernel locally on Linux/MacOS you can execute the following
 
 ```bash
 cd ../..
 micromamba create -n xeus-lite-host jupyterlite-core=0.6 jupyterlite-xeus jupyter_server jupyterlab notebook python-libarchive-c -c conda-forge
 micromamba activate xeus-lite-host
 jupyter lite serve --XeusAddon.prefix=$PREFIX \
                    --contents xeus-cpp/notebooks/xeus-cpp-lite-demo.ipynb \
-                   --contents xeus-cpp/notebooks/smallpt.ipynb \
                    --contents xeus-cpp/notebooks/images/marie.png \
                    --contents xeus-cpp/notebooks/audio/audio.wav \
                    --XeusAddon.mounts="$PREFIX/share/xeus-cpp/tagfiles:/share/xeus-cpp/tagfiles" \
                    --XeusAddon.mounts="$PREFIX/etc/xeus-cpp/tags.d:/etc/xeus-cpp/tags.d"
 ```
+
+and on Windows execute
+
+```powershell
+cd ..\..
+micromamba create -n xeus-lite-host jupyterlite-core=0.6 jupyterlite-xeus jupyter_server jupyterlab notebook python-libarchive-c -c conda-forge
+micromamba activate xeus-lite-host
+jupyter lite serve --XeusAddon.prefix="$env:PREFIX" `
+                   --contents xeus-cpp/notebooks/xeus-cpp-lite-demo.ipynb `
+                   --contents xeus-cpp/notebooks/images/marie.png `
+                   --contents xeus-cpp/notebooks/audio/audio.wav `
+                   --XeusAddon.mounts="$env:PREFIX/share/xeus-cpp/tagfiles:/share/xeus-cpp/tagfiles" `
+                   --XeusAddon.mounts="$env:PREFIX/etc/xeus-cpp/tags.d:/etc/xeus-cpp/tags.d"
+```

interpreter/CppInterOp/README.md

@@ -111,6 +111,13 @@ git clone --depth=1 --branch release/20.x https://github.com/llvm/llvm-project.g
 cd llvm-project
 ```
 
+If you want to have out-of-process JIT execution enabled in CppInterOp, then apply this patch on Linux-x86_64 and MacOS arm64 environment.
+> Note that this patch will not work for Windows because out-of-process JIT execution is currently implemented for Linux-x86_64 and MacOS arm64 only.
+
+```bash
+git apply -v ../CppInterOp/patches/llvm/clang20-1-out-of-process.patch
+```
+
 ##### Build Clang-REPL
 
 Clang-REPL is an interpreter that CppInterOp works alongside. Build Clang (and
@@ -140,6 +147,45 @@ export LLVM_DIR=$PWD
 cd ../
 ```
 
+##### Build Clang-REPL with Out-of-Process JIT Execution
+
+To have ``Out-of-Process JIT Execution`` enabled, run following commands to build clang and clang-repl to support this feature:
+> Only for Linux x86_64 and MacOS arm64
+
+```bash
+mkdir build 
+cd build 
+cmake -DLLVM_ENABLE_PROJECTS="clang;compiler-rt"                   \
+              -DLLVM_TARGETS_TO_BUILD="host;NVPTX"                \
+              -DCMAKE_BUILD_TYPE=Release                          \
+              -DLLVM_ENABLE_ASSERTIONS=ON                         \
+              -DCLANG_ENABLE_STATIC_ANALYZER=OFF                  \
+              -DCLANG_ENABLE_ARCMT=OFF                            \
+              -DCLANG_ENABLE_FORMAT=OFF                           \
+              -DCLANG_ENABLE_BOOTSTRAP=OFF                        \
+              ../llvm
+```
+
+###### For Linux x86_64
+
+```bash
+cmake --build . --target clang clang-repl llvm-jitlink-executor orc_rt-x86_64 --parallel $(nproc --all)
+```
+
+###### For MacOS arm64
+
+```bash
+cmake --build . --target clang clang-repl llvm-jitlink-executor orc_rt_osx --parallel $(sysctl -n hw.ncpu)
+```
+
+Note the 'llvm-project' directory location by executing
+
+```bash
+cd ../
+export LLVM_DIR=$PWD
+cd ../
+```
+
 #### Environment variables
 
 You will need to define the following environment variables for the build of CppInterOp and cppyy (as they clear for a new session, it is recommended that you also add these to your .bashrc in linux, .bash_profile if on MacOS). On Linux and MacOS you define as follows
@@ -163,10 +209,14 @@ Now CppInterOp can be built. This can be done by executing
 ```bash
 mkdir CppInterOp/build/
 cd CppInterOp/build/
-cmake -DBUILD_SHARED_LIBS=ON -DCPPINTEROP_USE_CLING=ON -DCPPINTEROP_USE_REPL=Off -DCling_DIR=$LLVM_DIR/build/tools/cling -DLLVM_DIR=$LLVM_DIR/build/lib/cmake/llvm -DClang_DIR=$LLVM_DIR/build/lib/cmake/clang -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR ..
+cmake -DBUILD_SHARED_LIBS=ON -DLLVM_DIR=$LLVM_DIR/build/lib/cmake/llvm -DClang_DIR=$LLVM_DIR/build/lib/cmake/clang -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR ..
 cmake --build . --target install --parallel $(nproc --all)
 ```
 
+and
+
+> Do make sure to pass ``DLLVM_BUILT_WITH_OOP_JIT=ON``, if you want to have out-of-process JIT execution feature enabled.
+
 #### Testing CppInterOp
 
 To test the built CppInterOp execute the following command in the CppInterOP build folder on Linux and MacOS

interpreter/CppInterOp/cmake/modules/GoogleTest.cmake

@@ -34,8 +34,8 @@ endif()
 ExternalProject_Add(
   googletest
   GIT_REPOSITORY https://github.com/google/googletest.git
-  GIT_SHALLOW 1
-  GIT_TAG v1.17.0
+  GIT_SHALLOW FALSE
+  GIT_TAG fa8438ae6b70c57010177de47a9f13d7041a6328
   UPDATE_COMMAND ""
   # # Force separate output paths for debug and release builds to allow easy
   # # identification of correct lib in subsequent TARGET_LINK_LIBRARIES commands

interpreter/CppInterOp/cppinterop-version.tag

@@ -1 +1 @@
-33a3e7e7629c0818bd92420568671325b016d6b2
+0e125033a00d1bc23ec8ed663e57ddc067ec5b9e

interpreter/CppInterOp/docs/DevelopersDocumentation.rst

@@ -41,6 +41,15 @@ Clone the 20.x release of the LLVM project repository.
    git clone --depth=1 --branch release/20.x https://github.com/llvm/llvm-project.git
    cd llvm-project
 
+If you want to have out-of-process JIT execution enabled in CppInterOp, then apply this patch on Linux-x86_64 and MacOS arm64 environment.
+.. note::
+
+   This patch will not work for Windows because out-of-process JIT execution is currently implemented for Linux and MacOS only.
+
+.. code:: bash
+
+   git apply -v ../CppInterOp/patches/llvm/clang20-1-out-of-process.patch
+
 ******************
  Build Clang-REPL
 ******************
@@ -99,6 +108,34 @@ On Windows you execute the following
    $env:LLVM_DIR= $PWD.Path
    cd ..\
 
+***************************************************
+Build Clang-REPL with Out-of-Process JIT Execution
+***************************************************
+
+To have `Out-of-Process JIT Execution` enabled, run following commands to build clang and clang-repl to support this feature:
+
+.. note::
+
+   Only for Linux x86_64 and MacOS arm64
+
+.. code:: bash
+   mkdir build 
+   cd build 
+   cmake -DLLVM_ENABLE_PROJECTS="clang;compiler-rt"                   \
+                  -DLLVM_TARGETS_TO_BUILD="host;NVPTX"                \
+                  -DCMAKE_BUILD_TYPE=Release                          \
+                  -DLLVM_ENABLE_ASSERTIONS=ON                         \
+                  -DCLANG_ENABLE_STATIC_ANALYZER=OFF                  \
+                  -DCLANG_ENABLE_ARCMT=OFF                            \
+                  -DCLANG_ENABLE_FORMAT=OFF                           \
+                  -DCLANG_ENABLE_BOOTSTRAP=OFF                        \
+                  ../llvm
+
+   # For Linux x86_64
+   cmake --build . --target clang clang-repl llvm-jitlink-executor orc_rt-x86_64 --parallel $(nproc --all)
+   # For MacOS arm64
+   cmake --build . --target clang clang-repl llvm-jitlink-executor orc_rt_osx --parallel $(sysctl -n hw.ncpu)
+
 **************************************
  Build Cling and related dependencies
 **************************************
@@ -261,6 +298,10 @@ commands on Linux and MacOS
    cmake -DBUILD_SHARED_LIBS=ON -DLLVM_DIR=$LLVM_DIR/build/lib/cmake/llvm -DClang_DIR=$LLVM_DIR/build/lib/cmake/clang -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR ..
    cmake --build . --target install --parallel $(nproc --all)
 
+.. note::
+
+   Do make sure to pass ``DLLVM_BUILT_WITH_OOP_JIT=ON``, if you want to have out-of-process JIT execution feature enabled.
+
 and
 
 .. code:: powershell

interpreter/CppInterOp/docs/Emscripten-build-instructions.rst

@@ -32,27 +32,27 @@ Now move into this directory using the following command
 
 To create a wasm build of CppInterOp we make use of the emsdk toolchain.
 This can be installed by executing (we only currently support version
-3.1.73)
+4.0.9)
 
 .. code:: bash
 
    git clone https://github.com/emscripten-core/emsdk.git
-   ./emsdk/emsdk install  3.1.73
+   ./emsdk/emsdk install  4.0.9
 
 and to activate the emsdk environment on Linux and osx execute 
 (we are defining SYSROOT_PATH for use later)
 
 .. code:: bash
 
-   ./emsdk/emsdk activate 3.1.73
+   ./emsdk/emsdk activate 4.0.9
    source ./emsdk/emsdk_env.sh
    export SYSROOT_PATH=$PWD/emsdk/upstream/emscripten/cache/sysroot
 
 and on Windows execute in Powershell
 
 .. code:: powershell
 
-   .\emsdk\emsdk activate 3.1.73
+   .\emsdk\emsdk activate 4.0.9
    .\emsdk\emsdk_env.ps1
    $env:PWD_DIR= $PWD.Path
    $env:SYSROOT_PATH="$env:EMSDK/upstream/emscripten/cache/sysroot"
@@ -203,7 +203,7 @@ and
 
 .. code:: powershell
 
-   $env:PREFIX="%CONDA_PREFIX%/envs/CppInterOp-wasm"
+   $env:PREFIX="$env:MAMBA_ROOT_PREFIX/envs/CppInterOp-wasm"
    $env:CMAKE_PREFIX_PATH=$env:PREFIX
    $env:CMAKE_SYSTEM_PREFIX_PATH=$env:PREFIX
 
@@ -358,8 +358,8 @@ Assuming it passes all test you can install by executing the following.
 
 A project which makes use of the wasm build of CppInterOp is xeus-cpp.
 xeus-cpp is a C++ Jupyter kernel. Assuming you are in the CppInterOp
-build folder, you can build the wasm version of xeus-cpp by executing
-(replace LLVM_VERSION with the version of llvm you are building against)
+build folder, you can build the wasm version of xeus-cpp on Linux/MacOS 
+by executing (replace LLVM_VERSION with the version of llvm you are building against)
 
 .. code:: bash
 
@@ -380,7 +380,28 @@ build folder, you can build the wasm version of xeus-cpp by executing
            ..
    emmake make -j $(nproc --all) install
 
-To build and test Jupyter Lite with this kernel locally you can execute the following
+and on Windows by executing 
+
+.. code:: powershell
+
+   cd ..\..
+   git clone --depth=1 https://github.com/compiler-research/xeus-cpp.git
+   $env:LLVM_VERSION=20
+   cd .\xeus-cpp
+   mkdir build
+   cd build
+   emcmake cmake `
+           -DCMAKE_BUILD_TYPE=Release                                     `
+           -DCMAKE_PREFIX_PATH="$env:PREFIX"                              `
+           -DCMAKE_INSTALL_PREFIX="$env:PREFIX"                           `
+           -DXEUS_CPP_EMSCRIPTEN_WASM_BUILD=ON                            `
+           -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ON                         `
+	        -DXEUS_CPP_RESOURCE_DIR="$env:LLVM_BUILD_DIR/lib/clang/$env:LLVM_VERSION" `
+           -DSYSROOT_PATH="$env:SYSROOT_PATH"                              `
+           ..
+   emmake make -j $(nproc --all) install
+
+To build and test Jupyter Lite with this kernel locally on Linux/MacOS you can execute the following
 
 .. code:: bash
 
@@ -389,8 +410,21 @@ To build and test Jupyter Lite with this kernel locally you can execute the foll
    micromamba activate xeus-lite-host
    jupyter lite serve --XeusAddon.prefix=$PREFIX \
                       --contents xeus-cpp/notebooks/xeus-cpp-lite-demo.ipynb \
-                      --contents xeus-cpp/notebooks/smallpt.ipynb \
                       --contents xeus-cpp/notebooks/images/marie.png \ 
                       --contents xeus-cpp/notebooks/audio/audio.wav \
                       --XeusAddon.mounts="$PREFIX/share/xeus-cpp/tagfiles:/share/xeus-cpp/tagfiles" \
                       --XeusAddon.mounts="$PREFIX/etc/xeus-cpp/tags.d:/etc/xeus-cpp/tags.d"
+
+and on Windows execute
+
+.. code:: powershell
+
+   cd ..\..
+   micromamba create -n xeus-lite-host jupyterlite-core=0.6 jupyterlite-xeus jupyter_server jupyterlab notebook python-libarchive-c -c conda-forge
+   micromamba activate xeus-lite-host
+   jupyter lite serve --XeusAddon.prefix="$env:PREFIX" `
+                      --contents xeus-cpp/notebooks/xeus-cpp-lite-demo.ipynb `
+                      --contents xeus-cpp/notebooks/images/marie.png ` 
+                      --contents xeus-cpp/notebooks/audio/audio.wav `
+                      --XeusAddon.mounts="$env:PREFIX/share/xeus-cpp/tagfiles:/share/xeus-cpp/tagfiles" `
+                      --XeusAddon.mounts="$env:PREFIX/etc/xeus-cpp/tags.d:/etc/xeus-cpp/tags.d"