Restore relative paths & merge formatting change into dev

.github/workflows/build_check.yml

@@ -0,0 +1,41 @@
+name: Build Matrix
+
+on: [pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        gcc: [10]
+        python: [3.8, 3.12]
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        ref: ${{ github.head_ref }}
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python }}
+
+    - name: Install GCC
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }}
+        sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${{ matrix.gcc }} 100
+        sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${{ matrix.gcc }} 100
+
+    - name: Build
+      run: |
+        # Your build and test commands go here
+        gcc --version
+        python --version
+        pip install . --verbose
+
+    - name: Test Install
+      run:
+        python -c "from parla import Parla; from parla.tasks import TaskSpace"
+
+

.github/workflows/lint_cpp.yml

@@ -0,0 +1,21 @@
+name: CPP Linter
+
+on: pull_request
+
+jobs:
+  cpp-linter:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: cpp-linter/cpp-linter-action@v2
+        id: linter
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          style: llvm
+
+      - name: Fail fast?!
+        if: steps.linter.outputs.checks-failed > 0
+        run: echo "Some files failed the linting checks!"
+        # for actual deployment
+        # run: exit 1

.github/workflows/lint_cython.yml

@@ -0,0 +1,22 @@
+name: Cython Linter
+
+on: [pull_request]
+
+jobs:
+  format:
+    runs-on: ubuntu-latest
+    permissions:
+      # Give the default GITHUB_TOKEN write permission to commit and push the changed files back to the repository.
+      contents: write
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        ref: ${{ github.head_ref }}
+    - name: Setup Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.11'
+    - name: Install cython-lint
+      run: pip install cython-lint
+    - name: Check code formatting
+      run: cython-lint src/python/parla/cython

.github/workflows/lint_python.yml

@@ -0,0 +1,29 @@
+name: Python Linter & Format
+
+on: [pull_request]
+
+jobs:
+  format:
+    runs-on: ubuntu-latest
+    permissions:
+      # Give the default GITHUB_TOKEN write permission to commit and push the changed files back to the repository.
+      contents: write
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        ref: ${{ github.head_ref }}
+
+    - name: Setup Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.11'
+    - name: Install ruff
+      run: pip install ruff
+    - name: Check code formatting
+      run: ruff check src
+    - name: Format code
+      run: ruff format src
+      if: success()  # Only format if check success
+    - uses: stefanzweifel/git-auto-commit-action@v5
+      with:
+        commit_message: "style: apply ruff format"

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.14)
 set(CMAKE_CXX_STANDARD 17)
 
-project(parla LANGUAGES C CXX)
+project(${SKBUILD_PROJECT_NAME} LANGUAGES C CXX)
 
 set(3RD_PARTY_DIR ${PROJECT_SOURCE_DIR}/3rd_party)
 option(PARLA_BUILD_TESTS "Enables building of C++ tests" OFF)
@@ -12,70 +12,115 @@ option(PARLA_ENABLE_NVTX "Enable tracing with nvtx" OFF)
 option(PARLA_ENABLE_CUDA "Enable compilation for CUDA architectures" OFF)
 
 message(STATUS "PARLA_BUILD_TESTS: " ${PARLA_BUILD_TESTS})
-message(STATUS "PARLA_DEBUG: " ${PARLA_DEBUG})
+message(STATUS "CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE})
 message(STATUS "PARLA_ENABLE_LOGGING: " ${PARLA_ENABLE_LOGGING})
 message(STATUS "PARLA_ENABLE_NVTX: " ${PARLA_ENABLE_NVTX})
 message(STATUS "PARLA_ENABLE_CUDA: " ${PARLA_ENABLE_CUDA})
 
 set(CMAKE_VERBOSE_MAKEFILE ON)
 include(CTest)
 
-find_package(PythonExtensions REQUIRED)
-find_package(Cython REQUIRED)
-
-find_package(Python3 REQUIRED COMPONENTS Interpreter)
-#include_directories(${NumPy_INCLUDE_DIRS})
+find_package(
+  Python
+  COMPONENTS Interpreter Development.Module
+  REQUIRED)
+
+set(CYTHON cython)
+message(STATUS "Cython: " ${CYTHON})
+# Test if cython is installed
+if(NOT CYTHON)
+  message(
+    FATAL_ERROR
+      "Cython is required to build Parla. Please install cython and try again.")
+endif()
+# Test if cython exists
+execute_process(
+  COMMAND ${CYTHON} --version
+  ERROR_QUIET
+  OUTPUT_VARIABLE CYTHON_VERSION)
+if(NOT CYTHON_VERSION)
+  message(
+    FATAL_ERROR
+      "Cython is required to build Parla. Please install cython and try again.")
+endif()
+
+message(STATUS "Python: " ${Python_EXECUTABLE})
 include_directories(${NUMPY_INCLUDE})
 
-if(PARLA_DEBUG)
-    SET(PARLA_DEBUG_C_COMPILE_FLAGS "-fno-omit-frame-pointer -O0 -g -static-libstdc++")
-    SET(PARLA_DEBUG_CYTHON_COMPILE_FLAGS "--gdb")
-    SET(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} ${PARLA_DEBUG_C_COMPILE_FLAGS}")
-    SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PARLA_DEBUG_C_COMPILE_FLAGS}")
-    SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}")
-    set(CYTHON_FLAGS "${CYTHON_FLAGS} ${PARLA_DEBUG_CYTHON_COMPILE_FLAGS}")
-endif(PARLA_DEBUG)
+set(CYTHON_FLAGS "--cplus;--fast-fail")
+
+if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+  set(PARLA_DEBUG_C_COMPILE_FLAGS "-fno-omit-frame-pointer")
+  set(PARLA_DEBUG_CYTHON_FLAGS "--line-directives;-p;--gdb")
+
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${PARLA_DEBUG_C_COMPILE_FLAGS}")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PARLA_DEBUG_C_COMPILE_FLAGS}")
+  list(APPEND CYTHON_FLAGS ${PARLA_DEBUG_CYTHON_FLAGS})
+  list(APPEND CYTHON_FLAGS "-X;linetrace=True")
+endif()
+
+if(UNIX)
+  if(APPLE)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+  else(NOT APPLE)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+  endif()
+else(NOT UNIX)
+  message(
+    ERROR
+    "Windows is not supported. Please raise an issue if you need Windows support."
+  )
+endif()
 
-SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lrt -lpthread")
 if(PARLA_ENABLE_NVTX)
-    set(NVTX_INCLUDE ${3RD_PARTY_DIR}/nvtx/c/include)
+  set(NVTX_INCLUDE ${3RD_PARTY_DIR}/nvtx/c/include)
 endif(PARLA_ENABLE_NVTX)
 
 if(PARLA_ENABLE_CUDA)
-    enable_language(CUDA)
-    find_package(CUDAToolkit REQUIRED)
+  enable_language(CUDA)
+  find_package(CUDAToolkit REQUIRED)
 endif(PARLA_ENABLE_CUDA)
 
+if(PARLA_ENABLE_HIP)
+  enable_language(HIP)
+endif(PARLA_ENABLE_HIP)
+
 if(PARLA_ENABLE_LOGGING)
-    #TODO: figure out binlog cmake support
-    message(STATUS "Finding binlog...")
-    list(APPEND CMAKE_PREFIX_PATH "${3RD_PARTY_DIR}/binlog/install")
-    find_package(binlog 0.1.0)
-
-    if (binlog_FOUND)
-        install(IMPORTED_RUNTIME_ARTIFACTS binlog::bread binlog::brecovery DESTINATION bin)
-        install(RUNTIME_DEPENDENCY_SET binlog::binlog DESTINATION lib)
-    endif (binlog_FOUND)
-
-    if(NOT binlog_FOUND)
-        message(STATUS "Logging library `binlog` not found. Building binlog from source")
-        set(BINLOG_BUILD_EXAMPLES OFF CACHE BOOL "Build binlog examples" FORCE)
-        set(BINLOG_BUILD_UNIT_TESTS OFF CACHE BOOL "Build binlog unit tests" FORCE)
-        set(BINLOG_BUILD_INTEGRATION_TESTS OFF CACHE BOOL "Build binlog integration tests" FORCE)
-        add_subdirectory(${3RD_PARTY_DIR}/binlog)
-        #find_package(binlog 0.1.0)
-    endif(NOT binlog_FOUND)
+  # TODO: figure out binlog cmake support
+  message(STATUS "Finding binlog...")
+  list(APPEND CMAKE_PREFIX_PATH "${3RD_PARTY_DIR}/binlog/install")
+  find_package(binlog 0.1.0)
+
+  if(binlog_FOUND)
+    install(IMPORTED_RUNTIME_ARTIFACTS binlog::bread binlog::brecovery
+            DESTINATION ${SKBUILD_SCRIPTS_DIR})
+    install(RUNTIME_DEPENDENCY_SET binlog::binlog DESTINATION lib)
+  endif(binlog_FOUND)
+
+  if(NOT binlog_FOUND)
+    message(
+      STATUS "Logging library `binlog` not found. Building binlog from source")
+    set(BINLOG_BUILD_EXAMPLES
+        OFF
+        CACHE BOOL "Build binlog examples" FORCE)
+    set(BINLOG_BUILD_UNIT_TESTS
+        OFF
+        CACHE BOOL "Build binlog unit tests" FORCE)
+    set(BINLOG_BUILD_INTEGRATION_TESTS
+        OFF
+        CACHE BOOL "Build binlog integration tests" FORCE)
+    add_subdirectory(${3RD_PARTY_DIR}/binlog)
+    # find_package(binlog 0.1.0)
+  endif(NOT binlog_FOUND)
 
 endif(PARLA_ENABLE_LOGGING)
 
 add_subdirectory(src/c/backend)
 add_subdirectory(src/python/parla)
 
-if(PARLA_BUILD_TESTS)
-    add_subdirectory(testing)
-endif(PARLA_BUILD_TESTS)
+# if(PARLA_BUILD_TESTS) add_subdirectory(testing) endif(PARLA_BUILD_TESTS)
 
-set(test_path_file ${CMAKE_SOURCE_DIR}/testing/run_tests.sh)
-file(WRITE ${test_path_file} "export PARLA_TESTS=${CMAKE_BINARY_DIR}/testing\n")
-file(APPEND ${test_path_file} "py.test $PARLA_TESTS\n")
-file(APPEND ${test_path_file} "ctest --test-dir $PARLA_TESTS\n")
+# set(test_path_file ${CMAKE_SOURCE_DIR}/testing/run_tests.sh) file(WRITE
+# ${test_path_file} "export PARLA_TESTS=${CMAKE_BINARY_DIR}/testing\n")
+# file(APPEND ${test_path_file} "py.test $PARLA_TESTS\n") file(APPEND
+# ${test_path_file} "ctest --test-dir $PARLA_TESTS\n")

README.md

@@ -4,36 +4,37 @@ Repository for refactoring Parla to a C++ runtime to avoid GIL during scheduling
 
 # Installation and Use
 
-Requires: 
-- psutil, cython, scikit-build, Python>=3.9, nvtx (python module)
-- CuPy>=9.2 (for GPU support)
-- A C++ compiler that supports C++20 float atomics
-- An internet connection. I don't know how to make scikit-build stop building in its own venv. 
-
 Note: Ensure that the Cupy version and your CUDA Toolkit are compatible with each other and the architecture of your machine.
+On TACC systems we recommend installing CuPy (and if needed numba) via pip against the system CUDA version instead of pulling in binaries from conda-forge's cudatoolkit.
 
 Can be built with:
 ```
     git submodule update --init --recursive
-    make clean; make
+    pip install .
 ```
 
-RUNNING `make clean` IS VERY IMPORTANT! 
 You may need to set C compilers explicily on Frontera. 
-
 ```
 module load gcc/12
 make clean; CC=gcc CXX=g++ make
 ```
 
-Build options are set through enviornment variables:
+This will populate a `build` directory and an install into your active Python environment's `site-packages` folder. 
+If you are a developer, it may sometimes be necessary to remove the build directory manually to apply changes and remove the cached version.
+Changes to most files in the src tree will trigger a rebuild of the relevant files without needing to clear the build directory. 
+Additionally, `scikit-build-core` supports editable installations that recompile changes on each run, please see this documentation. 
 
-PARLA_ENABLE_NVTX=ON/OFF :   Enables NVTX hooks in the C++ backend.
+You may need to set C compilers explicily on Frontera. 
+```
+module load gcc/12
+make clean; CC=gcc CXX=g++ make
+```
 
+Build options are set through the pyproject.toml:
+PARLA_ENABLE_NVTX=ON/OFF :   Enables NVTX hooks in the C++ backend.
 PARLA_ENABLE_LOGGING=ON/OFF : Enables Binlog hooks in the Python & C++ backend for logging. 
-
-There are currently no C++ tests for Parla but if there were, the CMAKE is configured to build and run them with `ctest`. 
-There is a pytest driven folder for tests from Python. 
+PARLA_ENABLE_CUDA=ON/OFF : Enables CUDA C++ backend for runahead scheduling
+PARLA_ENABLE_HIP=ON/OFF : Enables HIP C++ backend for runahead scheduling
 
 # Logging
 
@@ -44,12 +45,12 @@ Logs are dumped when the Parla runtime exits. The name of the log file defaults
 
 If a Parla program is interrupted with a KeyboardInterrupt Exception (aka Ctrl+C) it should dump the log as the runtime exits gracefully. The same *should* hopefully now be true of exceptions raised within tasks or workers. But please raise an issue if you found one that isn't handled. 
 
-If a Parla program is interuppted with an error that causes a crash and core dump, the logfile can be recovered with `brecovery`. See the binlog documentation for more information. 
-
+If a Parla program is interuppted with an error that causes a crash and core dump, the logfile can be recovered from the coredump directly with `brecovery`. See the binlog documentation for more information. 
 
 To read a logfile run:
 `bread <logfile>`
 
+
 # Number of threads
 
  The number of workers in the Parla runtime is controlled by the PARLA_NUM_THREADS enviornment variable. If it is not set it defaults to the number of cores on the host machine. 

makefile

@@ -2,10 +2,7 @@ all:
 	pip install . --verbose
 
 clean:
-	python setup.py clean --all
-
-local:
-	python setup.py build_ext --inplace
+	rm -rf build
 
 test: local
 	. testing/run_tests.sh

miniapps/independent-ffts/pyproject.toml

@@ -11,5 +11,5 @@ requires = [
 ]
 
 [tool.ruff]
-extend-ignore = ["F841", "401"]
+extend-ignore = ["F841", "F401"]