Merge pull request #386 from rapidsai/branch-24.06

.devcontainer/cuda11.8-conda/devcontainer.json

@@ -5,12 +5,12 @@
     "args": {
       "CUDA": "11.8",
       "PYTHON_PACKAGE_MANAGER": "conda",
-      "BASE": "rapidsai/devcontainers:24.04-cpp-llvm16-cuda11.8-mambaforge-ubuntu22.04"
+      "BASE": "rapidsai/devcontainers:24.06-cpp-llvm16-cuda11.8-mambaforge-ubuntu22.04"
     }
   },
   "hostRequirements": {"gpu": "optional"},
   "features": {
-    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.4": {}
+    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.6": {}
   },
   "overrideFeatureInstallOrder": [
     "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils"

.devcontainer/cuda11.8-pip/devcontainer.json

@@ -5,12 +5,12 @@
     "args": {
       "CUDA": "11.8",
       "PYTHON_PACKAGE_MANAGER": "pip",
-      "BASE": "rapidsai/devcontainers:24.04-cpp-cuda11.8-ubuntu22.04"
+      "BASE": "rapidsai/devcontainers:24.06-cpp-cuda11.8-ubuntu22.04"
     }
   },
   "hostRequirements": {"gpu": "optional"},
   "features": {
-    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.4": {}
+    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.6": {}
   },
   "overrideFeatureInstallOrder": [
     "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils"

.devcontainer/cuda12.2-conda/devcontainer.json

@@ -5,12 +5,12 @@
     "args": {
       "CUDA": "12.2",
       "PYTHON_PACKAGE_MANAGER": "conda",
-      "BASE": "rapidsai/devcontainers:24.04-cpp-mambaforge-ubuntu22.04"
+      "BASE": "rapidsai/devcontainers:24.06-cpp-mambaforge-ubuntu22.04"
     }
   },
   "hostRequirements": {"gpu": "optional"},
   "features": {
-    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.4": {}
+    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.6": {}
   },
   "overrideFeatureInstallOrder": [
     "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils"

.devcontainer/cuda12.2-pip/devcontainer.json

@@ -5,12 +5,12 @@
     "args": {
       "CUDA": "12.2",
       "PYTHON_PACKAGE_MANAGER": "pip",
-      "BASE": "rapidsai/devcontainers:24.04-cpp-cuda12.2-ubuntu22.04"
+      "BASE": "rapidsai/devcontainers:24.06-cpp-cuda12.2-ubuntu22.04"
     }
   },
   "hostRequirements": {"gpu": "optional"},
   "features": {
-    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.4": {}
+    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.6": {}
   },
   "overrideFeatureInstallOrder": [
     "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils"

.github/workflows/build.yaml

@@ -28,7 +28,7 @@ concurrency:
 jobs:
   cpp-build:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.04
+    uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.06
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -37,7 +37,7 @@ jobs:
   python-build:
     needs: [cpp-build]
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.04
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.06
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -46,7 +46,7 @@ jobs:
   upload-conda:
     needs: [cpp-build, python-build]
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.04
+    uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.06
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -56,7 +56,7 @@ jobs:
     if: github.ref_type == 'branch'
     needs: python-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.06
     with:
       arch: "amd64"
       branch: ${{ inputs.branch }}

.github/workflows/pr.yaml

@@ -20,38 +20,38 @@ jobs:
       - docs-build
       - devcontainer
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.04
+    uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.06
   checks:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.04
+    uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.06
   conda-cpp-build:
     needs: checks
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.04
+    uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.06
     with:
       build_type: pull-request
   conda-cpp-tests:
     needs: conda-cpp-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.04
+    uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.06
     with:
       build_type: pull-request
   conda-python-build:
     needs: conda-cpp-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.04
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.06
     with:
       build_type: pull-request
   conda-python-tests:
     needs: conda-python-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.06
     with:
       build_type: pull-request
   docs-build:
     needs: conda-python-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.06
     with:
       build_type: pull-request
       node_type: "gpu-v100-latest-1"
@@ -60,7 +60,7 @@ jobs:
       run_script: "ci/build_docs.sh"
   devcontainer:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.04
+    uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.06
     with:
       arch: '["amd64"]'
       cuda: '["12.2"]'

.github/workflows/test.yaml

@@ -16,15 +16,15 @@ on:
 jobs:
   cpp-tests:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.04
+    uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.06
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
   python-tests:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.06
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}

CHANGELOG.md

@@ -1,3 +1,20 @@
+# kvikio 24.06.00 (5 Jun 2024)
+
+## 🐛 Bug Fixes
+
+- Fix conda recipe for kvikio ([#381](https://github.com/rapidsai/kvikio/pull/381)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA)
+- Move cuFile linking to kvikio target ([#379](https://github.com/rapidsai/kvikio/pull/379)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA)
+
+## 🛠️ Improvements
+
+- Initial Python Interface for cufile Async IO ([#376](https://github.com/rapidsai/kvikio/pull/376)) [@K-Wu](https://github.com/K-Wu)
+- Enable Warnings as errors for Python tests ([#375](https://github.com/rapidsai/kvikio/pull/375)) [@mroeschke](https://github.com/mroeschke)
+- POSIX IO: use async memcpy ([#374](https://github.com/rapidsai/kvikio/pull/374)) [@madsbk](https://github.com/madsbk)
+- No-CUDA build ([#373](https://github.com/rapidsai/kvikio/pull/373)) [@madsbk](https://github.com/madsbk)
+- Refactor: move cufile header checks to the shim layer ([#372](https://github.com/rapidsai/kvikio/pull/372)) [@madsbk](https://github.com/madsbk)
+- Migrate to `{{ stdlib("c") }}` ([#371](https://github.com/rapidsai/kvikio/pull/371)) [@hcho3](https://github.com/hcho3)
+- Add deflate codec ([#364](https://github.com/rapidsai/kvikio/pull/364)) [@ap--](https://github.com/ap--)
+
 # kvikio 24.04.00 (10 Apr 2024)
 
 ## 🚨 Breaking Changes

VERSION

@@ -1 +1 @@
-24.04.00
+24.06.00

build.sh

@@ -117,6 +117,11 @@ if hasArg -n; then
     INSTALL_TARGET=""
 fi
 
+# Append `-DFIND_KVIKIO_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option.
+if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_KVIKIO_CPP"* ]]; then
+    EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DFIND_KVIKIO_CPP=ON"
+fi
+
 # If clean given, run it prior to any other steps
 if hasArg clean; then
     # If the dirs to clean are mounted dirs in a container, the

ci/release/update-version.sh

@@ -46,3 +46,6 @@ find .devcontainer/ -type f -name devcontainer.json -print0 | while IFS= read -r
     sed_runner "s@rapidsai/devcontainers:[0-9.]*@rapidsai/devcontainers:${NEXT_SHORT_TAG}@g" "${filename}"
     sed_runner "s@rapidsai/devcontainers/features/rapids-build-utils:[0-9.]*@rapidsai/devcontainers/features/rapids-build-utils:${NEXT_SHORT_TAG_PEP440}@" "${filename}"
 done
+
+# The example of a downstream project
+sed_runner "s/find_and_configure_kvikio(.*)/find_and_configure_kvikio(\"${NEXT_SHORT_TAG}\")/g" "cpp/examples/downstream/cmake/get_kvikio.cmake"

conda/recipes/kvikio/build.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# Copyright (c) 2018-2024, NVIDIA CORPORATION.
+
+./build.sh kvikio

conda/recipes/kvikio/conda_build_config.yaml

@@ -10,7 +10,10 @@ cuda_compiler:
 cuda11_compiler:
   - nvcc
 
-sysroot_version:
+c_stdlib:
+  - sysroot
+
+c_stdlib_version:
   - "2.17"
 
 cmake_version:

conda/recipes/kvikio/meta.yaml

@@ -32,9 +32,6 @@ build:
     - SCCACHE_S3_KEY_PREFIX=kvikio-linux64 # [linux64]
     - SCCACHE_S3_USE_SSL
     - SCCACHE_S3_NO_CREDENTIALS
-  script:
-    - cd python
-    - python -m pip install . -vv
   ignore_run_exports_from:
     {% if cuda_major == "11" %}
     - {{ compiler('cuda11') }}
@@ -54,7 +51,7 @@ requirements:
     {% else %}
     - {{ compiler('cuda') }}
     {% endif %}
-    - sysroot_{{ target_platform }} {{ sysroot_version }}
+    - {{ stdlib("c") }}
   host:
     - python
     - setuptools

conda/recipes/libkvikio/conda_build_config.yaml

@@ -13,7 +13,10 @@ cuda_compiler:
 cuda11_compiler:
   - nvcc
 
-sysroot_version:
+c_stdlib:
+  - sysroot
+
+c_stdlib_version:
   - "2.17"
 
 # The CTK libraries below are missing from the conda-forge::cudatoolkit package

conda/recipes/libkvikio/meta.yaml

@@ -42,7 +42,7 @@ requirements:
     - {{ compiler('cuda') }}
     {% endif %}
     - ninja
-    - sysroot_{{ target_platform }} {{ sysroot_version }}
+    - {{ stdlib("c") }}
   host:
     - cuda-version ={{ cuda_version }}
     {% if cuda_major == "11" %}

cpp/CMakeLists.txt

@@ -44,13 +44,13 @@ rapids_cmake_support_conda_env(conda_env MODIFY_PREFIX_PATH)
 rapids_cpm_init()
 
 rapids_find_package(
-  CUDAToolkit REQUIRED
+  Threads REQUIRED
   BUILD_EXPORT_SET kvikio-exports
   INSTALL_EXPORT_SET kvikio-exports
 )
 
 rapids_find_package(
-  Threads REQUIRED
+  CUDAToolkit
   BUILD_EXPORT_SET kvikio-exports
   INSTALL_EXPORT_SET kvikio-exports
 )
@@ -84,13 +84,45 @@ endif()
 add_library(kvikio INTERFACE)
 add_library(kvikio::kvikio ALIAS kvikio)
 
+# We enable CUDA and cuFile both here and in the FINAL_CODE_BLOCK export block. While the code block
+# below (in FINAL_CODE_BLOCK) sets this information when KvikIO is imported from a
+# kvikio-config.cmake file, this code block is intended to be used by projects that include KvikIO's
+# source directory in their own CMake build.
+#
+# Normally we would just set the below without using $<BUILD_LOCAL_INTERFACE:...>, and without the
+# final_code_string, but in this case we want to conditionally set these things at import time, not
+# export time, since KvikIO is a header-only library that can adapt to different build environments.
+
+# Enable CUDA in KvikIO
+if(CUDAToolkit_FOUND)
+  target_link_libraries(kvikio INTERFACE $<BUILD_LOCAL_INTERFACE:CUDA::toolkit>)
+  target_compile_definitions(kvikio INTERFACE $<BUILD_LOCAL_INTERFACE:KVIKIO_CUDA_FOUND>)
+else()
+  message(WARNING "Building KvikIO without CUDA")
+endif()
+
+# Enable supported cuFile features in KvikIO
+if(cuFile_FOUND)
+  target_link_libraries(kvikio INTERFACE $<BUILD_LOCAL_INTERFACE:cufile::cuFile_interface>)
+  target_compile_definitions(kvikio INTERFACE $<BUILD_LOCAL_INTERFACE:KVIKIO_CUFILE_FOUND>)
+  if(cuFile_BATCH_API_FOUND)
+    target_compile_definitions(
+      kvikio INTERFACE $<BUILD_LOCAL_INTERFACE:KVIKIO_CUFILE_BATCH_API_FOUND>
+    )
+  endif()
+  if(cuFile_STREAM_API_FOUND)
+    target_compile_definitions(
+      kvikio INTERFACE $<BUILD_LOCAL_INTERFACE:KVIKIO_CUFILE_STREAM_API_FOUND>
+    )
+  endif()
+endif()
+
 target_include_directories(
   kvikio INTERFACE "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>"
                    "$<INSTALL_INTERFACE:include>"
 )
 
 target_link_libraries(kvikio INTERFACE Threads::Threads)
-target_link_libraries(kvikio INTERFACE CUDA::toolkit)
 target_link_libraries(kvikio INTERFACE ${CMAKE_DL_LIBS})
 target_compile_features(kvikio INTERFACE cxx_std_17)
 
@@ -133,6 +165,14 @@ get_property(already_set_kvikio DIRECTORY PROPERTY kvikio_already_set_defines SE
 if(NOT already_set_kvikio)
   set_property(DIRECTORY PROPERTY kvikio_already_set_defines "ON")
 
+  find_package(CUDAToolkit QUIET)
+  if(CUDAToolkit_FOUND)
+    target_link_libraries(kvikio::kvikio INTERFACE CUDA::toolkit)
+    target_compile_definitions(kvikio::kvikio INTERFACE KVIKIO_CUDA_FOUND)
+  else()
+    message(WARNING "Building KvikIO without CUDA")
+  endif()
+
   # Find cuFile and determine which features are supported
   find_package(cuFile QUIET)
   if(NOT cuFile_FOUND)
@@ -179,7 +219,6 @@ rapids_export(
   FINAL_CODE_BLOCK final_code_string
 )
 
-# build export targets
 rapids_export(
   BUILD kvikio
   EXPORT_SET kvikio-exports