v23.09.00

v23.09.00

Author: marcinz

.github/copy-pr-bot.yaml

@@ -0,0 +1,4 @@
+# Configuration file for `copy-pr-bot` GitHub App
+# https://docs.gha-runners.nvidia.com/apps/copy-pr-bot/
+
+enabled: true

.github/ops-bot.yaml

@@ -4,6 +4,7 @@ on:
   push:
     branches-ignore:
       - gh-pages  # deployment target branch (this workflow should not exist on that branch anyway)
+      - "pull-request/[0-9]+"
   pull_request:
     branches-ignore:
       - gh-pages  # deployment target branch (this workflow should not exist on that branch anyway)
@@ -44,14 +45,14 @@ jobs:
         env:
           MATRIX_CONTEXT: ${{ toJSON(matrix) }}
         run: echo "$MATRIX_CONTEXT"
-      
+
       ####################################
       # Actual build process starts here #
       ####################################
-      
+
       - name: Checkout
         uses: actions/checkout@v3
-      
+
       - name: Generate YAML file listing dependencies
         run: scripts/generate-conda-envs.py --python 3.10 --ctk 11.8 --os linux --compilers --openmpi
 
@@ -61,7 +62,7 @@ jobs:
       - name: Build doxygen documentation through install.py
         run: |
           conda run -n legate /bin/bash -c "./install.py --docs"
-      
+
       - name: Build documentation using Makefile
         working-directory: ./docs/legate/core
         run: |

.github/workflows/ci-gh-docs.yml

@@ -27,14 +27,15 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: Pytest Unit Tests, test-scope: unit }
-          - { name: mypy, test-scope: mypy }
+          - { name: Pytest Unit Tests, test-scope: unit, runner: linux-amd64-gpu-v100-latest-1 }
+          - { name: Pytest Unit Tests, test-scope: unit, runner: linux-amd64-2gpu }
+          - { name: mypy, test-scope: mypy, runner: 'linux-amd64-gpu-v100-latest-1' }
     name: ${{ matrix.name }}
     uses:
       ./.github/workflows/gh-test.yml
     with:
       build-target: gpu
-      runs-on: linux-amd64-gpu-v100-latest-1
+      runs-on: ${{ matrix.runner }}
       sha: ${{ github.sha }}
       test-scope: ${{ matrix.test-scope }}
 

.github/workflows/ci-gh-gpu-build-and-test.yml

@@ -15,6 +15,7 @@ on:
 
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  BASE_IMAGE: rapidsai/devcontainers:23.06-cpp-mambaforge-ubuntu22.04
   IMAGE_NAME: legate.core-${{ inputs.build-target }}
   USE_CUDA: ${{ (inputs.build-target == 'cpu' && 'OFF') || 'ON' }}
 
@@ -43,44 +44,55 @@ jobs:
           role-duration-seconds: 28800 # 8 hours
           role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-nv-legate
 
+      - name: Docker system prune
+        run: |
+          docker version
+          docker system prune --all --force
+
       - name: Build docker image
         run: |
           echo BUILD_TARGET: ${{ inputs.build-target }}
           echo USE_CUDA: ${{ env.USE_CUDA }}
 
-          docker build \
-            --build-arg AWS_SESSION_TOKEN=${{ env.AWS_SESSION_TOKEN }} \
-            --build-arg AWS_ACCESS_KEY_ID=${{ env.AWS_ACCESS_KEY_ID }} \
-            --build-arg AWS_SECRET_ACCESS_KEY=${{ env.AWS_SECRET_ACCESS_KEY }} \
-            --build-arg GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} \
-            --build-arg USE_CUDA=${{ env.USE_CUDA }} \
-            --progress=plain \
-            --tag=$IMAGE_NAME:${{ inputs.sha }} \
-            --label "git-commit=${{ inputs.sha }}" \
-            -f continuous_integration/Dockerfile .
+          IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }}
+
+          continuous_integration/build-docker-image \
+            --base-image "$BASE_IMAGE" \
+            --image-tag "$IMAGE_TAG" \
+            --source-dir .
+
+      - name: Dump docker history of image before upload
+        run: |
+          IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }}
+          docker history $IMAGE_TAG
+
+      - name: Dump docker history of image before upload
+        run: |
+          IMAGE_TAG=${{ env.IMAGE_NAME_CUNUMERIC }}:${{ inputs.sha }}
 
       - name: Log in to container image registry
         run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
 
       - name: Push image
         run: |
-          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+          IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }}
+
+          IMAGE_ID=ghcr.io/${{ github.repository_owner }}
 
           # Change all uppercase to lowercase
           IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
 
-          VERSION=${{ inputs.sha }}
+          IMAGE_ID=$IMAGE_ID/$IMAGE_TAG
 
-          docker tag $IMAGE_NAME:$VERSION $IMAGE_ID:$VERSION
-          docker push $IMAGE_ID:$VERSION
+          docker tag $IMAGE_TAG $IMAGE_ID
+          docker push $IMAGE_ID
 
       - name: Copy artifacts back to the host
         run: |
+          IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }}
           mkdir -p artifacts
-          docker run  -v "$(pwd)/artifacts:/home/coder/.artifacts" --rm -t $IMAGE_NAME:${{ inputs.sha }} copy-artifacts
-          echo --------- DOCKER HISTORY START -----------
-          docker history $IMAGE_NAME:${{ inputs.sha }}
-          echo --------- DOCKER HISTORY END -----------
+          docker run  -v "$(pwd)/artifacts:/home/coder/.artifacts" --rm -t $IMAGE_TAG copy-artifacts
+
 
       - name: Display structure of workdir
         run: ls -R

.github/workflows/ci.yml

@@ -33,8 +33,8 @@ environment file listing all the packages that are required to build, run and
 test Legate Core and all downstream libraries. For example:
 
 ```shell
-$ ./scripts/generate-conda-envs.py --python 3.10 --ctk 11.7 --os linux --compilers --openmpi
---- generating: environment-test-linux-py310-cuda-11.7-compilers-openmpi.yaml
+$ ./scripts/generate-conda-envs.py --python 3.10 --ctk 11.8 --os linux --compilers --openmpi
+--- generating: environment-test-linux-py310-cuda-11.8-compilers-openmpi.yaml
 ```
 
 Run this script with `-h` to see all available configuration options for the

.github/workflows/gh-build.yml

@@ -72,7 +72,7 @@ include(rapids-cuda)
 include(rapids-export)
 include(rapids-find)
 
-set(legate_core_version 23.07.00)
+set(legate_core_version 23.09.00)
 
 # For now we want the optimization flags to match on both normal make and cmake
 # builds so we override the cmake defaults here for release, this changes

BUILD.md

@@ -17,34 +17,34 @@
 function(set_cuda_arch_from_names)
   set(cuda_archs "")
   # translate legacy arch names into numbers
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "fermi")
+  if(Legion_CUDA_ARCH MATCHES "fermi")
     list(APPEND cuda_archs 20)
   endif()
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "kepler")
+  if(Legion_CUDA_ARCH MATCHES "kepler")
     list(APPEND cuda_archs 30)
   endif()
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "k20")
+  if(Legion_CUDA_ARCH MATCHES "k20")
     list(APPEND cuda_archs 35)
   endif()
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "k80")
+  if(Legion_CUDA_ARCH MATCHES "k80")
     list(APPEND cuda_archs 37)
   endif()
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "maxwell")
+  if(Legion_CUDA_ARCH MATCHES "maxwell")
     list(APPEND cuda_archs 52)
   endif()
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "pascal")
+  if(Legion_CUDA_ARCH MATCHES "pascal")
     list(APPEND cuda_archs 60)
   endif()
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "volta")
+  if(Legion_CUDA_ARCH MATCHES "volta")
     list(APPEND cuda_archs 70)
   endif()
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "turing")
+  if(Legion_CUDA_ARCH MATCHES "turing")
     list(APPEND cuda_archs 75)
   endif()
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "ampere")
+  if(Legion_CUDA_ARCH MATCHES "ampere")
     list(APPEND cuda_archs 80)
   endif()
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "hopper")
+  if(Legion_CUDA_ARCH MATCHES "hopper")
     list(APPEND cuda_archs 90)
   endif()
 
@@ -60,19 +60,24 @@ function(set_cuda_arch_from_names)
     else()
       list(TRANSFORM cuda_archs APPEND "-real")
     endif()
-    set(CMAKE_CUDA_ARCHITECTURES ${cuda_archs} PARENT_SCOPE)
+    set(Legion_CUDA_ARCH ${cuda_archs} PARENT_SCOPE)
   endif()
 endfunction()
 
-function(add_cuda_architecture_defines defs)
-  message(VERBOSE "legate.core: CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}")
+function(add_cuda_architecture_defines)
+  set(options )
+  set(oneValueArgs DEFS)
+  set(multiValueArgs ARCHS)
+  cmake_parse_arguments(cuda "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
 
-  set(_defs ${${defs}})
+  message(VERBOSE "legate.core: CUDA_ARCHITECTURES=${cuda_ARCHS}")
+
+  set(_defs ${${cuda_DEFS}})
 
   macro(add_def_if_arch_enabled arch def)
-    if("${arch}" IN_LIST CMAKE_CUDA_ARCHITECTURES OR
-      ("${arch}-real" IN_LIST CMAKE_CUDA_ARCHITECTURES) OR
-      ("${arch}-virtual" IN_LIST CMAKE_CUDA_ARCHITECTURES))
+    if("${arch}" IN_LIST cuda_ARCHS OR
+      ("${arch}-real" IN_LIST cuda_ARCHS) OR
+      ("${arch}-virtual" IN_LIST cuda_ARCHS))
       list(APPEND _defs ${def})
     endif()
   endmacro()
@@ -88,5 +93,5 @@ function(add_cuda_architecture_defines defs)
   add_def_if_arch_enabled("80" "AMPERE_ARCH")
   add_def_if_arch_enabled("90" "HOPPER_ARCH")
 
-  set(${defs} ${_defs} PARENT_SCOPE)
+  set(${cuda_DEFS} ${_defs} PARENT_SCOPE)
 endfunction()

CMakeLists.txt

@@ -87,7 +87,7 @@ option(legate_core_BUILD_DOCS "Build doxygen docs" OFF)
 set_or_default(NCCL_DIR NCCL_PATH)
 set_or_default(Thrust_DIR THRUST_PATH)
 set_or_default(CUDA_TOOLKIT_ROOT_DIR CUDA)
-set_or_default(CMAKE_CUDA_ARCHITECTURES GPU_ARCH NATIVE)
+set_or_default(Legion_CUDA_ARCH GPU_ARCH all-major)
 set_or_default(Legion_HIJACK_CUDART USE_CUDART_HIJACK OFF)
 
 include(CMakeDependentOption)