Release v23.11.00

Release 23.11.00

Author: marcinz

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,97 @@
+name: Bug report
+description: Submit a bug report
+title: "[BUG] "
+labels: TRIAGE
+body:
+  - type: markdown
+    attributes:
+      value: "# Bug report"
+  - type: markdown
+    attributes:
+      value: Thank you for reporting a bug and helping us improve Legate!
+  - type: markdown
+    attributes:
+      value: >
+        Please fill out all of the required information.
+  - type: markdown
+    attributes:
+      value: |
+        ---
+        ## Environment information
+  - type: textarea
+    id: legate_issue
+    attributes:
+      label: Software versions
+      description: >-
+        Run `legate-issue` and paste the output here.
+      placeholder: |
+        Python      :  3.10.11 | packaged by conda-forge | (main, May 10 2023, 18:58:44) [GCC 11.3.0]
+        Platform    :  Linux-5.14.0-1042-oem-x86_64-with-glibc2.31
+        Legion      :  v23.11.00.dev-16-g2499f878
+        Legate      :  23.11.00.dev+17.gb7b50313
+        Cunumeric   :  (ImportError: cannot import name 'LogicalArray' from 'legate.core')
+        Numpy       :  1.24.4
+        Scipy       :  1.10.1
+        Numba       :  (not installed)
+        CTK package : cuda-version-11.8-h70ddcb2_2 (conda-forge)
+        GPU Driver  : 515.65.01
+        GPU Devices :
+          GPU 0: Quadro RTX 8000
+          GPU 1: Quadro RTX 8000
+    validations:
+      required: true
+  - type: input
+    id: jupyter
+    attributes:
+      label: Jupyter notebook / Jupyter Lab version
+      description: >-
+        Please supply if the issue you are reporting is related to Jupyter
+        notebook or Jupyter Lab.
+    validations:
+      required: false
+  - type: markdown
+    attributes:
+      value: |
+        ## Issue details
+  - type: textarea
+    id: expected-behavior
+    attributes:
+      label: Expected behavior
+      description: What did you expect to happen?
+    validations:
+      required: true
+  - type: textarea
+    id: observed-behavior
+    attributes:
+      label: Observed behavior
+      description: What did actually happen?
+    validations:
+      required: true
+  - type: markdown
+    attributes:
+      value: |
+        ## Directions to reproduce
+  - type: textarea
+    id: example
+    attributes:
+      label: Example code or instructions
+      description: >
+        Please provide detailed instructions to reproduce the issue. Ideally this includes a 
+        [Complete, minimal, self-contained example code](https://stackoverflow.com/help/minimal-reproducible-example)
+        given here or as a link to code in another repository. 
+      render: Python
+    validations:
+      required: true
+  - type: markdown
+    attributes:
+      value: |
+        ## Additional information
+  - type: textarea
+    id: traceback-console
+    attributes:
+      label: Stack traceback or browser console output
+      description: >
+        Add any error messages or logs that might be helpful in reproducing and
+        identifying the bug, for example a Python stack traceback.
+    validations:
+      required: false

.github/workflows/ci-gh-build-release.yml

@@ -0,0 +1,23 @@
+name: Build conda release packages
+
+concurrency:
+  group: ci-release-on-${{ github.event_name }}-from-${{ github.ref_name }}
+  cancel-in-progress: true
+
+on:
+  push:
+    workflow_dispatch:
+    branches:
+      - "pull-request/[0-9]+"
+      - "branch-*"
+
+jobs:
+  build:
+    uses:
+      ./.github/workflows/gh-build.yml
+    with:
+      build-target: all
+      repos-name: ${{ github.event.repository.name }}
+      runs-on: ${{ github.repository_owner == 'nv-legate' && 'linux-amd64-32cpu' || 'ubuntu-latest' }}
+      sha: ${{ github.sha }}
+      build-type: release

.github/workflows/ci-gh-cpu-build-and-test.yml

@@ -8,17 +8,19 @@ on:
   push:
     branches:
       - "pull-request/[0-9]+"
-      - "branch-*"
+      - "*branch-*"
 
 jobs:
   build-cpu:
+    if: ${{ github.repository_owner == 'nv-legate' }}
     uses:
       ./.github/workflows/gh-build.yml
     with:
       build-target: cpu
-      # Ref: https://docs.rapids.ai/resources/github-actions/#cpu-labels for `linux-amd64-cpu4`
-      runs-on: ${{ github.repository == 'nv-legate/legate.core' && 'linux-amd64-cpu4' || 'ubuntu-latest' }}
+      repos-name: ${{ github.event.repository.name }}
+      runs-on: ${{ contains(github.repository, 'nv-legate/legate.core') && 'linux-amd64-cpu4' || 'ubuntu-latest' }}
       sha: ${{ github.sha }}
+      build-type: ci
 
   test-cpu:
     needs:
@@ -30,11 +32,13 @@ jobs:
           - { name: Pytest Unit Tests, test-scope: unit }
           - { name: mypy, test-scope: mypy }
     name: ${{ matrix.name }}
+    if: ${{ github.repository_owner == 'nv-legate' }}
     uses:
       ./.github/workflows/gh-test.yml
     with:
       build-target: cpu
-      runs-on: ${{ github.repository == 'nv-legate/legate.core' && 'linux-amd64-cpu4' || 'ubuntu-latest' }}
+      repos-name: ${{ github.event.repository.name }}
+      runs-on: ${{ contains(github.repository, 'nv-legate/legate.core') && 'linux-amd64-32cpu' || 'ubuntu-latest' }}
       sha: ${{ github.sha }}
       test-scope: ${{ matrix.test-scope }}
 

.github/workflows/ci-gh-docs.yml

@@ -54,10 +54,10 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Generate YAML file listing dependencies
-        run: scripts/generate-conda-envs.py --python 3.10 --ctk 11.8 --os linux --compilers --openmpi
+        run: scripts/generate-conda-envs.py --python 3.10 --ctk 12.0 --os linux --compilers --openmpi
 
       - name: Install dependencies from generated YAML file
-        run: conda env create -n legate -f environment-test-linux-py3.10-cuda11.8-compilers-openmpi.yaml
+        run: conda env create -n legate -f environment-test-linux-py3.10-cuda12.0-compilers-openmpi.yaml
 
       - name: Build doxygen documentation through install.py
         run: |

.github/workflows/ci-gh-gpu-build-and-test.yml

@@ -8,17 +8,19 @@ on:
   push:
     branches:
       - "pull-request/[0-9]+"
-      - "branch-*"
+      - "*branch-*"
 
 jobs:
   build-gpu:
+    if: ${{ github.repository_owner == 'nv-legate' }}
     uses:
       ./.github/workflows/gh-build.yml
     with:
       build-target: gpu
-      # Ref: https://docs.rapids.ai/resources/github-actions/#cpu-labels for `linux-amd64-cpu4`
-      runs-on: ${{ github.repository == 'nv-legate/legate.core' && 'linux-amd64-cpu4' || 'ubuntu-latest' }}
+      repos-name: ${{ github.event.repository.name }}
+      runs-on: ${{ contains(github.repository, 'nv-legate/legate.core') && 'linux-amd64-32cpu' || 'ubuntu-latest' }}
       sha: ${{ github.sha }}
+      build-type: ci
 
   test-gpu:
     needs:
@@ -31,10 +33,12 @@ jobs:
           - { name: Pytest Unit Tests, test-scope: unit, runner: linux-amd64-2gpu }
           - { name: mypy, test-scope: mypy, runner: 'linux-amd64-gpu-v100-latest-1' }
     name: ${{ matrix.name }}
+    if: ${{ github.repository_owner == 'nv-legate' }}
     uses:
       ./.github/workflows/gh-test.yml
     with:
       build-target: gpu
+      repos-name: ${{ github.event.repository.name }}
       runs-on: ${{ matrix.runner }}
       sha: ${{ github.sha }}
       test-scope: ${{ matrix.test-scope }}

.github/workflows/gh-build.yml

@@ -6,18 +6,28 @@ on:
       build-target:
         required: true
         type: string
+        description: One of CPU / GPU
+      repos-name:
+        required: true
+        type: string
       runs-on:
         required: true
         type: string
       sha:
         required: true
         type: string
+        description: A unique identifier for labeling the images / artifacts
+      build-type:
+        required: true
+        type: string
+        description: One of ci / release
 
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   BASE_IMAGE: rapidsai/devcontainers:23.06-cpp-mambaforge-ubuntu22.04
-  IMAGE_NAME: legate.core-${{ inputs.build-target }}
+  IMAGE_NAME: ${{ inputs.repos-name }}-${{ inputs.build-target }}
   USE_CUDA: ${{ (inputs.build-target == 'cpu' && 'OFF') || 'ON' }}
+  PUSH_IMAGE: ${{ inputs.build-type == 'ci' && 'true' || 'false'}}
 
 jobs:
   build:
@@ -31,7 +41,7 @@ jobs:
     runs-on: ${{ inputs.runs-on }}
 
     steps:
-      - name: Checkout legate.core (= this repo)
+      - name: Checkout ${{ inputs.repos-name }} (= this repo)
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
@@ -51,29 +61,26 @@ jobs:
 
       - name: Build docker image
         run: |
-          echo BUILD_TARGET: ${{ inputs.build-target }}
-          echo USE_CUDA: ${{ env.USE_CUDA }}
-
           IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }}
 
           continuous_integration/build-docker-image \
             --base-image "$BASE_IMAGE" \
             --image-tag "$IMAGE_TAG" \
-            --source-dir .
+            --use-cuda "$USE_CUDA" \
+            --source-dir . \
+            --build-type "${{ inputs.build-type }}"
 
       - name: Dump docker history of image before upload
         run: |
           IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }}
           docker history $IMAGE_TAG
 
-      - name: Dump docker history of image before upload
-        run: |
-          IMAGE_TAG=${{ env.IMAGE_NAME_CUNUMERIC }}:${{ inputs.sha }}
-
-      - name: Log in to container image registry
+      - if: env.PUSH_IMAGE == 'true'
+        name: Log in to container image registry
         run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
 
-      - name: Push image
+      - if: env.PUSH_IMAGE == 'true'
+        name: Push image
         run: |
           IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }}
 
@@ -91,14 +98,13 @@ jobs:
         run: |
           IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }}
           mkdir -p artifacts
-          docker run  -v "$(pwd)/artifacts:/home/coder/.artifacts" --rm -t $IMAGE_TAG copy-artifacts
-
+          docker run  -v "$(pwd)/artifacts:/home/coder/.artifacts" --rm -t $IMAGE_TAG copy-artifacts ${{ inputs.build-type }}
 
       - name: Display structure of workdir
         run: ls -R
 
       - name: Upload build artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: "legate.core-${{ inputs.build-target }}-${{ inputs.sha }}"
+          name: "${{ inputs.repos-name }}-${{ inputs.build-target }}-${{ inputs.sha }}"
           path: artifacts

.github/workflows/gh-test.yml

@@ -6,6 +6,9 @@ on:
       build-target:
         required: true
         type: string
+      repos-name:
+        required: true
+        type: string
       runs-on:
         required: true
         type: string
@@ -23,7 +26,7 @@ jobs:
     runs-on: ${{ inputs.runs-on }}
     container:
       options: -u root
-      image: ghcr.io/nv-legate/legate.core-${{ inputs.build-target }}:${{ inputs.sha }}
+      image: ghcr.io/nv-legate/${{ inputs.repos-name }}-${{ inputs.build-target }}:${{ inputs.sha }}
       env:
         PYTHONDONTWRITEBYTECODE: 1
         NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}

.pre-commit-config.yaml

@@ -4,11 +4,11 @@ repos:
       hooks:
             - id: isort
     - repo: https://github.com/psf/black
-      rev: 23.3.0
+      rev: 23.9.1
       hooks:
             - id: black
     - repo: https://github.com/PyCQA/flake8
-      rev: 6.0.0
+      rev: 6.1.0
       hooks:
             - id: flake8
     - repo: https://github.com/pre-commit/mirrors-clang-format
@@ -18,7 +18,7 @@ repos:
           files: \.(cu|cuh|h|cc|inl)$
           types_or: []
     - repo: https://github.com/pre-commit/mirrors-mypy
-      rev: 'v1.4.1'
+      rev: 'v1.5.1'
       hooks:
         - id: mypy
           pass_filenames: false