Merge branch 'development' into apply_constant_fields_directly

dpgrote · Oct 11, 2023 · 45e4096 · 45e4096
2 parents 75185dc + b71b89c
commit 45e4096
Show file tree

Hide file tree

Showing 36 changed files with 1,299 additions and 775 deletions.
diff --git a/.clang-tidy b/.clang-tidy
@@ -33,10 +33,10 @@ Checks: '-*,
     modernize-replace-auto-ptr,
     modernize-replace-disallow-copy-and-assign-macro,
     modernize-replace-random-shuffle,
-    modernize-return-braced-init-list,
     modernize-shrink-to-fit,
     modernize-unary-static-assert,
     modernize-use-nullptr,
+        -modernize-return-braced-init-list,
     mpi-*,
     performance-faster-string-find,
     performance-for-range-copy,

diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml
@@ -111,7 +111,7 @@ jobs:
         which nvcc || echo "nvcc not in PATH!"
 
         git clone https://github.com/AMReX-Codes/amrex.git ../amrex
-        cd ../amrex && git checkout --detach 2e99628138df3b5b0ecf50b0c1201d5547f821a0 && cd -
+        cd ../amrex && git checkout --detach 23.10 && cd -
         make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2
 
   build_nvhpc21-11-nvcc:

diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -45,7 +45,7 @@ jobs:
         brew install ccache
         brew install fftw
         brew install libomp
-        brew link --force libomp
+        brew link --overwrite --force libomp
         brew install ninja
         brew install open-mpi
         brew install pkg-config

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -18,7 +18,7 @@ exclude: '^share/openPMD/thirdParty'
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
     - id: trailing-whitespace
       args: [--markdown-linebreak-ext=md]

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,7 +1,7 @@
 # Preamble ####################################################################
 #
 cmake_minimum_required(VERSION 3.20.0)
-project(WarpX VERSION 23.09)
+project(WarpX VERSION 23.10)
 
 include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake)
 

diff --git a/Docs/source/conf.py b/Docs/source/conf.py
@@ -79,9 +79,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'23.09'
+version = u'23.10'
 # The full version, including alpha/beta/rc tags.
-release = u'23.09'
+release = u'23.10'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

diff --git a/Docs/source/highlights.rst b/Docs/source/highlights.rst
@@ -14,6 +14,11 @@ Plasma-Based Acceleration
 
 Scientific works in laser-plasma and beam-plasma acceleration.
 
+#. Peng, H. and Huang, T. W. and Jiang, K. and Li, R. and Wu, C. N. and Yu, M. Y. and Riconda, C. and Weber, S. and Zhou, C. T. and Ruan, S. C.
+   **Coherent Subcycle Optical Shock from a Superluminal Plasma Wake**.
+   Phys. Rev. Lett. **131**, 145003, 2023
+   `DOI:10.1103/PhysRevLett.131.145003 <https://doi.org/10.1103/PhysRevLett.131.145003>`__
+
 #. Mewes SM, Boyle GJ, Ferran Pousa A, Shalloo RJ, Osterhoff J, Arran C, Corner L, Walczak R, Hooker SM, Thévenet M.
    **Demonstration of tunability of HOFI waveguides via start-to-end simulations**.
    Phys. Rev. Research **5**, 033112, 2023
@@ -27,8 +32,8 @@ Scientific works in laser-plasma and beam-plasma acceleration.
 
 #. Wang J, Zeng M, Li D, Wang X, Gao J.
    **High quality beam produced by tightly focused laser driven wakefield accelerators**.
-   arXiv pre-print, 2023.
-   `DOI:10.48550/arXiv.2304.10730 <https://doi.org/10.48550/arXiv.2304.10730>`__
+   Phys. Rev. Accel. Beams, **26**, 091303, 2023.
+   `DOI:10.1103/PhysRevAccelBeams.26.091303 <https://doi.org/10.1103/PhysRevAccelBeams.26.091303>`__
 
 #. Fedeli L, Huebl A, Boillod-Cerneux F, Clark T, Gott K, Hillairet C, Jaure S, Leblanc A, Lehe R, Myers A, Piechurski C, Sato M, Zaim N, Zhang W, Vay J-L, Vincenti H.
    **Pushing the Frontier in the Design of Laser-Based Electron Accelerators with Groundbreaking Mesh-Refined Particle-In-Cell Simulations on Exascale-Class Supercomputers**.

diff --git a/Docs/source/install/hpc/lassen.rst b/Docs/source/install/hpc/lassen.rst
@@ -24,12 +24,27 @@ If you are new to this system, **please see the following resources**:
 Login
 -----
 
-.. note::
+.. tab-set::
 
-   Lassen is currently transitioning to RHEL8.
-   During this transition, first SSH into lassen and then ``ssh eatoss4`` next to work with the updated RHEL8/TOSS4 nodes.
+   .. tab-item:: TOSS4 (RHEL8)
 
-   Approximately October 2023, the new software environment on these nodes will be the new default.
+      Lassen is currently transitioning to RHEL8.
+      During this transition, first SSH into lassen and then to the updated RHEL8/TOSS4 nodes.
+
+      .. code-block:: bash
+
+         ssh lassen.llnl.gov
+         ssh eatoss4
+
+      Approximately October/November 2023, the new software environment on these nodes will be the new default.
+
+   .. tab-item:: TOSS3 (RHEL7)
+
+      .. code-block:: bash
+
+         ssh lassen.llnl.gov
+
+      Approximately October/November 2023, this partition will become TOSS4 (RHEL8) as well.
 
 
 .. _building-lassen-preparation:
@@ -43,78 +58,135 @@ Use the following commands to download the WarpX source code:
 
    git clone https://github.com/ECP-WarpX/WarpX.git /usr/workspace/${USER}/lassen/src/warpx
 
-We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx.profile``.
-Create it now:
+.. tab-set::
 
-.. code-block:: bash
+   .. tab-item:: TOSS4 (RHEL8)
 
-   cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example $HOME/lassen_v100_warpx.profile
+      We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx.profile``.
+      Create it now:
 
-.. dropdown:: Script Details
-   :color: light
-   :icon: info
-   :animate: fade-in-slide-down
+      .. code-block:: bash
 
-   .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example
-      :language: bash
+         cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example $HOME/lassen_v100_warpx.profile
 
-Edit the 2nd line of this script, which sets the ``export proj=""`` variable.
-For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx.profile``.
-Enter the edit mode by typing ``i`` and edit line 2 to read:
+      .. dropdown:: Script Details
+         :color: light
+         :icon: info
+         :animate: fade-in-slide-down
 
-.. code-block:: bash
+         .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example
+            :language: bash
 
-   export proj="nsldt"
+      Edit the 2nd line of this script, which sets the ``export proj=""`` variable.
+      For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx.profile``.
+      Enter the edit mode by typing ``i`` and edit line 2 to read:
 
-Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit).
+      .. code-block:: bash
 
-.. important::
+         export proj="nsldt"
 
-   Now, and as the first step on future logins to lassen, activate these environment settings:
+      Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit).
 
-   .. code-block:: bash
+      .. important::
+
+         Now, and as the first step on future logins to lassen, activate these environment settings:
+
+         .. code-block:: bash
+
+            source $HOME/lassen_v100_warpx.profile
+
+   .. tab-item:: TOSS3 (RHEL7)
+
+      We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx_toss3.profile``.
+      Create it now:
+
+      .. code-block:: bash
+
+         cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example $HOME/lassen_v100_warpx_toss3.profile
+
+      .. dropdown:: Script Details
+         :color: light
+         :icon: info
+         :animate: fade-in-slide-down
 
-      source $HOME/lassen_v100_warpx.profile
+         .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example
+            :language: bash
+
+      Edit the 2nd line of this script, which sets the ``export proj=""`` variable.
+      For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx_toss3.profile``.
+      Enter the edit mode by typing ``i`` and edit line 2 to read:
+
+      .. code-block:: bash
+
+         export proj="nsldt"
+
+      Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit).
+
+      .. important::
+
+         Now, and as the first step on future logins to lassen, activate these environment settings:
+
+         .. code-block:: bash
+
+            source $HOME/lassen_v100_warpx_toss3.profile
 
 Finally, since lassen does not yet provide software modules for some of our dependencies, install them once:
 
-.. code-block:: bash
+.. tab-set::
 
-   bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies.sh
-   source /usr/workspace/${USER}/lassen/gpu/venvs/warpx-lassen/bin/activate
+   .. tab-item:: TOSS4 (RHEL8)
 
-.. dropdown:: Script Details
-   :color: light
-   :icon: info
-   :animate: fade-in-slide-down
+      .. code-block:: bash
 
-   .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies.sh
-      :language: bash
+         bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies.sh
+         source /usr/workspace/${USER}/lassen/gpu/venvs/warpx-lassen/bin/activate
 
-.. dropdown:: AI/ML Dependencies (Optional)
-   :animate: fade-in-slide-down
+      .. dropdown:: Script Details
+         :color: light
+         :icon: info
+         :animate: fade-in-slide-down
 
-   If you plan to run AI/ML workflows depending on pyTorch, run the next step as well.
-   This will take a while and should be skipped if not needed.
+         .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies.sh
+            :language: bash
 
-   .. code-block:: bash
+      .. dropdown:: AI/ML Dependencies (Optional)
+         :animate: fade-in-slide-down
 
-      runNode bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_ml.sh
+         If you plan to run AI/ML workflows depending on pyTorch, run the next step as well.
+         This will take a while and should be skipped if not needed.
 
-   .. dropdown:: Script Details
-      :color: light
-      :icon: info
-      :animate: fade-in-slide-down
+         .. code-block:: bash
 
-      .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_ml.sh
-         :language: bash
+            runNode bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_ml.sh
 
-   For `optimas dependencies <https://github.com/optimas-org/optimas>`__ (incl. scikit-learn), plan another hour of build time:
+         .. dropdown:: Script Details
+            :color: light
+            :icon: info
+            :animate: fade-in-slide-down
 
-   .. code-block:: bash
+            .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_ml.sh
+               :language: bash
+
+         For `optimas dependencies <https://github.com/optimas-org/optimas>`__ (incl. scikit-learn), plan another hour of build time:
+
+         .. code-block:: bash
+
+            python3 -m pip install -r /usr/workspace/${USER}/lassen/src/warpx/Tools/optimas/requirements.txt
+
+   .. tab-item:: TOSS3 (RHEL7)
+
+      .. code-block:: bash
+
+         bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh
+         source /usr/workspace/${USER}/lassen/gpu/venvs/warpx-lassen-toss3/bin/activate
 
-      python3 -m pip install -r /usr/workspace/${USER}/lassen/src/warpx/Tools/optimas/requirements.txt
+      .. dropdown:: Script Details
+         :color: light
+         :icon: info
+         :animate: fade-in-slide-down
 
+         .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh
+            :language: bash
 
 .. _building-lassen-compilation:
 

diff --git a/Python/pywarpx/fields.py b/Python/pywarpx/fields.py
@@ -298,10 +298,14 @@ def _get_field(self, mfi):
         # self.mf.array(mfi) is in C ordering.
         # Note: transposing creates a view and not a copy.
         device_arr4 = self.mf.array(mfi)
-        if cp is not None:
-            device_arr = cp.array(device_arr4, copy=False).T
+        if libwarpx.libwarpx_so.Config.have_gpu:
+            if cp is not None:
+                device_arr = device_arr4.to_cupy(copy=False)
+            else:
+                # Relies on managed memory
+                device_arr = device_arr4.to_numpy(copy=False)
         else:
-            device_arr = np.array(device_arr4, copy=False).T
+            device_arr = device_arr4.to_numpy(copy=False)
         if not self.include_ghosts:
             nghosts = self._get_n_ghosts()
             device_arr = device_arr[tuple([slice(ng, -ng) for ng in nghosts[:self.dim]])]
@@ -423,11 +427,10 @@ def __getitem__(self, index):
             if global_slices is not None:
                 # Note that the array will always have 4 dimensions.
                 device_arr = self._get_field(mfi)
-                if cp is not None:
-                    # Copy the data from the device to the host
-                    slice_arr = cp.asnumpy(device_arr[block_slices])
-                else:
-                    slice_arr = device_arr[block_slices]
+                slice_arr = device_arr[block_slices]
+                if (cp is not None) and (type(slice_arr) is cp.ndarray):
+                    # Copy data from host to device using cupy syntax
+                    slice_arr = slice_arr.get()
                 datalist.append((global_slices, slice_arr))
 
         # Gather the data from all processors
@@ -528,9 +531,9 @@ def __setitem__(self, index, value):
                 mf_arr = self._get_field(mfi)
                 if isinstance(value, np.ndarray):
                     slice_value = value3d[global_slices]
-                    if cp is not None:
+                    if libwarpx.libwarpx_so.Config.have_gpu:
                         # Copy data from host to device
-                        slice_value = cp.asarray(value3d[global_slices])
+                        slice_value = cp.asarray(slice_value)
                     mf_arr[block_slices] = slice_value
                 else:
                     mf_arr[block_slices] = value

diff --git a/Python/setup.py b/Python/setup.py
@@ -54,7 +54,7 @@
     package_data = {}
 
 setup(name = 'pywarpx',
-      version = '23.09',
+      version = '23.10',
       packages = ['pywarpx'],
       package_dir = {'pywarpx': 'pywarpx'},
       description = """Wrapper of WarpX""",

diff --git a/README.md b/README.md
@@ -2,7 +2,7 @@
 
 [![Code Status development](https://dev.azure.com/ECP-WarpX/WarpX/_apis/build/status/ECP-WarpX.WarpX?branchName=development)](https://dev.azure.com/ECP-WarpX/WarpX/_build/latest?definitionId=1&branchName=development)
 [![Nightly Installation Tests](https://dev.azure.com/ECP-WarpX/WarpX/_apis/build/status/ECP-WarpX.Nightly?branchName=nightly&label=nightly%20packages)](https://dev.azure.com/ECP-WarpX/WarpX/_build?definitionId=2)
-[![Documentation Status](https://readthedocs.org/projects/warpx/badge/?version=latest)](https://warpx.readthedocs.io/en/latest/?badge=latest)
+[![Documentation Status](https://readthedocs.org/projects/warpx/badge/?version=latest)](https://warpx.readthedocs.io)
 [![Spack Version](https://img.shields.io/spack/v/warpx)](https://spack.readthedocs.io/en/latest/package_list.html#warpx)
 [![Conda Version](https://img.shields.io/conda/vn/conda-forge/warpx)](https://anaconda.org/conda-forge/warpx)
 [![Discussions](https://img.shields.io/badge/chat-discussions-turquoise.svg)](https://github.com/ECP-WarpX/WarpX/discussions)