Merge branch 'development' into add_insulator_BC

.github/workflows/cuda.yml

@@ -115,7 +115,7 @@ jobs:
         which nvcc || echo "nvcc not in PATH!"
 
         git clone https://github.com/AMReX-Codes/amrex.git ../amrex
-        cd ../amrex && git checkout --detach 24.06 && cd -
+        cd ../amrex && git checkout --detach 259db7cfb99e7d1d2ab4bec9b1587fdf624a138a && cd -
         make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4
 
         ccache -s

.github/workflows/macos.yml

@@ -44,7 +44,7 @@ jobs:
     - name: CCache Cache
       uses: actions/cache@v4
       with:
-        path: /Users/runner/Library/Caches/ccache
+        path: ~/Library/Caches/ccache
         key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
         restore-keys: |
              ccache-${{ github.workflow }}-${{ github.job }}-git-
@@ -53,7 +53,7 @@ jobs:
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10
         export CCACHE_MAXSIZE=100M
-        export CCACHE_DEPEND=1
+        export CCACHE_SLOPPINESS=time_macros
         ccache -z
 
         source py-venv/bin/activate
@@ -76,6 +76,7 @@ jobs:
         cmake --build build_sp -j 3
         cmake --build build_sp --target pip_install
 
+        du -hs ~/Library/Caches/ccache
         ccache -s
 
     - name: run pywarpx

Docs/source/acknowledge_us.rst

@@ -23,14 +23,14 @@ Please add the following sentence to your publications, it helps contributors ke
 
 **Plain text:**
 
-  This research used the open-source particle-in-cell code WarpX https://github.com/ECP-WarpX/WarpX, primarily funded by the US DOE Exascale Computing Project. Primary WarpX contributors are with LBNL, LLNL, CEA-LIDYL, SLAC, DESY, CERN, and TAE Technologies. We acknowledge all WarpX contributors.
+  This research used the open-source particle-in-cell code WarpX https://github.com/ECP-WarpX/WarpX. Primary WarpX contributors are with LBNL, LLNL, CEA-LIDYL, SLAC, DESY, CERN, and TAE Technologies. We acknowledge all WarpX contributors.
 
 **LaTeX:**
 
 .. code-block:: latex
 
   \usepackage{hyperref}
-  This research used the open-source particle-in-cell code WarpX \url{https://github.com/ECP-WarpX/WarpX}, primarily funded by the US DOE Exascale Computing Project.
+  This research used the open-source particle-in-cell code WarpX \url{https://github.com/ECP-WarpX/WarpX}.
   Primary WarpX contributors are with LBNL, LLNL, CEA-LIDYL, SLAC, DESY, CERN, and TAE Technologies.
   We acknowledge all WarpX contributors.
 
@@ -55,8 +55,8 @@ If your project uses a specific algorithm or component, please consider citing t
 
 - Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A.
   **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**.
-  Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024.
-  `preprint <http://arxiv.org/abs/2402.17248>__`
+  Proc. of Platform for Advanced Scientific Computing (PASC'24), *PASC24 Best Paper Award*, 2024.
+  `DOI:10.1145/3659914.3659937 <https://doi.org/10.1145/3659914.3659937>`__
 
 - Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A.
   **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**.

Docs/source/highlights.rst

@@ -14,7 +14,17 @@ Plasma-Based Acceleration
 
 Scientific works in laser-plasma and beam-plasma acceleration.
 
-#. Peng, H. and Huang, T. W. and Jiang, K. and Li, R. and Wu, C. N. and Yu, M. Y. and Riconda, C. and Weber, S. and Zhou, C. T. and Ruan, S. C.
+#. Ross AJ, Chappell J, van de Wetering JJ, Cowley J, Archer E, Bourgeois N, Corner L, Emerson DR, Feder L, Gu XJ, Jakobsson O, Jones H, Picksley A, Reid L, Wang W, Walczak R, Hooker SM.
+   **Resonant excitation of plasma waves in a plasma channel**.
+   Phys. Rev. Research **6**, L022001, 2024
+   `DOI:10.1103/PhysRevResearch.6.L022001 <https://doi.org/10.1103/PhysRevResearch.6.L022001>`__
+
+#. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A.
+   **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**.
+   Proc. of Platform for Advanced Scientific Computing (PASC'24), *PASC24 Best Paper Award*, 2024.
+   `DOI:10.1145/3659914.3659937 <https://doi.org/10.1145/3659914.3659937>`__
+
+#. Peng H, Huang TW, Jiang K, Li R, Wu CN, Yu MY, Riconda C, Weber S, Zhou CT, Ruan SC.
    **Coherent Subcycle Optical Shock from a Superluminal Plasma Wake**.
    Phys. Rev. Lett. **131**, 145003, 2023
    `DOI:10.1103/PhysRevLett.131.145003 <https://doi.org/10.1103/PhysRevLett.131.145003>`__
@@ -24,11 +34,6 @@ Scientific works in laser-plasma and beam-plasma acceleration.
    Phys. Rev. Research **5**, 033112, 2023
    `DOI:10.1103/PhysRevResearch.5.033112 <https://doi.org/10.1103/PhysRevResearch.5.033112>`__
 
-#. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A.
-   **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**.
-   Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024.
-   `preprint <http://arxiv.org/abs/2402.17248>`__
-
 #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A.
    **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**.
    14th International Particle Accelerator Conference (IPAC'23), WEPA101, 2023.
@@ -103,14 +108,20 @@ Scientific works in particle and beam modeling.
 
 #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A.
    **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**.
-   Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024.
-   `preprint <http://arxiv.org/abs/2402.17248>`__
+   Proc. of Platform for Advanced Scientific Computing (PASC'24), *PASC24 Best Paper Award*, 2024.
+   `DOI:10.1145/3659914.3659937 <https://doi.org/10.1145/3659914.3659937>`__
+
+#. Nguyen B, Formenti A, Lehe R, Vay J-L, Gessner S, and Fedeli L.
+   **Comparison of WarpX and GUINEA-PIG for electron positron collisions**.
+   15th International Particle Accelerator Conference (IPAC'24), WEPC84, 2024.
+   `preprint <https://arxiv.org/abs/2405.09583>`__,
+   `DOI:10.18429/JACoW-IPAC2024-WEPC84 <https://doi.org/10.18429/JACoW-IPAC2024-WEPC84>`__
 
 #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L, Huebl A.
    **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**.
-   14th International Particle Accelerator Conference (IPAC'23), WEPA101, *in print*, 2023.
+   14th International Particle Accelerator Conference (IPAC'23), WEPA101, 2023.
    `preprint <https://www.ipac23.org/preproc/pdf/WEPA101.pdf>`__,
-   `DOI:10.18429/JACoW-IPAC-23-WEPA101 <https://doi.org/10.18429/JACoW-IPAC-23-WEPA101>`__
+   `DOI:10.18429/JACoW-IPAC2023-WEPA101 <https://doi.org/10.18429/JACoW-IPAC2023-WEPA101>`__
 
 #. Tan W H, Piot P, Myers A, Zhang W, Rheaume T, Jambunathan R, Huebl A, Lehe R, Vay J-L.
    **Simulation studies of drive-beam instability in a dielectric wakefield accelerator**.

Docs/source/refs.bib

@@ -205,17 +205,20 @@ @article{Roedel2010
 year = {2010}
 }
 
-@misc{SandbergPASC24,
-address = {Zuerich, Switzerland},
-author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Andrew Myers and Ji Qiang and Jean-Luc Vay and Axel Huebl},
-booktitle = {Proc. of PASC24},
-note = {accepted},
-series = {PASC'24 - Platform for Advanced Scientific Computing},
-title = {{Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines}},
-venue = {Zuerich, Switzerland},
-year = {2024},
-doi = {10.48550/arXiv.2402.17248},
-url = {https://arxiv.org/abs/2402.17248}
+@inproceedings{SandbergPASC24,
+author = {Sandberg, Ryan and Lehe, Remi and Mitchell, Chad and Garten, Marco and Myers, Andrew and Qiang, Ji and Vay, Jean-Luc and Huebl, Axel},
+title = {{Synthesizing Particle-In-Cell Simulations through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines}},
+series = {PASC '24},
+booktitle = {Proceedings of the Platform for Advanced Scientific Computing Conference},
+location = {Zuerich, Switzerland},
+publisher = {Association for Computing Machinery},
+address = {New York, NY, USA},
+articleno = {23},
+numpages = {11},
+isbn = {9798400706394},
+doi = {10.1145/3659914.3659937},
+note = {{PASC24 Best Paper Award}},
+year = {2024}
 }
 
 @inproceedings{SandbergIPAC23,

Docs/source/usage/parameters.rst

@@ -88,7 +88,7 @@ Overall simulation parameters
 
     * ``theta_implicit_em``: Use a fully implicit electromagnetic solver with a time-biasing parameter theta bound between 0.5 and 1.0. Exact energy conservation is achieved using theta = 0.5. Maximal damping of high-k modes is obtained using theta = 1.0. Choices for the nonlinear solver include a Picard iteration scheme and particle-suppressed (PS) JNFK.
       The algorithm itself is numerical stable for large time steps. That is, it does not require time steps that resolve the plasma period or the CFL condition for light waves. However, the practicality of using a large time step depends on the nonlinear solver. Note that the Picard solver is for demonstration only. It is inefficient and will most like not converge when
-      :math:`\omega_{pe} \Delta t` is close to or greater than one or when the CFL condition for light waves is violated. The PS-JFNK method must be used in order to use large time steps. However, the current implementation of PS-JFNK is still inefficient because the JFNK solver is not preconditioned and there is no use of the mass matrices to minimize the cost of a linear iteration. The time step is limited by how many cells a particle can cross in a time step (MPI-related) and by the need to resolve the relavent physics.
+      :math:`\omega_{pe} \Delta t` is close to or greater than one or when the CFL condition for light waves is violated. The PS-JFNK method must be used in order to use large time steps. However, the current implementation of PS-JFNK is still inefficient because the JFNK solver is not preconditioned and there is no use of the mass matrices to minimize the cost of a linear iteration. The time step is limited by how many cells a particle can cross in a time step (MPI-related) and by the need to resolve the relevant physics.
       The Picard method is described in `Angus et al., On numerical energy conservation for an implicit particle-in-cell method coupled with a binary Monte-Carlo algorithm for Coulomb collisions <https://doi.org/10.1016/j.jcp.2022.111030>`__.
       The PS-JFNK method is described in `Angus et al., An implicit particle code with exact energy and charge conservation for electromagnetic studies of dense plasmas <https://doi.org/10.1016/j.jcp.2023.112383>`__ . (The version implemented in WarpX is an updated version that includes the relativistic gamma factor for the particles.) Also see `Chen et al., An energy- and charge-conserving, implicit, electrostatic particle-in-cell algorithm. <https://doi.org/10.1016/j.jcp.2011.05.031>`__ .
       Exact energy conservation requires that the interpolation stencil used for the field gather match that used for the current deposition. ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1``. If using ``algo.current_deposition = villasenor``, the corresponding field gather routine will automatically be selected and the ``interpolation.galerkin_scheme`` flag does not need to be specified. The Esirkepov and villasenor deposition schemes are charge-conserving.
@@ -612,11 +612,26 @@ Additional PML parameters
 Embedded Boundary Conditions
 ----------------------------
 
-* ``warpx.eb_implicit_function`` (`string`)
-    A function of `x`, `y`, `z` that defines the surface of the embedded
-    boundary. That surface lies where the function value is 0 ;
-    the physics simulation area is where the function value is negative ;
-    the interior of the embeddded boundary is where the function value is positive.
+In WarpX, the embedded boundary can be defined in either of two ways:
+
+    - **From an analytical function:**
+        In that case, you will need to set the following parameter in the input file.
+
+        * ``warpx.eb_implicit_function`` (`string`)
+            A function of `x`, `y`, `z` that defines the surface of the embedded
+            boundary. That surface lies where the function value is 0 ;
+            the physics simulation area is where the function value is negative ;
+            the interior of the embeddded boundary is where the function value is positive.
+
+    - **From an STL file:**
+        In that case, you will need to set the following parameters in the input file.
+
+        * ``eb2.stl_file`` (`string`)
+            The path to an STL file. In addition, you also need to set ``eb2.geom_type = stl``,
+            in order for the file to be read by WarpX.
+
+Whether the embedded boundary is defined with an analytical function or an STL file, you can
+additionally define the electric potential at the embedded boundary with an analytical function:
 
 * ``warpx.eb_potential(x,y,z,t)`` (`string`)
     Gives the value of the electric potential at the surface of the embedded boundary,

Examples/Tests/collision/inputs_3d

@@ -22,6 +22,7 @@ boundary.field_hi = periodic periodic periodic
 warpx.serialize_initial_conditions = 1
 warpx.verbose = 1
 warpx.const_dt = 1.224744871e-07
+warpx.random_seed = 2034958209
 
 # Do not evolve the E and B fields
 algo.maxwell_solver = none

Examples/Tests/repelling_particles/analysis_repelling.py

@@ -36,7 +36,7 @@
 
 # Check plotfile name specified in command line
 last_filename = sys.argv[1]
-filename_radical = re.findall('(.*?)\d+/*$', last_filename)[0]
+filename_radical = re.findall(r'(.*?)\d+/*$', last_filename)[0]
 
 # Loop through files, and extract the position and velocity of both particles
 x1 = []
@@ -48,10 +48,10 @@
     ds = yt.load(filename)
     ad = ds.all_data()
 
-    x1.append( float(ad[('electron1','particle_position_x')]) )
-    x2.append( float(ad[('electron2','particle_position_x')]) )
-    beta1.append( float(ad[('electron1','particle_momentum_x')])/(m_e*c) )
-    beta2.append( float(ad[('electron2','particle_momentum_x')])/(m_e*c) )
+    x1.append( float(ad[('electron1','particle_position_x')][0]) )
+    x2.append( float(ad[('electron2','particle_position_x')][0]) )
+    beta1.append( float(ad[('electron1','particle_momentum_x')][0])/(m_e*c) )
+    beta2.append( float(ad[('electron2','particle_momentum_x')][0])/(m_e*c) )
 
 # Convert to numpy array
 x1 = np.array(x1)

Regression/Checksum/benchmarks_json/BTD_rz.json

@@ -1,13 +1,13 @@
 {
   "lev=0": {
-    "Br": 1.8705552264208163e-08,
-    "Bt": 2380179.5677922587,
-    "Bz": 2.4079077116116535e-08,
+    "Br": 1.8705569155952808e-08,
+    "Bt": 2380179.567792259,
+    "Bz": 2.4079063431898616e-08,
     "Er": 497571119514841.25,
-    "Et": 7.048225464720808,
-    "Ez": 137058870936728.28,
+    "Et": 7.048223219552,
+    "Ez": 137058870936728.23,
     "jr": 0.0,
     "jt": 0.0,
     "jz": 0.0
   }
-}
+}

Regression/Checksum/benchmarks_json/PEC_particle.json

@@ -1,31 +1,31 @@
 {
   "lev=0": {
-    "Bx": 6.52080713617892e-07,
-    "By": 5.323555640562078e-18,
-    "Bz": 1.1834601056649145e-06,
-    "Ex": 300.06725745121753,
-    "Ey": 320.82687281173185,
-    "Ez": 185.62208934593673,
-    "jx": 1.857782359527958e-06,
-    "jy": 257766.55567863808,
+    "Bx": 6.5208071361789e-07,
+    "By": 5.323579900863346e-18,
+    "Bz": 1.1834601056649094e-06,
+    "Ex": 300.0672574512165,
+    "Ey": 320.82687281173,
+    "Ez": 185.62208934593605,
+    "jx": 1.8577823595279576e-06,
+    "jy": 257766.5556786359,
     "jz": 0.0
   },
+  "electron": {
+    "particle_momentum_x": 1.1142179380730402e-32,
+    "particle_momentum_y": 4.5160903214571184e-36,
+    "particle_momentum_z": 5.735409609139057e-50,
+    "particle_position_x": 3.199800000000006e-05,
+    "particle_position_y": 6.267072536608953e-23,
+    "particle_position_z": 3.3938629027782103e-37,
+    "particle_weight": 1.0
+  },
   "proton": {
-    "particle_momentum_x": 1.0975605406855061e-33,
+    "particle_momentum_x": 1.097560540685506e-33,
     "particle_momentum_y": 1.002878875615427e-18,
-    "particle_momentum_z": 5.084424741580567e-51,
+    "particle_momentum_z": 8.484417024161229e-51,
     "particle_position_x": 3.1998e-05,
     "particle_position_y": 6.572670690061996e-06,
-    "particle_position_z": 5.681444715216788e-38,
-    "particle_weight": 1.0
-  },
-  "electron": {
-    "particle_momentum_x": 1.1142179380730476e-32,
-    "particle_momentum_y": 4.516090321457367e-36,
-    "particle_momentum_z": 2.2354046360787e-50,
-    "particle_position_x": 3.199800000000006e-05,
-    "particle_position_y": 6.267072536609136e-23,
-    "particle_position_z": 1.4145775327446577e-37,
+    "particle_position_z": 4.372139867361774e-39,
     "particle_weight": 1.0
   }
 }

Regression/Checksum/benchmarks_json/Python_ionization.json

@@ -3,30 +3,30 @@
     "Bx": 0.0,
     "By": 26296568.434868,
     "Bz": 0.0,
-    "Ex": 7878103122971888.0,
+    "Ex": 7878103122971890.0,
     "Ey": 0.0,
-    "Ez": 3027.995293554466,
-    "jx": 1.2111358330750162e+16,
+    "Ez": 3027.738911163427,
+    "jx": 1.2111358330750164e+16,
     "jy": 0.0,
-    "jz": 1.3483401471475687e-07
+    "jz": 1.3523766702993914e-07
+  },
+  "electrons": {
+    "particle_momentum_x": 4.4237309006289776e-18,
+    "particle_momentum_y": 0.0,
+    "particle_momentum_z": 2.643146520891287e-18,
+    "particle_orig_z": 0.4306994012391905,
+    "particle_position_x": 0.11013574385450274,
+    "particle_position_y": 0.6415447480129455,
+    "particle_weight": 3.4456249999999997e-10
   },
   "ions": {
     "particle_ionizationLevel": 72897.0,
-    "particle_momentum_x": 1.76132401934254e-18,
+    "particle_momentum_x": 1.7613240193425407e-18,
     "particle_momentum_y": 0.0,
-    "particle_momentum_z": 3.644887053263054e-23,
+    "particle_momentum_z": 3.6448870533515125e-23,
     "particle_orig_z": 0.128,
     "particle_position_x": 0.03200001189420337,
     "particle_position_y": 0.1280000046901387,
     "particle_weight": 9.999999999999999e-11
-  },
-  "electrons": {
-    "particle_momentum_x": 4.4206237143449475e-18,
-    "particle_momentum_y": 0.0,
-    "particle_momentum_z": 2.6361297302081026e-18,
-    "particle_orig_z": 0.4305565137391907,
-    "particle_position_x": 0.11009154442846772,
-    "particle_position_y": 0.6414658436421568,
-    "particle_weight": 3.4450781249999996e-10
   }
 }

Regression/Checksum/benchmarks_json/RepellingParticles.json

@@ -1,32 +1,32 @@
 {
+  "lev=0": {
+    "Bx": 0.0,
+    "By": 10603.681421961119,
+    "Bz": 0.0,
+    "Ex": 12084698236273.04,
+    "Ey": 0.0,
+    "Ez": 16282812598779.527,
+    "F": 21124.39775810062,
+    "divE": 1.495787831639977e+18,
+    "jx": 496020344341010.9,
+    "jy": 0.0,
+    "jz": 10.641859867595102,
+    "rho": 6408706.535999998
+  },
   "electron1": {
-    "particle_momentum_x": 7.303686586478214e-23,
+    "particle_momentum_x": 7.303686586478215e-23,
     "particle_momentum_y": 0.0,
-    "particle_momentum_z": 1.556862195127266e-36,
-    "particle_position_x": 1.293487226551532e-05,
-    "particle_position_y": 1.777889117487881e-19,
+    "particle_momentum_z": 1.552083282915395e-36,
+    "particle_position_x": 1.293487226551533e-05,
+    "particle_position_y": 1.7783750855614515e-19,
     "particle_weight": 5000000000000.0
   },
   "electron2": {
-    "particle_momentum_x": 7.303686586477884e-23,
+    "particle_momentum_x": 7.303686586477887e-23,
     "particle_momentum_y": 0.0,
-    "particle_momentum_z": 1.567350238825104e-36,
+    "particle_momentum_z": 1.5731022908002534e-36,
     "particle_position_x": 1.293487226551495e-05,
-    "particle_position_y": 1.797289839952283e-19,
+    "particle_position_y": 1.7989574553330234e-19,
     "particle_weight": 5000000000000.0
-  },
-  "lev=0": {
-    "Bx": 0.0,
-    "By": 10603.68142196111,
-    "Bz": 0.0,
-    "Ex": 12084698236273.04,
-    "Ey": 0.0,
-    "Ez": 16282812598779.53,
-    "F": 21124.39775810061,
-    "divE": 1.495787831639981e+18,
-    "jx": 496020344341011.4,
-    "jy": 0.0,
-    "jz": 10.93513947024536,
-    "rho": 6408706.535999998
   }
-}
+}