Merge branch 'development' into add_insulator_BC

.github/workflows/dependencies/hip.sh

@@ -12,12 +12,24 @@ set -eu -o pipefail
 #   failed files the given number of times.
 echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries
 
-# Ref.: https://rocmdocs.amd.com/en/latest/Installation_Guide/Installation-Guide.html#ubuntu
-wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key \
-  | sudo apt-key add -
-echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/debian/ ubuntu main' \
-  | sudo tee /etc/apt/sources.list.d/rocm.list
+# Ref.: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/native-install/ubuntu.html
+
+# Make the directory if it doesn't exist yet.
+# This location is recommended by the distribution maintainers.
+sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+
+# Download the key, convert the signing-key to a full
+# keyring required by apt and store in the keyring directory
+wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | \
+    gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
 
+curl -O https://repo.radeon.com/rocm/rocm.gpg.key
+sudo apt-key add rocm.gpg.key
+
+source /etc/os-release # set UBUNTU_CODENAME: focal or jammy or ...
+
+echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${1-latest} ${UBUNTU_CODENAME} main" \
+  | sudo tee /etc/apt/sources.list.d/rocm.list
 echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' \
   | sudo tee -a /etc/profile.d/rocm.sh
 

Docs/source/highlights.rst

@@ -25,9 +25,9 @@ Scientific works in laser-plasma and beam-plasma acceleration.
    `DOI:10.1103/PhysRevResearch.5.033112 <https://doi.org/10.1103/PhysRevResearch.5.033112>`__
 
 #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A.
-  **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**.
-  Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024.
-  `preprint <http://arxiv.org/abs/2402.17248>__`
+   **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**.
+   Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024.
+   `preprint <http://arxiv.org/abs/2402.17248>`__
 
 #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A.
    **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**.
@@ -102,9 +102,9 @@ Particle Accelerator & Beam Physics
 Scientific works in particle and beam modeling.
 
 #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A.
-  **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**.
-  Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024.
-  `preprint <http://arxiv.org/abs/2402.17248>__`
+   **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**.
+   Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024.
+   `preprint <http://arxiv.org/abs/2402.17248>`__
 
 #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L, Huebl A.
    **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**.

Docs/source/usage/parameters.rst

@@ -2689,10 +2689,11 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a
 
 * ``<diag_name>.<species_name>.variables`` (list of `strings` separated by spaces, optional)
     List of particle quantities to write to output.
-    Choices are ``w`` for the particle weight and ``ux`` ``uy`` ``uz`` for the particle momenta.
+    Choices are ``x``, ``y``, ``z`` for the particle positions (3D and RZ), ``x`` & ``z`` in 2D, ``z`` in 1D,
+    ``w`` for the particle weight and ``ux``, ``uy``, ``uz`` for the particle momenta.
     When using the lab-frame electrostatic solver, ``phi`` (electrostatic potential, on the macroparticles) is also available.
     By default, all particle quantities (except ``phi``) are written.
-    If ``<diag_name>.<species_name>.variables = none``, no particle data are written, except for particle positions, which are always included.
+    If ``<diag_name>.<species_name>.variables = none``, no particle data are written.
 
 * ``<diag_name>.<species_name>.random_fraction`` (`float`) optional
     If provided ``<diag_name>.<species_name>.random_fraction = a``, only `a` fraction of the particle data of this species will be dumped randomly in diag ``<diag_name>``, i.e. if `rand() < a`, this particle will be dumped, where `rand()` denotes a random number generator.

Examples/Physics_applications/laser_acceleration/inputs_rz

@@ -89,5 +89,5 @@ diagnostics.diags_names = diag1
 diag1.intervals = 10
 diag1.diag_type = Full
 diag1.fields_to_plot = Er Et Ez Br Bt Bz jr jt jz rho
-diag1.electrons.variables = w ux uy uz orig_x orig_z
-diag1.beam.variables = w ux uy uz
+diag1.electrons.variables = x y z w ux uy uz orig_x orig_z
+diag1.beam.variables = x y z w ux uy uz

Examples/Physics_applications/uniform_plasma/inputs_3d

@@ -45,7 +45,7 @@ electrons.uz_th  = 0.01 # uth the std of the (unitless) momentum
 diagnostics.diags_names = diag1 chk
 diag1.intervals = 4
 diag1.diag_type = Full
-diag1.electrons.variables = ux uy uz w
+diag1.electrons.variables = x y z ux uy uz w
 diag1.fields_to_plot = Bx By Bz Ex Ey Ez jx jy jz rho
 
 chk.intervals = 6

Examples/Tests/Implicit/inputs_1d

@@ -71,8 +71,8 @@ diagnostics.diags_names = diag1
 diag1.intervals = 100
 diag1.diag_type = Full
 diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE
-diag1.electrons.variables = w ux uy uz
-diag1.protons.variables = w ux uy uz
+diag1.electrons.variables = z w ux uy uz
+diag1.protons.variables = z w ux uy uz
 
 warpx.reduced_diags_names = particle_energy field_energy
 particle_energy.type = ParticleEnergy

Examples/Tests/Implicit/inputs_1d_semiimplicit

@@ -71,8 +71,8 @@ diagnostics.diags_names = diag1
 diag1.intervals = 100
 diag1.diag_type = Full
 diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE
-diag1.electrons.variables = w ux uy uz
-diag1.protons.variables = w ux uy uz
+diag1.electrons.variables = z w ux uy uz
+diag1.protons.variables = z w ux uy uz
 
 warpx.reduced_diags_names = particle_energy field_energy
 particle_energy.type = ParticleEnergy

Examples/Tests/Implicit/inputs_vandb_2d

@@ -82,8 +82,8 @@ diagnostics.diags_names = diag1
 diag1.intervals = 20
 diag1.diag_type = Full
 diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE
-diag1.electrons.variables = w ux uy uz
-diag1.protons.variables = w ux uy uz
+diag1.electrons.variables = x z w ux uy uz
+diag1.protons.variables = x z w ux uy uz
 
 warpx.reduced_diags_names = particle_energy field_energy
 particle_energy.type = ParticleEnergy

Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py

@@ -18,6 +18,7 @@
 inverse t(r) can be solved for exactly.
 """
 import os
+import re
 import sys
 
 import numpy as np
@@ -36,6 +37,15 @@
 ds = yt.load( filename )
 t_max = ds.current_time.item()  # time of simulation
 
+# Parse test name and check if particle_shape = 4 is used
+emass_10 = True if re.search('emass_10', filename) else False
+
+if emass_10:
+    l2_tolerance = 0.096
+    m_e = 10
+else:
+    l2_tolerance = 0.05
+    m_e = 9.10938356e-31 #Electron mass in kg
 ndims = np.count_nonzero(ds.domain_dimensions > 1)
 
 if ndims == 2:
@@ -60,7 +70,6 @@
 
 # Constants
 eps_0 = 8.8541878128e-12  #Vacuum Permittivity in C/(V*m)
-m_e = 9.10938356e-31  #Electron mass in kg
 q_e = -1.60217662e-19  #Electron charge in C
 pi = np.pi  #Circular constant of the universe
 r_0 = 0.1  #Initial radius of sphere
@@ -134,9 +143,9 @@ def calculate_error(E_axis, xmin, dx, nx):
 print("L2 error along y-axis = %s" %L2_error_y)
 print("L2 error along z-axis = %s" %L2_error_z)
 
-assert L2_error_x < 0.05
-assert L2_error_y < 0.05
-assert L2_error_z < 0.05
+assert L2_error_x < l2_tolerance
+assert L2_error_y < l2_tolerance
+assert L2_error_z < l2_tolerance
 
 # Check conservation of energy
 def return_energies(iteration):

Examples/Tests/electrostatic_sphere/inputs_rz

@@ -39,5 +39,5 @@ diag1.fields_to_plot = Er Et Ez rho
 diag2.intervals = 30
 diag2.diag_type = Full
 diag2.fields_to_plot = none
-diag2.electron.variables = ux uy uz w phi
+diag2.electron.variables = x y z ux uy uz w phi
 diag2.format = openpmd

Examples/Tests/langmuir/inputs_3d

@@ -93,5 +93,5 @@ diagnostics.diags_names = diag1
 diag1.intervals = max_step
 diag1.diag_type = Full
 diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho
-diag1.electrons.variables = w ux
-diag1.positrons.variables = uz
+diag1.electrons.variables = x y z w ux
+diag1.positrons.variables = x y z uz

Examples/Tests/photon_pusher/inputs_3d

@@ -140,19 +140,19 @@ p_dn_10.single_particle_weight = 1.0
 diagnostics.diags_names = diag1
 diag1.intervals = 50
 diag1.diag_type = Full
-diag1.p_xp_1.variables = ux uy uz
-diag1.p_xn_1.variables = ux uy uz
-diag1.p_yp_1.variables = ux uy uz
-diag1.p_yn_1.variables = ux uy uz
-diag1.p_zp_1.variables = ux uy uz
-diag1.p_zn_1.variables = ux uy uz
-diag1.p_dp_1.variables = ux uy uz
-diag1.p_dn_1.variables = ux uy uz
-diag1.p_xp_10.variables = ux uy uz
-diag1.p_xn_10.variables = ux uy uz
-diag1.p_yp_10.variables = ux uy uz
-diag1.p_yn_10.variables = ux uy uz
-diag1.p_zp_10.variables = ux uy uz
-diag1.p_zn_10.variables = ux uy uz
-diag1.p_dp_10.variables = ux uy uz
-diag1.p_dn_10.variables = ux uy uz
+diag1.p_xp_1.variables = x y z ux uy uz
+diag1.p_xn_1.variables = x y z ux uy uz
+diag1.p_yp_1.variables = x y z ux uy uz
+diag1.p_yn_1.variables = x y z ux uy uz
+diag1.p_zp_1.variables = x y z ux uy uz
+diag1.p_zn_1.variables = x y z ux uy uz
+diag1.p_dp_1.variables = x y z ux uy uz
+diag1.p_dn_1.variables = x y z ux uy uz
+diag1.p_xp_10.variables = x y z ux uy uz
+diag1.p_xn_10.variables = x y z ux uy uz
+diag1.p_yp_10.variables = x y z ux uy uz
+diag1.p_yn_10.variables = x y z ux uy uz
+diag1.p_zp_10.variables = x y z ux uy uz
+diag1.p_zn_10.variables = x y z ux uy uz
+diag1.p_dp_10.variables = x y z ux uy uz
+diag1.p_dn_10.variables = x y z ux uy uz

Examples/Tests/plasma_lens/inputs_3d

@@ -48,4 +48,4 @@ particles.repeated_plasma_lens_strengths_B = 0.0 0.0 0.0 0.0
 diagnostics.diags_names = diag1
 diag1.intervals = 84
 diag1.diag_type = Full
-diag1.electrons.variables = ux uy uz
+diag1.electrons.variables = x y z ux uy uz

Examples/Tests/plasma_lens/inputs_boosted_3d

@@ -50,4 +50,4 @@ particles.repeated_plasma_lens_strengths_B = 0.0 0.0 0.0 0.0
 diagnostics.diags_names = diag1
 diag1.intervals = 84
 diag1.diag_type = Full
-diag1.electrons.variables = ux uy uz
+diag1.electrons.variables = x y z ux uy uz

Examples/Tests/plasma_lens/inputs_lattice_3d

@@ -70,4 +70,4 @@ plasmalens4.dEdx = 200000.
 diagnostics.diags_names = diag1
 diag1.intervals = 84
 diag1.diag_type = Full
-diag1.electrons.variables = ux uy uz
+diag1.electrons.variables = x y z ux uy uz

Examples/Tests/plasma_lens/inputs_short_3d

@@ -52,4 +52,4 @@ particles.repeated_plasma_lens_strengths_B = 0.0 0.0 0.0 0.0
 diagnostics.diags_names = diag1
 diag1.intervals = 1 #84
 diag1.diag_type = Full
-diag1.electrons.variables = ux uy uz
+diag1.electrons.variables = x y z ux uy uz

Examples/Tests/qed/breit_wheeler/inputs_2d

@@ -201,19 +201,19 @@ diagnostics.diags_names = diag1
 diag1.intervals = 2
 diag1.diag_type = Full
 diag1.fields_to_plot = Ex
-diag1.p1.variables = ux uy uz w opticalDepthBW
-diag1.p2.variables = ux uy uz w opticalDepthBW
-diag1.p3.variables = ux uy uz w opticalDepthBW
-diag1.p4.variables = ux uy uz w opticalDepthBW
-
-diag1.ele1.variables = ux uy uz w opticalDepthQSR
-diag1.ele2.variables = ux uy uz w opticalDepthQSR
-diag1.ele3.variables = ux uy uz w opticalDepthQSR
-diag1.ele4.variables = ux uy uz w opticalDepthQSR
-
-diag1.pos1.variables = ux uy uz w opticalDepthQSR
-diag1.pos2.variables = ux uy uz w opticalDepthQSR
-diag1.pos3.variables = ux uy uz w opticalDepthQSR
-diag1.pos4.variables = ux uy uz w opticalDepthQSR
+diag1.p1.variables = x z ux uy uz w opticalDepthBW
+diag1.p2.variables = x z ux uy uz w opticalDepthBW
+diag1.p3.variables = x z ux uy uz w opticalDepthBW
+diag1.p4.variables = x z ux uy uz w opticalDepthBW
+
+diag1.ele1.variables = x z ux uy uz w opticalDepthQSR
+diag1.ele2.variables = x z ux uy uz w opticalDepthQSR
+diag1.ele3.variables = x z ux uy uz w opticalDepthQSR
+diag1.ele4.variables = x z ux uy uz w opticalDepthQSR
+
+diag1.pos1.variables = x z ux uy uz w opticalDepthQSR
+diag1.pos2.variables = x z ux uy uz w opticalDepthQSR
+diag1.pos3.variables = x z ux uy uz w opticalDepthQSR
+diag1.pos4.variables = x z ux uy uz w opticalDepthQSR
 
 diag1.format = plotfile

Examples/Tests/qed/breit_wheeler/inputs_3d

@@ -201,19 +201,19 @@ diagnostics.diags_names = diag1
 diag1.intervals = 2
 diag1.diag_type = Full
 diag1.fields_to_plot = Ex
-diag1.p1.variables = ux uy uz w opticalDepthBW
-diag1.p2.variables = ux uy uz w opticalDepthBW
-diag1.p3.variables = ux uy uz w opticalDepthBW
-diag1.p4.variables = ux uy uz w opticalDepthBW
-
-diag1.ele1.variables = ux uy uz w opticalDepthQSR
-diag1.ele2.variables = ux uy uz w opticalDepthQSR
-diag1.ele3.variables = ux uy uz w opticalDepthQSR
-diag1.ele4.variables = ux uy uz w opticalDepthQSR
-
-diag1.pos1.variables = ux uy uz w opticalDepthQSR
-diag1.pos2.variables = ux uy uz w opticalDepthQSR
-diag1.pos3.variables = ux uy uz w opticalDepthQSR
-diag1.pos4.variables = ux uy uz w opticalDepthQSR
+diag1.p1.variables = x y z ux uy uz w opticalDepthBW
+diag1.p2.variables = x y z ux uy uz w opticalDepthBW
+diag1.p3.variables = x y z ux uy uz w opticalDepthBW
+diag1.p4.variables = x y z ux uy uz w opticalDepthBW
+
+diag1.ele1.variables = x y z ux uy uz w opticalDepthQSR
+diag1.ele2.variables = x y z ux uy uz w opticalDepthQSR
+diag1.ele3.variables = x y z ux uy uz w opticalDepthQSR
+diag1.ele4.variables = x y z ux uy uz w opticalDepthQSR
+
+diag1.pos1.variables = x y z ux uy uz w opticalDepthQSR
+diag1.pos2.variables = x y z ux uy uz w opticalDepthQSR
+diag1.pos3.variables = x y z ux uy uz w opticalDepthQSR
+diag1.pos4.variables = x y z ux uy uz w opticalDepthQSR
 
 diag1.format = plotfile

Examples/Tests/qed/quantum_synchrotron/inputs_2d

@@ -178,12 +178,12 @@ diagnostics.diags_names = diag1
 diag1.intervals = 2
 diag1.diag_type = Full
 diag1.fields_to_plot = Ex
-diag1.p1.variables = ux uy uz w opticalDepthQSR
-diag1.p2.variables = ux uy uz w opticalDepthQSR
-diag1.p3.variables = ux uy uz w opticalDepthQSR
-diag1.p4.variables = ux uy uz w opticalDepthQSR
-
-diag1.qsp_1.variables = ux uy uz w opticalDepthBW
-diag1.qsp_2.variables = ux uy uz w opticalDepthBW
-diag1.qsp_3.variables = ux uy uz w opticalDepthBW
-diag1.qsp_4.variables = ux uy uz w opticalDepthBW
+diag1.p1.variables = x z ux uy uz w opticalDepthQSR
+diag1.p2.variables = x z ux uy uz w opticalDepthQSR
+diag1.p3.variables = x z ux uy uz w opticalDepthQSR
+diag1.p4.variables = x z ux uy uz w opticalDepthQSR
+
+diag1.qsp_1.variables = x z ux uy uz w opticalDepthBW
+diag1.qsp_2.variables = x z ux uy uz w opticalDepthBW
+diag1.qsp_3.variables = x z ux uy uz w opticalDepthBW
+diag1.qsp_4.variables = x z ux uy uz w opticalDepthBW

Examples/Tests/qed/quantum_synchrotron/inputs_3d

@@ -178,12 +178,12 @@ diagnostics.diags_names = diag1
 diag1.intervals = 2
 diag1.diag_type = Full
 diag1.fields_to_plot = Ex
-diag1.p1.variables = ux uy uz w opticalDepthQSR
-diag1.p2.variables = ux uy uz w opticalDepthQSR
-diag1.p3.variables = ux uy uz w opticalDepthQSR
-diag1.p4.variables = ux uy uz w opticalDepthQSR
-
-diag1.qsp_1.variables = ux uy uz w opticalDepthBW
-diag1.qsp_2.variables = ux uy uz w opticalDepthBW
-diag1.qsp_3.variables = ux uy uz w opticalDepthBW
-diag1.qsp_4.variables = ux uy uz w opticalDepthBW
+diag1.p1.variables = x y z ux uy uz w opticalDepthQSR
+diag1.p2.variables = x y z ux uy uz w opticalDepthQSR
+diag1.p3.variables = x y z ux uy uz w opticalDepthQSR
+diag1.p4.variables = x y z ux uy uz w opticalDepthQSR
+
+diag1.qsp_1.variables = x y z ux uy uz w opticalDepthBW
+diag1.qsp_2.variables = x y z ux uy uz w opticalDepthBW
+diag1.qsp_3.variables = x y z ux uy uz w opticalDepthBW
+diag1.qsp_4.variables = x y z ux uy uz w opticalDepthBW

Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning_cartesian

@@ -0,0 +1,45 @@
+max_step = 4
+warpx.verbose = 1
+warpx.const_dt = 1e-10
+amr.n_cell = 256
+amr.max_grid_size = 64
+amr.max_level = 0
+geometry.dims = 1
+geometry.prob_lo = 0
+geometry.prob_hi = 0.1
+
+# Boundary condition and field solver
+boundary.field_lo = periodic
+boundary.field_hi = periodic
+boundary.particle_lo = periodic
+boundary.particle_hi = periodic
+algo.particle_shape = 1
+algo.maxwell_solver = none
+
+particles.species_names = hydrogen
+hydrogen.mass = 1.67262192369e-27
+hydrogen.charge = 1.602176634e-19
+hydrogen.injection_style = nrandompercell
+hydrogen.initialize_self_fields = 0
+hydrogen.do_not_push = 1
+hydrogen.do_resampling = 1
+hydrogen.resampling_min_ppc = 10
+hydrogen.resampling_trigger_intervals = 1::2
+hydrogen.resampling_algorithm = velocity_coincidence_thinning
+hydrogen.resampling_algorithm_velocity_grid_type = cartesian
+hydrogen.resampling_algorithm_delta_u = 10000000.0 5000000.0 7000000.0
+hydrogen.num_particles_per_cell = 2500
+hydrogen.momentum_distribution_type = gaussian
+hydrogen.ux_m = 0.0
+hydrogen.uy_m = 0.0
+hydrogen.uz_m = 0.0
+hydrogen.ux_th = 0.000326
+hydrogen.uy_th = 0.000326
+hydrogen.uz_th = 0.000326
+hydrogen.profile = constant
+hydrogen.density = 1e+19
+
+# Diagnostics
+diagnostics.diags_names = diag1
+diag1.intervals = 4
+diag1.diag_type = Full