Skip to content

Commit

Permalink
Merge branch 'development' into cleanup_deposition_gather
Browse files Browse the repository at this point in the history
  • Loading branch information
dpgrote committed Jun 18, 2024
2 parents d9384a4 + 8283320 commit 68628b1
Show file tree
Hide file tree
Showing 34 changed files with 230 additions and 59 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/cuda.yml
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ jobs:
which nvcc || echo "nvcc not in PATH!"
git clone https://github.com/AMReX-Codes/amrex.git ../amrex
cd ../amrex && git checkout --detach 28b010126a1b39297d8a496ba81f171d8563953b && cd -
cd ../amrex && git checkout --detach 24.06 && cd -
make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4
ccache -s
Expand Down
39 changes: 38 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Preamble ####################################################################
#
cmake_minimum_required(VERSION 3.20.0)
project(WarpX VERSION 24.05)
project(WarpX VERSION 24.06)

include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake)

Expand Down Expand Up @@ -79,6 +79,7 @@ option(WarpX_LIB "Build WarpX as a library" OFF)
option(WarpX_MPI "Multi-node support (message-passing)" ON)
option(WarpX_OPENPMD "openPMD I/O (HDF5, ADIOS)" ON)
option(WarpX_FFT "FFT-based solvers" OFF)
option(WarpX_HEFFTE "Multi-node FFT-based solvers" OFF)
option(WarpX_PYTHON "Python bindings" OFF)
option(WarpX_SENSEI "SENSEI in situ diagnostics" OFF)
option(WarpX_QED "QED support (requires PICSAR)" ON)
Expand Down Expand Up @@ -136,6 +137,10 @@ mark_as_advanced(WarpX_MPI_THREAD_MULTIPLE)

option(WarpX_amrex_internal "Download & build AMReX" ON)

if(WarpX_HEFFTE AND NOT WarpX_MPI)
message(FATAL_ERROR "WarpX_HEFFTE (${WarpX_HEFFTE}) can only be used if WarpX_MPI is ON.")
endif()

# change the default build type to Release (or RelWithDebInfo) instead of Debug
set_default_build_type("Release")

Expand Down Expand Up @@ -174,6 +179,10 @@ option(ABLASTR_FFT "compile AnyFFT wrappers" ${WarpX_FFT})
if(WarpX_FFT)
set(ABLASTR_FFT ON CACHE STRING "FFT-based solvers" FORCE)
endif()
option(ABLASTR_HEFFTE "compile AnyFFT wrappers" ${WarpX_HEFFTE})
if(WarpX_HEFFTE)
set(ABLASTR_HEFFTE ON CACHE STRING "Multi-Node FFT-based solvers" FORCE)
endif()

# this defined the variable BUILD_TESTING which is ON by default
#include(CTest)
Expand Down Expand Up @@ -215,6 +224,23 @@ if(WarpX_FFT)
endif()
endif()

# multi-node FFT
if(WarpX_HEFFTE)
if(WarpX_COMPUTE STREQUAL CUDA)
set(_heFFTe_COMPS CUDA)
elseif(WarpX_COMPUTE STREQUAL HIP)
set(_heFFTe_COMPS ROCM)
elseif(WarpX_COMPUTE STREQUAL SYCL)
set(_heFFTe_COMPS ONEAPI)
else() # NOACC, OMP
set(_heFFTe_COMPS FFTW) # or MKL
endif()
# note: we could also enforce GPUAWARE for CUDA and HIP, which can still be
# disabled at runtime

find_package(Heffte REQUIRED COMPONENTS ${_heFFTe_COMPS})
endif()

# Python
if(WarpX_PYTHON)
find_package(Python COMPONENTS Interpreter Development.Module REQUIRED)
Expand Down Expand Up @@ -455,6 +481,10 @@ foreach(D IN LISTS WarpX_DIMS)
endif()
endif()

if(ABLASTR_HEFFTE)
target_link_libraries(ablastr_${SD} PUBLIC Heffte::Heffte)
endif()

if(WarpX_PYTHON)
target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::module pybind11::windows_extras)
if(WarpX_PYTHON_IPO)
Expand Down Expand Up @@ -539,6 +569,13 @@ foreach(D IN LISTS WarpX_DIMS)
target_compile_definitions(ablastr_${SD} PUBLIC ABLASTR_USE_FFT)
endif()

if(WarpX_HEFFTE)
target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_HEFFTE)
endif()
if(ABLASTR_HEFFTE)
target_compile_definitions(ablastr_${SD} PUBLIC ABLASTR_USE_HEFFTE)
endif()

if(WarpX_PYTHON AND pyWarpX_VERSION_INFO)
# for module __version__
target_compile_definitions(pyWarpX_${SD} PRIVATE
Expand Down
4 changes: 2 additions & 2 deletions Docs/source/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,9 @@ def __init__(self, *args, **kwargs):
# built documents.
#
# The short X.Y version.
version = u'24.05'
version = u'24.06'
# The full version, including alpha/beta/rc tags.
release = u'24.05'
release = u'24.06'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
Expand Down
2 changes: 2 additions & 0 deletions Docs/source/install/cmake.rst
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ CMake Option Default & Values Descr
``WarpX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double)
``WarpX_PARTICLE_PRECISION`` SINGLE/**DOUBLE** Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set
``WarpX_FFT`` ON/**OFF** FFT-based solvers
``WarpX_HEFFTE`` ON/**OFF** Multi-Node FFT-based solvers
``WarpX_PYTHON`` ON/**OFF** Python bindings
``WarpX_QED`` **ON**/OFF QED support (requires PICSAR)
``WarpX_QED_TABLE_GEN`` ON/**OFF** QED table generation support (requires PICSAR and Boost)
Expand Down Expand Up @@ -271,6 +272,7 @@ Environment Variable Default & Values Descr
``WARPX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double)
``WARPX_PARTICLE_PRECISION`` SINGLE/**DOUBLE** Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set
``WARPX_FFT`` ON/**OFF** FFT-based solvers
``WARPX_HEFFTE`` ON/**OFF** Multi-Node FFT-based solvers
``WARPX_QED`` **ON**/OFF PICSAR QED (requires PICSAR)
``WARPX_QED_TABLE_GEN`` ON/**OFF** QED table generation (requires PICSAR and Boost)
``BUILD_PARALLEL`` ``2`` Number of threads to use for parallel builds
Expand Down
7 changes: 4 additions & 3 deletions Docs/source/install/dependencies.rst
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,13 @@ Optional dependencies include:
- for on-node accelerated compute *one of either*:

- `OpenMP 3.1+ <https://www.openmp.org>`__: for threaded CPU execution or
- `CUDA Toolkit 11.7+ <https://developer.nvidia.com/cuda-downloads>`__: for Nvidia GPU support (see `matching host-compilers <https://gist.github.com/ax3l/9489132>`_) or
- `CUDA Toolkit 11.7+ <https://developer.nvidia.com/cuda-downloads>`__: for Nvidia GPU support (see `matching host-compilers <https://gist.github.com/ax3l/9489132>`__) or
- `ROCm 5.2+ (5.5+ recommended) <https://gpuopen.com/learn/amd-lab-notes/amd-lab-notes-rocm-installation-readme/>`__: for AMD GPU support
- `FFTW3 <http://www.fftw.org>`_: for spectral solver (PSATD) support when running on CPU or SYCL
- `FFTW3 <http://www.fftw.org>`__: for spectral solver (PSATD or IGF) support when running on CPU or SYCL

- also needs the ``pkg-config`` tool on Unix
- `BLAS++ <https://github.com/icl-utk-edu/blaspp>`_ and `LAPACK++ <https://github.com/icl-utk-edu/lapackpp>`_: for spectral solver (PSATD) support in RZ geometry
- `heFFTe 2.4.0+ <https://github.com/icl-utk-edu/heffte`__: for multi-node spectral solver (IGF) support
- `BLAS++ <https://github.com/icl-utk-edu/blaspp>`__ and `LAPACK++ <https://github.com/icl-utk-edu/lapackpp>`__: for spectral solver (PSATD) support in RZ geometry
- `Boost 1.66.0+ <https://www.boost.org/>`__: for QED lookup tables generation support
- `openPMD-api 0.15.1+ <https://github.com/openPMD/openPMD-api>`__: we automatically download and compile a copy of openPMD-api for openPMD I/O support

Expand Down
8 changes: 4 additions & 4 deletions Docs/source/install/hpc/perlmutter.rst
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
cd $HOME/src/warpx
rm -rf build_pm_gpu
cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
cmake --build build_pm_gpu -j 16
The WarpX application executables are now in ``$HOME/src/warpx/build_pm_gpu/bin/``.
Expand All @@ -164,7 +164,7 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
cd $HOME/src/warpx
rm -rf build_pm_gpu_py
cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3"
cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3"
cmake --build build_pm_gpu_py -j 16 --target pip_install
.. tab-item:: CPU Nodes
Expand All @@ -174,7 +174,7 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
cd $HOME/src/warpx
rm -rf build_pm_cpu
cmake -S . -B build_pm_cpu -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
cmake -S . -B build_pm_cpu -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
cmake --build build_pm_cpu -j 16
The WarpX application executables are now in ``$HOME/src/warpx/build_pm_cpu/bin/``.
Expand All @@ -184,7 +184,7 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
rm -rf build_pm_cpu_py
cmake -S . -B build_pm_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3"
cmake -S . -B build_pm_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3"
cmake --build build_pm_cpu_py -j 16 --target pip_install
Now, you can :ref:`submit Perlmutter compute jobs <running-cpp-perlmutter>` for WarpX :ref:`Python (PICMI) scripts <usage-picmi>` (:ref:`example scripts <usage-examples>`).
Expand Down
2 changes: 1 addition & 1 deletion Examples/Tests/embedded_circle/inputs_2d
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
max_step = 11
warpx.const_dt = 3.99e-13
warpx.do_electrostatic = labframe
warpx.self_fields_required_precision = 1e-06
warpx.self_fields_required_precision = 2e-06
warpx.eb_implicit_function = -((x-0.00005)**2+(z-0.00005)**2-1e-05**2)
warpx.eb_potential(x,y,z,t) = -10
warpx.self_fields_absolute_tolerance = 0.02
Expand Down
2 changes: 1 addition & 1 deletion Python/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
package_data = {}

setup(name = 'pywarpx',
version = '24.05',
version = '24.06',
packages = ['pywarpx'],
package_dir = {'pywarpx': 'pywarpx'},
description = """Wrapper of WarpX""",
Expand Down
4 changes: 2 additions & 2 deletions Regression/Checksum/benchmarks_json/collisionXYZ.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"Ex": 0.0,
"Ey": 0.0,
"Ez": 0.0,
"T_electron": 362230.52300397365,
"T_ion": 338312.83502136066
"T_electron": 358778.6506903592,
"T_ion": 341562.3085776466
}
}
2 changes: 1 addition & 1 deletion Regression/WarpX-GPU-tests.ini
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more

[AMReX]
dir = /home/regtester/git/amrex/
branch = 28b010126a1b39297d8a496ba81f171d8563953b
branch = 24.06

[source]
dir = /home/regtester/git/WarpX
Expand Down
2 changes: 1 addition & 1 deletion Regression/WarpX-tests.ini
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det

[AMReX]
dir = /home/regtester/AMReX_RegTesting/amrex/
branch = 28b010126a1b39297d8a496ba81f171d8563953b
branch = 24.06

[source]
dir = /home/regtester/AMReX_RegTesting/warpx
Expand Down
7 changes: 6 additions & 1 deletion Source/Diagnostics/ReducedDiags/FieldProbe.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,12 @@ FieldProbe::FieldProbe (const std::string& rd_name)
ablastr::warn_manager::WarnPriority::low);
}

WARPX_ALWAYS_ASSERT_WITH_MESSAGE(interp_order <= WarpX::nox ,
// ensure assumption holds: we read the fields in the interpolation kernel as they are,
// without further communication of guard/ghost/halo regions
int particle_shape;
const ParmParse pp_algo("algo");
utils::parser::getWithParser(pp_algo, "particle_shape", particle_shape);
WARPX_ALWAYS_ASSERT_WITH_MESSAGE(interp_order <= particle_shape ,
"Field probe interp_order should be less than or equal to algo.particle_shape");
if (ParallelDescriptor::IOProcessor())
{
Expand Down
3 changes: 3 additions & 0 deletions Source/Evolve/WarpXEvolve.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,9 @@ WarpX::Evolve (int numsteps)
multi_diags->FilterComputePackFlushLastTimestep( istep[0] );
if (m_exit_loop_due_to_interrupt_signal) { ExecutePythonCallback("onbreaksignal"); }
}

amrex::Print() <<
ablastr::warn_manager::GetWMInstance().PrintGlobalWarnings("THE END");
}

/* /brief Perform one PIC iteration, without subcycling
Expand Down
1 change: 1 addition & 0 deletions Source/Initialization/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ foreach(D IN LISTS WarpX_DIMS)
TemperatureProperties.cpp
VelocityProperties.cpp
WarpXAMReXInit.cpp
WarpXInit.cpp
WarpXInitData.cpp
)
endforeach()
1 change: 1 addition & 0 deletions Source/Initialization/Make.package
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ CEXE_sources += PlasmaInjector.cpp
CEXE_sources += TemperatureProperties.cpp
CEXE_sources += VelocityProperties.cpp
CEXE_sources += WarpXAMReXInit.cpp
CEXE_sources += WarpXInit.cpp
CEXE_sources += WarpXInitData.cpp

VPATH_LOCATIONS += $(WARPX_HOME)/Source/Initialization
30 changes: 30 additions & 0 deletions Source/Initialization/WarpXInit.H
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/* Copyright 2024 Luca Fedeli
*
* This file is part of WarpX.
*
* License: BSD-3-Clause-LBNL
*/
#ifndef WARPX_INIT_H_
#define WARPX_INIT_H_

namespace warpx::initialization
{
/** Initializes, in the following order:
* - the MPI library through the mpi_init helper function in ablastr
* - the AMReX library
* - the FFT library through the anyfft::setup() function in ablastr
*
* @param[in] argc number of arguments from main()
* @param[in] argv argument strings from main()
*/
void initialize_external_libraries(int argc, char* argv[]);

/** Initializes, in the following order:
* - the FFT library through the anyfft::cleanup() function in ablastr
* - the AMReX library
* - the MPI library through the mpi_finalize helper function in ablastr
*/
void finalize_external_libraries();
}

#endif //WARPX_INIT_H_
29 changes: 29 additions & 0 deletions Source/Initialization/WarpXInit.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/* Copyright 2024 Luca Fedeli
*
* This file is part of WarpX.
*
* License: BSD-3-Clause-LBNL
*/

#include "WarpXInit.H"

#include "Initialization/WarpXAMReXInit.H"

#include <AMReX.H>

#include <ablastr/math/fft/AnyFFT.H>
#include <ablastr/parallelization/MPIInitHelpers.H>

void warpx::initialization::initialize_external_libraries(int argc, char* argv[])
{
ablastr::parallelization::mpi_init(argc, argv);
warpx::initialization::amrex_init(argc, argv);
ablastr::math::anyfft::setup();
}

void warpx::initialization::finalize_external_libraries()
{
ablastr::math::anyfft::cleanup();
amrex::Finalize();
ablastr::parallelization::mpi_finalize();
}
2 changes: 1 addition & 1 deletion Source/Parallelization/GuardCellManager.H
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public:
*/
void Init(
amrex::Real dt,
amrex::RealVect dx,
const amrex::Real * dx,
bool do_subcycling,
bool do_fdtd_nci_corr,
short grid_type,
Expand Down
2 changes: 1 addition & 1 deletion Source/Parallelization/GuardCellManager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ using namespace amrex;
void
guardCellManager::Init (
const amrex::Real dt,
const amrex::RealVect dx,
const amrex::Real *dx,
const bool do_subcycling,
const bool do_fdtd_nci_corr,
const short grid_type,
Expand Down
8 changes: 1 addition & 7 deletions Source/WarpX.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2120,13 +2120,7 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d
{
const bool aux_is_nodal = (field_gathering_algo == GatheringAlgo::MomentumConserving);

#if defined(WARPX_DIM_1D_Z)
const amrex::RealVect dx(WarpX::CellSize(lev)[2]);
#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ)
const amrex::RealVect dx = {WarpX::CellSize(lev)[0], WarpX::CellSize(lev)[2]};
#elif defined(WARPX_DIM_3D)
const amrex::RealVect dx = {WarpX::CellSize(lev)[0], WarpX::CellSize(lev)[1], WarpX::CellSize(lev)[2]};
#endif
const Real* dx = Geom(lev).CellSize();

// Initialize filter before guard cells manager
// (needs info on length of filter's stencil)
Expand Down
Loading

0 comments on commit 68628b1

Please sign in to comment.